ubi 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 330dfff07aab154b3fc714a71b10cf0f148d541d
+  data.tar.gz: 81ff08e9a049671a6f2c51c43641007466fa30c3
+SHA512:
+  metadata.gz: fae80b06b1e6f5bac23357f8b4c09ef45879ba62b4e1e6e90c9e42a16490f7496cc614b27538beaed7d6529d8784fce29a79722e4a351379079cc0e4be6bf1cb
+  data.tar.gz: 4a4ec085b39e27a0e64addcf9c22fff3a969bdff21a72c8684bcb500451d3c4503eb1935383fd96715dd72257082f4851476b9b2c8d4a26244b6363b9264588b

data/.gitignore

@@ -0,0 +1,9 @@
+.DS_Store
+coverage
+rdoc
+pkg
+.bundle
+Gemfile.lock
+*.gem
+*.log
+tmp/*

data/.rspec

@@ -0,0 +1,3 @@
+--tty
+--color
+--format documentation

data/Gemfile

@@ -0,0 +1,11 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'pry'
+gem 'rake'
+gem 'rspec', '>= 3'
+gem 'rubocop'
+gem 'guard'
+gem 'guard-rspec'
+gem 'guard-rubocop'

data/Guardfile

@@ -0,0 +1,16 @@
+#
+# Ubi Guard
+#
+guard :rubocop, all_on_start: false, keep_failed: false, cli: ['-D'] do
+  watch(/.+\.rb$/)
+  watch(/(?:.+\/)?\.rubocop\.yml$/) { |m| File.dirname(m[0]) }
+end
+
+guard :rspec, cmd: 'bundle exec rspec' do
+  watch(/^spec\/.+_spec\.rb$/)
+  watch(/^lib\/(.+)\.rb$/)     { |m| "spec/#{m[1]}_spec.rb" }
+  watch(/^generators\/(.+)\.rb$/) { |_m| 'spec/schemaless/worker_spec' }
+
+  watch('lib/ubi.rb')     { 'spec' }
+  watch('spec/spec_helper.rb')  { 'spec' }
+end

data/MIT-LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2014-2015 Ubi Authors
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,57 @@
+                                              _      _
+                                       /\ /\ | |__  (_)
+                                      / / \ \| '_ \ | |
+                                      \ \_/ /| |_) || |
+                                       \___/ |_.__/ |_|
+
+
+Ubi finds information in the subject's webpage(s).
+A forager.
+
+
+## Thema -> Subject/Matter
+
+- Name
+- Email
+- URL
+
+
+## Memoria -> Attribute/Trait/Memory
+
+- URLs
+- Emails
+- Phones
+- Social
+- Files
+- Documents
+- Address
+- Logo/Images
+
+
+## Aranea -> Spider/Crawler
+
+Search
+API
+
+Others
+
+
+## Use
+
+```
+Ubi::Thema.new('A Company on City').urls
+Ubi::Thema.new('A Company on City', 'company.com').phones
+```
+
+### Memorias Directly
+
+```
+Ubi::Memoria::Email.parse('A text with some valid@emails.com')
+Ubi::Memoria::Phone.parse('A text with some +55-5555-5555')
+Ubi::Memoria::Site.parse('A text with some http://urls.com')
+```
+
+##### Keep crawling
+
+> Ubi, and the class names are latin, in case you're curious.
+> Yeah, apparently I got plenty of free time.

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+RSpec::Core::RakeTask.new
+RuboCop::RakeTask.new
+
+task default: [:spec, :rubocop]

data/bin/ubi

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require 'ubi'
+require 'ubi/impero'
+
+puts
+puts '                                                    Ubi!'
+
+Ubi::Impero.start(ARGV)

data/lib/ubi.rb

@@ -0,0 +1,37 @@
+require 'pry'
+require 'thor'
+require 'net/http'
+require 'net/https'
+require 'uri'
+require 'open-uri'
+require 'json'
+require 'phonelib'
+require 'nokogiri'
+require 'active_model'
+require 'active_support'
+require 'active_support/core_ext/hash'
+require 'ubi/version'
+
+# Ubiquous getters
+module Ubi
+  cattr_accessor(:memorias) { [] }
+  cattr_accessor(:araneas) { [] }
+
+  # Phonie.configuration.default_country_code = '55'
+  # Phonie.configuration.n1_length = 4
+end
+
+require 'ubi/aranea'
+require 'ubi/memoria'
+require 'ubi/consultor'
+
+%w( memorias consultores ).each do |ns|
+  Dir["#{File.dirname(__FILE__)}/ubi/#{ns}/**.rb"].sort.each do |lib|
+    require lib
+  end
+end
+
+# Loads after all
+require 'ubi/artifex'
+require 'ubi/datum'
+require 'ubi/thema'

data/lib/ubi/aranea.rb

@@ -0,0 +1,42 @@
+require 'polipus'
+
+module Ubi
+  # Base for araneas (spiders)
+  class Aranea
+    OPTIONS = {
+      workers: 3,
+      user_agent: "Ubi v#{Ubi::VERSION}",
+      depth_limit: 1
+
+      # storage: MemoryStore
+    }
+    attr_accessor :thema, :url, :datum
+
+    def initialize(thema, url, opts = {})
+      @thema = thema
+      @url   = url
+      @opts  = opts
+    end
+
+    delegate :name, to: :thema
+
+    def crawl!
+      Polipus.crawler(name, url, OPTIONS.merge(@opts)) do |crawler|
+        # In-place page processing
+        crawler.on_page_downloaded do |page|
+          # A nokogiri object
+          puts "'#{page.doc.css('title').text}' (#{page.url})"
+        end
+      end
+    end
+
+    def parser(chunk)
+      Nokogiri::HTML(chunk)
+    end
+
+    def datum
+      crawl! unless @datum
+      @datum
+    end
+  end
+end

data/lib/ubi/artifex.rb

@@ -0,0 +1,28 @@
+module Ubi
+  # Artifex: I work!
+  class Artifex
+    include Consultor
+    attr_accessor :thema
+
+    def initialize(query)
+      @thema = Thema.new(query)
+      start_with_search
+      social_search
+      other_search
+    end
+
+    def start_with_search
+      [Google, Yahoo, Bing, DuckDuckGo].each do |s|
+        @thema.try_consultor(s)
+      end
+    end
+
+    def social_search
+    end
+
+    def other_search
+    end
+
+    delegate :spec, to: :thema
+  end
+end

data/lib/ubi/consultor.rb

@@ -0,0 +1,61 @@
+module Ubi
+  module Consultor
+    # Base for araneas (spiders)
+    class Base
+      HEADERS = { 'User-Agent' => "Ubi v#{Ubi::VERSION}" }
+
+      def initialize(thema)
+        @thema = thema
+      end
+
+      def query_url
+        self.class.url + query.to_query
+      end
+
+      def links
+        '//a'
+      end
+
+      def parser(chunk)
+        Nokogiri::HTML(chunk)
+      end
+
+      def datum
+        @datum ||= Ubi::Datum.new(parser(request), words, links)
+      end
+
+      #
+      # Make an HTTP(S) request to a geocoding API and
+      # return the response object.
+      #
+      def request(opts = {})
+        timeout(10) do
+          uri = URI.parse(query_url)
+          puts "#{self} working on `#{@thema}` (#{query_url}) #{opts}"
+          uri.open(HEADERS).read
+        end
+      end
+
+      class << self
+        def inherited(base)
+          puts "Using aranea #{base}"
+          Ubi.araneas << base
+        end
+
+        #
+        # Human-readable name of the aranea
+        #
+        def name
+          fail "Not implemented by #{self}"
+        end
+
+        #
+        # Url to query
+        #
+        def url
+          fail "Not implemented by #{self}"
+        end
+      end
+    end
+  end
+end

data/lib/ubi/consultores/bing.rb

@@ -0,0 +1,26 @@
+module Ubi
+  module Consultor
+    # URL finder consultor
+    class Bing < Base
+      #
+      # query
+      def query
+        { q: @thema.name, pc: 'ubi' }
+      end
+
+      def links
+        '//a[not (contains(@href,"bing"))]'
+      end
+
+      def words
+        '//li[@class="g"]'
+      end
+
+      class << self
+        def url
+          'https://bing.com/search?'
+        end
+      end
+    end
+  end
+end

data/lib/ubi/consultores/duckduckgo.rb

@@ -0,0 +1,26 @@
+module Ubi
+  module Consultor
+    # URL finder consultor
+    class DuckDuckGo < Base
+      #
+      # query
+      def query
+        { q: @thema.name, t: 'ubi' }
+      end
+
+      def links
+        '//a[not (contains(@href,"duckduckgo"))]'
+      end
+
+      def words
+        '//li[@class="g"]'
+      end
+
+      class << self
+        def url
+          'https://duckduckgo.com/?'
+        end
+      end
+    end
+  end
+end

data/lib/ubi/consultores/facebook.rb

@@ -0,0 +1,6 @@
+module Ubi
+  module Consultor
+    class Facebook < Base
+    end
+  end
+end

data/lib/ubi/consultores/foursquare.rb

@@ -0,0 +1,6 @@
+module Ubi
+  module Consultor
+    class Foursquare < Base
+    end
+  end
+end

data/lib/ubi/consultores/google.rb

@@ -0,0 +1,24 @@
+module Ubi
+  module Consultor
+    # URL finder consultor
+    class Google < Base # HTML
+      def query
+        { q: @thema.name }
+      end
+
+      def links
+        '//a[not (contains(@href,"google"))]'
+      end
+
+      def words
+        '//div[@class="res"]'
+      end
+
+      class << self
+        def url
+          'https://www.google.com.br/search?'
+        end
+      end
+    end
+  end
+end

data/lib/ubi/consultores/twitter.rb

@@ -0,0 +1,6 @@
+module Ubi
+  module Consultor
+    class Twitter < Base
+    end
+  end
+end

data/lib/ubi/consultores/wikipedia.rb

@@ -0,0 +1,6 @@
+module Ubi
+  module Consultor
+    class Wikipedia < Base
+    end
+  end
+end

data/lib/ubi/consultores/yahoo.rb

@@ -0,0 +1,26 @@
+module Ubi
+  module Consultor
+    # URL finder consultor
+    class Yahoo < Base
+      #
+      # query
+      def query
+        { p: @thema.name, hspart: 'ubi' }
+      end
+
+      def links
+        '//a[not (contains(@href,"yahoo"))]'
+      end
+
+      def words
+        '//li[@class="g"]'
+      end
+
+      class << self
+        def url
+          'https://search.yahoo.com/yhs/search?'
+        end
+      end
+    end
+  end
+end