ubi 0.0.5 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9a9073e8291c6e092b1f3585191aa4c2c91e524d
-  data.tar.gz: ceee4c37c817147c7f4a8f0d4627b3c5108ccbf5
+  metadata.gz: 7fdab4d682e936e1a133a4dc7be3d8b8e94e5911
+  data.tar.gz: a159c4494b80dd627fe1e977beea018dc85dcd4f
 SHA512:
-  metadata.gz: 796e569c777e65d7a8aca7ceb5cb9579fc5f52bfdef22a16cb8b54ec2b48854e0ee207c66d90e2c6bead5fa5622ae328760cf8bad111859059695002871ad6c3
-  data.tar.gz: 7791238d12cf001ed5e2d36d86645e42debe21f3a968ed9cc3c21e18636dcba4c1c44ce7714e526f78330827cb812826c3d261f4542a17729799452b75aa996e
+  metadata.gz: 2dfc19bf88633329bbe9ed38ff7181c5f9d772217319b03d4dc1c8ef8ef7bd5375590acf7ef3545ed6472d62b7a2e7b1ee1646fe1be1f1bd399db6ae90102a8f
+  data.tar.gz: f921d122f54a59b8997e36decfdd3072fff344d1b0d8b241fe0cd797e06427a8dbc6524d08c2ab3035ead12f9738ac0d4dbaf649663b7591d578d37791fb69cc

data/Rakefile

@@ -6,4 +6,4 @@ require 'rubocop/rake_task'
 RSpec::Core::RakeTask.new
 RuboCop::RakeTask.new
 
-task default: [:rubocop, :spec]
+task default: [:spec]

data/lib/ubi.rb

@@ -1,15 +1,14 @@
-require 'pry'
-require 'thor'
-require 'net/http'
-require 'net/https'
+# require 'net/http'
+# require 'net/https'
 require 'uri'
-require 'open-uri'
+# require 'open-uri'
 require 'json'
 require 'phonelib'
 require 'nokogiri'
 require 'active_model'
 require 'active_support'
 require 'active_support/core_ext/hash'
+require 'public_suffix'
 require 'ubi/version'
 
 # Ubiquous getters

data/lib/ubi/aranea.rb

@@ -1,42 +1,54 @@
-require 'polipus'
+require 'retriever'
 
 module Ubi
   # Base for araneas (spiders)
   class Aranea
     OPTIONS = {
-      workers: 3,
       user_agent: "Ubi v#{Ubi::VERSION}",
-      depth_limit: 1
-
-      # storage: MemoryStore
+      depth_limit: 3,
+      logger: Logger.new(STDOUT),
+      # redis_options: {
+      #   host: 'localhost',
+      #   db: 5,
+      #   driver: 'hiredis'
+      # },
     }
-    attr_accessor :thema, :url, :datum
+    attr_accessor :thema, :url, :datum, :html, :text, :opts
 
     def initialize(thema, url, opts = {})
       @thema = thema
       @url   = url
-      @opts  = opts
+      @opts  = OPTIONS.merge(opts)
+      @datum = []
+      @html = []
+      @text = ''
     end
 
     delegate :name, to: :thema
 
     def crawl!
-      Polipus.crawler(name, url, OPTIONS.merge(@opts)) do |crawler|
-        # In-place page processing
-        crawler.on_page_downloaded do |page|
-          # A nokogiri object
-          puts "'#{page.doc.css('title').text}' (#{page.url})"
-        end
+      @last_run = Time.now
+
+      puts "Crawler start #{name} #{url}"
+      Retriever::PageIterator.new(url, opts) do |page|
+        parse page.source
+        p [page.title, page.h1, page.h2]
       end
     end
 
-    def parser(chunk)
-      Nokogiri::HTML(chunk)
+    def parse(chunk)
+      @datum << chunk
+      @html << Nokogiri::HTML(chunk)
+      @text << html.last.text
+    end
+
+    def work
+      crawl! unless @last_run
+      true
     end
 
-    def datum
-      crawl! unless @datum
-      @datum
+    def to_s
+      "#{thema} html: #{html.size} txt: #{text.size}"
     end
-  end
-end
+  end # Aranea
+end # Ubi

data/lib/ubi/artifex.rb

@@ -5,10 +5,11 @@ module Ubi
     attr_accessor :thema
 
     def initialize(query)
-      @thema = Thema.new(query)
-      start_with_search
-      social_search
+      @thema = Thema.new(query[:name], query[:urls])
+      query[:urls] ? fetch_pages : start_with_search
+      social_search if query[:mail]
       other_search
+      do_the_twist
     end
 
     def start_with_search
@@ -23,6 +24,16 @@ module Ubi
     def other_search
     end
 
+    def fetch_pages
+      thema.araneas.each(&:work)
+    end
+
+    def do_the_twist
+      thema.araneas.each { |a| thema.try_datum(a) }
+      # pp thema.spec
+      # binding.pry if binding.respond_to?(:pry)
+    end
+
     delegate :spec, to: :thema
   end
 end

data/lib/ubi/impero.rb

@@ -1,3 +1,5 @@
+require 'pry'
+require 'thor'
 require 'paint'
 
 module Ubi
@@ -14,7 +16,19 @@ module Ubi
     LONG
     option :address, type: :string # 'Subject\'s address'
     def find(name)
-      Ubi::Artifex.new(name).spec
+      Ubi::Artifex.new(name: name).spec
+    end
+
+    # desc 'init', 'creates settings on ~'
+    desc 'site', 'Lookup in site'
+    long_desc <<-LONG
+
+    Find data on URL
+
+    LONG
+    option :address, type: :string # 'Subject\'s address'
+    def site(*url)
+      Ubi::Artifex.new(urls: url).spec
     end
   end
 end

data/lib/ubi/memoria.rb

@@ -24,6 +24,11 @@ module Ubi
         format
       end
 
+      def ==(other)
+        return unless other.respond_to?(:text)
+        text == other.text
+      end
+
       class << self
         #
         # Account for memorias
@@ -37,6 +42,7 @@ module Ubi
         def extract_text(datum)
           case datum
           when String then datum
+          when Ubi::Aranea then datum.text
           when Nokogiri::HTML then datum.data.text
           # when PDF / DOC / IMG (tesseract it =) then datum.data.text
           else fail "Can't parse `#{datum.class}`"
@@ -79,11 +85,6 @@ module Ubi
         def plural
           "#{key}s"
         end
-
-        def ==(other)
-          return unless other.respond_to?(:key)
-          key == other.key
-        end
       end
     end
   end

data/lib/ubi/memorias/address.rb

@@ -38,7 +38,8 @@ module Ubi
 
       def fetch_possible
         parse_zip
-        @region = clean.scan(/\W([A-Z]{2})\W/).first.first
+        @region = clean.scan(/\W([A-Z]{2})\W/).first
+        @region = @region.first if @region
         @number = clean.scan(/\d+/).join(' ')
       end
       #
@@ -53,7 +54,7 @@ module Ubi
       end
 
       def format(location = :br)
-        text.sub(*self.class.formats[location])
+        text #.sub(*self.class.formats[location])
       end
 
       class << self
@@ -85,7 +86,7 @@ module Ubi
         end
 
         def regex(hint)
-          /((?:#{REGEXES[hint][:prefix].join('|')}).*)/i
+          /(\b(?:#{REGEXES[hint][:prefix].join('|')})\s.*)\b/i
         end
 
         def plural

data/lib/ubi/memorias/email.rb

@@ -2,6 +2,16 @@ module Ubi
   module Memoria
     # An Electronic Mail
     class Email < Base
+      #
+      # Clean up regex on init
+      #
+      def initialize(text, _hint = nil, opts = {})
+        text = text.downcase.gsub(/^\(|\.$/, '')
+        @text = text
+        # @addr = text
+        @opts = opts
+      end
+
       #
       #
       # Class methods
@@ -15,7 +25,7 @@ module Ubi
             ([a-z0-9!#$%&'*+/=?^_`{|}~-]+
             (?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@
             (?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+
-            (?:[a-z0-9-]*[a-z0-9])?)
+            (?:[a-z0-9-]*[a-z0-9])?)(?:\W|\s|$)
           }x
         end
       end

data/lib/ubi/memorias/image.rb

@@ -0,0 +1,25 @@
+module Ubi
+  module Memoria
+    # Show me your papers!
+
+    class Image < Base
+      attr_reader :size
+
+      def parser
+        @size = []
+      end
+      #
+      #
+      # Class methods
+      #
+      class << self
+        #
+        # Email regex
+        #
+        def regex(_hint)
+          /\.(jpg|png|svg)$/i
+        end
+      end
+    end
+  end
+end

data/lib/ubi/memorias/phone.rb

@@ -9,16 +9,20 @@ module Ubi
       end
 
       def format
+        number && number.e164
+      end
+
+      def national
         number && number.national
       end
 
-      def rfc
+      def international
         number && number.international
       end
 
       class << self
         # http://rubular.com/r/tEHB6KcZzk
-        def regex(hint = nil)
+        def regex(_hint = nil)
           /
             (?:^|\s)((?:\+\(?\d{1,3}\W)?[\._\-\/\s]*
             \(?\s*?\d{2,3}\s*?\)?[\._\-\/\s]*\d{3,5}

data/lib/ubi/memorias/site.rb

@@ -2,10 +2,27 @@ module Ubi
   module Memoria
     # A site, url and title?
     class Site < Base
+      attr_accessor :link, :uri
+
+      def initialize(text, _hint = nil, opts = {})
+        text = text.downcase.gsub(/^\(|\.$/, '')
+        text = "http://#{text}" unless text =~ %r{^\w{3,5}://}
+        @text = text
+        @link = tld_parser
+        @opts = opts
+      end
+
+      def tld_parser
+        @uri = URI.parse(text)
+        @link = PublicSuffix.parse(uri.host)
+      rescue PublicSuffix::DomainInvalid
+        nil
+      end
+
       #
       # Prefix http:// if there isn't one defined
       def format
-        text =~ /^http/ ? text : "http://#{text}"
+        text
       end
 
       class << self

data/lib/ubi/memorias/social.rb

@@ -11,7 +11,7 @@ module Ubi
           }
         end
 
-        def regex
+        def regex(url = nil)
           %r{https?://(?:\w+\.)*(#{url}/.*)}
         end
       end

data/lib/ubi/memorias/who.rb

@@ -11,7 +11,7 @@ module Ubi
           }
         end
 
-        def regex
+        def regex(url)
           %r{https?://(?:\w+\.)*(#{url}/.*)}
         end
       end

data/lib/ubi/thema.rb

@@ -12,6 +12,8 @@ module Ubi
     def initialize(name, urls = [], opts = {})
       @name = name
       @urls = urls
+      @name ||= urls.first.split('.').first.gsub(%r{^\w+://}, '')
+      @urls = @urls.map { |url| url =~ %r{://} ? url : "http://#{url}" }
       @opts = opts
       @cache = Ubi.memorias.reduce({}) { |a, e| a.merge(e => opts[e]) }
       reduce_names
@@ -22,36 +24,38 @@ module Ubi
     end
 
     def reduce_names
+      return unless name
       @ascii = name.mb_chars.downcase
       @downcase = name.mb_chars.downcase
       @clean = @downcase.gsub(/\W/, ' ')
     end
 
+    # Define memorias on thema
     Ubi.memorias.each do |memoria|
       define_method memoria.plural do
-        instance_variable_get('@' + memoria.plural) ||
-          instance_variable_set('@' + memoria.plural, [])
+        instance_variable_get("@#{memoria.plural}") ||
+          instance_variable_set("@#{memoria.plural}", [])
       end
     end
 
-    def [](arg)
-      @cache[arg]
-    end
-
     def spec
       puts self
       Ubi.memorias.each do |memoria|
-        print Paint[memoria.name, :black]
-        puts self[memoria.key]
+        d = send(memoria.plural)
+        puts Paint["#{memoria.name} (#{d.size})", :black]
+        puts d
       end
     end
 
-    def try_consultor(a)
-      a = a.new(self)
+    def try_datum(a)
       Ubi.memorias.each do |m|
-        puts Paint["Trying to find #{m} in #{a.class}", :green]
-        @cache[m] = matches = m.parse(a.datum)
-        puts matches if matches && !matches.empty?
+        print Paint["Trying to find #{m} in page ", :green]
+        matches = m.parse(a)
+        puts Paint[matches.size, :black]
+        next unless matches && !matches.empty?
+        matches.each do |match|
+          send(m.plural) << match unless send(m.plural).include?(match)
+        end
       end
     end
 

data/lib/ubi/version.rb

@@ -1,4 +1,4 @@
 # :nodoc:
 module Ubi
-  VERSION = '0.0.5'
+  VERSION = '0.0.7'
 end

data/spec/spec_helper.rb

@@ -38,5 +38,4 @@ RSpec.configure do |config|
   #   # VCR.use_cassette(vcr) { example.run }
   #   puts Paint[" [#{Time.now - t}s]", :black]
   # end
-  # config.before(:each) { Polipus::SignalHandler.disable }
 end

data/spec/ubi/aranea_spec.rb

@@ -8,7 +8,7 @@ describe Aranea do
 
   it 'should use only one url' do
     aranea = Thema.new('Rock', ['r1.com', 'r2.com']).araneas.first
-    expect(aranea.url).to eq('r1.com')
+    expect(aranea.url).to eq('http://r1.com')
   end
 
   # it 'should crawl a domain VCR ruby-lang', :vcr do

data/spec/ubi/memorias/email_spec.rb

@@ -12,6 +12,24 @@ describe Memoria::Email do
     end
   end
 
+  describe 'parsed' do
+    def parse(site)
+      Memoria::Email.parse(site).first.to_s
+    end
+
+    describe 'http' do
+      let(:parsed) { 'hoho@fubah.com' }
+
+      it { expect(parse('hoho@fubah.com')).to eq(parsed) }
+      it { expect(parse('hoho@fubah.com.')).to eq(parsed) }
+      it { expect(parse('"hoho@fubah.com"')).to eq(parsed) }
+      it { expect(parse('( hoho@fubah.com )')).to eq(parsed) }
+      it { expect(parse('(hoho@fubah.com)')).to eq(parsed) }
+      it { expect(parse('(hoho@fubah.com.)')).to eq(parsed) }
+      it { expect(parse('hoho@fubah.com/')).to eq(parsed) }
+    end
+  end
+
   describe 'valid' do
     %w(
       foo@foo.com

data/spec/ubi/memorias/phone_spec.rb

@@ -1,6 +1,10 @@
 require 'spec_helper'
 
 describe Memoria::Phone do
+  def parse(phone)
+    Memoria::Phone.parse(phone).first.to_s
+  end
+
   describe 'simple test' do
     subject { Memoria::Phone.new('+551112345678') }
 
@@ -29,11 +33,11 @@ describe Memoria::Phone do
     end
   end
 
-  describe 'parsed landlines' do
-    def parse(phone)
-      Memoria::Phone.parse(phone).first.rfc
-    end
+  describe 'random tests' do
+    it { expect(parse('(16) 3919-0723')).to eq('+551639190723') }
+  end
 
+  describe 'parsed landlines' do
     let(:parsed) { '+1112345678' }
 
     load_fixture('phone.txt').each_line do |l|
@@ -44,11 +48,7 @@ describe Memoria::Phone do
   end
 
   describe 'parsed mobiles txt' do
-    def parse(phone)
-      Memoria::Phone.parse(phone).first.rfc
-    end
-
-    let(:parsed) { '+55 11 99814-5678' }
+    let(:parsed) { '+5511998145678' }
 
     load_fixture('mobile.txt').each_line do |l|
       it "Should parse phone #{l}" do
@@ -58,11 +58,7 @@ describe Memoria::Phone do
   end
 
   describe 'parsed mobiles' do
-    def parse(phone)
-      Memoria::Phone.parse(phone).first.rfc
-    end
-
-    let(:parsed) { '+55 11 98234-5678' }
+    let(:parsed) { '+5511982345678' }
 
     it { expect(parse('11982345678')).to eq(parsed) }
     it { expect(parse('11 982345678')).to eq(parsed) }

data/spec/ubi/memorias/site_spec.rb

@@ -8,7 +8,7 @@ describe Memoria::Site do
     it { is_expected.to be_an Array }
 
     it 'should have text reader method' do
-      expect(subject[0].text).to eq('somesite.com')
+      expect(subject[0].text).to eq('http://somesite.com')
     end
   end
 
@@ -21,6 +21,8 @@ describe Memoria::Site do
       let(:parsed) { 'http://fubah.com' }
 
       it { expect(parse('fubah.com')).to eq(parsed) }
+      it { expect(parse('fubah.com.')).to eq(parsed) }
+      it { expect(parse('(fubah.com.)')).to eq(parsed) }
       it { expect(parse('@fubah.com')).to eq(parsed) } # 'http://@fubah.com') }
       it { expect(parse('fu@fubah.com')).to eq(parsed) } # 'http://fu@fubah.com') }
       it { expect(parse('http://fubah.com')).to eq(parsed) }

data/spec/ubi/thema_spec.rb

@@ -6,7 +6,7 @@ describe Thema do
   end
 
   it 'should accept urls too' do
-    expect(Thema.new('Hard Rock', ['hr.com']).urls).to include('hr.com')
+    expect(Thema.new('Hard Rock', ['hr.com']).urls).to include('http://hr.com')
   end
 
   it 'should convert urls to aranea' do

data/ubi.gemspec

@@ -26,9 +26,12 @@ Gem::Specification.new do |s|
   s.add_dependency 'paint'
   s.add_dependency 'phonelib'
   #  s.add_dependency 'addressie', '~> 0.0.0'
-  s.add_dependency 'polipus'
   s.add_dependency 'nokogiri'
+  s.add_dependency 'whatlanguage'
   s.add_dependency 'geopolitical'
+  s.add_dependency 'public_suffix'
+  s.add_dependency 'rubyretriever'
+  s.add_dependency 'charlock_holmes'
 
   s.add_development_dependency 'vcr'
   s.add_development_dependency 'rspec'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ubi
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.7
 platform: ruby
 authors:
 - Marcos Piccinini
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-22 00:00:00.000000000 Z
+date: 2015-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -53,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: polipus
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: whatlanguage
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -94,6 +94,48 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: public_suffix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubyretriever
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: charlock_holmes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
@@ -171,6 +213,7 @@ files:
 - lib/ubi/memorias/address.rb
 - lib/ubi/memorias/document.rb
 - lib/ubi/memorias/email.rb
+- lib/ubi/memorias/image.rb
 - lib/ubi/memorias/phone.rb
 - lib/ubi/memorias/site.rb
 - lib/ubi/memorias/social.rb
@@ -212,7 +255,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Find devices connected in LAN