ubi 0.0.3 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 330dfff07aab154b3fc714a71b10cf0f148d541d
-  data.tar.gz: 81ff08e9a049671a6f2c51c43641007466fa30c3
+  metadata.gz: 9a9073e8291c6e092b1f3585191aa4c2c91e524d
+  data.tar.gz: ceee4c37c817147c7f4a8f0d4627b3c5108ccbf5
 SHA512:
-  metadata.gz: fae80b06b1e6f5bac23357f8b4c09ef45879ba62b4e1e6e90c9e42a16490f7496cc614b27538beaed7d6529d8784fce29a79722e4a351379079cc0e4be6bf1cb
-  data.tar.gz: 4a4ec085b39e27a0e64addcf9c22fff3a969bdff21a72c8684bcb500451d3c4503eb1935383fd96715dd72257082f4851476b9b2c8d4a26244b6363b9264588b
+  metadata.gz: 796e569c777e65d7a8aca7ceb5cb9579fc5f52bfdef22a16cb8b54ec2b48854e0ee207c66d90e2c6bead5fa5622ae328760cf8bad111859059695002871ad6c3
+  data.tar.gz: 7791238d12cf001ed5e2d36d86645e42debe21f3a968ed9cc3c21e18636dcba4c1c44ce7714e526f78330827cb812826c3d261f4542a17729799452b75aa996e

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+env: CI="travis"
+rvm:
+  - 2.2
+  - rbx-2
+  - jruby-head
+  - ruby-head
+
+matrix:
+  allow_failures:
+    - rvm: jruby-head
+
+sudo: false

data/Gemfile

@@ -9,3 +9,5 @@ gem 'rubocop'
 gem 'guard'
 gem 'guard-rspec'
 gem 'guard-rubocop'
+
+gem 'coveralls', require: false

data/Guardfile

@@ -11,6 +11,6 @@ guard :rspec, cmd: 'bundle exec rspec' do
   watch(/^lib\/(.+)\.rb$/)     { |m| "spec/#{m[1]}_spec.rb" }
   watch(/^generators\/(.+)\.rb$/) { |_m| 'spec/schemaless/worker_spec' }
 
-  watch('lib/ubi.rb')     { 'spec' }
-  watch('spec/spec_helper.rb')  { 'spec' }
+  watch('lib/ubi.rb') { 'spec' }
+  watch('spec/spec_helper.rb') { 'spec' }
 end

data/README.md

@@ -5,6 +5,14 @@
                                        \___/ |_.__/ |_|
 
 
+
+[![Gem Version](https://badge.fury.io/rb/ubi.svg)](http://badge.fury.io/rb/ubi)
+[![Code Climate](https://codeclimate.com/github/nofxx/ubi.svg)](https://codeclimate.com/github/nofxx/ubi)
+[![Coverage Status](https://coveralls.io/repos/nofxx/ubi/badge.svg?branch=master)](https://coveralls.io/r/nofxx/ubi?branch=master)
+[![Dependency Status](https://gemnasium.com/nofxx/ubi.svg)](https://gemnasium.com/nofxx/ubi)
+[![Build Status](https://travis-ci.org/nofxx/ubi.svg?branch=master)](https://travis-ci.org/nofxx/ubi)
+
+
 Ubi finds information in the subject's webpage(s).
 A forager.
 
@@ -24,7 +32,7 @@ A forager.
 - Social
 - Files
 - Documents
-- Address
+- Addresses
 - Logo/Images
 
 

data/Rakefile

@@ -6,4 +6,4 @@ require 'rubocop/rake_task'
 RSpec::Core::RakeTask.new
 RuboCop::RakeTask.new
 
-task default: [:spec, :rubocop]
+task default: [:rubocop, :spec]

data/lib/ubi/consultor.rb

@@ -38,8 +38,8 @@ module Ubi
 
       class << self
         def inherited(base)
-          puts "Using aranea #{base}"
           Ubi.araneas << base
+          # puts "Using aranea #{base}"
         end
 
         #

data/lib/ubi/memoria.rb

@@ -3,17 +3,21 @@ module Ubi
     # Memoria Base
     class Base
       include ActiveModel::Validations
-      attr_accessor :value
+      attr_accessor :text, :hint, :opts
 
-      def initialize(value, aranea = nil, thema = nil)
-        @value = value
-        @aranea = aranea
-        @thema = thema
+      def initialize(text, hint = nil, opts = {})
+        @text = text
+        @hint = hint
+        @opts = opts
+        parser
+      end
+
+      def parser
+        # Implemented on subclasses
       end
 
-      # Format for #to_s
       def format
-        value.downcase
+        text
       end
 
       def to_s
@@ -26,7 +30,7 @@ module Ubi
         #
         def inherited(base)
           fail "Already defined #{base.key}" if Ubi.memorias.include?(base)
-          puts "With memoria #{base}"
+          # puts "With memoria #{base}"
           Ubi.memorias << base
         end
 
@@ -34,17 +38,31 @@ module Ubi
           case datum
           when String then datum
           when Nokogiri::HTML then datum.data.text
+          # when PDF / DOC / IMG (tesseract it =) then datum.data.text
           else fail "Can't parse `#{datum.class}`"
           end
         end
 
-        def parse(datum)
-          fail "Not implemented by #{self}" unless regex
-          extract_text(datum).scan(regex).map { |r| new(r.first) }
+        #
+        # Scan for memoria regex and map to new memoria if found
+        #
+        # @returns Array [Memoria, Memoria...]
+        def parse(datum, hint = :br)
+          fail "Not implemented by #{self}" unless defined?(:regex)
+          extract_text(datum).scan(regex(hint))
+            .map { |r| new(r.first, hint) }
+        end
+
+        # Scans and removes matches from original
+        #
+        def parse!(datum, hint = :br)
+          res = parse(datum, hint)
+          res.each { |m| datum.tap { |d| d.slice!(m.text) }.strip! }
+          res
         end
 
         #
-        # Human-readable name of the aranea
+        # Machine-readable name of this class
         #
         def key
           @key ||= to_s.split('::').last.downcase.to_sym
@@ -52,7 +70,7 @@ module Ubi
         end
 
         #
-        # Human-readable name of the aranea
+        # Human-readable name of this class
         #
         def name
           to_s.split('::').last

data/lib/ubi/memorias/address.rb

@@ -1,15 +1,24 @@
 module Ubi
   module Memoria
-    # An adress in this world
+    # An address in this world
+    #
+    # Ignorace < Bliss
+    #
+    # Yeah, this needs lots of work.
+    # Geonames in memory country with zip/cities and regions,
+    # and some neural thing. And geocode to use openstreet or else.
+    #
+    # Don't hesitate to improve your AI skills here.
+    #
     class Address < Base
-      DIVIDERS = /[,\-\|\/]/
+      DIVIDERS = %r{[,\-\|/]}
       SPLIT = /(?<=\D)#{DIVIDERS}|#{DIVIDERS}(?=\D)/
       REGEXES = {
         br: {
-          prefix: %w( r rua av avenida pç pça praça pc pca praca tv travessa est estrada rod rodovia ),
+          prefix: %w( r rua av avenida pc pca praça pc pca praca tv travessa est estrada rod rodovia ),
           number: %w( n no nº num numero km ),
-          ext: %w( comp obs ap apto apart apartamento andar  ),
-          zip: /\d{5}[-]\d{3}/
+          ext: %w( comp obs ap apto apart apartamento andar ),
+          zip: /\d{5}\s?[-]\s?\d{3}/
         },
         us: {
           prefix: %w( st street av avenue road ),
@@ -18,23 +27,33 @@ module Ubi
       }
 
       attr_accessor :name, :parts, :words, :zip, :place, :number,
-                    :city, :region, :nation, :extra
+                    :city, :region, :nation, :extra, :clean
+
+      def parse_zip
+        @zip = text.scan(REGEXES[:br][:zip]).first
+        return unless zip
+        @zip = zip.gsub(/\D/, '').sub(*Address.zip_format[:br])
+        clean.slice!(zip)
+      end
+
+      def fetch_possible
+        parse_zip
+        @region = clean.scan(/\W([A-Z]{2})\W/).first.first
+        @number = clean.scan(/\d+/).join(' ')
+      end
       #
       #
       # Init, remove non word chars
       #
-      def initialize(val, _location = :br)
-        @value = Address.sanitize(val)
-        # @zip = value.match(REGEXES[location][:zip])
-        # @region = value.match(/\W([A-Z]{2})\W/)[1]
-        # @number = value.match(/\w*\d+\w*/)
-
-        @parts = value.split(SPLIT).map { |v| v.strip.chomp }
+      def parser
+        @clean = Address.sanitize(text)
+        @parts = clean.split(SPLIT).map { |v| v.strip.chomp }
         @words = parts.map { |pt| pt.split(/\s+/) }
+        fetch_possible
       end
 
       def format(location = :br)
-        value.sub(*self.class.formats[location])
+        text.sub(*self.class.formats[location])
       end
 
       class << self
@@ -45,8 +64,9 @@ module Ubi
         # "\n" -> "-"
         # " -" -> "-"
         #
-        def sanitize(value)
-          value.gsub(/\s+/, ' ').gsub(/\\n/, '-')
+        def sanitize(txt)
+          v = ActiveSupport::Inflector.transliterate(txt)
+          v.gsub(/\s+/, ' ').gsub(/\\n/, '-')
             .gsub(/\s?(#{DIVIDERS})\s?/, '\1')
         end
 
@@ -58,8 +78,14 @@ module Ubi
           }
         end
 
-        def regex
-          /((?:#{REGEXES[:br][:prefix].join('|')}).*)/i
+        def zip_format
+          {
+            br: [/(\d{5})(\d{3})/, '\1-\2']
+          }
+        end
+
+        def regex(hint)
+          /((?:#{REGEXES[hint][:prefix].join('|')}).*)/i
         end
 
         def plural

data/lib/ubi/memorias/document.rb

@@ -9,16 +9,14 @@ module Ubi
     # cl: RUN/RUT
     #
     class Document < Base
-      #
-      #
-      # Init, remove non word chars
-      #
-      def initialize(value)
-        @value = value.gsub(/\W/, '')
+      attr_reader :number
+
+      def parser
+        @number = text.gsub(/\D/, '')
       end
 
       def format(location = :br)
-        value.sub(*self.class.formats[location])
+        number.sub(*self.class.formats[location])
       end
 
       #
@@ -27,7 +25,7 @@ module Ubi
       class << self
         def regexes
           {
-            br: /(\d{14}|\d{2}\.?\d{3}\.?\d{3}\/?\d{4}[-]?\d{2})/,
+            br: /(\d{14}|\d{2}\.?\d{3}\.?\d{3}\/?\d{4}\-?\d{2})/,
             cl: /\d{2}\.\d{3}\.\d{3}[-][0-9kK]/,
             us: /\d{3}[-]\d{2}[-]\d{4}/
           }

data/lib/ubi/memorias/email.rb

@@ -10,8 +10,13 @@ module Ubi
         #
         # Email regex
         #
-        def regex
-          %r{([a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)} # rubocop:disable Metrics/LineLength
+        def regex(_hint)
+          %r{
+            ([a-z0-9!#$%&'*+/=?^_`{|}~-]+
+            (?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@
+            (?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+
+            (?:[a-z0-9-]*[a-z0-9])?)
+          }x
         end
       end
     end

data/lib/ubi/memorias/phone.rb

@@ -2,19 +2,13 @@ module Ubi
   module Memoria
     # A Phone! mobile? landline? who is calling???
     class Phone < Base
-      attr_reader :number, :chunk
+      attr_reader :number
 
-      def initialize(chunk, hint = nil)
-        @hint = hint
-        @chunk = chunk
-        parse_number
+      def parser
+        @number = Phonelib.parse(text.gsub(/\D/, ''), hint)
       end
 
-      def parse_number
-        @number = Phonelib.parse(chunk.gsub(/\D/, ''), @hint)
-      end
-
-      def to_s
+      def format
         number && number.national
       end
 
@@ -24,10 +18,14 @@ module Ubi
 
       class << self
         # http://rubular.com/r/tEHB6KcZzk
-        def regex
-          /(?:^|\s)((?:\+\(?\d{1,3}\W)?[\._\-\/\s]*\(?\s*?\d{2,3}\s*?\)?[\._\-\/\s]*\d{3,5}[\._\-\/\s]*\d{4,5})(?:\s|$)/
+        def regex(hint = nil)
+          /
+            (?:^|\s)((?:\+\(?\d{1,3}\W)?[\._\-\/\s]*
+            \(?\s*?\d{2,3}\s*?\)?[\._\-\/\s]*\d{3,5}
+            [\._\-\/\s]*\d{4,5})(?:\s|$)
+          /x
         end
       end
-    end
-  end
-end
+    end # Phone
+  end # Memoria
+end # Ubi

data/lib/ubi/memorias/site.rb

@@ -5,19 +5,31 @@ module Ubi
       #
       # Prefix http:// if there isn't one defined
       def format
-        value =~ /http/ ? value : "http://#{value}"
+        text =~ /^http/ ? text : "http://#{text}"
       end
 
       class << self
         #
         # Regex only for *.tld
-        def regex
-          # %r{https?://((?:\w+[\./]?)+)(?:/|\.)}
-          # (?:\??)[a-zA-Z0-9\-\._\?\,\'\/\\\+&%\$#\=~]+
-          # %r{(?:https?\://)?(?:www\.)?[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}}
-          # ((?:https\:\/\/)|(?:http\:\/\/)|(?:www\.))?([a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(?:\??)[a-zA-Z0-9\-\._\?\,\'\/\\\+&%\$#\=~]+)
-          # http://www.regexr.com/3bkne
-          /([(https?):\/\/(www\.)?a-zA-Z0-9@:%\._\+~#=]{2,256}\.[a-z]{2,6}\b(?:[-a-zA-Z0-9@:%_\+.~#?&\/\/?=]*))/
+        # %r{https?://((?:\w+[\./]?)+)(?:/|\.)}
+        # (?:\??)[a-zA-Z0-9\-\._\?\,\'\/\\\+&%\$#\=~]+
+        #
+        # %r{(?:https?\://)?(?:www\.)?[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}}
+        #
+        # ((?:https\:\/\/)|(?:http\:\/\/)|(?:www\.))?([a-zA-Z0-9\-\.]+\.
+        # [a-zA-Z]{2,3}(?:\??)[a-zA-Z0-9\-\._\?\,\'\/\\\+&%\$#\=~]+)
+        #
+        # http://www.regexr.com/3bkne
+        # /([(https?):\/\/(www\.)?a-zA-Z0-9:@%\._\+~#=]{2,256}\.[a-z]{2,6}\b
+        # (?:[-a-zA-Z0-9@:%_\+.~#?&\/\/?=]*))/
+        #
+        # without @
+        def regex(_hint)
+          %r{
+           ([(https?)://(www\.)?a-zA-Z0-9:%\._\+~#=]{2,256}
+           \.[a-z]{2,6}\b
+           (?:[-a-zA-Z0-9@:%_\+.~#?&//?=]*))
+          }x
         end
 
         def key

data/lib/ubi/version.rb

@@ -1,4 +1,4 @@
 # :nodoc:
 module Ubi
-  VERSION = '0.0.3'
+  VERSION = '0.0.5'
 end

data/spec/spec_helper.rb

@@ -3,6 +3,8 @@ require 'paint'
 require 'pry'
 require 'vcr'
 require 'webmock/rspec'
+require 'coveralls'
+Coveralls.wear!
 
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))

data/spec/ubi/memorias/address_spec.rb

@@ -2,13 +2,13 @@ require 'spec_helper'
 
 describe Memoria::Address do
   [
-    'R Bahia,55 - Una/BR - 12345-678',
-    'R Bahia,55 - Una/BR - 12345- 678',
-    'R Bahia, 55 - Una/BR - 12345-678',
-    'R Bahia, 55  - Una/BR - 12345-678',
-    'R Bahia, 55 /  Una/BR - 12345-678',
-    'R Bahia, 55 \n  Una/BR \n 12345-678',
-    'R Bahia, 55,   Una/BR - 12345-678'
+    'R Bahia,55 - Una/BA - 12345-678',
+    'R Bahia,55 - Una/BA - 12345- 678',
+    'R Bahia, 55 - Una/BA - 12345-678',
+    'R Bahia, 55  - Una/BA - 12345-678',
+    'R Bahia, 55 /  Una/BA - 12345-678',
+    'R Bahia, 55 \n  Una/BA \n 12345-678',
+    'R Bahia, 55,   Una/BA - 12345-678'
   ].each do |chunk|
     describe "simple delimited `#{chunk}`" do
       subject { Memoria::Address.parse(chunk) }
@@ -22,14 +22,14 @@ describe Memoria::Address do
 
         it { expect(addr.zip.to_s).to eq('12345-678') }
         it { expect(addr.number.to_s).to eq('55') }
-        it { expect(addr.region.to_s).to eq('BR') }
+        it { expect(addr.region.to_s).to eq('BA') }
 
         it 'should split into parts' do
-          expect(addr.parts).to eq(['R Bahia', '55', 'Una', 'BR', '12345-678'])
+          expect(addr.parts).to eq(['R Bahia', '55', 'Una', 'BA', '12345-678'])
         end
 
         it 'should split into words' do
-          expect(addr.words).to eq([%w(R Bahia), ['55'], ['Una'], ['BR'], ['12345-678']])
+          expect(addr.words).to eq([%w(R Bahia), ['55'], ['Una'], ['BA'], ['12345-678']])
         end
       end
     end

data/spec/ubi/memorias/email_spec.rb

@@ -6,6 +6,10 @@ describe Memoria::Email do
 
     it { is_expected.to include(Memoria::Email) }
     it { is_expected.to be_an Array }
+
+    it 'should have text reader method' do
+      expect(subject[0].text).to eq('me@somesite.com')
+    end
   end
 
   describe 'valid' do

data/spec/ubi/memorias/phone_spec.rb

@@ -5,6 +5,28 @@ describe Memoria::Phone do
     subject { Memoria::Phone.new('+551112345678') }
 
     it { is_expected.to be_a Memoria::Phone }
+
+    it 'should have text reader method' do
+      expect(subject.text).to eq('+551112345678')
+    end
+
+    it 'should have text reader method' do
+      expect(subject.number).to be_a Phonelib::Phone
+    end
+  end
+
+  describe 'ill formated simple test' do
+    subject { Memoria::Phone.new('+55(11) 1234-5678') }
+
+    it { is_expected.to be_a Memoria::Phone }
+
+    it 'should have text reader method' do
+      expect(subject.text).to eq('+55(11) 1234-5678')
+    end
+
+    it 'should have text reader method' do
+      expect(subject.number).to be_a Phonelib::Phone
+    end
   end
 
   describe 'parsed landlines' do
@@ -26,7 +48,7 @@ describe Memoria::Phone do
       Memoria::Phone.parse(phone).first.rfc
     end
 
-    let(:parsed) { '+11998145678' }
+    let(:parsed) { '+55 11 99814-5678' }
 
     load_fixture('mobile.txt').each_line do |l|
       it "Should parse phone #{l}" do
@@ -40,7 +62,7 @@ describe Memoria::Phone do
       Memoria::Phone.parse(phone).first.rfc
     end
 
-    let(:parsed) { '+11982345678' }
+    let(:parsed) { '+55 11 98234-5678' }
 
     it { expect(parse('11982345678')).to eq(parsed) }
     it { expect(parse('11 982345678')).to eq(parsed) }

data/spec/ubi/memorias/site_spec.rb

@@ -2,10 +2,14 @@ require 'spec_helper'
 
 describe Memoria::Site do
   describe 'simple test' do
-    subject { Memoria::Site.parse('bla bla me@somesite.com') }
+    subject { Memoria::Site.parse('bla bla somesite.com') }
 
     it { is_expected.to include(Memoria::Site) }
     it { is_expected.to be_an Array }
+
+    it 'should have text reader method' do
+      expect(subject[0].text).to eq('somesite.com')
+    end
   end
 
   describe 'parsed' do
@@ -17,8 +21,8 @@ describe Memoria::Site do
       let(:parsed) { 'http://fubah.com' }
 
       it { expect(parse('fubah.com')).to eq(parsed) }
-      it { expect(parse('@fubah.com')).to eq('http://@fubah.com') }
-      it { expect(parse('fu@fubah.com')).to eq('http://fu@fubah.com') }
+      it { expect(parse('@fubah.com')).to eq(parsed) } # 'http://@fubah.com') }
+      it { expect(parse('fu@fubah.com')).to eq(parsed) } # 'http://fu@fubah.com') }
       it { expect(parse('http://fubah.com')).to eq(parsed) }
       it { expect(parse('http://fubah.com/56')).to eq(parsed + '/56') }
     end

data/ubi.gemspec

@@ -22,13 +22,12 @@ Gem::Specification.new do |s|
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.require_paths = ['lib']
 
-  s.add_dependency 'thor' #, '~> 0.19.1'
-  s.add_dependency 'paint' #, '~> 0.9.0'
+  s.add_dependency 'thor'
+  s.add_dependency 'paint'
   s.add_dependency 'phonelib'
-#  s.add_dependency 'phonie', '~> 3.1.0'
-#  s.add_dependency 'addressie', '~> 0.0.0'
-  s.add_dependency 'polipus' #, '~> 0.4.0'
-  s.add_dependency 'nokogiri' #, '~> 1.6.0'
+  #  s.add_dependency 'addressie', '~> 0.0.0'
+  s.add_dependency 'polipus'
+  s.add_dependency 'nokogiri'
   s.add_dependency 'geopolitical'
 
   s.add_development_dependency 'vcr'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ubi
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.5
 platform: ruby
 authors:
 - Marcos Piccinini
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-21 00:00:00.000000000 Z
+date: 2015-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -145,6 +145,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".travis.yml"
 - Gemfile
 - Guardfile
 - MIT-LICENSE