gimme_poc 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0e6506c640bcafe56d6906bfaae60f8a17e72865
-  data.tar.gz: 21e4fc7176c2d5aa28a0fe31ab993889a7fbf74d
+  metadata.gz: c2765f0ef20a443c6c37e7ffc7f2cfa83454ec6b
+  data.tar.gz: 9761000cf90a2aba6549dfa774a5aff7f6daebff
 SHA512:
-  metadata.gz: 39c35961983b31246c7414067d607bbc6c7a6dbddc5af9dfa97122ff094b881248db7a4a9c6415c33e393b3f59e5c632f0b06c3fe04e70dd9035892c5c2b11cf
-  data.tar.gz: b1a194c4f0d51d7b3ea77f81786cc5b2d981b7c27f22dbeb6db98f8c1cca688437d04a1af5c9193d25e92072f88c664468d59ff89a2a853695e59809792ee457
+  metadata.gz: 48f94a4ecd20ab3db3197abcedecabeb8fb8a0fe7b4274fbff3d23bea395363bae69fe7bd3f42b56a5e701599e4780c367cae513b2ad95f9622167d99844011f
+  data.tar.gz: 132fad4c4fdc308a0cc3b630314525d17e197e0dcc568f5690ffdcfd351e434b8a9b2d5d1a0e9be613077b155c7cd51ed2f17c7a3ad1e2919121c88a0126dc78

data/README.md

@@ -1,4 +1,5 @@
 # Gimme POC
+[![Build Status](https://travis-ci.org/m8ss/gimme_poc.svg?branch=master)](https://travis-ci.org/m8ss/gimme_poc) [![Code Climate](https://codeclimate.com/github/m8ss/gimme_poc/badges/gpa.svg)](https://codeclimate.com/github/m8ss/gimme_poc) [![Gem Version](https://badge.fury.io/rb/gimme_poc.svg)](https://badge.fury.io/rb/gimme_poc)
 
 Gimme POC (Point of Contact) simplifies the process of extracting the common 'contact us' information from a website.
 
@@ -13,6 +14,13 @@ gem install gimme_poc
 
 ```
 
+## Set Up
+
+```ruby
+require 'gimme_poc' # => that's it!
+
+```
+
 ## How it works
 
 Gimme POC is easy to use! Simply run this command.
@@ -60,7 +68,7 @@ Gimme.memory
 
 ## Clearing the search results
 
- To clear search results and start afresh, simply run:
+ To clear search results and start afresh, run:
 
 ```ruby
 
@@ -71,7 +79,7 @@ Gimme.reset!
 ## To do:
 
  - Convenience methods for returning specific information from all sites, (ie. just facebook or just twitter)
- - Work on false positives of bad urls.  (DNS redirects don't give 404 errors)
+ - Work on false positives of bad urls.  (Bad urls should be skipped + DNS redirects don't give 404 errors)
 
 
 More to follow...

data/Rakefile

@@ -1,22 +1,6 @@
 require 'rubygems'
-require 'bundler'
-
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts 'Run `bundle install` to install missing gems'
-  exit e.status_code
-end
 require 'rake'
 
-require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_gimme_poc*.rb'
-  test.verbose = true
-end
-
 desc 'Open console with gimme_poc loaded'
 task :console do
   exec 'pry -r ./lib/gimme_poc.rb'

data/lib/gimme_poc/version.rb

@@ -1,3 +1,3 @@
 module Gimme
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end

data/lib/gimme_poc.rb

@@ -9,6 +9,9 @@ module Gimme
 
     # Simple regex that looks for ###.#### or ###-####
     PHONE_REGEX = /\d{3}[-]\d{4}|\d{3}[.]\d{4}/
+    
+    # Captures http:// and https://
+    HTTP_REGEX = /\A\bhttps:\/\/|\bhttp:\/\//
 
     ## ----------------------------------------------------------------
     # Questions
@@ -16,51 +19,7 @@ module Gimme
     #
     #
 
-    # Boolean, returns true if contact link is present.
-    def contact_link?
-      @url = link_with_href('contact')
-
-      !@url.nil?
-    end
-
-    # True if contact page '../contact' does NOT get a 404 error.
-    def contact_page?
-      @url = page.uri.merge('../contact').to_s
-      begin
-        true if Mechanize.new.get(@url)
-      rescue Mechanize::ResponseCodeError
-        false
-      end
-    end
-
-    # Boolean, returns true if link to English version is present.
-    def english_link?
-      return false if page.link_with(href: /english/).nil?
-      @url = page.uri.merge(page.link_with(href: /english/).uri.to_s).to_s
-
-      !@url.nil?
-    end
-
-    # True if english page '../en' does NOT get a 404 error.
-    def en_page?
-      @url = page.uri.merge('../en').to_s
-      begin
-        true if Mechanize.new.get(@url)
-      rescue Mechanize::ResponseCodeError
-        false
-      end
-    end
-
-    # True if english page '../english' does NOT get a 404 error.
-    def english_page?
-      @url = page.uri.merge('../english').to_s
-      begin
-        true if Mechanize.new.get(@url)
-      rescue Mechanize::ResponseCodeError
-        false
-      end
-    end
-
+    ##
     # Boolean, returns true if anything is present
     # after running scan_for_contacts.
     def something_to_save?
@@ -77,8 +36,9 @@ module Gimme
       !(page.body =~ PHONE_REGEX).nil?
     end
 
+    ##
     # TODO: build better conditional to prevent false positives.
-    # There could be other forms like newsletter signup, etc.
+    #   There could be other forms like newsletter signup, etc.
     #
     # If there is a form with more than one field, this returns true.
     # Forms with one field are typically search boxes.
@@ -94,25 +54,43 @@ module Gimme
     #
     #
 
+    ##
     # The main method!
     # Takes array of urls and gets contact info for each if possible.
+    # If url is bad, it's converted to nil in 'get' method and skipped over.
     def poc(arr)
       arr = arr.split unless arr.is_a?(Array)
       arr.each do |url|
         puts '-' * 50
         puts "starting: #{url}"
-        get(url)
+        next if get(url).nil?
+        puts "now looking for contact pages"
         start_contact_links
-        go_to_contact_page
-        save_available_contacts
+        mechpage = go_to_contact_page
+        save_available_contacts(mechpage.uri.to_s)
       end
       Search.all_sites
     end
+    
+    # Mechanize needs absolute urls to work.
+    # If http:// or https:// isn't present, append http://.
+    def format_url(str)
+      str.prepend('http://') if (str =~ HTTP_REGEX).nil?
+      str
+    end
 
+    ##
     # Go to a page using Mechanize.
-    def get(url)
-      puts "sending GET request: #{url}..."
+    # Sleep for a split second to not overload any servers.
+    #
+    # Returns nil if bad url is given.
+    def get(str)
+      url = format_url(str)
+      puts "sending GET request to: #{url}"
+      sleep(0.1)
       @page = Mechanize.new { |a| a.user_agent_alias = 'Mac Safari' }.get(url)
+    rescue SocketError => e
+      puts "#{'skipping:'.red} -- #{e}"
     end
 
     # Starts/Restarts @contacts_links hash
@@ -121,55 +99,107 @@ module Gimme
       @contact_links = {}
     end
 
+    ##
     # Scans for contact page.  If it doesn't work on the first try,
-    # It will look for english versions and try again.
+    # It will look for english versions and try again. Processes left to right.
     #
-    # If contact page is found, go directly there and don't try again.
+    # Returns nil if no contact page can be found.
     def go_to_contact_page
-      1.times do
-        case
-        when contact_link?
-          puts 'found contact link!'.green
-          get(@url)
-        when contact_page?
-          puts 'found contact page!'.green
-          get(@url)
-        else
-          if english_link? # look for link first, not check the page.
-            puts 'found english link!'
-            get(@url)
-            redo
-          elsif en_page?
-            puts 'found en page!'
-            get(@url)
-            redo
-          elsif english_page?
-            puts 'found english page!'
-            redo
-          end
-        end
+      contact_page || english_contact_page
+    end
+
+    ##
+    # Looks for contact page.  Gets page if available.
+    # If no contact link is available, it will blind test '../contact'.
+    # Returns nil if nothing can be found.
+    def contact_page
+      contact_link = link_with_href(/contact|Contact/)
+      contact_test_page = merged_link('../contact')
+      
+      case
+      when !contact_link.nil?
+        puts "#{'success:'.green} Found contact link!\n"
+        get(merged_link(contact_link))
+      else
+        puts "#{'warning:'.yellow}couldn't find contact link"
+        blind_test(contact_test_page)
+      end
+    end
+
+    ##
+    # Looks for english page.  Gets page if available then looks for
+    # english contact page.
+    #
+    # If no english link is available,
+    # it will blind test '../en' and '../english'.
+    # Returns nil if nothing can be found.
+    def english_contact_page
+      puts "\nLooking for english page..."
+      english_link = page.link_with(href: /english|English/)
+      test_en_page = merged_link('../en')
+      test_english_page = merged_link('../english')
+      
+      case
+      when !english_link.nil?
+        puts "#{'success:'.green} found english link!"
+        get(merged(english_link)) # already merged link
+      else
+        blind_test(test_en_page) || blind_test(test_english_page)
+        puts "ready to start again"
+        contact_page
       end
     end
 
+    # TODO: Sometimes DNS will do a redirect and not give a 404.  
+    #   Need to prevent redirects.
+    #
+    # Blindly tests to see if a url goes through.  If there is a 404 error,
+    # this will return nil.
+    def blind_test(url)
+      begin
+        puts "\nblind testing: #{url}"
+        get(url)
+      rescue Mechanize::ResponseCodeError
+        puts "#{'404 Error:'.red} #{url}"
+      end
+    end
+    
+    ##
+    # Used in case of relative paths. Merging guarantees correct url.
+    # This needs a url string as argument to work.
+    # Produces a merged uri string.
+    def merged_link(url_str)
+      page.uri.merge(url_str).to_s
+    end
+
+    ##
     # Expects relative paths and merges everything.
     # Returns a string.  If there's nothing, return nil.
+    #
+    # Add \b word block to ensure whole word is searched.
     def link_with_href(str)
-      page.uri.merge(page.link_with(href: /#{str}/).uri.to_s).to_s rescue nil
+      merged_link(page.link_with(href: /\b#{str}/).uri.to_s)
+    rescue
+        nil
     end
 
+    ##
     # Returns anything that is possible to save, otherwise returns nil.
     # Booleans for phone, email, or contact form will display True or False.
+    #
+    # Add periods to link hrefs to prevent false positives. Must escape periods
+    # with a backslash or else it will be a regex wild card.
     def scan_for_contacts
       {
         contactpage: link_with_href('contact'),
         email_present: "#{email_available?}",
         phone_present: "#{phone_available?}",
         contact_form: "#{contactform_available?}",
-        facebook: link_with_href('facebook'),
-        twitter: link_with_href('twitter'),
-        youtube: link_with_href('youtube'),
-        googleplus: link_with_href('plus.google'),
-        linkedin: link_with_href('linkedin')
+        facebook: link_with_href('facebook\.'),
+        twitter: link_with_href('twitter\.'),
+        youtube: link_with_href('youtube\.'),
+        googleplus: link_with_href('plus\.google\.'),
+        linkedin: link_with_href('linkedin\.')
       }
     end
 
@@ -179,6 +209,7 @@ module Gimme
       @contact_links[key] = url
     end
 
+    ##
     # Remove negatives from the contacts hash.
     # Deletes a key value pair with a value of either nil or false.
     # Remember that false is a string.
@@ -187,8 +218,8 @@ module Gimme
     end
 
     # Saves any available contact info to @contact_links.
-    def save_available_contacts(hsh = scan_for_contacts)
-      puts 'saving available contact information...'
+    def save_available_contacts(url, hsh = scan_for_contacts)
+      puts "\nsaving available contact information from #{url}"
       return unless something_to_save?
       if hsh.is_a?(Hash)
         hsh.each do |k, v|
@@ -199,10 +230,10 @@ module Gimme
       else
         fail ArgumentError, "expected hash but got #{hsh.class}"
       end
-      Search::POC.new(@contact_links)
+      Search::POC.new(url, @contact_links)
     end
 
-    # Convenience method
+    # Convenience method.
     def memory
       Search.all_sites
     end
@@ -214,7 +245,7 @@ module Gimme
   end
 end
 
-# Collectino of sites searched
+# Collection of sites searched.
 class Search
   @all_sites = []
 
@@ -224,8 +255,9 @@ class Search
 
   # Each site is saved to this class
   class POC
-    def initialize(contact_info_hsh)
-      @contact_info = contact_info_hsh
+    def initialize(url, contact_info_hsh)
+      @host = url
+      @info = contact_info_hsh
       Search.all_sites << self
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gimme_poc
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - John Mason
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -98,7 +112,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: Get a point of contact.  Given a url or array of urls, extracts social media