gimme_poc 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6984d721d974f11b5dab3136323f68d924867ad6
-  data.tar.gz: 78503da039ce280bb1bb69898065b5d60a2d5463
+  metadata.gz: 6ca0656c586244edfaaac44d81b092956e1ed801
+  data.tar.gz: b2bbc85c51a79ba5a10a1a5e77d8084ac28adeee
 SHA512:
-  metadata.gz: 52770e7b95a98abc5638e7b017674f5c58b61b22bbed847bb009cc5092aea3f533922f0da9bd0a571c5e40050f26779c5734bb21c308f065742c66994f0faf16
-  data.tar.gz: b8405f565d973365c4bd05d6cab1f52295bd4b83c0fac858654e87d738cb3cf758096b6f5e2402716a2c768e11182479b71a6cad7c18869751a0fdadd9e6a7cf
+  metadata.gz: f2e131c8e68fb8a55169f62b1d9a5662901f6fcb45907878c30d8d700d784b66aa8cae0305c060a611da7d60b810cc2d56bb56959db1da378a3088fa6568d707
+  data.tar.gz: 6f32874d9fc287ce588baa0e91fa854ed7f75f045f88d798ebf02a0e781918f50d6922ffd06578965d7c033cacdb6a927d03efc7b7967c6a225d7579a8631853

data/lib/gimme_poc/contactpage.rb

@@ -0,0 +1,55 @@
+# Find the contact
+module Gimme
+  class << self
+    ##
+    # Scans for contact page.  If it doesn't work on the first try,
+    # It will look for english versions and try again. Processes left to right.
+    #
+    # Returns nil if no contact page can be found.
+    def go_to_contact_page(url)
+      contact_page(url) || english_contact_page(url)
+    end
+
+    ##
+    # Looks for contact page.  Gets page if available.
+    # If no contact link is available, it will blind test '../contact'.
+    # Returns nil if nothing can be found.
+    def contact_page(url)
+      contact_link = link_with_href(/contact|Contact/)
+      contact_test_page = merged_link('../contact')
+
+      case
+      when !contact_link.nil?
+        puts "#{'Success:'.green} Found contact link!\n"
+        get(merged_link(contact_link))
+      else
+        puts "#{'Warning:'.yellow} couldn't find contact link"
+        blind_test(contact_test_page) || get(orig_domain(url))
+      end
+    end
+
+    ##
+    # Looks for english page.  Gets page if available then looks for
+    # english contact page.
+    #
+    # If no english link is available,
+    # it will blind test '../en' and '../english'.
+    # Returns nil if nothing can be found.
+    def english_contact_page(url)
+      puts "\nLooking for english page..."
+      english_link = page.link_with(href: /english|English/)
+      test_en_page = merged_link('../en')
+      test_english_page = merged_link('../english')
+
+      case
+      when !english_link.nil?
+        puts "#{'Success:'.green} found english link!"
+        get(merged(english_link)) # already merged link
+      else
+        blind_test(test_en_page) || blind_test(test_english_page)
+        puts 'ready to start again'
+        contact_page(url)
+      end
+    end
+  end
+end

data/lib/gimme_poc/poc.rb

@@ -0,0 +1,21 @@
+module Gimme
+  # Collection of sites searched.
+  class Search
+    @all_sites = []
+
+    class << self
+      attr_accessor :all_sites
+    end
+
+    # Each site is saved to this class
+    class POC
+      attr_accessor :host, :info
+
+      def initialize(url, contact_info_hsh)
+        @host = url
+        @info = contact_info_hsh
+        Search.all_sites << self
+      end
+    end
+  end
+end

data/lib/gimme_poc/questions.rb

@@ -0,0 +1,33 @@
+# Find the contact
+module Gimme
+  class << self
+    ##
+    # Boolean, returns true if anything is present
+    # after running scan_for_contacts and deleting failures.
+    def something_to_save?(hsh)
+      delete_failures(hsh).any?
+    end
+
+    # Boolean, returns true if email is present.
+    def email_available?
+      !link_with_href('mailto').nil?
+    end
+
+    # Boolean, returns true if phone number is present.
+    def phone_available?
+      !(page.body =~ PHONE_REGEX).nil?
+    end
+
+    ##
+    # TODO: build better conditional to prevent false positives.
+    #   There could be other forms like newsletter signup, etc.
+    #
+    # If there is a form with more than one field, this returns true.
+    # Forms with one field are typically search boxes.
+    #
+    # Boolean, returns true if form is present on page.
+    def contactform_available?
+      !(page.forms.select { |x| x.fields.length > 1 }.empty?)
+    end
+  end
+end

data/lib/gimme_poc/save.rb

@@ -0,0 +1,53 @@
+module Gimme
+  class << self
+    ##
+    # Returns anything that is possible to save, otherwise returns nil.
+    # Booleans for phone, email, or contact form will display True or False.
+    #
+    # Add periods to link hrefs to prevent false positives. Must escape periods
+    # with a backslash or else it will be a regex wild card.
+    def scan_for_contacts
+      {
+        contactpage: link_with_href('contact'),
+        email_present: "#{email_available?}",
+        phone_present: "#{phone_available?}",
+        contact_form: "#{contactform_available?}",
+        facebook: link_with_href('facebook\.'),
+        twitter: link_with_href('twitter\.'),
+        youtube: link_with_href('youtube\.'),
+        googleplus: link_with_href('plus\.google\.'),
+        linkedin: link_with_href('linkedin\.')
+      }
+    end
+
+    # Used in save_available_contacts to save each valid link.
+    def save_link(key, url)
+      return if key.nil? || url.nil?
+      @contact_links[key] = url
+    end
+
+    ##
+    # Remove negatives from the contacts hash.
+    # Deletes a key value pair with a value of either nil or false.
+    # Remember that false is a string.
+    def delete_failures(hsh)
+      hsh.delete_if { |_k, v| v.nil? || v == 'false' }
+    end
+
+    # Saves any available contact info to @contact_links.
+    def save_available_contacts(url, hsh = scan_for_contacts)
+      return unless something_to_save?(hsh)
+      puts "\nsaving available contact information from #{url}"
+      if hsh.is_a?(Hash)
+        hsh.each do |k, v|
+          save_link(k, v) # saves to @contact_links
+        end
+        delete_failures(@contact_links)
+        puts "#{@contact_links}".cyan # same as @contact_links
+      else
+        fail ArgumentError, "expected hash but got #{hsh.class}"
+      end
+      Search::POC.new(url, @contact_links)
+    end
+  end
+end

data/lib/gimme_poc/version.rb

@@ -1,3 +1,3 @@
 module Gimme
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
 end

data/lib/gimme_poc/web.rb

@@ -0,0 +1,79 @@
+# Find the contact
+module Gimme
+  class << self
+    ##
+    # Go to a page using Mechanize.
+    # Sleep for a split second to not overload any servers.
+    #
+    # Returns nil if bad url is given.
+    def get(str)
+      url = format_url(str)
+      puts "sending GET request to: #{url}"
+      sleep(0.1)
+      @page = Mechanize.new { |a| a.user_agent_alias = 'Mac Safari' }.get(url)
+    rescue Mechanize::ResponseCodeError => e
+      puts "#{'Response Error:'.red} #{e}"
+    rescue SocketError => e
+      puts "#{'Socket Error:'.red} #{e}"
+    rescue Errno::ETIMEDOUT => e
+      puts "#{'Connection Timeout:'.red} #{e}"
+    end
+
+    # Starts/Restarts @contacts_links hash
+    def start_contact_links
+      puts 'setting contact links hash to {}'
+      @contact_links = {}
+    end
+
+    ##
+    # Mechanize needs absolute urls to work.
+    # If http:// or https:// isn't present, append http://.
+    def format_url(str)
+      LazyDomain.autohttp(str)
+    end
+
+    ##
+    # Outputs domain of a url. Useful if subdomains are given to GimmePOC
+    # and they don't work.
+    #
+    # For example:
+    # Given http://maps.google.com, returns 'google.com'.
+    def orig_domain(str)
+      LazyDomain.parse(str).domain
+    end
+
+    ##
+    # Used in case of relative paths. Merging guarantees correct url.
+    # This needs a url string as argument to work.
+    # Produces a merged uri string.
+    def merged_link(url_str)
+      page.uri.merge(url_str).to_s
+    end
+
+    ##
+    # Expects relative paths and merges everything.
+    # Returns a string.  If there's nothing, return nil.
+    #
+    # Add \b word block to ensure whole word is searched.
+    def link_with_href(str)
+      merged_link(page.link_with(href: /\b#{str}/).uri.to_s)
+    rescue
+      nil
+    end
+
+    # Boolean, returns true if url is not identical to original domain.
+    def subdomain?(str)
+      (str != orig_domain(str))
+    end
+
+    # TODO: Sometimes DNS will do a redirect and not give a 404.
+    #   Need to prevent redirects.
+    #
+    # Blindly tests to see if a url goes through.  If there is a 404 error,
+    # this will return nil.
+    def blind_test(url)
+      puts "\nblind testing: #{url}"
+      get(url)
+    end
+  end
+end

data/lib/gimme_poc.rb

@@ -1,6 +1,12 @@
-require 'mechanize'
 require 'colored'
+require 'lazy_domain'
+require 'mechanize'
+require_relative './gimme_poc/contactpage'
+require_relative './gimme_poc/poc'
+require_relative './gimme_poc/questions'
+require_relative './gimme_poc/save'
 require_relative './gimme_poc/version'
+require_relative './gimme_poc/web'
 
 # Find the contact
 module Gimme
@@ -8,51 +14,10 @@ module Gimme
     attr_accessor :page, :contact, :contact_links, :url
 
     # Simple regex that looks for ###.#### or ###-####
-    PHONE_REGEX = /\d{3}[-]\d{4}|\d{3}[.]\d{4}/
+    PHONE_REGEX = %r{/\d{3}[-]\d{4}|\d{3}[.]\d{4}/}
 
     # Captures http:// and https://
-    HTTP_REGEX = /\A\bhttps:\/\/|\bhttp:\/\//
-
-    ## ----------------------------------------------------------------
-    # Questions
-    #
-    #
-    #
-
-    ##
-    # Boolean, returns true if anything is present
-    # after running scan_for_contacts.
-    def something_to_save?
-      scan_for_contacts.any?
-    end
-
-    # Boolean, returns true if email is present.
-    def email_available?
-      !link_with_href('mailto').nil?
-    end
-
-    # Boolean, returns true if phone number is present.
-    def phone_available?
-      !(page.body =~ PHONE_REGEX).nil?
-    end
-
-    ##
-    # TODO: build better conditional to prevent false positives.
-    #   There could be other forms like newsletter signup, etc.
-    #
-    # If there is a form with more than one field, this returns true.
-    # Forms with one field are typically search boxes.
-    #
-    # Boolean, returns true if form is present on page.
-    def contactform_available?
-      !(page.forms.select { |x| x.fields.length > 1 }.empty?)
-    end
-
-    ## ----------------------------------------------------------------
-    # Actions
-    #
-    #
-    #
+    HTTP_REGEX = %r{/\A\bhttps:\/\/|\bhttp:\/\//}
 
     ##
     # The main method!
@@ -63,175 +28,21 @@ module Gimme
       arr.each do |url|
         puts '-' * 50
         puts "starting: #{url}"
-        next if get(url).nil?
+        case
+        when subdomain?(url)
+          get(orig_domain(url)) if get(url).nil?
+        else
+          next if get(url).nil?
+        end
         puts 'now looking for contact pages'
         start_contact_links
-        mechpage = go_to_contact_page
+        mechpage = go_to_contact_page(url)
         next if mechpage.nil?
         save_available_contacts(mechpage.uri.to_s)
       end
       Search.all_sites
     end
 
-    # Mechanize needs absolute urls to work.
-    # If http:// or https:// isn't present, append http://.
-    def format_url(str)
-      str.prepend('http://') if (str =~ HTTP_REGEX).nil?
-      str
-    end
-
-    ##
-    # Go to a page using Mechanize.
-    # Sleep for a split second to not overload any servers.
-    #
-    # Returns nil if bad url is given.
-    def get(str)
-      url = format_url(str)
-      puts "sending GET request to: #{url}"
-      sleep(0.1)
-      @page = Mechanize.new { |a| a.user_agent_alias = 'Mac Safari' }.get(url)
-    rescue SocketError => e
-      puts "#{'skipping:'.red} -- #{e}"
-    end
-
-    # Starts/Restarts @contacts_links hash
-    def start_contact_links
-      puts 'setting contact links hash to {}'
-      @contact_links = {}
-    end
-
-    ##
-    # Scans for contact page.  If it doesn't work on the first try,
-    # It will look for english versions and try again. Processes left to right.
-    #
-    # Returns nil if no contact page can be found.
-    def go_to_contact_page
-      contact_page || english_contact_page
-    end
-
-    ##
-    # Looks for contact page.  Gets page if available.
-    # If no contact link is available, it will blind test '../contact'.
-    # Returns nil if nothing can be found.
-    def contact_page
-      contact_link = link_with_href(/contact|Contact/)
-      contact_test_page = merged_link('../contact')
-
-      case
-      when !contact_link.nil?
-        puts "#{'success:'.green} Found contact link!\n"
-        get(merged_link(contact_link))
-      else
-        puts "#{'warning:'.yellow}couldn't find contact link"
-        blind_test(contact_test_page)
-      end
-    end
-
-    ##
-    # Looks for english page.  Gets page if available then looks for
-    # english contact page.
-    #
-    # If no english link is available,
-    # it will blind test '../en' and '../english'.
-    # Returns nil if nothing can be found.
-    def english_contact_page
-      puts "\nLooking for english page..."
-      english_link = page.link_with(href: /english|English/)
-      test_en_page = merged_link('../en')
-      test_english_page = merged_link('../english')
-
-      case
-      when !english_link.nil?
-        puts "#{'success:'.green} found english link!"
-        get(merged(english_link)) # already merged link
-      else
-        blind_test(test_en_page) || blind_test(test_english_page)
-        puts 'ready to start again'
-        contact_page
-      end
-    end
-
-    # TODO: Sometimes DNS will do a redirect and not give a 404.
-    #   Need to prevent redirects.
-    #
-    # Blindly tests to see if a url goes through.  If there is a 404 error,
-    # this will return nil.
-    def blind_test(url)
-      puts "\nblind testing: #{url}"
-      get(url)
-    rescue Mechanize::ResponseCodeError
-      puts "#{'404 Error:'.red} #{url}"
-    end
-
-    ##
-    # Used in case of relative paths. Merging guarantees correct url.
-    # This needs a url string as argument to work.
-    # Produces a merged uri string.
-    def merged_link(url_str)
-      page.uri.merge(url_str).to_s
-    end
-
-    ##
-    # Expects relative paths and merges everything.
-    # Returns a string.  If there's nothing, return nil.
-    #
-    # Add \b word block to ensure whole word is searched.
-    def link_with_href(str)
-      merged_link(page.link_with(href: /\b#{str}/).uri.to_s)
-    rescue
-      nil
-    end
-
-    ##
-    # Returns anything that is possible to save, otherwise returns nil.
-    # Booleans for phone, email, or contact form will display True or False.
-    #
-    # Add periods to link hrefs to prevent false positives. Must escape periods
-    # with a backslash or else it will be a regex wild card.
-    def scan_for_contacts
-      {
-        contactpage: link_with_href('contact'),
-        email_present: "#{email_available?}",
-        phone_present: "#{phone_available?}",
-        contact_form: "#{contactform_available?}",
-        facebook: link_with_href('facebook\.'),
-        twitter: link_with_href('twitter\.'),
-        youtube: link_with_href('youtube\.'),
-        googleplus: link_with_href('plus\.google\.'),
-        linkedin: link_with_href('linkedin\.')
-      }
-    end
-
-    # Used in save_available_contacts to save each valid link.
-    def save_link(key, url)
-      return if key.nil? || url.nil?
-      @contact_links[key] = url
-    end
-
-    ##
-    # Remove negatives from the contacts hash.
-    # Deletes a key value pair with a value of either nil or false.
-    # Remember that false is a string.
-    def delete_failures(hsh)
-      hsh.delete_if { |_k, v| v.nil? || v == 'false' }
-    end
-
-    # Saves any available contact info to @contact_links.
-    def save_available_contacts(url, hsh = scan_for_contacts)
-      puts "\nsaving available contact information from #{url}"
-      return unless something_to_save?
-      if hsh.is_a?(Hash)
-        hsh.each do |k, v|
-          save_link(k, v) # saves to @contact_links
-        end
-        delete_failures(@contact_links)
-        puts "#{@contact_links}".cyan # same as @contact_links
-      else
-        fail ArgumentError, "expected hash but got #{hsh.class}"
-      end
-      Search::POC.new(url, @contact_links)
-    end
-
     # Convenience method.
     def memory
       Search.all_sites
@@ -243,21 +54,3 @@ module Gimme
     end
   end
 end
-
-# Collection of sites searched.
-class Search
-  @all_sites = []
-
-  class << self
-    attr_accessor :all_sites
-  end
-
-  # Each site is saved to this class
-  class POC
-    def initialize(url, contact_info_hsh)
-      @host = url
-      @info = contact_info_hsh
-      Search.all_sites << self
-    end
-  end
-end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gimme_poc
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - John Mason
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-09 00:00:00.000000000 Z
+date: 2015-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: lazy_domain
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.1
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -91,7 +105,12 @@ files:
 - README.md
 - Rakefile
 - lib/gimme_poc.rb
+- lib/gimme_poc/contactpage.rb
+- lib/gimme_poc/poc.rb
+- lib/gimme_poc/questions.rb
+- lib/gimme_poc/save.rb
 - lib/gimme_poc/version.rb
+- lib/gimme_poc/web.rb
 homepage: http://github.com/m8ss/gimme_poc
 licenses:
 - MIT