gimme_poc 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ca0656c586244edfaaac44d81b092956e1ed801
-  data.tar.gz: b2bbc85c51a79ba5a10a1a5e77d8084ac28adeee
+  metadata.gz: 35a045a491109a5ae34152577508585667371af0
+  data.tar.gz: cae3580199f6cea7f2d5ceac4e689c931f743ff3
 SHA512:
-  metadata.gz: f2e131c8e68fb8a55169f62b1d9a5662901f6fcb45907878c30d8d700d784b66aa8cae0305c060a611da7d60b810cc2d56bb56959db1da378a3088fa6568d707
-  data.tar.gz: 6f32874d9fc287ce588baa0e91fa854ed7f75f045f88d798ebf02a0e781918f50d6922ffd06578965d7c033cacdb6a927d03efc7b7967c6a225d7579a8631853
+  metadata.gz: 200a0ba0bedded51c4b6aa50ceb46dc41d01a211fc52dbfdd7596989c9429e05db802f18b5f0c625d8353dbe7800da6447ed2a4fe43735438f16a9cf24b12728
+  data.tar.gz: e8c6def1d4e085c3c3dc05c92cdd894c43b2be287ce98ed79bc91111377b5c8e27db1898501de8726f86e3888577d0d207dbb3a23bba69991134d19cf1b03b70

data/lib/gimme_poc.rb

@@ -14,10 +14,10 @@ module Gimme
     attr_accessor :page, :contact, :contact_links, :url
 
     # Simple regex that looks for ###.#### or ###-####
-    PHONE_REGEX = %r{/\d{3}[-]\d{4}|\d{3}[.]\d{4}/}
+    PHONE_REGEX = /(\d{3}[-]\d{4}|\d{3}[.]\d{4})/
 
     # Captures http:// and https://
-    HTTP_REGEX = %r{/\A\bhttps:\/\/|\bhttp:\/\//}
+    HTTP_REGEX = %r{(\A\bhttps:\/\/|\bhttp:\/\/)}
 
     ##
     # The main method!
@@ -28,19 +28,26 @@ module Gimme
       arr.each do |url|
         puts '-' * 50
         puts "starting: #{url}"
+        unless LazyDomain.valid?(url)
+          puts "#{'Invalid Domain:'.red} `#{url}' is not a valid domain"
+          next
+        end
         case
         when subdomain?(url)
-          get(orig_domain(url)) if get(url).nil?
+          puts '(This url is a subdomain.  Will try both sub and root domain.)'
+          next if get(url).nil? && get(orig_domain(url)).nil?
         else
           next if get(url).nil?
         end
-        puts 'now looking for contact pages'
         start_contact_links
         mechpage = go_to_contact_page(url)
-        next if mechpage.nil?
-        save_available_contacts(mechpage.uri.to_s)
+        if mechpage.nil?
+          puts '(empty page, exiting.)'
+        else
+          save_available_contacts(mechpage.uri.to_s)
+        end
       end
-      Search.all_sites
+      Search.all_sites # Return results from all sites.
     end
 
     # Convenience method.

data/lib/gimme_poc/contactpage.rb

@@ -15,6 +15,7 @@ module Gimme
     # If no contact link is available, it will blind test '../contact'.
     # Returns nil if nothing can be found.
     def contact_page(url)
+      puts 'now looking for contact pages'
       contact_link = link_with_href(/contact|Contact/)
       contact_test_page = merged_link('../contact')
 
@@ -37,17 +38,17 @@ module Gimme
     # Returns nil if nothing can be found.
     def english_contact_page(url)
       puts "\nLooking for english page..."
-      english_link = page.link_with(href: /english|English/)
+      english_link = page.link_with(href: %r{en\/|english|English})
       test_en_page = merged_link('../en')
       test_english_page = merged_link('../english')
 
       case
       when !english_link.nil?
         puts "#{'Success:'.green} found english link!"
-        get(merged(english_link)) # already merged link
+        get(merged_link(english_link.uri))
       else
         blind_test(test_en_page) || blind_test(test_english_page)
-        puts 'ready to start again'
+        puts "\n(restarting)\n"
         contact_page(url)
       end
     end

data/lib/gimme_poc/save.rb

@@ -20,6 +20,11 @@ module Gimme
       }
     end
 
+    # Starts/Restarts @contacts_links hash
+    def start_contact_links
+      @contact_links = {}
+    end
+
     # Used in save_available_contacts to save each valid link.
     def save_link(key, url)
       return if key.nil? || url.nil?
@@ -36,18 +41,22 @@ module Gimme
 
     # Saves any available contact info to @contact_links.
     def save_available_contacts(url, hsh = scan_for_contacts)
-      return unless something_to_save?(hsh)
-      puts "\nsaving available contact information from #{url}"
-      if hsh.is_a?(Hash)
-        hsh.each do |k, v|
-          save_link(k, v) # saves to @contact_links
+      if something_to_save?(hsh)
+        puts "\nsaving available contact information from #{url}"
+        if hsh.is_a?(Hash)
+          hsh.each do |k, v|
+            save_link(k, v) # saves to @contact_links
+          end
+          delete_failures(@contact_links)
+          puts "#{@contact_links}".cyan # same as @contact_links
+        else
+          fail ArgumentError, "expected hash but got #{hsh.class}"
         end
-        delete_failures(@contact_links)
-        puts "#{@contact_links}".cyan # same as @contact_links
+        Search::POC.new(url, @contact_links)
       else
-        fail ArgumentError, "expected hash but got #{hsh.class}"
+        puts '(nothing to save)'
+        return
       end
-      Search::POC.new(url, @contact_links)
     end
   end
 end

data/lib/gimme_poc/version.rb

@@ -1,3 +1,3 @@
 module Gimme
-  VERSION = '0.0.4'
+  VERSION = '0.0.5'
 end

data/lib/gimme_poc/web.rb

@@ -10,19 +10,24 @@ module Gimme
       url = format_url(str)
       puts "sending GET request to: #{url}"
       sleep(0.1)
-      @page = Mechanize.new { |a| a.user_agent_alias = 'Mac Safari' }.get(url)
+      @page = Mechanize.new do |a|
+        a.user_agent_alias = 'Mac Safari'
+        a.open_timeout = 7
+        a.read_timeout = 7
+        a.idle_timeout = 7
+        a.redirect_ok = true
+      end.get(url)
+
     rescue Mechanize::ResponseCodeError => e
       puts "#{'Response Error:'.red} #{e}"
     rescue SocketError => e
       puts "#{'Socket Error:'.red} #{e}"
+    rescue Net::OpenTimeout => e
+      puts "#{'Connection Timeout:'.red} #{e}"
     rescue Errno::ETIMEDOUT => e
       puts "#{'Connection Timeout:'.red} #{e}"
-    end
-
-    # Starts/Restarts @contacts_links hash
-    def start_contact_links
-      puts 'setting contact links hash to {}'
-      @contact_links = {}
+    rescue Net::HTTP::Persistent::Error
+      puts "#{'Connection Timeout:'.red} read timeout, too many resets."
     end
 
     ##
@@ -32,6 +37,11 @@ module Gimme
       LazyDomain.autohttp(str)
     end
 
+    # Used for subdomain check.  Not a permanent change to url variable.
+    def unformat_url(str)
+      str.gsub(HTTP_REGEX, '')
+    end
+
     ##
     # Outputs domain of a url. Useful if subdomains are given to GimmePOC
     # and they don't work.
@@ -40,6 +50,8 @@ module Gimme
     # Given http://maps.google.com, returns 'google.com'.
     def orig_domain(str)
       LazyDomain.parse(str).domain
+    rescue PublicSuffix::DomainInvalid => e
+      puts "#{'Invalid Domain:'.red} #{e}"
     end
 
     ##
@@ -63,7 +75,7 @@ module Gimme
 
     # Boolean, returns true if url is not identical to original domain.
     def subdomain?(str)
-      (str != orig_domain(str))
+      (unformat_url(str) != orig_domain(str))
     end
 
     # TODO: Sometimes DNS will do a redirect and not give a 404.
@@ -72,7 +84,7 @@ module Gimme
     # Blindly tests to see if a url goes through.  If there is a 404 error,
     # this will return nil.
     def blind_test(url)
-      puts "\nblind testing: #{url}"
+      puts "\n(blind testing: #{url})"
       get(url)
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gimme_poc
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - John Mason
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-10 00:00:00.000000000 Z
+date: 2015-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize