arachnid2 0.3.6 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0f21f9cc6f752a43952d694abe4419d8c5d2ba4f3e5154543dab9d39e415753f
-  data.tar.gz: c29401a47c0291cb5087e237b72431167d3d5f59210a49fb9d73a8ef399fbb36
+  metadata.gz: 1df2624ea1b73392b92e3fcdc229e0816f2aab2dcac07e160749b29fdd0149e6
+  data.tar.gz: fc26566bd947099e80b028e0523cb7bbe469a4304ceea4d69d262dd33ad4e410
 SHA512:
-  metadata.gz: f6114133ad6c4b26bebfa762c676f5c4b544c55e3fb0802c3f5571b7d7ae2f4235218871f75e2bf284bdd1ebb5b64e992d83e291fd6c900038fc68bb6afe54a3
-  data.tar.gz: b543fb52387b4188c8738f8dbc44f0be5aec764ead8a53c2babf1e4d51c8d778c45ead9389478942afe6d16f3629fe7a81d6f4d965217079528ec7f968e2f89f
+  metadata.gz: 2af5a70e62c2194566c855edaebe805201d3cca0d612e32cbd15dde06cd49869538fcbb4af2ad14b5ed0c9f45b968ce39085b36b0faf74eac115f0a7ba5f92cc
+  data.tar.gz: bc250f87e6376fa05e1756490a5a8a7693f3cdd88df1c536e227a9a26b57a734f299a8c6cbcfa7f2a6ad16721c7c6ef1f4022ce7822b4cd5ca17f883b57a1fdd

data/Gemfile.lock

@@ -1,11 +1,11 @@
 PATH
   remote: .
   specs:
-    arachnid2 (0.3.5)
+    arachnid2 (0.3.7)
       addressable
       adomain
       bloomfilter-rb
-      nokogiri (>= 1.8.5)
+      nokogiri (>= 1.10.4)
       typhoeus
       watir
       webdriver-user-agent (>= 7.6)
@@ -14,30 +14,31 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.6.0)
-      public_suffix (>= 2.0.2, < 4.0)
-    adomain (0.1.1)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    adomain (0.2.3)
       addressable (~> 2.5)
+      logger
     bloomfilter-rb (2.1.1)
       redis
-    childprocess (0.9.0)
-      ffi (~> 1.0, >= 1.0.11)
+    childprocess (1.0.1)
+      rake (< 13.0)
     diff-lcs (1.3)
     ethon (0.12.0)
       ffi (>= 1.3.0)
     facets (3.1.0)
-    ffi (1.10.0)
+    ffi (1.11.1)
     json (2.2.0)
+    logger (1.4.1)
     mini_portile2 (2.4.0)
-    net_http_ssl_fix (0.0.10)
-    nokogiri (1.10.4)
+    nokogiri (1.10.7)
       mini_portile2 (~> 2.4.0)
-    os (1.0.0)
+    os (1.0.1)
     psych (3.1.0)
-    public_suffix (3.0.3)
+    public_suffix (4.0.1)
     rake (10.5.0)
-    redis (4.1.0)
-    regexp_parser (1.3.0)
+    redis (4.1.2)
+    regexp_parser (1.6.0)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
@@ -51,9 +52,9 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    rubyzip (1.2.2)
-    selenium-webdriver (3.141.0)
-      childprocess (~> 0.5)
+    rubyzip (1.2.3)
+    selenium-webdriver (3.142.3)
+      childprocess (>= 0.5, < 2.0)
       rubyzip (~> 1.2, >= 1.2.2)
     typhoeus (1.3.1)
       ethon (>= 0.9.0)
@@ -66,11 +67,10 @@ GEM
       os
       psych
       selenium-webdriver (>= 3.4.0)
-    webdrivers (3.6.0)
-      net_http_ssl_fix
+    webdrivers (4.1.0)
       nokogiri (~> 1.6)
       rubyzip (~> 1.0)
-      selenium-webdriver (~> 3.0)
+      selenium-webdriver (>= 3.0, < 4.0)
 
 PLATFORMS
   ruby
@@ -82,4 +82,4 @@ DEPENDENCIES
   rspec (~> 3.0)
 
 BUNDLED WITH
-   1.16.6
+   1.17.3

data/lib/arachnid2/version.rb

@@ -1,3 +1,3 @@
 class Arachnid2
-  VERSION = "0.3.6"
+  VERSION = "0.3.7"
 end

data/lib/arachnid2/watir.rb

@@ -17,6 +17,7 @@ class Arachnid2
       until @global_queue.empty?
         @already_retried = false
         q = @global_queue.shift
+        links = nil
 
         break if @global_visited.size >= crawl_options[:max_urls]
         break if Time.now > crawl_options[:time_limit]
@@ -37,19 +38,18 @@ class Arachnid2
             raise e unless e.message =~ /.*Reached error page.*/i
             next
           end
-          links = process(browser.url, browser.body.html)
+          links = process(browser.url, browser.body.html) if browser.body.exists?
           next unless links
 
           yield browser
 
           vacuum(links, browser.url)
         rescue => e
-          raise e if @already_retried
-          raise e unless "#{e.class}".include?("Selenium") || "#{e.class}".include?("Watir")
-          @browser.close if @browser rescue nil
-          @headless.destroy if @headless rescue nil
-          @browser = nil
-          @already_retried = true
+          next if e.class == Net::ReadTimeout
+
+          raise e if raise_before_retry?(e.class)
+
+          reset_for_retry
           retry
         end
 
@@ -60,6 +60,19 @@ class Arachnid2
     end
 
     private
+      def raise_before_retry?(klass)
+        @already_retried || \
+          "#{klass}".include?("Selenium") || \
+          "#{klass}".include?("Watir")
+      end
+
+      def reset_for_retry
+        @browser.close if @browser rescue nil
+        @headless.destroy if @headless rescue nil
+        @browser = nil
+        @already_retried = true
+      end
+
       def browser
         unless @browser
           behead if @make_headless

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arachnid2
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.3.7
 platform: ruby
 authors:
 - Sam Nissen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-09-05 00:00:00.000000000 Z
+date: 2020-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -208,8 +208,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.10
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: A simple, fast web crawler