arachnid2 0.3.4 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 917f030ae3bfe4eb2183020eaab33463abd80940f2d3ad228ba7e86fcb9d1c3d
-  data.tar.gz: dd7f349e22f52cf54c45740c4393371f9547c7b5f649048de92863e6a41facc1
+  metadata.gz: 5e25353806a447177f129c56d4c57c38c70223849f2bbd858c932f3f4ec8a4ef
+  data.tar.gz: d2725c9981671ee010692d82b97801ccc00a1f2b28663fb72b23bc08f6be890e
 SHA512:
-  metadata.gz: d868813ceba88a7a078e666626c5de3bad85dbf3b2e834626dab9672f6a5b4f32858c0262bc46831b4c7294e4b8b349cfeaab8ef7625eeada588e9604cfdc525
-  data.tar.gz: 1c1b3221ba6db4a12673197162e2be0328879124bd5280f71c63a34d801d4ed74f0cf6526eaff197cca7d805c1dc370f511c66e37467b42856735fc421bfa586
+  metadata.gz: 52a0b49101ca136ddee4c4ae8e976bd81cc9f3c559df3a94463bee7f42a2e4ce591330e2a587f5285bac98be52723ab518870ac8a8197413df8cd06267892858
+  data.tar.gz: 2514be62a0ae76a2d594f14d5ad8b66a45696bafa455a6347bb04b07ae99e48f322936d0afb6bd9e025c67ac9ce52213519f398a8f5deec54e508d6c4f1b4d84

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    arachnid2 (0.3.4)
+    arachnid2 (0.3.5)
       addressable
       adomain
       bloomfilter-rb

data/lib/arachnid2.rb

@@ -106,9 +106,11 @@ class Arachnid2
   # @return nil
   #
   def crawl(opts = {}, with_watir = false)
-    crawl_watir(opts, &Proc.new) and return if with_watir
-
-    Arachnid2::Typhoeus.new(@url).crawl(opts, &Proc.new)
+    if with_watir
+      crawl_watir(opts, &Proc.new)
+    else
+      Arachnid2::Typhoeus.new(@url).crawl(opts, &Proc.new)
+    end
   end
 
   def crawl_watir(opts)

data/lib/arachnid2/version.rb

@@ -1,3 +1,3 @@
 class Arachnid2
-  VERSION = "0.3.4"
+  VERSION = "0.3.5"
 end

data/lib/arachnid2/watir.rb

@@ -25,7 +25,18 @@ class Arachnid2
         @global_visited.insert(q)
 
         begin
-          browser.goto q
+          begin
+            browser.goto q
+          rescue Selenium::WebDriver::Error::UnknownError => e
+            # Firefox and Selenium, in their infinite wisdom
+            # raise an error when a page cannot be loaded.
+            # At the time of writing this, the page at
+            # thewirecutter.com/cars/accessories-auto
+            # causes such an issue (too many redirects).
+            # This error handling moves us on from those pages.
+            raise e unless e.message =~ /.*Reached error page.*/i
+            next
+          end
           links = process(browser.url, browser.body.html)
           next unless links
 
@@ -35,6 +46,8 @@ class Arachnid2
         rescue => e
           raise e if @already_retried
           raise e unless "#{e.class}".include?("Selenium") || "#{e.class}".include?("Watir")
+          @browser.close if @browser rescue nil
+          @headless.destroy if @headless rescue nil
           @browser = nil
           @already_retried = true
           retry

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arachnid2
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.5
 platform: ruby
 authors:
 - Sam Nissen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-15 00:00:00.000000000 Z
+date: 2019-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler