arachnid2 0.3.5 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e25353806a447177f129c56d4c57c38c70223849f2bbd858c932f3f4ec8a4ef
-  data.tar.gz: d2725c9981671ee010692d82b97801ccc00a1f2b28663fb72b23bc08f6be890e
+  metadata.gz: 6e4e32b14e6ad9a1f4a71bbe4099ec014176a2919e6f560ee36e38d93064cf3d
+  data.tar.gz: 501f5e7d3e8cf5c94391f8f5b70c2e08c96fd404d1409c8815792ceceaadc33d
 SHA512:
-  metadata.gz: 52a0b49101ca136ddee4c4ae8e976bd81cc9f3c559df3a94463bee7f42a2e4ce591330e2a587f5285bac98be52723ab518870ac8a8197413df8cd06267892858
-  data.tar.gz: 2514be62a0ae76a2d594f14d5ad8b66a45696bafa455a6347bb04b07ae99e48f322936d0afb6bd9e025c67ac9ce52213519f398a8f5deec54e508d6c4f1b4d84
+  metadata.gz: cd6fbad6aaab1e7da49f4fe178c00215ff236264f3f3ed99903b3d81338f54360bb2197845d50150df1ab5d19ccd9d53c9084d096d957cd6005f690c65d38e41
+  data.tar.gz: '0158f5b7469b33dafd07206654cf7793838b6644b623a6882c0057a29e994b1ae415fa97c56b3898ce06ece142ecd1f84853732426e356a1bb8cabda8b0fdcd1'

data/Gemfile.lock

@@ -1,11 +1,11 @@
 PATH
   remote: .
   specs:
-    arachnid2 (0.3.5)
+    arachnid2 (0.3.9)
       addressable
       adomain
       bloomfilter-rb
-      nokogiri (>= 1.8.5)
+      nokogiri (>= 1.10.4)
       typhoeus
       watir
       webdriver-user-agent (>= 7.6)
@@ -14,30 +14,30 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.6.0)
-      public_suffix (>= 2.0.2, < 4.0)
-    adomain (0.1.1)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    adomain (0.2.3)
       addressable (~> 2.5)
+      logger
     bloomfilter-rb (2.1.1)
       redis
-    childprocess (0.9.0)
-      ffi (~> 1.0, >= 1.0.11)
+    childprocess (3.0.0)
     diff-lcs (1.3)
     ethon (0.12.0)
       ffi (>= 1.3.0)
     facets (3.1.0)
-    ffi (1.10.0)
-    json (2.2.0)
+    ffi (1.12.2)
+    json (2.3.0)
+    logger (1.4.2)
     mini_portile2 (2.4.0)
-    net_http_ssl_fix (0.0.10)
-    nokogiri (1.10.1)
+    nokogiri (1.10.9)
       mini_portile2 (~> 2.4.0)
-    os (1.0.0)
+    os (1.0.1)
     psych (3.1.0)
-    public_suffix (3.0.3)
-    rake (10.5.0)
-    redis (4.1.0)
-    regexp_parser (1.3.0)
+    public_suffix (4.0.3)
+    rake (13.0.1)
+    redis (4.1.3)
+    regexp_parser (1.7.0)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
@@ -51,10 +51,10 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    rubyzip (1.2.2)
-    selenium-webdriver (3.141.0)
-      childprocess (~> 0.5)
-      rubyzip (~> 1.2, >= 1.2.2)
+    rubyzip (2.2.0)
+    selenium-webdriver (3.142.7)
+      childprocess (>= 0.5, < 4.0)
+      rubyzip (>= 1.2.2)
     typhoeus (1.3.1)
       ethon (>= 0.9.0)
     watir (6.16.5)
@@ -66,11 +66,10 @@ GEM
       os
       psych
       selenium-webdriver (>= 3.4.0)
-    webdrivers (3.6.0)
-      net_http_ssl_fix
+    webdrivers (4.2.0)
       nokogiri (~> 1.6)
-      rubyzip (~> 1.0)
-      selenium-webdriver (~> 3.0)
+      rubyzip (>= 1.3.0)
+      selenium-webdriver (>= 3.0, < 4.0)
 
 PLATFORMS
   ruby
@@ -78,8 +77,8 @@ PLATFORMS
 DEPENDENCIES
   arachnid2!
   bundler (~> 1.16)
-  rake (~> 10.0)
+  rake (>= 12.3.3)
   rspec (~> 3.0)
 
 BUNDLED WITH
-   1.16.5
+   1.17.3

data/README.md

@@ -186,6 +186,36 @@ with_watir = true
 Arachnid2.new(url).crawl(opts, with_watir)
 ```
 
+Arachnid2 has base defaults which you might want to address when
+employing Watir.
+
+* First, the default crawl time is 15 seconds.
+As browser page loads can take this long, you will probably want to
+set a higher crawl time.
+* Simply storing the browser is not a great idea, since it will
+be inaccessible after it is closed. Instead, consider nabbing the
+HTML, cookies, or whatever content is required during the crawl.
+* Finally, note that Firefox is the default browser.
+
+
+```ruby
+require 'arachnid2'
+
+with_watir = true
+responses = []
+url = "http://maximumfun.org"
+max = 60
+browser = :chrome
+opts = {time_box: max, browser_type: browser}
+
+spider = Arachnid2.new(url)
+spider.crawl(opts, with_watir) do |response|
+  response.body.wait_until(&:present?)
+  responses << response.body.html if response.body.present?
+end
+
+```
+
 #### Options
 
 See the Typhoeus options above &mdash; most apply to Watir as well, with

data/arachnid2.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", ">= 12.3.3"
   spec.add_development_dependency "rspec", "~> 3.0"
 
   spec.add_dependency "webdriver-user-agent", ">= 7.6"
@@ -32,5 +32,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency "bloomfilter-rb"
   spec.add_dependency "adomain"
   spec.add_dependency "addressable"
-  spec.add_dependency "nokogiri", ">= 1.8.5"
+  spec.add_dependency "nokogiri", ">= 1.10.4"
 end

data/lib/arachnid2.rb

@@ -1,5 +1,5 @@
 require "arachnid2/version"
-require "arachnid2/cached_arachnid_responses"
+require "arachnid2/cached_responses"
 require "arachnid2/exoskeleton"
 require "arachnid2/typhoeus"
 require "arachnid2/watir"

data/lib/arachnid2/{cached_arachnid_responses.rb → cached_responses.rb}

@@ -1,6 +1,6 @@
 require 'net/http'
 require 'json'
-module CachedArachnidResponses
+module CachedResponses
   CACHE_SERVICE_URL = ENV['ARACHNID_CACHED_SERVICE_ADDRESS'].freeze
 
   def load_data(_url, _options)
@@ -15,7 +15,7 @@ module CachedArachnidResponses
 
       body = ::JSON.parse(response.body)
       responses_list = Base64.decode64(body['encrypted_response'])
-      return Marshal.load responses_list # here we get array of Typhoeus::Response
+      return Marshal.load responses_list # here we get an Array of `Typhoeus::Response`s
     end
   rescue StandardError
     nil

data/lib/arachnid2/exoskeleton.rb

@@ -10,7 +10,7 @@ class Arachnid2
     end
 
     def process(url, html)
-      return false unless Adomain["#{url}"].include? @domain
+      return false unless Adomain["#{url}"]&.include? @domain
 
       extract_hrefs(html)
     end

data/lib/arachnid2/typhoeus.rb

@@ -1,6 +1,6 @@
 class Arachnid2
   class Typhoeus
-    include CachedArachnidResponses
+    include CachedResponses
     include Arachnid2::Exoskeleton
 
     def initialize(url)
@@ -17,39 +17,58 @@ class Arachnid2
         max_concurrency.times do
           q = @global_queue.shift
 
-          break if @global_visited.size >= crawl_options[:max_urls] || \
-                   Time.now > crawl_options[:time_limit] || \
-                   memory_danger?
-
+          break if time_to_stop?
           @global_visited.insert(q)
 
-          request = ::Typhoeus::Request.new(q, request_options)
-
-          data = load_data(@url, opts)
-          data.each { |response| yield response } and return unless data.nil?
-
-          request.on_complete do |response|
-            @cached_data.push(response)
-            links = process(response.effective_url, response.body)
-            next unless links
-
-            yield response
-
-            vacuum(links, response.effective_url)
-          end
+          found_in_cache = use_cache(q, opts, &Proc.new)
+          return if found_in_cache
 
-          @hydra.queue(request)
+          request = ::Typhoeus::Request.new(q, request_options)
+          requestable = after_request(request, &Proc.new)
+          @hydra.queue(request) if requestable
         end # max_concurrency.times do
 
         @hydra.run
-
       end # until @global_queue.empty?
-      put_cached_data(@url, opts, @cached_data) unless @cached_data.empty?
     ensure
       @cookie_file.close! if @cookie_file
     end # def crawl(opts = {})
 
     private
+      def after_request(request)
+        request.on_complete do |response|
+          cacheable = use_response(response, &Proc.new)
+          return unless cacheable
+
+          put_cached_data(response.effective_url, @options, response)
+        end
+
+        true
+      end
+
+      def use_response(response)
+        links = process(response.effective_url, response.body)
+        return unless links
+
+        yield response
+
+        vacuum(links, response.effective_url)
+        true
+      end
+
+      def use_cache(url, options)
+        data = load_data(url, options)
+        use_response(data, &Proc.new) if data
+
+        data
+      end
+
+      def time_to_stop?
+        @global_visited.size >= crawl_options[:max_urls] || \
+                 Time.now > crawl_options[:time_limit] || \
+                 memory_danger?
+      end
+
       def typhoeus_preflight
         @hydra = ::Typhoeus::Hydra.new(:max_concurrency => max_concurrency)
         typhoeus_proxy_options

data/lib/arachnid2/version.rb

@@ -1,3 +1,3 @@
 class Arachnid2
-  VERSION = "0.3.5"
+  VERSION = "0.4.0"
 end

data/lib/arachnid2/watir.rb

@@ -13,46 +13,17 @@ class Arachnid2
     def crawl(opts)
       preflight(opts)
       watir_preflight
+      @already_retried = false
 
       until @global_queue.empty?
-        @already_retried = false
         q = @global_queue.shift
+        links = nil
 
-        break if @global_visited.size >= crawl_options[:max_urls]
-        break if Time.now > crawl_options[:time_limit]
-        break if memory_danger?
+        break if time_to_stop?
 
         @global_visited.insert(q)
 
-        begin
-          begin
-            browser.goto q
-          rescue Selenium::WebDriver::Error::UnknownError => e
-            # Firefox and Selenium, in their infinite wisdom
-            # raise an error when a page cannot be loaded.
-            # At the time of writing this, the page at
-            # thewirecutter.com/cars/accessories-auto
-            # causes such an issue (too many redirects).
-            # This error handling moves us on from those pages.
-            raise e unless e.message =~ /.*Reached error page.*/i
-            next
-          end
-          links = process(browser.url, browser.body.html)
-          next unless links
-
-          yield browser
-
-          vacuum(links, browser.url)
-        rescue => e
-          raise e if @already_retried
-          raise e unless "#{e.class}".include?("Selenium") || "#{e.class}".include?("Watir")
-          @browser.close if @browser rescue nil
-          @headless.destroy if @headless rescue nil
-          @browser = nil
-          @already_retried = true
-          retry
-        end
-
+        make_request(q, &Proc.new)
       end # until @global_queue.empty?
     ensure
       @browser.close if @browser rescue nil
@@ -60,6 +31,74 @@ class Arachnid2
     end
 
     private
+      def make_request(q)
+        begin
+          links = browse_links(q, &Proc.new)
+          return unless links
+
+          vacuum(links, browser.url)
+        rescue Selenium::WebDriver::Error::NoSuchWindowError, Net::ReadTimeout => e
+          msg = "WARNING [arachnid2] Arachnid2::Watir#make_request " \
+                "is ignoring an error: " \
+                "#{e.class} - #{e.message}"
+          puts msg
+        rescue => e
+          raise e if raise_before_retry?(e.class)
+          msg = "WARNING [arachnid2] Arachnid2::Watir#make_request " \
+                "is retrying once after an error: " \
+                "#{e.class} - #{e.message}"
+          puts msg
+          e.backtrace[0..4].each{|l| puts "\t#{l}"}; puts "..."
+          reset_for_retry
+        end
+      end
+
+      def browse_links(url)
+        return unless navigate(url)
+
+        yield browser
+
+        process(browser.url, browser.body.html) if browser.body.exists?
+      end
+
+      def navigate(url)
+        begin
+          browser.goto url
+        rescue Selenium::WebDriver::Error::UnknownError => e
+          # Firefox and Selenium, in their infinite wisdom
+          # raise an error when a page cannot be loaded.
+          # At the time of writing this, the page at
+          # thewirecutter.com/cars/accessories-auto
+          # causes such an issue (too many redirects).
+          # This error handling moves us on from those pages.
+          raise e unless e.message =~ /.*Reached error page.*/i
+          return
+        end
+
+        true
+      end
+
+      def time_to_stop?
+        @global_visited.size >= crawl_options[:max_urls] || \
+                 Time.now > crawl_options[:time_limit] || \
+                 memory_danger?
+      end
+
+      def raise_before_retry?(klass)
+        @already_retried || \
+          "#{klass}".include?("Selenium") || \
+          "#{klass}".include?("Watir")
+      end
+
+      def reset_for_retry
+        @browser.close if @browser rescue nil
+        @headless.destroy if @headless rescue nil
+        @driver.quit if @headless rescue nil
+        @driver = nil
+        @browser = nil
+        @already_retried = true
+      end
+
       def browser
         unless @browser
           behead if @make_headless

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arachnid2
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.4.0
 platform: ruby
 authors:
 - Sam Nissen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-18 00:00:00.000000000 Z
+date: 2020-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -28,16 +28,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -156,14 +156,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.8.5
+        version: 1.10.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.8.5
+        version: 1.10.4
 description: 
 email:
 - scnissen@gmail.com
@@ -184,7 +184,7 @@ files:
 - bin/console
 - bin/setup
 - lib/arachnid2.rb
-- lib/arachnid2/cached_arachnid_responses.rb
+- lib/arachnid2/cached_responses.rb
 - lib/arachnid2/exoskeleton.rb
 - lib/arachnid2/typhoeus.rb
 - lib/arachnid2/version.rb
@@ -208,8 +208,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A simple, fast web crawler