grell 1.6 → 1.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2be8992c96b83e9b1a98474ada3b49ea7e5adb69
-  data.tar.gz: 3eed1bea205812e8e9ab7dc8678da57efea1fea1
+  metadata.gz: 0ef86064ca2938505dec12137ac353c08087695c
+  data.tar.gz: 22483420d8592db3d8e633c56a1c83b3c812a9ba
 SHA512:
-  metadata.gz: baa6e37b2ce80491b05688618b6ad0576149236c2367b2f6c52a84dfeae25edb6d340abfdcae4e3b6f7363072db0dc0c8c052cd83410e1f28e1725305db99993
-  data.tar.gz: 7c246e8b2a02494d5e44dc6fc4b0029ab254e63764b46791e9135ed9ec1657627d4b6f7e5cd921a951c062cfe815ac1fd7b4e7d87ffb11f786e0989d44c3083a
+  metadata.gz: 64c2b2d0b7e1478faed2a44d78e38c0cda20fc2742f558b610cb1a009120928ce3fa7215a5919f4e28554f8c9cc032957a775858dae5ce5870abaa1d847017fb
+  data.tar.gz: caf16e853905923720299543fde15318b62187fd21f4600136f135a6386eeef2669674e8eff1074ab5c66d6ed3432adc9e68561768f334b0c89052b246a4de20

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 1.6.1
+  * Use non-static name to support registering Poltergeist crawler multiple times
+  * More exception handling, store redirected URLs in addition to original URL
+
 # 1.6
   * Support custom URL comparison when adding new pages during crawling
   * Don't rescue Timeout error, so that Delayed Job can properly terminate hanging jobs

data/grell.gemspec

@@ -31,4 +31,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "webmock", '~> 1.18'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'puffing-billy', '~> 0.5'
+  spec.add_development_dependency 'timecop', '~> 0.8'
 end

data/lib/grell/capybara_driver.rb

@@ -13,7 +13,13 @@ module Grell
 
     def setup_capybara
       @poltergeist_driver = nil
-      Capybara.register_driver :poltergeist_crawler do |app|
+
+      # Capybara will not re-run the block if the driver name already exists, so the driver name
+      # will have a time integer appended to ensure uniqueness.
+      driver_name = "poltergeist_crawler_#{Time.now.to_i}".to_sym
+      Grell.logger.info "GRELL Registering poltergeist driver with name '#{driver_name}'"
+
+      Capybara.register_driver driver_name do |app|
         @poltergeist_driver = Capybara::Poltergeist::Driver.new(app, {
           js_errors: false,
           inspector: false,
@@ -24,13 +30,13 @@ module Grell
 
       Capybara.default_max_wait_time = 3
       Capybara.run_server = false
-      Capybara.default_driver = :poltergeist_crawler
+      Capybara.default_driver = driver_name
       page.driver.headers = {
         "DNT" => 1,
         "User-Agent" => USER_AGENT
       }
 
-      fail "Poltergeist Driver could not be properly initialized" unless @poltergeist_driver
+      raise 'Poltergeist Driver could not be properly initialized' unless @poltergeist_driver
 
       @poltergeist_driver
     end

data/lib/grell/crawler.rb

@@ -8,13 +8,13 @@ module Grell
     # Creates a crawler
     # options allows :logger to point to an object with the same interface than Logger in the standard library
     def initialize(options = {})
-      @driver = CapybaraDriver.setup(options)
-
       if options[:logger]
         Grell.logger = options[:logger]
       else
         Grell.logger = Logger.new(STDOUT)
       end
+
+      @driver = CapybaraDriver.setup(options)
     end
 
     # Restarts the PhantomJS process without modifying the state of visited and discovered pages.
@@ -51,12 +51,14 @@ module Grell
       Grell.logger.info "Visiting #{site.url}, visited_links: #{@collection.visited_pages.size}, discovered #{@collection.discovered_pages.size}"
       site.navigate
       filter!(site.links)
+      add_redirect_url(site)
 
-      if block #The user of this block can send us a :retry to retry accessing the page
-        while block.call(site) == :retry
+      if block # The user of this block can send us a :retry to retry accessing the page
+        while crawl_block(block, site) == :retry
           Grell.logger.info "Retrying our visit to #{site.url}"
           site.navigate
           filter!(site.links)
+          add_redirect_url(site)
         end
       end
 
@@ -67,6 +69,15 @@ module Grell
 
     private
 
+    # Treat any exceptions from the block as an unavailable page
+    def crawl_block(block, site)
+      block.call(site)
+    rescue Capybara::Poltergeist::BrowserError, Capybara::Poltergeist::DeadClient,
+           Capybara::Poltergeist::JavascriptError, Capybara::Poltergeist::StatusFailError,
+           Capybara::Poltergeist::TimeoutError, Errno::ECONNRESET, URI::InvalidURIError => e
+      site.unavailable_page(404, e)
+    end
+
     def filter!(links)
       links.select! { |link| link =~ @whitelist_regexp } if @whitelist_regexp
       links.delete_if { |link| link =~ @blacklist_regexp } if @blacklist_regexp
@@ -80,6 +91,13 @@ module Grell
       end
     end
 
+    # Store the resulting redirected URL along with the original URL
+    def add_redirect_url(site)
+      if site.url != site.current_url
+        @collection.create_page(site.current_url, site.id)
+      end
+    end
+
   end
 
 end

data/lib/grell/page.rb

@@ -34,15 +34,9 @@ module Grell
       @result_page = VisitedPage.new(@rawpage)
       @timestamp = Time.now
       @times_visited += 1
-    rescue Capybara::Poltergeist::JavascriptError => e
-      unavailable_page(404, e)
-    rescue Capybara::Poltergeist::BrowserError => e #This may happen internally on Poltergeist, they claim is a bug.
-      unavailable_page(404, e)
-    rescue URI::InvalidURIError => e #No cool URL means we report error
-      unavailable_page(404, e)
-    rescue Capybara::Poltergeist::TimeoutError => e #Poltergeist has its own timeout which is similar to Chromes.
-      unavailable_page(404, e)
-    rescue Capybara::Poltergeist::StatusFailError => e
+    rescue Capybara::Poltergeist::BrowserError, Capybara::Poltergeist::DeadClient,
+           Capybara::Poltergeist::JavascriptError, Capybara::Poltergeist::StatusFailError,
+           Capybara::Poltergeist::TimeoutError, Errno::ECONNRESET, URI::InvalidURIError => e
       unavailable_page(404, e)
     end
 
@@ -73,13 +67,14 @@ module Grell
       @url
     end
 
-    private
     def unavailable_page(status, exception)
       Grell.logger.warn "The page with the URL #{@url} was not available. Exception #{exception}"
       @result_page = ErroredPage.new(status, exception)
       @timestamp = Time.now
     end
 
+    private
+
     # Private class.
     # This is a result page when it has not been visited yet. Essentially empty of information
     #

data/lib/grell/version.rb

@@ -1,3 +1,3 @@
 module Grell
-  VERSION = "1.6"
+  VERSION = "1.6.1"
 end

data/spec/lib/capybara_driver_spec.rb

@@ -0,0 +1,32 @@
+
+RSpec.describe Grell::CapybaraDriver do
+  let(:ts) { Time.now }
+
+  describe 'setup_capybara' do
+    it 'properly registers the poltergeist driver' do
+      Timecop.freeze(ts)
+      driver = Grell::CapybaraDriver.new.setup_capybara
+      expect(driver).to be_instance_of(Capybara::Poltergeist::Driver)
+    end
+
+    it 'raises an exception if the driver cannot be initialized' do
+      Timecop.freeze(ts + 60)
+
+      # Attempt to register twice with the same driver name
+      Grell::CapybaraDriver.new.setup_capybara
+      expect { Grell::CapybaraDriver.new.setup_capybara }.
+        to raise_error "Poltergeist Driver could not be properly initialized"
+    end
+
+    it 'can register the poltergeist driver multiple times in a row' do
+      Timecop.freeze(ts + 120)
+      driver = Grell::CapybaraDriver.new.setup_capybara
+      expect(driver).to be_instance_of(Capybara::Poltergeist::Driver)
+    end
+
+    after do
+      Timecop.return
+    end
+  end
+
+end

data/spec/lib/crawler_spec.rb

@@ -36,13 +36,19 @@ RSpec.describe Grell::Crawler do
 
     it 'yields the result if a block is given' do
       result = []
-      block = Proc.new {|n| result.push(n) }
+      block = Proc.new { |n| result.push(n) }
       crawler.crawl(page, block)
       expect(result.size).to eq(1)
       expect(result.first.url).to eq(url)
       expect(result.first.visited?).to eq(true)
     end
 
+    it 'rescues any specified exceptions raised during the block execution' do
+      block = Proc.new { |n| raise Capybara::Poltergeist::BrowserError, 'Exception' }
+      expect{ crawler.crawl(page, block) }.to_not raise_error
+      expect(page.status).to eq(404)
+    end
+
     it 'logs interesting information' do
       crawler
       expect(Grell.logger).to receive(:info).with(/Visiting #{url}, visited_links: 0, discovered 0/)
@@ -61,6 +67,13 @@ RSpec.describe Grell::Crawler do
       crawler.crawl(page, block)
       expect(counter).to eq(times_retrying)
     end
+
+    it 'handles redirects by adding the current_url to the page collection' do
+      redirect_url = 'http://www.example.com/test/landing_page'
+      allow(page).to receive(:current_url).and_return(redirect_url)
+      expect_any_instance_of(Grell::PageCollection).to receive(:create_page).with(redirect_url, page_id)
+      crawler.crawl(page, nil)
+    end
   end
 
   context '#start_crawling' do
@@ -80,7 +93,7 @@ RSpec.describe Grell::Crawler do
 
     it 'calls the block we used to start_crawling' do
       result = []
-      block = Proc.new {|n| result.push(n) }
+      block = Proc.new { |n| result.push(n) }
       crawler.start_crawling(url, &block)
       expect(result.size).to eq(2)
       expect(result[0].url).to eq(url)

data/spec/lib/page_spec.rb

@@ -106,7 +106,8 @@ RSpec.describe Grell::Page do
   end
 
   [ Capybara::Poltergeist::JavascriptError, Capybara::Poltergeist::BrowserError, URI::InvalidURIError,
-    Capybara::Poltergeist::TimeoutError, Capybara::Poltergeist::StatusFailError ].each do |error_type|
+    Capybara::Poltergeist::TimeoutError, Capybara::Poltergeist::StatusFailError,
+    Capybara::Poltergeist::DeadClient, Errno::ECONNRESET ].each do |error_type|
 
     context "#{error_type}" do
       let(:headers) do

data/spec/spec_helper.rb

@@ -1,5 +1,6 @@
 require 'grell'
 require 'byebug'
+require 'timecop'
 require 'webmock/rspec'
 require 'billy/rspec'
 require 'rack'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: grell
 version: !ruby/object:Gem::Version
-  version: '1.6'
+  version: 1.6.1
 platform: ruby
 authors:
 - Jordi Polo Carres
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-02 00:00:00.000000000 Z
+date: 2016-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -136,6 +136,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
 description: Ruby web crawler using PhantomJS
 email:
 - jcarres@mdsol.com
@@ -159,6 +173,7 @@ files:
 - lib/grell/rawpage.rb
 - lib/grell/reader.rb
 - lib/grell/version.rb
+- spec/lib/capybara_driver_spec.rb
 - spec/lib/crawler_spec.rb
 - spec/lib/page_collection_spec.rb
 - spec/lib/page_spec.rb
@@ -184,11 +199,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Ruby web crawler
 test_files:
+- spec/lib/capybara_driver_spec.rb
 - spec/lib/crawler_spec.rb
 - spec/lib/page_collection_spec.rb
 - spec/lib/page_spec.rb