grell 1.6.10 → 1.6.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 866f1b7117624455b79791bacba549710eb7dc2b
-  data.tar.gz: 4a3646c053bb7b4884fa8b82ebb24809d2d37f97
+  metadata.gz: 9a3668b60b187d3b4dab183bf6b386738813a1ec
+  data.tar.gz: 32ddba97d8dca3abca1cffb4d44760d0679eea0b
 SHA512:
-  metadata.gz: dfb07e7c0a6a7fb2fe53a40a3da248a24ba683972c696ab7064481448cb4067403dc7350d34cbba21ff4f240fb09b3571adbb36b38bfecd28aeee1ea1551638e
-  data.tar.gz: 7f91a206fd4cb264d73b05468dce66d858637c6d675b92088b800a85b67599201f957f879d618a0f4fc7a538de79c3211f82ffc3d73b063fec760da09209578a
+  metadata.gz: 6ba545ee45bbdc4a43b85045d6e09c0a1cfd7f0d7605dd5abee455434426b84cf11fe706a46a0128cc2b35ab04ff0f0d2106482a7548989a1aaa2bdb4928cf90
+  data.tar.gz: 0d1e2eb5dbcd688d00a39e7ff39781b577647eb6db47ac2155709d81af7e004d435521a38067258026960982f97f60cbf991993366dc61b89ae8f2d08345c4d1

data/.travis.yml

@@ -9,6 +9,6 @@ script: bundle exec rspec
 
 before_install:
   - mkdir travis-phantomjs
-  - wget https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2 -O $PWD/travis-phantomjs/phantomjs-2.1.1-linux-x86_64.tar.bz2
+  - wget https://github.com/JordiPolo/phantomjs/blob/master/phantomjs-2.1.1-linux-x86_64.tar.bz2?raw=true -O $PWD/travis-phantomjs/phantomjs-2.1.1-linux-x86_64.tar.bz2
   - tar -xvf $PWD/travis-phantomjs/phantomjs-2.1.1-linux-x86_64.tar.bz2 -C $PWD/travis-phantomjs
   - export PATH=$PWD/travis-phantomjs/phantomjs-2.1.1-linux-x86_64/bin:$PATH

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 1.6.11
+  * Ensure all links are loaded by waiting for Ajax requests to complete
+  * Add '@evaluate_in_each_page' option to evaluate before extracting links (e.g. $('.dropdown').addClass('open');)
+
 # 1.6.10
   * Avoid following JS href links, add missing dependencies to fix Travis build
 

data/README.md

@@ -151,6 +151,17 @@ The page object generated by accessing the first URL passed to the start_crawlin
 Using this information it is possible to construct a directed graph.
 
 
+### Evaluate script
+
+You can evalute a JavaScript snippet in each page before extracting links by passing the snippet to the 'evaluate_in_each_page' option:
+
+```ruby
+require 'grell'
+
+crawler = Grell::Crawler.new(evaluate_in_each_page: "typeof jQuery !== 'undefined' && $('.dropdown').addClass('open');")
+
+```
+
 ### Errors
 When there is an error in the page or an internal error in the crawler (Javascript crashed the browser, etc). Grell will return with status 404 and the headers will have the following keys:
 - grellStatus: 'Error'

data/lib/grell/crawler.rb

@@ -15,6 +15,7 @@ module Grell
       end
 
       @driver = CapybaraDriver.setup(options)
+      @evaluate_in_each_page = options[:evaluate_in_each_page]
     end
 
     # Restarts the PhantomJS process without modifying the state of visited and discovered pages.
@@ -55,16 +56,12 @@ module Grell
 
     def crawl(site, block)
       Grell.logger.info "Visiting #{site.url}, visited_links: #{@collection.visited_pages.size}, discovered #{@collection.discovered_pages.size}"
-      site.navigate
-      filter!(site.links)
-      add_redirect_url(site)
+      crawl_site(site)
 
       if block # The user of this block can send us a :retry to retry accessing the page
         while crawl_block(block, site) == :retry
           Grell.logger.info "Retrying our visit to #{site.url}"
-          site.navigate
-          filter!(site.links)
-          add_redirect_url(site)
+          crawl_site(site)
         end
       end
 
@@ -75,6 +72,13 @@ module Grell
 
     private
 
+    def crawl_site(site)
+      site.navigate
+      site.rawpage.page.evaluate_script(@evaluate_in_each_page) if @evaluate_in_each_page
+      filter!(site.links)
+      add_redirect_url(site)
+    end
+
     # Treat any exceptions from the block as an unavailable page
     def crawl_block(block, site)
       block.call(site)

data/lib/grell/page.rb

@@ -26,11 +26,12 @@ module Grell
     end
 
     def navigate
-      # We wait a maximum of WAIT_TIME seconds to get an HTML page. We try or best to workaround inconsistencies on poltergeist
+      # We wait a maximum of WAIT_TIME seconds to get an HTML page. We try our best to workaround inconsistencies on poltergeist
       Reader.wait_for(->{@rawpage.navigate(url)}, WAIT_TIME, WAIT_INTERVAL ) do
         @rawpage.status && !@rawpage.headers.empty? &&
           @rawpage.headers["Content-Type"] && @rawpage.headers["Content-Type"].include?('text/html').equal?(true)
       end
+      @rawpage.wait_for_all_ajax_requests(WAIT_TIME, WAIT_INTERVAL)
       @result_page = VisitedPage.new(@rawpage)
       @timestamp = Time.now
     rescue Capybara::Poltergeist::BrowserError, Capybara::Poltergeist::DeadClient,

data/lib/grell/rawpage.rb

@@ -27,7 +27,6 @@ module Grell
       all('[href]', visible: true).to_a + all('[data-href]', visible: true).to_a
     end
 
-
     def host
       page.current_host
     end
@@ -36,6 +35,17 @@ module Grell
       page.has_selector?(selector)
     end
 
+    def wait_for_all_ajax_requests(timeout, interval)
+      Timeout::timeout(timeout) do
+        (timeout / interval).ceil.times do
+          jquery_active = page.evaluate_script("typeof jQuery !== 'undefined' && jQuery.active;")
+          break if (!jquery_active || jquery_active.zero?)
+          sleep(interval)
+        end
+      end
+      true
+    end
+
     private
 
     def follow_redirects!

data/lib/grell/version.rb

@@ -1,3 +1,3 @@
 module Grell
-  VERSION = "1.6.10".freeze
+  VERSION = "1.6.11".freeze
 end

data/spec/lib/crawler_spec.rb

@@ -5,7 +5,8 @@ RSpec.describe Grell::Crawler do
   let(:page) { Grell::Page.new(url, page_id, parent_page_id) }
   let(:host) { 'http://www.example.com' }
   let(:url) { 'http://www.example.com/test' }
-  let(:crawler) { Grell::Crawler.new(logger: Logger.new(nil), external_driver: true) }
+  let(:crawler) { Grell::Crawler.new(logger: Logger.new(nil), external_driver: true, evaluate_in_each_page: script) }
+  let(:script) { nil }
   let(:body) { 'body' }
   let(:custom_add_match) do
     Proc.new do |collection_page, page|
@@ -85,6 +86,21 @@ RSpec.describe Grell::Crawler do
       expect_any_instance_of(Grell::PageCollection).to receive(:create_page).with(redirect_url, page_id)
       crawler.crawl(page, nil)
     end
+
+    context 'without script' do
+      it 'does not evaluate a script' do
+        expect_any_instance_of(Capybara::Session).not_to receive(:evaluate_script)
+        crawler.crawl(page, nil)
+      end
+    end
+
+    context 'with script' do
+      let(:script) { "(typeof(jQuery)!='undefined') && $('.dropdown').addClass('open');" }
+      it 'evaluates a script' do
+        expect_any_instance_of(Capybara::Session).to receive(:evaluate_script).with(script)
+        crawler.crawl(page, nil)
+      end
+    end
   end
 
   context '#start_crawling' do

data/spec/lib/page_spec.rb

@@ -92,6 +92,17 @@ RSpec.describe Grell::Page do
     end
   end
 
+  describe '#navigate' do
+    before do
+      proxy.stub(url).and_return(body: '', code: 200, headers: {})
+    end
+
+    it 'waits for all ajax requests' do
+      expect_any_instance_of(Grell::RawPage).to receive(:wait_for_all_ajax_requests).with(0, 0.5)
+      page.navigate
+    end
+  end
+
   shared_examples_for 'an errored grell page' do
     it 'returns empty status 404 page after navigating' do
       expect(page.status).to eq(404)

data/spec/spec_helper.rb

@@ -23,6 +23,7 @@ RSpec.configure do |config|
   # We do not need to wait for pages to return all the data
   config.before do
     stub_const("Grell::Page::WAIT_TIME", 0)
+    allow_any_instance_of(Capybara::Session).to receive(:evaluate_script).and_return(nil)
   end
 
   config.expect_with :rspec do |expectations|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: grell
 version: !ruby/object:Gem::Version
-  version: 1.6.10
+  version: 1.6.11
 platform: ruby
 authors:
 - Jordi Polo Carres
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-27 00:00:00.000000000 Z
+date: 2016-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara