grell 1.6.10 → 2.1.2

data/lib/grell/crawler_manager.rb

@@ -0,0 +1,84 @@
+module Grell
+  # Manages the state of the process crawling, does not care about individual pages but about logging,
+  # restarting and quiting the crawler correctly.
+  class CrawlerManager
+    # logger: logger to use for Grell's messages
+    # on_periodic_restart: if set, the driver will restart every :each visits (100 default) and execute the :do block
+    # driver_options: Any extra options for the Capybara driver
+    def initialize(logger: nil, on_periodic_restart: {}, driver: nil)
+      Grell.logger = logger ? logger : Logger.new(STDOUT)
+      @periodic_restart_block = on_periodic_restart[:do]
+      @periodic_restart_period = on_periodic_restart[:each] || PAGES_TO_RESTART
+      @driver = driver || CapybaraDriver.new.setup_capybara
+      if @periodic_restart_period <= 0
+        Grell.logger.warn "GRELL. Restart option misconfigured with a negative period. Ignoring option."
+      end
+    end
+
+    # Restarts the PhantomJS process without modifying the state of visited and discovered pages.
+    def restart
+      Grell.logger.info "GRELL. Driver restarting"
+      @driver.restart
+      Grell.logger.info "GRELL. Driver restarted"
+    end
+
+    # Quits the poltergeist driver.
+    def quit
+      Grell.logger.info "GRELL. Driver quitting"
+      @driver.quit
+    end
+
+    # PhantomJS seems to consume memory increasingly as it crawls, periodic restart allows to restart
+    # the driver, potentially calling a block.
+    def check_periodic_restart(collection)
+      return unless @periodic_restart_block
+      return unless @periodic_restart_period > 0
+      return unless (collection.visited_pages.size % @periodic_restart_period).zero?
+      restart
+      @periodic_restart_block.call
+    end
+
+    def self.cleanup_all_processes
+      PhantomJSManager.new.cleanup_all_processes
+    end
+
+    private
+
+    PAGES_TO_RESTART = 100  # Default number of pages before we restart the driver.
+    KILL_TIMEOUT = 2        # Number of seconds we wait till we kill the process.
+
+    # Manages the PhantomJS process
+    class PhantomJSManager
+      def cleanup_all_processes
+        pids = running_phantomjs_pids
+        return if pids.empty?
+        Grell.logger.warn "GRELL. Killing PhantomJS processes: #{pids.inspect}"
+        pids.each do |pid|
+          Grell.logger.warn "GRELL. Sending KILL to PhantomJS process #{pid}"
+          kill_process(pid.to_i)
+        end
+      end
+
+      def running_phantomjs_pids
+        list_phantomjs_processes_cmd = "ps -ef | grep -E 'bin/phantomjs' | grep -v grep"
+        `#{list_phantomjs_processes_cmd} | awk '{print $2;}'`.split("\n")
+      end
+
+      def kill_process(pid)
+        Process.kill('TERM', pid)
+        force_kill(pid)
+      rescue Errno::ESRCH, Errno::ECHILD
+        # successfully terminated
+      rescue => e
+        Grell.logger.error ["GRELL. PhantomJS process could not be killed", e.message, *e.backtrace].join($/)
+      end
+
+      def force_kill(pid)
+        Timeout.timeout(KILL_TIMEOUT) { Process.wait(pid) }
+      rescue Timeout::Error
+        Process.kill('KILL', pid)
+        Process.wait(pid)
+      end
+    end
+  end
+end

data/lib/grell/page.rb

@@ -26,11 +26,12 @@ module Grell
     end
 
     def navigate
-      # We wait a maximum of WAIT_TIME seconds to get an HTML page. We try or best to workaround inconsistencies on poltergeist
+      # We wait a maximum of WAIT_TIME seconds to get an HTML page. We try our best to workaround inconsistencies on poltergeist
       Reader.wait_for(->{@rawpage.navigate(url)}, WAIT_TIME, WAIT_INTERVAL ) do
         @rawpage.status && !@rawpage.headers.empty? &&
           @rawpage.headers["Content-Type"] && @rawpage.headers["Content-Type"].include?('text/html').equal?(true)
       end
+      @rawpage.wait_for_all_ajax_requests(WAIT_TIME, WAIT_INTERVAL)
       @result_page = VisitedPage.new(@rawpage)
       @timestamp = Time.now
     rescue Capybara::Poltergeist::BrowserError, Capybara::Poltergeist::DeadClient,

data/lib/grell/page_collection.rb

@@ -10,7 +10,7 @@ module Grell
     # to the collection or if it is already present will be passed to the initializer.
     def initialize(add_match_block)
       @collection = []
-      @add_match_block = add_match_block
+      @add_match_block = add_match_block || default_add_match
     end
 
     def create_page(url, parent_id)
@@ -50,5 +50,13 @@ module Grell
       end
     end
 
+    # If add_match_block is not provided, url matching to determine if a new page should be added
+    # to the page collection will default to this proc
+    def default_add_match
+      Proc.new do |collection_page, page|
+        collection_page.url.downcase == page.url.downcase
+      end
+    end
+
   end
 end

data/lib/grell/rawpage.rb

@@ -27,7 +27,6 @@ module Grell
       all('[href]', visible: true).to_a + all('[data-href]', visible: true).to_a
     end
 
-
     def host
       page.current_host
     end
@@ -36,6 +35,17 @@ module Grell
       page.has_selector?(selector)
     end
 
+    def wait_for_all_ajax_requests(timeout, interval)
+      Timeout::timeout(timeout) do
+        (timeout / interval).ceil.times do
+          jquery_active = page.evaluate_script("typeof jQuery !== 'undefined' && jQuery.active;")
+          break if (!jquery_active || jquery_active.zero?)
+          sleep(interval)
+        end
+      end
+      true
+    end
+
     private
 
     def follow_redirects!

data/lib/grell/version.rb

@@ -1,3 +1,3 @@
 module Grell
-  VERSION = "1.6.10".freeze
+  VERSION = "2.1.2".freeze
 end

data/spec/lib/capybara_driver_spec.rb

@@ -1,6 +1,9 @@
 
 RSpec.describe Grell::CapybaraDriver do
   let(:ts) { Time.now }
+  before do
+    Grell.logger = Logger.new(nil)
+  end
 
   describe 'setup_capybara' do
     it 'properly registers the poltergeist driver' do
@@ -25,14 +28,6 @@ RSpec.describe Grell::CapybaraDriver do
     end
   end
 
-  describe 'quit' do
-    let(:driver) { Grell::CapybaraDriver.new.setup_capybara }
-    it 'quits the poltergeist driver' do
-      expect_any_instance_of(Capybara::Poltergeist::Driver).to receive(:quit)
-      driver.quit
-    end
-  end
-
   after do
     Timecop.return
 

data/spec/lib/crawler_manager_spec.rb

@@ -0,0 +1,174 @@
+RSpec.describe Grell::CrawlerManager do
+  let(:page) { Grell::Page.new(url, page_id, parent_page_id) }
+  let(:host) { 'http://www.example.com' }
+  let(:url) { 'http://www.example.com/test' }
+  let(:driver) { double(Grell::CapybaraDriver) }
+  let(:logger) { Logger.new(nil) }
+  let(:crawler_manager) do
+    described_class.new(logger: logger, driver: driver)
+  end
+
+  describe 'initialize' do
+    context 'provides a logger' do
+      let(:logger) { 33 }
+
+      it 'sets custom logger' do
+        crawler_manager
+        expect(Grell.logger).to eq(33)
+        Grell.logger = Logger.new(nil)
+      end
+    end
+
+    context 'does not provides a logger' do
+      let(:logger) { nil }
+
+      it 'sets default logger' do
+        crawler_manager
+        expect(Grell.logger).to be_instance_of(Logger)
+        Grell.logger = Logger.new(nil)
+      end
+    end
+
+    context 'does not provide a driver' do
+      let(:driver) { nil }
+
+      it 'setups a new Capybara driver' do
+        expect_any_instance_of(Grell::CapybaraDriver).to receive(:setup_capybara)
+        crawler_manager
+      end
+    end
+  end
+
+  describe '#quit' do
+    let(:driver) { double }
+
+    it 'quits the poltergeist driver' do
+      expect(logger).to receive(:info).with("GRELL. Driver quitting")
+      expect(driver).to receive(:quit)
+      crawler_manager.quit
+    end
+  end
+
+  describe '#restart' do
+    let(:driver) { double }
+
+    it 'restarts the poltergeist driver' do
+      expect(driver).to receive(:restart)
+      expect(logger).to receive(:info).with("GRELL. Driver restarted")
+      expect(logger).to receive(:info).with("GRELL. Driver restarting")
+      crawler_manager.restart
+    end
+  end
+
+  describe '#check_periodic_restart' do
+    let(:collection) { double }
+
+    context 'Periodic restart not setup' do
+      it 'does not restart' do
+        allow(collection).to receive_message_chain(:visited_pages, :size) { 100 }
+        expect(crawler_manager).not_to receive(:restart)
+        crawler_manager.check_periodic_restart(collection)
+      end
+    end
+
+    context 'Periodic restart setup with default period' do
+      let(:do_something) { proc {} }
+      let(:crawler_manager) do
+        Grell::CrawlerManager.new(
+          logger: logger,
+          driver: driver,
+          on_periodic_restart: { do: do_something }
+        )
+      end
+
+      it 'does not restart after visiting 99 pages' do
+        allow(collection).to receive_message_chain(:visited_pages, :size) { 99 }
+        expect(crawler_manager).not_to receive(:restart)
+        crawler_manager.check_periodic_restart(collection)
+      end
+
+      it 'restarts after visiting 100 pages' do
+        allow(collection).to receive_message_chain(:visited_pages, :size) { 100 }
+        expect(crawler_manager).to receive(:restart)
+        crawler_manager.check_periodic_restart(collection)
+      end
+    end
+
+    context 'Periodic restart setup with custom period' do
+      let(:do_something) { proc {} }
+      let(:period) { 50 }
+      let(:crawler_manager) do
+        Grell::CrawlerManager.new(
+          logger: logger,
+          driver: driver,
+          on_periodic_restart: { do: do_something, each: period }
+        )
+      end
+
+      context 'restart option is not positive' do
+        let(:period) { 0 }
+
+        it 'logs a warning' do
+          message = 'GRELL. Restart option misconfigured with a negative period. Ignoring option.'
+          expect(logger).to receive(:warn).with(message)
+          crawler_manager
+        end
+      end
+
+      it 'does not restart after visiting a number different from custom period pages' do
+        allow(collection).to receive_message_chain(:visited_pages, :size) { period * 1.2 }
+        expect(crawler_manager).not_to receive(:restart)
+        crawler_manager.check_periodic_restart(collection)
+      end
+
+      it 'restarts after visiting custom period pages' do
+        allow(collection).to receive_message_chain(:visited_pages, :size) { period }
+        expect(crawler_manager).to receive(:restart)
+        crawler_manager.check_periodic_restart(collection)
+      end
+    end
+  end
+
+  describe '.cleanup_all_processes' do
+    let(:driver) { double }
+
+    context 'There are some phantomjs processes running' do
+      let(:pids) { [10, 11] }
+      before do
+        allow_any_instance_of(Grell::CrawlerManager::PhantomJSManager)
+          .to receive(:running_phantomjs_pids).and_return(pids)
+      end
+
+      it 'logs processes pids' do
+        expect(Grell.logger).to receive(:warn).with('GRELL. Killing PhantomJS processes: [10, 11]')
+        expect(Grell.logger).to receive(:warn).with('GRELL. Sending KILL to PhantomJS process 10')
+        expect(Grell.logger).to receive(:warn).with('GRELL. Sending KILL to PhantomJS process 11')
+        described_class.cleanup_all_processes
+      end
+
+      it 'kills all phantomjs processes' do
+        expect_any_instance_of(Grell::CrawlerManager::PhantomJSManager).to receive(:kill_process).with(10)
+        expect_any_instance_of(Grell::CrawlerManager::PhantomJSManager).to receive(:kill_process).with(11)
+        described_class.cleanup_all_processes
+      end
+    end
+
+    context 'There are no phantomjs processes running' do
+      let(:pids) { [] }
+      before do
+        allow_any_instance_of(Grell::CrawlerManager::PhantomJSManager)
+          .to receive(:running_phantomjs_pids).and_return(pids)
+      end
+
+      it 'no warning is logged' do
+        expect(Grell.logger).not_to receive(:warn)
+        described_class.cleanup_all_processes
+      end
+
+      it 'No process is killed' do
+        expect_any_instance_of(Grell::CrawlerManager::PhantomJSManager).not_to receive(:kill_process)
+        described_class.cleanup_all_processes
+      end
+    end
+  end
+end

data/spec/lib/crawler_spec.rb

@@ -5,7 +5,19 @@ RSpec.describe Grell::Crawler do
   let(:page) { Grell::Page.new(url, page_id, parent_page_id) }
   let(:host) { 'http://www.example.com' }
   let(:url) { 'http://www.example.com/test' }
-  let(:crawler) { Grell::Crawler.new(logger: Logger.new(nil), external_driver: true) }
+  let(:add_match_block) { nil }
+  let(:denylist) { /a^/ }
+  let(:allowlist) { /.*/ }
+  let(:crawler) do
+    Grell::Crawler.new(
+      logger: Logger.new(nil),
+      driver: double(nil),
+      evaluate_in_each_page: script,
+      add_match_block: add_match_block,
+      denylist: denylist,
+      allowlist: allowlist)
+  end
+  let(:script) { nil }
   let(:body) { 'body' }
   let(:custom_add_match) do
     Proc.new do |collection_page, page|
@@ -17,29 +29,6 @@ RSpec.describe Grell::Crawler do
     proxy.stub(url).and_return(body: body, code: 200)
   end
 
-  describe 'initialize' do
-    it 'can provide your own logger' do
-      Grell::Crawler.new(external_driver: true, logger: 33)
-      expect(Grell.logger).to eq(33)
-      Grell.logger = Logger.new(nil)
-    end
-
-    it 'provides a stdout logger if nothing provided' do
-      crawler
-      expect(Grell.logger).to be_instance_of(Logger)
-    end
-  end
-
-  describe '#quit' do
-    let(:driver) { double }
-    before { allow(Grell::CapybaraDriver).to receive(:setup).and_return(driver) }
-
-    it 'quits the poltergeist driver' do
-      expect(driver).to receive(:quit)
-      crawler.quit
-    end
-  end
-
   describe '#crawl' do
     before do
       crawler.instance_variable_set('@collection', Grell::PageCollection.new(custom_add_match))
@@ -85,6 +74,21 @@ RSpec.describe Grell::Crawler do
       expect_any_instance_of(Grell::PageCollection).to receive(:create_page).with(redirect_url, page_id)
       crawler.crawl(page, nil)
     end
+
+    context 'without script' do
+      it 'does not evaluate a script' do
+        expect_any_instance_of(Capybara::Session).not_to receive(:evaluate_script)
+        crawler.crawl(page, nil)
+      end
+    end
+
+    context 'with script' do
+      let(:script) { "(typeof(jQuery)!='undefined') && $('.dropdown').addClass('open');" }
+      it 'evaluates a script' do
+        expect_any_instance_of(Capybara::Session).to receive(:evaluate_script).with(script)
+        crawler.crawl(page, nil)
+      end
+    end
   end
 
   context '#start_crawling' do
@@ -111,15 +115,6 @@ RSpec.describe Grell::Crawler do
       expect(result[1].url).to eq(url_visited)
     end
 
-    it 'can use a custom url add matcher block' do
-      expect(crawler).to_not receive(:default_add_match)
-      crawler.start_crawling(url, add_match_block: custom_add_match)
-    end
-
-    it 'uses a default url add matched if not provided' do
-      expect(crawler).to receive(:default_add_match).and_return(custom_add_match)
-      crawler.start_crawling(url)
-    end
   end
 
   shared_examples_for 'visits all available pages' do
@@ -133,7 +128,7 @@ RSpec.describe Grell::Crawler do
       expect(crawler.collection.discovered_pages.size).to eq(0)
     end
 
-    it 'contains the whitelisted page and the base page only' do
+    it 'contains the allowlisted page and the base page only' do
       crawler.start_crawling(url)
       expect(crawler.collection.visited_pages.map(&:url)).
         to eq(visited_pages)
@@ -173,7 +168,7 @@ RSpec.describe Grell::Crawler do
     it_behaves_like 'visits all available pages'
   end
 
-  describe '#whitelist' do
+  describe '#allowlist' do
     let(:body) do
       "<html><head></head><body>
       <a href=\"/trusmis.html\">trusmis</a>
@@ -188,10 +183,7 @@ RSpec.describe Grell::Crawler do
     end
 
     context 'using a single string' do
-      before do
-        crawler.whitelist('/trusmis.html')
-      end
-
+      let(:allowlist) { '/trusmis.html' }
       let(:visited_pages_count) { 2 } # my own page + trusmis
       let(:visited_pages) do
         ['http://www.example.com/test', 'http://www.example.com/trusmis.html']
@@ -201,10 +193,7 @@ RSpec.describe Grell::Crawler do
     end
 
     context 'using an array of strings' do
-      before do
-        crawler.whitelist(['/trusmis.html', '/nothere', 'another.html'])
-      end
-
+      let(:allowlist) { ['/trusmis.html', '/nothere', 'another.html'] }
       let(:visited_pages_count) { 2 }
       let(:visited_pages) do
         ['http://www.example.com/test', 'http://www.example.com/trusmis.html']
@@ -214,10 +203,7 @@ RSpec.describe Grell::Crawler do
     end
 
     context 'using a regexp' do
-      before do
-        crawler.whitelist(/\/trusmis\.html/)
-      end
-
+      let(:allowlist) { /\/trusmis\.html/ }
       let(:visited_pages_count) { 2 }
       let(:visited_pages) do
         ['http://www.example.com/test', 'http://www.example.com/trusmis.html']
@@ -227,10 +213,7 @@ RSpec.describe Grell::Crawler do
     end
 
     context 'using an array of regexps' do
-      before do
-        crawler.whitelist([/\/trusmis\.html/])
-      end
-
+      let(:allowlist) { [/\/trusmis\.html/] }
       let(:visited_pages_count) { 2 }
       let(:visited_pages) do
         ['http://www.example.com/test', 'http://www.example.com/trusmis.html']
@@ -240,10 +223,7 @@ RSpec.describe Grell::Crawler do
     end
 
     context 'using an empty array' do
-      before do
-        crawler.whitelist([])
-      end
-
+      let(:allowlist) { [] }
       let(:visited_pages_count) { 1 } # my own page only
       let(:visited_pages) do
         ['http://www.example.com/test']
@@ -252,11 +232,8 @@ RSpec.describe Grell::Crawler do
       it_behaves_like 'visits all available pages'
     end
 
-    context 'adding all links to the whitelist' do
-      before do
-        crawler.whitelist(['/trusmis', '/help'])
-      end
-
+    context 'adding all links to the allowlist' do
+      let(:allowlist) { ['/trusmis', '/help'] }
       let(:visited_pages_count) { 3 } # all links
       let(:visited_pages) do
         ['http://www.example.com/test','http://www.example.com/trusmis.html', 'http://www.example.com/help.html']
@@ -267,7 +244,7 @@ RSpec.describe Grell::Crawler do
   end
 
 
-  describe '#blacklist' do
+  describe '#denylist' do
     let(:body) do
       "<html><head></head><body>
       <a href=\"/trusmis.html\">trusmis</a>
@@ -282,9 +259,7 @@ RSpec.describe Grell::Crawler do
     end
 
     context 'using a single string' do
-      before do
-        crawler.blacklist('/trusmis.html')
-      end
+      let(:denylist) { '/trusmis.html' }
       let(:visited_pages_count) {2}
       let(:visited_pages) do
         ['http://www.example.com/test','http://www.example.com/help.html']
@@ -294,9 +269,7 @@ RSpec.describe Grell::Crawler do
     end
 
     context 'using an array of strings' do
-      before do
-        crawler.blacklist(['/trusmis.html', '/nothere', 'another.html'])
-      end
+      let(:denylist) { ['/trusmis.html', '/nothere', 'another.html'] }
       let(:visited_pages_count) {2}
       let(:visited_pages) do
         ['http://www.example.com/test','http://www.example.com/help.html']
@@ -306,9 +279,7 @@ RSpec.describe Grell::Crawler do
     end
 
     context 'using a regexp' do
-      before do
-        crawler.blacklist(/\/trusmis\.html/)
-      end
+      let(:denylist) { /\/trusmis\.html/ }
       let(:visited_pages_count) {2}
       let(:visited_pages) do
         ['http://www.example.com/test','http://www.example.com/help.html']
@@ -318,9 +289,7 @@ RSpec.describe Grell::Crawler do
     end
 
     context 'using an array of regexps' do
-      before do
-        crawler.blacklist([/\/trusmis\.html/])
-      end
+      let(:denylist) { [/\/trusmis\.html/] }
       let(:visited_pages_count) {2}
       let(:visited_pages) do
         ['http://www.example.com/test','http://www.example.com/help.html']
@@ -330,9 +299,7 @@ RSpec.describe Grell::Crawler do
     end
 
     context 'using an empty array' do
-      before do
-        crawler.blacklist([])
-      end
+      let(:denylist) { [] }
       let(:visited_pages_count) { 3 } # all links
       let(:visited_pages) do
         ['http://www.example.com/test','http://www.example.com/trusmis.html', 'http://www.example.com/help.html']
@@ -341,10 +308,8 @@ RSpec.describe Grell::Crawler do
       it_behaves_like 'visits all available pages'
     end
 
-    context 'adding all links to the whitelist' do
-      before do
-        crawler.blacklist(['/trusmis', '/help'])
-      end
+    context 'adding all links to the denylist' do
+      let(:denylist) { ['/trusmis', '/help'] }
       let(:visited_pages_count) { 1 }
       let(:visited_pages) do
         ['http://www.example.com/test']
@@ -355,7 +320,7 @@ RSpec.describe Grell::Crawler do
   end
 
 
-  describe 'Whitelisting and blacklisting' do
+  describe 'allowlisting and denylisting' do
     let(:body) do
       "<html><head></head><body>
       <a href=\"/trusmis.html\">trusmis</a>
@@ -369,12 +334,9 @@ RSpec.describe Grell::Crawler do
       proxy.stub('http://www.example.com/help.html').and_return(body: 'body', code: 200)
     end
 
-    context 'we blacklist the only whitelisted page' do
-      before do
-        crawler.whitelist('/trusmis.html')
-        crawler.blacklist('/trusmis.html')
-      end
-
+    context 'we denylist the only allowlisted page' do
+      let(:allowlist) { '/trusmis.html' }
+      let(:denylist) { '/trusmis.html' }
       let(:visited_pages_count) { 1 }
       let(:visited_pages) do
         ['http://www.example.com/test']
@@ -383,12 +345,9 @@ RSpec.describe Grell::Crawler do
       it_behaves_like 'visits all available pages'
     end
 
-    context 'we blacklist none of the whitelisted pages' do
-      before do
-        crawler.whitelist('/trusmis.html')
-        crawler.blacklist('/raistlin.html')
-      end
-
+    context 'we denylist none of the allowlisted pages' do
+      let(:allowlist) { '/trusmis.html' }
+      let(:denylist) { '/raistlin.html' }
       let(:visited_pages_count) { 2 }
       let(:visited_pages) do
         ['http://www.example.com/test', 'http://www.example.com/trusmis.html']