grell 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bbd2a19c7858d2e755e7d51a038b84901835f32b
-  data.tar.gz: 7c65a731bfdacb7b65cf523c8ba128f1f3390f96
+  metadata.gz: 8d9acf34b301934669bf458555b29ef7c429f936
+  data.tar.gz: 3949e320d75862e04325581986c24209440fd46e
 SHA512:
-  metadata.gz: fcf31c442f8d51cd4a9534270cc3afd7f5bac432a74a9ec2e429914145df710131e0a99aca61e4cd98d2e831a5cedb511c21dce9d1562881541fd1bfb4016014
-  data.tar.gz: c413503a4a8765fadbec10ed64467ba140b8739b6209ae4a19e585549ad689231b64b3d670e1bdceec560597051d93f8b90d696f66571cb1dad175f15af1b4a0
+  metadata.gz: 3cd06fcba285f5f894d0127c7de96575f032fff8d10cdcfa2ee5042b287b01babdbd3119c5c1d979c8610ae9ee9d571f8c6144711415acbcf09f3d43ceac567c
+  data.tar.gz: 4dfb9e9d2e83095ca66df36bc8f41082fc37a29995d75e0553f8466eeb0dd36b62c3578861f171c9c29e4815ca257807dbd53dab4eee875301d1613fde6a437a

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 2.1.0
+  * Delete `driver_options` configuration key as it was never used.
+  * `cleanup_all_processes` is a self method as intended to.
+
 # 2.0.0
   * New configuration key `on_periodic_restart`.
   * CrawlerManager.cleanup_all_processes method destroy all instances of phantomjs in this machine.

data/README.md

@@ -96,7 +96,6 @@ The `Grell:Crawler` class can be passed options to customize its behavior:
 - `blacklist`: Setups a blacklist filter for URLs to be avoided. Default: no URL is blacklisted.
 - `add_match_block`: Block evaluated to consider if a given page should be part of the pages to be visited. Default: add unique URLs.
 - `evaluate_in_each_page`: Javascript block to be evaluated on each page visited. Default: Nothing evaluated.
-- `driver_options`: Driver options will be passed to the Capybara driver which connects to PhantomJS.
 
 Grell by default will follow all the links it finds in the site being crawled.
 It will never follow links linking outside your site.

data/lib/grell/capybara_driver.rb

@@ -1,16 +1,10 @@
-
 module Grell
-
-  #The driver for Capybara. It uses Portelgeist to control PhantomJS
+  # This class setups the driver for capybara. Used internally by the CrawlerManager
+  # It uses Portelgeist to control PhantomJS
   class CapybaraDriver
-    include Capybara::DSL
-
     USER_AGENT = "Mozilla/5.0 (Grell Crawler)".freeze
 
-    def self.setup(options)
-      new.setup_capybara unless options[:external_driver]
-    end
-
+    # Returns a poltergeist driver
     def setup_capybara
       @poltergeist_driver = nil
 
@@ -20,18 +14,17 @@ module Grell
       Grell.logger.info "GRELL Registering poltergeist driver with name '#{driver_name}'"
 
       Capybara.register_driver driver_name do |app|
-        @poltergeist_driver = Capybara::Poltergeist::Driver.new(app, {
+        @poltergeist_driver = Capybara::Poltergeist::Driver.new(app,
           js_errors: false,
           inspector: false,
           phantomjs_logger: FakePoltergeistLogger,
-          phantomjs_options: ['--debug=no', '--load-images=no', '--ignore-ssl-errors=yes', '--ssl-protocol=TLSv1']
-         })
+          phantomjs_options: ['--debug=no', '--load-images=no', '--ignore-ssl-errors=yes', '--ssl-protocol=TLSv1'])
       end
 
       Capybara.default_max_wait_time = 3
       Capybara.run_server = false
       Capybara.default_driver = driver_name
-      page.driver.headers = {
+      Capybara.current_session.driver.headers = { # The driver gets initialized when modified here
         "DNT" => 1,
         "User-Agent" => USER_AGENT
       }
@@ -41,14 +34,11 @@ module Grell
       @poltergeist_driver
     end
 
-    def quit
-      @poltergeist_driver.quit
-    end
-
+    # Poltergeist driver needs a class with this signature. The javascript console.log is sent here.
+    # We just discard that information.
     module FakePoltergeistLogger
       def self.puts(*)
       end
     end
   end
-
 end

data/lib/grell/crawler_manager.rb

@@ -5,26 +5,26 @@ module Grell
     # logger: logger to use for Grell's messages
     # on_periodic_restart: if set, the driver will restart every :each visits (100 default) and execute the :do block
     # driver_options: Any extra options for the Capybara driver
-    def initialize(logger: nil, on_periodic_restart: {}, driver: nil, driver_options: {})
+    def initialize(logger: nil, on_periodic_restart: {}, driver: nil)
       Grell.logger = logger ? logger : Logger.new(STDOUT)
       @periodic_restart_block = on_periodic_restart[:do]
       @periodic_restart_period = on_periodic_restart[:each] || PAGES_TO_RESTART
-      @driver = driver || CapybaraDriver.setup(driver_options)
+      @driver = driver || CapybaraDriver.new.setup_capybara
       if @periodic_restart_period <= 0
-        Grell.logger.warn "GRELL being misconfigured with a negative period to restart. Ignoring option."
+        Grell.logger.warn "GRELL. Restart option misconfigured with a negative period. Ignoring option."
       end
     end
 
     # Restarts the PhantomJS process without modifying the state of visited and discovered pages.
     def restart
-      Grell.logger.info "GRELL is restarting"
+      Grell.logger.info "GRELL. Driver restarting"
       @driver.restart
-      Grell.logger.info "GRELL has restarted"
+      Grell.logger.info "GRELL. Driver restarted"
     end
 
     # Quits the poltergeist driver.
     def quit
-      Grell.logger.info "GRELL is quitting the poltergeist driver"
+      Grell.logger.info "GRELL. Driver quitting"
       @driver.quit
     end
 
@@ -38,14 +38,8 @@ module Grell
       @periodic_restart_block.call
     end
 
-    def cleanup_all_processes
-      pids = running_phantomjs_pids
-      return if pids.empty?
-      Grell.logger.warn "GRELL. Killing PhantomJS processes: #{pids.inspect}"
-      pids.each do |pid|
-        Grell.logger.warn "Sending KILL to PhantomJS process #{pid}"
-        kill_process(pid.to_i)
-      end
+    def self.cleanup_all_processes
+      PhantomJSManager.new.cleanup_all_processes
     end
 
     private
@@ -53,25 +47,38 @@ module Grell
     PAGES_TO_RESTART = 100  # Default number of pages before we restart the driver.
     KILL_TIMEOUT = 2        # Number of seconds we wait till we kill the process.
 
-    def running_phantomjs_pids
-      list_phantomjs_processes_cmd = "ps -ef | grep -E 'bin/phantomjs' | grep -v grep"
-      `#{list_phantomjs_processes_cmd} | awk '{print $2;}'`.split("\n")
-    end
+    # Manages the PhantomJS process
+    class PhantomJSManager
+      def cleanup_all_processes
+        pids = running_phantomjs_pids
+        return if pids.empty?
+        Grell.logger.warn "GRELL. Killing PhantomJS processes: #{pids.inspect}"
+        pids.each do |pid|
+          Grell.logger.warn "GRELL. Sending KILL to PhantomJS process #{pid}"
+          kill_process(pid.to_i)
+        end
+      end
 
-    def kill_process(pid)
-      Process.kill('TERM', pid)
-      force_kill(pid)
-    rescue Errno::ESRCH, Errno::ECHILD
-      # successfully terminated
-    rescue => e
-      Grell.logger.exception e, "PhantomJS process could not be killed"
-    end
+      def running_phantomjs_pids
+        list_phantomjs_processes_cmd = "ps -ef | grep -E 'bin/phantomjs' | grep -v grep"
+        `#{list_phantomjs_processes_cmd} | awk '{print $2;}'`.split("\n")
+      end
+
+      def kill_process(pid)
+        Process.kill('TERM', pid)
+        force_kill(pid)
+      rescue Errno::ESRCH, Errno::ECHILD
+        # successfully terminated
+      rescue => e
+        Grell.logger.exception e, "GRELL. PhantomJS process could not be killed"
+      end
 
-    def force_kill(pid)
-      Timeout.timeout(KILL_TIMEOUT) { Process.wait(pid) }
-    rescue Timeout::Error
-      Process.kill('KILL', pid)
-      Process.wait(pid)
+      def force_kill(pid)
+        Timeout.timeout(KILL_TIMEOUT) { Process.wait(pid) }
+      rescue Timeout::Error
+        Process.kill('KILL', pid)
+        Process.wait(pid)
+      end
     end
   end
 end

data/lib/grell/version.rb

@@ -1,3 +1,3 @@
 module Grell
-  VERSION = "2.0.0".freeze
+  VERSION = "2.1.0".freeze
 end

data/spec/lib/capybara_driver_spec.rb

@@ -1,6 +1,9 @@
 
 RSpec.describe Grell::CapybaraDriver do
   let(:ts) { Time.now }
+  before do
+    Grell.logger = Logger.new(nil)
+  end
 
   describe 'setup_capybara' do
     it 'properly registers the poltergeist driver' do
@@ -25,14 +28,6 @@ RSpec.describe Grell::CapybaraDriver do
     end
   end
 
-  describe 'quit' do
-    let(:driver) { Grell::CapybaraDriver.new.setup_capybara }
-    it 'quits the poltergeist driver' do
-      expect_any_instance_of(Capybara::Poltergeist::Driver).to receive(:quit)
-      driver.quit
-    end
-  end
-
   after do
     Timecop.return
 

data/spec/lib/crawler_manager_spec.rb

@@ -11,26 +11,39 @@ RSpec.describe Grell::CrawlerManager do
   describe 'initialize' do
     context 'provides a logger' do
       let(:logger) { 33 }
+
       it 'sets custom logger' do
         crawler_manager
         expect(Grell.logger).to eq(33)
         Grell.logger = Logger.new(nil)
       end
     end
+
     context 'does not provides a logger' do
       let(:logger) { nil }
+
       it 'sets default logger' do
         crawler_manager
         expect(Grell.logger).to be_instance_of(Logger)
         Grell.logger = Logger.new(nil)
       end
     end
+
+    context 'does not provide a driver' do
+      let(:driver) { nil }
+
+      it 'setups a new Capybara driver' do
+        expect_any_instance_of(Grell::CapybaraDriver).to receive(:setup_capybara)
+        crawler_manager
+      end
+    end
   end
 
   describe '#quit' do
     let(:driver) { double }
 
     it 'quits the poltergeist driver' do
+      expect(logger).to receive(:info).with("GRELL. Driver quitting")
       expect(driver).to receive(:quit)
       crawler_manager.quit
     end
@@ -41,14 +54,15 @@ RSpec.describe Grell::CrawlerManager do
 
     it 'restarts the poltergeist driver' do
       expect(driver).to receive(:restart)
-      expect(logger).to receive(:info).with("GRELL is restarting")
-      expect(logger).to receive(:info).with("GRELL has restarted")
+      expect(logger).to receive(:info).with("GRELL. Driver restarted")
+      expect(logger).to receive(:info).with("GRELL. Driver restarting")
       crawler_manager.restart
     end
   end
 
   describe '#check_periodic_restart' do
     let(:collection) { double }
+
     context 'Periodic restart not setup' do
       it 'does not restart' do
         allow(collection).to receive_message_chain(:visited_pages, :size) { 100 }
@@ -56,6 +70,7 @@ RSpec.describe Grell::CrawlerManager do
         crawler_manager.check_periodic_restart(collection)
       end
     end
+
     context 'Periodic restart setup with default period' do
       let(:do_something) { proc {} }
       let(:crawler_manager) do
@@ -71,12 +86,14 @@ RSpec.describe Grell::CrawlerManager do
         expect(crawler_manager).not_to receive(:restart)
         crawler_manager.check_periodic_restart(collection)
       end
+
       it 'restarts after visiting 100 pages' do
         allow(collection).to receive_message_chain(:visited_pages, :size) { 100 }
         expect(crawler_manager).to receive(:restart)
         crawler_manager.check_periodic_restart(collection)
       end
     end
+
     context 'Periodic restart setup with custom period' do
       let(:do_something) { proc {} }
       let(:period) { 50 }
@@ -88,11 +105,22 @@ RSpec.describe Grell::CrawlerManager do
         )
       end
 
+      context 'restart option is not positive' do
+        let(:period) { 0 }
+
+        it 'logs a warning' do
+          message = 'GRELL. Restart option misconfigured with a negative period. Ignoring option.'
+          expect(logger).to receive(:warn).with(message)
+          crawler_manager
+        end
+      end
+
       it 'does not restart after visiting a number different from custom period pages' do
         allow(collection).to receive_message_chain(:visited_pages, :size) { period * 1.2 }
         expect(crawler_manager).not_to receive(:restart)
         crawler_manager.check_periodic_restart(collection)
       end
+
       it 'restarts after visiting custom period pages' do
         allow(collection).to receive_message_chain(:visited_pages, :size) { period }
         expect(crawler_manager).to receive(:restart)
@@ -101,13 +129,46 @@ RSpec.describe Grell::CrawlerManager do
     end
   end
 
-  describe '#cleanup_all_processes' do
+  describe '.cleanup_all_processes' do
     let(:driver) { double }
 
-    it 'kills all phantomjs processes' do
-      allow(crawler_manager).to receive(:running_phantomjs_pids).and_return([10])
-      expect(crawler_manager).to receive(:kill_process).with(10)
-      crawler_manager.cleanup_all_processes
+    context 'There are some phantomjs processes running' do
+      let(:pids) { [10, 11] }
+      before do
+        allow_any_instance_of(Grell::CrawlerManager::PhantomJSManager)
+          .to receive(:running_phantomjs_pids).and_return(pids)
+      end
+
+      it 'logs processes pids' do
+        expect(Grell.logger).to receive(:warn).with('GRELL. Killing PhantomJS processes: [10, 11]')
+        expect(Grell.logger).to receive(:warn).with('GRELL. Sending KILL to PhantomJS process 10')
+        expect(Grell.logger).to receive(:warn).with('GRELL. Sending KILL to PhantomJS process 11')
+        described_class.cleanup_all_processes
+      end
+
+      it 'kills all phantomjs processes' do
+        expect_any_instance_of(Grell::CrawlerManager::PhantomJSManager).to receive(:kill_process).with(10)
+        expect_any_instance_of(Grell::CrawlerManager::PhantomJSManager).to receive(:kill_process).with(11)
+        described_class.cleanup_all_processes
+      end
+    end
+
+    context 'There are no phantomjs processes running' do
+      let(:pids) { [] }
+      before do
+        allow_any_instance_of(Grell::CrawlerManager::PhantomJSManager)
+          .to receive(:running_phantomjs_pids).and_return(pids)
+      end
+
+      it 'no warning is logged' do
+        expect(Grell.logger).not_to receive(:warn)
+        described_class.cleanup_all_processes
+      end
+
+      it 'No process is killed' do
+        expect_any_instance_of(Grell::CrawlerManager::PhantomJSManager).not_to receive(:kill_process)
+        described_class.cleanup_all_processes
+      end
     end
   end
 end

data/spec/lib/crawler_spec.rb

@@ -11,7 +11,7 @@ RSpec.describe Grell::Crawler do
   let(:crawler) do
     Grell::Crawler.new(
       logger: Logger.new(nil),
-      driver_options: { external_driver: true },
+      driver: double(nil),
       evaluate_in_each_page: script,
       add_match_block: add_match_block,
       blacklist: blacklist,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: grell
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Jordi Polo Carres
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-16 00:00:00.000000000 Z
+date: 2016-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -228,4 +228,3 @@ test_files:
 - spec/lib/page_spec.rb
 - spec/lib/reader_spec.rb
 - spec/spec_helper.rb
-has_rdoc: