wayfarer 0.4.7 → 0.4.9

data/spec/spec_helpers.rb

@@ -1,44 +1,52 @@
 # frozen_string_literal: true
 
+require "active_support/core_ext/object/deep_dup"
+require "capybara/cuprite"
 require "factory_bot"
+require "puma"
 require "pry"
+require "rake"
+require "rspec-parameterized"
 require "sinatra"
-require "capybara/cuprite"
+require "webrick"
 
 require_relative "../lib/wayfarer"
+
 require_relative "support/test_app"
+require_relative "support/redis_helpers"
+require_relative "support/active_job_helpers"
+require_relative "support/integration_helpers"
 
 module SpecHelpers
-  def test_app_path(path)
-    File.join("http://", test_app_host, path)
+  WAYFARER_TEST_APP_PORT = ENV.fetch("WAYFARER_PORT", "9876").to_s
+
+  def ci?
+    ENV.key?("CI")
+  end
+
+  def test_redis_host
+    ci? ? "redis" : "localhost"
   end
 
   def test_app_host
-    if ENV["CI"]
-      "test:9876"
-    else
-      "localhost:9876"
-    end
+    ci? ? "test" : "localhost"
   end
 
-  def redis_host
-    if ENV["CI"]
-      "redis://redis:6379"
-    else
-      "redis://localhost:6379"
-    end
+  def redis_url
+    "redis://#{test_redis_host}:6379/0"
   end
 
-  def write_file(path, contents)
-    FileUtils.mkdir_p(Pathname.new(path).dirname)
-    IO.write(path, contents)
+  def test_app_path(path = "")
+    File.join("http://", "#{test_app_host}:#{WAYFARER_TEST_APP_PORT}", path)
   end
-end
 
-module ActiveJobTestHelper
-  # perform_enqueued_jobs calls assert_equal
-  def assert_equal(expected, actual, _)
-    expect(actual).to eq(expected)
+  def test_app_hostname
+    test_app_host.split(":").first
+  end
+
+  def write_file(path, contents)
+    FileUtils.mkdir_p(Pathname.new(path).dirname)
+    File.write(path, contents)
   end
 end
 
@@ -47,57 +55,46 @@ RSpec.configure do |config|
   config.include(ActiveJob::TestHelper)
   config.include(SpecHelpers)
   config.include(ActiveJobTestHelper)
+  config.include(IntegrationHelpers)
 
-  config.before(:suite) do
+  # The shared context enables a global `let(:redis)` declaration
+  config.include_context "with Redis", :redis
+
+  config.before :suite do
     Wayfarer::Logging.logger = Logger.new($stdout)
 
     FactoryBot.find_definitions
-    FactoryBot::SyntaxRunner.include SpecHelpers
-
-    mutex = Mutex.new
-    cvar = ConditionVariable.new
-
-    Thread.new do
-      Rack::Handler::WEBrick.run(TestApp,
-                                 Host: "0.0.0.0",
-                                 Port: 9876,
-                                 Logger: WEBrick::Log.new("/dev/null"),
-                                 AccessLog: [],
-                                 StartCallback: proc { cvar.signal })
-    end
+    FactoryBot::SyntaxRunner.include(SpecHelpers)
 
-    mutex.lock
-    cvar.wait(mutex)
+    Support::TestApp.start!(SpecHelpers::WAYFARER_TEST_APP_PORT)
   end
 
   config.before do
-    Wayfarer.config = Wayfarer::DEFAULT_CONFIG.clone
-    Wayfarer.config[:redis][:url] = redis_host
+    Wayfarer.config = Wayfarer::DEFAULT_CONFIG.deep_dup
+    Wayfarer.config[:redis][:url] = redis_url
 
-    ActiveJob::Base.queue_adapter = :test
     ActiveJob::Base.logger = Logger.new(nil)
+    ActiveJob::Base.queue_adapter = :test
     ActiveJob::Base.queue_adapter.enqueued_jobs.clear
     ActiveJob::Base.queue_adapter.performed_jobs.clear
 
-    if ENV["CI"]
-      # For unknown reasons, Wayfarer::Networking::Ferrum#renew can take a very
-      # long time to instantiate a new ::Ferrum::Browser and requires the 60s
-      # timeout. TODO: Figure out what's going on
+    if ENV.key?("CI")
       Wayfarer.config[:ferrum][:options] = { url: "http://chrome:3000", timeout: 120 }
-      Wayfarer.config[:selenium][:driver] = :firefox
-      Wayfarer.config[:selenium][:options] = { url: "http://firefox:4444" }
-      Wayfarer.config[:selenium][:client_timeout] = 10
+
+      Wayfarer.config[:selenium][:driver] = :remote
+      Wayfarer.config[:selenium][:options] = {
+        url: "http://firefox:4444",
+        options: Selenium::WebDriver::Options.firefox
+      }
     end
 
     # TODO: Undo side-effect
     Capybara.register_driver(:cuprite) do |app|
-      Capybara::Cuprite::Driver.new(app, Wayfarer.config[:ferrum][:options])
+      Capybara::Cuprite::Driver.new(app, Wayfarer.config.dig(:ferrum, :options))
     end
     Wayfarer.config[:capybara][:driver] = :cuprite
-  end
 
-  config.before(:each, redis: true) do
-    ::Redis.new(url: redis_host).flushall
+    Wayfarer::Networking::Pool.finalizer = ->(_pool) {}
   end
 
   config.around(cli: true) do |example|
@@ -107,10 +104,6 @@ RSpec.configure do |config|
     example.run
   ensure
     Dir.chdir(origin)
-    # FileUtils.rm_r(tmp)
-  end
-
-  config.before(:each, cli: true) do
-    Object.send(:remove_const, :DummyJob) if defined?(DummyJob)
+    FileUtils.rm_r(tmp)
   end
 end

data/spec/support/active_job_helpers.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module ActiveJobTestHelper
+  # perform_enqueued_jobs expects assert_equal to be defined
+  def assert_equal(expected, actual, _)
+    expect(actual).to eq(expected)
+  end
+end

data/spec/support/integration_helpers.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module IntegrationHelpers
+  def mock_job!(symbol)
+    mock!(symbol) do
+      Class.new(ActiveJob::Base).include(Wayfarer::Base)
+    end
+  end
+
+  def mock_handler!(symbol)
+    mock!(symbol) do
+      Class.new.include(Wayfarer::Handler)
+    end
+  end
+
+private
+
+  def mock!(symbol, &block)
+    stub_const(symbol.to_s.camelize, block.call)
+  end
+end

data/spec/support/redis_helpers.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+RSpec.shared_context "with Redis", :redis do
+  let(:redis) { Redis.new(url: redis_url) }
+
+  before do
+    redis.flushall
+  end
+end

data/spec/support/test_app.rb

@@ -1,60 +1,81 @@
 # frozen_string_literal: true
 
-class TestApp < Sinatra::Base
-  set :root, File.dirname(__FILE__)
-  set :public_folder, -> { File.join(root, "static") }
+module Support
+  class TestApp < Sinatra::Base
+    def self.start!(port)
+      mutex = Mutex.new
+      cvar = ConditionVariable.new
 
-  get "/status_code/:code" do
-    status params[:code]
-  end
+      Thread.new do
+        Rackup::Handler::WEBrick.run(
+          TestApp,
+          Host: "0.0.0.0",
+          Port: port,
+          Logger: WEBrick::Log.new(File::NULL),
+          AccessLog: [],
+          StartCallback: proc { cvar.signal }
+        )
+      end
 
-  get "/headers/:header" do
-    request.env[params[:header]]
-  end
+      mutex.lock
+      cvar.wait(mutex)
+    end
 
-  get "/response_header/:key/:val" do
-    headers params[:key] => params[:val]
-  end
+    set :root, File.dirname(__FILE__)
+    set :public_folder, -> { File.join(root, "static") }
 
-  get "/response_header/Content-Type/*" do
-    headers "Content-Type" => params[:splat].first
-  end
+    get "/status_code/:code" do
+      status params[:code]
+    end
 
-  get "/hello_world" do
-    headers "hello" => "world"
-    "Hello world!"
-  end
+    get "/headers/:header" do
+      request.env[params[:header]]
+    end
 
-  get "/body/:content" do
-    params[:content]
-  end
+    get "/response_header/:key/:val" do
+      headers params[:key] => params[:val]
+    end
 
-  get "/redirect_loop" do
-    redirect to "/redirect_loop"
-  end
+    get "/response_header/Content-Type/*" do
+      content_type params[:splat].first
+    end
 
-  get "/redirect" do
-    n = params[:times].to_i
-    n.zero? ? "You arrived!" : (redirect to "/redirect?times=#{n - 1}")
-  end
+    get "/hello_world" do
+      headers "hello" => "world"
+      "Hello world!"
+    end
 
-  get "/malformed_redirect" do
-    redirect to "hptt://bro.ken"
-  end
+    get "/body/:content" do
+      params[:content]
+    end
 
-  get "/json/:file" do
-    content_type "application/json"
-    send_file(static_file("json/#{params[:file]}"))
-  end
+    get "/redirect_loop" do
+      redirect to "/redirect_loop"
+    end
 
-  get "/xml/:file" do
-    content_type "application/xml"
-    send_file(static_file("xml/#{params[:file]}"))
-  end
+    get "/redirect" do
+      n = params[:times].to_i
+      n.zero? ? "You arrived!" : (redirect to "/redirect?times=#{n - 1}")
+    end
+
+    get "/malformed_redirect" do
+      redirect to "hptt://bro.ken"
+    end
+
+    get "/json/:file" do
+      content_type "application/json"
+      send_file(static_file("json/#{params[:file]}"))
+    end
+
+    get "/xml/:file" do
+      content_type "application/xml"
+      send_file(static_file("xml/#{params[:file]}"))
+    end
 
-private
+  private
 
-  def static_file(file_path)
-    File.expand_path(file_path, settings.public_folder)
+    def static_file(file_path)
+      File.expand_path(file_path, settings.public_folder)
+    end
   end
 end

data/spec/{base_spec.rb → wayfarer/base_spec.rb}

@@ -2,17 +2,18 @@
 
 require "spec_helpers"
 
-describe Wayfarer::Base, redis: true do
+describe Wayfarer::Base, :integration, :redis do
   let(:task) { build(:task, :redis_pool) }
 
   before do
-    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+    mock_job! :dummy_job
   end
 
   describe "::crawl" do
     it "enqueues a task" do
-      expect(DummyJob).to receive(:perform_later).with(task)
+      allow(DummyJob).to receive(:perform_later)
       DummyJob.crawl(task.url, batch: task.batch)
+      expect(DummyJob).to have_received(:perform_later).with(task)
     end
 
     it "returns a task" do
@@ -21,10 +22,13 @@ describe Wayfarer::Base, redis: true do
   end
 
   describe "#perform" do
-    subject(:counter) { Wayfarer::Redis::Counter.new(task) }
+    let(:counter) { build(:counter, task: task) }
     let!(:initial) { counter.increment }
 
     before do
+      allow(Wayfarer::Redis::Counter).to receive(:new).and_return(counter)
+      allow(counter).to receive(:increment).and_call_original
+
       DummyJob.class_eval do
         route.to :index
       end
@@ -38,15 +42,14 @@ describe Wayfarer::Base, redis: true do
       end
 
       specify do
-        expect_any_instance_of(Wayfarer::Redis::Counter)
-          .to receive(:increment).at_least(:once).and_call_original
-
         expect {
           DummyJob.perform_later(task)
           perform_enqueued_jobs
           assert_performed_jobs 1
           expect(enqueued_jobs).to be_empty
-        }.not_to change { counter.value }.from(initial)
+        }.not_to change(counter, :value).from(initial)
+
+        expect(counter).to have_received(:increment).at_least(:once)
       end
     end
 
@@ -60,15 +63,14 @@ describe Wayfarer::Base, redis: true do
       end
 
       specify do
-        expect_any_instance_of(Wayfarer::Redis::Counter)
-          .to receive(:increment).at_least(:once).and_call_original
-
         expect {
           DummyJob.perform_later(task)
           expect { perform_enqueued_jobs }.to raise_error(RuntimeError)
           assert_performed_jobs 1
           expect(enqueued_jobs).to be_empty
-        }.not_to change { counter.value }.from(initial)
+        }.not_to change(counter, :value).from(initial)
+
+        expect(counter).to have_received(:increment).at_least(:once)
       end
     end
 
@@ -84,27 +86,25 @@ describe Wayfarer::Base, redis: true do
       end
 
       specify do
-        expect_any_instance_of(Wayfarer::Redis::Counter)
-          .to receive(:increment).at_least(:once).and_call_original
-
         expect {
           DummyJob.perform_later(task)
           2.times { perform_enqueued_jobs }
           expect { perform_enqueued_jobs }.to raise_error(RuntimeError)
           assert_performed_jobs 3
           expect(enqueued_jobs).to be_empty
-        }.not_to change { counter.value }.from(initial)
+        }.not_to change(counter, :value).from(initial)
+
+        expect(counter).to have_received(:increment).at_least(:once)
       end
     end
 
     describe "discarded after retries" do
       before do
-        DummyJob.class_eval do
-          ErrorA = Class.new(StandardError)
-          ErrorB = Class.new(StandardError)
+        stub_const("ErrorA", Class.new(StandardError))
+        stub_const("ErrorB", Class.new(StandardError))
 
+        DummyJob.class_eval do
           retry_on ErrorA, attempts: 5
-
           discard_on ErrorB
 
           def index
@@ -116,16 +116,15 @@ describe Wayfarer::Base, redis: true do
       end
 
       specify do
-        expect_any_instance_of(Wayfarer::Redis::Counter)
-          .to receive(:increment).at_least(:once).and_call_original
-
         expect {
           DummyJob.perform_later(task)
           4.times { perform_enqueued_jobs }
           perform_enqueued_jobs
           assert_performed_jobs 5
           expect(enqueued_jobs).to be_empty
-        }.not_to change { counter.value }.from(initial)
+        }.not_to change(counter, :value).from(initial)
+
+        expect(counter).to have_received(:increment).at_least(:once)
       end
     end
 
@@ -141,15 +140,14 @@ describe Wayfarer::Base, redis: true do
       end
 
       specify do
-        expect_any_instance_of(Wayfarer::Redis::Counter)
-          .to receive(:increment).at_least(:once).and_call_original
-
         expect {
           DummyJob.perform_later(task)
           perform_enqueued_jobs
           assert_performed_jobs 1
           expect(enqueued_jobs).to be_empty
-        }.not_to change { counter.value }.from(initial)
+        }.not_to change(counter, :value).from(initial)
+
+        expect(counter).to have_received(:increment).at_least(:once)
       end
     end
 
@@ -166,15 +164,14 @@ describe Wayfarer::Base, redis: true do
         end
 
         specify do
-          expect_any_instance_of(Wayfarer::Redis::Counter)
-            .to receive(:increment).at_least(:once).and_call_original
-
           expect {
             DummyJob.perform_later(task)
             expect { perform_enqueued_jobs }.to raise_error(RuntimeError)
             assert_performed_jobs 1
             expect(enqueued_jobs).to be_empty
-          }.not_to change { counter.value }.from(initial)
+          }.not_to change(counter, :value).from(initial)
+
+          expect(counter).to have_received(:increment).at_least(:once)
         end
       end
 
@@ -188,15 +185,14 @@ describe Wayfarer::Base, redis: true do
         end
 
         specify do
-          expect_any_instance_of(Wayfarer::Redis::Counter)
-            .to receive(:increment).at_least(:once).and_call_original
-
           expect {
             DummyJob.perform_later(task)
             expect { perform_enqueued_jobs }.to raise_error(RuntimeError)
             assert_performed_jobs 1
             expect(enqueued_jobs).to be_empty
-          }.not_to change { counter.value }.from(initial)
+          }.not_to change(counter, :value).from(initial)
+
+          expect(counter).to have_received(:increment).at_least(:once)
         end
       end
     end

data/spec/wayfarer/batch_completion_spec.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+# The RSpec/SpecFilePathFormat offense was not addressed as it requires renaming
+# the file, not changing its content.
+describe Wayfarer::BatchCompletion, :redis do
+  let(:task) { build(:task, :redis_pool) }
+  let(:job) { instance_spy(ActiveJob::Base, arguments: [task]) }
+
+  describe "::call" do
+    subject(:call_batch_completion) { described_class.call(name, nil, nil, nil, { job: job }) }
+
+    let(:name) { "foo" }
+
+    context "with Wayfarer job" do
+      before { job.extend(Wayfarer::Base) }
+
+      specify do
+        allow(described_class).to receive(:handle)
+
+        call_batch_completion
+
+        expect(described_class).to have_received(:handle).with(name, job, task)
+      end
+
+      it "does not reassign Redis pool" do
+        expect { call_batch_completion }.not_to change { task[:redis_pool] }.from(Wayfarer::Redis::Pool.instance)
+      end
+
+      context "without Redis pool" do
+        before do
+          task[:redis_pool] = nil
+        end
+
+        it "assigns Redis pool" do
+          expect { call_batch_completion }.to change { task[:redis_pool] }.from(nil).to(Wayfarer::Redis::Pool.instance)
+        end
+      end
+    end
+
+    context "with other job" do
+      specify do
+        allow(described_class).to receive(:handle)
+
+        call_batch_completion
+
+        expect(described_class).not_to have_received(:handle)
+      end
+    end
+  end
+
+  describe "::handle" do
+    subject(:handle) { described_class.handle(event, job, task) }
+
+    let(:job) do
+      instance_spy(ActiveJob::Base,
+                   arguments: [task],
+                   executions: executions,
+                   exception_executions: exception_executions)
+    end
+
+    let(:initial_exception_executions) { {} }
+    let(:exception_executions) { initial_exception_executions }
+    let(:executions) { 0 }
+    let(:counter) { build(:counter, task: task) }
+
+    before do
+      task[:job] = job
+      task[:initial_exception_executions] = initial_exception_executions
+    end
+
+    context "with an 'enqueue.active_job' event" do
+      let(:event) { "enqueue.active_job" }
+
+      specify do
+        expect { handle }.to change(counter, :value).by(1)
+      end
+
+      context "with retry" do
+        let(:executions) { 1 }
+
+        specify do
+          expect { handle }.not_to(change(counter, :value))
+        end
+      end
+    end
+
+    context "with a 'perform.active_job' event" do
+      let(:event) { "perform.active_job" }
+
+      context "when job succeeds" do
+        specify do
+          expect { handle }.to change(counter, :value).by(-1)
+        end
+
+        context "when batch completes" do
+          before do
+            counter.increment
+            allow(Wayfarer::GC).to receive(:run).and_call_original
+          end
+
+          specify do
+            handle
+
+            expect(job).to have_received(:run_callbacks)
+            expect(Wayfarer::GC).to have_received(:run).with(task)
+          end
+        end
+
+        context "when batch does not complete" do
+          before do
+            allow(Wayfarer::GC).to receive(:run)
+          end
+
+          specify do
+            handle
+
+            expect(job).not_to have_received(:run_callbacks)
+            expect(Wayfarer::GC).not_to have_received(:run)
+          end
+        end
+      end
+
+      context "when an exception occurrs" do
+        let(:exception_executions) { { "[RuntimeError]" => 1 } }
+
+        specify do
+          expect { handle }.not_to(change(counter, :value))
+        end
+      end
+    end
+
+    context "with a 'retry_stopped.active_job' event" do
+      let(:event) { "retry_stopped.active_job" }
+
+      specify do
+        expect { handle }.to change(counter, :value).by(-1)
+      end
+    end
+  end
+end