wayfarer 0.4.7 → 0.4.8

data/rake/tests.rake

@@ -8,21 +8,25 @@ RSpec::Core::RakeTask.new(:test)
 namespace :test do
   desc "Run only isolated tests"
   RSpec::Core::RakeTask.new :isolated do |task|
-    task.rspec_opts = ["--tag ~selenium --tag ~ferrum --tag ~cli"]
+    task.rspec_opts = %w[--tag ~selenium --tag ~ferrum --tag ~cli --tag ~redis]
+  end
+
+  RSpec::Core::RakeTask.new :integration do |task|
+    task.rspec_opts = %w[--tag redis --tag cli]
   end
 
   desc "Run only Selenium tests"
   RSpec::Core::RakeTask.new :selenium do |task|
-    task.rspec_opts = ["--tag selenium"]
+    task.rspec_opts = %w[--tag selenium]
   end
 
   desc "Run only Ferrum tests"
   RSpec::Core::RakeTask.new :ferrum do |task|
-    task.rspec_opts = ["--tag ferrum"]
+    task.rspec_opts = %w[--tag ferrum]
   end
 
   desc "Run only CLI tests"
   RSpec::Core::RakeTask.new :cli do |task|
-    task.rspec_opts = ["--tag cli"]
+    task.rspec_opts = %w[--tag cli]
   end
 end

data/requirements.txt

@@ -1 +1 @@
-mkdocs-material == 9.5.9
+mkdocs-material == 9.6.12

data/spec/factories/job.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :job, class: ActiveJob::Base do
+    executions { 1 }
+    exception_executions { Object.new }
+  end
+end

data/spec/factories/middleware.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
 FactoryBot.define do
-  factory :middleware, class: OpenStruct do
+  factory :middleware, class: Struct.new(:receiver) do
     receiver { -> {} }
 
     initialize_with do
-      new(receiver: receiver).tap do |middleware|
+      new(receiver).tap do |middleware|
         middleware.define_singleton_method(:call) do |task, &block|
           receiver.call(task, &block)
         end

data/spec/factories/path_finder.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :path_finder, class: Wayfarer::Routing::PathFinder do
+    url { "https://example.com" }
+
+    initialize_with do
+      new(build(:task, :uri, url: url))
+    end
+  end
+end

data/spec/factories/redis.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :counter, class: Wayfarer::Redis::Counter do
+    task { build(:task, :redis_pool) }
+
+    initialize_with do
+      new(task)
+    end
+  end
+
+  factory :barrier, class: Wayfarer::Redis::Barrier do
+    task { build(:task, :redis_pool) }
+
+    initialize_with do
+      new(task)
+    end
+  end
+end

data/spec/factories/task.rb

@@ -5,8 +5,24 @@ FactoryBot.define do
     url { test_app_path("/") }
     batch { "batch" }
 
+    transient do
+      job { nil }
+      controller { nil }
+      action { nil }
+    end
+
     initialize_with do
-      new(url, batch)
+      new(url, batch).tap do |task|
+        task[:job] = job
+        task[:controller] = controller
+        task[:action] = action
+      end
+    end
+
+    trait :staged_urls do
+      after(:build) do |task|
+        task[:staged_urls] = SortedSet.new
+      end
     end
 
     trait :redis_pool do
@@ -14,5 +30,27 @@ FactoryBot.define do
         task[:redis_pool] = Wayfarer::Redis::Pool.instance
       end
     end
+
+    trait :barrier do
+      redis_pool
+
+      after(:build) do |task|
+        task[:barrier] = Wayfarer::Redis::Barrier.new(task)
+      end
+    end
+
+    trait :uri do
+      after(:build) do |task|
+        task[:uri] = Addressable::URI.parse(task.url)
+      end
+    end
+
+    trait :normalized do
+      uri
+
+      after(:build) do |task|
+        task[:uri] = Wayfarer::URI::Normalization.canonical!(task[:uri])
+      end
+    end
   end
 end

data/spec/spec_helpers.rb

@@ -1,44 +1,52 @@
 # frozen_string_literal: true
 
+require "active_support/core_ext/object/deep_dup"
+require "capybara/cuprite"
 require "factory_bot"
+require "puma"
 require "pry"
+require "rake"
+require "rspec-parameterized"
 require "sinatra"
-require "capybara/cuprite"
+require "webrick"
 
 require_relative "../lib/wayfarer"
+
 require_relative "support/test_app"
+require_relative "support/redis_helpers"
+require_relative "support/active_job_helpers"
+require_relative "support/integration_helpers"
 
 module SpecHelpers
-  def test_app_path(path)
-    File.join("http://", test_app_host, path)
+  WAYFARER_TEST_APP_PORT = ENV.fetch("WAYFARER_PORT", "9876").to_s
+
+  def ci?
+    ENV.key?("CI")
+  end
+
+  def test_redis_host
+    ci? ? "redis" : "localhost"
   end
 
   def test_app_host
-    if ENV["CI"]
-      "test:9876"
-    else
-      "localhost:9876"
-    end
+    ci? ? "test" : "localhost"
   end
 
-  def redis_host
-    if ENV["CI"]
-      "redis://redis:6379"
-    else
-      "redis://localhost:6379"
-    end
+  def redis_url
+    "redis://#{test_redis_host}:6379/0"
   end
 
-  def write_file(path, contents)
-    FileUtils.mkdir_p(Pathname.new(path).dirname)
-    IO.write(path, contents)
+  def test_app_path(path = "")
+    File.join("http://", "#{test_app_host}:#{WAYFARER_TEST_APP_PORT}", path)
   end
-end
 
-module ActiveJobTestHelper
-  # perform_enqueued_jobs calls assert_equal
-  def assert_equal(expected, actual, _)
-    expect(actual).to eq(expected)
+  def test_app_hostname
+    test_app_host.split(":").first
+  end
+
+  def write_file(path, contents)
+    FileUtils.mkdir_p(Pathname.new(path).dirname)
+    File.write(path, contents)
   end
 end
 
@@ -47,57 +55,46 @@ RSpec.configure do |config|
   config.include(ActiveJob::TestHelper)
   config.include(SpecHelpers)
   config.include(ActiveJobTestHelper)
+  config.include(IntegrationHelpers)
 
-  config.before(:suite) do
+  # The shared context enables a global `let(:redis)` declaration
+  config.include_context "with Redis", :redis
+
+  config.before :suite do
     Wayfarer::Logging.logger = Logger.new($stdout)
 
     FactoryBot.find_definitions
-    FactoryBot::SyntaxRunner.include SpecHelpers
-
-    mutex = Mutex.new
-    cvar = ConditionVariable.new
-
-    Thread.new do
-      Rack::Handler::WEBrick.run(TestApp,
-                                 Host: "0.0.0.0",
-                                 Port: 9876,
-                                 Logger: WEBrick::Log.new("/dev/null"),
-                                 AccessLog: [],
-                                 StartCallback: proc { cvar.signal })
-    end
+    FactoryBot::SyntaxRunner.include(SpecHelpers)
 
-    mutex.lock
-    cvar.wait(mutex)
+    Support::TestApp.start!(SpecHelpers::WAYFARER_TEST_APP_PORT)
   end
 
   config.before do
-    Wayfarer.config = Wayfarer::DEFAULT_CONFIG.clone
-    Wayfarer.config[:redis][:url] = redis_host
+    Wayfarer.config = Wayfarer::DEFAULT_CONFIG.deep_dup
+    Wayfarer.config[:redis][:url] = redis_url
 
-    ActiveJob::Base.queue_adapter = :test
     ActiveJob::Base.logger = Logger.new(nil)
+    ActiveJob::Base.queue_adapter = :test
     ActiveJob::Base.queue_adapter.enqueued_jobs.clear
     ActiveJob::Base.queue_adapter.performed_jobs.clear
 
-    if ENV["CI"]
-      # For unknown reasons, Wayfarer::Networking::Ferrum#renew can take a very
-      # long time to instantiate a new ::Ferrum::Browser and requires the 60s
-      # timeout. TODO: Figure out what's going on
+    if ENV.key?("CI")
       Wayfarer.config[:ferrum][:options] = { url: "http://chrome:3000", timeout: 120 }
-      Wayfarer.config[:selenium][:driver] = :firefox
-      Wayfarer.config[:selenium][:options] = { url: "http://firefox:4444" }
-      Wayfarer.config[:selenium][:client_timeout] = 10
+
+      Wayfarer.config[:selenium][:driver] = :remote
+      Wayfarer.config[:selenium][:options] = {
+        url: "http://firefox:4444",
+        options: Selenium::WebDriver::Options.firefox
+      }
     end
 
     # TODO: Undo side-effect
     Capybara.register_driver(:cuprite) do |app|
-      Capybara::Cuprite::Driver.new(app, Wayfarer.config[:ferrum][:options])
+      Capybara::Cuprite::Driver.new(app, Wayfarer.config.dig(:ferrum, :options))
     end
     Wayfarer.config[:capybara][:driver] = :cuprite
-  end
 
-  config.before(:each, redis: true) do
-    ::Redis.new(url: redis_host).flushall
+    Wayfarer::Networking::Pool.finalizer = ->(_pool) {}
   end
 
   config.around(cli: true) do |example|
@@ -107,10 +104,6 @@ RSpec.configure do |config|
     example.run
   ensure
     Dir.chdir(origin)
-    # FileUtils.rm_r(tmp)
-  end
-
-  config.before(:each, cli: true) do
-    Object.send(:remove_const, :DummyJob) if defined?(DummyJob)
+    FileUtils.rm_r(tmp)
   end
 end

data/spec/support/active_job_helpers.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module ActiveJobTestHelper
+  # perform_enqueued_jobs expects assert_equal to be defined
+  def assert_equal(expected, actual, _)
+    expect(actual).to eq(expected)
+  end
+end

data/spec/support/integration_helpers.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module IntegrationHelpers
+  def mock_job!(symbol)
+    mock!(symbol) do
+      Class.new(ActiveJob::Base).include(Wayfarer::Base)
+    end
+  end
+
+  def mock_handler!(symbol)
+    mock!(symbol) do
+      Class.new.include(Wayfarer::Handler)
+    end
+  end
+
+private
+
+  def mock!(symbol, &block)
+    stub_const(symbol.to_s.camelize, block.call)
+  end
+end

data/spec/support/redis_helpers.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+RSpec.shared_context "with Redis", :redis do
+  let(:redis) { Redis.new(url: redis_url) }
+
+  before do
+    redis.flushall
+  end
+end

data/spec/support/test_app.rb

@@ -1,60 +1,81 @@
 # frozen_string_literal: true
 
-class TestApp < Sinatra::Base
-  set :root, File.dirname(__FILE__)
-  set :public_folder, -> { File.join(root, "static") }
+module Support
+  class TestApp < Sinatra::Base
+    def self.start!(port)
+      mutex = Mutex.new
+      cvar = ConditionVariable.new
 
-  get "/status_code/:code" do
-    status params[:code]
-  end
+      Thread.new do
+        Rackup::Handler::WEBrick.run(
+          TestApp,
+          Host: "0.0.0.0",
+          Port: port,
+          Logger: WEBrick::Log.new(File::NULL),
+          AccessLog: [],
+          StartCallback: proc { cvar.signal }
+        )
+      end
 
-  get "/headers/:header" do
-    request.env[params[:header]]
-  end
+      mutex.lock
+      cvar.wait(mutex)
+    end
 
-  get "/response_header/:key/:val" do
-    headers params[:key] => params[:val]
-  end
+    set :root, File.dirname(__FILE__)
+    set :public_folder, -> { File.join(root, "static") }
 
-  get "/response_header/Content-Type/*" do
-    headers "Content-Type" => params[:splat].first
-  end
+    get "/status_code/:code" do
+      status params[:code]
+    end
 
-  get "/hello_world" do
-    headers "hello" => "world"
-    "Hello world!"
-  end
+    get "/headers/:header" do
+      request.env[params[:header]]
+    end
 
-  get "/body/:content" do
-    params[:content]
-  end
+    get "/response_header/:key/:val" do
+      headers params[:key] => params[:val]
+    end
 
-  get "/redirect_loop" do
-    redirect to "/redirect_loop"
-  end
+    get "/response_header/Content-Type/*" do
+      content_type params[:splat].first
+    end
 
-  get "/redirect" do
-    n = params[:times].to_i
-    n.zero? ? "You arrived!" : (redirect to "/redirect?times=#{n - 1}")
-  end
+    get "/hello_world" do
+      headers "hello" => "world"
+      "Hello world!"
+    end
 
-  get "/malformed_redirect" do
-    redirect to "hptt://bro.ken"
-  end
+    get "/body/:content" do
+      params[:content]
+    end
 
-  get "/json/:file" do
-    content_type "application/json"
-    send_file(static_file("json/#{params[:file]}"))
-  end
+    get "/redirect_loop" do
+      redirect to "/redirect_loop"
+    end
 
-  get "/xml/:file" do
-    content_type "application/xml"
-    send_file(static_file("xml/#{params[:file]}"))
-  end
+    get "/redirect" do
+      n = params[:times].to_i
+      n.zero? ? "You arrived!" : (redirect to "/redirect?times=#{n - 1}")
+    end
+
+    get "/malformed_redirect" do
+      redirect to "hptt://bro.ken"
+    end
+
+    get "/json/:file" do
+      content_type "application/json"
+      send_file(static_file("json/#{params[:file]}"))
+    end
+
+    get "/xml/:file" do
+      content_type "application/xml"
+      send_file(static_file("xml/#{params[:file]}"))
+    end
 
-private
+  private
 
-  def static_file(file_path)
-    File.expand_path(file_path, settings.public_folder)
+    def static_file(file_path)
+      File.expand_path(file_path, settings.public_folder)
+    end
   end
 end

data/spec/{base_spec.rb → wayfarer/base_spec.rb}

@@ -2,17 +2,18 @@
 
 require "spec_helpers"
 
-describe Wayfarer::Base, redis: true do
+describe Wayfarer::Base, :integration, :redis do
   let(:task) { build(:task, :redis_pool) }
 
   before do
-    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+    mock_job! :dummy_job
   end
 
   describe "::crawl" do
     it "enqueues a task" do
-      expect(DummyJob).to receive(:perform_later).with(task)
+      allow(DummyJob).to receive(:perform_later)
       DummyJob.crawl(task.url, batch: task.batch)
+      expect(DummyJob).to have_received(:perform_later).with(task)
     end
 
     it "returns a task" do
@@ -21,10 +22,13 @@ describe Wayfarer::Base, redis: true do
   end
 
   describe "#perform" do
-    subject(:counter) { Wayfarer::Redis::Counter.new(task) }
+    let(:counter) { build(:counter, task: task) }
     let!(:initial) { counter.increment }
 
     before do
+      allow(Wayfarer::Redis::Counter).to receive(:new).and_return(counter)
+      allow(counter).to receive(:increment).and_call_original
+
       DummyJob.class_eval do
         route.to :index
       end
@@ -38,15 +42,14 @@ describe Wayfarer::Base, redis: true do
       end
 
       specify do
-        expect_any_instance_of(Wayfarer::Redis::Counter)
-          .to receive(:increment).at_least(:once).and_call_original
-
         expect {
           DummyJob.perform_later(task)
           perform_enqueued_jobs
           assert_performed_jobs 1
           expect(enqueued_jobs).to be_empty
-        }.not_to change { counter.value }.from(initial)
+        }.not_to change(counter, :value).from(initial)
+
+        expect(counter).to have_received(:increment).at_least(:once)
       end
     end
 
@@ -60,15 +63,14 @@ describe Wayfarer::Base, redis: true do
       end
 
       specify do
-        expect_any_instance_of(Wayfarer::Redis::Counter)
-          .to receive(:increment).at_least(:once).and_call_original
-
         expect {
           DummyJob.perform_later(task)
           expect { perform_enqueued_jobs }.to raise_error(RuntimeError)
           assert_performed_jobs 1
           expect(enqueued_jobs).to be_empty
-        }.not_to change { counter.value }.from(initial)
+        }.not_to change(counter, :value).from(initial)
+
+        expect(counter).to have_received(:increment).at_least(:once)
       end
     end
 
@@ -84,27 +86,25 @@ describe Wayfarer::Base, redis: true do
       end
 
       specify do
-        expect_any_instance_of(Wayfarer::Redis::Counter)
-          .to receive(:increment).at_least(:once).and_call_original
-
         expect {
           DummyJob.perform_later(task)
           2.times { perform_enqueued_jobs }
           expect { perform_enqueued_jobs }.to raise_error(RuntimeError)
           assert_performed_jobs 3
           expect(enqueued_jobs).to be_empty
-        }.not_to change { counter.value }.from(initial)
+        }.not_to change(counter, :value).from(initial)
+
+        expect(counter).to have_received(:increment).at_least(:once)
       end
     end
 
     describe "discarded after retries" do
       before do
-        DummyJob.class_eval do
-          ErrorA = Class.new(StandardError)
-          ErrorB = Class.new(StandardError)
+        stub_const("ErrorA", Class.new(StandardError))
+        stub_const("ErrorB", Class.new(StandardError))
 
+        DummyJob.class_eval do
           retry_on ErrorA, attempts: 5
-
           discard_on ErrorB
 
           def index
@@ -116,16 +116,15 @@ describe Wayfarer::Base, redis: true do
       end
 
       specify do
-        expect_any_instance_of(Wayfarer::Redis::Counter)
-          .to receive(:increment).at_least(:once).and_call_original
-
         expect {
           DummyJob.perform_later(task)
           4.times { perform_enqueued_jobs }
           perform_enqueued_jobs
           assert_performed_jobs 5
           expect(enqueued_jobs).to be_empty
-        }.not_to change { counter.value }.from(initial)
+        }.not_to change(counter, :value).from(initial)
+
+        expect(counter).to have_received(:increment).at_least(:once)
       end
     end
 
@@ -141,15 +140,14 @@ describe Wayfarer::Base, redis: true do
       end
 
       specify do
-        expect_any_instance_of(Wayfarer::Redis::Counter)
-          .to receive(:increment).at_least(:once).and_call_original
-
         expect {
           DummyJob.perform_later(task)
           perform_enqueued_jobs
           assert_performed_jobs 1
           expect(enqueued_jobs).to be_empty
-        }.not_to change { counter.value }.from(initial)
+        }.not_to change(counter, :value).from(initial)
+
+        expect(counter).to have_received(:increment).at_least(:once)
       end
     end
 
@@ -166,15 +164,14 @@ describe Wayfarer::Base, redis: true do
         end
 
         specify do
-          expect_any_instance_of(Wayfarer::Redis::Counter)
-            .to receive(:increment).at_least(:once).and_call_original
-
           expect {
             DummyJob.perform_later(task)
             expect { perform_enqueued_jobs }.to raise_error(RuntimeError)
             assert_performed_jobs 1
             expect(enqueued_jobs).to be_empty
-          }.not_to change { counter.value }.from(initial)
+          }.not_to change(counter, :value).from(initial)
+
+          expect(counter).to have_received(:increment).at_least(:once)
         end
       end
 
@@ -188,15 +185,14 @@ describe Wayfarer::Base, redis: true do
         end
 
         specify do
-          expect_any_instance_of(Wayfarer::Redis::Counter)
-            .to receive(:increment).at_least(:once).and_call_original
-
           expect {
             DummyJob.perform_later(task)
             expect { perform_enqueued_jobs }.to raise_error(RuntimeError)
             assert_performed_jobs 1
             expect(enqueued_jobs).to be_empty
-          }.not_to change { counter.value }.from(initial)
+          }.not_to change(counter, :value).from(initial)
+
+          expect(counter).to have_received(:increment).at_least(:once)
         end
       end
     end