wayfarer 0.4.7 → 0.4.8

data/spec/wayfarer/integration/custom_routing_spec.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Custom routing", :redis do
+  let(:task) { build(:task) }
+
+  before do
+    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+    DummyJob.class_eval do
+      class_attribute :action,
+                      :params
+
+      route.custom do |root, uri, _task|
+        if uri.host == "example.com"
+          root.path ":segment" do
+            path "users", path: "/:id", to: :foo
+          end
+        else
+          root.to :bar
+        end
+      end
+
+      after_action do
+        self.class.params = params
+      end
+
+      def foo
+        self.class.action = :foo
+      end
+
+      def bar
+        self.class.action = :bar
+      end
+    end
+  end
+
+  specify do
+    DummyJob.crawl("https://example.com/foobar/users/42")
+    perform_enqueued_jobs
+    expect(DummyJob.action).to be(:foo)
+    assert_performed_jobs 1
+    expect(enqueued_jobs).to be_empty
+
+    DummyJob.crawl(test_app_path("/"))
+    perform_enqueued_jobs
+    expect(DummyJob.action).to be(:bar)
+    assert_performed_jobs 2
+    expect(enqueued_jobs).to be_empty
+  end
+end

data/spec/{integration → wayfarer/integration}/gc_spec.rb

@@ -2,32 +2,28 @@
 
 require "spec_helpers"
 
-describe "Garbage collection", redis: true do
+describe "Garbage collection", :redis do
+  subject(:perform) do
+    DummyJob.perform_later(task)
+    perform_enqueued_jobs
+  end
+
   let(:task) { build(:task, :redis_pool) }
-  let(:counter) { Wayfarer::Redis::Counter.new(task) }
-  let(:barrier) { Wayfarer::Redis::Barrier.new(task) }
-  let(:redis) { ::Redis.new(url: redis_host) }
+  let(:counter) { build(:counter, task: task) }
+  let(:barrier) { build(:barrier, task: task) }
 
   before do
-    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+    mock_job! :dummy_job
 
     DummyJob.class_eval do
       route.to :index
 
       def index; end
     end
-  end
-
-  before do
     redis.set(counter.redis_key, 0)
     redis.hset(barrier.redis_key, "", "")
   end
 
-  subject(:perform) do
-    DummyJob.perform_later(task)
-    perform_enqueued_jobs
-  end
-
   it "resets counter" do
     expect { perform }.to change { redis.exists?(counter.redis_key) }.from(true).to(false)
 

data/spec/{integration → wayfarer/integration}/handler_spec.rb

@@ -2,11 +2,13 @@
 
 require "spec_helpers"
 
-describe "Handlers", redis: true do
+describe "Handlers", :redis do
   before do
-    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
-    stub_const("DummyHandler", Class.new.include(Wayfarer::Handler))
-    stub_const("OtherHandler", Class.new.include(Wayfarer::Handler))
+    mock_job! :dummy_job
+    mock_job! :other_job
+
+    mock_handler! :dummy_handler
+    mock_handler! :other_handler
   end
 
   describe "bypassing the router" do
@@ -45,11 +47,6 @@ describe "Handlers", redis: true do
   end
 
   describe "dispatching to another Wayfarer::Base" do
-    before do
-      stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
-      stub_const("OtherJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
-    end
-
     before do
       DummyJob.class_eval do
         extend SpecHelpers
@@ -58,9 +55,11 @@ describe "Handlers", redis: true do
       end
     end
 
+    let(:error) { Wayfarer::Middleware::Dispatch::InvalidTargetError }
+
     specify do
       DummyJob.crawl(test_app_path("/"))
-      expect { perform_enqueued_jobs }.to raise_error(ArgumentError, "invalid action: OtherJob")
+      expect { perform_enqueued_jobs }.to raise_error(error, "routed to invalid action: #{OtherJob}")
     end
   end
 end

data/spec/{integration → wayfarer/integration}/page_spec.rb

@@ -2,14 +2,14 @@
 
 require "spec_helpers"
 
-describe "Pages" do
-  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
-
+describe "Pages", :redis do
   before do
-    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
-    stub_const("DummyHandler", Class.new.include(Wayfarer::Handler))
+    mock_job! :dummy_job
+    mock_handler! :dummy_handler
   end
 
+  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
+
   shared_examples "executes" do
     specify do
       DummyJob.crawl(url)
@@ -46,6 +46,8 @@ describe "Pages" do
 
   describe "page content with handler" do
     before do
+      handler = DummyHandler
+
       DummyJob.class_eval do
         include RSpec::Matchers
         include SpecHelpers
@@ -68,7 +70,7 @@ describe "Pages" do
         extend SpecHelpers
         include SpecHelpers
 
-        route.host test_app_host, to: DummyHandler
+        route.host test_app_host, to: handler
       end
     end
 

data/spec/{integration → wayfarer/integration}/params_spec.rb

@@ -2,7 +2,7 @@
 
 require "spec_helpers"
 
-describe "URL parameters" do
+describe "Path parameters", :redis do
   let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
 
   before do
@@ -25,7 +25,7 @@ describe "URL parameters" do
         extend SpecHelpers
         include RSpec::Matchers
 
-        route.to :index, host: test_app_host do
+        route.to :index, host: test_app_host, scheme: :https do
           path "git-scm.com/book/:lang/:file"
         end
 
@@ -58,7 +58,7 @@ describe "URL parameters" do
         end
       end
     end
-  end
 
-  it_behaves_like "executes"
+    it_behaves_like "executes"
+  end
 end

data/spec/{integration → wayfarer/integration}/parsing_spec.rb

@@ -2,10 +2,10 @@
 
 require "spec_helpers"
 
-describe "Response body parsing", redis: true do
+describe "Response body parsing", :redis do
   before do
-    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
-    stub_const("DummyHandler", Class.new.include(Wayfarer::Handler))
+    mock_job! :dummy_job
+    mock_handler! :dummy_handler
   end
 
   shared_examples "executes" do
@@ -17,24 +17,6 @@ describe "Response body parsing", redis: true do
     end
   end
 
-  context "with XML" do
-    let(:url) { test_app_path("xml/dummy.xml") }
-
-    before do
-      DummyJob.class_eval do
-        include RSpec::Matchers
-
-        route.to :index
-
-        def index
-          expect(page.doc).to be_a(Nokogiri::XML::Document)
-        end
-      end
-
-      it_behaves_like "executes"
-    end
-  end
-
   context "with HTML" do
     let(:url) { test_app_path("finders.html") }
 
@@ -90,16 +72,8 @@ describe "Response body parsing", redis: true do
       end
 
       before do
-        Wayfarer::Parsing.registry["foo/bar"] = parser
-        Wayfarer::Parsing.registry["bar/qux"] = [parser_with_options, :ok]
-      end
-
-      after do
-        Wayfarer::Parsing.registry.delete("foo/bar")
-        Wayfarer::Parsing.registry.delete("bar/qux")
-      end
-
-      before do
+        Wayfarer.config[:parsing][:registry]["foo/bar"] = parser
+        Wayfarer.config[:parsing][:registry]["bar/qux"] = [parser_with_options, :ok]
         DummyJob.class_eval do
           route.to :index
 
@@ -115,10 +89,10 @@ describe "Response body parsing", redis: true do
         )
       end
 
-      specify do
+      specify :aggregate_failures do
         expect(perform("foo/bar")).to be(:ok)
         expect(perform("bar/qux")).to be(:ok)
-        expect(perform("image/jpeg")).to be(nil)
+        expect(perform("image/jpeg")).to be_nil
       end
     end
   end

data/spec/wayfarer/integration/retry_spec.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Retries", :redis do
+  # rubocop:disable RSpec/MultipleExpectations
+  let(:task) { build(:task, :normalized, :redis_pool) }
+  let(:counter) { build(:counter, task: task) }
+  let(:barrier) { build(:barrier, task: task) }
+
+  before do
+    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+
+    Wayfarer::Redis::Counter.new(task).increment # avoid garbage collection
+  end
+
+  context "when job fails" do
+    before do
+      DummyJob.class_eval do
+        class_attribute :attempts, default: 0
+
+        retry_on RuntimeError, wait: 0, attempts: 3
+
+        route.to :index
+
+        def index
+          self.class.attempts += 1
+
+          raise
+        end
+      end
+    end
+
+    specify do
+      DummyJob.perform_later(task)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(0).to(1)
+
+      assert_performed_jobs 1
+      expect(enqueued_jobs.pluck("executions")).to contain_exactly(1)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(1).to(2)
+      assert_performed_jobs 2
+      expect(enqueued_jobs.pluck("executions")).to contain_exactly(2)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(2).to(3).and raise_error(RuntimeError)
+      assert_performed_jobs 3
+      expect(enqueued_jobs).to be_empty
+
+      expect(redis.hget(barrier.redis_key, task[:uri].to_s)).to eq(Wayfarer::Redis::Barrier::VALUE)
+    end
+  end
+
+  context "when job fails and eventually succeeds" do
+    before do
+      DummyJob.class_eval do
+        class_attribute :attempts, default: 0
+
+        retry_on RuntimeError, wait: 0, attempts: 3
+
+        route.to :index
+
+        def index
+          raise if [1, 2].include?(self.class.attempts += 1)
+        end
+      end
+    end
+
+    specify do
+      DummyJob.perform_later(task)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(0).to(1)
+      assert_performed_jobs 1
+      expect(enqueued_jobs.pluck("executions")).to contain_exactly(1)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(1).to(2)
+      assert_performed_jobs 2
+      expect(enqueued_jobs.pluck("executions")).to contain_exactly(2)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(2).to(3)
+      assert_performed_jobs 3
+      expect(enqueued_jobs).to be_empty
+
+      expect(redis.hget(barrier.redis_key, task[:uri].to_s)).to eq(Wayfarer::Redis::Barrier::VALUE)
+    end
+  end
+
+  context "when job succeeds" do
+    before do
+      DummyJob.class_eval do
+        class_attribute :attempts, default: 0
+
+        route.to :index
+
+        def index
+          self.class.attempts += 1
+        end
+      end
+    end
+
+    specify do
+      DummyJob.perform_later(task)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(0).to(1)
+      assert_performed_jobs 1
+      expect(enqueued_jobs).to be_empty
+
+      expect(redis.hget(barrier.redis_key, task[:uri].to_s)).to eq(Wayfarer::Redis::Barrier::VALUE)
+    end
+  end
+  # rubocop:enable RSpec/MultipleExpectations
+end

data/spec/{integration → wayfarer/integration}/stage_spec.rb

@@ -2,7 +2,7 @@
 
 require "spec_helpers"
 
-describe "Staging", redis: true do
+describe "Staging", :redis do
   let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
 
   before do
@@ -15,7 +15,7 @@ describe "Staging", redis: true do
       DummyJob.class_eval do
         extend SpecHelpers
 
-        route.host test_app_host, to: :index
+        route.host test_app_hostname, to: :index
 
         def index
           stage page.meta.links.all
@@ -27,7 +27,7 @@ describe "Staging", redis: true do
       expect {
         DummyJob.crawl(url)
         perform_enqueued_jobs
-      }.to change { enqueued_jobs.size }.by(157)
+      }.to change(enqueued_jobs, :size).by(157)
     end
   end
 
@@ -36,7 +36,7 @@ describe "Staging", redis: true do
       DummyJob.class_eval do
         extend SpecHelpers
 
-        route.host test_app_host, to: DummyHandler
+        route.host test_app_hostname, to: DummyHandler
       end
 
       DummyHandler.class_eval do
@@ -52,7 +52,7 @@ describe "Staging", redis: true do
       expect {
         DummyJob.crawl(url)
         perform_enqueued_jobs
-      }.to change { enqueued_jobs.size }.by(157)
+      }.to change(enqueued_jobs, :size).by(157)
     end
   end
 end

data/spec/{middleware → wayfarer/middleware}/batch_completion_spec.rb

@@ -2,13 +2,12 @@
 
 require "spec_helpers"
 
-describe Wayfarer::Middleware::BatchCompletion, "#call" do
-  let(:task) { build(:task) }
-  let(:job) { double(exception_executions: exception_executions) }
-  let(:exception_executions) { {} }
+describe Wayfarer::Middleware::BatchCompletion do
   subject(:batch_completion) { described_class.new }
 
-  before { task[:job] = job }
+  let(:task) { build(:task, job: job) }
+  let(:job) { build(:job, exception_executions: exception_executions) }
+  let(:exception_executions) { {} }
 
   it "assigns cloned exception_executions" do
     expect {

data/spec/{middleware → wayfarer/middleware}/chain_spec.rb

@@ -3,9 +3,10 @@
 require "spec_helpers"
 
 describe Wayfarer::Middleware::Chain do
+  subject(:chain) { described_class.new(middlewares) }
+
   let(:task) { build(:task) }
   let(:middlewares) { [] }
-  subject(:chain) { described_class.new(middlewares) }
 
   describe "::empty" do
     it "returns an empty chain" do
@@ -13,7 +14,7 @@ describe Wayfarer::Middleware::Chain do
     end
   end
 
-  describe "#call" do
+  describe "#push" do
     it "adds middleware" do
       expect {
         chain.push(Class.new)
@@ -35,20 +36,23 @@ describe Wayfarer::Middleware::Chain do
     context "when middleware yields" do
       let(:middlewares) { [spy, spy] }
 
-      it "yields" do
+      it "yields", :aggregate_failures do
         allow(middlewares.first).to receive(:call).and_yield
-        expect(middlewares.last).to receive(:call).with(task).and_yield
+        allow(middlewares.last).to receive(:call).with(task).and_yield
+
         expect { |spy| chain.call(task, &spy) }.to yield_control
+        expect(middlewares.last).to have_received(:call).with(task)
       end
     end
 
     context "when middleware does not yield" do
       let(:middlewares) { [spy, spy] }
 
-      it "does not yield" do
+      it "does not yield", :aggregate_failures do
         allow(middlewares.first).to receive(:call)
-        expect(middlewares.last).not_to receive(:call)
+
         expect { |spy| chain.call(task, &spy) }.not_to yield_control
+        expect(middlewares.last).not_to have_received(:call)
       end
     end
 
@@ -58,7 +62,8 @@ describe Wayfarer::Middleware::Chain do
       it "does not alter its middleware" do
         allow(middlewares.first).to receive(:call).and_yield
         allow(middlewares.second).to receive(:call).and_yield
-        expect { chain.call(task) }.not_to(change { chain.middlewares })
+
+        expect { chain.call(task) }.not_to(change(chain, :middlewares))
       end
     end
 
@@ -81,22 +86,22 @@ describe Wayfarer::Middleware::Chain do
     describe "Metadata" do
       let(:first) do
         build(:middleware, receiver: lambda do |task, &block|
-                                       task[:foobar] = 42
-                                       block.call
-                                       task[:barqux] = 1337
-                                     end)
+          task[:foobar] = 42
+          block.call
+          task[:barqux] = 1337
+        end)
       end
 
       let(:last) do
         build(:middleware, receiver: lambda do |task|
-                                       raise unless task[:foobar] == 42
-                                       raise if task[:barqux]
-                                     end)
+          raise unless task[:foobar] == 42
+          raise if task[:barqux]
+        end)
       end
 
       let(:middlewares) { [first, last] }
 
-      it "is accessible across middleware" do
+      it "is accessible across middleware", :aggregate_failures do
         expect {
           chain.call(task)
         }.not_to raise_error

data/spec/{middleware → wayfarer/middleware}/content_type_spec.rb

@@ -4,66 +4,63 @@ require "spec_helpers"
 
 describe Wayfarer::Middleware::ContentType do
   let(:content_type) { "text/html" }
-  let(:task) { build(:task) }
+  let(:task) { build(:task, :normalized) }
   let(:page) { build(:page, headers: { "Content-Type" => content_type }) }
 
   describe "#call" do
-    subject { Class.new(described_class).include(described_class::API).new }
+    subject(:middleware) { Class.new(described_class).include(described_class::API).new }
 
     before do
       task[:page] = page
-      task[:controller] = subject
+      task[:controller] = middleware
+      middleware.class.content_type "text/html"
     end
 
-    before { subject.class.content_type "text/html" }
-
     context "with permitted Content-Type" do
-      it "yields" do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
       end
     end
 
     context "when permitted Content-Type has parameters" do
-      let(:page) { build(:page, headers: { "Content-Type" => "#{content_type}; charset=UTF-" }) }
+      let(:page) { build(:page, headers: { "Content-Type" => "#{content_type}; charset=UTF-8" }) }
 
-      it "yields" do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
       end
     end
 
     context "with forbidden Content-Type" do
       let(:content_type) { "application/json" }
 
-      it "does not yield" do
-        expect { |spy| subject.call(task, &spy) }.not_to yield_control
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.not_to yield_control
       end
     end
 
     context "with permitted Regexp Content-Type" do
       before do
-        subject.class.content_type(/text/)
+        middleware.class.content_type(/text/)
       end
 
-      it "yields" do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
       end
     end
   end
 
   describe described_class::API do
-    subject(:controller) do
-      Struct.new(:task).include(described_class).new(task)
-    end
+    subject(:controller) { Class.new.include(described_class).new }
 
     describe "::allowed_content_types" do
       describe "index" do
-        subject { controller.class.allowed_content_types[:index] }
+        subject(:index) { controller.class.allowed_content_types[:index] }
 
         it { is_expected.to be_empty }
       end
 
       describe "patterns" do
-        subject { controller.class.allowed_content_types[:patterns] }
+        subject(:patterns) { controller.class.allowed_content_types[:patterns] }
 
         it { is_expected.to be_empty }
       end
@@ -73,7 +70,7 @@ describe Wayfarer::Middleware::ContentType do
       it "allows Content-Types" do
         controller.class.content_type(content_type)
 
-        expect(controller.class.allowed_content_types[:index][content_type]).to be(true)
+        expect(controller.class.allowed_content_types.dig(:index, content_type)).to be(true)
       end
 
       it "allows Content-Type patterns" do