wayfarer 0.4.7 → 0.4.9

data/spec/{integration → wayfarer/integration}/page_spec.rb

@@ -2,14 +2,14 @@
 
 require "spec_helpers"
 
-describe "Pages" do
-  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
-
+describe "Pages", :redis do
   before do
-    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
-    stub_const("DummyHandler", Class.new.include(Wayfarer::Handler))
+    mock_job! :dummy_job
+    mock_handler! :dummy_handler
   end
 
+  let(:url) { test_app_path("finders.html") }
+
   shared_examples "executes" do
     specify do
       DummyJob.crawl(url)
@@ -29,14 +29,14 @@ describe "Pages" do
         route.host test_app_host, to: :index
 
         def index
-          expect(page.url).to eq(test_app_path("git-scm.com/book/en/v2.html"))
+          expect(page.url).to eq(test_app_path("finders.html"))
           expect(page.status_code).to be(200)
           expect(page.body).not_to be_empty
           expect(page.headers.count).to be(9)
 
-          expect(page.meta.links.all.count).to be(157)
-          expect(page.meta.links.internal.count).to be(102)
-          expect(page.meta.links.external.count).to be(55)
+          expect(page.meta.links.all.count).to be(6)
+          expect(page.meta.links.internal.count).to be(3)
+          expect(page.meta.links.external.count).to be(3)
         end
       end
     end
@@ -46,6 +46,8 @@ describe "Pages" do
 
   describe "page content with handler" do
     before do
+      handler = DummyHandler
+
       DummyJob.class_eval do
         include RSpec::Matchers
         include SpecHelpers
@@ -53,14 +55,14 @@ describe "Pages" do
         route.to :index
 
         def index
-          expect(page.url).to eq(test_app_path("git-scm.com/book/en/v2.html"))
+          expect(page.url).to eq(test_app_path("finders.html"))
           expect(page.status_code).to be(200)
           expect(page.body).not_to be_empty
           expect(page.headers.count).to be(9)
 
-          expect(page.meta.links.all.count).to be(157)
-          expect(page.meta.links.internal.count).to be(102)
-          expect(page.meta.links.external.count).to be(55)
+          expect(page.meta.links.all.count).to be(6)
+          expect(page.meta.links.internal.count).to be(3)
+          expect(page.meta.links.external.count).to be(3)
         end
       end
 
@@ -68,7 +70,7 @@ describe "Pages" do
         extend SpecHelpers
         include SpecHelpers
 
-        route.host test_app_host, to: DummyHandler
+        route.host test_app_host, to: handler
       end
     end
 

data/spec/{integration → wayfarer/integration}/params_spec.rb

@@ -2,7 +2,7 @@
 
 require "spec_helpers"
 
-describe "URL parameters" do
+describe "Path parameters", :redis do
   let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
 
   before do
@@ -25,7 +25,7 @@ describe "URL parameters" do
         extend SpecHelpers
         include RSpec::Matchers
 
-        route.to :index, host: test_app_host do
+        route.to :index, host: test_app_host, scheme: :https do
           path "git-scm.com/book/:lang/:file"
         end
 
@@ -58,7 +58,7 @@ describe "URL parameters" do
         end
       end
     end
-  end
 
-  it_behaves_like "executes"
+    it_behaves_like "executes"
+  end
 end

data/spec/{integration → wayfarer/integration}/parsing_spec.rb

@@ -2,10 +2,10 @@
 
 require "spec_helpers"
 
-describe "Response body parsing", redis: true do
+describe "Response body parsing", :redis do
   before do
-    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
-    stub_const("DummyHandler", Class.new.include(Wayfarer::Handler))
+    mock_job! :dummy_job
+    mock_handler! :dummy_handler
   end
 
   shared_examples "executes" do
@@ -17,24 +17,6 @@ describe "Response body parsing", redis: true do
     end
   end
 
-  context "with XML" do
-    let(:url) { test_app_path("xml/dummy.xml") }
-
-    before do
-      DummyJob.class_eval do
-        include RSpec::Matchers
-
-        route.to :index
-
-        def index
-          expect(page.doc).to be_a(Nokogiri::XML::Document)
-        end
-      end
-
-      it_behaves_like "executes"
-    end
-  end
-
   context "with HTML" do
     let(:url) { test_app_path("finders.html") }
 
@@ -90,16 +72,9 @@ describe "Response body parsing", redis: true do
       end
 
       before do
-        Wayfarer::Parsing.registry["foo/bar"] = parser
-        Wayfarer::Parsing.registry["bar/qux"] = [parser_with_options, :ok]
-      end
-
-      after do
-        Wayfarer::Parsing.registry.delete("foo/bar")
-        Wayfarer::Parsing.registry.delete("bar/qux")
-      end
+        Wayfarer.config[:parsing][:content_types]["foo/bar"] = parser
+        Wayfarer.config[:parsing][:content_types]["bar/qux"] = [parser_with_options, :ok]
 
-      before do
         DummyJob.class_eval do
           route.to :index
 
@@ -115,10 +90,10 @@ describe "Response body parsing", redis: true do
         )
       end
 
-      specify do
+      specify :aggregate_failures do
         expect(perform("foo/bar")).to be(:ok)
         expect(perform("bar/qux")).to be(:ok)
-        expect(perform("image/jpeg")).to be(nil)
+        expect(perform("image/jpeg")).to be_nil
       end
     end
   end

data/spec/wayfarer/integration/retry_spec.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Retries", :redis do
+  # rubocop:disable RSpec/MultipleExpectations
+  let(:task) { build(:task, :normalized, :redis_pool) }
+  let(:counter) { build(:counter, task: task) }
+  let(:barrier) { build(:barrier, task: task) }
+
+  before do
+    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+
+    Wayfarer::Redis::Counter.new(task).increment # avoid garbage collection
+  end
+
+  context "when job fails" do
+    before do
+      DummyJob.class_eval do
+        class_attribute :attempts, default: 0
+
+        retry_on RuntimeError, wait: 0, attempts: 3
+
+        route.to :index
+
+        def index
+          self.class.attempts += 1
+
+          raise
+        end
+      end
+    end
+
+    specify do
+      DummyJob.perform_later(task)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(0).to(1)
+
+      assert_performed_jobs 1
+      expect(enqueued_jobs.pluck("executions")).to contain_exactly(1)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(1).to(2)
+      assert_performed_jobs 2
+      expect(enqueued_jobs.pluck("executions")).to contain_exactly(2)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(2).to(3).and raise_error(RuntimeError)
+      assert_performed_jobs 3
+      expect(enqueued_jobs).to be_empty
+
+      expect(redis.hget(barrier.redis_key, task[:uri].to_s)).to eq(Wayfarer::Redis::Barrier::VALUE)
+    end
+  end
+
+  context "when job fails and eventually succeeds" do
+    before do
+      DummyJob.class_eval do
+        class_attribute :attempts, default: 0
+
+        retry_on RuntimeError, wait: 0, attempts: 3
+
+        route.to :index
+
+        def index
+          raise if [1, 2].include?(self.class.attempts += 1)
+        end
+      end
+    end
+
+    specify do
+      DummyJob.perform_later(task)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(0).to(1)
+      assert_performed_jobs 1
+      expect(enqueued_jobs.pluck("executions")).to contain_exactly(1)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(1).to(2)
+      assert_performed_jobs 2
+      expect(enqueued_jobs.pluck("executions")).to contain_exactly(2)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(2).to(3)
+      assert_performed_jobs 3
+      expect(enqueued_jobs).to be_empty
+
+      expect(redis.hget(barrier.redis_key, task[:uri].to_s)).to eq(Wayfarer::Redis::Barrier::VALUE)
+    end
+  end
+
+  context "when job succeeds" do
+    before do
+      DummyJob.class_eval do
+        class_attribute :attempts, default: 0
+
+        route.to :index
+
+        def index
+          self.class.attempts += 1
+        end
+      end
+    end
+
+    specify do
+      DummyJob.perform_later(task)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(0).to(1)
+      assert_performed_jobs 1
+      expect(enqueued_jobs).to be_empty
+
+      expect(redis.hget(barrier.redis_key, task[:uri].to_s)).to eq(Wayfarer::Redis::Barrier::VALUE)
+    end
+  end
+  # rubocop:enable RSpec/MultipleExpectations
+end

data/spec/{integration → wayfarer/integration}/stage_spec.rb

@@ -2,8 +2,8 @@
 
 require "spec_helpers"
 
-describe "Staging", redis: true do
-  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
+describe "Staging", :redis do
+  let(:url) { test_app_path("finders.html") }
 
   before do
     stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
@@ -15,7 +15,7 @@ describe "Staging", redis: true do
       DummyJob.class_eval do
         extend SpecHelpers
 
-        route.host test_app_host, to: :index
+        route.host test_app_hostname, to: :index
 
         def index
           stage page.meta.links.all
@@ -27,7 +27,7 @@ describe "Staging", redis: true do
       expect {
         DummyJob.crawl(url)
         perform_enqueued_jobs
-      }.to change { enqueued_jobs.size }.by(157)
+      }.to change(enqueued_jobs, :size).by(6)
     end
   end
 
@@ -36,7 +36,7 @@ describe "Staging", redis: true do
       DummyJob.class_eval do
         extend SpecHelpers
 
-        route.host test_app_host, to: DummyHandler
+        route.host test_app_hostname, to: DummyHandler
       end
 
       DummyHandler.class_eval do
@@ -52,7 +52,7 @@ describe "Staging", redis: true do
       expect {
         DummyJob.crawl(url)
         perform_enqueued_jobs
-      }.to change { enqueued_jobs.size }.by(157)
+      }.to change(enqueued_jobs, :size).by(6)
     end
   end
 end

data/spec/{middleware → wayfarer/middleware}/batch_completion_spec.rb

@@ -2,13 +2,12 @@
 
 require "spec_helpers"
 
-describe Wayfarer::Middleware::BatchCompletion, "#call" do
-  let(:task) { build(:task) }
-  let(:job) { double(exception_executions: exception_executions) }
-  let(:exception_executions) { {} }
+describe Wayfarer::Middleware::BatchCompletion do
   subject(:batch_completion) { described_class.new }
 
-  before { task[:job] = job }
+  let(:task) { build(:task, job: job) }
+  let(:job) { build(:job, exception_executions: exception_executions) }
+  let(:exception_executions) { {} }
 
   it "assigns cloned exception_executions" do
     expect {

data/spec/{middleware → wayfarer/middleware}/chain_spec.rb

@@ -3,9 +3,10 @@
 require "spec_helpers"
 
 describe Wayfarer::Middleware::Chain do
+  subject(:chain) { described_class.new(middlewares) }
+
   let(:task) { build(:task) }
   let(:middlewares) { [] }
-  subject(:chain) { described_class.new(middlewares) }
 
   describe "::empty" do
     it "returns an empty chain" do
@@ -13,7 +14,7 @@ describe Wayfarer::Middleware::Chain do
     end
   end
 
-  describe "#call" do
+  describe "#push" do
     it "adds middleware" do
       expect {
         chain.push(Class.new)
@@ -35,20 +36,23 @@ describe Wayfarer::Middleware::Chain do
     context "when middleware yields" do
       let(:middlewares) { [spy, spy] }
 
-      it "yields" do
+      it "yields", :aggregate_failures do
         allow(middlewares.first).to receive(:call).and_yield
-        expect(middlewares.last).to receive(:call).with(task).and_yield
+        allow(middlewares.last).to receive(:call).with(task).and_yield
+
         expect { |spy| chain.call(task, &spy) }.to yield_control
+        expect(middlewares.last).to have_received(:call).with(task)
       end
     end
 
     context "when middleware does not yield" do
       let(:middlewares) { [spy, spy] }
 
-      it "does not yield" do
+      it "does not yield", :aggregate_failures do
         allow(middlewares.first).to receive(:call)
-        expect(middlewares.last).not_to receive(:call)
+
         expect { |spy| chain.call(task, &spy) }.not_to yield_control
+        expect(middlewares.last).not_to have_received(:call)
       end
     end
 
@@ -58,7 +62,8 @@ describe Wayfarer::Middleware::Chain do
       it "does not alter its middleware" do
         allow(middlewares.first).to receive(:call).and_yield
         allow(middlewares.second).to receive(:call).and_yield
-        expect { chain.call(task) }.not_to(change { chain.middlewares })
+
+        expect { chain.call(task) }.not_to(change(chain, :middlewares))
       end
     end
 
@@ -81,22 +86,22 @@ describe Wayfarer::Middleware::Chain do
     describe "Metadata" do
       let(:first) do
         build(:middleware, receiver: lambda do |task, &block|
-                                       task[:foobar] = 42
-                                       block.call
-                                       task[:barqux] = 1337
-                                     end)
+          task[:foobar] = 42
+          block.call
+          task[:barqux] = 1337
+        end)
       end
 
       let(:last) do
         build(:middleware, receiver: lambda do |task|
-                                       raise unless task[:foobar] == 42
-                                       raise if task[:barqux]
-                                     end)
+          raise unless task[:foobar] == 42
+          raise if task[:barqux]
+        end)
       end
 
       let(:middlewares) { [first, last] }
 
-      it "is accessible across middleware" do
+      it "is accessible across middleware", :aggregate_failures do
         expect {
           chain.call(task)
         }.not_to raise_error

data/spec/{middleware → wayfarer/middleware}/content_type_spec.rb

@@ -4,66 +4,63 @@ require "spec_helpers"
 
 describe Wayfarer::Middleware::ContentType do
   let(:content_type) { "text/html" }
-  let(:task) { build(:task) }
+  let(:task) { build(:task, :normalized) }
   let(:page) { build(:page, headers: { "Content-Type" => content_type }) }
 
   describe "#call" do
-    subject { Class.new(described_class).include(described_class::API).new }
+    subject(:middleware) { Class.new(described_class).include(described_class::API).new }
 
     before do
       task[:page] = page
-      task[:controller] = subject
+      task[:controller] = middleware
+      middleware.class.content_type "text/html"
     end
 
-    before { subject.class.content_type "text/html" }
-
     context "with permitted Content-Type" do
-      it "yields" do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
       end
     end
 
     context "when permitted Content-Type has parameters" do
-      let(:page) { build(:page, headers: { "Content-Type" => "#{content_type}; charset=UTF-" }) }
+      let(:page) { build(:page, headers: { "Content-Type" => "#{content_type}; charset=UTF-8" }) }
 
-      it "yields" do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
       end
     end
 
     context "with forbidden Content-Type" do
       let(:content_type) { "application/json" }
 
-      it "does not yield" do
-        expect { |spy| subject.call(task, &spy) }.not_to yield_control
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.not_to yield_control
       end
     end
 
     context "with permitted Regexp Content-Type" do
       before do
-        subject.class.content_type(/text/)
+        middleware.class.content_type(/text/)
       end
 
-      it "yields" do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
       end
     end
   end
 
   describe described_class::API do
-    subject(:controller) do
-      Struct.new(:task).include(described_class).new(task)
-    end
+    subject(:controller) { Class.new.include(described_class).new }
 
     describe "::allowed_content_types" do
       describe "index" do
-        subject { controller.class.allowed_content_types[:index] }
+        subject(:index) { controller.class.allowed_content_types[:index] }
 
         it { is_expected.to be_empty }
       end
 
       describe "patterns" do
-        subject { controller.class.allowed_content_types[:patterns] }
+        subject(:patterns) { controller.class.allowed_content_types[:patterns] }
 
         it { is_expected.to be_empty }
       end
@@ -73,7 +70,7 @@ describe Wayfarer::Middleware::ContentType do
       it "allows Content-Types" do
         controller.class.content_type(content_type)
 
-        expect(controller.class.allowed_content_types[:index][content_type]).to be(true)
+        expect(controller.class.allowed_content_types.dig(:index, content_type)).to be(true)
       end
 
       it "allows Content-Type patterns" do

data/spec/{middleware → wayfarer/middleware}/controller_spec.rb

@@ -3,32 +3,29 @@
 require "spec_helpers"
 
 describe Wayfarer::Middleware::Controller do
+  subject(:controller) { Class.new.include(described_class) }
+
   let(:task) { build(:task) }
 
   let(:middleware) do
-    Class.new do
-      extend Wayfarer::Middleware::Base
-
-      module API; end
-
-      def call(task); end
+    Class.new.tap do |klass|
+      klass.extend(Wayfarer::Middleware::Base)
+      klass.define_method(:call, &->(_) {})
+      klass.const_set("API", Module.new)
     end
   end
 
-  subject(:controller) { Class.new.include(described_class) }
-
   describe "::use" do
-    it "adds middleware" do
+    specify do
       expect {
         controller.use(middleware)
       }.to change { controller.chain.middlewares.count }.by(1)
     end
 
-    it "adds lazy middleware" do
+    specify :aggregate_failures do
       controller.use(middleware)
-      expect(controller.chain.middlewares.last).to be_a(Wayfarer::Middleware::Lazy)
-      expect(controller.chain.middlewares.last.klass).to be(middleware)
-      expect_any_instance_of(middleware).to receive(:call).with(task)
+      expect(controller.chain.middlewares.last).to be_a(middleware)
+      expect_any_instance_of(middleware).to receive(:call).with(task) # rubocop:disable RSpec/AnyInstance
       controller.new.call(task)
     end
   end
@@ -42,19 +39,19 @@ describe Wayfarer::Middleware::Controller do
   end
 
   describe "#call" do
-    subject { controller.new }
+    let(:instance) { controller.new }
 
     it "assigns the task" do
       expect {
-        subject.call(task)
-      }.to change { subject.task }.to(task)
+        instance.call(task)
+      }.to change(instance, :task).to(task)
     end
 
     context "without job assigned" do
       it "assigns itself" do
         expect {
-          subject.call(task)
-        }.to change { task[:job] }.to(subject)
+          instance.call(task)
+        }.to change { task[:job] }.to(instance)
       end
     end
 
@@ -63,24 +60,28 @@ describe Wayfarer::Middleware::Controller do
 
       it "does not override the job" do
         expect {
-          subject.call(task)
+          instance.call(task)
         }.not_to(change { task[:job] })
       end
     end
 
     it "assigns itself as controller" do
       expect {
-        subject.call(task)
-      }.to change { task[:controller] }.to(subject)
+        instance.call(task)
+      }.to change { task[:controller] }.to(instance)
     end
 
     it "calls the chain" do
-      expect(subject.class.chain).to receive(:call)
-      subject.call(task)
+      chain = instance.class.chain
+      allow(chain).to receive(:call)
+
+      instance.call(task)
+
+      expect(chain).to have_received(:call)
     end
 
     it "yields" do
-      expect { |spy| subject.call(task, &spy) }.to yield_control
+      expect { |spy| instance.call(task, &spy) }.to yield_control
     end
   end
 end