wayfarer 0.4.6 → 0.4.8

data/spec/wayfarer/integration/handler_spec.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Handlers", :redis do
+  before do
+    mock_job! :dummy_job
+    mock_job! :other_job
+
+    mock_handler! :dummy_handler
+    mock_handler! :other_handler
+  end
+
+  describe "bypassing the router" do
+    before do
+      DummyJob.class_eval do
+        route.to DummyHandler
+      end
+
+      DummyHandler.class_eval do
+        class_attribute :called
+
+        before_action do
+          self.class.called = true
+        end
+
+        route.to [OtherHandler, :foobar]
+      end
+
+      OtherHandler.class_eval do
+        class_attribute :called
+
+        def foobar
+          self.class.called = true
+        end
+      end
+    end
+
+    specify do
+      DummyJob.crawl(test_app_path("/"))
+      perform_enqueued_jobs
+      assert_performed_jobs 1
+      expect(enqueued_jobs).to be_empty
+      expect(DummyHandler.called).to be(true)
+      expect(OtherHandler.called).to be(true)
+    end
+  end
+
+  describe "dispatching to another Wayfarer::Base" do
+    before do
+      DummyJob.class_eval do
+        extend SpecHelpers
+
+        route.to OtherJob
+      end
+    end
+
+    let(:error) { Wayfarer::Middleware::Dispatch::InvalidTargetError }
+
+    specify do
+      DummyJob.crawl(test_app_path("/"))
+      expect { perform_enqueued_jobs }.to raise_error(error, "routed to invalid action: #{OtherJob}")
+    end
+  end
+end

data/spec/wayfarer/integration/page_spec.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Pages", :redis do
+  before do
+    mock_job! :dummy_job
+    mock_handler! :dummy_handler
+  end
+
+  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
+
+  shared_examples "executes" do
+    specify do
+      DummyJob.crawl(url)
+      perform_enqueued_jobs
+      assert_performed_jobs 1
+      expect(enqueued_jobs).to be_empty
+    end
+  end
+
+  describe "page content" do
+    before do
+      DummyJob.class_eval do
+        include RSpec::Matchers
+        extend SpecHelpers
+        include SpecHelpers
+
+        route.host test_app_host, to: :index
+
+        def index
+          expect(page.url).to eq(test_app_path("git-scm.com/book/en/v2.html"))
+          expect(page.status_code).to be(200)
+          expect(page.body).not_to be_empty
+          expect(page.headers.count).to be(9)
+
+          expect(page.meta.links.all.count).to be(157)
+          expect(page.meta.links.internal.count).to be(102)
+          expect(page.meta.links.external.count).to be(55)
+        end
+      end
+    end
+
+    it_behaves_like "executes"
+  end
+
+  describe "page content with handler" do
+    before do
+      handler = DummyHandler
+
+      DummyJob.class_eval do
+        include RSpec::Matchers
+        include SpecHelpers
+
+        route.to :index
+
+        def index
+          expect(page.url).to eq(test_app_path("git-scm.com/book/en/v2.html"))
+          expect(page.status_code).to be(200)
+          expect(page.body).not_to be_empty
+          expect(page.headers.count).to be(9)
+
+          expect(page.meta.links.all.count).to be(157)
+          expect(page.meta.links.internal.count).to be(102)
+          expect(page.meta.links.external.count).to be(55)
+        end
+      end
+
+      DummyHandler.class_eval do
+        extend SpecHelpers
+        include SpecHelpers
+
+        route.host test_app_host, to: handler
+      end
+    end
+
+    it_behaves_like "executes"
+  end
+end

data/spec/wayfarer/integration/params_spec.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Path parameters", :redis do
+  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
+
+  before do
+    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+    stub_const("DummyHandler", Class.new.include(Wayfarer::Handler))
+  end
+
+  shared_examples "executes" do
+    specify do
+      DummyJob.crawl(url)
+      perform_enqueued_jobs
+      assert_performed_jobs 1
+      expect(enqueued_jobs).to be_empty
+    end
+  end
+
+  describe "path pattern matching" do
+    before do
+      DummyJob.class_eval do
+        extend SpecHelpers
+        include RSpec::Matchers
+
+        route.to :index, host: test_app_host, scheme: :https do
+          path "git-scm.com/book/:lang/:file"
+        end
+
+        def index
+          expect(params).to eq("lang" => "en", "file" => "v2.html")
+        end
+      end
+    end
+
+    it_behaves_like "executes"
+  end
+
+  describe "path pattern matching with handler" do
+    before do
+      DummyJob.class_eval do
+        extend SpecHelpers
+
+        route.to DummyHandler, host: test_app_host do
+          path "git-scm.com/book/:lang/:file"
+        end
+      end
+
+      DummyHandler.class_eval do
+        include RSpec::Matchers
+
+        route.to :index
+
+        def index
+          expect(params).to eq("lang" => "en", "file" => "v2.html")
+        end
+      end
+    end
+
+    it_behaves_like "executes"
+  end
+end

data/spec/wayfarer/integration/parsing_spec.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Response body parsing", :redis do
+  before do
+    mock_job! :dummy_job
+    mock_handler! :dummy_handler
+  end
+
+  shared_examples "executes" do
+    specify do
+      DummyJob.crawl(url)
+      perform_enqueued_jobs
+      assert_performed_jobs 1
+      expect(enqueued_jobs).to be_empty
+    end
+  end
+
+  context "with HTML" do
+    let(:url) { test_app_path("finders.html") }
+
+    before do
+      DummyJob.class_eval do
+        include RSpec::Matchers
+
+        route.to :index
+
+        def index
+          expect(page.doc).to be_a(Nokogiri::HTML::Document)
+        end
+      end
+    end
+
+    it_behaves_like "executes"
+  end
+
+  context "with JSON" do
+    let(:url) { test_app_path("json/dummy.json") }
+
+    before do
+      DummyJob.class_eval do
+        include RSpec::Matchers
+
+        route.to :index
+
+        def index
+          expect(page.doc).to be_a(Hash)
+        end
+      end
+    end
+
+    it_behaves_like "executes"
+  end
+
+  describe "custom Content-Type parsers" do
+    context "with registered Content-Types" do
+      let(:parser) do
+        Class.new do
+          def self.parse(_body)
+            :ok
+          end
+        end
+      end
+
+      let(:parser_with_options) do
+        Class.new do
+          def self.parse(_body, options)
+            options
+          end
+        end
+      end
+
+      before do
+        Wayfarer.config[:parsing][:registry]["foo/bar"] = parser
+        Wayfarer.config[:parsing][:registry]["bar/qux"] = [parser_with_options, :ok]
+        DummyJob.class_eval do
+          route.to :index
+
+          def index
+            page.doc
+          end
+        end
+      end
+
+      def perform(content_type)
+        DummyJob.new.perform(
+          build(:task, url: test_app_path("response_header/Content-Type/#{content_type}"))
+        )
+      end
+
+      specify :aggregate_failures do
+        expect(perform("foo/bar")).to be(:ok)
+        expect(perform("bar/qux")).to be(:ok)
+        expect(perform("image/jpeg")).to be_nil
+      end
+    end
+  end
+end

data/spec/wayfarer/integration/retry_spec.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Retries", :redis do
+  # rubocop:disable RSpec/MultipleExpectations
+  let(:task) { build(:task, :normalized, :redis_pool) }
+  let(:counter) { build(:counter, task: task) }
+  let(:barrier) { build(:barrier, task: task) }
+
+  before do
+    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+
+    Wayfarer::Redis::Counter.new(task).increment # avoid garbage collection
+  end
+
+  context "when job fails" do
+    before do
+      DummyJob.class_eval do
+        class_attribute :attempts, default: 0
+
+        retry_on RuntimeError, wait: 0, attempts: 3
+
+        route.to :index
+
+        def index
+          self.class.attempts += 1
+
+          raise
+        end
+      end
+    end
+
+    specify do
+      DummyJob.perform_later(task)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(0).to(1)
+
+      assert_performed_jobs 1
+      expect(enqueued_jobs.pluck("executions")).to contain_exactly(1)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(1).to(2)
+      assert_performed_jobs 2
+      expect(enqueued_jobs.pluck("executions")).to contain_exactly(2)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(2).to(3).and raise_error(RuntimeError)
+      assert_performed_jobs 3
+      expect(enqueued_jobs).to be_empty
+
+      expect(redis.hget(barrier.redis_key, task[:uri].to_s)).to eq(Wayfarer::Redis::Barrier::VALUE)
+    end
+  end
+
+  context "when job fails and eventually succeeds" do
+    before do
+      DummyJob.class_eval do
+        class_attribute :attempts, default: 0
+
+        retry_on RuntimeError, wait: 0, attempts: 3
+
+        route.to :index
+
+        def index
+          raise if [1, 2].include?(self.class.attempts += 1)
+        end
+      end
+    end
+
+    specify do
+      DummyJob.perform_later(task)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(0).to(1)
+      assert_performed_jobs 1
+      expect(enqueued_jobs.pluck("executions")).to contain_exactly(1)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(1).to(2)
+      assert_performed_jobs 2
+      expect(enqueued_jobs.pluck("executions")).to contain_exactly(2)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(2).to(3)
+      assert_performed_jobs 3
+      expect(enqueued_jobs).to be_empty
+
+      expect(redis.hget(barrier.redis_key, task[:uri].to_s)).to eq(Wayfarer::Redis::Barrier::VALUE)
+    end
+  end
+
+  context "when job succeeds" do
+    before do
+      DummyJob.class_eval do
+        class_attribute :attempts, default: 0
+
+        route.to :index
+
+        def index
+          self.class.attempts += 1
+        end
+      end
+    end
+
+    specify do
+      DummyJob.perform_later(task)
+
+      expect { perform_enqueued_jobs }.to change(DummyJob, :attempts).from(0).to(1)
+      assert_performed_jobs 1
+      expect(enqueued_jobs).to be_empty
+
+      expect(redis.hget(barrier.redis_key, task[:uri].to_s)).to eq(Wayfarer::Redis::Barrier::VALUE)
+    end
+  end
+  # rubocop:enable RSpec/MultipleExpectations
+end

data/spec/wayfarer/integration/stage_spec.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Staging", :redis do
+  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
+
+  before do
+    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+    stub_const("DummyHandler", Class.new.include(Wayfarer::Handler))
+  end
+
+  describe "staging URLs" do
+    before do
+      DummyJob.class_eval do
+        extend SpecHelpers
+
+        route.host test_app_hostname, to: :index
+
+        def index
+          stage page.meta.links.all
+        end
+      end
+    end
+
+    specify do
+      expect {
+        DummyJob.crawl(url)
+        perform_enqueued_jobs
+      }.to change(enqueued_jobs, :size).by(157)
+    end
+  end
+
+  context "with handler" do
+    before do
+      DummyJob.class_eval do
+        extend SpecHelpers
+
+        route.host test_app_hostname, to: DummyHandler
+      end
+
+      DummyHandler.class_eval do
+        route.to :index
+
+        def index
+          stage page.meta.links.all
+        end
+      end
+    end
+
+    specify do
+      expect {
+        DummyJob.crawl(url)
+        perform_enqueued_jobs
+      }.to change(enqueued_jobs, :size).by(157)
+    end
+  end
+end

data/spec/wayfarer/middleware/batch_completion_spec.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::BatchCompletion do
+  subject(:batch_completion) { described_class.new }
+
+  let(:task) { build(:task, job: job) }
+  let(:job) { build(:job, exception_executions: exception_executions) }
+  let(:exception_executions) { {} }
+
+  it "assigns cloned exception_executions" do
+    expect {
+      batch_completion.call(task)
+    }.to change {
+      task[:initial_exception_executions]
+    }.from(nil).to(exception_executions)
+
+    expect(task[:initial_exception_executions]).not_to be(exception_executions)
+  end
+
+  context "when already assigned" do
+    before { task[:initial_exception_executions] = Object.new }
+
+    specify do
+      expect { |spy| batch_completion.call(task, &spy) }.to yield_control
+    end
+
+    it "doesn't assign exception_executions" do
+      expect { batch_completion.call(task) }.not_to(change { task[:initial_exception_executions] })
+    end
+  end
+end

data/spec/{middleware → wayfarer/middleware}/chain_spec.rb

@@ -3,9 +3,10 @@
 require "spec_helpers"
 
 describe Wayfarer::Middleware::Chain do
+  subject(:chain) { described_class.new(middlewares) }
+
   let(:task) { build(:task) }
   let(:middlewares) { [] }
-  subject(:chain) { described_class.new(middlewares) }
 
   describe "::empty" do
     it "returns an empty chain" do
@@ -13,7 +14,7 @@ describe Wayfarer::Middleware::Chain do
     end
   end
 
-  describe "#call" do
+  describe "#push" do
     it "adds middleware" do
       expect {
         chain.push(Class.new)
@@ -35,20 +36,23 @@ describe Wayfarer::Middleware::Chain do
     context "when middleware yields" do
       let(:middlewares) { [spy, spy] }
 
-      it "yields" do
+      it "yields", :aggregate_failures do
         allow(middlewares.first).to receive(:call).and_yield
-        expect(middlewares.last).to receive(:call).with(task).and_yield
+        allow(middlewares.last).to receive(:call).with(task).and_yield
+
         expect { |spy| chain.call(task, &spy) }.to yield_control
+        expect(middlewares.last).to have_received(:call).with(task)
       end
     end
 
     context "when middleware does not yield" do
       let(:middlewares) { [spy, spy] }
 
-      it "does not yield" do
+      it "does not yield", :aggregate_failures do
         allow(middlewares.first).to receive(:call)
-        expect(middlewares.last).not_to receive(:call)
+
         expect { |spy| chain.call(task, &spy) }.not_to yield_control
+        expect(middlewares.last).not_to have_received(:call)
       end
     end
 
@@ -58,18 +62,19 @@ describe Wayfarer::Middleware::Chain do
       it "does not alter its middleware" do
         allow(middlewares.first).to receive(:call).and_yield
         allow(middlewares.second).to receive(:call).and_yield
-        expect { chain.call(task) }.not_to(change { chain.middlewares })
+
+        expect { chain.call(task) }.not_to(change(chain, :middlewares))
       end
     end
 
     describe "Return value" do
       let(:middlewares) do
         [build(:middleware, receiver: lambda do |task, &block|
-                                        task.metadata.foobar = 42
+                                        task[:foobar] = 42
                                         block.call
                                       end),
          build(:middleware, receiver: lambda do |task|
-                                        task.metadata.foobar *= 1337
+                                        task[:foobar] *= 1337
                                       end)]
       end
 
@@ -81,28 +86,28 @@ describe Wayfarer::Middleware::Chain do
     describe "Metadata" do
       let(:first) do
         build(:middleware, receiver: lambda do |task, &block|
-                                       task.metadata.foobar = 42
-                                       block.call
-                                       task.metadata.barqux = 1337
-                                     end)
+          task[:foobar] = 42
+          block.call
+          task[:barqux] = 1337
+        end)
       end
 
       let(:last) do
         build(:middleware, receiver: lambda do |task|
-                                       raise unless task.metadata.foobar == 42
-                                       raise if task.metadata.barqux
-                                     end)
+          raise unless task[:foobar] == 42
+          raise if task[:barqux]
+        end)
       end
 
       let(:middlewares) { [first, last] }
 
-      it "is accessible across middleware" do
+      it "is accessible across middleware", :aggregate_failures do
         expect {
           chain.call(task)
         }.not_to raise_error
 
-        expect(task.metadata.foobar).to be(42)
-        expect(task.metadata.barqux).to be(1337)
+        expect(task[:foobar]).to be(42)
+        expect(task[:barqux]).to be(1337)
       end
     end
   end

data/spec/wayfarer/middleware/content_type_spec.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::ContentType do
+  let(:content_type) { "text/html" }
+  let(:task) { build(:task, :normalized) }
+  let(:page) { build(:page, headers: { "Content-Type" => content_type }) }
+
+  describe "#call" do
+    subject(:middleware) { Class.new(described_class).include(described_class::API).new }
+
+    before do
+      task[:page] = page
+      task[:controller] = middleware
+      middleware.class.content_type "text/html"
+    end
+
+    context "with permitted Content-Type" do
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
+      end
+    end
+
+    context "when permitted Content-Type has parameters" do
+      let(:page) { build(:page, headers: { "Content-Type" => "#{content_type}; charset=UTF-8" }) }
+
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
+      end
+    end
+
+    context "with forbidden Content-Type" do
+      let(:content_type) { "application/json" }
+
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.not_to yield_control
+      end
+    end
+
+    context "with permitted Regexp Content-Type" do
+      before do
+        middleware.class.content_type(/text/)
+      end
+
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
+      end
+    end
+  end
+
+  describe described_class::API do
+    subject(:controller) { Class.new.include(described_class).new }
+
+    describe "::allowed_content_types" do
+      describe "index" do
+        subject(:index) { controller.class.allowed_content_types[:index] }
+
+        it { is_expected.to be_empty }
+      end
+
+      describe "patterns" do
+        subject(:patterns) { controller.class.allowed_content_types[:patterns] }
+
+        it { is_expected.to be_empty }
+      end
+    end
+
+    describe "#content_type" do
+      it "allows Content-Types" do
+        controller.class.content_type(content_type)
+
+        expect(controller.class.allowed_content_types.dig(:index, content_type)).to be(true)
+      end
+
+      it "allows Content-Type patterns" do
+        controller.class.content_type(/text/)
+
+        expect(controller.class.allowed_content_types[:patterns]).to contain_exactly(/text/)
+      end
+    end
+  end
+end