wayfarer 0.4.5 → 0.4.7

data/spec/integration/page_spec.rb

@@ -5,16 +5,31 @@ require "spec_helpers"
 describe "Pages" do
   let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
 
-  describe Wayfarer::Base do
+  before do
+    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+    stub_const("DummyHandler", Class.new.include(Wayfarer::Handler))
+  end
+
+  shared_examples "executes" do
     specify do
-      class self.class::DummyJob < Wayfarer::Base
-        extend SpecHelpers
+      DummyJob.crawl(url)
+      perform_enqueued_jobs
+      assert_performed_jobs 1
+      expect(enqueued_jobs).to be_empty
+    end
+  end
+
+  describe "page content" do
+    before do
+      DummyJob.class_eval do
         include RSpec::Matchers
+        extend SpecHelpers
+        include SpecHelpers
 
-        route { host test_app_host, to: :index }
+        route.host test_app_host, to: :index
 
         def index
-          expect(page.url).to eq("http://test:9876/git-scm.com/book/en/v2.html")
+          expect(page.url).to eq(test_app_path("git-scm.com/book/en/v2.html"))
           expect(page.status_code).to be(200)
           expect(page.body).not_to be_empty
           expect(page.headers.count).to be(9)
@@ -24,39 +39,39 @@ describe "Pages" do
           expect(page.meta.links.external.count).to be(55)
         end
       end
-
-      self.class::DummyJob.crawl(url)
-      perform_enqueued_jobs
     end
-  end
-
-  describe Wayfarer::Handler do
-    specify do
-      class self.class::DummyJob < Wayfarer::Base
-        extend SpecHelpers
 
-        route { host test_app_host, to: DummyHandler }
+    it_behaves_like "executes"
+  end
 
-        class DummyHandler < Wayfarer::Handler
-          include RSpec::Matchers
+  describe "page content with handler" do
+    before do
+      DummyJob.class_eval do
+        include RSpec::Matchers
+        include SpecHelpers
 
-          route { to :index }
+        route.to :index
 
-          def index
-            expect(page.url).to eq("http://test:9876/git-scm.com/book/en/v2.html")
-            expect(page.status_code).to be(200)
-            expect(page.body).not_to be_empty
-            expect(page.headers.count).to be(9)
+        def index
+          expect(page.url).to eq(test_app_path("git-scm.com/book/en/v2.html"))
+          expect(page.status_code).to be(200)
+          expect(page.body).not_to be_empty
+          expect(page.headers.count).to be(9)
 
-            expect(page.meta.links.all.count).to be(157)
-            expect(page.meta.links.internal.count).to be(102)
-            expect(page.meta.links.external.count).to be(55)
-          end
+          expect(page.meta.links.all.count).to be(157)
+          expect(page.meta.links.internal.count).to be(102)
+          expect(page.meta.links.external.count).to be(55)
         end
       end
 
-      self.class::DummyJob.crawl(url)
-      perform_enqueued_jobs
+      DummyHandler.class_eval do
+        extend SpecHelpers
+        include SpecHelpers
+
+        route.host test_app_host, to: DummyHandler
+      end
     end
+
+    it_behaves_like "executes"
   end
 end

data/spec/integration/params_spec.rb

@@ -5,52 +5,60 @@ require "spec_helpers"
 describe "URL parameters" do
   let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
 
-  describe Wayfarer::Base do
+  before do
+    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+    stub_const("DummyHandler", Class.new.include(Wayfarer::Handler))
+  end
+
+  shared_examples "executes" do
     specify do
-      class self.class::DummyJob < Wayfarer::Base
+      DummyJob.crawl(url)
+      perform_enqueued_jobs
+      assert_performed_jobs 1
+      expect(enqueued_jobs).to be_empty
+    end
+  end
+
+  describe "path pattern matching" do
+    before do
+      DummyJob.class_eval do
         extend SpecHelpers
         include RSpec::Matchers
 
-        route do
-          to :index, host: test_app_host do
-            path "git-scm.com/book/:lang/:file"
-          end
+        route.to :index, host: test_app_host do
+          path "git-scm.com/book/:lang/:file"
         end
 
         def index
           expect(params).to eq("lang" => "en", "file" => "v2.html")
         end
       end
-
-      self.class::DummyJob.crawl(url)
-      perform_enqueued_jobs
     end
+
+    it_behaves_like "executes"
   end
 
-  describe Wayfarer::Handler do
-    specify do
-      class self.class::DummyJob < Wayfarer::Base
+  describe "path pattern matching with handler" do
+    before do
+      DummyJob.class_eval do
         extend SpecHelpers
 
-        route do
-          to DummyHandler, host: test_app_host do
-            path "git-scm.com/book/:lang/:file"
-          end
+        route.to DummyHandler, host: test_app_host do
+          path "git-scm.com/book/:lang/:file"
         end
+      end
 
-        class DummyHandler < Wayfarer::Handler
-          include RSpec::Matchers
+      DummyHandler.class_eval do
+        include RSpec::Matchers
 
-          route { to :index }
+        route.to :index
 
-          def index
-            expect(params).to eq("lang" => "en", "file" => "v2.html")
-          end
+        def index
+          expect(params).to eq("lang" => "en", "file" => "v2.html")
         end
       end
-
-      self.class::DummyJob.crawl(url)
-      perform_enqueued_jobs
     end
   end
+
+  it_behaves_like "executes"
 end

data/spec/integration/parsing_spec.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Response body parsing", redis: true do
+  before do
+    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+    stub_const("DummyHandler", Class.new.include(Wayfarer::Handler))
+  end
+
+  shared_examples "executes" do
+    specify do
+      DummyJob.crawl(url)
+      perform_enqueued_jobs
+      assert_performed_jobs 1
+      expect(enqueued_jobs).to be_empty
+    end
+  end
+
+  context "with XML" do
+    let(:url) { test_app_path("xml/dummy.xml") }
+
+    before do
+      DummyJob.class_eval do
+        include RSpec::Matchers
+
+        route.to :index
+
+        def index
+          expect(page.doc).to be_a(Nokogiri::XML::Document)
+        end
+      end
+
+      it_behaves_like "executes"
+    end
+  end
+
+  context "with HTML" do
+    let(:url) { test_app_path("finders.html") }
+
+    before do
+      DummyJob.class_eval do
+        include RSpec::Matchers
+
+        route.to :index
+
+        def index
+          expect(page.doc).to be_a(Nokogiri::HTML::Document)
+        end
+      end
+    end
+
+    it_behaves_like "executes"
+  end
+
+  context "with JSON" do
+    let(:url) { test_app_path("json/dummy.json") }
+
+    before do
+      DummyJob.class_eval do
+        include RSpec::Matchers
+
+        route.to :index
+
+        def index
+          expect(page.doc).to be_a(Hash)
+        end
+      end
+    end
+
+    it_behaves_like "executes"
+  end
+
+  describe "custom Content-Type parsers" do
+    context "with registered Content-Types" do
+      let(:parser) do
+        Class.new do
+          def self.parse(_body)
+            :ok
+          end
+        end
+      end
+
+      let(:parser_with_options) do
+        Class.new do
+          def self.parse(_body, options)
+            options
+          end
+        end
+      end
+
+      before do
+        Wayfarer::Parsing.registry["foo/bar"] = parser
+        Wayfarer::Parsing.registry["bar/qux"] = [parser_with_options, :ok]
+      end
+
+      after do
+        Wayfarer::Parsing.registry.delete("foo/bar")
+        Wayfarer::Parsing.registry.delete("bar/qux")
+      end
+
+      before do
+        DummyJob.class_eval do
+          route.to :index
+
+          def index
+            page.doc
+          end
+        end
+      end
+
+      def perform(content_type)
+        DummyJob.new.perform(
+          build(:task, url: test_app_path("response_header/Content-Type/#{content_type}"))
+        )
+      end
+
+      specify do
+        expect(perform("foo/bar")).to be(:ok)
+        expect(perform("bar/qux")).to be(:ok)
+        expect(perform("image/jpeg")).to be(nil)
+      end
+    end
+  end
+end

data/spec/integration/routing_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Routing" do
+  let(:task) { build(:task) }
+
+  before do
+    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+  end
+
+  describe "custom routing" do
+    before do
+      DummyJob.class_eval do
+      end
+    end
+  end
+end

data/spec/integration/stage_spec.rb

@@ -2,50 +2,57 @@
 
 require "spec_helpers"
 
-describe "Staging" do
+describe "Staging", redis: true do
   let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
 
-  describe Wayfarer::Base do
-    specify do
-      class self.class::DummyJob < Wayfarer::Base
+  before do
+    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+    stub_const("DummyHandler", Class.new.include(Wayfarer::Handler))
+  end
+
+  describe "staging URLs" do
+    before do
+      DummyJob.class_eval do
         extend SpecHelpers
 
-        route { host test_app_host, to: :index }
+        route.host test_app_host, to: :index
 
         def index
           stage page.meta.links.all
         end
       end
+    end
 
+    specify do
       expect {
-        self.class::DummyJob.crawl(url)
+        DummyJob.crawl(url)
         perform_enqueued_jobs
-      }.to change { enqueued_jobs.size }.by(156)
+      }.to change { enqueued_jobs.size }.by(157)
     end
   end
 
-  describe Wayfarer::Handler do
-    specify do
-      class self.class::DummyJob < Wayfarer::Base
+  context "with handler" do
+    before do
+      DummyJob.class_eval do
         extend SpecHelpers
 
-        route do
-          host test_app_host, to: DummyHandler
-        end
+        route.host test_app_host, to: DummyHandler
+      end
 
-        class DummyHandler < Wayfarer::Handler
-          route { to :index }
+      DummyHandler.class_eval do
+        route.to :index
 
-          def index
-            stage page.meta.links.all
-          end
+        def index
+          stage page.meta.links.all
         end
       end
+    end
 
+    specify do
       expect {
-        self.class::DummyJob.crawl(url)
+        DummyJob.crawl(url)
         perform_enqueued_jobs
-      }.to change { enqueued_jobs.size }.by(156)
+      }.to change { enqueued_jobs.size }.by(157)
     end
   end
 end

data/spec/middleware/batch_completion_spec.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::BatchCompletion, "#call" do
+  let(:task) { build(:task) }
+  let(:job) { double(exception_executions: exception_executions) }
+  let(:exception_executions) { {} }
+  subject(:batch_completion) { described_class.new }
+
+  before { task[:job] = job }
+
+  it "assigns cloned exception_executions" do
+    expect {
+      batch_completion.call(task)
+    }.to change {
+      task[:initial_exception_executions]
+    }.from(nil).to(exception_executions)
+
+    expect(task[:initial_exception_executions]).not_to be(exception_executions)
+  end
+
+  context "when already assigned" do
+    before { task[:initial_exception_executions] = Object.new }
+
+    specify do
+      expect { |spy| batch_completion.call(task, &spy) }.to yield_control
+    end
+
+    it "doesn't assign exception_executions" do
+      expect { batch_completion.call(task) }.not_to(change { task[:initial_exception_executions] })
+    end
+  end
+end

data/spec/middleware/chain_spec.rb

@@ -65,11 +65,11 @@ describe Wayfarer::Middleware::Chain do
     describe "Return value" do
       let(:middlewares) do
         [build(:middleware, receiver: lambda do |task, &block|
-                                        task.metadata.foobar = 42
+                                        task[:foobar] = 42
                                         block.call
                                       end),
          build(:middleware, receiver: lambda do |task|
-                                        task.metadata.foobar *= 1337
+                                        task[:foobar] *= 1337
                                       end)]
       end
 
@@ -81,16 +81,16 @@ describe Wayfarer::Middleware::Chain do
     describe "Metadata" do
       let(:first) do
         build(:middleware, receiver: lambda do |task, &block|
-                                       task.metadata.foobar = 42
+                                       task[:foobar] = 42
                                        block.call
-                                       task.metadata.barqux = 1337
+                                       task[:barqux] = 1337
                                      end)
       end
 
       let(:last) do
         build(:middleware, receiver: lambda do |task|
-                                       raise unless task.metadata.foobar == 42
-                                       raise if task.metadata.barqux
+                                       raise unless task[:foobar] == 42
+                                       raise if task[:barqux]
                                      end)
       end
 
@@ -101,8 +101,8 @@ describe Wayfarer::Middleware::Chain do
           chain.call(task)
         }.not_to raise_error
 
-        expect(task.metadata.foobar).to be(42)
-        expect(task.metadata.barqux).to be(1337)
+        expect(task[:foobar]).to be(42)
+        expect(task[:barqux]).to be(1337)
       end
     end
   end

data/spec/middleware/content_type_spec.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::ContentType do
+  let(:content_type) { "text/html" }
+  let(:task) { build(:task) }
+  let(:page) { build(:page, headers: { "Content-Type" => content_type }) }
+
+  describe "#call" do
+    subject { Class.new(described_class).include(described_class::API).new }
+
+    before do
+      task[:page] = page
+      task[:controller] = subject
+    end
+
+    before { subject.class.content_type "text/html" }
+
+    context "with permitted Content-Type" do
+      it "yields" do
+        expect { |spy| subject.call(task, &spy) }.to yield_control
+      end
+    end
+
+    context "when permitted Content-Type has parameters" do
+      let(:page) { build(:page, headers: { "Content-Type" => "#{content_type}; charset=UTF-" }) }
+
+      it "yields" do
+        expect { |spy| subject.call(task, &spy) }.to yield_control
+      end
+    end
+
+    context "with forbidden Content-Type" do
+      let(:content_type) { "application/json" }
+
+      it "does not yield" do
+        expect { |spy| subject.call(task, &spy) }.not_to yield_control
+      end
+    end
+
+    context "with permitted Regexp Content-Type" do
+      before do
+        subject.class.content_type(/text/)
+      end
+
+      it "yields" do
+        expect { |spy| subject.call(task, &spy) }.to yield_control
+      end
+    end
+  end
+
+  describe described_class::API do
+    subject(:controller) do
+      Struct.new(:task).include(described_class).new(task)
+    end
+
+    describe "::allowed_content_types" do
+      describe "index" do
+        subject { controller.class.allowed_content_types[:index] }
+
+        it { is_expected.to be_empty }
+      end
+
+      describe "patterns" do
+        subject { controller.class.allowed_content_types[:patterns] }
+
+        it { is_expected.to be_empty }
+      end
+    end
+
+    describe "#content_type" do
+      it "allows Content-Types" do
+        controller.class.content_type(content_type)
+
+        expect(controller.class.allowed_content_types[:index][content_type]).to be(true)
+      end
+
+      it "allows Content-Type patterns" do
+        controller.class.content_type(/text/)
+
+        expect(controller.class.allowed_content_types[:patterns]).to contain_exactly(/text/)
+      end
+    end
+  end
+end

data/spec/middleware/controller_spec.rb

@@ -54,28 +54,28 @@ describe Wayfarer::Middleware::Controller do
       it "assigns itself" do
         expect {
           subject.call(task)
-        }.to change { task.metadata.job }.to(subject)
+        }.to change { task[:job] }.to(subject)
       end
     end
 
     context "with job assigned" do
-      before { task.metadata.job = Object.new }
+      before { task[:job] = Object.new }
 
       it "does not override the job" do
         expect {
           subject.call(task)
-        }.not_to(change { task.metadata.job })
+        }.not_to(change { task[:job] })
       end
     end
 
     it "assigns itself as controller" do
       expect {
         subject.call(task)
-      }.to change { task.metadata.controller }.to(subject)
+      }.to change { task[:controller] }.to(subject)
     end
 
     it "calls the chain" do
-      expect(subject.chain).to receive(:call)
+      expect(subject.class.chain).to receive(:call)
       subject.call(task)
     end