wayfarer 0.4.5 → 0.4.7

data/spec/factories/task.rb

@@ -2,11 +2,17 @@
 
 FactoryBot.define do
   factory :task, class: Wayfarer::Task do
-    url { "https://example.com" }
+    url { test_app_path("/") }
     batch { "batch" }
 
     initialize_with do
       new(url, batch)
     end
+
+    trait :redis_pool do
+      after(:build) do |task|
+        task[:redis_pool] = Wayfarer::Redis::Pool.instance
+      end
+    end
   end
 end

data/spec/fixtures/dummy_job.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
-class DummyJob < Wayfarer::Base
-  route { to :index }
+class Router < ActiveJob::Base
+  include Wayfarer::Base
 
-  def hello; end
+  def hello
+    stage page.meta.links.external
+  end
 end

data/spec/gc_spec.rb

@@ -2,58 +2,16 @@
 
 require "spec_helpers"
 
-describe Wayfarer::GC, redis: true do
-  include Wayfarer::Redis::Connection
+describe Wayfarer::GC, "::run", redis: true do
+  let(:task) { build(:task, :redis_pool) }
+  let(:barrier) { instance_double(Wayfarer::Redis::Barrier) }
 
-  let(:task) { build(:task) }
-  subject(:gc) { described_class.new(task) }
+  subject(:run) { described_class.run(task) }
 
-  before do
-    task.metadata.job = spy
-    task.barrier.seen?(task.url)
-  end
-
-  describe "#run" do
-    context "when counter reaches 0" do
-      before { task.counter.increment }
-
-      it "resets the barrier" do
-        expect {
-          gc.run
-        }.to change { redis { |conn| conn.exists?(task.barrier.redis_key) } }.to(false)
-      end
-
-      it "resets the counter" do
-        expect {
-          gc.run
-        }.to change { redis { |conn| conn.exists?(task.counter.redis_key) } }.to(false)
-      end
-
-      it "runs after batch callbacks" do
-        expect(task.metadata.job).to receive(:run_callbacks).with(:batch).exactly(:once)
-        gc.run
-      end
-    end
-
-    context "when counter does not reach 0" do
-      before { 2.times { task.counter.increment } }
-
-      it "does not reset the barrier" do
-        expect {
-          gc.run
-        }.not_to(change { redis { |conn| conn.exists?(task.barrier.redis_key) } })
-      end
-
-      it "does not reset the counter" do
-        expect {
-          gc.run
-        }.not_to(change { redis { |conn| conn.exists?(task.counter.redis_key) } })
-      end
+  it "resets barrier and counter" do
+    expect(barrier).to receive(:reset!)
+    expect(Wayfarer::Redis::Barrier).to receive(:new).with(task).and_return(barrier)
 
-      it "does not run after batch callbacks" do
-        expect(task.metadata.job).not_to receive(:run_callbacks).with(:batch)
-        gc.run
-      end
-    end
+    run
   end
 end

data/spec/handler_spec.rb

@@ -3,7 +3,7 @@
 require "spec_helpers"
 
 describe Wayfarer::Handler do
-  subject { Class.new(described_class) }
+  subject { Class.new.include(described_class) }
 
   it "undefines ::after_batch" do
     expect(subject).not_to respond_to(:after_batch)

data/spec/integration/callbacks_spec.rb

@@ -2,84 +2,196 @@
 
 require "spec_helpers"
 
-describe "Callbacks" do
-  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
+describe Wayfarer::Callbacks, redis: true do
+  before do
+    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+    stub_const("DummyHandler", Class.new.include(Wayfarer::Handler))
+  end
 
-  describe Wayfarer::Base do
-    specify do
-      class self.class::DummyJob < Wayfarer::Base
-        extend SpecHelpers
-        include RSpec::Matchers
+  describe "job callbacks" do
+    before do
+      DummyJob.class_eval do
+        route.path "alpha", to: :alpha
+        route.path "beta", to: :beta
 
-        route { host test_app_host, to: :index }
+        def callbacks_fired
+          @callbacks_fired ||= []
+        end
 
-        attr_accessor :callbacks_fired
+        before_fetch do |job|
+          callbacks_fired << [:before_fetch, job]
+        end
 
-        before_fetch do
-          self.callbacks_fired = %i[before_fetch]
+        before_fetch if: -> { %i[alpha beta].exclude?(action) } do |job|
+          callbacks_fired << [:before_fetch_except_alpha_beta, job]
         end
 
-        before_action do
-          callbacks_fired.push(:before_action)
+        before_action do |job|
+          callbacks_fired << [:before_action, job]
         end
 
-        after_batch do
-          expect(callbacks_fired).to eq(%i[before_fetch before_action])
+        before_action if: -> { action == :alpha } do |job|
+          callbacks_fired << [:before_action_only_alpha, job]
         end
 
-        def index; end
+        before_action if: -> { action == :beta } do |job|
+          callbacks_fired << [:before_action_only_beta, job]
+        end
+
+        before_action if: -> { %i[alpha gamma].include?(action) } do |job|
+          callbacks_fired << [:before_action_only_alpha_gamma, job]
+        end
+
+        before_action if: -> { action != :alpha } do |job|
+          callbacks_fired << [:before_action_except_alpha, job]
+        end
+
+        before_action if: -> { %i[beta gamma].exclude?(action) } do |job|
+          callbacks_fired << [:before_action_except_beta_gamma, job]
+        end
+
+        before_action :callback_a, if: -> { action == :alpha }
+
+        before_action :callback_b
+
+        before_action :callback_c, if: -> { action != :beta }
+
+        def alpha; end
+
+        def beta; end
+
+      private
+
+        def callback_a
+          callbacks_fired << :before_action_callback_a
+        end
+
+        def callback_b
+          callbacks_fired << :before_action_callback_b
+        end
+
+        def callback_c
+          callbacks_fired << :before_action_callback_c
+        end
       end
+    end
 
-      self.class::DummyJob.crawl(url)
-      perform_enqueued_jobs
+    specify do
+      job = DummyJob.new
+      job.perform(build(:task, url: test_app_path("alpha")))
+      expect(job.callbacks_fired).to eq([
+                                          [:before_fetch, job],
+                                          [:before_action, job],
+                                          [:before_action_only_alpha, job],
+                                          [:before_action_only_alpha_gamma, job],
+                                          [:before_action_except_beta_gamma, job],
+                                          :before_action_callback_a,
+                                          :before_action_callback_b,
+                                          :before_action_callback_c
+                                        ])
+
+      job = DummyJob.new
+      job.perform(build(:task, url: test_app_path("beta")))
+      expect(job.callbacks_fired).to eq([
+                                          [:before_fetch, job],
+                                          [:before_action, job],
+                                          [:before_action_only_beta, job],
+                                          [:before_action_except_alpha, job],
+                                          :before_action_callback_b
+                                        ])
     end
   end
 
-  describe Wayfarer::Handler do
-    specify do
-      class self.class::DummyJob < Wayfarer::Base
-        extend SpecHelpers
-        include RSpec::Matchers
+  describe "job callbacks with handler" do
+    before { stub_const("CallbacksFired", []) }
+
+    before do
+      DummyJob.class_eval do
+        route.to [DummyHandler, :index]
 
-        route { host test_app_host, to: DummyHandler }
+        before_fetch do |job|
+          CallbacksFired << [:before_fetch, job.class]
+        end
 
-        attr_accessor :callbacks_fired
+        around_fetch do |job, block|
+          CallbacksFired << [:around_fetch, job.class]
 
-        before_fetch do
-          self.callbacks_fired = %i[before_fetch_job]
+          block.call
         end
 
-        before_action do
-          callbacks_fired.push(:before_action_job)
+        after_fetch do |job|
+          CallbacksFired << [:after_fetch, job.class]
         end
 
-        after_batch do
-          expect(callbacks_fired).to eq(%i[before_fetch_job before_action_job])
+        before_action do |job|
+          CallbacksFired << [:before_action, job.class]
         end
 
-        class DummyHandler < Wayfarer::Handler
-          include RSpec::Matchers
+        around_action do |job, block|
+          CallbacksFired << [:around_action, job.class]
 
-          route { to :index }
+          block.call
+        end
 
-          attr_accessor :callbacks_fired
+        after_action do |job|
+          CallbacksFired << [:after_action, job.class]
+        end
+      end
 
-          before_fetch do
-            raise "before_fetch ran in handler"
-          end
+      DummyHandler.class_eval do
+        before_action do |handler|
+          CallbacksFired << [:before_action, handler.class]
+        end
 
-          before_action do
-            self.callbacks_fired = %i[before_action_handler]
-          end
+        around_action do |handler, block|
+          CallbacksFired << [:around_action, handler.class]
 
-          def index
-            expect(callbacks_fired).to eq(%i[before_action_handler])
-          end
+          block.call
         end
+
+        after_action do |handler|
+          CallbacksFired << [:after_action, handler.class]
+        end
+
+        def index; end
       end
+    end
 
-      self.class::DummyJob.crawl(url)
+    specify do
+      DummyJob.crawl(test_app_path("/"))
       perform_enqueued_jobs
+
+      expect(CallbacksFired).to eq([
+                                     [:before_fetch, DummyJob],
+                                     [:around_fetch, DummyJob],
+                                     [:after_fetch, DummyJob],
+                                     [:before_action, DummyJob],
+                                     [:around_action, DummyJob],
+                                     [:before_action, DummyHandler],
+                                     [:around_action, DummyHandler],
+                                     [:after_action, DummyHandler],
+                                     [:after_action, DummyJob]
+                                   ])
+    end
+  end
+
+  describe "accessing user agent in before_fetch" do
+    before do
+      DummyJob.class_eval do
+        include RSpec::Matchers
+
+        route.to :index
+
+        before_fetch do
+          expect(user_agent).to be_a(Net::HTTP::Persistent)
+        end
+
+        def index; end
+      end
+    end
+
+    specify do
+      DummyJob.new.perform(build(:task, url: test_app_path("alpha")))
     end
   end
 end

data/spec/integration/content_type_spec.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Content-Type allow listing", redis: true do
+  def perform(content_type)
+    DummyJob.new.perform(
+      build(:task, url: test_app_path("response_header/Content-Type/#{content_type}"))
+    )
+  end
+
+  before do
+    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+    stub_const("DummyHandler", Class.new.include(Wayfarer::Handler))
+  end
+
+  context "with registered Content-Types" do
+    before do
+      DummyJob.class_eval do
+        route.to :index
+
+        content_type "foo", "bar"
+
+        def index
+          :ok
+        end
+      end
+    end
+
+    specify do
+      expect(perform("foo")).to be(:ok)
+      expect(perform("bar")).to be(:ok)
+      expect(perform("qux")).to be(nil)
+    end
+  end
+
+  context "with registered Content-Type media type" do
+    before do
+      DummyJob.class_eval do
+        route.to :index
+
+        content_type(/text/)
+        content_type "application/vnd.scenario.custom+json"
+
+        def index
+          :ok
+        end
+      end
+    end
+
+    specify do
+      expect(perform("text/xml")).to be(:ok)
+      expect(perform("application/xml")).to be(nil)
+      expect(perform("application/rss+xml")).to be(nil)
+      expect(perform("image/svg+xml")).to be(nil)
+      expect(perform("application/vnd.scenario.custom+json")).to be(:ok)
+    end
+  end
+
+  context "without registered Content-Types" do
+    before do
+      DummyJob.class_eval do
+        route.to :index
+
+        def index
+          :ok
+        end
+      end
+    end
+
+    specify do
+      expect(perform("foo")).to be(:ok)
+    end
+  end
+
+  context "with regular expression Content-Type" do
+    before do
+      DummyJob.class_eval do
+        route.to :index
+
+        content_type(/xml/)
+
+        def index
+          :ok
+        end
+      end
+    end
+
+    specify do
+      expect(perform("text/xml")).to be(:ok)
+      expect(perform("application/xml")).to be(:ok)
+      expect(perform("application/rss+xml")).to be(:ok)
+      expect(perform("image/svg+xml")).to be(:ok)
+      expect(perform("application/vnd.scenario.custom+json")).to be(nil)
+    end
+  end
+
+  context "with registered Content-Types" do
+    before do
+      DummyJob.class_eval do
+        route.to :index
+
+        content_type(/text/, "foo")
+
+        def index
+          :ok
+        end
+      end
+    end
+
+    specify do
+      expect(perform("text/application")).to be(:ok)
+      expect(perform("foo")).to be(:ok)
+      expect(perform("bar")).to be(nil)
+    end
+  end
+
+  context "with handler" do
+    before do
+      DummyJob.class_eval do
+        route.to DummyHandler
+
+        content_type(/text/, "foo")
+      end
+    end
+
+    before do
+      DummyHandler.class_eval do
+        route.to :index
+
+        content_type "foo"
+
+        def index
+          :handler
+        end
+      end
+    end
+
+    specify do
+      expect(perform("text/application")).to be(nil)
+      expect(perform("foo")).to be(:handler)
+      expect(perform("bar")).to be(nil)
+    end
+  end
+end

data/spec/integration/gc_spec.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Garbage collection", redis: true do
+  let(:task) { build(:task, :redis_pool) }
+  let(:counter) { Wayfarer::Redis::Counter.new(task) }
+  let(:barrier) { Wayfarer::Redis::Barrier.new(task) }
+  let(:redis) { ::Redis.new(url: redis_host) }
+
+  before do
+    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+
+    DummyJob.class_eval do
+      route.to :index
+
+      def index; end
+    end
+  end
+
+  before do
+    redis.set(counter.redis_key, 0)
+    redis.hset(barrier.redis_key, "", "")
+  end
+
+  subject(:perform) do
+    DummyJob.perform_later(task)
+    perform_enqueued_jobs
+  end
+
+  it "resets counter" do
+    expect { perform }.to change { redis.exists?(counter.redis_key) }.from(true).to(false)
+
+    assert_performed_jobs 1
+    expect(enqueued_jobs).to be_empty
+  end
+
+  it "resets barrier" do
+    expect { perform }.to change { redis.exists?(barrier.redis_key) }.from(true).to(false)
+
+    assert_performed_jobs 1
+    expect(enqueued_jobs).to be_empty
+  end
+end

data/spec/integration/handler_spec.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Handlers", redis: true do
+  before do
+    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+    stub_const("DummyHandler", Class.new.include(Wayfarer::Handler))
+    stub_const("OtherHandler", Class.new.include(Wayfarer::Handler))
+  end
+
+  describe "bypassing the router" do
+    before do
+      DummyJob.class_eval do
+        route.to DummyHandler
+      end
+
+      DummyHandler.class_eval do
+        class_attribute :called
+
+        before_action do
+          self.class.called = true
+        end
+
+        route.to [OtherHandler, :foobar]
+      end
+
+      OtherHandler.class_eval do
+        class_attribute :called
+
+        def foobar
+          self.class.called = true
+        end
+      end
+    end
+
+    specify do
+      DummyJob.crawl(test_app_path("/"))
+      perform_enqueued_jobs
+      assert_performed_jobs 1
+      expect(enqueued_jobs).to be_empty
+      expect(DummyHandler.called).to be(true)
+      expect(OtherHandler.called).to be(true)
+    end
+  end
+
+  describe "dispatching to another Wayfarer::Base" do
+    before do
+      stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+      stub_const("OtherJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+    end
+
+    before do
+      DummyJob.class_eval do
+        extend SpecHelpers
+
+        route.to OtherJob
+      end
+    end
+
+    specify do
+      DummyJob.crawl(test_app_path("/"))
+      expect { perform_enqueued_jobs }.to raise_error(ArgumentError, "invalid action: OtherJob")
+    end
+  end
+end