wayfarer 0.4.7 → 0.4.8

data/spec/wayfarer/batch_completion_spec.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+# The RSpec/SpecFilePathFormat offense was not addressed as it requires renaming
+# the file, not changing its content.
+describe Wayfarer::BatchCompletion, :redis do
+  let(:task) { build(:task, :redis_pool) }
+  let(:job) { instance_spy(ActiveJob::Base, arguments: [task]) }
+
+  describe "::call" do
+    subject(:call_batch_completion) { described_class.call(name, nil, nil, nil, { job: job }) }
+
+    let(:name) { "foo" }
+
+    context "with Wayfarer job" do
+      before { job.extend(Wayfarer::Base) }
+
+      specify do
+        allow(described_class).to receive(:handle)
+
+        call_batch_completion
+
+        expect(described_class).to have_received(:handle).with(name, job, task)
+      end
+
+      it "does not reassign Redis pool" do
+        expect { call_batch_completion }.not_to change { task[:redis_pool] }.from(Wayfarer::Redis::Pool.instance)
+      end
+
+      context "without Redis pool" do
+        before do
+          task[:redis_pool] = nil
+        end
+
+        it "assigns Redis pool" do
+          expect { call_batch_completion }.to change { task[:redis_pool] }.from(nil).to(Wayfarer::Redis::Pool.instance)
+        end
+      end
+    end
+
+    context "with other job" do
+      specify do
+        allow(described_class).to receive(:handle)
+
+        call_batch_completion
+
+        expect(described_class).not_to have_received(:handle)
+      end
+    end
+  end
+
+  describe "::handle" do
+    subject(:handle) { described_class.handle(event, job, task) }
+
+    let(:job) do
+      instance_spy(ActiveJob::Base,
+                   arguments: [task],
+                   executions: executions,
+                   exception_executions: exception_executions)
+    end
+
+    let(:initial_exception_executions) { {} }
+    let(:exception_executions) { initial_exception_executions }
+    let(:executions) { 0 }
+    let(:counter) { build(:counter, task: task) }
+
+    before do
+      task[:job] = job
+      task[:initial_exception_executions] = initial_exception_executions
+    end
+
+    context "with an 'enqueue.active_job' event" do
+      let(:event) { "enqueue.active_job" }
+
+      specify do
+        expect { handle }.to change(counter, :value).by(1)
+      end
+
+      context "with retry" do
+        let(:executions) { 1 }
+
+        specify do
+          expect { handle }.not_to(change(counter, :value))
+        end
+      end
+    end
+
+    context "with a 'perform.active_job' event" do
+      let(:event) { "perform.active_job" }
+
+      context "when job succeeds" do
+        specify do
+          expect { handle }.to change(counter, :value).by(-1)
+        end
+
+        context "when batch completes" do
+          before do
+            counter.increment
+            allow(Wayfarer::GC).to receive(:run).and_call_original
+          end
+
+          specify do
+            handle
+
+            expect(job).to have_received(:run_callbacks)
+            expect(Wayfarer::GC).to have_received(:run).with(task)
+          end
+        end
+
+        context "when batch does not complete" do
+          before do
+            allow(Wayfarer::GC).to receive(:run)
+          end
+
+          specify do
+            handle
+
+            expect(job).not_to have_received(:run_callbacks)
+            expect(Wayfarer::GC).not_to have_received(:run)
+          end
+        end
+      end
+
+      context "when an exception occurrs" do
+        let(:exception_executions) { { "[RuntimeError]" => 1 } }
+
+        specify do
+          expect { handle }.not_to(change(counter, :value))
+        end
+      end
+    end
+
+    context "with a 'retry_stopped.active_job' event" do
+      let(:event) { "retry_stopped.active_job" }
+
+      specify do
+        expect { handle }.to change(counter, :value).by(-1)
+      end
+    end
+  end
+end

data/spec/wayfarer/cli/job_spec.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Jobs", :cli, :integration, :redis do
+  include Wayfarer::Redis
+
+  subject(:cli) { Wayfarer::CLI }
+
+  let(:url) { test_app_path("/hello_world") }
+  let(:batch) { "my-batch" }
+
+  before do
+    mock_job! :dummy_job
+
+    write_file "dummy_job.rb", <<~FILE
+      Wayfarer::CLI.new.send(:load_environment, "dummy_job.rb")
+      class DummyJob < ActiveJob::Base
+        include Wayfarer::Base
+      end
+    FILE
+  end
+
+  describe "perform" do
+    it "performs the worker" do
+      # rubocop:disable RSpec/AnyInstance
+      # The CLI reloads DummyJob from file. #expect_any_instance_of alters the
+      # object initialization in a way that survives this class reloading.
+      expect_any_instance_of(DummyJob).to receive(:perform).with(kind_of(Wayfarer::Task)) do |job|
+        # rubocop:enable RSpec/AnyInstance
+        task = job.arguments.first
+        task[:job] = job
+      end
+
+      cli.start(["perform", "-r", "dummy_job.rb", "DummyJob", url])
+    end
+
+    context "when using MockRedis" do
+      it "performs the worker using MockRedis" do
+        cli.start(["perform", "--mock-redis", "DummyJob", url])
+        expect(Wayfarer.config[:redis][:factory].call(nil)).to be_a(MockRedis)
+      end
+    end
+  end
+
+  describe "enqueue" do
+    before do
+      allow(DummyJob).to receive(:crawl)
+    end
+
+    it "enqueues the job" do
+      cli.start(["enqueue", "DummyJob", url])
+      expect(DummyJob).to have_received(:crawl).with(Addressable::URI.parse(url), batch: kind_of(String))
+    end
+
+    context "with a batch provided" do
+      it "enqueues the job" do
+        cli.start(["enqueue", "--batch", batch, "DummyJob", url])
+        expect(DummyJob).to have_received(:crawl).with(Addressable::URI.parse(url), batch: batch)
+      end
+    end
+  end
+
+  describe "execute" do
+    before do
+      allow(DummyJob).to receive(:crawl)
+    end
+
+    it "executes the job" do
+      cli.start(["execute", "--retain-pool", "DummyJob", url])
+      expect(DummyJob).to have_received(:crawl).with(Addressable::URI.parse(url), batch: kind_of(String))
+    end
+
+    context "with a batch provided" do
+      it "executes the job" do
+        cli.start(["execute", "--retain-pool", "--batch", batch, "DummyJob", url])
+        expect(DummyJob).to have_received(:crawl).with(Addressable::URI.parse(url), batch: batch)
+      end
+    end
+
+    context "with mock Redis" do
+      it "executes the worker using MockRedis" do
+        cli.start(["execute", "--retain-pool", "--mock-redis", "DummyJob", url])
+        expect(Wayfarer.config[:redis][:factory].call(nil)).to be_a(MockRedis)
+      end
+    end
+  end
+end

data/spec/wayfarer/cli/routing_spec.rb

@@ -0,0 +1,322 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Routing", :cli, :integration do
+  subject(:route) { Wayfarer::CLI.start(["route", "-r", "dummy_job.rb", "DummyJob", url]) }
+
+  before do
+    write_file("dummy_job.rb", contents)
+
+    mock_job! :dummy_job
+  end
+
+  let(:contents) do
+    <<~RUBY
+      class DummyJob < ActiveJob::Base
+        include Wayfarer::Base
+
+        #{routes}
+      end
+    RUBY
+  end
+
+  context "without routes" do
+    let(:url) { "http://#{test_app_host}" }
+    let(:routes) { "" }
+
+    let(:expected_output) do
+      <<~OUTPUT
+        ---
+        routed: false
+        root_route:
+          match: false
+          params: {}
+          children: []
+      OUTPUT
+    end
+
+    specify do
+      expect { route }.to output(expected_output).to_stdout
+    end
+  end
+
+  context "with target route only" do
+    let(:url) { "http://#{test_app_host}" }
+    let(:routes) do
+      <<~RUBY
+        route.to :index
+      RUBY
+    end
+
+    let(:expected_output) do
+      <<~OUTPUT
+        ---
+        routed: true
+        params: {}
+        action: :index
+        root_route:
+          match: true
+          params: {}
+          children:
+          - target_route:
+              action: :index
+              children: []
+      OUTPUT
+    end
+
+    specify do
+      expect { route }.to output(expected_output).to_stdout
+    end
+  end
+
+  context "with a symbol target" do
+    let(:routes) do
+      <<~RUBY
+        route.host #{test_app_hostname.inspect}, to: :index
+      RUBY
+    end
+
+    context "with matching URL" do
+      let(:url) { "http://#{test_app_host}" }
+
+      let(:expected_output) do
+        <<~OUTPUT
+          ---
+          routed: true
+          params: {}
+          action: :index
+          root_route:
+            match: true
+            params: {}
+            children:
+            - route:
+                host:
+                  name: #{test_app_hostname}
+                match: true
+                params: {}
+                children:
+                - target_route:
+                    action: :index
+                    children: []
+        OUTPUT
+      end
+
+      specify do
+        expect { route }.to output(expected_output).to_stdout
+      end
+    end
+
+    context "with mismatching URL" do
+      let(:url) { "http://example.com" }
+
+      let(:expected_output) do
+        <<~OUTPUT
+          ---
+          routed: false
+          root_route:
+            match: true
+            params: {}
+            children:
+            - route:
+                host:
+                  name: #{test_app_hostname}
+                match: false
+                params: {}
+                children: []
+        OUTPUT
+      end
+
+      specify do
+        expect { route }.to output(expected_output).to_stdout
+      end
+    end
+  end
+
+  context "with a symbol target and params" do
+    let(:url) { "http://#{test_app_host}/barqux" }
+    let(:routes) do
+      <<~RUBY
+        route.host #{test_app_hostname.inspect}, to: :index do
+          path ":foobar"
+        end
+      RUBY
+    end
+
+    let(:expected_output) do
+      <<~OUTPUT
+        ---
+        routed: true
+        params:
+          foobar: barqux
+        action: :index
+        root_route:
+          match: true
+          params: {}
+          children:
+          - route:
+              host:
+                name: #{test_app_hostname}
+              match: true
+              params: {}
+              children:
+              - target_route:
+                  action: :index
+                  children:
+                  - route:
+                      path:
+                        pattern: "/:foobar"
+                      match: true
+                      params:
+                        foobar: barqux
+                      children: []
+      OUTPUT
+    end
+
+    specify do
+      expect { route }.to output(expected_output).to_stdout
+    end
+  end
+
+  context "with a custom route, params, and a conditional target" do
+    let(:url) { "http://#{test_app_host}/users/42/edit" }
+    let(:routes) do
+      <<~RUBY
+        route.host #{test_app_hostname.inspect}, path: ":resource" do
+          custom do |root, uri|
+            root.path ":id", path: ":action" do
+              if uri.host == #{test_app_hostname.inspect}
+                root.to :foo
+              end
+            end
+          end
+        end
+      RUBY
+    end
+
+    let(:expected_output) do
+      <<~OUTPUT
+        ---
+        routed: true
+        params:
+          resource: users
+          id: '42'
+          action: edit
+        action: :foo
+        root_route:
+          match: true
+          params: {}
+          children:
+          - route:
+              host:
+                name: #{test_app_hostname}
+              match: true
+              params: {}
+              children:
+              - route:
+                  path:
+                    pattern: "/:resource"
+                  match: true
+                  params:
+                    resource: users
+                  children:
+                  - sub_route:
+                      custom:
+                        custom: Proc
+                      match: true
+                      params:
+                        resource: users
+                        id: '42'
+                        action: edit
+                      children: []
+      OUTPUT
+    end
+
+    specify do
+      expect { route }.to output(expected_output).to_stdout
+    end
+  end
+
+  describe "query parameter matching" do
+    let(:url) { "http://#{test_app_host}?page=10&order=desc&mobile=0&referrer=startpage&foo=bar" }
+    let(:routes) do
+      <<~RUBY
+        route.query({ page: 1..20, order: "desc", mobile: 0, referrer: /start/ }, to: :foo)
+      RUBY
+    end
+
+    let(:expected_output) do
+      <<~OUTPUT
+        ---
+        routed: true
+        params:
+          page: '10'
+          order: desc
+          mobile: '0'
+          referrer: startpage
+        action: :foo
+        root_route:
+          match: true
+          params: {}
+          children:
+          - route:
+              query:
+                page:
+                  min: 1
+                  max: 20
+                order: desc
+                mobile: 0
+                referrer: "/start/"
+              match: true
+              params:
+                page: '10'
+                order: desc
+                mobile: '0'
+                referrer: startpage
+              children:
+              - target_route:
+                  action: :foo
+                  children: []
+      OUTPUT
+    end
+
+    specify do
+      expect { route }.to output(expected_output).to_stdout
+    end
+  end
+
+  context "with URL matcher" do
+    let(:url) { "http://#{test_app_host}" }
+    let(:routes) do
+      <<~RUBY
+        route.url "http://#{test_app_host}", to: :index
+      RUBY
+    end
+
+    let(:expected_output) do
+      <<~OUTPUT
+        ---
+        routed: true
+        params: {}
+        action: :index
+        root_route:
+          match: true
+          params: {}
+          children:
+          - route:
+              url:
+                url: #{url}
+              match: true
+              params: {}
+              children:
+              - target_route:
+                  action: :index
+                  children: []
+      OUTPUT
+    end
+
+    specify do
+      expect { route }.to output(expected_output).to_stdout
+    end
+  end
+end

data/spec/{cli → wayfarer/cli}/version_spec.rb

@@ -2,7 +2,7 @@
 
 require "spec_helpers"
 
-describe Wayfarer::CLI, cli: true do
+describe Wayfarer::CLI, :cli do # rubocop:disable RSpec/SpecFilePathFormat
   describe "version" do
     it "prints the version" do
       expect {

data/spec/wayfarer/gc_spec.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::GC, :integration, :redis do
+  describe "::run" do
+    subject(:run) { described_class.run(task) }
+
+    let(:job) { instance_spy(Class.new.include(Wayfarer::Base)) }
+    let(:task) { build(:task, :redis_pool, job: job) }
+    let(:barrier) { instance_spy(Wayfarer::Redis::Barrier) }
+
+    before do
+      allow(Wayfarer::Redis::Barrier).to receive(:new).with(task).and_return(barrier)
+    end
+
+    specify do
+      run
+
+      expect(job).to have_received(:run_callbacks).with(:batch)
+    end
+
+    specify do
+      run
+
+      expect(barrier).to have_received(:reset!)
+    end
+  end
+end

data/spec/{handler_spec.rb → wayfarer/handler_spec.rb}

@@ -5,7 +5,5 @@ require "spec_helpers"
 describe Wayfarer::Handler do
   subject { Class.new.include(described_class) }
 
-  it "undefines ::after_batch" do
-    expect(subject).not_to respond_to(:after_batch)
-  end
+  it { is_expected.not_to respond_to(:after_batch) }
 end

data/spec/{integration → wayfarer/integration}/callbacks_spec.rb

@@ -2,10 +2,10 @@
 
 require "spec_helpers"
 
-describe Wayfarer::Callbacks, redis: true do
+describe "Callbacks", :integration, :redis do
   before do
-    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
-    stub_const("DummyHandler", Class.new.include(Wayfarer::Handler))
+    mock_job! :dummy_job
+    mock_handler! :dummy_handler
   end
 
   describe "job callbacks" do
@@ -103,9 +103,8 @@ describe Wayfarer::Callbacks, redis: true do
   end
 
   describe "job callbacks with handler" do
-    before { stub_const("CallbacksFired", []) }
-
     before do
+      stub_const("CallbacksFired", [])
       DummyJob.class_eval do
         route.to [DummyHandler, :index]
 
@@ -191,7 +190,11 @@ describe Wayfarer::Callbacks, redis: true do
     end
 
     specify do
-      DummyJob.new.perform(build(:task, url: test_app_path("alpha")))
+      DummyJob.crawl(test_app_path("alpha"))
+      perform_enqueued_jobs
+
+      assert_performed_jobs 1
+      expect(enqueued_jobs).to be_empty
     end
   end
 end

data/spec/wayfarer/integration/content_type_spec.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Content-Type allow listing", :redis do
+  using RSpec::Parameterized::TableSyntax
+
+  subject(:perform) { DummyJob.new.call(task) }
+
+  before do
+    mock_job! :dummy_job
+  end
+
+  with_them do
+    let(:task) { build(:task, url: url) }
+
+    specify do
+      types = content_types
+
+      DummyJob.class_eval do
+        route.to :index
+
+        content_type types
+
+        def index
+          true
+        end
+      end
+
+      expect(DummyJob.new.perform(task)).to be(expected)
+    end
+  end
+
+  where(:content_types, :url, :expected) do
+    "text/html" | test_app_path("response_header/Content-Type/text/html") | true
+  end
+end