wayfarer 0.4.6 → 0.4.8

data/spec/wayfarer/cli/routing_spec.rb

@@ -0,0 +1,322 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Routing", :cli, :integration do
+  subject(:route) { Wayfarer::CLI.start(["route", "-r", "dummy_job.rb", "DummyJob", url]) }
+
+  before do
+    write_file("dummy_job.rb", contents)
+
+    mock_job! :dummy_job
+  end
+
+  let(:contents) do
+    <<~RUBY
+      class DummyJob < ActiveJob::Base
+        include Wayfarer::Base
+
+        #{routes}
+      end
+    RUBY
+  end
+
+  context "without routes" do
+    let(:url) { "http://#{test_app_host}" }
+    let(:routes) { "" }
+
+    let(:expected_output) do
+      <<~OUTPUT
+        ---
+        routed: false
+        root_route:
+          match: false
+          params: {}
+          children: []
+      OUTPUT
+    end
+
+    specify do
+      expect { route }.to output(expected_output).to_stdout
+    end
+  end
+
+  context "with target route only" do
+    let(:url) { "http://#{test_app_host}" }
+    let(:routes) do
+      <<~RUBY
+        route.to :index
+      RUBY
+    end
+
+    let(:expected_output) do
+      <<~OUTPUT
+        ---
+        routed: true
+        params: {}
+        action: :index
+        root_route:
+          match: true
+          params: {}
+          children:
+          - target_route:
+              action: :index
+              children: []
+      OUTPUT
+    end
+
+    specify do
+      expect { route }.to output(expected_output).to_stdout
+    end
+  end
+
+  context "with a symbol target" do
+    let(:routes) do
+      <<~RUBY
+        route.host #{test_app_hostname.inspect}, to: :index
+      RUBY
+    end
+
+    context "with matching URL" do
+      let(:url) { "http://#{test_app_host}" }
+
+      let(:expected_output) do
+        <<~OUTPUT
+          ---
+          routed: true
+          params: {}
+          action: :index
+          root_route:
+            match: true
+            params: {}
+            children:
+            - route:
+                host:
+                  name: #{test_app_hostname}
+                match: true
+                params: {}
+                children:
+                - target_route:
+                    action: :index
+                    children: []
+        OUTPUT
+      end
+
+      specify do
+        expect { route }.to output(expected_output).to_stdout
+      end
+    end
+
+    context "with mismatching URL" do
+      let(:url) { "http://example.com" }
+
+      let(:expected_output) do
+        <<~OUTPUT
+          ---
+          routed: false
+          root_route:
+            match: true
+            params: {}
+            children:
+            - route:
+                host:
+                  name: #{test_app_hostname}
+                match: false
+                params: {}
+                children: []
+        OUTPUT
+      end
+
+      specify do
+        expect { route }.to output(expected_output).to_stdout
+      end
+    end
+  end
+
+  context "with a symbol target and params" do
+    let(:url) { "http://#{test_app_host}/barqux" }
+    let(:routes) do
+      <<~RUBY
+        route.host #{test_app_hostname.inspect}, to: :index do
+          path ":foobar"
+        end
+      RUBY
+    end
+
+    let(:expected_output) do
+      <<~OUTPUT
+        ---
+        routed: true
+        params:
+          foobar: barqux
+        action: :index
+        root_route:
+          match: true
+          params: {}
+          children:
+          - route:
+              host:
+                name: #{test_app_hostname}
+              match: true
+              params: {}
+              children:
+              - target_route:
+                  action: :index
+                  children:
+                  - route:
+                      path:
+                        pattern: "/:foobar"
+                      match: true
+                      params:
+                        foobar: barqux
+                      children: []
+      OUTPUT
+    end
+
+    specify do
+      expect { route }.to output(expected_output).to_stdout
+    end
+  end
+
+  context "with a custom route, params, and a conditional target" do
+    let(:url) { "http://#{test_app_host}/users/42/edit" }
+    let(:routes) do
+      <<~RUBY
+        route.host #{test_app_hostname.inspect}, path: ":resource" do
+          custom do |root, uri|
+            root.path ":id", path: ":action" do
+              if uri.host == #{test_app_hostname.inspect}
+                root.to :foo
+              end
+            end
+          end
+        end
+      RUBY
+    end
+
+    let(:expected_output) do
+      <<~OUTPUT
+        ---
+        routed: true
+        params:
+          resource: users
+          id: '42'
+          action: edit
+        action: :foo
+        root_route:
+          match: true
+          params: {}
+          children:
+          - route:
+              host:
+                name: #{test_app_hostname}
+              match: true
+              params: {}
+              children:
+              - route:
+                  path:
+                    pattern: "/:resource"
+                  match: true
+                  params:
+                    resource: users
+                  children:
+                  - sub_route:
+                      custom:
+                        custom: Proc
+                      match: true
+                      params:
+                        resource: users
+                        id: '42'
+                        action: edit
+                      children: []
+      OUTPUT
+    end
+
+    specify do
+      expect { route }.to output(expected_output).to_stdout
+    end
+  end
+
+  describe "query parameter matching" do
+    let(:url) { "http://#{test_app_host}?page=10&order=desc&mobile=0&referrer=startpage&foo=bar" }
+    let(:routes) do
+      <<~RUBY
+        route.query({ page: 1..20, order: "desc", mobile: 0, referrer: /start/ }, to: :foo)
+      RUBY
+    end
+
+    let(:expected_output) do
+      <<~OUTPUT
+        ---
+        routed: true
+        params:
+          page: '10'
+          order: desc
+          mobile: '0'
+          referrer: startpage
+        action: :foo
+        root_route:
+          match: true
+          params: {}
+          children:
+          - route:
+              query:
+                page:
+                  min: 1
+                  max: 20
+                order: desc
+                mobile: 0
+                referrer: "/start/"
+              match: true
+              params:
+                page: '10'
+                order: desc
+                mobile: '0'
+                referrer: startpage
+              children:
+              - target_route:
+                  action: :foo
+                  children: []
+      OUTPUT
+    end
+
+    specify do
+      expect { route }.to output(expected_output).to_stdout
+    end
+  end
+
+  context "with URL matcher" do
+    let(:url) { "http://#{test_app_host}" }
+    let(:routes) do
+      <<~RUBY
+        route.url "http://#{test_app_host}", to: :index
+      RUBY
+    end
+
+    let(:expected_output) do
+      <<~OUTPUT
+        ---
+        routed: true
+        params: {}
+        action: :index
+        root_route:
+          match: true
+          params: {}
+          children:
+          - route:
+              url:
+                url: #{url}
+              match: true
+              params: {}
+              children:
+              - target_route:
+                  action: :index
+                  children: []
+      OUTPUT
+    end
+
+    specify do
+      expect { route }.to output(expected_output).to_stdout
+    end
+  end
+end

data/spec/{cli → wayfarer/cli}/version_spec.rb

@@ -2,7 +2,7 @@
 
 require "spec_helpers"
 
-describe Wayfarer::CLI::Runner, cli: true do
+describe Wayfarer::CLI, :cli do # rubocop:disable RSpec/SpecFilePathFormat
   describe "version" do
     it "prints the version" do
       expect {

data/spec/wayfarer/gc_spec.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::GC, :integration, :redis do
+  describe "::run" do
+    subject(:run) { described_class.run(task) }
+
+    let(:job) { instance_spy(Class.new.include(Wayfarer::Base)) }
+    let(:task) { build(:task, :redis_pool, job: job) }
+    let(:barrier) { instance_spy(Wayfarer::Redis::Barrier) }
+
+    before do
+      allow(Wayfarer::Redis::Barrier).to receive(:new).with(task).and_return(barrier)
+    end
+
+    specify do
+      run
+
+      expect(job).to have_received(:run_callbacks).with(:batch)
+    end
+
+    specify do
+      run
+
+      expect(barrier).to have_received(:reset!)
+    end
+  end
+end

data/spec/wayfarer/handler_spec.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Handler do
+  subject { Class.new.include(described_class) }
+
+  it { is_expected.not_to respond_to(:after_batch) }
+end

data/spec/wayfarer/integration/callbacks_spec.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Callbacks", :integration, :redis do
+  before do
+    mock_job! :dummy_job
+    mock_handler! :dummy_handler
+  end
+
+  describe "job callbacks" do
+    before do
+      DummyJob.class_eval do
+        route.path "alpha", to: :alpha
+        route.path "beta", to: :beta
+
+        def callbacks_fired
+          @callbacks_fired ||= []
+        end
+
+        before_fetch do |job|
+          callbacks_fired << [:before_fetch, job]
+        end
+
+        before_fetch if: -> { %i[alpha beta].exclude?(action) } do |job|
+          callbacks_fired << [:before_fetch_except_alpha_beta, job]
+        end
+
+        before_action do |job|
+          callbacks_fired << [:before_action, job]
+        end
+
+        before_action if: -> { action == :alpha } do |job|
+          callbacks_fired << [:before_action_only_alpha, job]
+        end
+
+        before_action if: -> { action == :beta } do |job|
+          callbacks_fired << [:before_action_only_beta, job]
+        end
+
+        before_action if: -> { %i[alpha gamma].include?(action) } do |job|
+          callbacks_fired << [:before_action_only_alpha_gamma, job]
+        end
+
+        before_action if: -> { action != :alpha } do |job|
+          callbacks_fired << [:before_action_except_alpha, job]
+        end
+
+        before_action if: -> { %i[beta gamma].exclude?(action) } do |job|
+          callbacks_fired << [:before_action_except_beta_gamma, job]
+        end
+
+        before_action :callback_a, if: -> { action == :alpha }
+
+        before_action :callback_b
+
+        before_action :callback_c, if: -> { action != :beta }
+
+        def alpha; end
+
+        def beta; end
+
+      private
+
+        def callback_a
+          callbacks_fired << :before_action_callback_a
+        end
+
+        def callback_b
+          callbacks_fired << :before_action_callback_b
+        end
+
+        def callback_c
+          callbacks_fired << :before_action_callback_c
+        end
+      end
+    end
+
+    specify do
+      job = DummyJob.new
+      job.perform(build(:task, url: test_app_path("alpha")))
+      expect(job.callbacks_fired).to eq([
+                                          [:before_fetch, job],
+                                          [:before_action, job],
+                                          [:before_action_only_alpha, job],
+                                          [:before_action_only_alpha_gamma, job],
+                                          [:before_action_except_beta_gamma, job],
+                                          :before_action_callback_a,
+                                          :before_action_callback_b,
+                                          :before_action_callback_c
+                                        ])
+
+      job = DummyJob.new
+      job.perform(build(:task, url: test_app_path("beta")))
+      expect(job.callbacks_fired).to eq([
+                                          [:before_fetch, job],
+                                          [:before_action, job],
+                                          [:before_action_only_beta, job],
+                                          [:before_action_except_alpha, job],
+                                          :before_action_callback_b
+                                        ])
+    end
+  end
+
+  describe "job callbacks with handler" do
+    before do
+      stub_const("CallbacksFired", [])
+      DummyJob.class_eval do
+        route.to [DummyHandler, :index]
+
+        before_fetch do |job|
+          CallbacksFired << [:before_fetch, job.class]
+        end
+
+        around_fetch do |job, block|
+          CallbacksFired << [:around_fetch, job.class]
+
+          block.call
+        end
+
+        after_fetch do |job|
+          CallbacksFired << [:after_fetch, job.class]
+        end
+
+        before_action do |job|
+          CallbacksFired << [:before_action, job.class]
+        end
+
+        around_action do |job, block|
+          CallbacksFired << [:around_action, job.class]
+
+          block.call
+        end
+
+        after_action do |job|
+          CallbacksFired << [:after_action, job.class]
+        end
+      end
+
+      DummyHandler.class_eval do
+        before_action do |handler|
+          CallbacksFired << [:before_action, handler.class]
+        end
+
+        around_action do |handler, block|
+          CallbacksFired << [:around_action, handler.class]
+
+          block.call
+        end
+
+        after_action do |handler|
+          CallbacksFired << [:after_action, handler.class]
+        end
+
+        def index; end
+      end
+    end
+
+    specify do
+      DummyJob.crawl(test_app_path("/"))
+      perform_enqueued_jobs
+
+      expect(CallbacksFired).to eq([
+                                     [:before_fetch, DummyJob],
+                                     [:around_fetch, DummyJob],
+                                     [:after_fetch, DummyJob],
+                                     [:before_action, DummyJob],
+                                     [:around_action, DummyJob],
+                                     [:before_action, DummyHandler],
+                                     [:around_action, DummyHandler],
+                                     [:after_action, DummyHandler],
+                                     [:after_action, DummyJob]
+                                   ])
+    end
+  end
+
+  describe "accessing user agent in before_fetch" do
+    before do
+      DummyJob.class_eval do
+        include RSpec::Matchers
+
+        route.to :index
+
+        before_fetch do
+          expect(user_agent).to be_a(Net::HTTP::Persistent)
+        end
+
+        def index; end
+      end
+    end
+
+    specify do
+      DummyJob.crawl(test_app_path("alpha"))
+      perform_enqueued_jobs
+
+      assert_performed_jobs 1
+      expect(enqueued_jobs).to be_empty
+    end
+  end
+end

data/spec/wayfarer/integration/content_type_spec.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Content-Type allow listing", :redis do
+  using RSpec::Parameterized::TableSyntax
+
+  subject(:perform) { DummyJob.new.call(task) }
+
+  before do
+    mock_job! :dummy_job
+  end
+
+  with_them do
+    let(:task) { build(:task, url: url) }
+
+    specify do
+      types = content_types
+
+      DummyJob.class_eval do
+        route.to :index
+
+        content_type types
+
+        def index
+          true
+        end
+      end
+
+      expect(DummyJob.new.perform(task)).to be(expected)
+    end
+  end
+
+  where(:content_types, :url, :expected) do
+    "text/html" | test_app_path("response_header/Content-Type/text/html") | true
+  end
+end

data/spec/wayfarer/integration/custom_routing_spec.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Custom routing", :redis do
+  let(:task) { build(:task) }
+
+  before do
+    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
+    DummyJob.class_eval do
+      class_attribute :action,
+                      :params
+
+      route.custom do |root, uri, _task|
+        if uri.host == "example.com"
+          root.path ":segment" do
+            path "users", path: "/:id", to: :foo
+          end
+        else
+          root.to :bar
+        end
+      end
+
+      after_action do
+        self.class.params = params
+      end
+
+      def foo
+        self.class.action = :foo
+      end
+
+      def bar
+        self.class.action = :bar
+      end
+    end
+  end
+
+  specify do
+    DummyJob.crawl("https://example.com/foobar/users/42")
+    perform_enqueued_jobs
+    expect(DummyJob.action).to be(:foo)
+    assert_performed_jobs 1
+    expect(enqueued_jobs).to be_empty
+
+    DummyJob.crawl(test_app_path("/"))
+    perform_enqueued_jobs
+    expect(DummyJob.action).to be(:bar)
+    assert_performed_jobs 2
+    expect(enqueued_jobs).to be_empty
+  end
+end

data/spec/wayfarer/integration/gc_spec.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Garbage collection", :redis do
+  subject(:perform) do
+    DummyJob.perform_later(task)
+    perform_enqueued_jobs
+  end
+
+  let(:task) { build(:task, :redis_pool) }
+  let(:counter) { build(:counter, task: task) }
+  let(:barrier) { build(:barrier, task: task) }
+
+  before do
+    mock_job! :dummy_job
+
+    DummyJob.class_eval do
+      route.to :index
+
+      def index; end
+    end
+    redis.set(counter.redis_key, 0)
+    redis.hset(barrier.redis_key, "", "")
+  end
+
+  it "resets counter" do
+    expect { perform }.to change { redis.exists?(counter.redis_key) }.from(true).to(false)
+
+    assert_performed_jobs 1
+    expect(enqueued_jobs).to be_empty
+  end
+
+  it "resets barrier" do
+    expect { perform }.to change { redis.exists?(barrier.redis_key) }.from(true).to(false)
+
+    assert_performed_jobs 1
+    expect(enqueued_jobs).to be_empty
+  end
+end