wayfarer 0.4.7 → 0.4.9

data/spec/wayfarer/middleware/dedup_spec.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative "../../spec_helpers"
+
+RSpec.describe Wayfarer::Middleware::Dedup, :redis do
+  subject(:middleware) { described_class.new }
+
+  let(:job) { build(:job, executions: 1) }
+  let(:controller) { job }
+  let(:task) { build(:task, :normalized, :barrier, job: job, controller: controller, url: "http://example.com/") }
+  let(:barrier) { task[:barrier] }
+
+  context "with unseen URL" do
+    specify do
+      expect { |block| middleware.call(task, &block) }.to yield_control
+    end
+  end
+
+  context "with seen URL" do
+    before { barrier.check!(task[:uri].to_s) }
+
+    specify do
+      expect { |block| middleware.call(task, &block) }.not_to yield_control
+    end
+
+    context "when task is rerouted" do
+      let(:controller) { Class.new.include(Wayfarer::Handler) }
+
+      specify do
+        expect { |block| middleware.call(task, &block) }.to yield_control
+      end
+    end
+
+    context "with retry" do
+      let(:job) { build(:job, executions: 3) }
+
+      specify do
+        expect { |block| middleware.call(task, &block) }.to yield_control
+      end
+    end
+  end
+
+  describe "cache keys" do
+    before do
+      task[:barrier] = spy
+    end
+
+    specify do
+      middleware.call(task)
+
+      expect(task[:barrier]).to have_received(:check!).with(task[:uri].to_s)
+    end
+
+    context "with custom key" do
+      before do
+        Wayfarer.config[:deduplication][:key] = ->(task) { Digest::MD5.base64digest(task[:uri]) }
+      end
+
+      specify do
+        middleware.call(task)
+
+        expect(task[:barrier]).to have_received(:check!).with("qbnwQzbOAYGgjndOARE7MQ==")
+      end
+    end
+  end
+end

data/spec/wayfarer/middleware/normalize_spec.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::Normalize do
+  subject(:middleware) { described_class.new }
+
+  let(:task) { build(:task, :uri, url: "http://example.com/") }
+
+  context "with assigned URI" do
+    specify do
+      expect { |spy| middleware.call(task, &spy) }.to yield_control
+    end
+
+    specify do
+      expect { middleware.call(task) }.to(change { task[:uri] })
+    end
+  end
+
+  context "with invalid URL" do
+    before do
+      allow(Wayfarer::URI::Normalization)
+        .to receive(:canonical!)
+        .with(task[:uri])
+        .and_raise(Wayfarer::URI::Normalization::InvalidURIError)
+    end
+
+    specify do
+      expect { |spy| middleware.call(task, &spy) }.not_to yield_control
+    end
+  end
+end

data/spec/{middleware → wayfarer/middleware}/router_spec.rb

@@ -3,8 +3,9 @@
 require "spec_helpers"
 
 describe Wayfarer::Middleware::Router do
-  let(:task) { build(:task) }
-  subject { described_class.new }
+  subject(:middleware) { described_class.new }
+
+  let(:task) { build(:task, :uri) }
 
   describe "#call" do
     let(:controller) do
@@ -13,10 +14,9 @@ describe Wayfarer::Middleware::Router do
 
     before do
       allow(controller.class.route).to receive(:invoke)
-        .with(Addressable::URI.parse(task.url))
+        .with(task)
         .and_return(result)
       task[:controller] = controller
-      task[:uri] = Addressable::URI.parse(task.url)
     end
 
     context "with matching route" do
@@ -28,7 +28,7 @@ describe Wayfarer::Middleware::Router do
 
       it "assigns the action" do
         expect {
-          subject.call(task)
+          middleware.call(task)
         }.to change { task[:action] }.to(action)
       end
 
@@ -36,29 +36,29 @@ describe Wayfarer::Middleware::Router do
         task[:params] = ActiveSupport::HashWithIndifferentAccess.new("bar" => "qux")
 
         expect {
-          subject.call(task)
+          middleware.call(task)
         }.to change { task[:params] }.to("foo" => "bar", "bar" => "qux")
       end
 
       specify do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
       end
 
       specify do
+        # rubocop:disable RSpec/MessageSpies
         expect(Wayfarer::Logging.logger).to receive(:add).with(Logger::INFO, kind_of(String))
+        # rubocop:enable RSpec/MessageSpies
 
-        subject.call(task)
+        middleware.call(task)
       end
     end
 
     context "without matching route" do
-      let(:result) do
-        Wayfarer::Routing::Result::Mismatch.new
-      end
+      let(:result) { Wayfarer::Routing::Result::Mismatch.instance }
 
       it "does not assign the action" do
         expect {
-          subject.call(task)
+          middleware.call(task)
         }.not_to(change { task[:action] })
       end
 
@@ -66,34 +66,32 @@ describe Wayfarer::Middleware::Router do
         task[:params] = ActiveSupport::HashWithIndifferentAccess.new("bar" => "qux")
 
         expect {
-          subject.call(task)
+          middleware.call(task)
         }.not_to(change { task[:params] })
       end
 
       specify do
-        expect { |spy| subject.call(task, &spy) }.not_to yield_control
+        expect { |spy| middleware.call(task, &spy) }.not_to yield_control
       end
 
       specify do
+        # rubocop:disable RSpec/MessageSpies
         expect(Wayfarer::Logging.logger).to receive(:add).with(Logger::INFO, kind_of(String))
+        # rubocop:enable RSpec/MessageSpies
 
-        subject.call(task)
+        middleware.call(task)
       end
     end
   end
 
   describe described_class::API do
-    subject(:controller) do
-      Struct.new(:task).include(described_class).new(task)
-    end
+    subject(:controller) { Struct.new(:task).include(described_class).new(task) }
 
     describe "::route" do
       it "returns a root route" do
         expect(controller.class.route).to be_a(Wayfarer::Routing::RootRoute)
       end
-    end
 
-    describe "::route" do
       it "adds a routing block" do
         expect {
           controller.class.route.to(:index)

data/spec/{middleware → wayfarer/middleware}/stage_spec.rb

@@ -3,38 +3,39 @@
 require "spec_helpers"
 
 describe Wayfarer::Middleware::Stage do
+  subject(:middleware) { described_class.new }
+
   let(:task) { build(:task) }
-  subject { described_class.new }
 
   describe "#call" do
     it "assigns an empty set" do
-      subject.call(task)
+      middleware.call(task)
       expect(task[:staged_urls]).to eq(SortedSet.new)
     end
 
     it "yields" do
-      expect { |spy| subject.call(task, &spy) }.to yield_control
+      expect { |spy| middleware.call(task, &spy) }.to yield_control
     end
 
     it "enqueues tasks" do
+      job = double(crawl: nil) # rubocop:disable RSpec/VerifiedDoubles
       urls = [test_app_path("/alpha"), test_app_path("/beta")]
 
-      spy.tap do |job|
-        expect(job).to receive(:crawl).with(urls.first, batch: task.batch).ordered
-        expect(job).to receive(:crawl).with(urls.second, batch: task.batch).ordered
-        task[:job] = double(class: job)
-      end
+      task[:job] = instance_spy(Object, class: job)
 
-      subject.call(task) do
+      middleware.call(task) do
         task[:staged_urls] = SortedSet.new(urls)
       end
+
+      expect(job).to have_received(:crawl).with(urls.first, batch: task.batch).ordered
+      expect(job).to have_received(:crawl).with(urls.second, batch: task.batch).ordered
     end
 
     it "resets staged URLs" do
       task[:staged_urls] = SortedSet.new([test_app_path("/foo")])
 
       expect {
-        subject.call(task)
+        middleware.call(task)
       }.to change { task[:staged_urls].count }.to(0)
     end
   end

data/spec/wayfarer/middleware/uri_parser_spec.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::UriParser do
+  subject(:middleware) { described_class.new }
+
+  let(:task) { build(:task) }
+
+  it "parses URLs" do
+    expect { middleware.call(task) }.to change { task[:uri] }.to(Addressable::URI.parse(task.url))
+  end
+
+  specify do
+    expect { |spy| middleware.call(task, &spy) }.to yield_control
+  end
+
+  context "with already parsed URI" do
+    before do
+      task[:uri] = Addressable::URI.parse("http://example.com")
+    end
+
+    specify do
+      expect { |spy| middleware.call(task, &spy) }.to yield_control
+    end
+
+    specify do
+      expect { middleware.call(task) }.not_to(change { task[:uri] })
+    end
+  end
+
+  context "with invalid URL" do
+    let(:task) { build(:task, url: "ht%0atp://localhost/") }
+
+    specify do
+      expect { |spy| middleware.call(task, &spy) }.not_to yield_control
+    end
+
+    specify do
+      expect(Wayfarer::Logging.logger)
+        .to receive(:add) # rubocop:disable RSpec/MessageSpies
+        .with(Logger::INFO, "Not processing invalid URL (Invalid scheme format: 'ht%0atp')")
+
+      middleware.call(task)
+    end
+  end
+
+  describe described_class::API do
+    subject(:controller) do
+      Struct.new(:task).include(described_class).new(task)
+    end
+
+    describe "#uri" do
+      let(:uri) { Addressable::URI.parse(task.url) }
+
+      before { task[:uri] = uri }
+
+      specify do
+        expect(controller.uri).to be(uri)
+      end
+    end
+  end
+end

data/spec/{middleware → wayfarer/middleware}/user_agent_spec.rb

@@ -3,17 +3,19 @@
 require "spec_helpers"
 
 describe Wayfarer::Middleware::UserAgent do
-  let(:task) { build(:task) }
-  let(:page) { Object.new }
+  subject(:middleware) { described_class.new }
+
+  let(:task) { build(:task, :staged_urls) }
+  let(:page) { instance_double(Wayfarer::Page) }
   let(:agent) { Object.new }
   let(:result) { Wayfarer::Networking::Result::Success.new(page) }
   let(:context) do
-    double(instance: agent).tap do |context|
+    instance_spy(Wayfarer::Networking::Context, instance: agent).tap do |context|
       allow(context).to receive(:fetch).with(task.url).and_return(result)
     end
   end
 
-  subject { described_class.new }
+  before { task[:controller] = controller }
 
   describe "#call" do
     let(:controller) do
@@ -25,67 +27,66 @@ describe Wayfarer::Middleware::UserAgent do
 
       spy.tap do |pool|
         allow(pool).to receive(:with).and_yield(context)
-        allow(subject).to receive(:pool).and_return(pool)
+        allow(middleware).to receive(:pool).and_return(pool) # rubocop:disable RSpec/SubjectStub
       end
-
-      task[:staged_urls] = SortedSet.new
-      task[:controller] = controller
     end
 
     context "with page assigned" do
       before { task[:page] = page }
 
-      it "does not alter the page" do
+      specify do
         expect {
-          subject.call(task)
+          middleware.call(task)
         }.not_to(change { task[:page] })
       end
 
-      it "yields" do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
       end
     end
 
-    it "runs callbacks" do
-      expect(controller).to receive(:run_callbacks).with(:fetch)
-      subject.call(task)
+    specify do
+      middleware.call(task)
+
+      expect(controller).to have_received(:run_callbacks).with(:fetch)
     end
 
-    it "fetches the URL" do
-      expect(context).to receive(:fetch).with(task.url)
-      subject.call(task)
+    specify do
+      middleware.call(task)
+
+      expect(context).to have_received(:fetch).with(task.url)
     end
 
     context "with Redirect" do
       let(:redirect_url) { test_app_path("/foobar") }
       let(:result) { Wayfarer::Networking::Result::Redirect.new(page) }
 
-      it "stages the redirect URL" do
+      specify do
         expect {
-          subject.call(task)
+          middleware.call(task)
         }.to change { task[:staged_urls].count }.by(1)
       end
 
-      it "does not yield" do
-        expect { |spy| subject.call(task, &spy) }.not_to yield_control
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.not_to yield_control
       end
     end
 
     context "with Success" do
-      it "assigns the context" do
+      specify do
         expect {
-          subject.call(task)
+          middleware.call(task)
         }.to change { task[:context] }.to(context)
       end
 
-      it "assigns the page" do
+      specify do
         expect {
-          subject.call(task)
+          middleware.call(task)
         }.to change { task[:page] }.to(result.page)
       end
 
-      it "yields" do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
       end
     end
   end
@@ -98,7 +99,7 @@ describe Wayfarer::Middleware::UserAgent do
     describe "#user_agent" do
       before { task[:context] = context }
 
-      it "returns the agent" do
+      specify do
         expect(controller.user_agent).to be(context.instance)
       end
     end
@@ -106,7 +107,7 @@ describe Wayfarer::Middleware::UserAgent do
     describe "#page" do
       before { task[:page] = page }
 
-      it "returns the page" do
+      specify do
         expect(controller.page).to be(task[:page])
       end
 
@@ -139,8 +140,9 @@ describe Wayfarer::Middleware::UserAgent do
     end
 
     describe "#fetch" do
+      subject(:page) { controller.fetch(url) }
+
       let(:url) { test_app_path("/redirect?times=3") }
-      subject { controller.fetch(url) }
 
       it { is_expected.to be_a(Wayfarer::Page) }
 
@@ -148,7 +150,7 @@ describe Wayfarer::Middleware::UserAgent do
         let(:url) { test_app_path("/redirect?times=4") }
 
         specify do
-          expect { subject }.to raise_error(Wayfarer::Networking::Follow::RedirectsExhaustedError)
+          expect { page }.to raise_error(Wayfarer::Networking::Follow::RedirectsExhaustedError)
         end
       end
     end

data/spec/wayfarer/networking/capybara_spec.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+require_relative "strategy"
+
+describe Wayfarer::Networking::Capybara, :ferrum do
+  it_behaves_like "Network strategy", strategy: described_class,
+                                      browser: true,
+                                      request_headers: false,
+                                      response_headers: true,
+                                      status_code: true,
+                                      raises_on_error_response: true # uses Ferrum
+end

data/spec/{networking → wayfarer/networking}/context_spec.rb

@@ -3,62 +3,70 @@
 require "spec_helpers"
 
 describe Wayfarer::Networking::Context do
-  let(:instance) { Object.new }
-  let(:strategy) { spy }
-  let(:renewing_error) { Class.new(StandardError) }
-  subject(:context) { Wayfarer::Networking::Context.new(strategy) }
+  subject(:context) { described_class.new(strategy) }
 
-  before do
-    allow(strategy).to receive(:renew_on).and_return([renewing_error])
-    allow(strategy).to receive(:create).and_return(instance)
+  let(:instance) { Object.new }
+  let(:class_double) { double(renew_on: [renewing_error]) } # # rubocop:disable RSpec/VerifiedDoubles
+  let(:strategy) do
+    double( # rubocop:disable RSpec/VerifiedDoubles
+      create: instance,
+      destroy: nil,
+      class: class_double,
+      fetch: nil,
+      live: nil
+    )
   end
 
-  describe "#new" do
-    it "assigns the strategy" do
-      expect(context.strategy).to be(strategy)
-    end
+  let(:renewing_error) { Class.new(StandardError) }
 
-    it "assigns the instance" do
-      expect(context.strategy).to be(strategy)
-    end
+  before do
+    allow(strategy).to receive(:destroy)
   end
 
   describe "#fetch" do
     let(:url) { Object.new }
 
-    it "fetches" do
-      expect(strategy).to receive(:fetch).with(instance, url)
+    specify do
       context.fetch(url)
+
+      expect(strategy).to have_received(:fetch).with(instance, url)
     end
 
-    context "with renewing exception raised" do
+    context "with renewing exception" do
       before do
         allow(strategy).to receive(:fetch).and_raise(renewing_error)
       end
 
       it "renews and reraises" do
-        expect(context).to receive(:renew)
-
         expect {
           context.fetch(url)
         }.to raise_error(renewing_error)
+
+        expect(strategy).to have_received(:destroy).with(instance)
       end
     end
 
-    context "with configured renewing exception raised" do
+    context "with renewing exception raised" do
       let(:other_error) { Class.new(StandardError) }
 
-      before do
+      around do |example|
+        original_renew_on = Wayfarer.config[:network][:renew_on]
         Wayfarer.config[:network][:renew_on] = [other_error]
+        example.run
+        Wayfarer.config[:network][:renew_on] = original_renew_on
+      end
+
+      before do
+        allow(class_double).to receive(:renew_on).and_return([renewing_error, other_error])
         allow(strategy).to receive(:fetch).and_raise(other_error)
       end
 
       it "renews and reraises" do
-        expect(context).to receive(:renew)
-
         expect {
           context.fetch(url)
         }.to raise_error(other_error)
+
+        expect(strategy).to have_received(:destroy).with(instance)
       end
     end
 
@@ -67,12 +75,12 @@ describe Wayfarer::Networking::Context do
         allow(strategy).to receive(:fetch).and_raise(StandardError)
       end
 
-      it "reraises" do
-        expect(context).not_to receive(:renew)
-
+      it "reraises without renewing" do
         expect {
           context.fetch(url)
         }.to raise_error(StandardError)
+
+        expect(strategy).not_to have_received(:destroy)
       end
     end
   end
@@ -81,28 +89,29 @@ describe Wayfarer::Networking::Context do
     let(:page) { Object.new }
 
     it "calls live" do
-      expect(strategy).to receive(:live).with(instance)
       context.live
+
+      expect(strategy).to have_received(:live).with(instance)
     end
 
-    context "with renewing exception raised" do
+    context "with renewing exception" do
       before do
         allow(strategy).to receive(:live).and_raise(renewing_error)
       end
 
       it "renews and reraises" do
-        expect(context).to receive(:renew)
-
         expect {
           context.live
         }.to raise_error(renewing_error)
+
+        expect(strategy).to have_received(:destroy).with(instance)
       end
 
       context "when renewing raises" do
         let(:other_error) { Class.new(StandardError) }
 
         before do
-          allow(context).to receive(:renew).and_raise(other_error)
+          allow(strategy).to receive(:destroy).and_raise(other_error)
         end
 
         it "reraises the original exception" do
@@ -118,27 +127,27 @@ describe Wayfarer::Networking::Context do
         allow(strategy).to receive(:live).and_raise(StandardError)
       end
 
-      it "reraises" do
-        expect(context).not_to receive(:renew)
-
+      it "reraises without renewing" do
         expect {
           context.live(page)
         }.to raise_error(StandardError)
+
+        expect(strategy).not_to have_received(:destroy)
       end
     end
   end
 
   describe "#renew" do
-    it "destroys" do
-      expect(strategy).to receive(:destroy).with(instance)
+    specify do
       context.renew
+      expect(strategy).to have_received(:destroy).with(instance)
     end
   end
 
   describe "#instance" do
-    it "creates an instance" do
-      expect(strategy).to receive(:create).and_return(instance)
+    specify do
       expect(context.instance).to be(instance)
+      expect(strategy).to have_received(:create)
     end
   end
 end

data/spec/wayfarer/networking/ferrum_spec.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+require_relative "strategy"
+
+describe Wayfarer::Networking::Ferrum, :ferrum do
+  it_behaves_like "Network strategy", strategy: described_class,
+                                      browser: true,
+                                      request_headers: true,
+                                      response_headers: true,
+                                      status_code: true,
+                                      raises_on_error_response: true
+end