wayfarer 0.4.7 → 0.4.8

data/spec/{middleware → wayfarer/middleware}/controller_spec.rb

@@ -3,20 +3,18 @@
 require "spec_helpers"
 
 describe Wayfarer::Middleware::Controller do
+  subject(:controller) { Class.new.include(described_class) }
+
   let(:task) { build(:task) }
 
   let(:middleware) do
-    Class.new do
-      extend Wayfarer::Middleware::Base
-
-      module API; end
-
-      def call(task); end
+    Class.new.tap do |klass|
+      klass.extend(Wayfarer::Middleware::Base)
+      klass.define_method(:call, &->(_) {})
+      klass.const_set("API", Module.new)
     end
   end
 
-  subject(:controller) { Class.new.include(described_class) }
-
   describe "::use" do
     it "adds middleware" do
       expect {
@@ -28,7 +26,7 @@ describe Wayfarer::Middleware::Controller do
       controller.use(middleware)
       expect(controller.chain.middlewares.last).to be_a(Wayfarer::Middleware::Lazy)
       expect(controller.chain.middlewares.last.klass).to be(middleware)
-      expect_any_instance_of(middleware).to receive(:call).with(task)
+      expect_any_instance_of(middleware).to receive(:call).with(task) # rubocop:disable RSpec/AnyInstance
       controller.new.call(task)
     end
   end
@@ -42,19 +40,19 @@ describe Wayfarer::Middleware::Controller do
   end
 
   describe "#call" do
-    subject { controller.new }
+    let(:instance) { controller.new }
 
     it "assigns the task" do
       expect {
-        subject.call(task)
-      }.to change { subject.task }.to(task)
+        instance.call(task)
+      }.to change(instance, :task).to(task)
     end
 
     context "without job assigned" do
       it "assigns itself" do
         expect {
-          subject.call(task)
-        }.to change { task[:job] }.to(subject)
+          instance.call(task)
+        }.to change { task[:job] }.to(instance)
       end
     end
 
@@ -63,24 +61,28 @@ describe Wayfarer::Middleware::Controller do
 
       it "does not override the job" do
         expect {
-          subject.call(task)
+          instance.call(task)
         }.not_to(change { task[:job] })
       end
     end
 
     it "assigns itself as controller" do
       expect {
-        subject.call(task)
-      }.to change { task[:controller] }.to(subject)
+        instance.call(task)
+      }.to change { task[:controller] }.to(instance)
     end
 
     it "calls the chain" do
-      expect(subject.class.chain).to receive(:call)
-      subject.call(task)
+      chain = instance.class.chain
+      allow(chain).to receive(:call)
+
+      instance.call(task)
+
+      expect(chain).to have_received(:call)
     end
 
     it "yields" do
-      expect { |spy| subject.call(task, &spy) }.to yield_control
+      expect { |spy| instance.call(task, &spy) }.to yield_control
     end
   end
 end

data/spec/wayfarer/middleware/dedup_spec.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative "../../spec_helpers"
+
+RSpec.describe Wayfarer::Middleware::Dedup, :redis do
+  subject(:middleware) { described_class.new }
+
+  let(:job) { build(:job, executions: 1) }
+  let(:controller) { job }
+  let(:task) { build(:task, :normalized, :barrier, job: job, controller: controller, url: "http://example.com/") }
+  let(:barrier) { task[:barrier] }
+
+  context "with unseen URL" do
+    specify do
+      expect { |block| middleware.call(task, &block) }.to yield_control
+    end
+  end
+
+  context "with seen URL" do
+    before { barrier.check!(task[:uri].to_s) }
+
+    specify do
+      expect { |block| middleware.call(task, &block) }.not_to yield_control
+    end
+
+    context "when task is rerouted" do
+      let(:controller) { Class.new.include(Wayfarer::Handler) }
+
+      specify do
+        expect { |block| middleware.call(task, &block) }.to yield_control
+      end
+    end
+
+    context "with retry" do
+      let(:job) { build(:job, executions: 3) }
+
+      specify do
+        expect { |block| middleware.call(task, &block) }.to yield_control
+      end
+    end
+  end
+
+  describe "cache keys" do
+    before do
+      task[:barrier] = spy
+    end
+
+    specify do
+      middleware.call(task)
+
+      expect(task[:barrier]).to have_received(:check!).with(task[:uri].to_s)
+    end
+
+    context "with custom key" do
+      before do
+        Wayfarer.config[:deduplication][:key] = ->(task) { Digest::MD5.base64digest(task[:uri]) }
+      end
+
+      specify do
+        middleware.call(task)
+
+        expect(task[:barrier]).to have_received(:check!).with("qbnwQzbOAYGgjndOARE7MQ==")
+      end
+    end
+  end
+end

data/spec/wayfarer/middleware/normalize_spec.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::Normalize do
+  subject(:middleware) { described_class.new }
+
+  let(:task) { build(:task, :uri, url: "http://example.com/") }
+
+  context "with assigned URI" do
+    specify do
+      expect { |spy| middleware.call(task, &spy) }.to yield_control
+    end
+
+    specify do
+      expect { middleware.call(task) }.to(change { task[:uri] })
+    end
+  end
+
+  context "with invalid URL" do
+    before do
+      allow(Wayfarer::URI::Normalization)
+        .to receive(:canonical!)
+        .with(task[:uri])
+        .and_raise(Wayfarer::URI::Normalization::InvalidURIError)
+    end
+
+    specify do
+      expect { |spy| middleware.call(task, &spy) }.not_to yield_control
+    end
+  end
+end

data/spec/{middleware → wayfarer/middleware}/router_spec.rb

@@ -3,8 +3,9 @@
 require "spec_helpers"
 
 describe Wayfarer::Middleware::Router do
-  let(:task) { build(:task) }
-  subject { described_class.new }
+  subject(:middleware) { described_class.new }
+
+  let(:task) { build(:task, :uri) }
 
   describe "#call" do
     let(:controller) do
@@ -13,10 +14,9 @@ describe Wayfarer::Middleware::Router do
 
     before do
       allow(controller.class.route).to receive(:invoke)
-        .with(Addressable::URI.parse(task.url))
+        .with(task)
         .and_return(result)
       task[:controller] = controller
-      task[:uri] = Addressable::URI.parse(task.url)
     end
 
     context "with matching route" do
@@ -28,7 +28,7 @@ describe Wayfarer::Middleware::Router do
 
       it "assigns the action" do
         expect {
-          subject.call(task)
+          middleware.call(task)
         }.to change { task[:action] }.to(action)
       end
 
@@ -36,29 +36,29 @@ describe Wayfarer::Middleware::Router do
         task[:params] = ActiveSupport::HashWithIndifferentAccess.new("bar" => "qux")
 
         expect {
-          subject.call(task)
+          middleware.call(task)
         }.to change { task[:params] }.to("foo" => "bar", "bar" => "qux")
       end
 
       specify do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
       end
 
       specify do
+        # rubocop:disable RSpec/MessageSpies
         expect(Wayfarer::Logging.logger).to receive(:add).with(Logger::INFO, kind_of(String))
+        # rubocop:enable RSpec/MessageSpies
 
-        subject.call(task)
+        middleware.call(task)
       end
     end
 
     context "without matching route" do
-      let(:result) do
-        Wayfarer::Routing::Result::Mismatch.new
-      end
+      let(:result) { Wayfarer::Routing::Result::Mismatch.instance }
 
       it "does not assign the action" do
         expect {
-          subject.call(task)
+          middleware.call(task)
         }.not_to(change { task[:action] })
       end
 
@@ -66,34 +66,32 @@ describe Wayfarer::Middleware::Router do
         task[:params] = ActiveSupport::HashWithIndifferentAccess.new("bar" => "qux")
 
         expect {
-          subject.call(task)
+          middleware.call(task)
         }.not_to(change { task[:params] })
       end
 
       specify do
-        expect { |spy| subject.call(task, &spy) }.not_to yield_control
+        expect { |spy| middleware.call(task, &spy) }.not_to yield_control
       end
 
       specify do
+        # rubocop:disable RSpec/MessageSpies
         expect(Wayfarer::Logging.logger).to receive(:add).with(Logger::INFO, kind_of(String))
+        # rubocop:enable RSpec/MessageSpies
 
-        subject.call(task)
+        middleware.call(task)
       end
     end
   end
 
   describe described_class::API do
-    subject(:controller) do
-      Struct.new(:task).include(described_class).new(task)
-    end
+    subject(:controller) { Struct.new(:task).include(described_class).new(task) }
 
     describe "::route" do
       it "returns a root route" do
         expect(controller.class.route).to be_a(Wayfarer::Routing::RootRoute)
       end
-    end
 
-    describe "::route" do
       it "adds a routing block" do
         expect {
           controller.class.route.to(:index)

data/spec/{middleware → wayfarer/middleware}/stage_spec.rb

@@ -3,38 +3,39 @@
 require "spec_helpers"
 
 describe Wayfarer::Middleware::Stage do
+  subject(:middleware) { described_class.new }
+
   let(:task) { build(:task) }
-  subject { described_class.new }
 
   describe "#call" do
     it "assigns an empty set" do
-      subject.call(task)
+      middleware.call(task)
       expect(task[:staged_urls]).to eq(SortedSet.new)
     end
 
     it "yields" do
-      expect { |spy| subject.call(task, &spy) }.to yield_control
+      expect { |spy| middleware.call(task, &spy) }.to yield_control
     end
 
     it "enqueues tasks" do
+      job = double(crawl: nil) # rubocop:disable RSpec/VerifiedDoubles
       urls = [test_app_path("/alpha"), test_app_path("/beta")]
 
-      spy.tap do |job|
-        expect(job).to receive(:crawl).with(urls.first, batch: task.batch).ordered
-        expect(job).to receive(:crawl).with(urls.second, batch: task.batch).ordered
-        task[:job] = double(class: job)
-      end
+      task[:job] = instance_spy(Object, class: job)
 
-      subject.call(task) do
+      middleware.call(task) do
         task[:staged_urls] = SortedSet.new(urls)
       end
+
+      expect(job).to have_received(:crawl).with(urls.first, batch: task.batch).ordered
+      expect(job).to have_received(:crawl).with(urls.second, batch: task.batch).ordered
     end
 
     it "resets staged URLs" do
       task[:staged_urls] = SortedSet.new([test_app_path("/foo")])
 
       expect {
-        subject.call(task)
+        middleware.call(task)
       }.to change { task[:staged_urls].count }.to(0)
     end
   end

data/spec/wayfarer/middleware/uri_parser_spec.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::UriParser do
+  subject(:middleware) { described_class.new }
+
+  let(:task) { build(:task) }
+
+  it "parses URLs" do
+    expect { middleware.call(task) }.to change { task[:uri] }.to(Addressable::URI.parse(task.url))
+  end
+
+  specify do
+    expect { |spy| middleware.call(task, &spy) }.to yield_control
+  end
+
+  context "with already parsed URI" do
+    before do
+      task[:uri] = Addressable::URI.parse("http://example.com")
+    end
+
+    specify do
+      expect { |spy| middleware.call(task, &spy) }.to yield_control
+    end
+
+    specify do
+      expect { middleware.call(task) }.not_to(change { task[:uri] })
+    end
+  end
+
+  context "with invalid URL" do
+    let(:task) { build(:task, url: "ht%0atp://localhost/") }
+
+    specify do
+      expect { |spy| middleware.call(task, &spy) }.not_to yield_control
+    end
+
+    specify do
+      expect(Wayfarer::Logging.logger)
+        .to receive(:add) # rubocop:disable RSpec/MessageSpies
+        .with(Logger::INFO, "Not processing invalid URL (Invalid scheme format: 'ht%0atp')")
+
+      middleware.call(task)
+    end
+  end
+
+  describe described_class::API do
+    subject(:controller) do
+      Struct.new(:task).include(described_class).new(task)
+    end
+
+    describe "#uri" do
+      let(:uri) { Addressable::URI.parse(task.url) }
+
+      before { task[:uri] = uri }
+
+      specify do
+        expect(controller.uri).to be(uri)
+      end
+    end
+  end
+end

data/spec/{middleware → wayfarer/middleware}/user_agent_spec.rb

@@ -3,17 +3,19 @@
 require "spec_helpers"
 
 describe Wayfarer::Middleware::UserAgent do
-  let(:task) { build(:task) }
-  let(:page) { Object.new }
+  subject(:middleware) { described_class.new }
+
+  let(:task) { build(:task, :staged_urls) }
+  let(:page) { instance_double(Wayfarer::Page) }
   let(:agent) { Object.new }
   let(:result) { Wayfarer::Networking::Result::Success.new(page) }
   let(:context) do
-    double(instance: agent).tap do |context|
+    instance_spy(Wayfarer::Networking::Context, instance: agent).tap do |context|
       allow(context).to receive(:fetch).with(task.url).and_return(result)
     end
   end
 
-  subject { described_class.new }
+  before { task[:controller] = controller }
 
   describe "#call" do
     let(:controller) do
@@ -25,67 +27,66 @@ describe Wayfarer::Middleware::UserAgent do
 
       spy.tap do |pool|
         allow(pool).to receive(:with).and_yield(context)
-        allow(subject).to receive(:pool).and_return(pool)
+        allow(middleware).to receive(:pool).and_return(pool) # rubocop:disable RSpec/SubjectStub
       end
-
-      task[:staged_urls] = SortedSet.new
-      task[:controller] = controller
     end
 
     context "with page assigned" do
       before { task[:page] = page }
 
-      it "does not alter the page" do
+      specify do
         expect {
-          subject.call(task)
+          middleware.call(task)
         }.not_to(change { task[:page] })
       end
 
-      it "yields" do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
       end
     end
 
-    it "runs callbacks" do
-      expect(controller).to receive(:run_callbacks).with(:fetch)
-      subject.call(task)
+    specify do
+      middleware.call(task)
+
+      expect(controller).to have_received(:run_callbacks).with(:fetch)
     end
 
-    it "fetches the URL" do
-      expect(context).to receive(:fetch).with(task.url)
-      subject.call(task)
+    specify do
+      middleware.call(task)
+
+      expect(context).to have_received(:fetch).with(task.url)
     end
 
     context "with Redirect" do
       let(:redirect_url) { test_app_path("/foobar") }
       let(:result) { Wayfarer::Networking::Result::Redirect.new(page) }
 
-      it "stages the redirect URL" do
+      specify do
         expect {
-          subject.call(task)
+          middleware.call(task)
         }.to change { task[:staged_urls].count }.by(1)
       end
 
-      it "does not yield" do
-        expect { |spy| subject.call(task, &spy) }.not_to yield_control
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.not_to yield_control
       end
     end
 
     context "with Success" do
-      it "assigns the context" do
+      specify do
         expect {
-          subject.call(task)
+          middleware.call(task)
         }.to change { task[:context] }.to(context)
       end
 
-      it "assigns the page" do
+      specify do
         expect {
-          subject.call(task)
+          middleware.call(task)
         }.to change { task[:page] }.to(result.page)
       end
 
-      it "yields" do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
       end
     end
   end
@@ -98,7 +99,7 @@ describe Wayfarer::Middleware::UserAgent do
     describe "#user_agent" do
       before { task[:context] = context }
 
-      it "returns the agent" do
+      specify do
         expect(controller.user_agent).to be(context.instance)
       end
     end
@@ -106,7 +107,7 @@ describe Wayfarer::Middleware::UserAgent do
     describe "#page" do
       before { task[:page] = page }
 
-      it "returns the page" do
+      specify do
         expect(controller.page).to be(task[:page])
       end
 
@@ -139,8 +140,9 @@ describe Wayfarer::Middleware::UserAgent do
     end
 
     describe "#fetch" do
+      subject(:page) { controller.fetch(url) }
+
       let(:url) { test_app_path("/redirect?times=3") }
-      subject { controller.fetch(url) }
 
       it { is_expected.to be_a(Wayfarer::Page) }
 
@@ -148,7 +150,7 @@ describe Wayfarer::Middleware::UserAgent do
         let(:url) { test_app_path("/redirect?times=4") }
 
         specify do
-          expect { subject }.to raise_error(Wayfarer::Networking::Follow::RedirectsExhaustedError)
+          expect { page }.to raise_error(Wayfarer::Networking::Follow::RedirectsExhaustedError)
         end
       end
     end

data/spec/wayfarer/networking/capybara_spec.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+require_relative "strategy"
+
+describe Wayfarer::Networking::Capybara, :ferrum do
+  it_behaves_like "Network strategy", strategy: described_class,
+                                      browser: true,
+                                      request_headers: false,
+                                      response_headers: true,
+                                      status_code: true,
+                                      raises_on_error_response: true # uses Ferrum
+end