wayfarer 0.4.6 → 0.4.8

data/spec/{middleware → wayfarer/middleware}/controller_spec.rb

@@ -3,20 +3,18 @@
 require "spec_helpers"
 
 describe Wayfarer::Middleware::Controller do
+  subject(:controller) { Class.new.include(described_class) }
+
   let(:task) { build(:task) }
 
   let(:middleware) do
-    Class.new do
-      extend Wayfarer::Middleware::Base
-
-      module API; end
-
-      def call(task); end
+    Class.new.tap do |klass|
+      klass.extend(Wayfarer::Middleware::Base)
+      klass.define_method(:call, &->(_) {})
+      klass.const_set("API", Module.new)
     end
   end
 
-  subject(:controller) { Class.new.include(described_class) }
-
   describe "::use" do
     it "adds middleware" do
       expect {
@@ -28,7 +26,7 @@ describe Wayfarer::Middleware::Controller do
       controller.use(middleware)
       expect(controller.chain.middlewares.last).to be_a(Wayfarer::Middleware::Lazy)
       expect(controller.chain.middlewares.last.klass).to be(middleware)
-      expect_any_instance_of(middleware).to receive(:call).with(task)
+      expect_any_instance_of(middleware).to receive(:call).with(task) # rubocop:disable RSpec/AnyInstance
       controller.new.call(task)
     end
   end
@@ -42,45 +40,49 @@ describe Wayfarer::Middleware::Controller do
   end
 
   describe "#call" do
-    subject { controller.new }
+    let(:instance) { controller.new }
 
     it "assigns the task" do
       expect {
-        subject.call(task)
-      }.to change { subject.task }.to(task)
+        instance.call(task)
+      }.to change(instance, :task).to(task)
     end
 
     context "without job assigned" do
       it "assigns itself" do
         expect {
-          subject.call(task)
-        }.to change { task.metadata.job }.to(subject)
+          instance.call(task)
+        }.to change { task[:job] }.to(instance)
       end
     end
 
     context "with job assigned" do
-      before { task.metadata.job = Object.new }
+      before { task[:job] = Object.new }
 
       it "does not override the job" do
         expect {
-          subject.call(task)
-        }.not_to(change { task.metadata.job })
+          instance.call(task)
+        }.not_to(change { task[:job] })
       end
     end
 
     it "assigns itself as controller" do
       expect {
-        subject.call(task)
-      }.to change { task.metadata.controller }.to(subject)
+        instance.call(task)
+      }.to change { task[:controller] }.to(instance)
     end
 
     it "calls the chain" do
-      expect(subject.chain).to receive(:call)
-      subject.call(task)
+      chain = instance.class.chain
+      allow(chain).to receive(:call)
+
+      instance.call(task)
+
+      expect(chain).to have_received(:call)
     end
 
     it "yields" do
-      expect { |spy| subject.call(task, &spy) }.to yield_control
+      expect { |spy| instance.call(task, &spy) }.to yield_control
     end
   end
 end

data/spec/wayfarer/middleware/dedup_spec.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative "../../spec_helpers"
+
+RSpec.describe Wayfarer::Middleware::Dedup, :redis do
+  subject(:middleware) { described_class.new }
+
+  let(:job) { build(:job, executions: 1) }
+  let(:controller) { job }
+  let(:task) { build(:task, :normalized, :barrier, job: job, controller: controller, url: "http://example.com/") }
+  let(:barrier) { task[:barrier] }
+
+  context "with unseen URL" do
+    specify do
+      expect { |block| middleware.call(task, &block) }.to yield_control
+    end
+  end
+
+  context "with seen URL" do
+    before { barrier.check!(task[:uri].to_s) }
+
+    specify do
+      expect { |block| middleware.call(task, &block) }.not_to yield_control
+    end
+
+    context "when task is rerouted" do
+      let(:controller) { Class.new.include(Wayfarer::Handler) }
+
+      specify do
+        expect { |block| middleware.call(task, &block) }.to yield_control
+      end
+    end
+
+    context "with retry" do
+      let(:job) { build(:job, executions: 3) }
+
+      specify do
+        expect { |block| middleware.call(task, &block) }.to yield_control
+      end
+    end
+  end
+
+  describe "cache keys" do
+    before do
+      task[:barrier] = spy
+    end
+
+    specify do
+      middleware.call(task)
+
+      expect(task[:barrier]).to have_received(:check!).with(task[:uri].to_s)
+    end
+
+    context "with custom key" do
+      before do
+        Wayfarer.config[:deduplication][:key] = ->(task) { Digest::MD5.base64digest(task[:uri]) }
+      end
+
+      specify do
+        middleware.call(task)
+
+        expect(task[:barrier]).to have_received(:check!).with("qbnwQzbOAYGgjndOARE7MQ==")
+      end
+    end
+  end
+end

data/spec/wayfarer/middleware/normalize_spec.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::Normalize do
+  subject(:middleware) { described_class.new }
+
+  let(:task) { build(:task, :uri, url: "http://example.com/") }
+
+  context "with assigned URI" do
+    specify do
+      expect { |spy| middleware.call(task, &spy) }.to yield_control
+    end
+
+    specify do
+      expect { middleware.call(task) }.to(change { task[:uri] })
+    end
+  end
+
+  context "with invalid URL" do
+    before do
+      allow(Wayfarer::URI::Normalization)
+        .to receive(:canonical!)
+        .with(task[:uri])
+        .and_raise(Wayfarer::URI::Normalization::InvalidURIError)
+    end
+
+    specify do
+      expect { |spy| middleware.call(task, &spy) }.not_to yield_control
+    end
+  end
+end

data/spec/wayfarer/middleware/router_spec.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::Router do
+  subject(:middleware) { described_class.new }
+
+  let(:task) { build(:task, :uri) }
+
+  describe "#call" do
+    let(:controller) do
+      Struct.new(:task).include(described_class::API).new(task)
+    end
+
+    before do
+      allow(controller.class.route).to receive(:invoke)
+        .with(task)
+        .and_return(result)
+      task[:controller] = controller
+    end
+
+    context "with matching route" do
+      let(:action) { :action }
+      let(:params) { { "foo" => "bar" } }
+      let(:result) do
+        Wayfarer::Routing::Result::Match.new(action, params)
+      end
+
+      it "assigns the action" do
+        expect {
+          middleware.call(task)
+        }.to change { task[:action] }.to(action)
+      end
+
+      it "merges params" do
+        task[:params] = ActiveSupport::HashWithIndifferentAccess.new("bar" => "qux")
+
+        expect {
+          middleware.call(task)
+        }.to change { task[:params] }.to("foo" => "bar", "bar" => "qux")
+      end
+
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
+      end
+
+      specify do
+        # rubocop:disable RSpec/MessageSpies
+        expect(Wayfarer::Logging.logger).to receive(:add).with(Logger::INFO, kind_of(String))
+        # rubocop:enable RSpec/MessageSpies
+
+        middleware.call(task)
+      end
+    end
+
+    context "without matching route" do
+      let(:result) { Wayfarer::Routing::Result::Mismatch.instance }
+
+      it "does not assign the action" do
+        expect {
+          middleware.call(task)
+        }.not_to(change { task[:action] })
+      end
+
+      it "does not alter params" do
+        task[:params] = ActiveSupport::HashWithIndifferentAccess.new("bar" => "qux")
+
+        expect {
+          middleware.call(task)
+        }.not_to(change { task[:params] })
+      end
+
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.not_to yield_control
+      end
+
+      specify do
+        # rubocop:disable RSpec/MessageSpies
+        expect(Wayfarer::Logging.logger).to receive(:add).with(Logger::INFO, kind_of(String))
+        # rubocop:enable RSpec/MessageSpies
+
+        middleware.call(task)
+      end
+    end
+  end
+
+  describe described_class::API do
+    subject(:controller) { Struct.new(:task).include(described_class).new(task) }
+
+    describe "::route" do
+      it "returns a root route" do
+        expect(controller.class.route).to be_a(Wayfarer::Routing::RootRoute)
+      end
+
+      it "adds a routing block" do
+        expect {
+          controller.class.route.to(:index)
+        }.to change { controller.class.route.children.count }.by(1)
+      end
+    end
+  end
+end

data/spec/wayfarer/middleware/stage_spec.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::Stage do
+  subject(:middleware) { described_class.new }
+
+  let(:task) { build(:task) }
+
+  describe "#call" do
+    it "assigns an empty set" do
+      middleware.call(task)
+      expect(task[:staged_urls]).to eq(SortedSet.new)
+    end
+
+    it "yields" do
+      expect { |spy| middleware.call(task, &spy) }.to yield_control
+    end
+
+    it "enqueues tasks" do
+      job = double(crawl: nil) # rubocop:disable RSpec/VerifiedDoubles
+      urls = [test_app_path("/alpha"), test_app_path("/beta")]
+
+      task[:job] = instance_spy(Object, class: job)
+
+      middleware.call(task) do
+        task[:staged_urls] = SortedSet.new(urls)
+      end
+
+      expect(job).to have_received(:crawl).with(urls.first, batch: task.batch).ordered
+      expect(job).to have_received(:crawl).with(urls.second, batch: task.batch).ordered
+    end
+
+    it "resets staged URLs" do
+      task[:staged_urls] = SortedSet.new([test_app_path("/foo")])
+
+      expect {
+        middleware.call(task)
+      }.to change { task[:staged_urls].count }.to(0)
+    end
+  end
+
+  describe described_class::API do
+    subject(:controller) do
+      Struct.new(:task).include(described_class).new(task)
+    end
+
+    describe "#stage" do
+      before { task[:staged_urls] = SortedSet.new }
+
+      it "stages URLs" do
+        expect {
+          controller.stage(test_app_path("/foo"))
+        }.to change { task[:staged_urls].count }.by(1)
+      end
+
+      it "converts to strings" do
+        controller.stage(Addressable::URI.parse(test_app_path("/foo")))
+        expect(task[:staged_urls].to_a.first).to be_a(String)
+      end
+    end
+  end
+end

data/spec/wayfarer/middleware/uri_parser_spec.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::UriParser do
+  subject(:middleware) { described_class.new }
+
+  let(:task) { build(:task) }
+
+  it "parses URLs" do
+    expect { middleware.call(task) }.to change { task[:uri] }.to(Addressable::URI.parse(task.url))
+  end
+
+  specify do
+    expect { |spy| middleware.call(task, &spy) }.to yield_control
+  end
+
+  context "with already parsed URI" do
+    before do
+      task[:uri] = Addressable::URI.parse("http://example.com")
+    end
+
+    specify do
+      expect { |spy| middleware.call(task, &spy) }.to yield_control
+    end
+
+    specify do
+      expect { middleware.call(task) }.not_to(change { task[:uri] })
+    end
+  end
+
+  context "with invalid URL" do
+    let(:task) { build(:task, url: "ht%0atp://localhost/") }
+
+    specify do
+      expect { |spy| middleware.call(task, &spy) }.not_to yield_control
+    end
+
+    specify do
+      expect(Wayfarer::Logging.logger)
+        .to receive(:add) # rubocop:disable RSpec/MessageSpies
+        .with(Logger::INFO, "Not processing invalid URL (Invalid scheme format: 'ht%0atp')")
+
+      middleware.call(task)
+    end
+  end
+
+  describe described_class::API do
+    subject(:controller) do
+      Struct.new(:task).include(described_class).new(task)
+    end
+
+    describe "#uri" do
+      let(:uri) { Addressable::URI.parse(task.url) }
+
+      before { task[:uri] = uri }
+
+      specify do
+        expect(controller.uri).to be(uri)
+      end
+    end
+  end
+end

data/spec/wayfarer/middleware/user_agent_spec.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::UserAgent do
+  subject(:middleware) { described_class.new }
+
+  let(:task) { build(:task, :staged_urls) }
+  let(:page) { instance_double(Wayfarer::Page) }
+  let(:agent) { Object.new }
+  let(:result) { Wayfarer::Networking::Result::Success.new(page) }
+  let(:context) do
+    instance_spy(Wayfarer::Networking::Context, instance: agent).tap do |context|
+      allow(context).to receive(:fetch).with(task.url).and_return(result)
+    end
+  end
+
+  before { task[:controller] = controller }
+
+  describe "#call" do
+    let(:controller) do
+      Struct.new(:task).include(Wayfarer::Middleware::Stage::API).new(task)
+    end
+
+    before do
+      allow(controller).to receive(:run_callbacks).with(:fetch).and_yield
+
+      spy.tap do |pool|
+        allow(pool).to receive(:with).and_yield(context)
+        allow(middleware).to receive(:pool).and_return(pool) # rubocop:disable RSpec/SubjectStub
+      end
+    end
+
+    context "with page assigned" do
+      before { task[:page] = page }
+
+      specify do
+        expect {
+          middleware.call(task)
+        }.not_to(change { task[:page] })
+      end
+
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
+      end
+    end
+
+    specify do
+      middleware.call(task)
+
+      expect(controller).to have_received(:run_callbacks).with(:fetch)
+    end
+
+    specify do
+      middleware.call(task)
+
+      expect(context).to have_received(:fetch).with(task.url)
+    end
+
+    context "with Redirect" do
+      let(:redirect_url) { test_app_path("/foobar") }
+      let(:result) { Wayfarer::Networking::Result::Redirect.new(page) }
+
+      specify do
+        expect {
+          middleware.call(task)
+        }.to change { task[:staged_urls].count }.by(1)
+      end
+
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.not_to yield_control
+      end
+    end
+
+    context "with Success" do
+      specify do
+        expect {
+          middleware.call(task)
+        }.to change { task[:context] }.to(context)
+      end
+
+      specify do
+        expect {
+          middleware.call(task)
+        }.to change { task[:page] }.to(result.page)
+      end
+
+      specify do
+        expect { |spy| middleware.call(task, &spy) }.to yield_control
+      end
+    end
+  end
+
+  describe described_class::API do
+    subject(:controller) do
+      Struct.new(:task).include(described_class).new(task)
+    end
+
+    describe "#user_agent" do
+      before { task[:context] = context }
+
+      specify do
+        expect(controller.user_agent).to be(context.instance)
+      end
+    end
+
+    describe "#page" do
+      before { task[:page] = page }
+
+      specify do
+        expect(controller.page).to be(task[:page])
+      end
+
+      context "with live keyword" do
+        before { task[:context] = context }
+
+        context "with stateful agent" do
+          before do
+            result.page = Object.new
+            allow(context).to receive(:live).and_return(result)
+          end
+
+          it "replaces the page" do
+            expect {
+              controller.page(live: true)
+            }.to change { task[:page] }.to(result.page)
+          end
+        end
+
+        context "with stateless agent" do
+          before { allow(context).to receive(:live).and_return(nil) }
+
+          it "does not alter the page" do
+            expect {
+              controller.page(live: true)
+            }.not_to(change { task[:page] })
+          end
+        end
+      end
+    end
+
+    describe "#fetch" do
+      subject(:page) { controller.fetch(url) }
+
+      let(:url) { test_app_path("/redirect?times=3") }
+
+      it { is_expected.to be_a(Wayfarer::Page) }
+
+      context "with reries exhausted" do
+        let(:url) { test_app_path("/redirect?times=4") }
+
+        specify do
+          expect { page }.to raise_error(Wayfarer::Networking::Follow::RedirectsExhaustedError)
+        end
+      end
+    end
+  end
+end

data/spec/wayfarer/networking/capybara_spec.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+require_relative "strategy"
+
+describe Wayfarer::Networking::Capybara, :ferrum do
+  it_behaves_like "Network strategy", strategy: described_class,
+                                      browser: true,
+                                      request_headers: false,
+                                      response_headers: true,
+                                      status_code: true,
+                                      raises_on_error_response: true # uses Ferrum
+end