wayfarer 0.4.6 → 0.4.7

data/spec/middleware/dedup_spec.rb

@@ -3,85 +3,68 @@
 require "spec_helpers"
 
 describe Wayfarer::Middleware::Dedup, redis: true do
-  let(:task) { build(:task) }
+  let(:task) { build(:task, :redis_pool) }
+  let(:uri) { Addressable::URI.parse(task.url) }
+  let(:executions) { 1 }
+  let(:job) { double(executions: executions) }
   subject { described_class.new }
 
-  before { task.metadata.staged_urls = SortedSet.new }
+  before do
+    task[:job] = job
+    task[:controller] = job
+    task[:normalized_url] = task.url
+  end
 
   describe "#call" do
-    context "if already routed" do
-      before { task.metadata.action = :action }
+    it "assigns barrier" do
+      expect { subject.call(task) }.to change { task[:barrier] }.from(nil).to(instance_of(Wayfarer::Redis::Barrier))
+    end
 
-      it "does not call the barrier" do
-        expect(task.barrier).not_to receive(:seen?)
-      end
+    it "assigns barrier for batch" do
+      subject.call(task)
+
+      expect(task[:barrier].task).to be(task)
+    end
 
-      it "yields" do
+    context "with retry" do
+      let(:executions) { 2 }
+
+      specify do
         expect { |spy| subject.call(task, &spy) }.to yield_control
       end
-    end
 
-    context "without staged URLs" do
-      it "does not raise" do
-        expect { subject.call(task) }.not_to raise_error
+      specify do
+        expect(Wayfarer::Logging.logger).to receive(:add).with(Logger::DEBUG, "Not deduplicating retry")
+
+        subject.call(task)
       end
     end
 
-    context "with unseen URL" do
-      it "marks the URL as seen" do
-        expect {
-          subject.call(task)
-        }.to change { task.barrier.seen?(task.url) }.to(true)
-      end
+    context "when rerouted" do
+      before { task[:controller] = Object.new }
 
-      it "yields" do
+      specify do
         expect { |spy| subject.call(task, &spy) }.to yield_control
       end
-    end
 
-    context "with seen URL" do
-      before { task.barrier.seen?(task.url) }
+      specify do
+        expect(Wayfarer::Logging.logger).to receive(:add).with(Logger::DEBUG, "Not deduplicating rerouted task")
 
-      it "does not yield" do
-        expect { |spy| subject.call(task, &spy) }.not_to yield_control
+        subject.call(task)
       end
     end
 
-    context "with exception raised" do
-      it "does not mark the URL as seen" do
-        begin
-          subject.call(task) { raise }
-        rescue StandardError
-          nil
-        end
-        expect(task.barrier.seen?(task.url)).to be(false)
-      end
-
-      it "re-raises the exception" do
-        expect {
-          subject.call(task) { raise }
-        }.to raise_error(RuntimeError)
+    context "with unchecked URL" do
+      specify do
+        expect { |spy| subject.call(task, &spy) }.to yield_control
       end
     end
 
-    describe "staged URL filtering" do
-      let(:seen_urls) { %w[https://yahoo.com https://google.com] }
-      let(:unseen_urls) { %w[https://w3c.org https://nasa.gov] }
+    context "with checked URL" do
+      before { Wayfarer::Redis::Barrier.new(task).check!(task[:normalized_url]) }
 
-      before do
-        seen_urls.each do |url|
-          task.barrier.seen?(url)
-        end
-
-        [*seen_urls, *unseen_urls].each do |url|
-          task.metadata.staged_urls.add(url)
-        end
-      end
-
-      it "filters seen staged URLs" do
-        expect {
-          subject.call(task)
-        }.to change { task.metadata.staged_urls }.to(SortedSet.new(unseen_urls))
+      specify do
+        expect { |spy| subject.call(task, &spy) }.not_to yield_control
       end
     end
   end

data/spec/middleware/dispatch_spec.rb

@@ -8,27 +8,27 @@ describe Wayfarer::Middleware::Dispatch do
   subject(:chain) { described_class.new }
 
   before do
-    task.metadata.controller = spy
-    task.metadata.action = action
+    task[:controller] = spy
+    task[:action] = action
 
-    allow(task.metadata.controller).to receive(:run_callbacks).and_yield
+    allow(task[:controller]).to receive(:run_callbacks).and_yield
   end
 
   describe "#call" do
     it "runs callbacks" do
-      expect(task.metadata.controller).to receive(:run_callbacks).with(action)
+      expect(task[:controller]).to receive(:run_callbacks).with(action)
       subject.call(task)
     end
 
     context "when action is a Symbol" do
       it "calls the method" do
-        expect(task.metadata.controller).to receive(action)
+        expect(task[:controller]).to receive(action)
         subject.call(task)
       end
     end
 
-    context "with other action" do
-      let(:action) { Class.new }
+    context "with handler" do
+      let(:action) { Class.new.include(Wayfarer::Handler) }
 
       it "instantiates and calls" do
         expect_any_instance_of(action).to receive(:call).with(task)
@@ -36,6 +36,22 @@ describe Wayfarer::Middleware::Dispatch do
       end
     end
 
+    context "without action" do
+      let(:action) { nil }
+
+      it "instantiates and calls" do
+        expect { subject.call(task) }.to raise_error(ArgumentError)
+      end
+    end
+
+    context "without other action" do
+      let(:action) { Class.new }
+
+      it "instantiates and calls" do
+        expect { subject.call(task) }.to raise_error(ArgumentError)
+      end
+    end
+
     it "yields" do
       expect { |spy| subject.call(task, &spy) }.to yield_control
     end

data/spec/middleware/normalize_spec.rb

@@ -4,26 +4,57 @@ require "spec_helpers"
 
 describe Wayfarer::Middleware::Normalize do
   let(:task) { build(:task) }
+  let(:uri) { Addressable::URI.parse(url) }
   subject { described_class.new }
 
-  describe "#call" do
-    let(:urls) do
-      ["http://example.com/products?product_id=123",
-       "HTTP://EXAMPLE.COM/products/?product_id=123",
-       "http://example.com/products/?product_id=123",
-       "http://example.com/foo/../products?product_id=123",
-       "invalid@url-net"]
+  describe "::normalize" do
+    subject { described_class.normalize(uri).to_s }
+
+    context "without HTTP(S) URL" do
+      let(:url) { "localhost:3000" }
+
+      it { is_expected.to eq(url) }
     end
 
-    before { task.metadata.staged_urls = SortedSet.new(urls) }
+    context "with HTTP URL" do
+      let(:url) { "http://example.com" }
 
-    it "yields" do
-      expect { |spy| subject.call(task, &spy) }.to yield_control
+      it { is_expected.to eq("http://example.com/") }
     end
+  end
+
+  describe "#call" do
+    let(:url) { task.url }
+    before { task[:uri] = uri }
+
+    context "when already assigned" do
+      before { task[:normalized_url] = Object.new }
+
+      specify do
+        expect { |spy| subject.call(task, &spy) }.to yield_control
+      end
+
+      it "doesn't normalize URL" do
+        expect { subject.call(task) }.not_to(change { task[:normalized_url] })
+      end
+    end
+
+    context "with invalid URL" do
+      before do
+        # I can't come up with a URL that normalize_url raises on but
+        # Addressable::URI doesn't, hence the stub
+        allow(described_class).to receive(:normalize).with(uri).and_return(nil)
+      end
+
+      specify do
+        expect { |spy| subject.call(task, &spy) }.not_to yield_control
+      end
+
+      specify do
+        expect(Wayfarer::Logging.logger).to receive(:add).with(Logger::INFO, "Failed to normalize HTTP(S) URL")
 
-    it "normalizes and compacts URLs" do
-      subject.call(task)
-      expect(task.metadata.staged_urls).to eq(SortedSet[urls.first])
+        subject.call(task)
+      end
     end
   end
 end

data/spec/middleware/router_spec.rb

@@ -12,10 +12,11 @@ describe Wayfarer::Middleware::Router do
     end
 
     before do
-      allow(controller.class.router).to receive(:invoke)
-        .with(Addressable::URI.parse(task.url), controller.steer)
+      allow(controller.class.route).to receive(:invoke)
+        .with(Addressable::URI.parse(task.url))
         .and_return(result)
-      task.metadata.controller = controller
+      task[:controller] = controller
+      task[:uri] = Addressable::URI.parse(task.url)
     end
 
     context "with matching route" do
@@ -28,20 +29,26 @@ describe Wayfarer::Middleware::Router do
       it "assigns the action" do
         expect {
           subject.call(task)
-        }.to change { task.metadata.action }.to(action)
+        }.to change { task[:action] }.to(action)
       end
 
       it "merges params" do
-        task.metadata.params = ActiveSupport::HashWithIndifferentAccess.new("bar" => "qux")
+        task[:params] = ActiveSupport::HashWithIndifferentAccess.new("bar" => "qux")
 
         expect {
           subject.call(task)
-        }.to change { task.metadata.params }.to("foo" => "bar", "bar" => "qux")
+        }.to change { task[:params] }.to("foo" => "bar", "bar" => "qux")
       end
 
-      it "yields" do
+      specify do
         expect { |spy| subject.call(task, &spy) }.to yield_control
       end
+
+      specify do
+        expect(Wayfarer::Logging.logger).to receive(:add).with(Logger::INFO, kind_of(String))
+
+        subject.call(task)
+      end
     end
 
     context "without matching route" do
@@ -52,20 +59,26 @@ describe Wayfarer::Middleware::Router do
       it "does not assign the action" do
         expect {
           subject.call(task)
-        }.not_to(change { task.metadata.action })
+        }.not_to(change { task[:action] })
       end
 
       it "does not alter params" do
-        task.metadata.params = ActiveSupport::HashWithIndifferentAccess.new("bar" => "qux")
+        task[:params] = ActiveSupport::HashWithIndifferentAccess.new("bar" => "qux")
 
         expect {
           subject.call(task)
-        }.not_to(change { task.metadata.params })
+        }.not_to(change { task[:params] })
       end
 
-      it "does not yield" do
+      specify do
         expect { |spy| subject.call(task, &spy) }.not_to yield_control
       end
+
+      specify do
+        expect(Wayfarer::Logging.logger).to receive(:add).with(Logger::INFO, kind_of(String))
+
+        subject.call(task)
+      end
     end
   end
 
@@ -74,31 +87,17 @@ describe Wayfarer::Middleware::Router do
       Struct.new(:task).include(described_class).new(task)
     end
 
-    describe "::router" do
-      it "returns a router" do
-        expect(controller.class.router).to be_a(Wayfarer::Routing::Router)
+    describe "::route" do
+      it "returns a root route" do
+        expect(controller.class.route).to be_a(Wayfarer::Routing::RootRoute)
       end
     end
 
     describe "::route" do
       it "adds a routing block" do
         expect {
-          controller.class.route { to :index }
-        }.to change { controller.class.router.blocks.count }.by(1)
-      end
-    end
-
-    describe "::steer" do
-      it "overrides #steer" do
-        expect {
-          controller.class.steer { :foobar }
-        }.to change { controller.steer }.from([]).to(:foobar)
-      end
-    end
-
-    describe "#steer" do
-      it "returns [] by default" do
-        expect(controller.steer).to eq([])
+          controller.class.route.to(:index)
+        }.to change { controller.class.route.children.count }.by(1)
       end
     end
   end

data/spec/middleware/stage_spec.rb

@@ -9,7 +9,7 @@ describe Wayfarer::Middleware::Stage do
   describe "#call" do
     it "assigns an empty set" do
       subject.call(task)
-      expect(task.metadata.staged_urls).to eq(SortedSet.new)
+      expect(task[:staged_urls]).to eq(SortedSet.new)
     end
 
     it "yields" do
@@ -22,20 +22,20 @@ describe Wayfarer::Middleware::Stage do
       spy.tap do |job|
         expect(job).to receive(:crawl).with(urls.first, batch: task.batch).ordered
         expect(job).to receive(:crawl).with(urls.second, batch: task.batch).ordered
-        task.metadata.job = double(class: job)
+        task[:job] = double(class: job)
       end
 
       subject.call(task) do
-        task.metadata.staged_urls = SortedSet.new(urls)
+        task[:staged_urls] = SortedSet.new(urls)
       end
     end
 
     it "resets staged URLs" do
-      task.metadata.staged_urls = SortedSet.new([test_app_path("/foo")])
+      task[:staged_urls] = SortedSet.new([test_app_path("/foo")])
 
       expect {
         subject.call(task)
-      }.to change { task.metadata.staged_urls.count }.to(0)
+      }.to change { task[:staged_urls].count }.to(0)
     end
   end
 
@@ -45,17 +45,17 @@ describe Wayfarer::Middleware::Stage do
     end
 
     describe "#stage" do
-      before { task.metadata.staged_urls = SortedSet.new }
+      before { task[:staged_urls] = SortedSet.new }
 
       it "stages URLs" do
         expect {
           controller.stage(test_app_path("/foo"))
-        }.to change { task.metadata.staged_urls.count }.by(1)
+        }.to change { task[:staged_urls].count }.by(1)
       end
 
       it "converts to strings" do
         controller.stage(Addressable::URI.parse(test_app_path("/foo")))
-        expect(task.metadata.staged_urls.to_a.first).to be_a(String)
+        expect(task[:staged_urls].to_a.first).to be_a(String)
       end
     end
   end

data/spec/middleware/uri_parser_spec.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::UriParser, "#call" do
+  let(:task) { build(:task) }
+  subject { described_class.new }
+
+  it "parses URLs" do
+    expect { subject.call(task) }.to change { task[:uri] }.to(Addressable::URI.parse(task.url))
+  end
+
+  it "normalizes URLs" do
+    task = build(:task, url: "http://example.com")
+
+    expect { subject.call(task) }.to change { task[:uri].to_s }.to("http://example.com/")
+  end
+
+  specify do
+    expect { |spy| subject.call(task, &spy) }.to yield_control
+  end
+
+  context "with invalid URL" do
+    let(:task) { build(:task, url: "ht%0atp://localhost/") }
+
+    specify do
+      expect { |spy| subject.call(task, &spy) }.not_to yield_control
+    end
+
+    specify do
+      expect(Wayfarer::Logging.logger)
+        .to receive(:add).with(Logger::INFO, "Not processing invalid URL (Invalid scheme format: 'ht%0atp')")
+
+      subject.call(task)
+    end
+  end
+
+  describe described_class::API do
+    subject(:controller) do
+      Struct.new(:task).include(described_class).new(task)
+    end
+
+    describe "#uri" do
+      let(:uri) { Addressable::URI.parse(task.url) }
+
+      before { task[:uri] = uri }
+
+      it "returns the agent" do
+        expect(controller.uri).to be(uri)
+      end
+    end
+  end
+end

data/spec/middleware/{fetch_spec.rb → user_agent_spec.rb}

@@ -2,7 +2,7 @@
 
 require "spec_helpers"
 
-describe Wayfarer::Middleware::Fetch do
+describe Wayfarer::Middleware::UserAgent do
   let(:task) { build(:task) }
   let(:page) { Object.new }
   let(:agent) { Object.new }
@@ -28,17 +28,17 @@ describe Wayfarer::Middleware::Fetch do
         allow(subject).to receive(:pool).and_return(pool)
       end
 
-      task.metadata.staged_urls = SortedSet.new
-      task.metadata.controller = controller
+      task[:staged_urls] = SortedSet.new
+      task[:controller] = controller
     end
 
     context "with page assigned" do
-      before { task.metadata.page = page }
+      before { task[:page] = page }
 
       it "does not alter the page" do
         expect {
           subject.call(task)
-        }.not_to(change { task.metadata.page })
+        }.not_to(change { task[:page] })
       end
 
       it "yields" do
@@ -63,7 +63,7 @@ describe Wayfarer::Middleware::Fetch do
       it "stages the redirect URL" do
         expect {
           subject.call(task)
-        }.to change { task.metadata.staged_urls.count }.by(1)
+        }.to change { task[:staged_urls].count }.by(1)
       end
 
       it "does not yield" do
@@ -75,13 +75,13 @@ describe Wayfarer::Middleware::Fetch do
       it "assigns the context" do
         expect {
           subject.call(task)
-        }.to change { task.metadata.context }.to(context)
+        }.to change { task[:context] }.to(context)
       end
 
       it "assigns the page" do
         expect {
           subject.call(task)
-        }.to change { task.metadata.page }.to(result.page)
+        }.to change { task[:page] }.to(result.page)
       end
 
       it "yields" do
@@ -95,31 +95,23 @@ describe Wayfarer::Middleware::Fetch do
       Struct.new(:task).include(described_class).new(task)
     end
 
-    describe "#agent" do
-      before { task.metadata.context = context }
+    describe "#user_agent" do
+      before { task[:context] = context }
 
       it "returns the agent" do
-        expect(controller.agent).to be(context.instance)
-      end
-    end
-
-    describe "#context" do
-      before { task.metadata.context = context }
-
-      it "returns the context" do
-        expect(controller.context).to be(task.metadata.context)
+        expect(controller.user_agent).to be(context.instance)
       end
     end
 
     describe "#page" do
-      before { task.metadata.page = page }
+      before { task[:page] = page }
 
       it "returns the page" do
-        expect(controller.page).to be(task.metadata.page)
+        expect(controller.page).to be(task[:page])
       end
 
       context "with live keyword" do
-        before { task.metadata.context = context }
+        before { task[:context] = context }
 
         context "with stateful agent" do
           before do
@@ -130,7 +122,7 @@ describe Wayfarer::Middleware::Fetch do
           it "replaces the page" do
             expect {
               controller.page(live: true)
-            }.to change { task.metadata.page }.to(result.page)
+            }.to change { task[:page] }.to(result.page)
           end
         end
 
@@ -140,15 +132,24 @@ describe Wayfarer::Middleware::Fetch do
           it "does not alter the page" do
             expect {
               controller.page(live: true)
-            }.not_to(change { task.metadata.page })
+            }.not_to(change { task[:page] })
           end
         end
       end
     end
 
-    describe "#http" do
-      it "returns a redirect-following HTTP agent" do
-        expect(controller.http).to be_a(Wayfarer::Networking::Follow)
+    describe "#fetch" do
+      let(:url) { test_app_path("/redirect?times=3") }
+      subject { controller.fetch(url) }
+
+      it { is_expected.to be_a(Wayfarer::Page) }
+
+      context "with reries exhausted" do
+        let(:url) { test_app_path("/redirect?times=4") }
+
+        specify do
+          expect { subject }.to raise_error(Wayfarer::Networking::Follow::RedirectsExhaustedError)
+        end
       end
     end
   end

data/spec/networking/context_spec.rb

@@ -49,7 +49,7 @@ describe Wayfarer::Networking::Context do
       let(:other_error) { Class.new(StandardError) }
 
       before do
-        Wayfarer.config.network.renew_on = [other_error]
+        Wayfarer.config[:network][:renew_on] = [other_error]
         allow(strategy).to receive(:fetch).and_raise(other_error)
       end
 

data/spec/networking/follow_spec.rb

@@ -16,7 +16,7 @@ describe Wayfarer::Networking::Follow do
       before { allow(inner).to receive(:fetch).and_return(success) }
 
       it "returns the page" do
-        expect(outer.fetch(url)).to be(page)
+        expect(outer.fetch(url, follow: 3)).to be(page)
       end
     end
 
@@ -34,7 +34,7 @@ describe Wayfarer::Networking::Follow do
 
         expect {
           outer.fetch(url, follow: 0)
-        }.to raise_error(Wayfarer::Networking::RedirectsExhaustedError)
+        }.to raise_error(described_class::RedirectsExhaustedError)
       end
     end
   end

data/spec/networking/pool_spec.rb

@@ -17,7 +17,7 @@ describe Wayfarer::Networking::Pool do
     end
 
     context "when using Ferrum", ferrum: true do
-      before { Wayfarer.config.network.agent = :ferrum }
+      before { Wayfarer.config[:network][:agent] = :ferrum }
 
       it "yields Ferrum" do
         pool.with do |context|
@@ -27,7 +27,7 @@ describe Wayfarer::Networking::Pool do
     end
 
     context "when using Selenium", selenium: true do
-      before { Wayfarer.config.network.agent = :selenium }
+      before { Wayfarer.config[:network][:agent] = :selenium }
 
       it "yields Selenium" do
         pool.with do |context|
@@ -38,8 +38,8 @@ describe Wayfarer::Networking::Pool do
 
     context "when using Capybara", ferrum: true do
       before do
-        Wayfarer.config.network.agent = :capybara
-        Wayfarer.config.capybara.driver = :cuprite
+        Wayfarer.config[:network][:agent] = :capybara
+        Wayfarer.config[:capybara][:driver] = :cuprite
       end
 
       it "yields Capybara" do
@@ -55,7 +55,7 @@ describe Wayfarer::Networking::Pool do
 
     before do
       pool.class.registry[:foobar] = double(new: strategy)
-      Wayfarer.config.network.agent = :foobar
+      Wayfarer.config[:network][:agent] = :foobar
     end
 
     it "destroys the strategy" do