wayfarer 0.4.1 → 0.4.5

data/spec/integration/stage_spec.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Staging" do
+  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
+
+  describe Wayfarer::Base do
+    specify do
+      class self.class::DummyJob < Wayfarer::Base
+        extend SpecHelpers
+
+        route { host test_app_host, to: :index }
+
+        def index
+          stage page.meta.links.all
+        end
+      end
+
+      expect {
+        self.class::DummyJob.crawl(url)
+        perform_enqueued_jobs
+      }.to change { enqueued_jobs.size }.by(156)
+    end
+  end
+
+  describe Wayfarer::Handler do
+    specify do
+      class self.class::DummyJob < Wayfarer::Base
+        extend SpecHelpers
+
+        route do
+          host test_app_host, to: DummyHandler
+        end
+
+        class DummyHandler < Wayfarer::Handler
+          route { to :index }
+
+          def index
+            stage page.meta.links.all
+          end
+        end
+      end
+
+      expect {
+        self.class::DummyJob.crawl(url)
+        perform_enqueued_jobs
+      }.to change { enqueued_jobs.size }.by(156)
+    end
+  end
+end

data/spec/integration/steering_spec.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Steering" do
+  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
+
+  describe Wayfarer::Base do
+    specify do
+      class self.class::DummyJob < Wayfarer::Base
+        extend RSpec::Matchers
+
+        route do |a, b|
+          expect(a).to eq("foobar")
+          expect(b).to eq("barqux")
+        end
+
+        steer do |task|
+          [task.batch, "barqux"]
+        end
+      end
+
+      self.class::DummyJob.crawl(url, batch: "foobar")
+      perform_enqueued_jobs
+    end
+  end
+
+  describe Wayfarer::Handler do
+    specify do
+      class self.class::DummyJob < Wayfarer::Base
+        route do |_a, _b|
+          to DummyHandler
+        end
+
+        steer do |_task|
+          [123, "barqux"]
+        end
+
+        class DummyHandler < Wayfarer::Handler
+          extend RSpec::Matchers
+
+          route do |a, b|
+            expect(a).to eq("foobar")
+            expect(b).to eq("fooz")
+          end
+
+          steer do |task|
+            [task.batch, "fooz"]
+          end
+        end
+      end
+
+      self.class::DummyJob.crawl(url, batch: "foobar")
+      perform_enqueued_jobs
+    end
+  end
+end

data/spec/middleware/chain_spec.rb

@@ -4,12 +4,25 @@ require "spec_helpers"
 
 describe Wayfarer::Middleware::Chain do
   let(:task) { build(:task) }
-  subject(:chain) { build(:chain, middlewares: middlewares) }
+  let(:middlewares) { [] }
+  subject(:chain) { described_class.new(middlewares) }
+
+  describe "::empty" do
+    it "returns an empty chain" do
+      expect(described_class.empty.middlewares).to be_empty
+    end
+  end
 
   describe "#call" do
-    context "without middlewares" do
-      let(:middlewares) { [] }
+    it "adds middleware" do
+      expect {
+        chain.push(Class.new)
+      }.to change { chain.middlewares.count }.by(1)
+    end
+  end
 
+  describe "#call" do
+    context "without middlewares" do
       it "does not raise" do
         expect { chain.call(task) }.not_to raise_error
       end
@@ -51,13 +64,13 @@ describe Wayfarer::Middleware::Chain do
 
     describe "Return value" do
       let(:middlewares) do
-        [build(:middleware, callee: lambda do |task, &block|
-                                      task.metadata.foobar = 42
-                                      block.call
-                                    end),
-         build(:middleware, callee: lambda do |task|
-                                      task.metadata.foobar *= 1337
-                                    end)]
+        [build(:middleware, receiver: lambda do |task, &block|
+                                        task.metadata.foobar = 42
+                                        block.call
+                                      end),
+         build(:middleware, receiver: lambda do |task|
+                                        task.metadata.foobar *= 1337
+                                      end)]
       end
 
       it "is the last middleware's return value" do
@@ -67,18 +80,18 @@ describe Wayfarer::Middleware::Chain do
 
     describe "Metadata" do
       let(:first) do
-        build(:middleware, callee: lambda do |task, &block|
-                                     task.metadata.foobar = 42
-                                     block.call
-                                     task.metadata.barqux = 1337
-                                   end)
+        build(:middleware, receiver: lambda do |task, &block|
+                                       task.metadata.foobar = 42
+                                       block.call
+                                       task.metadata.barqux = 1337
+                                     end)
       end
 
       let(:last) do
-        build(:middleware, callee: lambda do |task|
-                                     raise unless task.metadata.foobar == 42
-                                     raise if task.metadata.barqux
-                                   end)
+        build(:middleware, receiver: lambda do |task|
+                                       raise unless task.metadata.foobar == 42
+                                       raise if task.metadata.barqux
+                                     end)
       end
 
       let(:middlewares) { [first, last] }

data/spec/middleware/controller_spec.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::Controller do
+  let(:task) { build(:task) }
+
+  let(:middleware) do
+    Class.new do
+      extend Wayfarer::Middleware::Base
+
+      module API; end
+
+      def call(task); end
+    end
+  end
+
+  subject(:controller) { Class.new.include(described_class) }
+
+  describe "::use" do
+    it "adds middleware" do
+      expect {
+        controller.use(middleware)
+      }.to change { controller.chain.middlewares.count }.by(1)
+    end
+
+    it "adds lazy middleware" do
+      controller.use(middleware)
+      expect(controller.chain.middlewares.last).to be_a(Wayfarer::Middleware::Lazy)
+      expect(controller.chain.middlewares.last.klass).to be(middleware)
+      expect_any_instance_of(middleware).to receive(:call).with(task)
+      controller.new.call(task)
+    end
+  end
+
+  describe "::api" do
+    it "includes middleware API" do
+      expect {
+        controller.api(middleware)
+      }.to change { controller.include?(middleware.const_get(:API)) }.to(true)
+    end
+  end
+
+  describe "#call" do
+    subject { controller.new }
+
+    it "assigns the task" do
+      expect {
+        subject.call(task)
+      }.to change { subject.task }.to(task)
+    end
+
+    context "without job assigned" do
+      it "assigns itself" do
+        expect {
+          subject.call(task)
+        }.to change { task.metadata.job }.to(subject)
+      end
+    end
+
+    context "with job assigned" do
+      before { task.metadata.job = Object.new }
+
+      it "does not override the job" do
+        expect {
+          subject.call(task)
+        }.not_to(change { task.metadata.job })
+      end
+    end
+
+    it "assigns itself as controller" do
+      expect {
+        subject.call(task)
+      }.to change { task.metadata.controller }.to(subject)
+    end
+
+    it "calls the chain" do
+      expect(subject.chain).to receive(:call)
+      subject.call(task)
+    end
+
+    it "yields" do
+      expect { |spy| subject.call(task, &spy) }.to yield_control
+    end
+  end
+end

data/spec/middleware/dedup_spec.rb

@@ -4,26 +4,38 @@ require "spec_helpers"
 
 describe Wayfarer::Middleware::Dedup, redis: true do
   let(:task) { build(:task) }
-  subject(:dedup) { Wayfarer::Middleware::Dedup.new }
+  subject { described_class.new }
 
   before { task.metadata.staged_urls = SortedSet.new }
 
   describe "#call" do
+    context "if already routed" do
+      before { task.metadata.action = :action }
+
+      it "does not call the barrier" do
+        expect(task.barrier).not_to receive(:seen?)
+      end
+
+      it "yields" do
+        expect { |spy| subject.call(task, &spy) }.to yield_control
+      end
+    end
+
     context "without staged URLs" do
       it "does not raise" do
-        expect { dedup.call(task) }.not_to raise_error
+        expect { subject.call(task) }.not_to raise_error
       end
     end
 
     context "with unseen URL" do
       it "marks the URL as seen" do
         expect {
-          dedup.call(task)
+          subject.call(task)
         }.to change { task.barrier.seen?(task.url) }.to(true)
       end
 
       it "yields" do
-        expect { |spy| dedup.call(task, &spy) }.to yield_control
+        expect { |spy| subject.call(task, &spy) }.to yield_control
       end
     end
 
@@ -31,14 +43,14 @@ describe Wayfarer::Middleware::Dedup, redis: true do
       before { task.barrier.seen?(task.url) }
 
       it "does not yield" do
-        expect { |spy| dedup.call(task, &spy) }.not_to yield_control
+        expect { |spy| subject.call(task, &spy) }.not_to yield_control
       end
     end
 
     context "with exception raised" do
       it "does not mark the URL as seen" do
         begin
-          dedup.call(task) { raise }
+          subject.call(task) { raise }
         rescue StandardError
           nil
         end
@@ -47,7 +59,7 @@ describe Wayfarer::Middleware::Dedup, redis: true do
 
       it "re-raises the exception" do
         expect {
-          dedup.call(task) { raise }
+          subject.call(task) { raise }
         }.to raise_error(RuntimeError)
       end
     end
@@ -68,7 +80,7 @@ describe Wayfarer::Middleware::Dedup, redis: true do
 
       it "filters seen staged URLs" do
         expect {
-          dedup.call(task)
+          subject.call(task)
         }.to change { task.metadata.staged_urls }.to(SortedSet.new(unseen_urls))
       end
     end

data/spec/middleware/dispatch_spec.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::Dispatch do
+  let(:task) { build(:task) }
+  let(:action) { :action }
+  subject(:chain) { described_class.new }
+
+  before do
+    task.metadata.controller = spy
+    task.metadata.action = action
+
+    allow(task.metadata.controller).to receive(:run_callbacks).and_yield
+  end
+
+  describe "#call" do
+    it "runs callbacks" do
+      expect(task.metadata.controller).to receive(:run_callbacks).with(action)
+      subject.call(task)
+    end
+
+    context "when action is a Symbol" do
+      it "calls the method" do
+        expect(task.metadata.controller).to receive(action)
+        subject.call(task)
+      end
+    end
+
+    context "with other action" do
+      let(:action) { Class.new }
+
+      it "instantiates and calls" do
+        expect_any_instance_of(action).to receive(:call).with(task)
+        subject.call(task)
+      end
+    end
+
+    it "yields" do
+      expect { |spy| subject.call(task, &spy) }.to yield_control
+    end
+  end
+end

data/spec/middleware/fetch_spec.rb

@@ -3,81 +3,152 @@
 require "spec_helpers"
 
 describe Wayfarer::Middleware::Fetch do
-  let(:klass) { Class.new(Wayfarer::Base) }
-  let(:job) { klass.new }
   let(:task) { build(:task) }
-  let(:agent) { spy }
-  subject(:fetch) { Wayfarer::Middleware::Fetch.new }
-
-  before do
-    task.job = job
-    allow(fetch.pool).to receive(:with).and_yield(agent)
+  let(:page) { Object.new }
+  let(:agent) { Object.new }
+  let(:result) { Wayfarer::Networking::Result::Success.new(page) }
+  let(:context) do
+    double(instance: agent).tap do |context|
+      allow(context).to receive(:fetch).with(task.url).and_return(result)
+    end
   end
 
+  subject { described_class.new }
+
   describe "#call" do
-    it "assigns the task" do
-      expect {
-        fetch.call(task)
-      }.to change { fetch.task }.to(task)
+    let(:controller) do
+      Struct.new(:task).include(Wayfarer::Middleware::Stage::API).new(task)
     end
 
-    it "assigns the agent" do
-      expect {
-        fetch.call(task)
-      }.to change { task.metadata.agent }.to(agent)
-    end
+    before do
+      allow(controller).to receive(:run_callbacks).with(:fetch).and_yield
 
-    it "retrieves the page" do
-      expect(agent).to receive(:fetch).with(task.url)
-      fetch.call(task)
-    end
+      spy.tap do |pool|
+        allow(pool).to receive(:with).and_yield(context)
+        allow(subject).to receive(:pool).and_return(pool)
+      end
 
-    it "runs callbacks" do
-      expect { |spy|
-        klass.before_fetch(&spy)
-        fetch.call(task)
-      }.to yield_control
+      task.metadata.staged_urls = SortedSet.new
+      task.metadata.controller = controller
     end
 
-    context "when encountering an HTTP redirect" do
-      let(:redirect_url) { "https://redirect.com" }
+    context "with page assigned" do
+      before { task.metadata.page = page }
 
-      let(:result) do
-        Wayfarer::Networking::Result::Redirect.new(redirect_url)
+      it "does not alter the page" do
+        expect {
+          subject.call(task)
+        }.not_to(change { task.metadata.page })
       end
 
-      before do
-        task.metadata.staged_urls = Set.new
-        allow(agent).to receive(:fetch).with(task.url).and_return(result)
+      it "yields" do
+        expect { |spy| subject.call(task, &spy) }.to yield_control
       end
+    end
+
+    it "runs callbacks" do
+      expect(controller).to receive(:run_callbacks).with(:fetch)
+      subject.call(task)
+    end
+
+    it "fetches the URL" do
+      expect(context).to receive(:fetch).with(task.url)
+      subject.call(task)
+    end
+
+    context "with Redirect" do
+      let(:redirect_url) { test_app_path("/foobar") }
+      let(:result) { Wayfarer::Networking::Result::Redirect.new(page) }
 
       it "stages the redirect URL" do
         expect {
-          fetch.call(task)
-        }.to change { task.metadata.staged_urls }.to([redirect_url])
+          subject.call(task)
+        }.to change { task.metadata.staged_urls.count }.by(1)
       end
 
       it "does not yield" do
-        expect { |spy| fetch.call(task, &spy) }.not_to yield_control
+        expect { |spy| subject.call(task, &spy) }.not_to yield_control
       end
     end
 
-    context "when retrieval succeeds" do
-      let(:page) { build(:page) }
-      let(:result) { Wayfarer::Networking::Result::Success.new(page) }
-
-      before do
-        allow(agent).to receive(:fetch).with(task.url).and_return(result)
+    context "with Success" do
+      it "assigns the context" do
+        expect {
+          subject.call(task)
+        }.to change { task.metadata.context }.to(context)
       end
 
       it "assigns the page" do
         expect {
-          fetch.call(task)
-        }.to change { task.metadata.page }.to(page)
+          subject.call(task)
+        }.to change { task.metadata.page }.to(result.page)
       end
 
       it "yields" do
-        expect { |spy| fetch.call(task, &spy) }.to yield_control
+        expect { |spy| subject.call(task, &spy) }.to yield_control
+      end
+    end
+  end
+
+  describe described_class::API do
+    subject(:controller) do
+      Struct.new(:task).include(described_class).new(task)
+    end
+
+    describe "#agent" do
+      before { task.metadata.context = context }
+
+      it "returns the agent" do
+        expect(controller.agent).to be(context.instance)
+      end
+    end
+
+    describe "#context" do
+      before { task.metadata.context = context }
+
+      it "returns the context" do
+        expect(controller.context).to be(task.metadata.context)
+      end
+    end
+
+    describe "#page" do
+      before { task.metadata.page = page }
+
+      it "returns the page" do
+        expect(controller.page).to be(task.metadata.page)
+      end
+
+      context "with live keyword" do
+        before { task.metadata.context = context }
+
+        context "with stateful agent" do
+          before do
+            result.page = Object.new
+            allow(context).to receive(:live).and_return(result)
+          end
+
+          it "replaces the page" do
+            expect {
+              controller.page(live: true)
+            }.to change { task.metadata.page }.to(result.page)
+          end
+        end
+
+        context "with stateless agent" do
+          before { allow(context).to receive(:live).and_return(nil) }
+
+          it "does not alter the page" do
+            expect {
+              controller.page(live: true)
+            }.not_to(change { task.metadata.page })
+          end
+        end
+      end
+    end
+
+    describe "#http" do
+      it "returns a redirect-following HTTP agent" do
+        expect(controller.http).to be_a(Wayfarer::Networking::Follow)
       end
     end
   end

data/spec/middleware/normalize_spec.rb

@@ -4,24 +4,25 @@ require "spec_helpers"
 
 describe Wayfarer::Middleware::Normalize do
   let(:task) { build(:task) }
-  subject(:normalize) { Wayfarer::Middleware::Normalize.new }
+  subject { described_class.new }
 
   describe "#call" do
     let(:urls) do
       ["http://example.com/products?product_id=123",
        "HTTP://EXAMPLE.COM/products/?product_id=123",
        "http://example.com/products/?product_id=123",
-       "http://example.com/foo/../products?product_id=123"]
+       "http://example.com/foo/../products?product_id=123",
+       "invalid@url-net"]
     end
 
     before { task.metadata.staged_urls = SortedSet.new(urls) }
 
     it "yields" do
-      expect { |spy| normalize.call(task, &spy) }.to yield_control
+      expect { |spy| subject.call(task, &spy) }.to yield_control
     end
 
     it "normalizes and compacts URLs" do
-      normalize.call(task)
+      subject.call(task)
       expect(task.metadata.staged_urls).to eq(SortedSet[urls.first])
     end
   end