wayfarer 0.4.2 → 0.4.6

data/lib/wayfarer/middleware/router.rb

@@ -3,24 +3,17 @@
 module Wayfarer
   module Middleware
     class Router
+      extend Base
+
       module API
         def self.included(base)
-          base.include(InstanceMethods)
           base.extend(ClassMethods)
-        end
-
-        module InstanceMethods
-          def steer
-            []
-          end
-
-          def params
-            task.metadata.params
-          end
+          base.include(InstanceMethods)
         end
 
         module ClassMethods
           def router
+            # TODO: Use cattr_accessor
             @router ||= Wayfarer::Routing::Router.new
           end
 
@@ -32,18 +25,31 @@ module Wayfarer
             define_method(:steer) { block.call(task) }
           end
         end
+
+        module InstanceMethods
+          def steer
+            []
+          end
+
+          def params
+            task.metadata.params
+          end
+        end
       end
 
       def call(task)
-        router = task.job.class.router
+        controller = task.metadata.controller
+        # TODO: The router has to be cloned because it's not thread-safe
+        router = controller.class.router.clone
         url = Addressable::URI.parse(task.url)
 
-        case result = router.invoke(url, task.job.steer)
+        case result = router.invoke(url, controller.steer)
         when Routing::Result::Mismatch
           return
         when Routing::Result::Match
           task.metadata.action = result.action
-          task.metadata.params = ActiveSupport::HashWithIndifferentAccess.new(result.params)
+          task.metadata.params ||= ActiveSupport::HashWithIndifferentAccess.new
+          task.metadata.params.merge!(result.params)
         end
 
         yield if block_given?

data/lib/wayfarer/middleware/stage.rb

@@ -3,9 +3,11 @@
 module Wayfarer
   module Middleware
     class Stage
+      extend Base
+
       module API
         def stage(urls)
-          Array.wrap(urls).each { |url| task.metadata.staged_urls.add(url) }
+          Array.wrap(urls).each { |url| task.metadata.staged_urls.add(url.to_s) }
         end
       end
 
@@ -15,8 +17,10 @@ module Wayfarer
         yield if block_given?
 
         task.metadata.staged_urls.each do |url|
-          task.job.class.crawl_later(url, batch: task.batch)
+          task.metadata.job.class.crawl(url, batch: task.batch)
         end
+
+        task.metadata.staged_urls.clear
       end
     end
   end

data/lib/wayfarer/networking/context.rb

@@ -25,10 +25,11 @@ module Wayfarer
 
       def supervise
         yield
-      rescue *strategy.renew_on => e
+      rescue *strategy.renew_on, *Wayfarer.config.network.renew_on => e
         renew
       ensure
         # If renewing raises, re-raise the originally caught exception
+        # TODO: Not nice this effectively swallows exceptions
         raise e if e
       end
     end

data/lib/wayfarer/networking/ferrum.rb

@@ -22,6 +22,7 @@ module Wayfarer
 
       def navigate(instance, url)
         instance.goto(url)
+        # instance.network.wait_for_idle
       end
 
       def live(instance)

data/lib/wayfarer/networking/follow.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Networking
+    RedirectsExhaustedError = Class.new(StandardError)
+
+    Follow = Struct.new(:context) do
+      extend Forwardable
+
+      delegate %i[live renew instance] => :context
+
+      def fetch(url, follow: 3)
+        raise RedirectsExhaustedError if follow.negative?
+
+        case result = context.fetch(url)
+        when Result::Success then result.page
+        when Result::Redirect then fetch(result.redirect_url, follow: follow - 1)
+        end
+      end
+    end
+  end
+end

data/lib/wayfarer/networking/result.rb

@@ -2,16 +2,8 @@
 
 module Wayfarer
   module Networking
-    # Wrapper structs for page retrieval results.
     module Result
-      # Signals that a page has been retrieved (2xx or no browser error).
-      # @!attribute [rw] page
-      #   @return [Page] the retrieved page.
       Success = Struct.new(:page)
-
-      # Signals that a URL resulted in response with 3xx.
-      # @!attribute [rw] redirect_url
-      #   @return [URI] Where to go next.
       Redirect = Struct.new(:redirect_url)
     end
   end

data/lib/wayfarer/routing/matchers/host.rb

@@ -6,7 +6,9 @@ module Wayfarer
       Host = Struct.new(:host) do
         # rubocop:disable Style/CaseEquality
         def match(url)
-          host === url.host
+          # url.host excludes the port
+          # TODO: Test case, docs
+          host === url.authority
         end
         # rubocop:enable Style/CaseEquality
 

data/lib/wayfarer/routing/matchers/path.rb

@@ -13,7 +13,7 @@ module Wayfarer
           @path = path
           @route = route
           @peeking = false
-          @matcher = Mustermann.new(path, type: "sinatra")
+          @matcher = Mustermann.new(path, type: "sinatra") # TODO: Add type constant
         end
 
         def match(url)

data/lib/wayfarer/routing/result.rb

@@ -3,12 +3,7 @@
 module Wayfarer
   module Routing
     module Result
-      # Signals that a URL matched a route.
-      # @!attribute [rw] action
-      #   @return [Page] the retrieved page.
       Match = Struct.new(:action, :params)
-
-      # Signals that a URL matched no route.
       Mismatch = Class.new
     end
   end

data/lib/wayfarer/routing/router.rb

@@ -3,11 +3,12 @@
 module Wayfarer
   module Routing
     class Router
-      ArgumentCountError = Class.new(StandardError)
+      ArgumentCountError = Class.new(StandardError) # TODO: Unused, remove
 
       extend Forwardable
 
-      attr_reader :root
+      attr_reader :root,
+                  :blocks
 
       def initialize
         @blocks = []

data/lib/wayfarer/serializer.rb

@@ -7,11 +7,11 @@ module Wayfarer
     end
 
     def serialize(task)
-      super(url: task.url, batch: task.batch)
+      super("url" => task.url, "batch" => task.batch)
     end
 
     def deserialize(hash)
-      Wayfarer::Task.new(*hash.values_at(:url, :batch))
+      Wayfarer::Task.new(*hash.values_at("url", "batch"))
     end
   end
 end

data/lib/wayfarer/task.rb

@@ -8,8 +8,6 @@ module Wayfarer
                 :batch,
                 :metadata
 
-    attr_accessor :job
-
     stringify :url,
               :batch
 
@@ -30,5 +28,9 @@ module Wayfarer
     def counter
       @counter ||= Wayfarer::Redis::Counter.new(batch)
     end
+
+    def gc
+      @gc ||= Wayfarer::GC.new(self)
+    end
   end
 end

data/lib/wayfarer.rb

@@ -8,7 +8,6 @@ require "uri"
 
 require "active_job"
 require "capybara"
-require "capybara/cuprite"
 require "connection_pool"
 require "docile"
 require "ferrum"
@@ -39,19 +38,13 @@ module Wayfarer
   module VERSION
     MAJOR = 0
     MINOR = 4
-    TINY = 2
+    TINY = 6
     STRING = [MAJOR, MINOR, TINY].join(".")
   end
 
   mattr_accessor :config, default: Wayfarer::Config::Root.new
 
-  mattr_accessor :middleware, default: [Wayfarer::Middleware::Stage.new,
-                                        Wayfarer::Middleware::Dedup.new,
-                                        Wayfarer::Middleware::Normalize.new,
-                                        Wayfarer::Middleware::Router.new,
-                                        Wayfarer::Middleware::Fetch.new]
-
-  UserAgentTimeoutError = Class.new(StandardError)
+  UserAgentTimeoutError = Class.new(StandardError) # TODO: Move to Networking namespace
 end
 
 loader.eager_load

data/spec/base_spec.rb

@@ -12,25 +12,14 @@ describe Wayfarer::Base, redis: true do
 
   before { stub_const("DummyJob", klass) }
 
-  describe "::crawl_later" do
+  describe "::crawl" do
     it "enqueues a task" do
       expect(DummyJob).to receive(:perform_later).with(task)
-      DummyJob.crawl_later(url, batch: batch)
-    end
-  end
-
-  describe "#perform" do
-    it "assigns itself to the task" do
-      job = DummyJob.new
-      job.perform(task)
-      expect(task.job).to be(job)
+      DummyJob.crawl(url, batch: batch)
     end
 
-    it "calls the middleware chain" do
-      job = DummyJob.new
-      allow(job).to receive(:chain).and_return(spy)
-      expect(job.chain).to receive(:call).with(task)
-      job.perform(task)
+    it "returns a task" do
+      expect(DummyJob.crawl(url)).to be_a(Wayfarer::Task)
     end
   end
 
@@ -40,21 +29,21 @@ describe Wayfarer::Base, redis: true do
     describe "after enqueue" do
       it "increments the counter" do
         expect {
-          DummyJob.crawl_later(url, batch: batch)
+          DummyJob.crawl(url, batch: batch)
         }.to change { counter.value }.by(1)
       end
     end
 
     describe "after perform" do
       it "decrements the counter" do
-        DummyJob.crawl_later(url, batch: batch)
+        DummyJob.crawl(url, batch: batch)
         task.counter.increment
         expect { perform_enqueued_jobs }.to change { task.counter.value }.by(-1)
       end
 
       context "when counter reaches 0" do
         it "resets the barrier" do
-          DummyJob.crawl_later(url, batch: batch)
+          DummyJob.crawl(url, batch: batch)
           perform_enqueued_jobs
           redis do |conn|
             expect(conn.exists?(task.barrier.redis_key)).to be(false)
@@ -62,7 +51,7 @@ describe Wayfarer::Base, redis: true do
         end
 
         it "resets the counter" do
-          DummyJob.crawl_later(url, batch: batch)
+          DummyJob.crawl(url, batch: batch)
           perform_enqueued_jobs
           redis do |conn|
             expect(conn.exists?(task.counter.redis_key)).to be(false)
@@ -72,7 +61,7 @@ describe Wayfarer::Base, redis: true do
         it "runs after batch callbacks" do
           expect { |spy|
             klass.after_batch(&spy)
-            DummyJob.crawl_later(url, batch: batch)
+            DummyJob.crawl(url, batch: batch)
             perform_enqueued_jobs
           }.to yield_control
         end
@@ -88,7 +77,7 @@ describe Wayfarer::Base, redis: true do
     end
 
     it "does not retry the job" do
-      DummyJob.crawl_later(url, batch: batch)
+      DummyJob.crawl(url, batch: batch)
 
       expect {
         begin
@@ -102,7 +91,7 @@ describe Wayfarer::Base, redis: true do
     it "decrements the counter" do
       3.times { task.counter.increment }
 
-      DummyJob.crawl_later(url, batch: batch)
+      DummyJob.crawl(url, batch: batch)
       begin
         perform_enqueued_jobs
       rescue StandardError
@@ -111,14 +100,6 @@ describe Wayfarer::Base, redis: true do
 
       expect(task.counter.value).to be(3)
     end
-
-    # it "runs after batch callbacks" do
-    #   expect do |spy|
-    #     klass.after_batch(&spy)
-    #     DummyJob.crawl_later(url, batch: batch)
-    #     3.times { perform_enqueued_jobs }
-    #   end.to yield_control
-    # end
   end
 
   describe "Retries" do
@@ -131,7 +112,13 @@ describe Wayfarer::Base, redis: true do
     end
 
     before do
-      allow_any_instance_of(DummyJob).to receive(:perform).and_raise(RuntimeError.new)
+      allow_any_instance_of(DummyJob).to receive(:perform) do |job|
+        task = job.arguments.first
+        task.metadata.job = job
+
+        raise RuntimeError
+      end
+
       stub_const("Spy", spy)
     end
 
@@ -140,7 +127,7 @@ describe Wayfarer::Base, redis: true do
                                    .with(kind_of(DummyJob),
                                          kind_of(RuntimeError))
 
-      DummyJob.crawl_later(url, batch: batch)
+      DummyJob.crawl(url, batch: batch)
 
       expect {
         perform_enqueued_jobs
@@ -157,7 +144,7 @@ describe Wayfarer::Base, redis: true do
 
     it "marks the URL seen" do
       task.counter.increment # otherwise barrier gets reset
-      DummyJob.crawl_later(url, batch: batch)
+      DummyJob.crawl(url, batch: batch)
       3.times { perform_enqueued_jobs }
       expect(task.barrier.seen?(task.url)).to be(true)
     end
@@ -165,7 +152,7 @@ describe Wayfarer::Base, redis: true do
     it "decrements the counter" do
       3.times { task.counter.increment }
 
-      DummyJob.crawl_later(url, batch: batch)
+      DummyJob.crawl(url, batch: batch)
       3.times { perform_enqueued_jobs }
 
       expect(task.counter.value).to be(3)
@@ -174,7 +161,7 @@ describe Wayfarer::Base, redis: true do
     it "runs after batch callbacks" do
       expect { |spy|
         klass.after_batch(&spy)
-        DummyJob.crawl_later(url, batch: batch)
+        DummyJob.crawl(url, batch: batch)
         3.times { perform_enqueued_jobs }
       }.to yield_control
     end
@@ -190,7 +177,13 @@ describe Wayfarer::Base, redis: true do
     end
 
     before do
-      allow_any_instance_of(DummyJob).to receive(:perform).and_raise(RuntimeError.new)
+      allow_any_instance_of(DummyJob).to receive(:perform) do |job|
+        task = job.arguments.first
+        task.metadata.job = job
+
+        raise RuntimeError
+      end
+
       stub_const("Spy", spy)
     end
 
@@ -199,7 +192,7 @@ describe Wayfarer::Base, redis: true do
                                    .with(kind_of(DummyJob),
                                          kind_of(RuntimeError))
 
-      DummyJob.crawl_later(url, batch: batch)
+      DummyJob.crawl(url, batch: batch)
 
       expect {
         perform_enqueued_jobs
@@ -208,24 +201,22 @@ describe Wayfarer::Base, redis: true do
 
     it "marks the URL seen" do
       task.counter.increment # otherwise barrier gets reset
-      DummyJob.crawl_later(url, batch: batch)
+      DummyJob.crawl(url, batch: batch)
       perform_enqueued_jobs
       expect(task.barrier.seen?(task.url)).to be(true)
     end
 
     it "decrements the counter" do
       3.times { task.counter.increment }
-
-      DummyJob.crawl_later(url, batch: batch)
+      DummyJob.crawl(url, batch: batch)
       perform_enqueued_jobs
-
       expect(task.counter.value).to be(3)
     end
 
     it "runs after batch callbacks" do
       expect { |spy|
         klass.after_batch(&spy)
-        DummyJob.crawl_later(url, batch: batch)
+        DummyJob.crawl(url, batch: batch)
         perform_enqueued_jobs
       }.to yield_control
     end

data/spec/cli/job_spec.rb

@@ -20,7 +20,11 @@ describe Wayfarer::CLI::Job, cli: true, redis: true do
 
   describe "job perform" do
     it "performs the worker" do
-      expect_any_instance_of(DummyJob).to receive(:perform).with(kind_of(Wayfarer::Task))
+      expect_any_instance_of(DummyJob).to receive(:perform).with(kind_of(Wayfarer::Task)) do |job|
+        task = job.arguments.first
+        task.metadata.job = job
+      end
+
       cli.start(["job", "perform", "DummyJob", url])
     end
 
@@ -39,13 +43,13 @@ describe Wayfarer::CLI::Job, cli: true, redis: true do
 
   describe "job enqueue" do
     it "enqueues the job" do
-      expect(DummyJob).to receive(:crawl_later).with(Addressable::URI.parse(url), batch: kind_of(String))
+      expect(DummyJob).to receive(:crawl).with(Addressable::URI.parse(url), batch: kind_of(String))
       cli.start(["job", "enqueue", "DummyJob", url])
     end
 
     context "with batch provided" do
       it "enqueues the job" do
-        expect(DummyJob).to receive(:crawl_later).with(Addressable::URI.parse(url), batch: batch)
+        expect(DummyJob).to receive(:crawl).with(Addressable::URI.parse(url), batch: batch)
         cli.start(["job", "enqueue", "--batch", batch, "DummyJob", url])
       end
     end
@@ -53,13 +57,13 @@ describe Wayfarer::CLI::Job, cli: true, redis: true do
 
   describe "job execute" do
     it "executes the job" do
-      expect(DummyJob).to receive(:crawl_later).with(Addressable::URI.parse(url), batch: kind_of(String))
+      expect(DummyJob).to receive(:crawl).with(Addressable::URI.parse(url), batch: kind_of(String))
       cli.start(["job", "execute", "DummyJob", url])
     end
 
     context "with batch provided" do
       it "enqueues the job" do
-        expect(DummyJob).to receive(:crawl_later).with(Addressable::URI.parse(url), batch: batch)
+        expect(DummyJob).to receive(:crawl).with(Addressable::URI.parse(url), batch: batch)
         cli.start(["job", "execute", "--batch", batch, "DummyJob", url])
       end
     end

data/spec/gc_spec.rb

@@ -6,12 +6,10 @@ describe Wayfarer::GC, redis: true do
   include Wayfarer::Redis::Connection
 
   let(:task) { build(:task) }
-  let(:klass) { Class.new(Wayfarer::Base) }
-  let(:job) { klass.new }
-  subject(:gc) { Wayfarer::GC.new(job) }
+  subject(:gc) { described_class.new(task) }
 
   before do
-    job.arguments = [task]
+    task.metadata.job = spy
     task.barrier.seen?(task.url)
   end
 
@@ -32,10 +30,8 @@ describe Wayfarer::GC, redis: true do
       end
 
       it "runs after batch callbacks" do
-        expect { |spy|
-          klass.after_batch(&spy)
-          gc.run
-        }.to yield_control
+        expect(task.metadata.job).to receive(:run_callbacks).with(:batch).exactly(:once)
+        gc.run
       end
     end
 
@@ -55,7 +51,7 @@ describe Wayfarer::GC, redis: true do
       end
 
       it "does not run after batch callbacks" do
-        expect(klass).not_to receive(:run_after_batch_callbacks)
+        expect(task.metadata.job).not_to receive(:run_callbacks).with(:batch)
         gc.run
       end
     end

data/spec/handler_spec.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Handler do
+  subject { Class.new(described_class) }
+
+  it "undefines ::after_batch" do
+    expect(subject).not_to respond_to(:after_batch)
+  end
+end

data/spec/integration/callbacks_spec.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Callbacks" do
+  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
+
+  describe Wayfarer::Base do
+    specify do
+      class self.class::DummyJob < Wayfarer::Base
+        extend SpecHelpers
+        include RSpec::Matchers
+
+        route { host test_app_host, to: :index }
+
+        attr_accessor :callbacks_fired
+
+        before_fetch do
+          self.callbacks_fired = %i[before_fetch]
+        end
+
+        before_action do
+          callbacks_fired.push(:before_action)
+        end
+
+        after_batch do
+          expect(callbacks_fired).to eq(%i[before_fetch before_action])
+        end
+
+        def index; end
+      end
+
+      self.class::DummyJob.crawl(url)
+      perform_enqueued_jobs
+    end
+  end
+
+  describe Wayfarer::Handler do
+    specify do
+      class self.class::DummyJob < Wayfarer::Base
+        extend SpecHelpers
+        include RSpec::Matchers
+
+        route { host test_app_host, to: DummyHandler }
+
+        attr_accessor :callbacks_fired
+
+        before_fetch do
+          self.callbacks_fired = %i[before_fetch_job]
+        end
+
+        before_action do
+          callbacks_fired.push(:before_action_job)
+        end
+
+        after_batch do
+          expect(callbacks_fired).to eq(%i[before_fetch_job before_action_job])
+        end
+
+        class DummyHandler < Wayfarer::Handler
+          include RSpec::Matchers
+
+          route { to :index }
+
+          attr_accessor :callbacks_fired
+
+          before_fetch do
+            raise "before_fetch ran in handler"
+          end
+
+          before_action do
+            self.callbacks_fired = %i[before_action_handler]
+          end
+
+          def index
+            expect(callbacks_fired).to eq(%i[before_action_handler])
+          end
+        end
+      end
+
+      self.class::DummyJob.crawl(url)
+      perform_enqueued_jobs
+    end
+  end
+end