wayfarer 0.4.1 → 0.4.2

data/lib/wayfarer/middleware/worker.rb

@@ -6,44 +6,24 @@ module Wayfarer
       def self.included(base)
         base.include(Wayfarer::Redis::Connection)
         base.include(Wayfarer::Middleware::Stage::API)
+        base.include(Wayfarer::Middleware::Router::API)
+        base.include(Wayfarer::Middleware::Fetch::API)
         base.include(Wayfarer::Callbacks)
         base.include(InstanceMethods)
-        base.extend(ClassMethods)
-      end
-
-      module ClassMethods
-        def route
-          @route ||= Wayfarer::Routing::RootRoute.new.tap do |route|
-            Docile.dsl_eval(route) { yield if block_given? }
-          end
-        end
       end
 
       module InstanceMethods
-        extend Forwardable
-
-        delegate %i[params agent] => "task.metadata"
-
         def call(task)
           run_callbacks :action do
             public_send(task.metadata.action)
-            yield if block_given? # TODO: Should be excluded from callback block
           end
+
+          yield if block_given?
         end
 
         def chain
           Wayfarer::Middleware::Chain.new([*Wayfarer.middleware, self])
         end
-
-        def browser
-          agent.instance
-        end
-
-        def page(live: false)
-          return task.metadata.page unless live
-
-          task.metadata.page = agent.live&.page || task.metadata.page
-        end
       end
     end
   end

data/lib/wayfarer/networking/pool.rb

@@ -4,22 +4,23 @@ module Wayfarer
   module Networking
     class Pool
       include Singleton
-      extend Forwardable
 
       cattr_accessor :registry, default: { http: HTTP,
                                            ferrum: Ferrum,
                                            selenium: Selenium,
                                            capybara: Capybara }
 
-      attr_reader :pool
-
-      def initialize
-        @pool = ConnectionPool.new(size: Wayfarer.config.network.pool_size,
-                                   timeout: Wayfarer.config.network.pool_timeout,
-                                   &method(:context))
+      def pool
+        @pool ||= ConnectionPool.new(size: Wayfarer.config.network.pool_size,
+                                     timeout: Wayfarer.config.network.pool_timeout,
+                                     &method(:context))
       end
 
-      delegate with: :pool
+      def with(&block)
+        pool.with(&block)
+      rescue ConnectionPool::TimeoutError => e
+        raise Wayfarer::UserAgentTimeoutError, e
+      end
 
       def free
         pool.shutdown(&:renew)

data/lib/wayfarer/page.rb

@@ -20,7 +20,7 @@ module Wayfarer
       # If no Content-Type field is present, assume HTML/XML
       return @doc = Wayfarer::Parsing::XML.parse_html(body) unless headers["content-type"]
 
-      content_type = headers["content-type"].first
+      content_type = headers["content-type"]
       sub_type = MIME::Types[content_type].first.sub_type
 
       @doc = case sub_type

data/lib/wayfarer/routing/matchers/custom.rb

@@ -4,6 +4,8 @@ module Wayfarer
   module Routing
     module Matchers
       class Custom
+        include Stringify
+
         attr_reader :delegate
 
         def initialize(delegate = proc)

data/lib/wayfarer/routing/matchers/path.rb

@@ -36,6 +36,7 @@ module Wayfarer
 
         def visit(route)
           return true if route == self.route
+          return true if route.is_a?(TargetRoute)
 
           return unless route.matcher.is_a?(self.class)
 

data/lib/wayfarer/routing/route.rb

@@ -4,6 +4,7 @@ module Wayfarer
   module Routing
     class Route
       include DSL
+      include Stringify
 
       attr_reader :children
 
@@ -12,6 +13,11 @@ module Wayfarer
                     :action,
                     :path_offset
 
+      stringify :matcher,
+                :action,
+                :path_offset,
+                :children
+
       def initialize(matcher = Matchers::Custom.new { children.any? }, path_offset = "/")
         @matcher = matcher
         @children = []

data/lib/wayfarer/routing/router.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    class Router
+      ArgumentCountError = Class.new(StandardError)
+
+      extend Forwardable
+
+      attr_reader :root
+
+      def initialize
+        @blocks = []
+      end
+
+      def draw(&block)
+        @blocks.push(block)
+      end
+
+      def invoke(url, arguments)
+        @root = Wayfarer::Routing::RootRoute.new
+        @blocks.each { |block| Docile.dsl_eval(@root, *arguments, &block) }
+        root.invoke(url)
+      end
+    end
+  end
+end

data/lib/wayfarer/stringify.rb

@@ -1,31 +1,38 @@
 # frozen_string_literal: true
 
+# 346325
+
 module Wayfarer
   module Stringify
     def self.included(base)
       base.include(InstanceMethods)
       base.extend(ClassMethods)
+      base.instance_eval do
+        cattr_accessor :stringified_attributes do
+          []
+        end
+      end
     end
 
     module ClassMethods
       def stringify(*variables)
         stringified_attributes.concat(variables)
       end
-
-      def stringified_attributes
-        @stringified_attributes ||= []
-      end
     end
 
     module InstanceMethods
       def to_s
-        "#<#{class_name} #{attributes}>"
+        if self.class.stringified_attributes.any?
+          "#<#{class_name} #{attributes.join(', ')}>"
+        else
+          "#<#{class_name}>"
+        end
       end
 
       alias inspect to_s
 
       def class_name
-        self.class.name.split("::").last
+        self.class.name
       end
 
       def attributes
@@ -34,7 +41,6 @@ module Wayfarer
             .map { |attr| [attr, public_send(attr)] }
             .to_h
             .map { |k, v| [k, "=", v.inspect].join }
-            .join(", ")
       end
     end
   end

data/lib/wayfarer.rb

@@ -39,7 +39,7 @@ module Wayfarer
   module VERSION
     MAJOR = 0
     MINOR = 4
-    TINY = 1
+    TINY = 2
     STRING = [MAJOR, MINOR, TINY].join(".")
   end
 
@@ -50,6 +50,8 @@ module Wayfarer
                                         Wayfarer::Middleware::Normalize.new,
                                         Wayfarer::Middleware::Router.new,
                                         Wayfarer::Middleware::Fetch.new]
+
+  UserAgentTimeoutError = Class.new(StandardError)
 end
 
 loader.eager_load

data/spec/callbacks_spec.rb

@@ -8,8 +8,8 @@ describe Wayfarer::Callbacks do
 
   let(:klass) do
     Class.new(Wayfarer::Base) do
-      route.host "alpha.com", to: :alpha
-      route.host "beta.com", to: :beta
+      route { host "alpha.com", to: :alpha }
+      route { host "beta.com", to: :beta }
 
       before_fetch do |job|
         Spy.before_fetch(self, job)

data/spec/config/networking_spec.rb

@@ -24,8 +24,8 @@ describe Wayfarer::Config::Networking do
 
   describe "#pool_size" do
     context "by default" do
-      it "is 3" do
-        expect(network.pool_size).to be(3)
+      it "is 1" do
+        expect(network.pool_size).to be(1)
       end
     end
 

data/spec/factories/{queue/middleware.rb → middleware.rb}

@@ -2,12 +2,12 @@
 
 FactoryBot.define do
   factory :middleware, class: OpenStruct do
-    callee { -> {} }
+    receiver { -> {} }
 
     initialize_with do
-      new(callee: callee).tap do |middleware|
+      new(receiver: receiver).tap do |middleware|
         middleware.define_singleton_method(:call) do |task, &block|
-          callee.call(task, &block)
+          receiver.call(task, &block)
         end
       end
     end

data/spec/factories/{queue/page.rb → page.rb}

@@ -8,7 +8,7 @@ FactoryBot.define do
     body { "" }
 
     trait :html do
-      headers { { "Content-Type" => %w[text/html] } }
+      headers { { "Content-Type" => "text/html" } }
       body do
         <<~HTML
           <!doctype html>
@@ -24,7 +24,7 @@ FactoryBot.define do
     end
 
     trait :xml do
-      headers { { "Content-Type" => %w[application/xml] } }
+      headers { { "Content-Type" => "application/xml" } }
       body do
         <<~XML
           <menu id="file" value="File">
@@ -39,7 +39,7 @@ FactoryBot.define do
     end
 
     trait :json do
-      headers { { "Content-Type" => %w[application/json] } }
+      headers { { "Content-Type" => "application/json" } }
       body do
         <<~JSON
           {

data/spec/fixtures/dummy_job.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class DummyJob < Wayfarer::Base
-  route.to :index
+  route { to :index }
 
   def hello; end
 end

data/spec/middleware/chain_spec.rb

@@ -4,7 +4,7 @@ require "spec_helpers"
 
 describe Wayfarer::Middleware::Chain do
   let(:task) { build(:task) }
-  subject(:chain) { build(:chain, middlewares: middlewares) }
+  subject(:chain) { described_class.new(middlewares) }
 
   describe "#call" do
     context "without middlewares" do
@@ -51,13 +51,13 @@ describe Wayfarer::Middleware::Chain do
 
     describe "Return value" do
       let(:middlewares) do
-        [build(:middleware, callee: lambda do |task, &block|
-                                      task.metadata.foobar = 42
-                                      block.call
-                                    end),
-         build(:middleware, callee: lambda do |task|
-                                      task.metadata.foobar *= 1337
-                                    end)]
+        [build(:middleware, receiver: lambda do |task, &block|
+                                        task.metadata.foobar = 42
+                                        block.call
+                                      end),
+         build(:middleware, receiver: lambda do |task|
+                                        task.metadata.foobar *= 1337
+                                      end)]
       end
 
       it "is the last middleware's return value" do
@@ -67,18 +67,18 @@ describe Wayfarer::Middleware::Chain do
 
     describe "Metadata" do
       let(:first) do
-        build(:middleware, callee: lambda do |task, &block|
-                                     task.metadata.foobar = 42
-                                     block.call
-                                     task.metadata.barqux = 1337
-                                   end)
+        build(:middleware, receiver: lambda do |task, &block|
+                                       task.metadata.foobar = 42
+                                       block.call
+                                       task.metadata.barqux = 1337
+                                     end)
       end
 
       let(:last) do
-        build(:middleware, callee: lambda do |task|
-                                     raise unless task.metadata.foobar == 42
-                                     raise if task.metadata.barqux
-                                   end)
+        build(:middleware, receiver: lambda do |task|
+                                       raise unless task.metadata.foobar == 42
+                                       raise if task.metadata.barqux
+                                     end)
       end
 
       let(:middlewares) { [first, last] }

data/spec/middleware/fetch_spec.rb

@@ -6,12 +6,14 @@ describe Wayfarer::Middleware::Fetch do
   let(:klass) { Class.new(Wayfarer::Base) }
   let(:job) { klass.new }
   let(:task) { build(:task) }
-  let(:agent) { spy }
+  let(:context) { double(instance: instance) }
+  let(:instance) { spy }
   subject(:fetch) { Wayfarer::Middleware::Fetch.new }
 
   before do
     task.job = job
-    allow(fetch.pool).to receive(:with).and_yield(agent)
+    allow(fetch.pool).to receive(:with).and_yield(context)
+    allow(context).to receive(:fetch)
   end
 
   describe "#call" do
@@ -21,14 +23,8 @@ describe Wayfarer::Middleware::Fetch do
       }.to change { fetch.task }.to(task)
     end
 
-    it "assigns the agent" do
-      expect {
-        fetch.call(task)
-      }.to change { task.metadata.agent }.to(agent)
-    end
-
     it "retrieves the page" do
-      expect(agent).to receive(:fetch).with(task.url)
+      expect(context).to receive(:fetch).with(task.url)
       fetch.call(task)
     end
 
@@ -48,7 +44,7 @@ describe Wayfarer::Middleware::Fetch do
 
       before do
         task.metadata.staged_urls = Set.new
-        allow(agent).to receive(:fetch).with(task.url).and_return(result)
+        allow(context).to receive(:fetch).with(task.url).and_return(result)
       end
 
       it "stages the redirect URL" do
@@ -67,7 +63,7 @@ describe Wayfarer::Middleware::Fetch do
       let(:result) { Wayfarer::Networking::Result::Success.new(page) }
 
       before do
-        allow(agent).to receive(:fetch).with(task.url).and_return(result)
+        allow(context).to receive(:fetch).with(task.url).and_return(result)
       end
 
       it "assigns the page" do
@@ -76,9 +72,29 @@ describe Wayfarer::Middleware::Fetch do
         }.to change { task.metadata.page }.to(page)
       end
 
+      it "assigns the instance" do
+        expect {
+          fetch.call(task)
+        }.to change { task.metadata.agent }.to(instance)
+      end
+
       it "yields" do
         expect { |spy| fetch.call(task, &spy) }.to yield_control
       end
     end
   end
+
+  describe described_class::API do
+    let(:klass) do
+      Class.new(Wayfarer::Base) do
+        route do |host, path|
+          to :index, host: host, path: path
+        end
+
+        steer do |task|
+          [task.batch, "/foobar"]
+        end
+      end
+    end
+  end
 end

data/spec/middleware/router_spec.rb

@@ -4,17 +4,22 @@ require "spec_helpers"
 
 describe Wayfarer::Middleware::Router do
   let(:task) { build(:task, url: url) }
-  let(:klass) do
-    Class.new(Wayfarer::Base) do
-      route.to :index, host: "example.com", path: "/foobar"
-    end
-  end
-
   subject(:router) { Wayfarer::Middleware::Router.new }
 
-  before { task.job = klass.new }
+  before do
+    task.job = klass.new
+    allow(task.job).to receive(:arguments).and_return([task])
+  end
 
   describe "#call" do
+    let(:klass) do
+      Class.new(Wayfarer::Base) do
+        route do
+          to :index, host: "example.com", path: "/foobar"
+        end
+      end
+    end
+
     context "when URL matches a route" do
       let(:url) { "https://example.com/foobar" }
 
@@ -50,4 +55,26 @@ describe Wayfarer::Middleware::Router do
       end
     end
   end
+
+  describe described_class::API do
+    let(:klass) do
+      Class.new(Wayfarer::Base) do
+        route do |host, path|
+          to :index, host: host, path: path
+        end
+
+        steer do |task|
+          [task.batch, "/foobar"]
+        end
+      end
+    end
+
+    let(:task) { build(:task, batch: "example.com", url: "https://example.com/foobar") }
+
+    it "passes arguments" do
+      router.call(task)
+      expect(task.metadata.action).to be(:index)
+      expect(task.metadata.params).to eq({})
+    end
+  end
 end

data/spec/middleware/worker_spec.rb

@@ -18,9 +18,9 @@ describe Wayfarer::Middleware::Worker do
     allow_any_instance_of(job).to receive(:arguments).and_return([task])
   end
 
-  describe "::route" do
-    it "returns a route" do
-      expect(job.route).to be_a(Wayfarer::Routing::Route)
+  describe "::router" do
+    it "returns a router" do
+      expect(job.router).to be_a(Wayfarer::Routing::Router)
     end
   end
 
@@ -67,16 +67,6 @@ describe Wayfarer::Middleware::Worker do
     end
   end
 
-  describe "#browser" do
-    it "returns the context's instance" do
-      worker = job.new
-      context = spy
-      expect(worker.task.metadata).to receive(:agent).and_return(context)
-      expect(context).to receive(:instance)
-      worker.browser
-    end
-  end
-
   describe "#page" do
     let(:worker) { job.new }
     let(:page) { Object.new }

data/spec/routing/router_spec.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Routing::Router do
+  subject(:router) do
+    described_class.new.tap do |router|
+      router.draw do |alpha, beta|
+        path alpha, to: :alpha
+        path beta, to: :beta
+      end
+    end
+  end
+
+  describe "#invoke" do
+    let(:url) { Addressable::URI.parse("https://example.com/alpha") }
+
+    it "evaluates routes" do
+      result = router.invoke(url, %w[/alpha /beta])
+      expect(result).to be_a(Wayfarer::Routing::Result::Match)
+      expect(result.action).to be(:alpha)
+    end
+  end
+end

data/wayfarer.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = "wayfarer"
-  s.version = "0.4.1"
+  s.version = "0.4.2"
   s.license = "MIT"
 
   s.homepage = "http://github.com/bauerd/wayfarer"

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: wayfarer
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Dominic Bauer
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
 date: 2021-09-26 00:00:00.000000000 Z
@@ -388,6 +388,9 @@ files:
 - Rakefile
 - bin/wayfarer
 - docker-compose.yml
+- docs/cookbook/batch_routing.md
+- docs/cookbook/consent_screen.md
+- docs/cookbook/executing_javascript.md
 - docs/cookbook/querying_html.md
 - docs/cookbook/screenshots.md
 - docs/cookbook/user_agent.md
@@ -397,11 +400,15 @@ files:
 - docs/guides/browser_automation/selenium.md
 - docs/guides/callbacks.md
 - docs/guides/configuration.md
+- docs/guides/debugging.md
 - docs/guides/error_handling.md
 - docs/guides/jobs.md
+- docs/guides/navigation.md
 - docs/guides/networking.md
 - docs/guides/pages.md
 - docs/guides/performance.md
+- docs/guides/reliability.md
+- docs/guides/routing/steering.md
 - docs/guides/tasks.md
 - docs/index.md
 - docs/reference/api/base.md
@@ -464,6 +471,7 @@ files:
 - lib/wayfarer/routing/result.rb
 - lib/wayfarer/routing/root_route.rb
 - lib/wayfarer/routing/route.rb
+- lib/wayfarer/routing/router.rb
 - lib/wayfarer/routing/target_route.rb
 - lib/wayfarer/serializer.rb
 - lib/wayfarer/stringify.rb
@@ -483,10 +491,9 @@ files:
 - spec/config/selenium_spec.rb
 - spec/config/strconv_spec.rb
 - spec/config/struct_spec.rb
-- spec/factories/queue/chain.rb
-- spec/factories/queue/middleware.rb
-- spec/factories/queue/page.rb
-- spec/factories/queue/task.rb
+- spec/factories/middleware.rb
+- spec/factories/page.rb
+- spec/factories/task.rb
 - spec/fixtures/dummy_job.rb
 - spec/gc_spec.rb
 - spec/middleware/chain_spec.rb
@@ -522,6 +529,7 @@ files:
 - spec/routing/path_finder_spec.rb
 - spec/routing/root_route_spec.rb
 - spec/routing/route_spec.rb
+- spec/routing/router_spec.rb
 - spec/spec_helpers.rb
 - spec/stringify_spec.rb
 - spec/support/static/finders.html
@@ -539,7 +547,7 @@ homepage: http://github.com/bauerd/wayfarer
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -555,7 +563,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.6
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Versatile web crawling with Ruby
 test_files: []

data/spec/factories/queue/chain.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-FactoryBot.define do
-  factory :chain, class: Wayfarer::Middleware::Chain do
-    middlewares { [] }
-
-    initialize_with do
-      new(middlewares)
-    end
-  end
-end