wayfarer 0.4.8 → 0.4.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 677b57fadbc7e01a8adfbc5c2280dfe81aa0de2fce3b778281acc33d42a9a8f3
-  data.tar.gz: 22c93784493e7a551c19c1f070defa7de8a99d14e8ee740cf866d62dbb9542b9
+  metadata.gz: 102ad4ff568c6a84fb710cbb46f3ca4839ce6a319265d33953f5ed601038d95d
+  data.tar.gz: 22f2f2460eb814a5534567cfc4222663d5819739df76e8c07bd0e5add4de15ec
 SHA512:
-  metadata.gz: 2c232c7760c3914f56b742ddbcdd7f9719e98a02f0019ffc14ec9f883499f63aaa9a92afb95c241b4d79a8369f8fb53727c9d7d5d694b79786a6d90753a9bc3b
-  data.tar.gz: 680a7e146e41d3eaf9c8bc48362a9ae8d59dd94baf26b436ee14e47329cfd924c83299aef5f59f10c2e1227f8e08170984885341e79222345ab9bbb460da0f1c
+  metadata.gz: 7c3c4780264a54b3ffc3b885027affb00d041c7d0d828435987d71d19cdaba03d69a03dda645901abfa59ab8f5b4f702dd85722fec87efb36a08d304a903296b
+  data.tar.gz: e05ab817b1125ebc8b644b79ec63e18fce3566ffdf4508e6409d5e8669a060b0bbd88e30a782db7f6abc9555198e1919fbb0feda4d9b75aba8f5b8823592c294

data/.github/workflows/release.yaml

@@ -13,8 +13,6 @@ jobs:
 
       - run: docker compose exec -T wayfarer git config --global --add safe.directory /opt/app
 
-      - run: git status
-
       - name: Release gem
         env:
           GIT_AUTHOR_NAME: Dominic Bauer

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    wayfarer (0.4.8)
+    wayfarer (0.4.9)
       activejob (>= 7.1)
       addressable (~> 2.8)
       capybara (~> 3.4)

data/docs/guides/networking/custom_adapters.md

@@ -1,9 +1,11 @@
 # User agent API
 
-Wayfarer retrieves web pages with user agents. There are two types of user
-agents: __stateful__ browsers which carry state and follow redirects implicitly
-as they navigate to a URL, and __stateless__ HTTP clients, which handle
-redirects explicitly.
+Wayfarer retrieves URL contents with user agents. It supports two types of user
+agents:
+
+* __stateless__ HTTP clients which handle redirects explicitly.
+* __stateful__ browsers which carry state and follow redirects implicitly
+as they navigate to a URL
 
 |                   | Stateless adapters | Stateful adapters |
 |-------------------|--------------------|-------------------|
@@ -12,151 +14,156 @@ redirects explicitly.
 
 Because spawning browser processes or instantiating HTTP clients is expensive,
 Wayfarer keeps user agents in a pool and reuses them across jobs. This means
-that browser state carries over between jobs, as a job checks out a previous
-job's user agent. Only on certain irrecoverable errors are individual user agents
+that browser state carries over between jobs, as each job checks out a previous
+job's user agent. When exceptions are raised, you must [handle them](/guides/jobs/#handling-errors).
+
+Only on certain irrecoverable errors are individual user agents
 destroyed and recreated. For example when a browser process crashes, it is
 replaced with a fresh browser process.
 
-## Base interface for custom user agents
+## Implementing the user agent interfaces
 
 You implement both stateful and stateless agents by including the
-`Wayfarer::Networking::Strategy` module and defining callback methods. The
-interfaces for stateful and stateless share the following base methods:
+`Wayfarer::Networking::Strategy` module and defining callback methods:
 
 ```mermaid
 classDiagram
-class Square~Shape~{
-    int id
-    List~int~ position
-    setPoints(List~int~ points)
-    getPoints() List~int~
+class BaseAgent {
+    <<Interface>>
+    +#create()*
+    +#destroy(instance)*
+    +::renew_on()$
+}
+
+class StatefulAgent {
+    <<Interface>>
+    +#navigate(instance, url)*
+    +#live(instance)*
 }
 
-Square : -List~string~ messages
-Square : +setMessages(List~string~ messages)
-Square : +getMessages() List~string~
-Square : +getDistanceMatrix() List~List~int~~
+class StatelessAgent {
+    <<Interface>>
+    +#fetch(instance, url)*
+}
+
+BaseAgent |>.. StatefulAgent : implements
+BaseAgent |>.. StatelessAgent : implements
 ```
 
-* `#create` (__required__): Called when a new instance (browser process or HTTP client) is
-  needed.
-* `#destroy(instance)` (optional): Called when an instance should be destroyed. Browser
-  processes should be quit, and HTTP clients should be freed.
-* `#renew_on` (optional): Returns a list of exception classes upon which the existing
-  instance gets destroyed and replaced with a newly created one.
+Every user agent implementation must provide the `#create` instance callback
+which returns an initialized user agent. Typically, the optional
+`#destroy(instance)` instance callback is also implemented to free resources
+of an existing user agent.
 
-## Stateless interface
+Also a class method `::renew_on` can be defined which returns an array of
+exception classes upon which an instance of the user agent should get recreated
+(destroy-and-create).
 
-The stateless interface indicate HTTP 3xx redirect responses explicitly. This is how
-Wayfarer provides redirect handling out of the box, as there is a configurable limit
-on the number of retries to follow.
+### Stateless interface
 
-In addition to the base interface, stateless user agents implement `#fetch`
+In addition to the base interface, stateless user agents implement `#fetch(instance, url)`
 which fetches [pages](../pages) or indicates redirects:
 
-* `#fetch(instance, url)` (__required__): Called to retrieve a URL. Responses with a
+* `#create()` __(required)__
+* `#fetch(instance, url)` __(required)__: Called to retrieve a URL. Responses with a
   3xx status code must indicate the redirect URL by returning `redirect(url)`, since Wayfarer
   deals with redirects on your behalf to avoid redirect loops. All other status
   codes, including 4xx and 5xx, are considered successful and are indicated by calling
   `success(url:, body:, status_code:, headers:)`.
+* `#destroy(instance)` (optional)
+* `::renew_on` (optional)
 
-## Stateful interface
+The stateless interface indicate HTTP 3xx redirect responses explicitly. This is how
+Wayfarer provides redirect handling out of the box, as there is a configurable limit
+on the number of retries to follow.
 
-In addition to the base interface, stateful user agents implement two additional
-methods:
+!!! info "`redirect(url)` enqueues a task for the URL and stops processing the current task."
 
-* `#navigate(instance, url)` (__required__): Navigates the user agent to the given URL.
-  Stateful user agents follow redirects implicitly.
-* `#live(instance) -> Wayfarer::Page` (__required__): Turns the current user agent state
-  into a [page](../pages).
+!!! info "Pages with HTTP error status codes get routed"
 
-## Recreating user agents on error with `#renew_on`
+    If a HTTP request to a URL results in an error status code (for example, 404),
+    page retrieval is considered successful. This allows job actions to record
+    such data.
 
-Agents can optionally implement `#renew_on` to get themselves rereated on
-certain errors.
+### Stateful interface
 
-If `#fetch` or `#navigate` raise an exception and the exception class is listed
-in `#renew_on`, the instance is destroyed and recreated.
+In addition to the base interface, stateful user agents implement two additional
+instance callbacks:
 
-* `#renew_on` (optional): A list of exception classes upon which the existing instance gets
-  destroyed and replaced with a newly created one.
+* `#create()` __(required)__
+* `#navigate(instance, url)` __(required)__: Navigates the user agent to the given URL.
+* `#live(instance)` __(required)__: Turns the current user agent state
+into a [page](../pages) by calling `success(url:, body:, status_code:, headers:)`.
+* `#destroy(instance)` (optional)
+* `::renew_on` (optional)
 
 ## Example implementations
 
-=== "Stateful"
+=== "Stateless"
 
     ```ruby
-    class StatefulAgent
+    class StatelessAgent
       include Wayfarer::Networking::Strategy
 
-      # Required methods
-
-      def create
-        MyBrowser.new
+      def self.renew_on # optional
+        [MyIrrecoverableError]
       end
 
-      def navigate(browser, url)
-        browser.goto(url)
+      def create # required
+        MyClient.new
       end
 
-      def live(browser)
-        success(url: browser.url,
-                body: browser.body,
-                status_code: browser.status_code,
-                headers: browser.headers)
-      end
+      def fetch(client, url) # required
+        response = client.get(url)
 
-      # Optional methods
+        return redirect(response.redirect_url) if response.redirect?
 
-      def destroy(browser)
-        browser.quit
+        success(url: url,
+                body: response.body,
+                status_code: response.status_code,
+                headers: response.headers)
       end
 
-      def renew_on
-        [MyBrowser::IrrecoverableError]
+      def destroy(client) # optional
+        client.close
       end
     end
     ```
 
-=== "Stateless"
+=== "Stateful"
 
     ```ruby
-    class StatelessAgent
+    class StatefulAgent
       include Wayfarer::Networking::Strategy
 
-      # Required methods
-
-      def create
-        MyClient.new
+      def self.renew_on # optional
+        [MyIrrecoverableError]
       end
 
-      def fetch(client, url)
-        response = client.get(url)
-
-        return redirect(response.redirect_url) if response.redirect?
-
-        success(url: url,
-                body: response.body,
-                status_code: response.status_code,
-                headers: response.headers)
+      def create # required
+        MyBrowser.new
       end
 
-      # Optional methods
+      def navigate(browser, url) # required
+        browser.goto(url)
+      end
 
-      def destroy(client)
-        client.close
+      def live(browser) # required
+        success(url: browser.url,
+                body: browser.body,
+                status_code: browser.status_code,
+                headers: browser.headers)
       end
 
-      def renew_on # optional
-        [MyClient::IrrecoverableError]
+      def destroy(browser) # optional
+        browser.quit
       end
     end
     ```
 
-
-Register and use the strategy:
+Register and use the agent:
 
 ```ruby
-Wayfarer::Networking::Pool.registry[:my_agent] = MyAgent.new
+Wayfarer.config[:network][:agents][:my_agent] = MyAgent.new
 Wayfarer.config[:network][:agent] = :my_agent
 ```

data/docs/guides/networking/http.md

@@ -6,7 +6,7 @@ also alongside automated browsers.
 ## Ad-hoc GET requests
 
 When automating browsers, it can be useful to additionally retrieve another page
-over plain HTTP. Jobs can fetch URLs to [pages](/pages) with `#http`:
+or even the current page over plain HTTP. Jobs can fetch URLs to [pages](/pages) with `#http`:
 
 ```ruby
 class DummyJob < Wayfarer::Base

data/docs/guides/routing.md

@@ -9,7 +9,7 @@ Actions are instance methods denoted by symbols, or [handlers](/guides/handlers)
     By default, Wayfarer [applies some transformations to each URL](../tasks/#url-normalization) to bring it
     into a canonical form. Routing happens based on this canonical form.
 
-    You can always access a task's raw string as it was enqueued with `task.batch`.
+    You can always access a task's raw string as it was enqueued with `task.url`.
 
 A job's route declarations equate to a predicate tree.
 When a URL is routed, the predicate tree is searched depth-first. If a

data/lib/wayfarer/middleware/base.rb

@@ -12,10 +12,6 @@ module Wayfarer
 
         Module.new
       end
-
-      def lazy
-        Wayfarer::Middleware::Lazy.new(self)
-      end
     end
   end
 end

data/lib/wayfarer/middleware/controller.rb

@@ -18,7 +18,7 @@ module Wayfarer
 
       class_methods do
         def use(middleware)
-          chain.push(middleware.lazy)
+          chain.push(middleware.new)
           api(middleware)
         end
 

data/lib/wayfarer/networking/context.rb

@@ -36,7 +36,7 @@ module Wayfarer
 
       def supervise
         yield
-      rescue *strategy.renew_on, *Wayfarer.config[:network][:renew_on] => e
+      rescue *strategy.class.renew_on, *Wayfarer.config[:network][:renew_on] => e
         renew # may raise
       ensure
         # If renewing raises, re-raise the originally caught exception

data/lib/wayfarer/networking/ferrum.rb

@@ -5,7 +5,7 @@ module Wayfarer
     class Ferrum
       include Strategy
 
-      def renew_on
+      def self.renew_on
         [::Ferrum::DeadBrowserError]
       end
 

data/lib/wayfarer/networking/pool.rb

@@ -10,14 +10,6 @@ module Wayfarer
                       instance_accessor: false,
                       instance_predicate: false
 
-      class_attribute :registry,
-                      default: { http: HTTP,
-                                 ferrum: Ferrum,
-                                 selenium: Selenium,
-                                 capybara: Capybara },
-                      instance_accessor: false,
-                      instance_predicate: false
-
       attr_reader :pool
 
       def initialize
@@ -43,7 +35,9 @@ module Wayfarer
       end
 
       def strategy
-        self.class.registry[Wayfarer.config.dig(:network, :agent)].new
+        Wayfarer.config.dig(:network, :agent).then do |agent|
+          Wayfarer.config.dig(:network, :agents).fetch(agent).new
+        end
       end
     end
   end

data/lib/wayfarer/networking/strategy.rb

@@ -3,8 +3,12 @@
 module Wayfarer
   module Networking
     module Strategy
-      def renew_on
-        []
+      extend ActiveSupport::Concern
+
+      class_methods do
+        def renew_on
+          []
+        end
       end
 
       def fetch(instance, url)

data/lib/wayfarer/parsing.rb

@@ -12,7 +12,7 @@ module Wayfarer
 
     # @!visibility private
     def parse(body, content_type = FALLBACK_CONTENT_TYPE)
-      parser, args = Wayfarer.config.dig(:parsing, :registry, content_type)
+      parser, args = Wayfarer.config.dig(:parsing, :content_types, content_type)
       return unless parser
 
       parser.parse(body, *args)

data/lib/wayfarer.rb

@@ -9,8 +9,7 @@ require "uri"
 require "yaml"
 
 require "active_job"
-require "active_support/core_ext/array/wrap"
-require "active_support/core_ext/object/deep_dup"
+require "active_support/all"
 require "capybara"
 require "connection_pool"
 require "ferrum"
@@ -42,7 +41,7 @@ module Wayfarer
   module VERSION
     MAJOR = 0
     MINOR = 4
-    TINY = 8
+    TINY = 9
     STRING = [MAJOR, MINOR, TINY].join(".")
   end
 
@@ -53,6 +52,12 @@ module Wayfarer
     },
     network: {
       agent: :http,
+      agents: {
+        http: Wayfarer::Networking::HTTP,
+        ferrum: Wayfarer::Networking::Ferrum,
+        selenium: Wayfarer::Networking::Selenium,
+        capybara: Wayfarer::Networking::Capybara
+      },
       pool: {
         size: 1,
         timeout: 10
@@ -61,7 +66,7 @@ module Wayfarer
       renew_on: []
     },
     parsing: {
-      registry: {
+      content_types: {
         "application/json" => Wayfarer::Parsing::JSON,
         "text/html" => [Wayfarer::Parsing::XML, :html],
         "application/xml" => [Wayfarer::Parsing::XML, :xml]

data/rake/release.rake

@@ -12,12 +12,6 @@ namespace :release do
 
     raise "Gem version #{gem_version} deviates from library version #{lib_version}" unless gem_version == lib_version
   end
-
-  task :guard_debug do
-    sh "git diff --exit-code"
-    sh "git diff-index --quiet --cached HEAD"
-  end
 end
 
 Rake::Task[:"release:rubygem_push"].enhance(%i[release:guard_versions])
-Rake::Task[:"release:guard_clean"].enhance(%i[release:guard_debug])

data/spec/wayfarer/integration/page_spec.rb

@@ -8,7 +8,7 @@ describe "Pages", :redis do
     mock_handler! :dummy_handler
   end
 
-  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
+  let(:url) { test_app_path("finders.html") }
 
   shared_examples "executes" do
     specify do
@@ -29,14 +29,14 @@ describe "Pages", :redis do
         route.host test_app_host, to: :index
 
         def index
-          expect(page.url).to eq(test_app_path("git-scm.com/book/en/v2.html"))
+          expect(page.url).to eq(test_app_path("finders.html"))
           expect(page.status_code).to be(200)
           expect(page.body).not_to be_empty
           expect(page.headers.count).to be(9)
 
-          expect(page.meta.links.all.count).to be(157)
-          expect(page.meta.links.internal.count).to be(102)
-          expect(page.meta.links.external.count).to be(55)
+          expect(page.meta.links.all.count).to be(6)
+          expect(page.meta.links.internal.count).to be(3)
+          expect(page.meta.links.external.count).to be(3)
         end
       end
     end
@@ -55,14 +55,14 @@ describe "Pages", :redis do
         route.to :index
 
         def index
-          expect(page.url).to eq(test_app_path("git-scm.com/book/en/v2.html"))
+          expect(page.url).to eq(test_app_path("finders.html"))
           expect(page.status_code).to be(200)
           expect(page.body).not_to be_empty
           expect(page.headers.count).to be(9)
 
-          expect(page.meta.links.all.count).to be(157)
-          expect(page.meta.links.internal.count).to be(102)
-          expect(page.meta.links.external.count).to be(55)
+          expect(page.meta.links.all.count).to be(6)
+          expect(page.meta.links.internal.count).to be(3)
+          expect(page.meta.links.external.count).to be(3)
         end
       end
 

data/spec/wayfarer/integration/parsing_spec.rb

@@ -72,8 +72,9 @@ describe "Response body parsing", :redis do
       end
 
       before do
-        Wayfarer.config[:parsing][:registry]["foo/bar"] = parser
-        Wayfarer.config[:parsing][:registry]["bar/qux"] = [parser_with_options, :ok]
+        Wayfarer.config[:parsing][:content_types]["foo/bar"] = parser
+        Wayfarer.config[:parsing][:content_types]["bar/qux"] = [parser_with_options, :ok]
+
         DummyJob.class_eval do
           route.to :index
 

data/spec/wayfarer/integration/stage_spec.rb

@@ -3,7 +3,7 @@
 require "spec_helpers"
 
 describe "Staging", :redis do
-  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
+  let(:url) { test_app_path("finders.html") }
 
   before do
     stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
@@ -27,7 +27,7 @@ describe "Staging", :redis do
       expect {
         DummyJob.crawl(url)
         perform_enqueued_jobs
-      }.to change(enqueued_jobs, :size).by(157)
+      }.to change(enqueued_jobs, :size).by(6)
     end
   end
 
@@ -52,7 +52,7 @@ describe "Staging", :redis do
       expect {
         DummyJob.crawl(url)
         perform_enqueued_jobs
-      }.to change(enqueued_jobs, :size).by(157)
+      }.to change(enqueued_jobs, :size).by(6)
     end
   end
 end

data/spec/wayfarer/middleware/controller_spec.rb

@@ -16,16 +16,15 @@ describe Wayfarer::Middleware::Controller do
   end
 
   describe "::use" do
-    it "adds middleware" do
+    specify do
       expect {
         controller.use(middleware)
       }.to change { controller.chain.middlewares.count }.by(1)
     end
 
-    it "adds lazy middleware" do
+    specify :aggregate_failures do
       controller.use(middleware)
-      expect(controller.chain.middlewares.last).to be_a(Wayfarer::Middleware::Lazy)
-      expect(controller.chain.middlewares.last.klass).to be(middleware)
+      expect(controller.chain.middlewares.last).to be_a(middleware)
       expect_any_instance_of(middleware).to receive(:call).with(task) # rubocop:disable RSpec/AnyInstance
       controller.new.call(task)
     end

data/spec/wayfarer/networking/context_spec.rb

@@ -6,11 +6,12 @@ describe Wayfarer::Networking::Context do
   subject(:context) { described_class.new(strategy) }
 
   let(:instance) { Object.new }
+  let(:class_double) { double(renew_on: [renewing_error]) } # # rubocop:disable RSpec/VerifiedDoubles
   let(:strategy) do
     double( # rubocop:disable RSpec/VerifiedDoubles
       create: instance,
       destroy: nil,
-      renew_on: [renewing_error],
+      class: class_double,
       fetch: nil,
       live: nil
     )
@@ -56,7 +57,7 @@ describe Wayfarer::Networking::Context do
       end
 
       before do
-        allow(strategy).to receive(:renew_on).and_return([renewing_error, other_error])
+        allow(class_double).to receive(:renew_on).and_return([renewing_error, other_error])
         allow(strategy).to receive(:fetch).and_raise(other_error)
       end
 

data/spec/wayfarer/networking/pool_spec.rb

@@ -54,7 +54,7 @@ describe Wayfarer::Networking::Pool do
     let(:strategy) { double(create: nil, destroy: nil) } # rubocop:disable RSpec/VerifiedDoubles
 
     before do
-      pool.class.registry[:foobar] = double(new: strategy) # rubocop:disable RSpec/VerifiedDoubles
+      Wayfarer.config[:network][:agents][:foobar] = double(new: strategy) # rubocop:disable RSpec/VerifiedDoubles
       pool.class.finalizer = ->(pool) { pool.reload(&:renew) }
 
       Wayfarer.config[:network][:agent] = :foobar

data/spec/wayfarer/networking/strategy.rb

@@ -15,9 +15,9 @@ RSpec.shared_examples "Network strategy" do |options|
   end
   # rubocop:enable RSpec/BeforeAfterAll
 
-  describe "#renew_on" do
+  describe "::renew_on" do
     it "returns an Array" do
-      expect(context.strategy.renew_on).to be_an(Array)
+      expect(context.strategy.class.renew_on).to be_an(Array)
     end
   end
 

data/wayfarer.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = "wayfarer"
-  s.version = "0.4.8"
+  s.version = "0.4.9"
   s.license = "MIT"
 
   s.homepage = "http://github.com/bauerd/wayfarer"