wayfarer 0.4.8 → 0.4.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 677b57fadbc7e01a8adfbc5c2280dfe81aa0de2fce3b778281acc33d42a9a8f3
-  data.tar.gz: 22c93784493e7a551c19c1f070defa7de8a99d14e8ee740cf866d62dbb9542b9
+  metadata.gz: 39fd7cefe2a4a7e07e2c3f69c0d13f96603e983432c56616444407a04af55442
+  data.tar.gz: 465b842759a03128413d66ff59b5116f0f7d444b193ad0929ed3a3914f55d5ad
 SHA512:
-  metadata.gz: 2c232c7760c3914f56b742ddbcdd7f9719e98a02f0019ffc14ec9f883499f63aaa9a92afb95c241b4d79a8369f8fb53727c9d7d5d694b79786a6d90753a9bc3b
-  data.tar.gz: 680a7e146e41d3eaf9c8bc48362a9ae8d59dd94baf26b436ee14e47329cfd924c83299aef5f59f10c2e1227f8e08170984885341e79222345ab9bbb460da0f1c
+  metadata.gz: 21111b9d58ab7d0286f2f0a8c575485f0553cc4ddb8f19ebcad9f82f67e70dd53290832de6a4da137839a22151f0d06a47b0c606bc139b1491c5ce3d2accf188
+  data.tar.gz: '049cc40ee6973ecc66ce921d1ca8bf746ae9066ef88ebe3a364a0b859d125d9857daba149dfffd0b630f9addf88d2330ca3f9de5e57dab521d1dc2908a1d5cde'

data/.github/workflows/release.yaml

@@ -13,8 +13,6 @@ jobs:
 
       - run: docker compose exec -T wayfarer git config --global --add safe.directory /opt/app
 
-      - run: git status
-
       - name: Release gem
         env:
           GIT_AUTHOR_NAME: Dominic Bauer

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    wayfarer (0.4.8)
+    wayfarer (0.4.10)
       activejob (>= 7.1)
       addressable (~> 2.8)
       capybara (~> 3.4)

data/docs/guides/development.md

@@ -2,21 +2,11 @@
 
 ## Release Procedure
 
-1. Ensure `Wayfarer::VERSION` was bumped appropriately.
-2. Ensure the version in wayfarer.gemspec matches.
-3. Open a release Pull Request develop -> master branch
-4. Merge the Pull Request
-5. Publish RubyGem and git tag as follows:
-
-```
-git checkout master
-git pull origin master --rebase
-bundle exec rake build
-gem push build/wayfarer-*.gem
-bundle exec rake clean
-git tag <VERSION>
-git push origin <VERSION>
-```
+1. Bump versions:
+  * `Wayfarer::VERSION` in `lib/wayfarer.rb`
+  * RubyGem version in `wayfarer.gemspec`
+  * Run `bundle install` to regenerate `Gemfile.lock`
+2. Push to `master` or `develop` and run the manual `release` worflow
 
 ## Conventions and guidelines
 
@@ -27,10 +17,11 @@ git push origin <VERSION>
 
 ### Navigate the web along URL patterns
 
-URLs are less prone to change than served markup.
-One reason for this is that changes to a URL's path can have negative
-consequences for its page ranking in search engines. Websites naturally implement
-architectural URL patterns like REST or expose surrogate keys.
+URLs are less prone to change than served markup. There are also SEO
+incentives to keep their paths stable. Since websites naturally implement
+architectural URL patterns like REST on the application layer, URL structure
+reflects internal domain concepts necessarily, for which Wayfarer's URL-based
+routing is designed.
 
 ### Follow URLs verbatim as they appear in responses
 
@@ -49,17 +40,18 @@ is ephemeral.
 
 There are two core features that depend on Redis. First, per-batch acylicity is
 achieved by maintaining the set of processed URLs per batch in Redis.
-There's no option to follow links in a cyclic manner. Second, batch completion
-requires updating an integer value in Redis, and batch completion is a very
-useful feature, since most crawls should end eventually, and often you want to
-know when.
+There's no configuration option to follow links in a cyclic manner. Second,
+batch completion requires updating an integer value in Redis, and batch completion
+is a very useful feature, since most crawls should end eventually, and often you want
+to know when.
 
 ### No configuration files
 
-Wayfarer can be configured through `Wayfarer.config` only, because `Wayfarer.config`
-may contain Ruby objects that don't de/serialize well, such as `Proc`s or `Set`s.
+Wayfarer can be configured through `Wayfarer.config` in Ruby code only,
+because `Wayfarer.config` may contain Ruby objects that don't de/serialize well,
+such as `Proc`s or `Set`s.
 
-### Features out of scope
+### Other features that are out of scope
 
 Wayfarer won't provide:
 

data/docs/guides/networking/custom_adapters.md

@@ -1,9 +1,11 @@
 # User agent API
 
-Wayfarer retrieves web pages with user agents. There are two types of user
-agents: __stateful__ browsers which carry state and follow redirects implicitly
-as they navigate to a URL, and __stateless__ HTTP clients, which handle
-redirects explicitly.
+Wayfarer retrieves URL contents with user agents. It supports two types of user
+agents:
+
+* __stateless__ HTTP clients which handle redirects explicitly.
+* __stateful__ browsers which carry state and follow redirects implicitly
+as they navigate to a URL
 
 |                   | Stateless adapters | Stateful adapters |
 |-------------------|--------------------|-------------------|
@@ -12,151 +14,156 @@ redirects explicitly.
 
 Because spawning browser processes or instantiating HTTP clients is expensive,
 Wayfarer keeps user agents in a pool and reuses them across jobs. This means
-that browser state carries over between jobs, as a job checks out a previous
-job's user agent. Only on certain irrecoverable errors are individual user agents
+that browser state carries over between jobs, as each job checks out a previous
+job's user agent. When exceptions are raised, you must [handle them](/guides/jobs/#handling-errors).
+
+Only on certain irrecoverable errors are individual user agents
 destroyed and recreated. For example when a browser process crashes, it is
 replaced with a fresh browser process.
 
-## Base interface for custom user agents
+## Implementing the user agent interfaces
 
 You implement both stateful and stateless agents by including the
-`Wayfarer::Networking::Strategy` module and defining callback methods. The
-interfaces for stateful and stateless share the following base methods:
+`Wayfarer::Networking::Strategy` module and defining callback methods:
 
 ```mermaid
 classDiagram
-class Square~Shape~{
-    int id
-    List~int~ position
-    setPoints(List~int~ points)
-    getPoints() List~int~
+class BaseAgent {
+    <<Interface>>
+    +#create()*
+    +#destroy(instance)*
+    +::renew_on()$
+}
+
+class StatefulAgent {
+    <<Interface>>
+    +#navigate(instance, url)*
+    +#live(instance)*
 }
 
-Square : -List~string~ messages
-Square : +setMessages(List~string~ messages)
-Square : +getMessages() List~string~
-Square : +getDistanceMatrix() List~List~int~~
+class StatelessAgent {
+    <<Interface>>
+    +#fetch(instance, url)*
+}
+
+BaseAgent |>.. StatefulAgent : implements
+BaseAgent |>.. StatelessAgent : implements
 ```
 
-* `#create` (__required__): Called when a new instance (browser process or HTTP client) is
-  needed.
-* `#destroy(instance)` (optional): Called when an instance should be destroyed. Browser
-  processes should be quit, and HTTP clients should be freed.
-* `#renew_on` (optional): Returns a list of exception classes upon which the existing
-  instance gets destroyed and replaced with a newly created one.
+Every user agent implementation must provide the `#create` instance callback
+which returns an initialized user agent. Typically, the optional
+`#destroy(instance)` instance callback is also implemented to free resources
+of an existing user agent.
 
-## Stateless interface
+Also a class method `::renew_on` can be defined which returns an array of
+exception classes upon which an instance of the user agent should get recreated
+(destroy-and-create).
 
-The stateless interface indicate HTTP 3xx redirect responses explicitly. This is how
-Wayfarer provides redirect handling out of the box, as there is a configurable limit
-on the number of retries to follow.
+### Stateless interface
 
-In addition to the base interface, stateless user agents implement `#fetch`
+In addition to the base interface, stateless user agents implement `#fetch(instance, url)`
 which fetches [pages](../pages) or indicates redirects:
 
-* `#fetch(instance, url)` (__required__): Called to retrieve a URL. Responses with a
+* `#create()` __(required)__
+* `#fetch(instance, url)` __(required)__: Called to retrieve a URL. Responses with a
   3xx status code must indicate the redirect URL by returning `redirect(url)`, since Wayfarer
   deals with redirects on your behalf to avoid redirect loops. All other status
   codes, including 4xx and 5xx, are considered successful and are indicated by calling
   `success(url:, body:, status_code:, headers:)`.
+* `#destroy(instance)` (optional)
+* `::renew_on` (optional)
 
-## Stateful interface
+The stateless interface indicate HTTP 3xx redirect responses explicitly. This is how
+Wayfarer provides redirect handling out of the box, as there is a configurable limit
+on the number of retries to follow.
 
-In addition to the base interface, stateful user agents implement two additional
-methods:
+!!! info "`redirect(url)` enqueues a task for the URL and stops processing the current task."
 
-* `#navigate(instance, url)` (__required__): Navigates the user agent to the given URL.
-  Stateful user agents follow redirects implicitly.
-* `#live(instance) -> Wayfarer::Page` (__required__): Turns the current user agent state
-  into a [page](../pages).
+!!! info "Pages with HTTP error status codes get routed"
 
-## Recreating user agents on error with `#renew_on`
+    If a HTTP request to a URL results in an error status code (for example, 404),
+    page retrieval is considered successful. This allows job actions to record
+    such data.
 
-Agents can optionally implement `#renew_on` to get themselves rereated on
-certain errors.
+### Stateful interface
 
-If `#fetch` or `#navigate` raise an exception and the exception class is listed
-in `#renew_on`, the instance is destroyed and recreated.
+In addition to the base interface, stateful user agents implement two additional
+instance callbacks:
 
-* `#renew_on` (optional): A list of exception classes upon which the existing instance gets
-  destroyed and replaced with a newly created one.
+* `#create()` __(required)__
+* `#navigate(instance, url)` __(required)__: Navigates the user agent to the given URL.
+* `#live(instance)` __(required)__: Turns the current user agent state
+into a [page](../pages) by calling `success(url:, body:, status_code:, headers:)`.
+* `#destroy(instance)` (optional)
+* `::renew_on` (optional)
 
 ## Example implementations
 
-=== "Stateful"
+=== "Stateless"
 
     ```ruby
-    class StatefulAgent
+    class StatelessAgent
       include Wayfarer::Networking::Strategy
 
-      # Required methods
-
-      def create
-        MyBrowser.new
+      def self.renew_on # optional
+        [MyIrrecoverableError]
       end
 
-      def navigate(browser, url)
-        browser.goto(url)
+      def create # required
+        MyClient.new
       end
 
-      def live(browser)
-        success(url: browser.url,
-                body: browser.body,
-                status_code: browser.status_code,
-                headers: browser.headers)
-      end
+      def fetch(client, url) # required
+        response = client.get(url)
 
-      # Optional methods
+        return redirect(response.redirect_url) if response.redirect?
 
-      def destroy(browser)
-        browser.quit
+        success(url: url,
+                body: response.body,
+                status_code: response.status_code,
+                headers: response.headers)
       end
 
-      def renew_on
-        [MyBrowser::IrrecoverableError]
+      def destroy(client) # optional
+        client.close
       end
     end
     ```
 
-=== "Stateless"
+=== "Stateful"
 
     ```ruby
-    class StatelessAgent
+    class StatefulAgent
       include Wayfarer::Networking::Strategy
 
-      # Required methods
-
-      def create
-        MyClient.new
+      def self.renew_on # optional
+        [MyIrrecoverableError]
       end
 
-      def fetch(client, url)
-        response = client.get(url)
-
-        return redirect(response.redirect_url) if response.redirect?
-
-        success(url: url,
-                body: response.body,
-                status_code: response.status_code,
-                headers: response.headers)
+      def create # required
+        MyBrowser.new
       end
 
-      # Optional methods
+      def navigate(browser, url) # required
+        browser.goto(url)
+      end
 
-      def destroy(client)
-        client.close
+      def live(browser) # required
+        success(url: browser.url,
+                body: browser.body,
+                status_code: browser.status_code,
+                headers: browser.headers)
       end
 
-      def renew_on # optional
-        [MyClient::IrrecoverableError]
+      def destroy(browser) # optional
+        browser.quit
       end
     end
     ```
 
-
-Register and use the strategy:
+Register and use the agent:
 
 ```ruby
-Wayfarer::Networking::Pool.registry[:my_agent] = MyAgent.new
+Wayfarer.config[:network][:agents][:my_agent] = MyAgent.new
 Wayfarer.config[:network][:agent] = :my_agent
 ```

data/docs/guides/networking/http.md

@@ -6,7 +6,7 @@ also alongside automated browsers.
 ## Ad-hoc GET requests
 
 When automating browsers, it can be useful to additionally retrieve another page
-over plain HTTP. Jobs can fetch URLs to [pages](/pages) with `#http`:
+or even the current page over plain HTTP. Jobs can fetch URLs to [pages](/pages) with `#http`:
 
 ```ruby
 class DummyJob < Wayfarer::Base

data/docs/guides/routing.md

@@ -9,7 +9,7 @@ Actions are instance methods denoted by symbols, or [handlers](/guides/handlers)
     By default, Wayfarer [applies some transformations to each URL](../tasks/#url-normalization) to bring it
     into a canonical form. Routing happens based on this canonical form.
 
-    You can always access a task's raw string as it was enqueued with `task.batch`.
+    You can always access a task's raw string as it was enqueued with `task.url`.
 
 A job's route declarations equate to a predicate tree.
 When a URL is routed, the predicate tree is searched depth-first. If a

data/docs/guides/tutorial.md

@@ -62,5 +62,4 @@ bundle exec wayfarer perform -r dummy_job.rb DummyJob https://example.com
 If you don't provide a batch, Wayfarer uses a generated UUID instead.
 We could have also used `DummyJob.crawl
 
-
-
+(To be continued.)

data/lib/wayfarer/cli.rb

@@ -13,6 +13,13 @@ module Wayfarer
 
     class_option :require, aliases: :r, type: :string, default: nil
 
+    class_option :log_level,
+                 aliases: :l,
+                 type: :string,
+                 desc: "Set the log level (debug, info, warn, error, fatal).",
+                 enum: %w[debug info warn error fatal unknown],
+                 default: "info"
+
     desc "route JOB URL", "Routing tree for URL for JOB"
     option :format, type: :string, enum: %w[yaml json ruby], default: "yaml"
     def route(job, url)
@@ -84,6 +91,8 @@ module Wayfarer
       require File.join(Dir.pwd, require_path) if require_path
 
       load_rails
+
+      set_log_level
     end
 
     def load_rails
@@ -99,5 +108,12 @@ module Wayfarer
       require File.expand_path("config/boot", Dir.pwd)
       require File.expand_path("config/environment", Dir.pwd)
     end
+
+    def set_log_level
+      level = options[:log_level] || return
+      severity = ActiveSupport::Logger::Severity.const_get(level.upcase)
+
+      Wayfarer::Logging.logger.level = severity
+    end
   end
 end

data/lib/wayfarer/logging.rb

@@ -2,7 +2,7 @@
 
 module Wayfarer
   module Logging
-    mattr_accessor :logger, default: ActiveSupport::Logger.new($stdout)
+    mattr_accessor :logger, default: ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new($stdout))
 
     def self.emit(...)
       Emitter.new(...)
@@ -23,9 +23,8 @@ module Wayfarer
             level, msg = messages[key] || raise(ArgumentError, "No log message for #{key.inspect}")
             severity = ActiveSupport::Logger::Severity.const_get(level.upcase)
 
-            ActiveSupport::TaggedLogging
-              .new(Logging.logger)
-              .tagged(task.batch, task.url, task[:controller]&.class&.name) do |logger|
+            Logging.logger
+                   .tagged(task.batch, task.url, task[:controller]&.class&.name) do |logger|
               logger.add(severity, msg % args)
             end
           end

data/lib/wayfarer/middleware/base.rb

@@ -12,10 +12,6 @@ module Wayfarer
 
         Module.new
       end
-
-      def lazy
-        Wayfarer::Middleware::Lazy.new(self)
-      end
     end
   end
 end

data/lib/wayfarer/middleware/controller.rb

@@ -18,7 +18,7 @@ module Wayfarer
 
       class_methods do
         def use(middleware)
-          chain.push(middleware.lazy)
+          chain.push(middleware.new)
           api(middleware)
         end
 

data/lib/wayfarer/networking/context.rb

@@ -36,7 +36,7 @@ module Wayfarer
 
       def supervise
         yield
-      rescue *strategy.renew_on, *Wayfarer.config[:network][:renew_on] => e
+      rescue *strategy.class.renew_on, *Wayfarer.config[:network][:renew_on] => e
         renew # may raise
       ensure
         # If renewing raises, re-raise the originally caught exception

data/lib/wayfarer/networking/ferrum.rb

@@ -5,7 +5,7 @@ module Wayfarer
     class Ferrum
       include Strategy
 
-      def renew_on
+      def self.renew_on
         [::Ferrum::DeadBrowserError]
       end
 

data/lib/wayfarer/networking/pool.rb

@@ -10,14 +10,6 @@ module Wayfarer
                       instance_accessor: false,
                       instance_predicate: false
 
-      class_attribute :registry,
-                      default: { http: HTTP,
-                                 ferrum: Ferrum,
-                                 selenium: Selenium,
-                                 capybara: Capybara },
-                      instance_accessor: false,
-                      instance_predicate: false
-
       attr_reader :pool
 
       def initialize
@@ -43,7 +35,9 @@ module Wayfarer
       end
 
       def strategy
-        self.class.registry[Wayfarer.config.dig(:network, :agent)].new
+        Wayfarer.config.dig(:network, :agent).then do |agent|
+          Wayfarer.config.dig(:network, :agents).fetch(agent).new
+        end
       end
     end
   end

data/lib/wayfarer/networking/strategy.rb

@@ -3,8 +3,12 @@
 module Wayfarer
   module Networking
     module Strategy
-      def renew_on
-        []
+      extend ActiveSupport::Concern
+
+      class_methods do
+        def renew_on
+          []
+        end
       end
 
       def fetch(instance, url)

data/lib/wayfarer/parsing.rb

@@ -12,7 +12,7 @@ module Wayfarer
 
     # @!visibility private
     def parse(body, content_type = FALLBACK_CONTENT_TYPE)
-      parser, args = Wayfarer.config.dig(:parsing, :registry, content_type)
+      parser, args = Wayfarer.config.dig(:parsing, :content_types, content_type)
       return unless parser
 
       parser.parse(body, *args)

data/lib/wayfarer.rb

@@ -9,8 +9,7 @@ require "uri"
 require "yaml"
 
 require "active_job"
-require "active_support/core_ext/array/wrap"
-require "active_support/core_ext/object/deep_dup"
+require "active_support/all"
 require "capybara"
 require "connection_pool"
 require "ferrum"
@@ -42,7 +41,7 @@ module Wayfarer
   module VERSION
     MAJOR = 0
     MINOR = 4
-    TINY = 8
+    TINY = 10
     STRING = [MAJOR, MINOR, TINY].join(".")
   end
 
@@ -53,6 +52,12 @@ module Wayfarer
     },
     network: {
       agent: :http,
+      agents: {
+        http: Wayfarer::Networking::HTTP,
+        ferrum: Wayfarer::Networking::Ferrum,
+        selenium: Wayfarer::Networking::Selenium,
+        capybara: Wayfarer::Networking::Capybara
+      },
       pool: {
         size: 1,
         timeout: 10
@@ -61,7 +66,7 @@ module Wayfarer
       renew_on: []
     },
     parsing: {
-      registry: {
+      content_types: {
         "application/json" => Wayfarer::Parsing::JSON,
         "text/html" => [Wayfarer::Parsing::XML, :html],
         "application/xml" => [Wayfarer::Parsing::XML, :xml]

data/rake/release.rake

@@ -12,12 +12,6 @@ namespace :release do
 
     raise "Gem version #{gem_version} deviates from library version #{lib_version}" unless gem_version == lib_version
   end
-
-  task :guard_debug do
-    sh "git diff --exit-code"
-    sh "git diff-index --quiet --cached HEAD"
-  end
 end
 
 Rake::Task[:"release:rubygem_push"].enhance(%i[release:guard_versions])
-Rake::Task[:"release:guard_clean"].enhance(%i[release:guard_debug])

data/spec/spec_helpers.rb

@@ -61,7 +61,7 @@ RSpec.configure do |config|
   config.include_context "with Redis", :redis
 
   config.before :suite do
-    Wayfarer::Logging.logger = Logger.new($stdout)
+    Wayfarer::Logging.logger = ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new($stdout))
 
     FactoryBot.find_definitions
     FactoryBot::SyntaxRunner.include(SpecHelpers)

data/spec/wayfarer/integration/page_spec.rb

@@ -8,7 +8,7 @@ describe "Pages", :redis do
     mock_handler! :dummy_handler
   end
 
-  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
+  let(:url) { test_app_path("finders.html") }
 
   shared_examples "executes" do
     specify do
@@ -29,14 +29,14 @@ describe "Pages", :redis do
         route.host test_app_host, to: :index
 
         def index
-          expect(page.url).to eq(test_app_path("git-scm.com/book/en/v2.html"))
+          expect(page.url).to eq(test_app_path("finders.html"))
           expect(page.status_code).to be(200)
           expect(page.body).not_to be_empty
           expect(page.headers.count).to be(9)
 
-          expect(page.meta.links.all.count).to be(157)
-          expect(page.meta.links.internal.count).to be(102)
-          expect(page.meta.links.external.count).to be(55)
+          expect(page.meta.links.all.count).to be(6)
+          expect(page.meta.links.internal.count).to be(3)
+          expect(page.meta.links.external.count).to be(3)
         end
       end
     end
@@ -55,14 +55,14 @@ describe "Pages", :redis do
         route.to :index
 
         def index
-          expect(page.url).to eq(test_app_path("git-scm.com/book/en/v2.html"))
+          expect(page.url).to eq(test_app_path("finders.html"))
           expect(page.status_code).to be(200)
           expect(page.body).not_to be_empty
           expect(page.headers.count).to be(9)
 
-          expect(page.meta.links.all.count).to be(157)
-          expect(page.meta.links.internal.count).to be(102)
-          expect(page.meta.links.external.count).to be(55)
+          expect(page.meta.links.all.count).to be(6)
+          expect(page.meta.links.internal.count).to be(3)
+          expect(page.meta.links.external.count).to be(3)
         end
       end