wayfarer 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1dad2cf380ed02bc82337e9e333f8203de7c9043bca7c30b317438e66e26828
-  data.tar.gz: 7c178b52bd9fccf8b46026aba4dd8e7e9f8f1fef9e9919e7b3841adc686d047c
+  metadata.gz: a2bbcc550d6799e1e3588b832905866116105fdcceb0eeef5ec244622f15bb10
+  data.tar.gz: b8b324d89d162e578cde829f15a44bb08b9e93ad2a3bb5b6619e96275a7fa6cd
 SHA512:
-  metadata.gz: 2b64c747f71682c052392a83c5993365641464a1df5fc3d6a05c239f3f2e9265c248779e8ace9ebd1cd0f37023345c59c9bd6298f8bec5b42a8a1e236233ae6e
-  data.tar.gz: ce8ddb1c4d7397f93699743587050c21bd645d2e3c37efcfbc8ed29bdcd5bd8146cef350e72eb08c3eab6e6c570379197a0d8fdb90d4fa68f8a032bd4dbba9d1
+  metadata.gz: 998c06776f7a7922aa2d36770dc7e4389c5814ac36a20062b20e7fe6986fb52e4fde9f538287510fccfca1689fb9f7f4e019ec23dcbeff777add1a99e24fba26
+  data.tar.gz: 239e3db3d5fffb8f81e74c655a648ce03febd346c25fb86b695bca8a8d328e6ff341d63d4becf63a2811defae3af5a1c4bf1c772e327b114df5909a06151a95b

data/.github/workflows/ci.yaml

@@ -22,7 +22,7 @@ jobs:
       - name: Run Ferrum tests
         run: docker-compose run --rm --name test --service-ports wayfarer bundle exec rake test:ferrum
 
-      - name: Run Ferrum tests
+      - name: Run Selenium tests
         run: docker-compose run --rm --name test --service-ports wayfarer bundle exec rake test:selenium
 
       - name: Run CLI tests

data/Gemfile.lock

@@ -1,21 +1,21 @@
 PATH
   remote: .
   specs:
-    wayfarer (0.4.0)
+    wayfarer (0.4.1)
       activejob (~> 6.0)
-      capybara (~> 2.5)
+      addressable (~> 2.8)
+      capybara (~> 3.0)
       connection_pool (~> 2.2)
-      cuprite (~> 0.11)
       docile (~> 1.1)
       ferrum (~> 0.9)
       metainspector (~> 5.0)
       mime-types (~> 3.0)
       mock_redis (~> 0.29)
       mustermann (~> 1.1)
-      net-http-persistent (~> 2.9)
+      net-http-persistent (~> 3.0)
       nokogiri (~> 1.11)
       normalize_url (~> 0.0.6)
-      redis (~> 4.0)
+      redis (~> 4.4, < 4.5)
       selenium-webdriver (~> 3.4)
       thor (~> 1.0)
 
@@ -34,13 +34,15 @@ GEM
     addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.2)
-    capybara (2.18.0)
+    capybara (3.36.0)
       addressable
+      matrix
       mini_mime (>= 0.1.3)
-      nokogiri (>= 1.3.3)
-      rack (>= 1.0.0)
-      rack-test (>= 0.5.4)
-      xpath (>= 2.0, < 4.0)
+      nokogiri (~> 1.8)
+      rack (>= 1.6.0)
+      rack-test (>= 0.6.3)
+      regexp_parser (>= 1.5, < 3.0)
+      xpath (~> 3.2)
     childprocess (3.0.0)
     cliver (0.3.2)
     coderay (1.1.3)
@@ -83,7 +85,7 @@ GEM
     faraday-net_http_persistent (1.2.0)
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
-    faraday_middleware (1.1.0)
+    faraday_middleware (1.2.0)
       faraday (~> 1.0)
     fastimage (2.2.5)
     ferrum (0.11)
@@ -97,6 +99,7 @@ GEM
       domain_name (~> 0.5)
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
+    matrix (0.4.2)
     metainspector (5.11.2)
       addressable (~> 2.7)
       faraday (~> 1.4)
@@ -111,7 +114,7 @@ GEM
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2021.0901)
-    mini_mime (1.1.1)
+    mini_mime (1.1.2)
     mini_portile2 (2.6.1)
     minitest (5.14.4)
     mock_redis (0.29.0)
@@ -120,8 +123,9 @@ GEM
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
     nesty (1.0.2)
-    net-http-persistent (2.9.4)
-    nokogiri (1.12.4)
+    net-http-persistent (3.1.0)
+      connection_pool (~> 2.2)
+    nokogiri (1.12.5)
       mini_portile2 (~> 2.6.1)
       racc (~> 1.4)
     normalize_url (0.0.6)
@@ -133,7 +137,7 @@ GEM
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (4.0.6)
-    racc (1.5.2)
+    racc (1.6.0)
     rack (1.6.13)
     rack-protection (1.5.5)
       rack
@@ -198,6 +202,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  cuprite (~> 0.13)
   factory_bot (~> 6.0)
   faker (~> 1.7)
   pry (~> 0.10)

data/docs/cookbook/user_agent.md

@@ -3,5 +3,5 @@
 See: [Guides: Networking: HTTP request headers](/guides/networking#http-request-headers)
 
 ```ruby
-Wayfarer.config.http_headers = { "User-Agent" => "MyCrawler ..." }
+Wayfarer.config.network.http_headers = { "User-Agent" => "MyCrawler ..." }
 ```

data/docs/guides/browser_automation/capybara.md

@@ -1,3 +1,66 @@
 # Capybara
 
-TODO
+[Capybara](https://github.com/teamcapybara/capybara) is originally a test
+framework for web applications.
+
+When Capybara is in use, a remote browser process is available as a Capybara
+session:
+
+```ruby
+class DummyWorker < Wayfarer::Worker
+  route.to :index
+
+  def index
+    browser # => #<Capybara::Session ...>
+  end
+end
+```
+
+
+## Configuring a driver
+
+1. Install the Capybara driver for the desired user agent.
+
+    For example, to automate Google Chrome with
+    [Ferrum](https://github.com/rubycdp/ferrum), install the
+    [Cuprite](https://github.com/rubycdp/cuprite) driver:
+
+    === "RubyGems"
+
+        ```ruby
+        gem install cuprite
+        ```
+
+    === "Bundler"
+
+        ```ruby
+        gem "cuprite" # Gemfile
+        ```
+
+2. Configure Wayfarer to use the `:capybara` user agent and set the desired
+    driver:
+
+    === "Runtime"
+
+        ```ruby
+        Wayfarer.config.network.agent = :capybara
+        Wayfarer.config.capybara.driver = :cuprite
+        ```
+
+    === "Environment variables"
+
+        ```ruby
+        WAYFARER_NETWORK_AGENT=capybara
+        WAYFARER_CAPYBARA_DRIVER=cuprite
+        ```
+
+3. Register the driver:
+
+    ```ruby
+    Capybara.javascript_driver = :cuprite
+
+    Capybara.register_driver(:cuprite) do |app|
+      # Wayfarer's Ferrum or Selenium options must be passed along manually
+      Capybara::Cuprite::Driver.new(app, Wayfare.config.ferrum.options)
+    end
+    ```

data/docs/guides/browser_automation/custom_adapters.md

@@ -0,0 +1,100 @@
+# Custom agents
+
+Wayfarer offers an interface for integrating third-party browsers and HTTP
+clients as user agents.
+
+There are two types of agents:
+
+1. Stateful agents, i.e. browsers, which carry state and support navigation.
+   These follow HTTP redirects implicitly.
+2. Stateless agents, which deal with HTTP requests/responses only.
+   These handle HTTP redirects explicitly.
+
+## Implementation
+
+Both types can be implemented with callback methods:
+
+=== "Stateful"
+
+    ```ruby
+    class StatefulAgent
+      include Wayfarer::Networking::Strategy
+
+      def renew_on # optional
+        [MyBrowser::IrrecoverableError]
+      end
+
+      def create
+        MyBrowser.new
+      end
+
+      def destroy(browser) # optional
+        browser.quit
+      end
+
+      def navigate(browser, url)
+        browser.goto(url)
+      end
+
+      def live(browser)
+        success(url: browser.url,
+                body: browser.body,
+                status_code: browser.status_code,
+                headers: browser.headers)
+      end
+    end
+    ```
+
+=== "Stateless"
+
+    ```ruby
+    class StatelessAgent
+      include Wayfarer::Networking::Strategy
+
+      def renew_on # optional
+        [MyClient::IrrecoverableError]
+      end
+
+      def create
+        MyClient.new
+      end
+
+      def destroy(client) # optional
+        client.close
+      end
+
+      def fetch(client, url)
+        response = client.get(url)
+
+        return redirect(response.redirect_url) if response.redirect?
+
+        success(url: url,
+                body: response.body,
+                status_code: response.status_code,
+                headers: response.headers)
+      end
+    end
+    ```
+
+
+Register the strategy:
+
+```ruby
+Wayfarer::Networking::Pool.registry[:my_agent] = MyAgent.new
+```
+
+Use the strategy:
+
+```ruby
+Wayfarer.config.network.agent = :my_agent
+```
+
+### Remarks
+
+#### Self-healing
+
+* A strategy's `#renew_on` method may return a list of exception classes upon
+  which the existing instance gets destroyed and replaced with a newly created
+  one.
+* Stateless clients must not raise exceptions when encountering certain HTTP
+  response codes (for example, 5xx).

data/docs/guides/browser_automation/ferrum.md

@@ -25,13 +25,13 @@ end
 === "Runtime"
 
     ```ruby
-    Wayfarer.config.adapter = :ferrum
-    Wayfarer.config.ferrum_options = { headless: false, url: "http://chrome:3000" }
+    Wayfarer.config.network.agent = :ferrum
+    Wayfarer.config.ferrum.options = { headless: false, url: "http://chrome:3000" }
     ```
 
 === "Environment variables"
 
     ```
-    WAYFARER_ADAPTER=ferrum
+    WAYFARER_AGENT=ferrum
     WAYFARER_FERRUM_OPTIONS=headless:false,url:http://chrome:3000
     ```

data/docs/guides/browser_automation/selenium.md

@@ -25,7 +25,7 @@ process.
     Pages retrieved with a Selenium WebDriver return fake values:
 
     ```ruby
-    Wayfarer.config.adapter = :selenium
+    Wayfarer.config.network.agent = :selenium
 
     class DummyJob < Wayfarer::Base
       route.to :index
@@ -47,13 +47,15 @@ process.
 === "Runtime"
 
     ```ruby
-    Wayfarer.config.adapter = :selenium
-    Wayfarer.config.selenium_argv = [:firefox]
+    Wayfarer.config.network.agent = :selenium
+    Wayfarer.config.selenium.driver = :firefox
+    Wayfarer.config.selenium.options = { url: "http://firefox" }
     ```
 
 === "Environment variables"
 
     ```
-    WAYFARER_ADAPTER=selenium
-    WAYFARER_SELENIUM_ARGV=firefox
+    WAYFARER_AGENT=selenium
+    WAYFARER_SELENIUM_DRIVER=firefox
+    WAYFARER_SELENIUM_OPTIONS=url:http://firefox
     ```

data/docs/guides/callbacks.md

@@ -1,30 +1,56 @@
 # Callbacks
 
-## Life cycle callbacks
+## Active Job callbacks
 
-Wayfarer supports all of Active Job's life cycle callbacks:
+Wayfarer naturally supports all of [Active Job's life cycle callbacks](https://edgeguides.rubyonrails.org/active_job_basics.html#callbacks).
 
-* [Active Job Basics: Callbacks](https://edgeguides.rubyonrails.org/active_job_basics.html#callbacks)
-* [ActiveJob::Callbacks](https://api.rubyonrails.org/classes/ActiveJob/Callbacks/ClassMethods.html)
+## `before_fetch`
 
-## `after_batch` callbacks
+Runs before a job fetches a page, either by making an HTTP request, or by
+navigating a browser to its task URL.
 
-Jobs can register callbacks to run once all jobs in their batch have concluded:
+```ruby
+class DummyJob < Wayfarer::Base
+  before_fetch :do_something
+
+  private
+
+  def do_something
+    # before the task.url is fetched
+  end
+end
+```
+
+## `before_action`
+
+Runs after a page was fetched, before an action method is called.
 
 ```ruby
 class DummyJob < Wayfarer::Base
-  after_batch do
-    # All jobs in batch done
+  before_action :do_something
+
+  private
+
+  def do_something
+    # page is available at this point
   end
+end
+```
+
+## `after_batch`
 
+Runs once the last job in a batch performed:
+
+```ruby
+class DummyJob < Wayfarer::Base
   after_batch do
-    # Multiple callbacks can be registered
+    # All jobs in batch done
   end
 end
 ```
 
 Internally, a batch counter is in-/decremented on certain events. Once the
-counter reaches zero, `after_batch` callbacks execute in declaration order.
+counter reaches zero, `after_batch` callbacks runs in declaration order.
 
 The counter is incremented when:
 
@@ -36,3 +62,84 @@ The counter is decremented when:
 * A job fails due to an unhandled exception.
 * A job fails due to a discarded exception.
 * A job fails and thereyby exhausts its maximum attempts.
+
+!!! attention "Batch callbacks can fail jobs"
+
+    If the last job's `after_batch` callbacks raises an exception, this can lead
+    to the job getting retried. If the exception raised by the callback is
+    unhandled or discarded, the callback never fully runs.
+
+## Callback options
+
+### Definition styles
+
+Callbacks can be registered either by supplying a block or a symbol identifying
+a callback instance method:
+
+```ruby
+class DummyJob < Wayfarer::Base
+  before_action do
+    # ...
+  end
+
+  before_action :my_callback
+
+  private
+
+  def my_callback
+    # ...
+  end
+end
+```
+
+### Conditionals
+
+Callbacks can be registered conditionally with the `:if` and `:unless` keywords:
+
+```ruby
+class DummyJob < Wayfarer::Base
+  before_fetch :my_callback, if: :my_condition
+
+  private
+
+  def my_callback
+  end
+
+  def my_condition
+  end
+end
+```
+
+Callbacks can be registered for certain action methods only with the `:only` and
+`:except` keywords:
+
+```ruby
+class DummyJob < Wayfarer::Base
+  before_fetch :do_something, only: :foo
+
+  before_fetch except: [:foo, :qux] do
+    # runs only before bar
+  end
+
+  def foo
+  end
+
+  def bar
+  end
+end
+
+```
+
+### Early termination
+
+Callbacks that return `false` halt the callback chain:
+
+```ruby
+class DummyJob < Wayfarer::Base
+  before_action { false }
+
+  before_action do
+    # never runs
+  end
+end
+```

data/docs/guides/configuration.md

@@ -12,22 +12,28 @@ Wayfarer parses environment variables into a runtime configuration
 `Wayfarer.config`:
 
 ```ruby
-# Which network adapter to use to process tasks
-Wayfarer.config.adapter = :net_http # or :ferrum, :selenium
+# Which user agent to use to process tasks
+Wayfarer.config.network.agent = :http # or :ferrum, :selenium
 
-# How many network adapters to instantiate
-Wayfarer.config.adapter_pool_size = 3
+# How many user agents to instantiate
+Wayfarer.config.network.pool_size = 3
 
-# How long an adapter may be used while processing a task
-Wayfarer.config.adapter_pool_timeout = 5000
+# How long an agent may be used while processing a task
+Wayfarer.config.network.pool_timeout = 5000
 
 # Ferrum options
-Wayfarer.config.ferrum_options = {}
+Wayfarer.config.ferrum.options = {}
 
-# Selenium arguments
-Wayfarer.config.selenium_argv = [:chrome]
+# Selenium driver to use
+Wayfarer.config.selenium.driver = :chrome
+
+# Selenium HTTP client read timeout
+Wayfarer.config.selenium.client_timeout = 10 # seconds
+
+# Selenium options
+Wayfarer.config.selenium.options = { url: "http://chrome" }
 
 # HTTP request headers (Selenium is unsupported)
-Wayfarer.config.http_headers = { "Field" => "Value" }
+Wayfarer.config.network.http_headers = { "Field" => "Value" }
 ```
 

data/docs/guides/error_handling.md

@@ -6,16 +6,12 @@ Wayfarer relies on Active Job's error handling facilities, `retry_on` and
 * [Active Job Basics: Exceptions](https://guides.rubyonrails.org/active_job_basics.html#exceptions)
 * [ActiveJob::Exceptions](https://edgeapi.rubyonrails.org/classes/ActiveJob/Exceptions/ClassMethods.html)
 
-## Unhandled exceptions
-
-Jobs with unhandled exceptions fail and are not retried.
-
 ## Retrying
 
 ```ruby
 class DummyJob < Wayfarer::Base
   retry_on MyError, attempts: 3 do |job, error|
-    # All 3 retry attempts have failed
+    # All 3 attempts have failed (1 initial attempt + 2 retries)
   end
 end
 ```
@@ -29,3 +25,11 @@ class DummyJob < Wayfarer::Base
   end
 end
 ```
+
+## Job failures
+
+Jobs are not retried and their URLs locked within their batch if:
+
+* A discarded exception is raised.
+* An unhandled exception is raised.
+* A handled exception is raised, but retry attempts are exhausted.

data/docs/guides/networking.md

@@ -1,18 +1,92 @@
 # Networking
 
-Wayfarer retrieves pages in two ways:
+Wayfarer navigates the web in two ways:
 
 1. Via plain HTTP requests
 2. By automating browsers
 
-TODO
+Both options are mutually exclusive per Ruby process.
+
+## User agents
+
+A user agent is an entity that knows how to retrieve the contents behind a URL.
+
+The user agent can be configured via the global configuration:
+
+```ruby
+Wayfarer.config.network.agent = :http # or :ferrum, :selenium
+```
+
+## Connection pooling
+
+Wayfarer keeps user agents within a connection pool. When a job executes
+and needs to retrieve the contents behind a URL, an agent is checked out from
+the pool.
+
+The pool has a constant size and it should equal the number of threads the
+underlying message queue operates with. The size can be configured via the
+global configuration:
+
+```ruby
+Wayfarer.config.network.pool_size = 8
+```
+
+### Timeouts
+
+user agents may stay checked out from the pool by jobs for a limited time
+only. Once this time limit is exceeded, a `ConnectionPool::TimeoutError`
+exception is raised. This places a hard time limit on every job.
+
+The timeout can be configured via the global configuration:
+
+```ruby
+Wayfarer.config.network.pool_timeout = 20 # seconds
+```
+
+Because jobs with unhandled exceptions fail, explicit error handling is required
+if retries are desired:
+
+```ruby
+class DummyJob < Wayfarer::Base
+  retry_on ConnectionPool::TimeoutError, attempts: 3
+end
+```
+
+## Agent-specific client timeouts
+
+The time in seconds it may take to communicate with remote browser processes can
+be configured globally per agent:
+
+```ruby
+Wayfarer.config.ferrum.options = { timeout: 5 }
+Wayfarer.config.selenium.client_timeout = 60
+```
+
+### Shared state
+
+As user agents get checked in and out continously between jobs, their state
+carries over from job to job, too.
+
+For browser automation, this means:
+
+* A job finds the browser at the last URL the previous job has left off.
+* The browser's cookies might have been set, or other client-side state might
+  exist that significantly affects a page's behaviour.
+
+## HTTP redirect handling
+
+Browsers follow redirects transparently when they are navigated to a URL.
+
+When using plain HTTP, redirect URLs are enqueued transparently within the same
+batch. URLs that result in 3xx responses will not be retrieved again within
+their batch.
 
 ## HTTP request headers
 
 Request headers can be configured via the global configuration:
 
 ```ruby
-Wayfarer.config.http_headers = { "Field" => "Value" }
+Wayfarer.config.network.http_headers = { "Field" => "Value" }
 ```
 
 !!! attention "Partial support"

data/docs/index.md

@@ -14,10 +14,18 @@ hide:
 * Data extraction
 * Browser automation
 
-!!! attention "Experimental software"
+!!! attention "Unstable software"
 
     Wayfarer is under development and releases should be considered unstable.
 
+    Wayfarer complies to
+    [Semantic Versioning 2.0.0](https://semver.org/spec/v2.0.0.html) in
+    which v0.x means that there could be backward-incompatible changes for every
+    release:
+
+    >Major version zero (0.y.z) is for initial development. Anything MAY change
+    at any time. The public API SHOULD NOT be considered stable.
+
 ### Installation
 
 Install the RubyGem: