wayfarer 0.4.0 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1dad2cf380ed02bc82337e9e333f8203de7c9043bca7c30b317438e66e26828
-  data.tar.gz: 7c178b52bd9fccf8b46026aba4dd8e7e9f8f1fef9e9919e7b3841adc686d047c
+  metadata.gz: caaf26f19cde178471b846bb33d8c347fc6ebdbb6951b9bfd40b1214516076be
+  data.tar.gz: 7347ecd76e6085091bbf21d31cccb24cffa9b0ab88ade4b8f7a388496fa345cc
 SHA512:
-  metadata.gz: 2b64c747f71682c052392a83c5993365641464a1df5fc3d6a05c239f3f2e9265c248779e8ace9ebd1cd0f37023345c59c9bd6298f8bec5b42a8a1e236233ae6e
-  data.tar.gz: ce8ddb1c4d7397f93699743587050c21bd645d2e3c37efcfbc8ed29bdcd5bd8146cef350e72eb08c3eab6e6c570379197a0d8fdb90d4fa68f8a032bd4dbba9d1
+  metadata.gz: 35851a6c27663ff0013c03938accc1d144fe83f29fd29b1fba3ac6b3d436e66145291333cddeb76519ec78507409f76c74633f45adca93a7ad9fc5efdf318d2f
+  data.tar.gz: db0cbbf7a2f0524f3a1c1405e77e7136efc1256d93690ca3ea1ed1630edf7e8fbb9ecbd56a6a2aabdbba8b392965e34084ded416fcab8ffe3496087f5be7236b

data/.github/workflows/ci.yaml

@@ -22,7 +22,7 @@ jobs:
       - name: Run Ferrum tests
         run: docker-compose run --rm --name test --service-ports wayfarer bundle exec rake test:ferrum
 
-      - name: Run Ferrum tests
+      - name: Run Selenium tests
         run: docker-compose run --rm --name test --service-ports wayfarer bundle exec rake test:selenium
 
       - name: Run CLI tests

data/.rubocop.yml

@@ -20,6 +20,10 @@ Style/BlockDelimiters:
     - subject
     - expect
 
+Style/ClassAndModuleChildren:
+  Exclude:
+    - "spec/**/*"
+
 Style/Documentation:
   Enabled: false
 
@@ -32,6 +36,12 @@ Metrics/BlockLength:
 
 Metrics/AbcSize:
   Max: 30
+  Exclude:
+    - "spec/**/*"
 
 Metrics/CyclomaticComplexity:
   Max: 15
+
+Lint/ConstantDefinitionInBlock:
+  Exclude:
+    - "spec/**/*"

data/Gemfile.lock

@@ -1,21 +1,21 @@
 PATH
   remote: .
   specs:
-    wayfarer (0.4.0)
-      activejob (~> 6.0)
-      capybara (~> 2.5)
+    wayfarer (0.4.4)
+      activejob (>= 6.0)
+      addressable (~> 2.8)
+      capybara (~> 3.0)
       connection_pool (~> 2.2)
-      cuprite (~> 0.11)
       docile (~> 1.1)
       ferrum (~> 0.9)
       metainspector (~> 5.0)
       mime-types (~> 3.0)
       mock_redis (~> 0.29)
       mustermann (~> 1.1)
-      net-http-persistent (~> 2.9)
+      net-http-persistent (~> 3.0)
       nokogiri (~> 1.11)
       normalize_url (~> 0.0.6)
-      redis (~> 4.0)
+      redis (~> 4.4, < 4.5)
       selenium-webdriver (~> 3.4)
       thor (~> 1.0)
 
@@ -34,13 +34,15 @@ GEM
     addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.2)
-    capybara (2.18.0)
+    capybara (3.36.0)
       addressable
+      matrix
       mini_mime (>= 0.1.3)
-      nokogiri (>= 1.3.3)
-      rack (>= 1.0.0)
-      rack-test (>= 0.5.4)
-      xpath (>= 2.0, < 4.0)
+      nokogiri (~> 1.8)
+      rack (>= 1.6.0)
+      rack-test (>= 0.6.3)
+      regexp_parser (>= 1.5, < 3.0)
+      xpath (~> 3.2)
     childprocess (3.0.0)
     cliver (0.3.2)
     coderay (1.1.3)
@@ -83,7 +85,7 @@ GEM
     faraday-net_http_persistent (1.2.0)
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
-    faraday_middleware (1.1.0)
+    faraday_middleware (1.2.0)
       faraday (~> 1.0)
     fastimage (2.2.5)
     ferrum (0.11)
@@ -97,6 +99,7 @@ GEM
       domain_name (~> 0.5)
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
+    matrix (0.4.2)
     metainspector (5.11.2)
       addressable (~> 2.7)
       faraday (~> 1.4)
@@ -111,7 +114,7 @@ GEM
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2021.0901)
-    mini_mime (1.1.1)
+    mini_mime (1.1.2)
     mini_portile2 (2.6.1)
     minitest (5.14.4)
     mock_redis (0.29.0)
@@ -120,8 +123,9 @@ GEM
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
     nesty (1.0.2)
-    net-http-persistent (2.9.4)
-    nokogiri (1.12.4)
+    net-http-persistent (3.1.0)
+      connection_pool (~> 2.2)
+    nokogiri (1.12.5)
       mini_portile2 (~> 2.6.1)
       racc (~> 1.4)
     normalize_url (0.0.6)
@@ -133,7 +137,7 @@ GEM
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (4.0.6)
-    racc (1.5.2)
+    racc (1.6.0)
     rack (1.6.13)
     rack-protection (1.5.5)
       rack
@@ -198,6 +202,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  cuprite (~> 0.13)
   factory_bot (~> 6.0)
   faker (~> 1.7)
   pry (~> 0.10)

data/docs/cookbook/batch_routing.md

@@ -0,0 +1,22 @@
+# Batch routing
+
+```ruby
+# Create a record in an external database and store the hostname
+record = Database::Row.create(hostname: "example.com")
+
+class DummyJob < Wayfarer::Base
+  route do |hostname|
+    host hostname, to: :index
+  end
+
+  steer do |task|
+    # Pass the external record's hostname to the router
+    [Database::Row.find(task.batch).hostname]
+  end
+
+  # ...
+end
+
+# Enqueue the task and use the database record's key as batch
+DummyJob.crawl("https://example.com", batch: record.id)
+```

data/docs/cookbook/consent_screen.md

@@ -0,0 +1,36 @@
+# Consent Screens
+
+Some websites have nag-screens that make visitors wait for a button to show up.
+Here is an example with Ferrum where the opt-in button is contained in an
+iframe, clicked, and makes the live page behind the screen accessible to
+`#index`:
+
+```ruby
+Wayfarer.config.network.agent = :ferrum
+
+class DummyJob < Wayfarer::Base
+  route { to :index, host: "example.com" }
+
+  before_action if: :consent_required? do
+    sleep(5) # If the consent form has a loading animation
+    consent_button&.click
+    sleep(5) # Wait for browser to get redirected behind nag-screen
+  end
+
+  def index
+    # Nag-screen passed
+    stage page(live: true).meta.links.internal
+  end
+
+  private
+
+  def consent_button
+    browser.frames.third.css("button#consent")&.first
+  end
+
+  def consent_required?
+    browser.css(".consent_screen").any?
+  end
+end
+
+```

data/docs/cookbook/executing_javascript.md

@@ -0,0 +1,41 @@
+# Executing JavaScript
+
+Executing JavaScript requires automating a browser.
+
+=== "Ferrum"
+
+    ```ruby
+    class DummyJob < Wayfarer::Base
+      route { to :index }
+
+      def index
+        browser.evaluate("[window.scrollX, window.scrollY]")
+      end
+    end
+    ```
+
+=== "Selenium"
+
+    ```ruby
+    class DummyJob < Wayfarer::Base
+      route { to :index }
+
+      def index
+        # Mind the explicit return
+        browser.execute_script("return [window.scrollX, window.scrollY]")
+      end
+    end
+    ```
+
+=== "Capybara"
+
+    ```ruby
+    class DummyJob < Wayfarer::Base
+      route { to :index }
+
+      def index
+        # Capybara does not return value of JavaScript execution
+        browser.execute_script("console.log('Foobar')") # => nil
+      end
+    end
+    ```

data/docs/cookbook/navigation.md

@@ -0,0 +1,43 @@
+# Navigation
+
+=== "Ferrum"
+
+    ```ruby
+    class DummyJob < Wayfarer::Base
+      route { to :index }
+
+      def index
+        agent.goto("https://example.com")
+        agent.back
+        agent.forward
+      end
+    end
+    ```
+
+=== "Selenium"
+
+    ```ruby
+    class DummyJob < Wayfarer::Base
+      route { to :index }
+
+      def index
+        agent.navigate.to("https://example.com")
+        agent.navigate.back
+        agent.navigate.forward
+      end
+    end
+    ```
+
+=== "Capybara"
+
+    ```ruby
+    class DummyJob < Wayfarer::Base
+      route { to :index }
+
+      def index
+        agent.visit("https://example.com")
+        agent.go_back
+        agent.go_forward
+      end
+    end
+    ```

data/docs/cookbook/querying_html.md

@@ -6,7 +6,7 @@ See: [Nokogiri: Searching an HTML / XML Document](https://nokogiri.org/tutorials
 
     ```ruby
     class DummyJob < Wayfarer::Base
-      route.to :index
+      route { to :index }
 
       def index
         page.doc.css("html")
@@ -19,7 +19,7 @@ See: [Nokogiri: Searching an HTML / XML Document](https://nokogiri.org/tutorials
 
     ```ruby
     class DummyJob < Wayfarer::Base
-      route.to :index
+      route { to :index }
 
       def index
         browser.at_css("html")
@@ -32,7 +32,7 @@ See: [Nokogiri: Searching an HTML / XML Document](https://nokogiri.org/tutorials
 
     ```ruby
     class DummyJob < Wayfarer::Base
-      route.to :index
+      route { to :index }
 
       def index
         browser.find_elements(css: "html")

data/docs/cookbook/screenshots.md

@@ -6,7 +6,7 @@ Taking screenshots requires automating a browser.
 
     ```ruby
     class DummyJob < Wayfarer::Base
-      route.to :index
+      route { to :index }
 
       def index
         browser.screenshot(path: "screenshot.png")
@@ -18,7 +18,7 @@ Taking screenshots requires automating a browser.
 
     ```ruby
     class DummyJob < Wayfarer::Base
-      route.to :index
+      route { to :index }
 
       def index
         browser.save_screenshot("screenshot.png")

data/docs/cookbook/user_agent.md

@@ -3,5 +3,5 @@
 See: [Guides: Networking: HTTP request headers](/guides/networking#http-request-headers)
 
 ```ruby
-Wayfarer.config.http_headers = { "User-Agent" => "MyCrawler ..." }
+Wayfarer.config.network.http_headers = { "User-Agent" => "MyCrawler ..." }
 ```

data/docs/guides/callbacks.md

@@ -1,38 +1,145 @@
 # Callbacks
 
-## Life cycle callbacks
+## Active Job callbacks
 
-Wayfarer supports all of Active Job's life cycle callbacks:
+Wayfarer naturally supports all of [Active Job's life cycle callbacks](https://edgeguides.rubyonrails.org/active_job_basics.html#callbacks).
 
-* [Active Job Basics: Callbacks](https://edgeguides.rubyonrails.org/active_job_basics.html#callbacks)
-* [ActiveJob::Callbacks](https://api.rubyonrails.org/classes/ActiveJob/Callbacks/ClassMethods.html)
+## `before_fetch`
 
-## `after_batch` callbacks
+Runs before a job fetches a page, either by making an HTTP request, or by
+navigating a browser to its task URL.
 
-Jobs can register callbacks to run once all jobs in their batch have concluded:
+```ruby
+class DummyJob < Wayfarer::Base
+  before_fetch :do_something
+
+  private
+
+  def do_something
+    # before the task.url is fetched
+  end
+end
+```
+
+## `before_action`
+
+Runs after a page was fetched, before an action method is called.
 
 ```ruby
 class DummyJob < Wayfarer::Base
-  after_batch do
-    # All jobs in batch done
+  before_action :do_something
+
+  private
+
+  def do_something
+    # page is available at this point
   end
+end
+```
+
+## `after_batch`
 
+Runs once the last job in a batch performed:
+
+```ruby
+class DummyJob < Wayfarer::Base
   after_batch do
-    # Multiple callbacks can be registered
+    # All jobs in batch done
   end
 end
 ```
 
 Internally, a batch counter is in-/decremented on certain events. Once the
-counter reaches zero, `after_batch` callbacks execute in declaration order.
+counter reaches zero, `after_batch` callbacks runs in declaration order.
 
-The counter is incremented when:
+The counter is incremented when within the batch:
 
-* A job is enqueued within the batch.
+* A job is enqueued.
 
 The counter is decremented when:
 
 * A job succeeds.
-* A job fails due to an unhandled exception.
-* A job fails due to a discarded exception.
-* A job fails and thereyby exhausts its maximum attempts.
+* A job errors due to an unhandled exception.
+* A job is discarded due to an exception.
+* A job errors and thereyby exhausts its maximum attempts.
+
+!!! attention "Batch callbacks can fail jobs"
+
+    If the last job's `after_batch` callbacks raises an exception, this can lead
+    to the job getting retried. If the exception raised by the callback is
+    unhandled or discarded, the callback never fully runs.
+
+## Callback options
+
+### Definition styles
+
+Callbacks can be registered either by supplying a block or a symbol identifying
+a callback instance method:
+
+```ruby
+class DummyJob < Wayfarer::Base
+  before_action do
+    # ...
+  end
+
+  before_action :my_callback
+
+  private
+
+  def my_callback
+    # ...
+  end
+end
+```
+
+### Conditionals
+
+Callbacks can be registered conditionally with the `:if` and `:unless` keywords:
+
+```ruby
+class DummyJob < Wayfarer::Base
+  before_fetch :my_callback, if: :my_condition
+
+  private
+
+  def my_callback
+  end
+
+  def my_condition
+  end
+end
+```
+
+Callbacks can be registered for certain action methods only with the `:only` and
+`:except` keywords:
+
+```ruby
+class DummyJob < Wayfarer::Base
+  before_fetch :do_something, only: :foo
+
+  before_fetch except: [:foo, :qux] do
+    # runs only before bar
+  end
+
+  def foo
+  end
+
+  def bar
+  end
+end
+
+```
+
+### Early termination
+
+Callbacks that return `false` halt the callback chain:
+
+```ruby
+class DummyJob < Wayfarer::Base
+  before_action { false }
+
+  before_action do
+    # never runs
+  end
+end
+```

data/docs/guides/configuration.md

@@ -12,22 +12,28 @@ Wayfarer parses environment variables into a runtime configuration
 `Wayfarer.config`:
 
 ```ruby
-# Which network adapter to use to process tasks
-Wayfarer.config.adapter = :net_http # or :ferrum, :selenium
+# Which user agent to use to process tasks
+Wayfarer.config.network.agent = :http # or :ferrum, :selenium
 
-# How many network adapters to instantiate
-Wayfarer.config.adapter_pool_size = 3
+# How many user agents to instantiate
+Wayfarer.config.network.pool_size = 3
 
-# How long an adapter may be used while processing a task
-Wayfarer.config.adapter_pool_timeout = 5000
+# How long an agent may be used while processing a task
+Wayfarer.config.network.pool_timeout = 5000
 
 # Ferrum options
-Wayfarer.config.ferrum_options = {}
+Wayfarer.config.ferrum.options = {}
 
-# Selenium arguments
-Wayfarer.config.selenium_argv = [:chrome]
+# Selenium driver to use
+Wayfarer.config.selenium.driver = :chrome
+
+# Selenium HTTP client read timeout
+Wayfarer.config.selenium.client_timeout = 10 # seconds
+
+# Selenium options
+Wayfarer.config.selenium.options = { url: "http://chrome" }
 
 # HTTP request headers (Selenium is unsupported)
-Wayfarer.config.http_headers = { "Field" => "Value" }
+Wayfarer.config.network.http_headers = { "Field" => "Value" }
 ```
 

data/docs/guides/debugging.md

@@ -0,0 +1,17 @@
+# Debugging
+
+[Wayfarer's CLI](/reference/cli/) has two sub-commands that come in handy when
+diagnosing problems in the development workflow.
+
+## Routing a URL from the shell
+
+## `wayfarer route`
+
+### `wayfarer route result JOB URL`
+
+:   Prints the result of invoking `JOB`'s router with `URL`.
+
+### `wayfarer route tree JOB URL`
+
+:   Visualises the routing tree result of invoking `JOB`'s router with `URL`.
+

data/docs/guides/error_handling.md

@@ -1,31 +1,31 @@
 # Error handling
 
-Wayfarer relies on Active Job's error handling facilities, `retry_on` and
-`discard_on`:
+## Wayfarer never swallows exceptions
 
-* [Active Job Basics: Exceptions](https://guides.rubyonrails.org/active_job_basics.html#exceptions)
-* [ActiveJob::Exceptions](https://edgeapi.rubyonrails.org/classes/ActiveJob/Exceptions/ClassMethods.html)
+* Wayfarer never swallows exceptions.
+* Jobs with unhandled exceptions are not retried.
 
-## Unhandled exceptions
+## Retrying and discarding
 
-Jobs with unhandled exceptions fail and are not retried.
+Wayfarer relies on [Active Job's two error handling facilities](https://guides.rubyonrails.org/active_job_basics.html#exceptions).
 
-## Retrying
+* `retry_on` to retry jobs a number of times on certain errors:
 
-```ruby
-class DummyJob < Wayfarer::Base
-  retry_on MyError, attempts: 3 do |job, error|
-    # All 3 retry attempts have failed
-  end
-end
-```
+    ```ruby
+    class DummyJob < Wayfarer::Base
+      retry_on MyError, attempts: 3 do |job, error|
+        # This block runs once all 3 attempts have failed
+        # (1 initial attempt + 2 retries)
+      end
+    end
+    ```
 
-## Discarding
+* `discard_on` to throw away jobs on certain errors:
 
-```ruby
-class DummyJob < Wayfarer::Base
-  discard_on MyError do |job, error|
-    # The job will not get retried
-  end
-end
-```
+    ```ruby
+    class DummyJob < Wayfarer::Base
+      discard_on MyError do |job, error|
+        # This block runs once and buries the job
+      end
+    end
+    ```