wayfarer 0.4.6 → 0.4.7

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wayfarer
 version: !ruby/object:Gem::Version
-  version: 0.4.6
+  version: 0.4.7
 platform: ruby
 authors:
 - Dominic Bauer
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '7.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '7.1'
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -123,33 +123,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: mustermann
+  name: mock_redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '0.29'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '0.29'
 - !ruby/object:Gem::Dependency
-  name: mock_redis
+  name: mustermann
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.29'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.29'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: net-http-persistent
   requirement: !ruby/object:Gem::Requirement
@@ -240,6 +240,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
 - !ruby/object:Gem::Dependency
   name: cuprite
   requirement: !ruby/object:Gem::Requirement
@@ -373,12 +387,15 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/workflows/ci.yaml"
+- ".github/workflows/lint.yaml"
+- ".github/workflows/release.yaml"
+- ".github/workflows/tests.yaml"
 - ".gitignore"
 - ".rbenv-gemsets"
 - ".rspec"
 - ".rubocop.yml"
 - ".ruby-version"
+- ".vale.ini"
 - ".yardopts"
 - Dockerfile
 - Gemfile
@@ -395,60 +412,53 @@ files:
 - docs/cookbook/querying_html.md
 - docs/cookbook/screenshots.md
 - docs/cookbook/user_agent.md
+- docs/design.md
 - docs/guides/callbacks.md
 - docs/guides/configuration.md
 - docs/guides/debugging.md
-- docs/guides/error_handling.md
+- docs/guides/handlers.md
+- docs/guides/index.md
 - docs/guides/jobs.md
+- docs/guides/jobs/error_handling.md
 - docs/guides/navigation.md
-- docs/guides/networking.md
 - docs/guides/networking/capybara.md
 - docs/guides/networking/custom_adapters.md
 - docs/guides/networking/ferrum.md
 - docs/guides/networking/http.md
 - docs/guides/networking/selenium.md
 - docs/guides/pages.md
-- docs/guides/performance.md
-- docs/guides/reliability.md
-- docs/guides/routing/steering.md
+- docs/guides/redis.md
+- docs/guides/routing.md
 - docs/guides/tasks.md
+- docs/guides/tutorial.md
+- docs/guides/user_agents.md
 - docs/index.md
-- docs/reference/api/base.md
 - docs/reference/api/route.md
 - docs/reference/cli.md
-- docs/reference/configuration_keys.md
-- docs/reference/environment_variables.md
+- docs/reference/configuration.md
 - lib/wayfarer.rb
 - lib/wayfarer/base.rb
+- lib/wayfarer/batch_completion.rb
 - lib/wayfarer/callbacks.rb
-- lib/wayfarer/cli/base.rb
-- lib/wayfarer/cli/generate.rb
-- lib/wayfarer/cli/job.rb
-- lib/wayfarer/cli/route.rb
+- lib/wayfarer/cli.rb
 - lib/wayfarer/cli/route_printer.rb
-- lib/wayfarer/cli/runner.rb
-- lib/wayfarer/cli/templates/Gemfile.tt
-- lib/wayfarer/cli/templates/job.rb.tt
-- lib/wayfarer/config/capybara.rb
-- lib/wayfarer/config/ferrum.rb
-- lib/wayfarer/config/networking.rb
-- lib/wayfarer/config/redis.rb
-- lib/wayfarer/config/root.rb
-- lib/wayfarer/config/selenium.rb
-- lib/wayfarer/config/strconv.rb
-- lib/wayfarer/config/struct.rb
 - lib/wayfarer/gc.rb
 - lib/wayfarer/handler.rb
+- lib/wayfarer/logging.rb
 - lib/wayfarer/middleware/base.rb
+- lib/wayfarer/middleware/batch_completion.rb
 - lib/wayfarer/middleware/chain.rb
+- lib/wayfarer/middleware/content_type.rb
 - lib/wayfarer/middleware/controller.rb
 - lib/wayfarer/middleware/dedup.rb
 - lib/wayfarer/middleware/dispatch.rb
-- lib/wayfarer/middleware/fetch.rb
 - lib/wayfarer/middleware/lazy.rb
 - lib/wayfarer/middleware/normalize.rb
+- lib/wayfarer/middleware/redis.rb
 - lib/wayfarer/middleware/router.rb
 - lib/wayfarer/middleware/stage.rb
+- lib/wayfarer/middleware/uri_parser.rb
+- lib/wayfarer/middleware/user_agent.rb
 - lib/wayfarer/networking/capybara.rb
 - lib/wayfarer/networking/context.rb
 - lib/wayfarer/networking/ferrum.rb
@@ -459,13 +469,13 @@ files:
 - lib/wayfarer/networking/selenium.rb
 - lib/wayfarer/networking/strategy.rb
 - lib/wayfarer/page.rb
+- lib/wayfarer/parsing.rb
 - lib/wayfarer/parsing/json.rb
 - lib/wayfarer/parsing/xml.rb
 - lib/wayfarer/redis/barrier.rb
-- lib/wayfarer/redis/connection.rb
 - lib/wayfarer/redis/counter.rb
 - lib/wayfarer/redis/pool.rb
-- lib/wayfarer/redis/version.rb
+- lib/wayfarer/redis/resettable.rb
 - lib/wayfarer/routing/dsl.rb
 - lib/wayfarer/routing/matchers/custom.rb
 - lib/wayfarer/routing/matchers/host.rb
@@ -478,26 +488,21 @@ files:
 - lib/wayfarer/routing/result.rb
 - lib/wayfarer/routing/root_route.rb
 - lib/wayfarer/routing/route.rb
-- lib/wayfarer/routing/router.rb
 - lib/wayfarer/routing/target_route.rb
 - lib/wayfarer/serializer.rb
 - lib/wayfarer/stringify.rb
 - lib/wayfarer/task.rb
 - mkdocs.yml
+- rake/docs.rake
+- rake/lint.rake
+- rake/release.rake
+- rake/tests.rake
 - requirements.txt
 - spec/base_spec.rb
-- spec/callbacks_spec.rb
-- spec/cli/generate_spec.rb
+- spec/batch_completion_spec.rb
 - spec/cli/job_spec.rb
+- spec/cli/routing_spec.rb
 - spec/cli/version_spec.rb
-- spec/config/capybara_spec.rb
-- spec/config/ferrum_spec.rb
-- spec/config/networking_spec.rb
-- spec/config/redis_spec.rb
-- spec/config/root_spec.rb
-- spec/config/selenium_spec.rb
-- spec/config/strconv_spec.rb
-- spec/config/struct_spec.rb
 - spec/factories/middleware.rb
 - spec/factories/page.rb
 - spec/factories/task.rb
@@ -505,18 +510,25 @@ files:
 - spec/gc_spec.rb
 - spec/handler_spec.rb
 - spec/integration/callbacks_spec.rb
+- spec/integration/content_type_spec.rb
+- spec/integration/gc_spec.rb
+- spec/integration/handler_spec.rb
 - spec/integration/page_spec.rb
 - spec/integration/params_spec.rb
+- spec/integration/parsing_spec.rb
+- spec/integration/routing_spec.rb
 - spec/integration/stage_spec.rb
-- spec/integration/steering_spec.rb
+- spec/middleware/batch_completion_spec.rb
 - spec/middleware/chain_spec.rb
+- spec/middleware/content_type_spec.rb
 - spec/middleware/controller_spec.rb
 - spec/middleware/dedup_spec.rb
 - spec/middleware/dispatch_spec.rb
-- spec/middleware/fetch_spec.rb
 - spec/middleware/normalize_spec.rb
 - spec/middleware/router_spec.rb
 - spec/middleware/stage_spec.rb
+- spec/middleware/uri_parser_spec.rb
+- spec/middleware/user_agent_spec.rb
 - spec/networking/capybara_spec.rb
 - spec/networking/context_spec.rb
 - spec/networking/ferrum_spec.rb
@@ -531,7 +543,6 @@ files:
 - spec/redis/barrier_spec.rb
 - spec/redis/counter_spec.rb
 - spec/redis/pool_spec.rb
-- spec/redis/version_spec.rb
 - spec/routing/dsl_spec.rb
 - spec/routing/integration_spec.rb
 - spec/routing/matchers/custom_spec.rb
@@ -544,7 +555,6 @@ files:
 - spec/routing/path_finder_spec.rb
 - spec/routing/root_route_spec.rb
 - spec/routing/route_spec.rb
-- spec/routing/router_spec.rb
 - spec/spec_helpers.rb
 - spec/stringify_spec.rb
 - spec/support/static/finders.html

data/.github/workflows/ci.yaml

@@ -1,32 +0,0 @@
-name: ci
-
-on:
-  push:
-    branches:
-      - '*'
-env:
-  CI: true
-
-jobs:
-  ci:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Start services
-        run: docker-compose up -d
-
-      - name: Run isolated tests
-        run: docker-compose run --rm --name test --service-ports wayfarer bundle exec rake test:isolated
-
-      - name: Run Ferrum tests
-        run: docker-compose run --rm --name test --service-ports wayfarer bundle exec rake test:ferrum
-
-      - name: Run Selenium tests
-        run: docker-compose run --rm --name test --service-ports wayfarer bundle exec rake test:selenium
-
-      - name: Run CLI tests
-        run: docker-compose run --rm --name test --service-ports wayfarer bundle exec rake test:cli
-
-      - name: Run RuboCop
-        run: docker-compose run --rm --name test --service-ports wayfarer bundle exec rake rubocop

data/docs/guides/error_handling.md

@@ -1,53 +0,0 @@
-# Error handling
-
-## Wayfarer never swallows exceptions
-
-* Wayfarer never swallows exceptions.
-* Jobs with unhandled exceptions are not retried.
-
-## Retrying or discarding failing jobs
-
-Wayfarer relies on [Active Job's two error handling facilities](https://guides.rubyonrails.org/active_job_basics.html#exceptions).
-
-* `retry_on` to retry jobs a number of times on certain errors:
-
-    ```ruby
-    class DummyJob < Wayfarer::Base
-      retry_on MyError, attempts: 3 do |job, error|
-        # This block runs once all 3 attempts have failed
-        # (1 initial attempt + 2 retries)
-
-        raise error
-      end
-    end
-    ```
-
-* `discard_on` to throw away jobs on certain errors:
-
-    ```ruby
-    class DummyJob < Wayfarer::Base
-      discard_on MyError do |job, error|
-        # This block runs once and buries the job
-
-        raise error
-      end
-    end
-    ```
-
-!!! attention "Always re-raise errors"
-
-    You should always re-raise errors from `retry_on` and `discard_on` blocks,
-    otherwise jobs will not get retried!
-
-## Renewing agents on certain errors
-
-```ruby
-Wayfarer.config.network.renew_on = [MyError]
-```
-
-For example, if you use the Capybara
-[Cuprite](https://github.com/rubycdp/cuprite) driver:
-
-```ruby
-Wayfarer.config.network.renew_on = [Ferrum::DeadBrowserError]
-```

data/docs/guides/networking.md

@@ -1,94 +0,0 @@
-# Networking
-
-Wayfarer navigates the web in two ways:
-
-1. Via plain HTTP requests
-2. By automating browsers
-
-Both options are mutually exclusive per Ruby process.
-
-## User agents
-
-A user agent is an entity that knows how to retrieve the contents behind a URL.
-
-The user agent can be configured via the global configuration:
-
-```ruby
-Wayfarer.config.network.agent = :http # or :ferrum, :selenium
-```
-
-## Connection pooling
-
-Wayfarer keeps user agents within a connection pool. When a job executes
-and needs to retrieve the contents behind a URL, an agent is checked out from
-the pool.
-
-The pool has a constant size and it should equal the number of threads the
-underlying message queue operates with. The size can be configured via the
-global configuration:
-
-```ruby
-Wayfarer.config.network.pool_size = 8
-```
-
-### Timeouts
-
-user agents may stay checked out from the pool by jobs for a limited time
-only. Once this time limit is exceeded, a `ConnectionPool::TimeoutError`
-exception is raised. This places a hard time limit on every job.
-
-The timeout can be configured via the global configuration:
-
-```ruby
-Wayfarer.config.network.pool_timeout = 20 # seconds
-```
-
-Because jobs with unhandled exceptions fail, explicit error handling is required
-if retries are desired:
-
-```ruby
-class DummyJob < Wayfarer::Base
-  retry_on ConnectionPool::TimeoutError, attempts: 3
-end
-```
-
-## Agent-specific client timeouts
-
-The time in seconds it may take to communicate with remote browser processes can
-be configured globally per agent:
-
-```ruby
-Wayfarer.config.ferrum.options = { timeout: 5 }
-Wayfarer.config.selenium.client_timeout = 60
-```
-
-### Shared state
-
-As user agents get checked in and out continously between jobs, their state
-carries over from job to job, too.
-
-For browser automation, this means:
-
-* A job finds the browser at the last URL the previous job has left off.
-* The browser's cookies might have been set, or other client-side state might
-  exist that significantly affects a page's behaviour.
-
-## HTTP redirect handling
-
-Browsers follow redirects transparently when they are navigated to a URL.
-
-When using plain HTTP, redirect URLs are enqueued transparently within the same
-batch. URLs that result in 3xx responses will not be retrieved again within
-their batch.
-
-## HTTP request headers
-
-Request headers can be configured via the global configuration:
-
-```ruby
-Wayfarer.config.network.http_headers = { "Field" => "Value" }
-```
-
-!!! attention "Partial support"
-
-    Selenium does not support configuring HTTP request headers.

data/docs/guides/performance.md

@@ -1,130 +0,0 @@
-# Performance
-
-How to write performant crawlers with Wayfarer.
-
-## Use a sufficiently sized user agent pool
-
-Automated browser processes or HTTP clients are kept in a [connection pool]() of
-static size. This avoids having to re-establish browser processes and enables
-their reuse.
-
-If the size of the pool is too small, the pool is a
-bottleneck. For example, if your message queue adapter uses 8 threads, but the
-pool only contains 1 user agent, the remaining 7 threads block until the agent
-is checked back in to the pool for use by one of the blocked threads.
-
-There is no reliable way to detect the number of threads of the underlying
-message queue adapter. The pool size should equal the number of threads;
-
-```ruby
-Wayfarer.config.network.pool_size = 8 # defaults to 1
-```
-
-### Job shedding
-
-There is a maximum number of seconds that jobs wait when checking out a user
-agent from the pool. Once this time is exceeded,
-a `Wayfarer::UserAgentTimeoutError` is raised. By default, the timeout is 10
-seconds.
-
-This hints there are more threads in use than user agents in the pool.
-
-## Stage less URLs
-
-Staging less URLs saves space and time:
-
-* Less tasks written to the message queue
-* Less time spent consuming tasks
-* Less time spent filtering URLs with Redis
-
-Wayfarer maintains a set of processed URLs for a batch in Redis. Every staged
-URL is checked for inclusion in this set before it gets appended as a task to
-the message queue.
-
-A common pattern is to stage all links of a page, and rely on routing to fetch
-only the relevant ones:
-
-```ruby
-class DummyJob < Wayfarer::Base
-  route { to: index, host: "example.com" }
-
-  def index
-    stage page.meta.links.all
-  end
-end
-```
-
-Pages commonly contain a large number of URLs.
-
-Every staged URL is:
-
-1. Normalized to a canonical form, for example by sorting query parameters
-   alphabetically.
-2. Checked for inclusion in the batch Redis set or discarded.
-3. Written to the message queue.
-4. Consumed from the queue and matched against the router.
-5. Fetched, if a route matches.
-
-Narrowing down the links in the document to follow speeds up the process.
-For example using Nokogiri, interesting links can be identified with a CSS
-selector:
-
-```ruby
-class DummyJob < Wayfarer::Base
-  route { to: index, host: "example.com" }
-
-  def index
-    stage interesting_links
-  end
-
-  private
-
-  def interesting_links
-    page.doc.css("a.interesting").map { |elem| elem["href"] }
-  end
-end
-```
-
-Because the router only accepts the single hostname `example.com`, the job can
-also ensure it stages only internal URLs by intersecting them with the
-interesting ones:
-
-```ruby
-class DummyJob < Wayfarer::Base
-  route { to: index, host: "example.com" }
-
-  def index
-    stage interesting_internal_links
-  end
-
-  private
-
-  def interesting_internal_links
-    page.meta.links.internal & interesting_links
-  end
-
-  def interesting_links
-    page.doc.css("a.interesting").map { |elem| elem["href"] }
-  end
-end
-```
-
-
-## Use Redis >= 6.2.0
-
-Redis 6.2.0 introduced the
-[`SMISMEMBER`](https://redis.io/commands/smismember) command which enables
-Wayfarer to check whether multiple URLs have been processed in a batch with a
-single command. With earlier versions, one command per URL is required.
-
-Wayfarer detects the Redis server version and uses `SMISMEMBER` without user
-configuration when supported.
-
-## Use Oj for JSON parsing
-
-Wayfarer uses [Oj](https://github.com/ohler55/oj) for JSON parsing if the gem
-has been required at runtime:
-
-```ruby
-require "oj"
-```

data/docs/guides/reliability.md

@@ -1,41 +0,0 @@
-# Reliablity
-
-## Durability
-
-Wayfarer executes atop reliable messages queues such as Sidekiq, Resque,
-RabbitMQ, etc. Its configuration is independent of the underlying queue
-infrastructure it reads from and writes to.
-
-## Self-healing user agents
-
-Wayfarer handles the scenario where a remote browser process has crashed and
-must be replaced by a fresh browser process.
-
-This can be tested locally by automating a browser with headless mode turned
-off, and then closing the opened browser window: The current job fails, but the
-next job has access to a newly established browser session again.
-
-For example Ferrum might raise `Ferrum::DeadBrowserError`. Wayfarer's
-user agents are self-healing and react to these kinds of errors internally. When
-a browser window is closed, the Ferrum user agent attempts to establish a new
-browser process as a replacement, for the next job to use.
-
-[Wayfarer never swallows exceptions](/guides/error_handling). This means
-that even though the user agent might heal itself, jobs still need to explicitly
-retry browser errors:
-
-```ruby
-class Foobar < Wayfarer::Base
-  route { to: :index }
-
-  retry_on Ferrum::DeadBrowserError, attempts: 3, wait: :exponentially_longer
-
-  # ...
-end
-```
-
-This leads to log entries like:
-
-```
-Retrying DummyJob in 3 seconds, due to a Ferrum::DeadBrowserError.
-```

data/docs/guides/routing/steering.md

@@ -1,30 +0,0 @@
-# Steering
-
-A job's router can receive arguments computed dynamically by `::steer`.
-Steering enables [batch routing](/cookbook/batch_routing).
-
-For example, the following router has hostname and path hard-coded:
-
-```ruby
-class DummyJob < Wayfarer::Base
-  route do
-    host "example.com", path: "/contact", to: :index
-  end
-end
-```
-
-Instead, hostname and path could be provided by `::steer`, too:
-
-```ruby
-class DummyJob < Wayfarer::Base
-  route do |hostname, path|
-    host hostname, path: path, to: :index
-  end
-
-  steer do |_task|
-    ["example.com", "/contact"]
-  end
-end
-```
-
-Note that `steer` yields the current [task](/guides/tasks).

data/docs/reference/api/base.md

@@ -1,48 +0,0 @@
----
-title: Wayfarer::Base
----
-
-# `Wayfarer::Base`
-
-Wayfarer's complete job API.
-
----
-
-### `::route`
-:   Draw routes to instance methods.
-
----
-
-### `::steer { (Wayfarer::Task) -> [any] }`
-:   Provide router arguments.
-
----
-
-### `#task -> Wayfarer::Task`
-:   The currently processing task.
-
----
-
-### `#params -> Hash`
-:   URL parameters collected from the matching route.
-
----
-
-### `#stage(String | [String]) -> void`
-:   Add URLs to a processing set. URLs already processed within the
-    current batch get discarded are not enqueued. Every staged URL gets
-    normalized.
-
----
-
-### `#browser -> Object`
-:   The user agent that retrieved the current page.
-
----
-
-### `#page(live: true | false) -> Page`
-:   The page representing the response retrieved from the currently
-    processing URL.
-
-    With `live: true` called, a fresh `Page` is returned that reflects the
-    current browser DOM. Calls to `#page` return the most recent page.