wayfarer 0.4.9 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 102ad4ff568c6a84fb710cbb46f3ca4839ce6a319265d33953f5ed601038d95d
-  data.tar.gz: 22f2f2460eb814a5534567cfc4222663d5819739df76e8c07bd0e5add4de15ec
+  metadata.gz: e46ffe5b40bd68f97cc1dc1c1777fc18f26ddf25e4b803efa99a6ca5ba8eb96e
+  data.tar.gz: 45359bcb3a30ac6d80052e05ecc35a78321c1b5d58fda6f6f2bf263375c9661d
 SHA512:
-  metadata.gz: 7c3c4780264a54b3ffc3b885027affb00d041c7d0d828435987d71d19cdaba03d69a03dda645901abfa59ab8f5b4f702dd85722fec87efb36a08d304a903296b
-  data.tar.gz: e05ab817b1125ebc8b644b79ec63e18fce3566ffdf4508e6409d5e8669a060b0bbd88e30a782db7f6abc9555198e1919fbb0feda4d9b75aba8f5b8823592c294
+  metadata.gz: 510c6839238f636cea908e27baed38b5f21999c56ef60486e1d8f23e006a205a25cd4e01f8ef017c52f35f91ae96fc198e582f8b517353128911cc7d355d76f0
+  data.tar.gz: a34c0c7a57688c4fc6442261391dcd8f55509711dbab4bd223303c376a8ca2a8f9e5e618cab472ef041c6cd3f1a660834a7d301b3d41624ea7cb79a42f19fa2e

data/.github/actions/setup-mise/action.yaml

@@ -0,0 +1,22 @@
+name: Setup mise
+description: Installs mise and tools
+runs:
+  using: composite
+  steps:
+    - name: Install mise
+      shell: bash
+      run: |
+        sudo apt update -y && sudo apt install -y curl
+        sudo install -dm 755 /etc/apt/keyrings
+        curl -fSs https://mise.jdx.dev/gpg-key.pub | sudo tee /etc/apt/keyrings/mise-archive-keyring.asc 1> /dev/null
+        echo "deb [signed-by=/etc/apt/keyrings/mise-archive-keyring.asc] https://mise.jdx.dev/deb stable main" | sudo tee /etc/apt/sources.list.d/mise.list
+        sudo apt update -y
+        sudo apt install -y mise
+
+    - name: Add mise shims to PATH
+      shell: bash
+      run: echo "$HOME/.local/share/mise/shims" >> $GITHUB_PATH
+
+    - name: Install tools
+      shell: bash
+      run: mise install -y

data/.github/workflows/push.yaml

@@ -0,0 +1,89 @@
+name: Push
+
+on:
+  push:
+    branches:
+      - '*'
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Start services
+        run: docker compose up -d wayfarer
+
+      - name: RuboCop
+        run: docker compose run --rm --name test wayfarer bundle exec rake lint:rubocop
+
+      - name: Verify RubyGem platforms
+        if: success() || failure()
+        run: docker compose run --rm wayfarer bundle exec rake lint:platforms
+
+      - name: Setup mise
+        if: success() || failure()
+        uses: ./.github/actions/setup-mise
+
+      - name: Vale
+        if: success() || failure()
+        env:
+          VALE_STYLES_PATH: /tmp/value
+        run: |
+          vale sync
+          vale --glob="*.md" $(git diff --name-only origin/${{ github.base_ref }}...HEAD)
+
+  unit-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Compose
+        run: docker compose up -d
+
+      - name: Run tests
+        run: docker compose exec -T -e CI=$CI wayfarer bundle exec rake test
+
+      - name: Verify gem version
+        run: docker compose exec -T -e CI=true wayfarer bundle exec rake release:guard_versions
+
+  check-links:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup mise
+        uses: ./.github/actions/setup-mise
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      - name: Build site (Strict)
+        run: python -m mkdocs build --strict
+
+      - name: Upload artifact
+        if: github.ref_name == github.event.repository.default_branch
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: 'site'
+
+  deploy-docs:
+    needs: check-links
+    if: github.ref_name == github.event.repository.default_branch
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

data/.github/workflows/release.yaml

@@ -4,10 +4,10 @@ on: workflow_dispatch
 
 jobs:
   release:
-    if: github.ref_name == 'master' || github.ref_name == 'develop'
+    if: github.ref_name == 'master'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - run: docker compose up -d
 

data/.gitignore

@@ -9,3 +9,4 @@ yard
 .tool-versions
 *.tar.gz
 styles
+.opencode

data/AGENTS.md

@@ -0,0 +1,265 @@
+# Wayfarer -- AI Assistant Guide
+
+## Project Overview
+
+Wayfarer is a Ruby gem (v0.5.0) for versatile web crawling, built on top of **ActiveJob**. It provides a middleware-based pipeline for fetching, parsing, deduplicating, and routing web pages, with support for plain HTTP, headless Chrome (Ferrum/Capybara), and Selenium browser automation.
+
+- **Gem name:** `wayfarer`
+- **Ruby version:** 3.4.4 (enforced via `mise.toml`)
+- **License:** MIT
+
+---
+
+## Technology Stack
+
+| Layer | Technology |
+|---|---|
+| Language | Ruby 3.4.4 |
+| Job framework | ActiveJob 7.1+ |
+| Deduplication/state | Redis 5.4+ |
+| Plain HTTP | net-http-persistent 4.0+ |
+| Headless Chrome (CDP) | Ferrum 0.17+ |
+| Browser (Selenium) | selenium-webdriver 4.35+ |
+| Browser wrapper | Capybara 3.4+ |
+| HTML/XML parsing | Nokogiri 1.18+ |
+| Meta-tag parsing | MetaInspector 5.16+ |
+| URL routing/matching | Mustermann 3.0+ |
+| URL normalization | Addressable 2.8+ |
+| Autoloading | Zeitwerk 2.7+ |
+| CLI | Thor 1.0+ |
+| Connection pooling | connection_pool 2.5+ |
+| Testing | RSpec 3.13+ |
+| Linting | RuboCop 1.80+ |
+| Documentation | MkDocs (Material theme) + YARD |
+
+---
+
+## Development Workflow
+
+### Initial Setup
+
+```bash
+mise install    # Install Ruby 3.4.4, Python 3.13, vale
+bundle install
+```
+
+### Docker (recommended for CI-equivalent runs)
+
+```bash
+docker compose up -d
+docker compose exec -T -e CI=true wayfarer bundle exec rake test
+docker compose run --rm wayfarer bundle exec rake lint:rubocop
+```
+
+### Interactive Console
+
+```bash
+bundle exec rake console   # Pry REPL with wayfarer loaded
+```
+
+---
+
+## Running Tests
+
+Tests use **RSpec** with tag-based filtering. Always run via Rake, not `rspec` directly.
+
+```bash
+bundle exec rake test             # All tests
+bundle exec rake test:isolated    # No browser/Redis -- fastest
+bundle exec rake test:integration # Redis + CLI
+bundle exec rake test:selenium    # Selenium browser
+bundle exec rake test:ferrum      # Ferrum (Chrome CDP)
+bundle exec rake test:cli         # CLI tests
+```
+
+**RSpec tags:** `:selenium`, `:ferrum`, `:cli`, `:redis`. Untagged tests run in `test:isolated`.
+
+A Sinatra test server starts automatically on port 9876 before the suite. FactoryBot factories are in `spec/factories/`. The shared context `"with Redis"` is auto-included for `:redis`-tagged examples.
+
+---
+
+## Linting
+
+```bash
+bundle exec rake lint:rubocop
+```
+
+Key RuboCop settings: 120-char line limit, double-quoted strings, outdented access modifiers, max 20-line methods, AbcSize <= 30, RSpec max 5 expectations/example and max 5 nested groups.
+
+---
+
+## Common Ruby Patterns
+
+### ActiveSupport::Concern for Mixins
+
+The codebase uses `extend ActiveSupport::Concern` throughout to structure reusable modules with `included` and `class_methods` blocks. The `included` block runs on inclusion; `class_methods` defines class-level behavior without verbose `def self.included` boilerplate.
+
+```ruby
+module Base
+  extend ActiveSupport::Concern
+
+  included do
+    include Wayfarer::Middleware::Controller
+    use Wayfarer::Middleware::Redis
+    use Wayfarer::Middleware::UriParser
+    # ...
+  end
+
+  class_methods do
+    def crawl(url, batch: SecureRandom.uuid)
+      Task.new(url, batch).tap { |task| perform_later(task) }
+    end
+  end
+end
+```
+
+### Middleware Chain via Recursive Lambda
+
+`Middleware::Chain` composes middleware as a recursive lambda. Each middleware receives a block representing the rest of the chain, and calls `yield` to continue or returns early to halt.
+
+```ruby
+def call(task)
+  traverse = lambda do |i|
+    if middlewares[i]
+      middlewares[i].call(task) { traverse.call(i + 1) }
+    elsif block_given?
+      yield
+    end
+  end
+  traverse.call(0)
+end
+```
+
+### Strategy Pattern via Module Inclusion
+
+`Networking::Strategy` defines the abstract interface (raising `NotImplementedError`) and shared helpers (`success`, `redirect`). Concrete agents (`HTTP`, `Ferrum`, `Selenium`, `Capybara`) include the mixin and override the required methods.
+
+```ruby
+module Strategy
+  extend ActiveSupport::Concern
+
+  def fetch(instance, url) = navigate(instance, url).then { live(instance) }
+  def navigate(_instance, _url) = raise NotImplementedError
+  def create = raise NotImplementedError
+  def destroy(_instance); end
+
+private
+  def success(...) = Networking::Result::Success.new(Page.new(...))
+  def redirect(...) = Networking::Result::Redirect.new(...)
+end
+```
+
+### Fluent Routing DSL
+
+`Routing::DSL` provides chainable builder methods. Each method calls `child_route`, which creates a child `Route` and appends it to `children`. Inline keyword arguments and blocks are both supported, enabling two equivalent syntaxes:
+
+```ruby
+# Inline
+route.host("example.com", path: "/articles/:id", to: :article)
+
+# Block
+route.host("example.com") do
+  path "/articles/:id", to: :article
+end
+```
+
+The method signature uses `**` and `&` forwarding to keep DSL methods concise. Last `.to()` call wins when routes conflict.
+
+### Visitor Pattern for Tree Traversal
+
+Routes form a tree searched depth-first by `Routing::PathFinder`. Routes call `accept(visitor)`, which calls `enter`, `visit`, and `leave` on the visitor. This separates traversal logic from matching logic while allowing the visitor to accumulate mutable state (params, matched actions).
+
+```ruby
+# In Route
+def accept(visitor)
+  visitor.enter(self)
+  return visitor.leave unless visitor.visit(self)
+  children.each { |child| child.accept(visitor) }
+  visitor.leave
+end
+```
+
+### Dynamic Method Generation via Module Subclasses
+
+`Logging.emit` returns an anonymous `Module` subclass whose `included` hook uses `define_method` to inject a `log` method. This lets each middleware declare its log messages declaratively at the class level.
+
+```ruby
+class Router
+  include Wayfarer::Logging.emit(
+    mismatch: [:info,  "No matching route"],
+    match:    [:info,  "Routing to %<action>s"],
+  )
+end
+```
+
+### Singleton + Connection Pooling
+
+`Networking::Pool` is a `Singleton` wrapping `ConnectionPool`. Cleanup is registered via `at_exit`. A `class_attribute :finalizer` lambda allows customizing shutdown behavior without subclassing. `ConnectionPool::TimeoutError` is wrapped into the gem's own error class at the boundary.
+
+### Configuration Syntax
+
+Use bracket syntax `Wayfarer.config[:key][:subkey]` instead of `dig` for predictable configuration access.
+
+### Value Objects with Declarative Stringify
+
+Value objects (`Task`, `Page`) include lightweight mixins like `Stringify` and `KV`. `stringify :url, :batch` registers which attributes appear in `#to_s`/`#inspect`. Custom `ActiveJob::Serializers::ObjectSerializer` subclasses handle serialization to/from JSON.
+
+### Inline Error Classes
+
+Lightweight custom errors are defined inline at module scope:
+
+```ruby
+UserAgentTimeoutError = Class.new(StandardError)
+```
+
+Rescue lists are built dynamically from configuration:
+
+```ruby
+rescue *strategy.class.renew_on, *Wayfarer.config[:network][:renew_on] => e
+  renew
+end
+```
+
+---
+
+## Documentation
+
+### Writing Style
+
+Documentation in `docs/` must strictly mimic the voice, tone, and typography of existing guides. Key rules:
+
+- **Address the reader directly** using second person ("you can", "use the")
+- **Use contractions** ("isn't", "doesn't", "it's") -- Vale enforces `Google.Contractions`
+- **Prefer active voice** over passive voice ("the graph compiles" not "the chain is compiled")
+- **Use ASCII characters** -- write `word--word` not Unicode em dashes. Avoid Unicode dashes, quotes, and other non-ASCII punctuation unless explicitly required.
+- **Headings use sentence-style capitalization** -- `## Adding middleware` not `## Adding Middleware`
+- **Keep paragraphs short** -- one concept per paragraph, 2-3 sentences max
+- **Code references use backticks** -- `` `use` ``, `` `dependency_graph.remove` ``
+- **Cross-link related guides** -- `[tasks](tasks.md)`, `[handlers](handlers.md)`
+- **Use mkdocs-material admonitions** for callouts -- `!!! note`, `!!! warning`, `!!! danger`
+
+### Linting with Vale
+
+Documentation is linted with [Vale](https://vale.sh) using the Google style guide. Configuration lives in `.vale.ini` and domain vocabulary in `styles/config/vocabularies/Wayfarer/accept.txt`.
+
+```bash
+vale sync                          # Download style packages (run once)
+vale docs/guides/middleware.md     # Lint a specific file
+vale docs/                         # Lint all documentation
+```
+
+Fix all errors and warnings. Suggestions (passive voice, acronym expansion) are acceptable when they match the existing documentation style.
+
+---
+
+## Key Conventions
+
+- **All files:** `# frozen_string_literal: true` at the top
+- **Strings:** Always double-quoted
+- **Modules vs Classes:** Namespaced under `Wayfarer::*`; use `ActiveSupport::Concern` for mixins
+- **Autoloading:** Zeitwerk with custom inflections -- file names must match constant names after inflection (`cli` -> `CLI`, `dsl` -> `DSL`, `http` -> `HTTP`, `uri` -> `URI`, `url` -> `URL`, etc.)
+- **Numeric comparisons:** Use `== 0` not `.zero?`
+- **No module documentation cop** -- use YARD `@param`/`@return`/`@yield`/`@example`/`@see` tags to document complex logic, public APIs, and overridden template methods instead. Use comments to explain *why* something is done.
+- **Specs:** Tag browser tests (`:selenium`, `:ferrum`), Redis tests (`:redis`), CLI tests (`:cli`); all others run isolated
+- **Do not skip RuboCop** without an inline disable comment with justification
+- **Version consistency:** `VERSION::STRING` in `lib/wayfarer.rb` must match `wayfarer.gemspec`; CI enforces via `rake release:guard_versions`

data/CLAUDE.md

@@ -0,0 +1 @@
+AGENTS.md

data/Dockerfile

@@ -2,6 +2,7 @@ FROM ruby:3.4.4-alpine
 RUN apk add --no-cache build-base git yaml-dev zlib-dev
 RUN git config --global --add safe.directory /opt/app
 WORKDIR /opt/app
+ENV BUNDLE_FROZEN=true
 COPY Gemfile Gemfile.lock *.gemspec ./
 RUN bundle install
 COPY . .

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    wayfarer (0.4.9)
+    wayfarer (0.5.0)
       activejob (>= 7.1)
       addressable (~> 2.8)
       capybara (~> 3.4)
@@ -98,6 +98,7 @@ GEM
       concurrent-ruby (~> 1.1)
       webrick (~> 1.7)
       websocket-driver (~> 0.7)
+    ffi (1.17.2-arm64-darwin)
     ffi (1.17.2-x86_64-darwin)
     ffi (1.17.2-x86_64-linux-musl)
     globalid (1.2.1)
@@ -148,6 +149,8 @@ GEM
     net-http-persistent (4.0.6)
       connection_pool (~> 2.2, >= 2.2.4)
     nio4r (2.7.4)
+    nokogiri (1.18.9-arm64-darwin)
+      racc (~> 1.4)
     nokogiri (1.18.9-x86_64-darwin)
       racc (~> 1.4)
     nokogiri (1.18.9-x86_64-linux-musl)
@@ -302,6 +305,7 @@ GEM
     zlib (3.2.1)
 
 PLATFORMS
+  arm64-darwin-25
   x86_64-darwin-24
   x86_64-linux-musl
 

data/docs/cookbook/batch_routing.md

@@ -4,7 +4,9 @@
 # Create a record in an external database and store the hostname
 record = Database::Row.create(hostname: "example.com")
 
-class DummyJob < Wayfarer::Base
+class DummyJob < ActiveJob::Base
+  include Wayfarer::Base
+
   route do |hostname|
     host hostname, to: :index
   end

data/docs/cookbook/consent_screen.md

@@ -8,7 +8,9 @@ iframe, clicked, and makes the live page behind the screen accessible to
 ```ruby
 Wayfarer.config[:network][:agent] = :ferrum
 
-class DummyJob < Wayfarer::Base
+class DummyJob < ActiveJob::Base
+  include Wayfarer::Base
+
   route.to :index, host: "example.com"
 
   before_action if: :consent_required? do

data/docs/cookbook/executing_javascript.md

@@ -5,7 +5,9 @@ Executing JavaScript requires automating a browser.
 === "Ferrum"
 
     ```ruby
-    class DummyJob < Wayfarer::Base
+    class DummyJob < ActiveJob::Base
+      include Wayfarer::Base
+
       route.to :index
 
       def index
@@ -17,7 +19,9 @@ Executing JavaScript requires automating a browser.
 === "Selenium"
 
     ```ruby
-    class DummyJob < Wayfarer::Base
+    class DummyJob < ActiveJob::Base
+      include Wayfarer::Base
+
       route.to :index
 
       def index
@@ -30,7 +34,9 @@ Executing JavaScript requires automating a browser.
 === "Capybara"
 
     ```ruby
-    class DummyJob < Wayfarer::Base
+    class DummyJob < ActiveJob::Base
+      include Wayfarer::Base
+
       route.to :index
 
       def index

data/docs/cookbook/navigation.md

@@ -3,7 +3,9 @@
 === "Ferrum"
 
     ```ruby
-    class DummyJob < Wayfarer::Base
+    class DummyJob < ActiveJob::Base
+      include Wayfarer::Base
+
       route.to :index
 
       def index
@@ -17,7 +19,9 @@
 === "Selenium"
 
     ```ruby
-    class DummyJob < Wayfarer::Base
+    class DummyJob < ActiveJob::Base
+      include Wayfarer::Base
+
       route.to :index
 
       def index
@@ -31,7 +35,9 @@
 === "Capybara"
 
     ```ruby
-    class DummyJob < Wayfarer::Base
+    class DummyJob < ActiveJob::Base
+      include Wayfarer::Base
+
       route.to :index
 
       def index

data/docs/cookbook/querying_html.md

@@ -5,7 +5,9 @@ See: [Nokogiri: Searching an HTML / XML Document](https://nokogiri.org/tutorials
 === "Nokogiri"
 
     ```ruby
-    class DummyJob < Wayfarer::Base
+    class DummyJob < ActiveJob::Base
+      include Wayfarer::Base
+
       route.to :index
 
       def index
@@ -18,7 +20,9 @@ See: [Nokogiri: Searching an HTML / XML Document](https://nokogiri.org/tutorials
 === "Ferrum"
 
     ```ruby
-    class DummyJob < Wayfarer::Base
+    class DummyJob < ActiveJob::Base
+      include Wayfarer::Base
+
       route.to :index
 
       def index
@@ -31,7 +35,9 @@ See: [Nokogiri: Searching an HTML / XML Document](https://nokogiri.org/tutorials
 === "Selenium"
 
     ```ruby
-    class DummyJob < Wayfarer::Base
+    class DummyJob < ActiveJob::Base
+      include Wayfarer::Base
+
       route.to :index
 
       def index

data/docs/cookbook/screenshots.md

@@ -5,7 +5,9 @@ Taking screenshots requires automating a browser.
 === "Ferrum"
 
     ```ruby
-    class DummyJob < Wayfarer::Base
+    class DummyJob < ActiveJob::Base
+      include Wayfarer::Base
+
       route.to :index
 
       def index
@@ -17,7 +19,9 @@ Taking screenshots requires automating a browser.
 === "Selenium"
 
     ```ruby
-    class DummyJob < Wayfarer::Base
+    class DummyJob < ActiveJob::Base
+      include Wayfarer::Base
+
       route.to :index
 
       def index

data/docs/guides/cli.md

@@ -8,7 +8,7 @@ The command-line interface to Wayfarer.
 wayfarer [OPTIONS] [perform|enqueue|execute|route|tree]
 ```
 
-See [Configuration](../reference/cli) for the respected environment variables.
+See [Configuration](configuration.md) for the respected environment variables.
 
 ---
 

data/docs/guides/debugging.md

@@ -1,6 +1,6 @@
 # Debugging
 
-[Wayfarer's CLI](/reference/cli/) has two sub-commands that come in handy when
+[Wayfarer's CLI](cli.md) has two sub-commands that come in handy when
 diagnosing problems in the development workflow.
 
 ## Routing a URL from the shell