activeagent 1.7.1 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14e5b0cec7ba170c446d1ed32ad5c9d0705ac8eae4fdc6e4e4a7d17bdae502a2
-  data.tar.gz: 28e12f0ef3727a97fc231e5f7ace15ad4cb020b379fc76c757c2206c89f2b2e6
+  metadata.gz: 3642068294052d483decf5260b5d7dd1793647f06920713ca658f7b89ba948b0
+  data.tar.gz: 1d0811bee23dc646e916d17abc711637c1e2d41055633ace8bb1c067615d26b4
 SHA512:
-  metadata.gz: 9cd766d568ab510b043e4902c1c5fa73ce877a268761168509f2772190ab0f2d6f259fe62e4654aa2a002d20228f876fce82c7c2f5f3e1018cd6cb01935aed27
-  data.tar.gz: 197cab65243e11561c7216f08e9b30369819e3ef73627d0f6e72092a7fc74c16a278638f5044e11f292149a1a39dbe11cab9593a41e7289049a3a90c6232a0a0
+  metadata.gz: 6d4492797c3d4b238bf86c49c48b9ec12e5876bbc9ef075c4cae7f369a3e3274b3db30781b9367643740bd9fa365eb8fe8ad81308b125f6843f3f828ccf5f339
+  data.tar.gz: 601676a4590808eef1469f3cfafc5136c43bccbd22e7b600c15940eb7a1fc957479c568676d699d7d41c10836fa621eb072a3ff5fd20d5f4cfa22a517e982416

data/CHANGELOG.md

@@ -7,6 +7,214 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.8.0] - 2026-09-29
+
+Releases `activeagent` and `actionagent` 1.8.0 from one tag. A minor release.
+Settings -> Integrations connects GitHub and Claude Code. A connected
+repository's checkout boots as a sandbox, on a developer's machine with the new
+`:local` backend, where Claude Code sessions run and an evaluation can run
+against the checkout without the agent being edited. The dashboard's MCP
+server gains evaluation and telemetry tools for a developer's own coding
+harness. Ollama hosts can be tested and can be remote, with an optional Bearer
+API key. Comparison runs lead with a per-model table and filter the fix list
+by model. A publisher can check its collector before a run.
+
+Upgrading: run `bin/rails generate action_agent:install --skip` and
+`bin/rails db:migrate`. The generator adds what an install lacks:
+`provider_keys.api_key` and the `github_connections` and `code_sessions`
+tables. An app on the RubyLLM provider needs ruby_llm 1.16 or later; 2.x
+works too.
+
+The Claude Code connection stores Anthropic API keys only. Anthropic does not
+let third-party products collect, store or route requests through Claude.ai
+subscription credentials
+([Claude Code legal and compliance](https://code.claude.com/docs/en/legal-and-compliance.md)),
+so a `claude setup-token` token (`sk-ant-oat…`) is refused. An install that
+stored one while running a pre-release build never hands it to a session: its
+owner sees Claude Code as needing an API key (`needs_replacing: true` in
+`GET /api/provider_keys`) until they paste one.
+`bin/rails action_agent:claude_code:purge_subscription_tokens` deletes the
+stored tokens and prints how many it removed. A developer who wants sessions on
+their own Claude login sets `config.claude_code_auth = :local_login` with the
+`:local` backend and runs `claude /login` on that machine instead.
+
+### Added
+
+- **Evaluation and telemetry tools on the MCP facade** (`actionagent`). The
+  dashboard's MCP server now offers `evaluations_list`, `evaluations_get`,
+  `evaluations_run`, `evaluation_runs_get`, `evaluation_runs_compare`,
+  `traces_search` and `traces_get`, so a developer's own coding harness can
+  run an agent's evaluations, read the fix items and failing traces, and
+  iterate on the agent in its own checkout without the dashboard holding a
+  model login. The tools read under the API key's owner exactly as the JSON
+  API reads under the signed-in owner, run through the same execution,
+  quota and sandbox checks as `POST /api/evaluations/:id/run`, bound their
+  output, and mask the owner's credentials. Their names cannot collide with
+  schema tools or agent tools. `ActionAgent.mcp_dashboard_tools = false`
+  turns them off.
+
+- **Local checkout sandboxes and Claude Code sessions** (`actionagent`, #489).
+  A new `:local` sandbox backend (`config.sandbox_service = :local`) makes
+  **Start sandbox** work on a developer's machine without containers. It
+  clones the repository under `tmp/action_agent/sandboxes`, runs the setup
+  the checkout's optional `.activeagents/sandbox.yml` names (`env`, `setup`,
+  `manifest`, `start`), and boots the app on `127.0.0.1`. It then registers
+  the app's MCP facade as the `sandbox:<session_id>` server. Every process
+  starts from a sanitized copy of the dashboard's environment: without its
+  database and Redis URLs, Rails keys and environment, Bundler and Ruby
+  settings, git repository and config variables, `SSH_AUTH_SOCK`,
+  model-provider and Claude Code settings, variables named like a secret, or
+  URLs carrying credentials. The GitHub token reaches
+  only the fetch, and the Claude Code API key reaches only Claude Code.
+  `:local` runs the owner's code with the dashboard's privileges, so it is off
+  outside development and test unless
+  `ActionAgent.local_sandboxes_enabled = true`.
+  - `bin/rails action_agent:sandbox:manifest` writes the booted app's
+    `{mcp_path, mcp_token}` for the backend.
+  - `bin/rails action_agent:sandbox:reap` expires overdue sandboxes and stops
+    them.
+  - A ready sandbox runs headless Claude Code sessions (`--permission-mode
+    acceptEdits` by default). You start them from Settings -> Integrations, or
+    through `/api/sandboxes/:session_id/code_sessions`. Their events stream
+    into the dashboard, and the checkout's diff follows.
+  - New options: `local_sandboxes_enabled`, `local_sandbox_root`,
+    `local_sandbox_boot_timeout`, `claude_code_command`,
+    `claude_code_permission_mode`, `claude_code_max_turns`,
+    `claude_code_timeout` and `claude_code_auth`.
+  - `claude_code_auth = :local_login` runs sessions on the machine's own
+    Claude Code login (`claude /login`), with no stored key: the backend
+    passes no credential and no `CLAUDE_CONFIG_DIR`, so `claude` uses the
+    dashboard user's own `~/.claude` or keychain, which the dashboard never
+    reads. `LocalSandboxBackend.claude_login_status` asks
+    `claude auth status --json` (cached for a minute) and keeps only
+    `loggedIn` and the login method. `GET /api/sandboxes` reports
+    `claude_code_auth` and, in this mode, `claude_code_login`
+    (`{ logged_in, auth_method }`), and the assistant's
+    `connections.claude_code` the same as `auth` and `login`. Other backends
+    refuse sessions in this mode (`code_sessions_supported: false`, and a
+    `422` naming the reason).
+  - `app_runtime` sandboxes now provision in the background and last 2 hours.
+  - Run `rails g action_agent:install` to add the
+    `create_active_agent_code_sessions` migration.
+  - This repository's own `.activeagents/sandbox.yml` boots `test/dummy`.
+  - Every `:local` sandbox boots on databases of its own, so a checkout of
+    the dashboard's own app no longer migrates the developer's development
+    database. The backend reads the adapter from the checkout's
+    `config/database.yml` without running its ERB, and sets `DATABASE_URL`
+    and `<NAME>_DATABASE_URL` (`QUEUE_DATABASE_URL`, `CACHE_DATABASE_URL`):
+    SQLite files in the workspace, or `<database>_sandbox_<id>` on
+    PostgreSQL and MySQL, which terminate drops with the checkout's
+    Rails database tasks restricted to the names and URLs recorded at boot.
+    Setting a variable in `sandbox.yml`'s `env` overrides it and excludes
+    that database from cleanup. Replica mappings follow their own writer;
+    ambiguous mappings require an explicit URL instead of guessing.
+  - The Claude Code panel has a **Model** select: Claude Code's own
+    default, the `sonnet`, `opus` and `haiku` aliases, or any model id under
+    *Other…*. It remembers the last choice per browser, and each session
+    shows the model it ran on.
+  - A run can use a checkout sandbox without the agent being edited:
+    `sandbox_id` on `POST /api/evaluations/:id/run` (and on the runner's
+    `/api/agents/:id/execute` and `/test`) gives that run's tool dispatcher
+    the sandbox's `sandbox:<session_id>` runtime, as if the agent listed it.
+    The sandbox must be the caller's, a ready `app_runtime` sandbox, and the
+    agent owner's; anything else is a `422`. The run records which sandbox
+    it used (`run.sandbox`), and a scenario suite's **Run against sandbox**
+    select, its Runs list and the run report show it. The selected sandbox
+    takes precedence for matching tool names, with one schema per name.
+    Queued agent runs fail if their selected sandbox stops, and failed
+    sandbox discovery never silently falls back to the original tools.
+- **Claude Code connection** (`actionagent`, #478). Settings -> Integrations
+  stores an Anthropic API key (`sk-ant-api…`, from the Claude Console) as the
+  `claude_code` provider key. It is encrypted, write-only, and not an agent
+  provider. `SandboxSession#runtime_environment` hands it to an
+  `app_runtime` backend as `ANTHROPIC_API_KEY`, so the checkout can run
+  Claude Code sessions. Claude subscription tokens (`claude setup-token`)
+  are refused, as Anthropic's terms require (see the upgrading note above).
+  `/api/provider_keys` rows now carry `kind` (`key`, `host` or
+  `connection`) and `needs_replacing`.
+- **GitHub connections and checkout sandboxes** (`actionagent`, #477).
+  Settings -> Integrations connects GitHub over OAuth
+  (`ActionAgent.github_client_id` / `github_client_secret`, or
+  `GITHUB_CLIENT_ID` / `GITHUB_CLIENT_SECRET`), stores the token encrypted,
+  and lets the owner choose which repositories the workspace may use. Only
+  repositories GitHub lists for the token can be selected. A new
+  `app_runtime` sandbox type checks out one of them: the backend receives
+  `sandbox_session.checkout_spec` (repository, ref, clone URL, token),
+  boots the app, and returns `mcp_url` / `mcp_token` from `create_sandbox`.
+  The session is then an MCP server keyed `sandbox:<session_id>`. An agent
+  that lists that key in `mcp_servers` runs, and is evaluated, with the
+  checkout's own tools. Lookups are scoped to the agent's owner. Run
+  `rails g action_agent:install` to add the
+  `create_active_agent_github_connections` migration.
+
+- **Ollama hosts are testable and can be remote** (`actionagent`). Settings ->
+  Provider API Keys gains a **Test connection** for Ollama that reports
+  whether the server is reachable, the round-trip time and the models it
+  serves, before or after saving (`POST <mount>/api/provider_keys/test`,
+  read-only). The host is accepted as a bare server address
+  (`http://localhost:11434`; the OpenAI-compatible `/v1` path is added) and
+  an optional **API key** is stored beside it and sent as a Bearer token,
+  for a server behind an authenticating proxy or Ollama Cloud. The agent
+  builder's live Ollama model list uses the same probe and key. When no host
+  is configured the card shows the host app's `config/active_agent.yml`
+  default. The install generator emits a guarded `add_provider_key_api_key`
+  migration for existing installs; re-run
+  `bin/rails generate action_agent:install --skip` and `bin/rails db:migrate`.
+- **A model comparison table on comparison runs** (`actionagent`,
+  `activeagent`). A run over several models now leads its Models section
+  with one row per model, best first: passed, mean score, average latency,
+  average tokens per scenario, cost (and per scenario), and the model's
+  typical fault — its most frequent one with the diagnosis of a result that
+  carries it. The scenario suite panel, the sampling run detail and the
+  standalone HTML report (`ActiveAgent::Evals::ReportHtml`) all render it.
+- **What to fix, filtered by model** (`actionagent`, `activeagent`). On a
+  comparison run the fix list takes a model chip, narrowing to the items
+  attributed to that model and counting what that model alone produced,
+  since one model may need more instruction than another. The standalone
+  report filters through radio chips and stylesheet rules — it still ships
+  no script.
+
+- **A publisher can check its collector before a run** (`activeagent`).
+  `ActiveAgent::Evals::Publisher#verify!` asks the collector whether it is up
+  and accepts the key before a run is paid for. It posts an empty JSON object,
+  which a compatible collector refuses with a 422 naming `version`, without
+  storing anything; anything else raises `Publisher::Error` with a delivery's
+  status, detail and guidance. `Publisher#endpoint` returns the collector URL.
+
+### Changed
+
+- **Collector rejections say what the status means** (`activeagent`). A
+  `Publisher::Error` for a 401, 403, 404, 415 or 501 rejection names a refused
+  key, an account an operator must act on, an endpoint that is not a
+  collector, a rewritten `Content-Type`, or an install with no evaluation
+  store, in place of the generic guidance.
+
+### Fixed
+
+- **A nested scenario expectation written as one value** (`activeagent`).
+  `ScenarioParser` now stores `{ expectations: { contains: "30" } }` as a
+  list of one, the shape the persisted scenario and the dashboard's matrix
+  read; an object-list import with a lone value used to break the suite
+  panel. The matrix also tolerates scenarios persisted before this.
+
+## [1.7.2] - 2026-09-29
+
+Releases `activeagent` and `actionagent` 1.7.2 from one tag. A patch on 1.7.1:
+the RubyLLM provider runs on ruby_llm 1.16 and 2.x, where 1.7.1 required 1.x.
+Apps on the RubyLLM provider need ruby_llm 1.16 or later; earlier 1.x
+releases lack the APIs the provider calls. No migrations.
+
+### Changed
+
+- **The RubyLLM provider supports ruby_llm 1.16 and 2.x** (`activeagent`).
+  The adapter handles 2.x's tool interface, token limits, embedding model
+  objects, usage and finish reasons while keeping the 1.16 API working.
+  Requiring `>= 1.16, < 3` prevents Bundler from selecting an older 1.x
+  release without the APIs the adapter calls. Rails main can now resolve
+  RubyLLM 2.x alongside Active Storage's Marcel 2 dependency. CI also runs
+  the full suite with RubyLLM 1.16 to retain coverage of that version.
+  Unsupported-version errors name the loaded version and both bounds (#508).
+
 ## [1.7.1] - 2026-09-29
 
 Releases `activeagent` and `actionagent` 1.7.1 from one tag. A patch on 1.7.0:

data/lib/active_agent/evals/publisher.rb

@@ -11,7 +11,8 @@ module ActiveAgent
     # Publishes a completed report without replaying the agent. The caller must
     # retain run_id when retrying: compatible collectors treat that identity as
     # immutable within the authenticated account. Delivery is blocking and does
-    # not follow redirects with the account's bearer credential.
+    # not follow redirects with the account's bearer credential. +verify!+ asks
+    # the collector whether it is up and accepts the key before a run is paid for.
     #
     # Every failure to deliver raises Error. Invalid arguments raise
     # ArgumentError before anything is sent.
@@ -20,6 +21,9 @@ module ActiveAgent
       MAX_BYTES = 2 * 1024 * 1024
       DETAIL_LIMIT = 200
 
+      # @return [String] the collector URL reports go to
+      attr_reader :endpoint
+
       # Raised for every failed delivery. Only a network failure keeps the
       # underlying error as its +cause+, so neither the response nor the
       # report reaches a log through the exception chain.
@@ -48,10 +52,15 @@ module ActiveAgent
       # Whether each rejection status is retryable, and what the caller should
       # do about it. Other statuses fall back to the rules in +rejection+.
       REJECTIONS = {
+        401 => [ false, "the collector refused the API key; check the key against the collector's account" ],
+        403 => [ false, "the account may not store this report until an operator acts, for example on a cap on observed agents, evaluations or scenarios; resolve that before retrying" ],
+        404 => [ false, "nothing at the endpoint takes evaluation reports; check that it is a collector's /v1/evaluations or <mount>/api/evaluation_reports URL" ],
         409 => [ false, "the collector already holds a different report under this run_id; never retry this report with the same run_id" ],
         413 => [ false, "the report exceeds the collector's size limit; publish a smaller selection" ],
+        415 => [ false, "the collector did not receive application/json; check anything between the publisher and the collector that rewrites the Content-Type" ],
         422 => [ false, "correct what the collector refused before retrying" ],
-        429 => [ true, "the account is over its quota or rate limit; retain the report and run_id and retry later" ]
+        429 => [ true, "the account is over its quota or rate limit; retain the report and run_id and retry later" ],
+        501 => [ false, "the collector has no evaluation store; migrate the install, or publish to one generated with evaluation tables" ]
       }.freeze
 
       # The key is sent as a bearer token and filtered from the collector's
@@ -65,6 +74,7 @@ module ActiveAgent
         unless @uri.scheme == "https" || %w[localhost 127.0.0.1 ::1].include?(@uri.hostname)
           raise ArgumentError, "Evaluation endpoint requires HTTPS except on loopback hosts"
         end
+        @endpoint = @uri.to_s
 
         @api_key = api_key.to_s.strip
         raise ArgumentError, "Evaluation API key is required" if @api_key.empty?
@@ -91,6 +101,52 @@ module ActiveAgent
         body = encode(identities.merge("version" => 1, "report" => report_hash(report)))
         raise Error, "Evaluation report exceeds the 2 MiB delivery limit; publish a smaller selection" if body.bytesize > MAX_BYTES
 
+        deliver("retain the report and run_id for retry") do
+          response = post(body)
+          raise rejection(response) unless %w[200 201].include?(response.code)
+
+          receipt = JSON.parse(response.body.to_s)
+          unless receipt.is_a?(Hash) && receipt["run_id"] == run_id && receipt["status"] == "complete" && receipt["id"] && receipt["evaluation_id"]
+            raise Error.new("Evaluation collector returned an invalid completion receipt; retain the report and run_id for retry", retryable: true)
+          end
+          receipt
+        end
+      end
+
+      # Returns true when the collector is up and accepts the API key, without
+      # storing anything. It posts an empty JSON object, which a compatible
+      # collector authenticates, parses, and then refuses with a 422 whose
+      # +error+ names +version+, as not a version-1 report. That refusal, and
+      # only that one, is the ready answer. Call it before an expensive run, so
+      # a stopped collector or a refused key fails before the first model call
+      # rather than after the last one.
+      #
+      # Raises Error for anything else: a rejection other than 422, with the
+      # status, detail and guidance +call+ would carry (401 for a refused key,
+      # 404 when nothing at the endpoint takes reports); a retryable delivery
+      # failure when the collector cannot be reached; a 422 that says anything
+      # else; or a collector that stores the empty object. The last two mean the
+      # endpoint is not a compatible collector.
+      def verify!
+        deliver("start the collector or check the endpoint, then verify again") do
+          response = post("{}")
+          if response.code == "422"
+            detail = collector_detail(response.body)
+            next true if detail&.match?(/\bversion\b/)
+
+            raise Error.new("Evaluation collector answered HTTP 422 without refusing the empty envelope as a version-1 report, " \
+                            "so it is not a compatible collector; check the endpoint", status: 422, detail: detail)
+          end
+          raise rejection(response) unless %w[200 201].include?(response.code)
+
+          raise Error, "Evaluation collector stored an empty report, so it is not a compatible collector; check the endpoint"
+        end
+      end
+
+      private
+
+      # Posts +body+ to the endpoint with the bearer key and the delivery timeouts.
+      def post(body)
         http = Net::HTTP.new(@uri.hostname, @uri.port)
         http.use_ssl = @uri.scheme == "https"
         http.open_timeout = @open_timeout
@@ -101,26 +157,25 @@ module ActiveAgent
         request["Content-Type"] = "application/json"
         request["Accept"] = "application/json"
         request.body = body
-        response = http.request(request)
-        raise rejection(response) unless %w[200 201].include?(response.code)
+        http.request(request)
+      end
 
-        receipt = JSON.parse(response.body.to_s)
-        unless receipt.is_a?(Hash) && receipt["run_id"] == run_id && receipt["status"] == "complete" && receipt["id"] && receipt["evaluation_id"]
-          raise Error.new("Evaluation collector returned an invalid completion receipt; retain the report and run_id for retry", retryable: true)
-        end
-        receipt
+      # Runs one exchange with the collector, turning every failure to reach it
+      # or to read its answer into a retryable Error that quotes nothing from
+      # the response and ends with +guidance+, what the caller should do next.
+      # An Error the block raises passes through unchanged.
+      def deliver(guidance)
+        yield
       rescue JSON::ParserError
         # The parser's message quotes the body.
-        raise Error.new("Evaluation collector returned invalid JSON; retain the report and run_id for retry", retryable: true), cause: nil
+        raise Error.new("Evaluation collector returned invalid JSON; #{guidance}", retryable: true), cause: nil
       rescue Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Zlib::Error => e
         # These messages can quote the response's status line, headers or body.
-        raise Error.new("Evaluation collector returned a malformed response (#{e.class}); retain the report and run_id for retry", retryable: true), cause: nil
+        raise Error.new("Evaluation collector returned a malformed response (#{e.class}); #{guidance}", retryable: true), cause: nil
       rescue IOError, SocketError, SystemCallError, Timeout::Error, OpenSSL::SSL::SSLError => e
-        raise Error.new("Evaluation delivery failed (#{e.class}); retain the report and run_id for retry", retryable: true)
+        raise Error.new("Evaluation delivery failed (#{e.class}); #{guidance}", retryable: true)
       end
 
-      private
-
       def report_hash(report)
         hash = report.to_h if report.respond_to?(:to_h) && !report.nil? && !report.is_a?(Array)
         raise ArgumentError, "report must be a Report or its saved JSON hash" unless hash.is_a?(Hash)

data/lib/active_agent/evals/report_html.rb

@@ -178,6 +178,9 @@ module ActiveAgent
           DesignTokens.css(scope: ":root.theme-dark", tokens: DesignTokens::DARK, color_scheme: "dark"),
           STYLES,
           ".mx { grid-template-columns: minmax(240px, 1.6fr) 150px repeat(#{@models.size}, minmax(170px, 1fr)); }",
+          # One rule per model: with that chip checked, hide every fix card
+          # attributed to other models (cards attributed to none stay).
+          *@models.each_index.map { |i| ".fix-section:has(input[value=\"m#{i}\"]:checked) .fix[data-models]:not([data-models~=\"m#{i}\"]) { display: none; }" },
           ".matrix .inner { min-width: #{390 + 185 * @models.size}px; }"
         ].join("\n")
       end
@@ -233,12 +236,78 @@ module ActiveAgent
         <<~PANEL
           <div class="panel">
           <div class="panel-head"><span class="micro">Models</span><span class="right">judged by #{h(judged_by)}</span></div>
+          #{html_comparison_table if comparing?}
           #{blocks.join}
           #{verdict_row}
           </div>
         PANEL
       end
 
+      # The comparison read across: one row per model, best first (pass rate,
+      # then mean score) — passed, mean score, average latency, average
+      # tokens per scenario, cost, and the model's typical fault. The blocks
+      # under it carry the same figures per model with bars and every fault.
+      def html_comparison_table
+        rows = summary_by_model.sort_by do |label, stats|
+          total = stats["scenarios"].to_i
+          [ total.positive? ? -stats["passed"].to_f / total : 0.0, -(stats["avg_score"] || -1).to_f, @models.index(model_by_label(label)).to_i ]
+        end
+
+        <<~TABLE
+          <div class="compare"><table>
+          <thead><tr><th>Model</th><th class="num">Passed</th><th class="num">Mean score</th><th class="num">Avg latency</th><th class="num" title="Average input + output tokens per scenario">Avg tokens</th><th class="num" title="Cohort spend, and per scenario">Cost</th><th class="fault">Typical fault</th></tr></thead>
+          <tbody>#{rows.map { |label, stats| html_comparison_row(label, stats) }.join}</tbody>
+          </table></div>
+        TABLE
+      end
+
+      def html_comparison_row(label, stats)
+        short, provider = split_label(model_by_label(label))
+        total = stats["scenarios"].to_i
+        ratio = total.positive? ? stats["passed"].to_f / total : 0.0
+        pick = comparing? && winner == label ? %(<span class="pick" title="picked by the judge">★ pick</span>) : ""
+        per = ->(value) { value.nil? || total.zero? ? nil : value.to_f / total }
+        avg_tokens = per.call(stats["input_tokens"].to_i + stats["output_tokens"].to_i)
+        tokens_cell = avg_tokens ? h(fmt_k(avg_tokens.round)) : "—"
+        tokens_title = avg_tokens ? %( title="#{per.call(stats['input_tokens']).to_f.round} in · #{per.call(stats['output_tokens']).to_f.round} out per scenario") : ""
+        per_cost = per.call(stats["cost"])
+        cost_cell = stats["cost"].nil? ? "—" : h(fmt_cost(stats["cost"]))
+        cost_cell += "<span class=\"per\">#{h(fmt_cost(per_cost))}/scenario</span>" if per_cost
+
+        <<~ROW
+          <tr>
+          <td class="model-cell"><span class="name">#{h(short)}</span>#{pick}<span class="provider">#{h(provider)}</span></td>
+          <td class="num ratio tone-#{tone_for(ratio)}">#{total.positive? ? "#{stats['passed']}/#{total}" : '—'}</td>
+          <td class="num">#{h(fmt_mean_score(stats['avg_score']))}</td>
+          <td class="num">#{h(fmt_ms(stats['avg_duration_ms']))}</td>
+          <td class="num"#{tokens_title}>#{tokens_cell}</td>
+          <td class="num">#{cost_cell}</td>
+          <td class="fault">#{typical_fault_text(label, stats)}</td>
+          </tr>
+        ROW
+      end
+
+      # "missing content ×2 · refund_window: The answer is missing expected
+      # content: 30." — the model's most frequent fault, and the diagnosis of
+      # the first result that carries it; "no faults" for a clean cohort.
+      def typical_fault_text(label, stats)
+        tally = stats["faults"] || {}
+        return %(<span class="clean">no faults</span>) if tally.empty?
+
+        mine = @results.select { |result| result.label == label }
+        example_of = ->(fault) { mine.find { |result| result.fault == fault } }
+        # Most frequent first; between equals, a fault a result can explain,
+        # then a specific fault over the judge's catch-all, then the name.
+        fault, count = tally.min_by { |name, n| [ -n, example_of.call(name) ? 0 : 1, name == "low_quality" ? 1 : 0, name ] }
+        example = example_of.call(fault)
+        head = "#{fault_name(fault)} ×#{count}"
+        return h(head) unless example&.summary.present?
+
+        detail = "#{example.scenario.key}: #{example.summary}"
+        detail = "#{detail[0, 119]}…" if detail.length > 120
+        "#{h(head)} <span class=\"detail\">· #{h(detail)}</span>"
+      end
+
       def html_model_block(label, stats)
         short, provider = split_label(model_by_label(label))
         total = stats["scenarios"]
@@ -277,13 +346,36 @@ module ActiveAgent
           end
 
         <<~FIXES
-          <section class="section" aria-label="Recommendations">
+          <section class="section fix-section" aria-label="Recommendations">
           <div class="section-head"><span class="micro">What to fix</span><span class="meta">#{h(meta)}</span></div>
+          #{html_fix_filter(items) if comparing? && items.any?}
           #{body}
           </section>
         FIXES
       end
 
+      # A model filter for the fix cards — a fault one model keeps making is
+      # that model's to fix, so the list narrows to what was attributed to
+      # it. Radio chips and stylesheet rules alone (the page carries no
+      # script): each card names its models in data-models, and a checked
+      # model hides every card that does not name it. Cards attributed to no
+      # model (an older run) stay under every filter.
+      def html_fix_filter(items)
+        chips = [ %(<label class="chip pick-model"><input type="radio" name="fix-model" value="all" checked><span>all models #{items.size}</span></label>) ]
+        @models.each_with_index do |spec, index|
+          count = items.count { |item| Array(item["models"]).empty? || item["models"].include?(spec.label) }
+          chips << %(<label class="chip pick-model"><input type="radio" name="fix-model" value="m#{index}"><span>#{h(short_name(spec))} #{count}</span></label>)
+        end
+        %(<div class="fix-filter"><span class="micro sm">for</span>#{chips.join}</div>)
+      end
+
+      def fix_model_tokens(item)
+        labels = Array(item["models"])
+        return "" if labels.empty?
+
+        labels.filter_map { |label| (index = @models.index(model_by_label(label))) && "m#{index}" }.join(" ")
+      end
+
       def html_fix_card(item)
         tone = item["kind"] == "instruction" ? "info" : "error"
         glyph = tone == "info" ? "[i]" : "[!]"
@@ -297,7 +389,8 @@ module ActiveAgent
         parts << html_fix_server(item["server"]) if item["server"]
         parts << %(<div class="note">#{h(item['note'])}</div>) if item["note"].present?
         parts << html_fix_action(item["action"]) if item["action"]
-        %(<div class="fix">#{parts.join}</div>)
+        models = fix_model_tokens(item)
+        %(<div class="fix"#{%( data-models="#{models}") if models.present?}>#{parts.join}</div>)
       end
 
       def html_fix_tools(item)
@@ -557,6 +650,24 @@ module ActiveAgent
         .tok .in { color: var(--color-token-in); }
         .tok .out { color: var(--color-token-out); }
         .faults { display: flex; gap: 6px; flex-wrap: wrap; }
+        .compare { overflow-x: auto; border-top: 1px solid var(--color-border-light); }
+        .compare table { width: 100%; border-collapse: collapse; }
+        .compare th { padding: 8px 12px; text-align: left; vertical-align: bottom; white-space: nowrap; font-family: var(--font-mono); font-size: 10px; font-weight: 600; letter-spacing: 0.05em; text-transform: uppercase; color: var(--color-text-muted); background: var(--color-muted); }
+        .compare td { padding: 9px 12px; vertical-align: top; border-top: 1px solid var(--color-border-light); font-size: 13px; color: var(--color-text-cell); }
+        .compare th.num, .compare td.num { text-align: right; }
+        .compare td.num { font-family: var(--font-mono); font-size: 12px; white-space: nowrap; }
+        .compare td.ratio { font-weight: 600; }
+        .compare .per { display: block; font-weight: 400; color: var(--color-text-muted); }
+        .compare .model-cell { white-space: nowrap; }
+        .compare .model-cell .name { font-family: var(--font-mono); font-size: 12px; font-weight: 600; color: var(--color-text-primary); }
+        .compare .model-cell .provider { display: block; font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
+        .compare .pick { margin-left: 6px; font-family: var(--font-mono); font-size: 10px; font-weight: 700; color: var(--color-warning-text); }
+        .compare th.fault, .compare td.fault { width: 34%; }
+        .compare td.fault .detail { color: var(--color-text-secondary); }
+        .fix-filter { display: flex; align-items: center; gap: 6px; flex-wrap: wrap; }
+        .pick-model { cursor: pointer; border: 1px solid var(--color-border); background: var(--color-card); }
+        .pick-model input { position: absolute; opacity: 0; width: 0; height: 0; }
+        .pick-model:has(input:checked) { border-color: var(--color-accent-ui); background: var(--color-accent-ui-tint); color: var(--color-accent-ui); }
         .clean { font-family: var(--font-mono); font-size: 11px; color: var(--color-success-text); }
         .verdict { padding: 10px 12px; border-top: 1px solid var(--color-border-light); font-size: 12px; line-height: 18px; color: var(--color-text-cell); }
         .verdict .micro { margin-right: 8px; }

data/lib/active_agent/evals/scenario_parser.rb

@@ -125,6 +125,10 @@ module ActiveAgent
         expectations = (entry["expectations"] || entry["expect"] || {}).to_h.stringify_keys
         %w[tools contains not_contains].each do |field|
           expectations[field] = Array(entry[field]) if entry.key?(field)
+          # A nested expectation written as one value ({ contains: "30" })
+          # is a list of one: the persisted scenario and the dashboard's
+          # matrix read each field as an array.
+          expectations[field] = Array(expectations[field]) if expectations.key?(field)
         end
 
         scenario(

data/lib/active_agent/providers/_base_provider.rb

@@ -10,8 +10,8 @@ require_relative "concerns/tool_choice_clearing"
 GEM_LOADERS = {
   anthropic: [ "anthropic", "~> 1.12", "anthropic" ],
   openai:    [ "openai",    "~> 0.34", "openai" ],
-  # ruby_llm 2.0 renamed the APIs RubyLLMProvider calls.
-  ruby_llm:  [ "ruby_llm",  "~> 1.0",  "ruby_llm" ]
+  # Keep a tested floor: 1.2 lacks the provider APIs this adapter uses.
+  ruby_llm:  [ "ruby_llm",  [ ">= 1.16", "< 3" ], "ruby_llm" ]
 }
 
 # Requires a provider's gem dependency.
@@ -23,16 +23,17 @@ GEM_LOADERS = {
 #   version is outside the supported range
 def require_gem!(type, file_name)
   gem_name, requirement, package_name = GEM_LOADERS.fetch(type)
+  requirements = Array(requirement)
   provider_name = file_name.split("/").last.delete_suffix(".rb").camelize
 
   begin
-    gem(gem_name, requirement)
+    gem(gem_name, *requirements)
     require(package_name)
   rescue LoadError
     loaded = Gem.loaded_specs[gem_name]
     if loaded && !Gem::Requirement.new(requirement).satisfied_by?(loaded.version)
-      raise LoadError, "#{provider_name} supports the '#{gem_name}' gem #{requirement}, but #{loaded.version} is loaded. " \
-                       "Add `gem \"#{gem_name}\", \"#{requirement}\"` to your Gemfile and run `bundle update #{gem_name}`."
+      raise LoadError, "#{provider_name} supports the '#{gem_name}' gem #{requirements.join(', ')}, but #{loaded.version} is loaded. " \
+                       "Add `gem \"#{gem_name}\", #{requirements.map(&:inspect).join(', ')}` to your Gemfile and run `bundle update #{gem_name}`."
     end
 
     raise LoadError, "The '#{gem_name}' gem is required for #{provider_name}. Please add it to your Gemfile and run `bundle install`."

data/lib/active_agent/providers/ruby_llm/tool_proxy.rb

@@ -27,6 +27,11 @@ module ActiveAgent
           {}
         end
 
+        # RubyLLM 2 names for the same tool interface.
+        alias_method :parameters_schema, :params_schema
+        alias_method :declared_parameters, :parameters
+        alias_method :provider_options, :provider_params
+
         private
 
         def deep_stringify(obj)

data/lib/active_agent/providers/ruby_llm_provider.rb

@@ -62,14 +62,19 @@ module ActiveAgent
         schema = ruby_llm_schema(parameters[:response_format])
         kwargs[:schema] = schema if schema
 
-        # Pass extra params (max_tokens, etc.) via RubyLLM's params: deep-merge
+        # RubyLLM 2 renamed params: and exposes a provider-neutral token limit.
         max_tokens = parameters[:max_tokens] || options.max_tokens
         if max_tokens
-          kwargs[:params] = { max_tokens: max_tokens }
+          if ruby_llm_v2?
+            kwargs[:max_output_tokens] = max_tokens
+          else
+            kwargs[:params] = { max_tokens: max_tokens }
+          end
         end
 
         if parameters[:stream]
           stream_proc = parameters[:stream]
+          @stream_tool_calls = {}
 
           # For streaming, pass a block that forwards chunks
           @ruby_llm_provider.complete(messages, **kwargs) do |chunk|
@@ -95,7 +100,8 @@ module ActiveAgent
         inputs = input.is_a?(Array) ? input : [ input ]
 
         data = inputs.map.with_index do |text, index|
-          embedding = @ruby_llm_provider.embed(text, model: model_id, dimensions: parameters[:dimensions])
+          embedding_model = ruby_llm_v2? ? @ruby_llm_model : model_id
+          embedding = @ruby_llm_provider.embed(text, model: embedding_model, dimensions: parameters[:dimensions])
 
           {
             object: "embedding",
@@ -138,12 +144,15 @@ module ActiveAgent
         # Handle tool calls in chunk
         if chunk.tool_calls&.any?
           message[:tool_calls] ||= []
-          chunk.tool_calls.each do |_id, tool_call|
-            existing = message[:tool_calls].find { |tc| tc[:id] == tool_call.id }
+          chunk.tool_calls.each do |key, tool_call|
+            # RubyLLM 2 keys OpenAI deltas by index; only the first delta
+            # includes the call ID and name. Keep each index tied to its call.
+            existing = message[:tool_calls].find { |tc| tool_call.id && tc[:id] == tool_call.id }
+            existing ||= @stream_tool_calls[key]
             if existing
               existing[:function][:arguments] += tool_call.arguments.to_s if tool_call.arguments
             else
-              message[:tool_calls] << {
+              existing = {
                 id: tool_call.id,
                 type: "function",
                 function: {
@@ -151,7 +160,9 @@ module ActiveAgent
                   arguments: tool_call.arguments.to_s
                 }
               }
+              message[:tool_calls] << existing
             end
+            @stream_tool_calls[key] = existing
           end
         end
 
@@ -254,6 +265,10 @@ module ActiveAgent
 
       private
 
+      def ruby_llm_v2?
+        Gem.loaded_specs.fetch("ruby_llm").version.segments.first >= 2
+      end
+
       # Resolves and caches the RubyLLM provider for the given model.
       #
       # Reuses the cached provider if the model hasn't changed (e.g., during
@@ -447,6 +462,8 @@ module ActiveAgent
         # Add stop_reason if available
         if response.respond_to?(:stop_reason) && response.stop_reason
           hash[:stop_reason] = response.stop_reason
+        elsif response.respond_to?(:finish_reason) && response.finish_reason
+          hash[:stop_reason] = { stop: "end_turn", tool_calls: "tool_use" }.fetch(response.finish_reason, response.finish_reason.to_s)
         elsif response.tool_calls&.any?
           hash[:stop_reason] = "tool_use"
         else
@@ -454,7 +471,10 @@ module ActiveAgent
         end
 
         # Add usage info if available
-        if response.respond_to?(:input_tokens) && response.input_tokens
+        if response.respond_to?(:tokens)
+          tokens = response.tokens
+          hash[:usage] = { input_tokens: tokens.input, output_tokens: tokens.output } if tokens.input
+        elsif response.respond_to?(:input_tokens) && response.input_tokens
           hash[:usage] = {
             input_tokens: response.input_tokens,
             output_tokens: response.output_tokens

data/lib/active_agent/version.rb

@@ -1,3 +1,3 @@
 module ActiveAgent
-  VERSION = "1.7.1"
+  VERSION = "1.8.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: activeagent
 version: !ruby/object:Gem::Version
-  version: 1.7.1
+  version: 1.8.0
 platform: ruby
 authors:
 - Justin Bowen
@@ -204,16 +204,22 @@ dependencies:
   name: ruby_llm
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.16'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.16'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: capybara
   requirement: !ruby/object:Gem::Requirement