activeagent 1.9.2 → 1.9.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +252 -0
  3. data/README.md +3 -0
  4. data/lib/active_agent/base.rb +2 -2
  5. data/lib/active_agent/concerns/preview.rb +2 -2
  6. data/lib/active_agent/evals/design_tokens.rb +12 -1
  7. data/lib/active_agent/evals/publisher.rb +19 -12
  8. data/lib/active_agent/evals/report.rb +12 -62
  9. data/lib/active_agent/evals/report_html.rb +85 -10
  10. data/lib/active_agent/evals/report_markdown.rb +202 -0
  11. data/lib/active_agent/evals.rb +1 -0
  12. data/lib/active_agent/inline_preview_interceptor.rb +1 -0
  13. data/lib/active_agent/preview/templates/layouts/active_agent/preview.html.erb +17 -0
  14. data/lib/active_agent/preview/templates/rails/agents/index.html.erb +14 -0
  15. data/lib/active_agent/preview/templates/rails/agents/preview.html.erb +3 -0
  16. data/lib/active_agent/providers/_base_provider.rb +2 -0
  17. data/lib/active_agent/providers/anthropic_provider.rb +1 -0
  18. data/lib/active_agent/providers/common/responses/base.rb +15 -2
  19. data/lib/active_agent/providers/common/usage.rb +4 -2
  20. data/lib/active_agent/providers/open_ai/chat_provider.rb +39 -0
  21. data/lib/active_agent/providers/open_ai/responses_provider.rb +3 -0
  22. data/lib/active_agent/providers/opper/_types.rb +16 -0
  23. data/lib/active_agent/providers/opper/options.rb +38 -0
  24. data/lib/active_agent/providers/opper_provider.rb +72 -0
  25. data/lib/active_agent/railtie.rb +11 -5
  26. data/lib/active_agent/version.rb +1 -1
  27. data/lib/generators/test_unit/agent_generator.rb +1 -1
  28. data/lib/generators/test_unit/templates/functional_test.rb.tt +1 -1
  29. data/lib/generators/test_unit/templates/preview.rb.tt +3 -3
  30. data/lib/rails/agents_controller.rb +43 -0
  31. metadata +10 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 2b52709687c94c3d6c989fc36bba7826b931a6d0aa6e193b1bd4482f916d945f
4
- data.tar.gz: 1801338ff01afc21837db5382aee49eb4cde1f002b322b479cf2256c283e8668
3
+ metadata.gz: 715323643179a060b32d624ba906586778645e818c0436d4e4efe9ea6585847e
4
+ data.tar.gz: 3efe81e8a8d5357baf0f46adcfa5ce2fedd96234bb8ed603ffae63bac17bec58
5
5
  SHA512:
6
- metadata.gz: 4d0299374c70e7e66b5edf3b141d171e79cb2f576ec32dcb6a2b4e11e9eb5bccaabbf2a5106fb5afd05f01ece8b6dc75e9a0cd956523c39ffdc4c3b5a17a0b22
7
- data.tar.gz: b60f377af9288f08cd5166ba7d71edc1060ccddf23c1d2912ed4b9d105e75ce6a6325a1f0e855e2141ded3c2ebdffbb7cd0e2f37796e662cc37ef833054d21d1
6
+ metadata.gz: 4a0ca1235fb0e8290ad261b61d77e38b4ee87372158384644612d54e81742f9f743968c8b8fa2c4e5cbebd57b62cdba7b0b0efbede6b3ad78176a7d7cca8dc37
7
+ data.tar.gz: 8fc1c553e0a008287280492dc401efccf1fdd675417985ef8acd8df3cb86ebe263a7cf6066b1740c8c387c1eb6de92ffffe428c5024b33a3ddf0464e729202a3
data/CHANGELOG.md CHANGED
@@ -5,6 +5,258 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## [1.9.4] - 2026-10-25
9
+
10
+ ### Added
11
+
12
+ - **Host links in the dashboard's user menu** (`actionagent`).
13
+ `ActionAgent.user_menu_links` takes an Array of `{ label:, href: }` or a
14
+ Proc receiving the signed-in user and the current owner, and the header's
15
+ avatar menu lists them above "Sign out". A hosted platform points them at
16
+ its workspace, members and account settings pages, and shows its admin
17
+ pages to the users who may use them; a self-hosted install leaves it unset
18
+ and sees the menu it had. Malformed entries are dropped and a raising Proc
19
+ is logged, so the menu seam cannot take the dashboard down.
20
+
21
+ ### Fixed
22
+
23
+ - **Keep the trace console usable offline** (`actionagent`). The console
24
+ ships its own inline styles and uses native details and ordinary links
25
+ instead of CDN-hosted Tailwind, Turbo and Stimulus. Automatic refresh is
26
+ opt-in (30 or 60 seconds) and uses the browser's refresh mechanism;
27
+ expanding a trace no longer depends on JavaScript or a host asset pipeline.
28
+ - **Discover and call generated agent previews** (`activeagent`). TestUnit
29
+ writes previews under `test/agents/previews`, with current URLs and query
30
+ parameters passed to the agent. Discovery checks for newly added files,
31
+ and the default image interceptor accepts generation objects.
32
+
33
+ - **Restore Rails browser previews** (`activeagent`). `/rails/agents` lists
34
+ previews and renders their prompt requests without executing a provider.
35
+ Namespaced actions and Markdown downloads work, unknown actions return
36
+ 404, and all endpoints enforce Rails' local-request restriction. Preview
37
+ directories participate in development reloading, including configured
38
+ paths and the legacy `test/docs/previews` directory. The dashboard gem is
39
+ not needed; the routes remain disabled outside development by default.
40
+
41
+ - **Generate runnable agent tests** (`activeagent`). TestUnit agent tests
42
+ inherit from `ActiveSupport::TestCase` instead of the nonexistent
43
+ `ActiveAgent::TestCase`. Plain and namespaced agents, including agents
44
+ without actions, can load in a Rails host without calling a provider.
45
+
46
+ ### Changed
47
+
48
+ - **One badge for every status and message role in the dashboard**
49
+ (`actionagent`). The agent cards' status pill and provider chip, the run
50
+ and sandbox session status pills (Agent Interactions, the sandbox runner)
51
+ and the message role pills in conversation streams all render through the
52
+ shared `Badge` (soft tint, strong mono text, radius 4) with one status
53
+ tone map (`statusTone`) and one role map (`roleBadge`), so an `active`
54
+ agent, a `complete` run and an `Assistant` turn look the same wherever
55
+ they appear and follow the dark theme, which the cards' Tailwind pills
56
+ did not. Two tones join the tokens for the system and developer roles.
57
+
58
+ ## [1.9.3] - 2026-10-21
59
+
60
+ Releases `activeagent` and `actionagent` 1.9.3, with the matching browser
61
+ sidecar: browser upload and code conversations (the sandbox browser's live
62
+ view takes files from the person driving it, and a Claude Code or Codex
63
+ session is a conversation with the sandbox's browser beside the chat and a
64
+ hand-off either way, Codex on this machine's own login), evaluation run
65
+ reports exported as JSON, HTML and Markdown with Copy as Markdown, an agent
66
+ builder that offers only tools that work, a model leaderboard on the
67
+ Evaluations page, complete trace window statistics, a judge pin that runs
68
+ on its own provider, Opper as a provider and Parallel Search in the MCP
69
+ catalog, and streaming fixes (unindexed tool call fragments, the finish
70
+ reason of streamed and Responses generations, Anthropic cache-token usage).
71
+ The framework, the dashboard engine and `@activeagents/browser-sidecar`
72
+ move to 1.9.3 together.
73
+
74
+ Upgrading: migration 028 adds the conversation columns to `code_sessions`.
75
+ A host runs `bin/rails generate action_agent:install --skip` then
76
+ `bin/rails db:migrate`, after 1.9.1's 027. Upgrade the sidecar with the
77
+ engine: the file chooser relay and the per-turn browser tokens are in the
78
+ 1.9.3 sidecar. On a single-tenant install the new report export
79
+ (`GET <mount>/api/evaluation_reports/:run_id`) answers 401 until
80
+ `ActionAgent.ingest_api_key` is set. A `judge_model` pin written
81
+ `anthropic/…` or `openai/…` now runs on that native provider; an install
82
+ that reached such a model through OpenRouter pins `openrouter/anthropic/…`.
83
+
84
+ ### Added
85
+
86
+ - **Export evaluation run reports as JSON, HTML and Markdown** (`activeagent`,
87
+ `actionagent`). `GET <mount>/api/evaluation_reports/:run_id`, with the
88
+ ingest key and scoped to the tenant like the POST (and refused on a
89
+ single-tenant install that set no key), returns a complete run
90
+ as the version-1 envelope the collector takes, so a run moves to another
91
+ install with one POST and posting it back is an identical retry (200);
92
+ `?format=html` returns the run's report page and `?format=md` its
93
+ Markdown, and `?download=1` an attachment of any of them.
94
+ An imported run exports under its published `run_id`, a dashboard run
95
+ under `dashboard-run-<id>-<created_at>`. `ActiveAgent::Evals::Publisher.envelope`
96
+ builds the envelope for both the publisher and the export. The dashboard's
97
+ run report adds `report.json` and `report.md` (one piece with
98
+ `?part=verdict|fix_items|scenario`), and `?download=1` saves any format
99
+ as `<evaluation>-run-<n>.<ext>`; the run report view and the suite's run
100
+ header get *Download HTML / JSON / Markdown*. New rake tasks:
101
+ `action_agent:evaluations:export RUN=… [EVALUATION=…] FORMAT=json|html|md [FILE=…]`
102
+ and `action_agent:evaluations:import FILE=… [ACCOUNT_ID=…]`.
103
+
104
+ - **Copy as Markdown** (`activeagent`, `actionagent`). `Report#to_markdown`
105
+ now has the HTML page's content in GitHub-flavored Markdown: the verdict's
106
+ rationale, What to fix from `Report#fix_items` (replacing the old
107
+ Recommendations list), and every scenario's answers in a `<details>` block
108
+ with a per-model table; table cells escape `|` and line breaks. Its pieces
109
+ are public — `verdict_markdown`, `fix_items_markdown(model:)`,
110
+ `fix_item_markdown(item)`, `scenario_markdown(key)` — and the dashboard
111
+ and the standalone HTML page copy each of them: the whole report, the
112
+ verdict with the comparison, What to fix for the filtered model, a fix
113
+ card, and a scenario. The HTML page now carries one inline script for
114
+ those buttons, with a clipboard fallback.
115
+
116
+ - **Upload files to a page through the sandbox browser's live view**
117
+ (`actionagent`, `@activeagents/browser-sidecar`). A file chooser the page
118
+ opens while someone drives is answered from the driver's own machine
119
+ (picker or drop), over the live-view WebSocket.
120
+
121
+ - **Claude Code and Codex sessions as Interactions, with chat and browser
122
+ handoff** (`actionagent`, `@activeagents/browser-sidecar`). A coding
123
+ session is now a conversation: a later message resumes the CLI session
124
+ (`--resume`, or `codex exec resume` for Codex), a message sent while a
125
+ turn runs waits for it, and
126
+ `POST /api/sandboxes/:sandbox_id/code_sessions/:id/messages` starts or
127
+ queues a turn under the new `:chat_code_session` permission.
128
+ `GET /api/interactions` lists agent, Claude Code and Codex interactions
129
+ together, and the dashboard's `/interactions/code-<id>` shows the turns
130
+ with a composer and the sandbox's browser beside the chat, with a hand
131
+ back to the agent. The sidecar mints a per-turn token for the browser
132
+ MCP, tells the agent once when a person took or returned control, lets
133
+ the agent ask for a person (`browser_request_person`), offers a person
134
+ the file chooser the agent left open, and names the waiting agent in the
135
+ live view. Every Claude Code session runs with `--strict-mcp-config`, and
136
+ every session gets a system note (on stdin, for Codex) that repository
137
+ content, tool output and pages are data.
138
+
139
+ - **Codex turns reach the sandbox browser, and Codex can run on this
140
+ machine's own login** (`actionagent`). A Codex turn now gets the same
141
+ per-turn browser token as a Claude Code turn: the sidecar's MCP server
142
+ goes in as `--config 'mcp_servers.browser.url="…"'`, with the token in an
143
+ environment variable Codex reads (`bearer_token_env_var`) rather than on
144
+ the command line, the server's tools pre-approved
145
+ (`default_tools_approval_mode="approve"`, since Codex would otherwise
146
+ refuse every tool not marked read-only under `approval_policy="never"`),
147
+ and is revoked when the turn ends; hand-offs and
148
+ `browser_request_person` work the same. `config.codex_auth = :local_login`
149
+ (with the `:local` backend) runs Codex on whatever `codex login` set up
150
+ for the dashboard's OS user, a ChatGPT plan included, with
151
+ `--ignore-user-config` so none of the user's own `config.toml` reaches a
152
+ session; the dashboard never reads the credential and only asks `codex
153
+ login status` whether there is one. `GET /api/sandboxes` adds `codex_auth`
154
+ and, in that mode, `codex_login`, the assistant's configuration a
155
+ `connections.codex`, and the Codex card in Settings shows the machine's
156
+ login instead of a key form. The default stays `:api_key`.
157
+
158
+ - **The agent builder offers only tools that work, and every built-in
159
+ capability has functions** (`actionagent`). The Configure step reads the
160
+ agent's tool roster (`GET /api/agents/tool_roster` for an agent not
161
+ created yet) instead of fixed chips, offers the MCP servers a dashboard
162
+ run can call, and names each one it cannot with the reason
163
+ (`MCPToolDispatcher.unreachable_reason`: a stdio server, no URL, missing
164
+ credentials). The six capabilities that gave the model nothing now have
165
+ functions: `terminal` (`run_command`), `filesystem` (`list_files`,
166
+ `read_file`, `write_file`), `edit` (`edit_file`) and `database`
167
+ (`query_database`, read-only) act in the run's checkout sandbox through
168
+ new `SandboxOrchestrator` verbs (on the `:local` backend; others return
169
+ `UnsupportedBackendError` to the model); `translate` (`translate_text`)
170
+ generates on the run's own provider; `slack` lists, reads and posts with
171
+ a bot token connected in Settings -> Integrations -> Slack (a `slack`
172
+ connection `ProviderKey`). Instruction sets are composed into the
173
+ instructions a run executes under. `run_command`, `write_file`,
174
+ `edit_file` and `slack_post_message` wait for approval by default: an
175
+ agent that gains `terminal`, `filesystem`, `edit` or `slack` — from the
176
+ builder, a template, the Tools tab or the agents API — has them on its
177
+ `approval_required_tools` unless the save names its own list, and the
178
+ Tools tab's Approval switch takes them off. An agent that enabled one of
179
+ those capabilities before this release keeps its list as it is. A host's
180
+ `ActionAgent.mcp_catalog` headers now reach the MCP client (they were
181
+ dropped).
182
+
183
+ - **Start an interaction with your own agent from Interactions and Run
184
+ Agents** (`actionagent`). Interactions has a "New interaction" picker and
185
+ Run Agents a "Your agents" list; both open the agent's runner on a fresh
186
+ conversation (`/agents/:id/run?new=1`). The runner lists the agent's tools
187
+ and MCP servers, and can run against a checkout sandbox (`sandbox_id`),
188
+ which the checkout tools need.
189
+
190
+ - **A model leaderboard on the Evaluations page** (`actionagent`). The Pass
191
+ rate tile listed each model's passes on a line of its own, a dozen lines
192
+ for seven models; it now shows the overall rate, the leading model and a
193
+ "compare N models" toggle, which opens a leaderboard below the tiles: one
194
+ row per model with its pooled passes and rate, and one column per counted
195
+ evaluation (`—` where the model did not run). Models rank by the lower
196
+ bound of the Wilson score interval (95%), so a lucky 7/8 does not outrank
197
+ 75/173.
198
+
199
+ - **Opper as an OpenAI-compatible provider** (`activeagent`). `service:
200
+ "Opper"` generates and embeds through Opper's compatibility endpoint
201
+ (`https://api.opper.ai/v3/compat`), with the key from `OPPER_API_KEY`
202
+ when none is configured, as the Requesty provider does (#583).
203
+
204
+ - **Parallel Search in the MCP catalog** (`actionagent`). An opt-in,
205
+ keyless `parallel` server (`web_search`, `web_fetch`) over HTTP; queries
206
+ and URLs go to parallel.ai. Every MCP HTTP request now sends
207
+ `User-Agent: ActionAgent/<version>` unless a header names another (#575).
208
+
209
+ ### Fixed
210
+
211
+ - **Trace totals and timeline buckets cover the whole filtered window**
212
+ (`actionagent`). `GET /api/traces` adds `total_count`, `returned_count`,
213
+ `truncated`, a `summary` (requests, errors, average latency, throughput
214
+ per minute) and zero-filled `buckets` computed over every trace the
215
+ filters match, independently of the capped detail list, and the Traces
216
+ view reports truncation and averages recorded request durations instead
217
+ of time-bucket means (part of #384).
218
+
219
+ - **A pinned evaluation judge runs on its own provider** (`actionagent`).
220
+ A `judge_model` pin selects the model's native provider, or the one its
221
+ qualifier names (`openrouter/anthropic/…`), instead of the first provider
222
+ with credentials; the qualifier is stripped before the request, the pin
223
+ wins over the credential resolver's default model, and a pinned route
224
+ with no credentials fails the run clearly when another provider is
225
+ configured. Native-to-gateway alias conversion stays out of scope (#431).
226
+ Upgrading: a pin written `anthropic/…` or `openai/…` now selects that
227
+ native provider, as candidate model names already did; an install that
228
+ reached such a model through OpenRouter pins `openrouter/anthropic/…`.
229
+
230
+ - **Streamed tool calls without an `index` are no longer split into
231
+ fragments** (`activeagent`). OpenAI-compatible servers (Gemini's
232
+ compatibility endpoint, some Ollama and OpenRouter backends) stream tool
233
+ calls without the `index` that ties argument fragments to their call, and
234
+ the Chat provider turned every fragment into its own tool call with
235
+ partial arguments. A fragment now continues the call with the same `id`,
236
+ or the last call when it has no `id` and no function name.
237
+
238
+ - **A streamed generation reports why it stopped** (`activeagent`).
239
+ `response.finish_reason` (and `stop_reason`), and the `finish_reason` of
240
+ the `prompt.active_agent` event, were `nil` for every streamed Chat and
241
+ Anthropic generation, since a stream has no response body to read it
242
+ from. The Chat provider now records the chunk's `finish_reason` and the
243
+ Anthropic provider the `message_delta`'s `stop_reason`, and the response
244
+ falls back to it.
245
+
246
+ - **OpenAI Responses generations report why they stopped** (`activeagent`).
247
+ `response.finish_reason` (and `stop_reason`) was `nil` for every
248
+ Responses generation, streamed or not. It is now
249
+ `incomplete_details.reason` when the response was cut short
250
+ (`max_output_tokens`, `content_filter`), else the response's `status`
251
+ (`completed`).
252
+
253
+ - **Anthropic usage with cache token fields no longer raises** (`activeagent`).
254
+ `Usage.from_provider_usage` recognised Anthropic's usage only by
255
+ `cache_creation` or `service_tier`; a payload carrying
256
+ `cache_read_input_tokens` or `cache_creation_input_tokens` without them
257
+ fell through to `Usage.new` and raised `UnknownAttributeError`. The cache
258
+ token fields now mark the payload as Anthropic's (#611).
259
+
8
260
  ## [1.9.2] - 2026-10-20
9
261
 
10
262
  Releases `activeagent` and `actionagent` 1.9.2, with the matching browser
data/README.md CHANGED
@@ -57,6 +57,9 @@ bundle add openai
57
57
  # Requesty (uses OpenAI-compatible API)
58
58
  bundle add openai
59
59
 
60
+ # Opper (uses OpenAI-compatible API)
61
+ bundle add openai
62
+
60
63
  # DeepSeek (uses OpenAI-compatible API)
61
64
  bundle add openai
62
65
 
@@ -26,7 +26,7 @@ module ActiveAgent
26
26
  #
27
27
  # @example Basic agent
28
28
  # class MyAgent < ActiveAgent::Base
29
- # generate_with :openai, model: "gpt-4"
29
+ # generate_with :openai, model: "gpt-5"
30
30
  #
31
31
  # def greet
32
32
  # prompt instructions: "Greet the user warmly"
@@ -112,7 +112,7 @@ module ActiveAgent
112
112
  # @return [void]
113
113
  #
114
114
  # @example
115
- # generate_with :openai, model: "gpt-4", temperature: 0.7
115
+ # generate_with :openai, model: "gpt-5"
116
116
  def self.generate_with(provider_reference, **agent_options)
117
117
  self.prompt_provider = provider_reference
118
118
 
@@ -68,7 +68,7 @@ module ActiveAgent
68
68
  class << self
69
69
  # Returns all agent preview classes.
70
70
  def all
71
- load_previews if descendants.empty?
71
+ load_previews
72
72
  descendants.sort_by { |agent| agent.name.titleize }
73
73
  end
74
74
 
@@ -111,7 +111,7 @@ module ActiveAgent
111
111
 
112
112
  def load_previews
113
113
  preview_paths.each do |preview_path|
114
- Dir["#{preview_path}/**/*_preview.rb"].sort.each { |file| require file }
114
+ Dir["#{preview_path}/**/*_preview.rb"].sort.each { |file| require_dependency file }
115
115
  end
116
116
  end
117
117
 
@@ -58,6 +58,13 @@ module ActiveAgent
58
58
  "--color-info" => "#3b82f6",
59
59
  "--color-info-soft" => "#dbeafe",
60
60
  "--color-info-text" => "#1e40af",
61
+ # The system and developer message roles: badge tones like the four above.
62
+ "--color-system" => "#8b5cf6",
63
+ "--color-system-soft" => "#ede9fe",
64
+ "--color-system-text" => "#6d28d9",
65
+ "--color-dev" => "#14b8a6",
66
+ "--color-dev-soft" => "#ccfbf1",
67
+ "--color-dev-text" => "#0f766e",
61
68
 
62
69
  # Trace span colors (observability)
63
70
  "--span-root" => "#9ca3af",
@@ -111,7 +118,11 @@ module ActiveAgent
111
118
  "--color-error-soft" => "rgba(220,38,38,0.15)",
112
119
  "--color-error-text" => "#f87171",
113
120
  "--color-info-soft" => "rgba(59,130,246,0.15)",
114
- "--color-info-text" => "#93c5fd"
121
+ "--color-info-text" => "#93c5fd",
122
+ "--color-system-soft" => "rgba(139,92,246,0.15)",
123
+ "--color-system-text" => "#c4b5fd",
124
+ "--color-dev-soft" => "rgba(20,184,166,0.15)",
125
+ "--color-dev-text" => "#5eead4"
115
126
  }.freeze
116
127
 
117
128
  # One CSS rule declaring `tokens` as custom properties on `scope`, with
@@ -94,11 +94,7 @@ module ActiveAgent
94
94
  # report may be a Report or its saved JSON hash. Full prompts, answers and
95
95
  # tool results are included; applications should make publication opt-in.
96
96
  def call(report:, run_id:, source:, agent_name:, suite:)
97
- identities = { "run_id" => run_id, "source" => source, "agent_name" => agent_name, "suite" => suite }
98
- identities.each do |key, value|
99
- raise ArgumentError, "#{key} must be a nonempty string" unless value.is_a?(String) && !value.strip.empty?
100
- end
101
- body = encode(identities.merge("version" => 1, "report" => report_hash(report)))
97
+ body = encode(self.class.envelope(report: report, run_id: run_id, source: source, agent_name: agent_name, suite: suite))
102
98
  raise Error, "Evaluation report exceeds the 2 MiB delivery limit; publish a smaller selection" if body.bytesize > MAX_BYTES
103
99
 
104
100
  deliver("retain the report and run_id for retry") do
@@ -143,6 +139,24 @@ module ActiveAgent
143
139
  end
144
140
  end
145
141
 
142
+ # The version-1 envelope a collector stores: the four identities, the
143
+ # version and the report's hash. What #call sends, and what a
144
+ # collector's export of a stored run returns, so the export can be
145
+ # posted again. report may be a Report or its saved JSON hash.
146
+ #
147
+ # Raises ArgumentError for an identity that is not a nonempty string,
148
+ # or a report that is neither.
149
+ def self.envelope(report:, run_id:, source:, agent_name:, suite:)
150
+ identities = { "run_id" => run_id, "source" => source, "agent_name" => agent_name, "suite" => suite }
151
+ identities.each do |key, value|
152
+ raise ArgumentError, "#{key} must be a nonempty string" unless value.is_a?(String) && !value.strip.empty?
153
+ end
154
+ hash = report.to_h if report.respond_to?(:to_h) && !report.nil? && !report.is_a?(Array)
155
+ raise ArgumentError, "report must be a Report or its saved JSON hash" unless hash.is_a?(Hash)
156
+
157
+ identities.merge("version" => 1, "report" => hash)
158
+ end
159
+
146
160
  private
147
161
 
148
162
  # Posts +body+ to the endpoint with the bearer key and the delivery timeouts.
@@ -176,13 +190,6 @@ module ActiveAgent
176
190
  raise Error.new("Evaluation delivery failed (#{e.class}); #{guidance}", retryable: true)
177
191
  end
178
192
 
179
- def report_hash(report)
180
- hash = report.to_h if report.respond_to?(:to_h) && !report.nil? && !report.is_a?(Array)
181
- raise ArgumentError, "report must be a Report or its saved JSON hash" unless hash.is_a?(Hash)
182
-
183
- hash
184
- end
185
-
186
193
  # Returns the envelope as JSON. A report holding invalid UTF-8, NaN,
187
194
  # Infinity or nesting over JSON's depth limit raises a non-retryable
188
195
  # Error, without the generator's message, which can quote the report.
@@ -8,6 +8,7 @@ module ActiveAgent
8
8
  # Markdown, or a self-contained HTML page (ReportHtml).
9
9
  class Report
10
10
  include ReportHtml
11
+ include ReportMarkdown
11
12
 
12
13
  # The judge a verdict names when the framework ranked the models by
13
14
  # pass rate itself, no judge having been available to rule on them.
@@ -307,24 +308,6 @@ module ActiveAgent
307
308
  JSON.pretty_generate(to_h, *args)
308
309
  end
309
310
 
310
- def to_markdown
311
- scenario_count = @results.map { |result| result.scenario.key }.uniq.size
312
- lines = [ "# Evaluation — #{scenario_count} scenario#{'s' unless scenario_count == 1} × #{@models.size} model#{'s' unless @models.size == 1}", "" ]
313
- label = @judge_label || @judge&.label
314
- lines << (label ? "Judged by `#{label}`." : "No judge; scored on rules and expectations alone.")
315
- lines << ""
316
- lines.concat(summary_table)
317
- lines << ""
318
- lines.concat(total_lines)
319
- lines << "**Best model: #{winner}**" if winner
320
- lines << ""
321
- lines.concat(comparison_lines)
322
- lines.concat(matrix_table)
323
- lines.concat(detail_lines)
324
- lines.concat(recommendation_lines)
325
- lines.join("\n")
326
- end
327
-
328
311
  private
329
312
 
330
313
  # Whether a model summary's cost is an estimate — any replay priced
@@ -559,7 +542,7 @@ module ActiveAgent
559
542
  latency = stats["avg_duration_ms"] ? "#{stats['avg_duration_ms']} ms" : "—"
560
543
  percentiles = stats["p50_duration_ms"] ? "#{stats['p50_duration_ms']} / #{stats['p95_duration_ms']} ms" : "—"
561
544
  cells = [
562
- "`#{label}`",
545
+ cell("`#{label}`"),
563
546
  Format.passes(stats["passed"], stats["scenarios"], style: :markdown),
564
547
  Format.score(stats["avg_score"]),
565
548
  latency,
@@ -568,7 +551,7 @@ module ActiveAgent
568
551
  Format.money(stats["cost"], estimated: estimated_cost?(stats)),
569
552
  per_thousand_cell(stats),
570
553
  (judge_cell(stats) if judged),
571
- faults.presence || "—"
554
+ cell(faults.presence || "—")
572
555
  ].compact
573
556
  "| #{cells.join(' | ')} |"
574
557
  end
@@ -617,13 +600,13 @@ module ActiveAgent
617
600
  lines = [ "## Comparison", "", "| Model | Pass rate | #{confidence} | Scenarios |", "|---|---|---|---|" ]
618
601
  comparison["ranking"].each do |label|
619
602
  stats = comparison["models"][label]
620
- lines << "| `#{label}` | #{rate_text(stats['pass_rate'])} | #{interval_text(stats['interval'])} | #{stats['scenarios']} |"
603
+ lines << "| #{cell("`#{label}`")} | #{rate_text(stats['pass_rate'])} | #{interval_text(stats['interval'])} | #{stats['scenarios']} |"
621
604
  end
622
605
  lines += [ "", "| Pair | Difference | #{confidence} | Shared scenarios | Decisive |", "|---|---|---|---|---|" ]
623
606
  comparison["pairs"].each do |pair|
624
607
  a, b = pair["models"]
625
608
  difference = pair["difference"] ? format("%+.1f points", pair["difference"]) : "—"
626
- lines << "| `#{a}` vs `#{b}` | #{difference} | #{interval_text(pair['interval'])} | #{pair['scenarios']} | #{pair['decisive'] ? 'yes' : 'no'} |"
609
+ lines << "| #{cell("`#{a}` vs `#{b}`")} | #{difference} | #{interval_text(pair['interval'])} | #{pair['scenarios']} | #{pair['decisive'] ? 'yes' : 'no'} |"
627
610
  end
628
611
  lines += [ "", comparison["conclusion"].to_s, "",
629
612
  "_#{confidence} from #{Format.delimited(method['resamples'])} resamples of whole scenarios (seed #{method['seed']})._", "" ]
@@ -642,16 +625,16 @@ module ActiveAgent
642
625
  # cost across every model, and in brackets what the judge spent on it.
643
626
  def matrix_table
644
627
  labels = @models.map(&:label)
645
- header = [ "| Scenario | #{labels.map { |label| "`#{label}`" }.join(' | ')} | Cost |", "|---|#{labels.map { '---' }.join('|')}|---|" ]
628
+ header = [ "| Scenario | #{labels.map { |label| cell("`#{label}`") }.join(' | ')} | Cost |", "|---|#{labels.map { '---' }.join('|')}|---|" ]
646
629
  rows = @results.group_by { |result| result.scenario.key }.map do |key, cohort|
647
630
  cells = labels.map do |label|
648
631
  result = cohort.find { |candidate| candidate.label == label }
649
632
  next "—" unless result
650
633
 
651
634
  mark = result.passed? ? "✅" : (result.errored? ? "⚠️" : "❌")
652
- [ mark, (Format.score(result.score) if result.score), result.fault&.tr("_", " ") ].compact.join(" ")
635
+ cell([ mark, (Format.score(result.score) if result.score), result.fault&.tr("_", " ") ].compact.join(" "))
653
636
  end
654
- "| `#{key}` #{cell(cohort.first.scenario.prompt.truncate(70))} | #{cells.join(' | ')} | #{scenario_cost_cell(key)} |"
637
+ "| #{cell("`#{key}` #{cohort.first.scenario.prompt.truncate(70)}")} | #{cells.join(' | ')} | #{scenario_cost_cell(key)} |"
655
638
  end
656
639
  header + rows
657
640
  end
@@ -664,30 +647,11 @@ module ActiveAgent
664
647
  text
665
648
  end
666
649
 
667
- # A prompt may contain " | " (ScenarioParser keeps it), which would
668
- # otherwise split the table cell.
650
+ # A prompt may contain " | " (ScenarioParser keeps it), and an answer
651
+ # or a label a line break, either of which would otherwise break the
652
+ # table row.
669
653
  def cell(text)
670
- text.to_s.gsub("|") { "\\|" }
671
- end
672
-
673
- def detail_lines
674
- lines = [ "", "## Answers", "" ]
675
- @results.each do |result|
676
- lines << "### `#{result.scenario.key}` · `#{result.label}` · #{result.status}" \
677
- "#{" · score #{Format.score(result.score)}" if result.score}#{" · #{answer_cost(result)}" if result.replay.cost}"
678
- lines << ""
679
- lines << "> #{result.scenario.prompt}"
680
- lines << ""
681
- if result.replay.tool_calls.any?
682
- lines << "Tools: #{result.replay.tool_calls.map { |call| "#{call['name']}#{' ✗' if call['error']}" }.join(', ')}"
683
- end
684
- lines << "Fault: #{result.fault.tr('_', ' ')} — #{result.summary} #{result.recommendation}" if result.fault
685
- lines << "Error: #{result.replay.error}" if result.replay.error
686
- lines << ""
687
- lines << (result.replay.answer.presence || "_(no answer)_").to_s.truncate(1_500)
688
- lines << ""
689
- end
690
- lines
654
+ text.to_s.gsub(/\s*\r?\n\s*/, " ").gsub("|") { "\\|" }
691
655
  end
692
656
 
693
657
  # "~$0.0243 · judge ~$0.0015": what one answer cost, and what judging
@@ -698,20 +662,6 @@ module ActiveAgent
698
662
  text += " · judge #{Format.money(judge_cost, estimated: estimated_usage?(result.judge_usage))}" if judge_cost
699
663
  text
700
664
  end
701
-
702
- # Renders after `detail_lines`, whose trailing blank line separates the two sections.
703
- def recommendation_lines
704
- return [] if recommendations.empty?
705
-
706
- lines = [ "## Recommendations", "" ]
707
- recommendations.each do |entry|
708
- lines << "- **#{entry['fault'].tr('_', ' ')}** ×#{entry['count']} (#{entry['scenario_keys'].join(', ')}): #{entry['recommendation']}"
709
- entry["suggested_tools"].each do |tool|
710
- lines << " - suggested tool `#{tool['name']}`: #{tool['description']}"
711
- end
712
- end
713
- lines << ""
714
- end
715
665
  end
716
666
  end
717
667
  end