activeagent 1.9.2 → 1.9.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +252 -0
- data/README.md +3 -0
- data/lib/active_agent/base.rb +2 -2
- data/lib/active_agent/concerns/preview.rb +2 -2
- data/lib/active_agent/evals/design_tokens.rb +12 -1
- data/lib/active_agent/evals/publisher.rb +19 -12
- data/lib/active_agent/evals/report.rb +12 -62
- data/lib/active_agent/evals/report_html.rb +85 -10
- data/lib/active_agent/evals/report_markdown.rb +202 -0
- data/lib/active_agent/evals.rb +1 -0
- data/lib/active_agent/inline_preview_interceptor.rb +1 -0
- data/lib/active_agent/preview/templates/layouts/active_agent/preview.html.erb +17 -0
- data/lib/active_agent/preview/templates/rails/agents/index.html.erb +14 -0
- data/lib/active_agent/preview/templates/rails/agents/preview.html.erb +3 -0
- data/lib/active_agent/providers/_base_provider.rb +2 -0
- data/lib/active_agent/providers/anthropic_provider.rb +1 -0
- data/lib/active_agent/providers/common/responses/base.rb +15 -2
- data/lib/active_agent/providers/common/usage.rb +4 -2
- data/lib/active_agent/providers/open_ai/chat_provider.rb +39 -0
- data/lib/active_agent/providers/open_ai/responses_provider.rb +3 -0
- data/lib/active_agent/providers/opper/_types.rb +16 -0
- data/lib/active_agent/providers/opper/options.rb +38 -0
- data/lib/active_agent/providers/opper_provider.rb +72 -0
- data/lib/active_agent/railtie.rb +11 -5
- data/lib/active_agent/version.rb +1 -1
- data/lib/generators/test_unit/agent_generator.rb +1 -1
- data/lib/generators/test_unit/templates/functional_test.rb.tt +1 -1
- data/lib/generators/test_unit/templates/preview.rb.tt +3 -3
- data/lib/rails/agents_controller.rb +43 -0
- metadata +10 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 715323643179a060b32d624ba906586778645e818c0436d4e4efe9ea6585847e
|
|
4
|
+
data.tar.gz: 3efe81e8a8d5357baf0f46adcfa5ce2fedd96234bb8ed603ffae63bac17bec58
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4a0ca1235fb0e8290ad261b61d77e38b4ee87372158384644612d54e81742f9f743968c8b8fa2c4e5cbebd57b62cdba7b0b0efbede6b3ad78176a7d7cca8dc37
|
|
7
|
+
data.tar.gz: 8fc1c553e0a008287280492dc401efccf1fdd675417985ef8acd8df3cb86ebe263a7cf6066b1740c8c387c1eb6de92ffffe428c5024b33a3ddf0464e729202a3
|
data/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,258 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.9.4] - 2026-10-25
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- **Host links in the dashboard's user menu** (`actionagent`).
|
|
13
|
+
`ActionAgent.user_menu_links` takes an Array of `{ label:, href: }` or a
|
|
14
|
+
Proc receiving the signed-in user and the current owner, and the header's
|
|
15
|
+
avatar menu lists them above "Sign out". A hosted platform points them at
|
|
16
|
+
its workspace, members and account settings pages, and shows its admin
|
|
17
|
+
pages to the users who may use them; a self-hosted install leaves it unset
|
|
18
|
+
and sees the menu it had. Malformed entries are dropped and a raising Proc
|
|
19
|
+
is logged, so the menu seam cannot take the dashboard down.
|
|
20
|
+
|
|
21
|
+
### Fixed
|
|
22
|
+
|
|
23
|
+
- **Keep the trace console usable offline** (`actionagent`). The console
|
|
24
|
+
ships its own inline styles and uses native details and ordinary links
|
|
25
|
+
instead of CDN-hosted Tailwind, Turbo and Stimulus. Automatic refresh is
|
|
26
|
+
opt-in (30 or 60 seconds) and uses the browser's refresh mechanism;
|
|
27
|
+
expanding a trace no longer depends on JavaScript or a host asset pipeline.
|
|
28
|
+
- **Discover and call generated agent previews** (`activeagent`). TestUnit
|
|
29
|
+
writes previews under `test/agents/previews`, with current URLs and query
|
|
30
|
+
parameters passed to the agent. Discovery checks for newly added files,
|
|
31
|
+
and the default image interceptor accepts generation objects.
|
|
32
|
+
|
|
33
|
+
- **Restore Rails browser previews** (`activeagent`). `/rails/agents` lists
|
|
34
|
+
previews and renders their prompt requests without executing a provider.
|
|
35
|
+
Namespaced actions and Markdown downloads work, unknown actions return
|
|
36
|
+
404, and all endpoints enforce Rails' local-request restriction. Preview
|
|
37
|
+
directories participate in development reloading, including configured
|
|
38
|
+
paths and the legacy `test/docs/previews` directory. The dashboard gem is
|
|
39
|
+
not needed; the routes remain disabled outside development by default.
|
|
40
|
+
|
|
41
|
+
- **Generate runnable agent tests** (`activeagent`). TestUnit agent tests
|
|
42
|
+
inherit from `ActiveSupport::TestCase` instead of the nonexistent
|
|
43
|
+
`ActiveAgent::TestCase`. Plain and namespaced agents, including agents
|
|
44
|
+
without actions, can load in a Rails host without calling a provider.
|
|
45
|
+
|
|
46
|
+
### Changed
|
|
47
|
+
|
|
48
|
+
- **One badge for every status and message role in the dashboard**
|
|
49
|
+
(`actionagent`). The agent cards' status pill and provider chip, the run
|
|
50
|
+
and sandbox session status pills (Agent Interactions, the sandbox runner)
|
|
51
|
+
and the message role pills in conversation streams all render through the
|
|
52
|
+
shared `Badge` (soft tint, strong mono text, radius 4) with one status
|
|
53
|
+
tone map (`statusTone`) and one role map (`roleBadge`), so an `active`
|
|
54
|
+
agent, a `complete` run and an `Assistant` turn look the same wherever
|
|
55
|
+
they appear and follow the dark theme, which the cards' Tailwind pills
|
|
56
|
+
did not. Two tones join the tokens for the system and developer roles.
|
|
57
|
+
|
|
58
|
+
## [1.9.3] - 2026-10-21
|
|
59
|
+
|
|
60
|
+
Releases `activeagent` and `actionagent` 1.9.3, with the matching browser
|
|
61
|
+
sidecar: browser upload and code conversations (the sandbox browser's live
|
|
62
|
+
view takes files from the person driving it, and a Claude Code or Codex
|
|
63
|
+
session is a conversation with the sandbox's browser beside the chat and a
|
|
64
|
+
hand-off either way, Codex on this machine's own login), evaluation run
|
|
65
|
+
reports exported as JSON, HTML and Markdown with Copy as Markdown, an agent
|
|
66
|
+
builder that offers only tools that work, a model leaderboard on the
|
|
67
|
+
Evaluations page, complete trace window statistics, a judge pin that runs
|
|
68
|
+
on its own provider, Opper as a provider and Parallel Search in the MCP
|
|
69
|
+
catalog, and streaming fixes (unindexed tool call fragments, the finish
|
|
70
|
+
reason of streamed and Responses generations, Anthropic cache-token usage).
|
|
71
|
+
The framework, the dashboard engine and `@activeagents/browser-sidecar`
|
|
72
|
+
move to 1.9.3 together.
|
|
73
|
+
|
|
74
|
+
Upgrading: migration 028 adds the conversation columns to `code_sessions`.
|
|
75
|
+
A host runs `bin/rails generate action_agent:install --skip` then
|
|
76
|
+
`bin/rails db:migrate`, after 1.9.1's 027. Upgrade the sidecar with the
|
|
77
|
+
engine: the file chooser relay and the per-turn browser tokens are in the
|
|
78
|
+
1.9.3 sidecar. On a single-tenant install the new report export
|
|
79
|
+
(`GET <mount>/api/evaluation_reports/:run_id`) answers 401 until
|
|
80
|
+
`ActionAgent.ingest_api_key` is set. A `judge_model` pin written
|
|
81
|
+
`anthropic/…` or `openai/…` now runs on that native provider; an install
|
|
82
|
+
that reached such a model through OpenRouter pins `openrouter/anthropic/…`.
|
|
83
|
+
|
|
84
|
+
### Added
|
|
85
|
+
|
|
86
|
+
- **Export evaluation run reports as JSON, HTML and Markdown** (`activeagent`,
|
|
87
|
+
`actionagent`). `GET <mount>/api/evaluation_reports/:run_id`, with the
|
|
88
|
+
ingest key and scoped to the tenant like the POST (and refused on a
|
|
89
|
+
single-tenant install that set no key), returns a complete run
|
|
90
|
+
as the version-1 envelope the collector takes, so a run moves to another
|
|
91
|
+
install with one POST and posting it back is an identical retry (200);
|
|
92
|
+
`?format=html` returns the run's report page and `?format=md` its
|
|
93
|
+
Markdown, and `?download=1` an attachment of any of them.
|
|
94
|
+
An imported run exports under its published `run_id`, a dashboard run
|
|
95
|
+
under `dashboard-run-<id>-<created_at>`. `ActiveAgent::Evals::Publisher.envelope`
|
|
96
|
+
builds the envelope for both the publisher and the export. The dashboard's
|
|
97
|
+
run report adds `report.json` and `report.md` (one piece with
|
|
98
|
+
`?part=verdict|fix_items|scenario`), and `?download=1` saves any format
|
|
99
|
+
as `<evaluation>-run-<n>.<ext>`; the run report view and the suite's run
|
|
100
|
+
header get *Download HTML / JSON / Markdown*. New rake tasks:
|
|
101
|
+
`action_agent:evaluations:export RUN=… [EVALUATION=…] FORMAT=json|html|md [FILE=…]`
|
|
102
|
+
and `action_agent:evaluations:import FILE=… [ACCOUNT_ID=…]`.
|
|
103
|
+
|
|
104
|
+
- **Copy as Markdown** (`activeagent`, `actionagent`). `Report#to_markdown`
|
|
105
|
+
now has the HTML page's content in GitHub-flavored Markdown: the verdict's
|
|
106
|
+
rationale, What to fix from `Report#fix_items` (replacing the old
|
|
107
|
+
Recommendations list), and every scenario's answers in a `<details>` block
|
|
108
|
+
with a per-model table; table cells escape `|` and line breaks. Its pieces
|
|
109
|
+
are public — `verdict_markdown`, `fix_items_markdown(model:)`,
|
|
110
|
+
`fix_item_markdown(item)`, `scenario_markdown(key)` — and the dashboard
|
|
111
|
+
and the standalone HTML page copy each of them: the whole report, the
|
|
112
|
+
verdict with the comparison, What to fix for the filtered model, a fix
|
|
113
|
+
card, and a scenario. The HTML page now carries one inline script for
|
|
114
|
+
those buttons, with a clipboard fallback.
|
|
115
|
+
|
|
116
|
+
- **Upload files to a page through the sandbox browser's live view**
|
|
117
|
+
(`actionagent`, `@activeagents/browser-sidecar`). A file chooser the page
|
|
118
|
+
opens while someone drives is answered from the driver's own machine
|
|
119
|
+
(picker or drop), over the live-view WebSocket.
|
|
120
|
+
|
|
121
|
+
- **Claude Code and Codex sessions as Interactions, with chat and browser
|
|
122
|
+
handoff** (`actionagent`, `@activeagents/browser-sidecar`). A coding
|
|
123
|
+
session is now a conversation: a later message resumes the CLI session
|
|
124
|
+
(`--resume`, or `codex exec resume` for Codex), a message sent while a
|
|
125
|
+
turn runs waits for it, and
|
|
126
|
+
`POST /api/sandboxes/:sandbox_id/code_sessions/:id/messages` starts or
|
|
127
|
+
queues a turn under the new `:chat_code_session` permission.
|
|
128
|
+
`GET /api/interactions` lists agent, Claude Code and Codex interactions
|
|
129
|
+
together, and the dashboard's `/interactions/code-<id>` shows the turns
|
|
130
|
+
with a composer and the sandbox's browser beside the chat, with a hand
|
|
131
|
+
back to the agent. The sidecar mints a per-turn token for the browser
|
|
132
|
+
MCP, tells the agent once when a person took or returned control, lets
|
|
133
|
+
the agent ask for a person (`browser_request_person`), offers a person
|
|
134
|
+
the file chooser the agent left open, and names the waiting agent in the
|
|
135
|
+
live view. Every Claude Code session runs with `--strict-mcp-config`, and
|
|
136
|
+
every session gets a system note (on stdin, for Codex) that repository
|
|
137
|
+
content, tool output and pages are data.
|
|
138
|
+
|
|
139
|
+
- **Codex turns reach the sandbox browser, and Codex can run on this
|
|
140
|
+
machine's own login** (`actionagent`). A Codex turn now gets the same
|
|
141
|
+
per-turn browser token as a Claude Code turn: the sidecar's MCP server
|
|
142
|
+
goes in as `--config 'mcp_servers.browser.url="…"'`, with the token in an
|
|
143
|
+
environment variable Codex reads (`bearer_token_env_var`) rather than on
|
|
144
|
+
the command line, the server's tools pre-approved
|
|
145
|
+
(`default_tools_approval_mode="approve"`, since Codex would otherwise
|
|
146
|
+
refuse every tool not marked read-only under `approval_policy="never"`),
|
|
147
|
+
and is revoked when the turn ends; hand-offs and
|
|
148
|
+
`browser_request_person` work the same. `config.codex_auth = :local_login`
|
|
149
|
+
(with the `:local` backend) runs Codex on whatever `codex login` set up
|
|
150
|
+
for the dashboard's OS user, a ChatGPT plan included, with
|
|
151
|
+
`--ignore-user-config` so none of the user's own `config.toml` reaches a
|
|
152
|
+
session; the dashboard never reads the credential and only asks `codex
|
|
153
|
+
login status` whether there is one. `GET /api/sandboxes` adds `codex_auth`
|
|
154
|
+
and, in that mode, `codex_login`, the assistant's configuration a
|
|
155
|
+
`connections.codex`, and the Codex card in Settings shows the machine's
|
|
156
|
+
login instead of a key form. The default stays `:api_key`.
|
|
157
|
+
|
|
158
|
+
- **The agent builder offers only tools that work, and every built-in
|
|
159
|
+
capability has functions** (`actionagent`). The Configure step reads the
|
|
160
|
+
agent's tool roster (`GET /api/agents/tool_roster` for an agent not
|
|
161
|
+
created yet) instead of fixed chips, offers the MCP servers a dashboard
|
|
162
|
+
run can call, and names each one it cannot with the reason
|
|
163
|
+
(`MCPToolDispatcher.unreachable_reason`: a stdio server, no URL, missing
|
|
164
|
+
credentials). The six capabilities that gave the model nothing now have
|
|
165
|
+
functions: `terminal` (`run_command`), `filesystem` (`list_files`,
|
|
166
|
+
`read_file`, `write_file`), `edit` (`edit_file`) and `database`
|
|
167
|
+
(`query_database`, read-only) act in the run's checkout sandbox through
|
|
168
|
+
new `SandboxOrchestrator` verbs (on the `:local` backend; others return
|
|
169
|
+
`UnsupportedBackendError` to the model); `translate` (`translate_text`)
|
|
170
|
+
generates on the run's own provider; `slack` lists, reads and posts with
|
|
171
|
+
a bot token connected in Settings -> Integrations -> Slack (a `slack`
|
|
172
|
+
connection `ProviderKey`). Instruction sets are composed into the
|
|
173
|
+
instructions a run executes under. `run_command`, `write_file`,
|
|
174
|
+
`edit_file` and `slack_post_message` wait for approval by default: an
|
|
175
|
+
agent that gains `terminal`, `filesystem`, `edit` or `slack` — from the
|
|
176
|
+
builder, a template, the Tools tab or the agents API — has them on its
|
|
177
|
+
`approval_required_tools` unless the save names its own list, and the
|
|
178
|
+
Tools tab's Approval switch takes them off. An agent that enabled one of
|
|
179
|
+
those capabilities before this release keeps its list as it is. A host's
|
|
180
|
+
`ActionAgent.mcp_catalog` headers now reach the MCP client (they were
|
|
181
|
+
dropped).
|
|
182
|
+
|
|
183
|
+
- **Start an interaction with your own agent from Interactions and Run
|
|
184
|
+
Agents** (`actionagent`). Interactions has a "New interaction" picker and
|
|
185
|
+
Run Agents a "Your agents" list; both open the agent's runner on a fresh
|
|
186
|
+
conversation (`/agents/:id/run?new=1`). The runner lists the agent's tools
|
|
187
|
+
and MCP servers, and can run against a checkout sandbox (`sandbox_id`),
|
|
188
|
+
which the checkout tools need.
|
|
189
|
+
|
|
190
|
+
- **A model leaderboard on the Evaluations page** (`actionagent`). The Pass
|
|
191
|
+
rate tile listed each model's passes on a line of its own, a dozen lines
|
|
192
|
+
for seven models; it now shows the overall rate, the leading model and a
|
|
193
|
+
"compare N models" toggle, which opens a leaderboard below the tiles: one
|
|
194
|
+
row per model with its pooled passes and rate, and one column per counted
|
|
195
|
+
evaluation (`—` where the model did not run). Models rank by the lower
|
|
196
|
+
bound of the Wilson score interval (95%), so a lucky 7/8 does not outrank
|
|
197
|
+
75/173.
|
|
198
|
+
|
|
199
|
+
- **Opper as an OpenAI-compatible provider** (`activeagent`). `service:
|
|
200
|
+
"Opper"` generates and embeds through Opper's compatibility endpoint
|
|
201
|
+
(`https://api.opper.ai/v3/compat`), with the key from `OPPER_API_KEY`
|
|
202
|
+
when none is configured, as the Requesty provider does (#583).
|
|
203
|
+
|
|
204
|
+
- **Parallel Search in the MCP catalog** (`actionagent`). An opt-in,
|
|
205
|
+
keyless `parallel` server (`web_search`, `web_fetch`) over HTTP; queries
|
|
206
|
+
and URLs go to parallel.ai. Every MCP HTTP request now sends
|
|
207
|
+
`User-Agent: ActionAgent/<version>` unless a header names another (#575).
|
|
208
|
+
|
|
209
|
+
### Fixed
|
|
210
|
+
|
|
211
|
+
- **Trace totals and timeline buckets cover the whole filtered window**
|
|
212
|
+
(`actionagent`). `GET /api/traces` adds `total_count`, `returned_count`,
|
|
213
|
+
`truncated`, a `summary` (requests, errors, average latency, throughput
|
|
214
|
+
per minute) and zero-filled `buckets` computed over every trace the
|
|
215
|
+
filters match, independently of the capped detail list, and the Traces
|
|
216
|
+
view reports truncation and averages recorded request durations instead
|
|
217
|
+
of time-bucket means (part of #384).
|
|
218
|
+
|
|
219
|
+
- **A pinned evaluation judge runs on its own provider** (`actionagent`).
|
|
220
|
+
A `judge_model` pin selects the model's native provider, or the one its
|
|
221
|
+
qualifier names (`openrouter/anthropic/…`), instead of the first provider
|
|
222
|
+
with credentials; the qualifier is stripped before the request, the pin
|
|
223
|
+
wins over the credential resolver's default model, and a pinned route
|
|
224
|
+
with no credentials fails the run clearly when another provider is
|
|
225
|
+
configured. Native-to-gateway alias conversion stays out of scope (#431).
|
|
226
|
+
Upgrading: a pin written `anthropic/…` or `openai/…` now selects that
|
|
227
|
+
native provider, as candidate model names already did; an install that
|
|
228
|
+
reached such a model through OpenRouter pins `openrouter/anthropic/…`.
|
|
229
|
+
|
|
230
|
+
- **Streamed tool calls without an `index` are no longer split into
|
|
231
|
+
fragments** (`activeagent`). OpenAI-compatible servers (Gemini's
|
|
232
|
+
compatibility endpoint, some Ollama and OpenRouter backends) stream tool
|
|
233
|
+
calls without the `index` that ties argument fragments to their call, and
|
|
234
|
+
the Chat provider turned every fragment into its own tool call with
|
|
235
|
+
partial arguments. A fragment now continues the call with the same `id`,
|
|
236
|
+
or the last call when it has no `id` and no function name.
|
|
237
|
+
|
|
238
|
+
- **A streamed generation reports why it stopped** (`activeagent`).
|
|
239
|
+
`response.finish_reason` (and `stop_reason`), and the `finish_reason` of
|
|
240
|
+
the `prompt.active_agent` event, were `nil` for every streamed Chat and
|
|
241
|
+
Anthropic generation, since a stream has no response body to read it
|
|
242
|
+
from. The Chat provider now records the chunk's `finish_reason` and the
|
|
243
|
+
Anthropic provider the `message_delta`'s `stop_reason`, and the response
|
|
244
|
+
falls back to it.
|
|
245
|
+
|
|
246
|
+
- **OpenAI Responses generations report why they stopped** (`activeagent`).
|
|
247
|
+
`response.finish_reason` (and `stop_reason`) was `nil` for every
|
|
248
|
+
Responses generation, streamed or not. It is now
|
|
249
|
+
`incomplete_details.reason` when the response was cut short
|
|
250
|
+
(`max_output_tokens`, `content_filter`), else the response's `status`
|
|
251
|
+
(`completed`).
|
|
252
|
+
|
|
253
|
+
- **Anthropic usage with cache token fields no longer raises** (`activeagent`).
|
|
254
|
+
`Usage.from_provider_usage` recognised Anthropic's usage only by
|
|
255
|
+
`cache_creation` or `service_tier`; a payload carrying
|
|
256
|
+
`cache_read_input_tokens` or `cache_creation_input_tokens` without them
|
|
257
|
+
fell through to `Usage.new` and raised `UnknownAttributeError`. The cache
|
|
258
|
+
token fields now mark the payload as Anthropic's (#611).
|
|
259
|
+
|
|
8
260
|
## [1.9.2] - 2026-10-20
|
|
9
261
|
|
|
10
262
|
Releases `activeagent` and `actionagent` 1.9.2, with the matching browser
|
data/README.md
CHANGED
data/lib/active_agent/base.rb
CHANGED
|
@@ -26,7 +26,7 @@ module ActiveAgent
|
|
|
26
26
|
#
|
|
27
27
|
# @example Basic agent
|
|
28
28
|
# class MyAgent < ActiveAgent::Base
|
|
29
|
-
# generate_with :openai, model: "gpt-
|
|
29
|
+
# generate_with :openai, model: "gpt-5"
|
|
30
30
|
#
|
|
31
31
|
# def greet
|
|
32
32
|
# prompt instructions: "Greet the user warmly"
|
|
@@ -112,7 +112,7 @@ module ActiveAgent
|
|
|
112
112
|
# @return [void]
|
|
113
113
|
#
|
|
114
114
|
# @example
|
|
115
|
-
# generate_with :openai, model: "gpt-
|
|
115
|
+
# generate_with :openai, model: "gpt-5"
|
|
116
116
|
def self.generate_with(provider_reference, **agent_options)
|
|
117
117
|
self.prompt_provider = provider_reference
|
|
118
118
|
|
|
@@ -68,7 +68,7 @@ module ActiveAgent
|
|
|
68
68
|
class << self
|
|
69
69
|
# Returns all agent preview classes.
|
|
70
70
|
def all
|
|
71
|
-
load_previews
|
|
71
|
+
load_previews
|
|
72
72
|
descendants.sort_by { |agent| agent.name.titleize }
|
|
73
73
|
end
|
|
74
74
|
|
|
@@ -111,7 +111,7 @@ module ActiveAgent
|
|
|
111
111
|
|
|
112
112
|
def load_previews
|
|
113
113
|
preview_paths.each do |preview_path|
|
|
114
|
-
Dir["#{preview_path}/**/*_preview.rb"].sort.each { |file|
|
|
114
|
+
Dir["#{preview_path}/**/*_preview.rb"].sort.each { |file| require_dependency file }
|
|
115
115
|
end
|
|
116
116
|
end
|
|
117
117
|
|
|
@@ -58,6 +58,13 @@ module ActiveAgent
|
|
|
58
58
|
"--color-info" => "#3b82f6",
|
|
59
59
|
"--color-info-soft" => "#dbeafe",
|
|
60
60
|
"--color-info-text" => "#1e40af",
|
|
61
|
+
# The system and developer message roles: badge tones like the four above.
|
|
62
|
+
"--color-system" => "#8b5cf6",
|
|
63
|
+
"--color-system-soft" => "#ede9fe",
|
|
64
|
+
"--color-system-text" => "#6d28d9",
|
|
65
|
+
"--color-dev" => "#14b8a6",
|
|
66
|
+
"--color-dev-soft" => "#ccfbf1",
|
|
67
|
+
"--color-dev-text" => "#0f766e",
|
|
61
68
|
|
|
62
69
|
# Trace span colors (observability)
|
|
63
70
|
"--span-root" => "#9ca3af",
|
|
@@ -111,7 +118,11 @@ module ActiveAgent
|
|
|
111
118
|
"--color-error-soft" => "rgba(220,38,38,0.15)",
|
|
112
119
|
"--color-error-text" => "#f87171",
|
|
113
120
|
"--color-info-soft" => "rgba(59,130,246,0.15)",
|
|
114
|
-
"--color-info-text" => "#93c5fd"
|
|
121
|
+
"--color-info-text" => "#93c5fd",
|
|
122
|
+
"--color-system-soft" => "rgba(139,92,246,0.15)",
|
|
123
|
+
"--color-system-text" => "#c4b5fd",
|
|
124
|
+
"--color-dev-soft" => "rgba(20,184,166,0.15)",
|
|
125
|
+
"--color-dev-text" => "#5eead4"
|
|
115
126
|
}.freeze
|
|
116
127
|
|
|
117
128
|
# One CSS rule declaring `tokens` as custom properties on `scope`, with
|
|
@@ -94,11 +94,7 @@ module ActiveAgent
|
|
|
94
94
|
# report may be a Report or its saved JSON hash. Full prompts, answers and
|
|
95
95
|
# tool results are included; applications should make publication opt-in.
|
|
96
96
|
def call(report:, run_id:, source:, agent_name:, suite:)
|
|
97
|
-
|
|
98
|
-
identities.each do |key, value|
|
|
99
|
-
raise ArgumentError, "#{key} must be a nonempty string" unless value.is_a?(String) && !value.strip.empty?
|
|
100
|
-
end
|
|
101
|
-
body = encode(identities.merge("version" => 1, "report" => report_hash(report)))
|
|
97
|
+
body = encode(self.class.envelope(report: report, run_id: run_id, source: source, agent_name: agent_name, suite: suite))
|
|
102
98
|
raise Error, "Evaluation report exceeds the 2 MiB delivery limit; publish a smaller selection" if body.bytesize > MAX_BYTES
|
|
103
99
|
|
|
104
100
|
deliver("retain the report and run_id for retry") do
|
|
@@ -143,6 +139,24 @@ module ActiveAgent
|
|
|
143
139
|
end
|
|
144
140
|
end
|
|
145
141
|
|
|
142
|
+
# The version-1 envelope a collector stores: the four identities, the
|
|
143
|
+
# version and the report's hash. What #call sends, and what a
|
|
144
|
+
# collector's export of a stored run returns, so the export can be
|
|
145
|
+
# posted again. report may be a Report or its saved JSON hash.
|
|
146
|
+
#
|
|
147
|
+
# Raises ArgumentError for an identity that is not a nonempty string,
|
|
148
|
+
# or a report that is neither.
|
|
149
|
+
def self.envelope(report:, run_id:, source:, agent_name:, suite:)
|
|
150
|
+
identities = { "run_id" => run_id, "source" => source, "agent_name" => agent_name, "suite" => suite }
|
|
151
|
+
identities.each do |key, value|
|
|
152
|
+
raise ArgumentError, "#{key} must be a nonempty string" unless value.is_a?(String) && !value.strip.empty?
|
|
153
|
+
end
|
|
154
|
+
hash = report.to_h if report.respond_to?(:to_h) && !report.nil? && !report.is_a?(Array)
|
|
155
|
+
raise ArgumentError, "report must be a Report or its saved JSON hash" unless hash.is_a?(Hash)
|
|
156
|
+
|
|
157
|
+
identities.merge("version" => 1, "report" => hash)
|
|
158
|
+
end
|
|
159
|
+
|
|
146
160
|
private
|
|
147
161
|
|
|
148
162
|
# Posts +body+ to the endpoint with the bearer key and the delivery timeouts.
|
|
@@ -176,13 +190,6 @@ module ActiveAgent
|
|
|
176
190
|
raise Error.new("Evaluation delivery failed (#{e.class}); #{guidance}", retryable: true)
|
|
177
191
|
end
|
|
178
192
|
|
|
179
|
-
def report_hash(report)
|
|
180
|
-
hash = report.to_h if report.respond_to?(:to_h) && !report.nil? && !report.is_a?(Array)
|
|
181
|
-
raise ArgumentError, "report must be a Report or its saved JSON hash" unless hash.is_a?(Hash)
|
|
182
|
-
|
|
183
|
-
hash
|
|
184
|
-
end
|
|
185
|
-
|
|
186
193
|
# Returns the envelope as JSON. A report holding invalid UTF-8, NaN,
|
|
187
194
|
# Infinity or nesting over JSON's depth limit raises a non-retryable
|
|
188
195
|
# Error, without the generator's message, which can quote the report.
|
|
@@ -8,6 +8,7 @@ module ActiveAgent
|
|
|
8
8
|
# Markdown, or a self-contained HTML page (ReportHtml).
|
|
9
9
|
class Report
|
|
10
10
|
include ReportHtml
|
|
11
|
+
include ReportMarkdown
|
|
11
12
|
|
|
12
13
|
# The judge a verdict names when the framework ranked the models by
|
|
13
14
|
# pass rate itself, no judge having been available to rule on them.
|
|
@@ -307,24 +308,6 @@ module ActiveAgent
|
|
|
307
308
|
JSON.pretty_generate(to_h, *args)
|
|
308
309
|
end
|
|
309
310
|
|
|
310
|
-
def to_markdown
|
|
311
|
-
scenario_count = @results.map { |result| result.scenario.key }.uniq.size
|
|
312
|
-
lines = [ "# Evaluation — #{scenario_count} scenario#{'s' unless scenario_count == 1} × #{@models.size} model#{'s' unless @models.size == 1}", "" ]
|
|
313
|
-
label = @judge_label || @judge&.label
|
|
314
|
-
lines << (label ? "Judged by `#{label}`." : "No judge; scored on rules and expectations alone.")
|
|
315
|
-
lines << ""
|
|
316
|
-
lines.concat(summary_table)
|
|
317
|
-
lines << ""
|
|
318
|
-
lines.concat(total_lines)
|
|
319
|
-
lines << "**Best model: #{winner}**" if winner
|
|
320
|
-
lines << ""
|
|
321
|
-
lines.concat(comparison_lines)
|
|
322
|
-
lines.concat(matrix_table)
|
|
323
|
-
lines.concat(detail_lines)
|
|
324
|
-
lines.concat(recommendation_lines)
|
|
325
|
-
lines.join("\n")
|
|
326
|
-
end
|
|
327
|
-
|
|
328
311
|
private
|
|
329
312
|
|
|
330
313
|
# Whether a model summary's cost is an estimate — any replay priced
|
|
@@ -559,7 +542,7 @@ module ActiveAgent
|
|
|
559
542
|
latency = stats["avg_duration_ms"] ? "#{stats['avg_duration_ms']} ms" : "—"
|
|
560
543
|
percentiles = stats["p50_duration_ms"] ? "#{stats['p50_duration_ms']} / #{stats['p95_duration_ms']} ms" : "—"
|
|
561
544
|
cells = [
|
|
562
|
-
"`#{label}`",
|
|
545
|
+
cell("`#{label}`"),
|
|
563
546
|
Format.passes(stats["passed"], stats["scenarios"], style: :markdown),
|
|
564
547
|
Format.score(stats["avg_score"]),
|
|
565
548
|
latency,
|
|
@@ -568,7 +551,7 @@ module ActiveAgent
|
|
|
568
551
|
Format.money(stats["cost"], estimated: estimated_cost?(stats)),
|
|
569
552
|
per_thousand_cell(stats),
|
|
570
553
|
(judge_cell(stats) if judged),
|
|
571
|
-
faults.presence || "—"
|
|
554
|
+
cell(faults.presence || "—")
|
|
572
555
|
].compact
|
|
573
556
|
"| #{cells.join(' | ')} |"
|
|
574
557
|
end
|
|
@@ -617,13 +600,13 @@ module ActiveAgent
|
|
|
617
600
|
lines = [ "## Comparison", "", "| Model | Pass rate | #{confidence} | Scenarios |", "|---|---|---|---|" ]
|
|
618
601
|
comparison["ranking"].each do |label|
|
|
619
602
|
stats = comparison["models"][label]
|
|
620
|
-
lines << "| `#{label}` | #{rate_text(stats['pass_rate'])} | #{interval_text(stats['interval'])} | #{stats['scenarios']} |"
|
|
603
|
+
lines << "| #{cell("`#{label}`")} | #{rate_text(stats['pass_rate'])} | #{interval_text(stats['interval'])} | #{stats['scenarios']} |"
|
|
621
604
|
end
|
|
622
605
|
lines += [ "", "| Pair | Difference | #{confidence} | Shared scenarios | Decisive |", "|---|---|---|---|---|" ]
|
|
623
606
|
comparison["pairs"].each do |pair|
|
|
624
607
|
a, b = pair["models"]
|
|
625
608
|
difference = pair["difference"] ? format("%+.1f points", pair["difference"]) : "—"
|
|
626
|
-
lines << "| `#{a}` vs `#{b}` | #{difference} | #{interval_text(pair['interval'])} | #{pair['scenarios']} | #{pair['decisive'] ? 'yes' : 'no'} |"
|
|
609
|
+
lines << "| #{cell("`#{a}` vs `#{b}`")} | #{difference} | #{interval_text(pair['interval'])} | #{pair['scenarios']} | #{pair['decisive'] ? 'yes' : 'no'} |"
|
|
627
610
|
end
|
|
628
611
|
lines += [ "", comparison["conclusion"].to_s, "",
|
|
629
612
|
"_#{confidence} from #{Format.delimited(method['resamples'])} resamples of whole scenarios (seed #{method['seed']})._", "" ]
|
|
@@ -642,16 +625,16 @@ module ActiveAgent
|
|
|
642
625
|
# cost across every model, and in brackets what the judge spent on it.
|
|
643
626
|
def matrix_table
|
|
644
627
|
labels = @models.map(&:label)
|
|
645
|
-
header = [ "| Scenario | #{labels.map { |label| "`#{label}`" }.join(' | ')} | Cost |", "|---|#{labels.map { '---' }.join('|')}|---|" ]
|
|
628
|
+
header = [ "| Scenario | #{labels.map { |label| cell("`#{label}`") }.join(' | ')} | Cost |", "|---|#{labels.map { '---' }.join('|')}|---|" ]
|
|
646
629
|
rows = @results.group_by { |result| result.scenario.key }.map do |key, cohort|
|
|
647
630
|
cells = labels.map do |label|
|
|
648
631
|
result = cohort.find { |candidate| candidate.label == label }
|
|
649
632
|
next "—" unless result
|
|
650
633
|
|
|
651
634
|
mark = result.passed? ? "✅" : (result.errored? ? "⚠️" : "❌")
|
|
652
|
-
[ mark, (Format.score(result.score) if result.score), result.fault&.tr("_", " ") ].compact.join(" ")
|
|
635
|
+
cell([ mark, (Format.score(result.score) if result.score), result.fault&.tr("_", " ") ].compact.join(" "))
|
|
653
636
|
end
|
|
654
|
-
"| `#{key}` #{
|
|
637
|
+
"| #{cell("`#{key}` #{cohort.first.scenario.prompt.truncate(70)}")} | #{cells.join(' | ')} | #{scenario_cost_cell(key)} |"
|
|
655
638
|
end
|
|
656
639
|
header + rows
|
|
657
640
|
end
|
|
@@ -664,30 +647,11 @@ module ActiveAgent
|
|
|
664
647
|
text
|
|
665
648
|
end
|
|
666
649
|
|
|
667
|
-
# A prompt may contain " | " (ScenarioParser keeps it),
|
|
668
|
-
# otherwise
|
|
650
|
+
# A prompt may contain " | " (ScenarioParser keeps it), and an answer
|
|
651
|
+
# or a label a line break, either of which would otherwise break the
|
|
652
|
+
# table row.
|
|
669
653
|
def cell(text)
|
|
670
|
-
text.to_s.gsub("|") { "\\|" }
|
|
671
|
-
end
|
|
672
|
-
|
|
673
|
-
def detail_lines
|
|
674
|
-
lines = [ "", "## Answers", "" ]
|
|
675
|
-
@results.each do |result|
|
|
676
|
-
lines << "### `#{result.scenario.key}` · `#{result.label}` · #{result.status}" \
|
|
677
|
-
"#{" · score #{Format.score(result.score)}" if result.score}#{" · #{answer_cost(result)}" if result.replay.cost}"
|
|
678
|
-
lines << ""
|
|
679
|
-
lines << "> #{result.scenario.prompt}"
|
|
680
|
-
lines << ""
|
|
681
|
-
if result.replay.tool_calls.any?
|
|
682
|
-
lines << "Tools: #{result.replay.tool_calls.map { |call| "#{call['name']}#{' ✗' if call['error']}" }.join(', ')}"
|
|
683
|
-
end
|
|
684
|
-
lines << "Fault: #{result.fault.tr('_', ' ')} — #{result.summary} #{result.recommendation}" if result.fault
|
|
685
|
-
lines << "Error: #{result.replay.error}" if result.replay.error
|
|
686
|
-
lines << ""
|
|
687
|
-
lines << (result.replay.answer.presence || "_(no answer)_").to_s.truncate(1_500)
|
|
688
|
-
lines << ""
|
|
689
|
-
end
|
|
690
|
-
lines
|
|
654
|
+
text.to_s.gsub(/\s*\r?\n\s*/, " ").gsub("|") { "\\|" }
|
|
691
655
|
end
|
|
692
656
|
|
|
693
657
|
# "~$0.0243 · judge ~$0.0015": what one answer cost, and what judging
|
|
@@ -698,20 +662,6 @@ module ActiveAgent
|
|
|
698
662
|
text += " · judge #{Format.money(judge_cost, estimated: estimated_usage?(result.judge_usage))}" if judge_cost
|
|
699
663
|
text
|
|
700
664
|
end
|
|
701
|
-
|
|
702
|
-
# Renders after `detail_lines`, whose trailing blank line separates the two sections.
|
|
703
|
-
def recommendation_lines
|
|
704
|
-
return [] if recommendations.empty?
|
|
705
|
-
|
|
706
|
-
lines = [ "## Recommendations", "" ]
|
|
707
|
-
recommendations.each do |entry|
|
|
708
|
-
lines << "- **#{entry['fault'].tr('_', ' ')}** ×#{entry['count']} (#{entry['scenario_keys'].join(', ')}): #{entry['recommendation']}"
|
|
709
|
-
entry["suggested_tools"].each do |tool|
|
|
710
|
-
lines << " - suggested tool `#{tool['name']}`: #{tool['description']}"
|
|
711
|
-
end
|
|
712
|
-
end
|
|
713
|
-
lines << ""
|
|
714
|
-
end
|
|
715
665
|
end
|
|
716
666
|
end
|
|
717
667
|
end
|