activeagent 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +158 -0
- data/lib/active_agent/evals/design_tokens.rb +130 -0
- data/lib/active_agent/evals/diagnosis.rb +238 -0
- data/lib/active_agent/evals/judge.rb +205 -0
- data/lib/active_agent/evals/model_spec.rb +80 -0
- data/lib/active_agent/evals/replay.rb +63 -0
- data/lib/active_agent/evals/report.rb +447 -0
- data/lib/active_agent/evals/report_html.rb +634 -0
- data/lib/active_agent/evals/result.rb +78 -0
- data/lib/active_agent/evals/runner.rb +149 -0
- data/lib/active_agent/evals/scenario.rb +68 -0
- data/lib/active_agent/evals/scenario_parser.rb +215 -0
- data/lib/active_agent/evals/scorer.rb +118 -0
- data/lib/active_agent/evals/suite.rb +99 -0
- data/lib/active_agent/evals.rb +60 -0
- data/lib/active_agent/providers/ruby_llm/options.rb +4 -0
- data/lib/active_agent/providers/ruby_llm_provider.rb +14 -1
- data/lib/active_agent/providers/rubyllm_provider.rb +1 -0
- data/lib/active_agent/telemetry/configuration.rb +11 -0
- data/lib/active_agent/telemetry/instrumentation.rb +26 -6
- data/lib/active_agent/version.rb +1 -1
- data/lib/active_agent.rb +1 -0
- metadata +22 -4
|
@@ -0,0 +1,634 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "cgi"
|
|
4
|
+
|
|
5
|
+
module ActiveAgent
|
|
6
|
+
module Evals
|
|
7
|
+
# Report#to_html: the run as one self-contained page on the dashboard's
|
|
8
|
+
# design system (DesignTokens) — inline styles, no scripts, no external
|
|
9
|
+
# assets — so a run's outcome can be archived next to a CI run or handed
|
|
10
|
+
# to a teammate, and so the dashboard can serve the same page for a
|
|
11
|
+
# persisted run.
|
|
12
|
+
#
|
|
13
|
+
# Same content as Report#to_markdown, laid out the way the dashboard's
|
|
14
|
+
# suite card is: header and stat tiles, the MODELS panel with the judge's
|
|
15
|
+
# pick and verdict, WHAT TO FIX cards from Report#fix_items, the
|
|
16
|
+
# SCENARIOS matrix, and a per-scenario disclosure with every answer.
|
|
17
|
+
module ReportHtml
|
|
18
|
+
THEMES = %w[light dark].freeze
|
|
19
|
+
ANSWER_LIMIT = 3_000
|
|
20
|
+
ERROR_LIMIT = 500
|
|
21
|
+
|
|
22
|
+
# @param theme [String, nil] "light" or "dark" pins the palette by putting
|
|
23
|
+
# `theme-light` / `theme-dark` on `<html>`; nil follows the viewer's
|
|
24
|
+
# `prefers-color-scheme`.
|
|
25
|
+
def to_html(theme: nil)
|
|
26
|
+
theme = theme.to_s.presence
|
|
27
|
+
theme = nil unless THEMES.include?(theme)
|
|
28
|
+
title = html_title
|
|
29
|
+
|
|
30
|
+
<<~HTML
|
|
31
|
+
<!doctype html>
|
|
32
|
+
<html lang="en"#{%( class="theme-#{theme}") if theme}>
|
|
33
|
+
<head>
|
|
34
|
+
<meta charset="utf-8">
|
|
35
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
36
|
+
<title>#{h(title)}</title>
|
|
37
|
+
<style>
|
|
38
|
+
#{html_styles}
|
|
39
|
+
</style>
|
|
40
|
+
</head>
|
|
41
|
+
<body>
|
|
42
|
+
<main class="page">
|
|
43
|
+
#{html_header(title)}
|
|
44
|
+
#{html_stat_tiles}
|
|
45
|
+
<section class="card">
|
|
46
|
+
#{html_models_panel}
|
|
47
|
+
#{html_fixes}
|
|
48
|
+
#{html_matrix}
|
|
49
|
+
#{html_details}
|
|
50
|
+
#{html_footer}
|
|
51
|
+
</section>
|
|
52
|
+
</main>
|
|
53
|
+
</body>
|
|
54
|
+
</html>
|
|
55
|
+
HTML
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
|
|
60
|
+
def h(value)
|
|
61
|
+
CGI.escapeHTML(value.to_s)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def plural(count, word)
|
|
65
|
+
"#{count} #{count == 1 ? word : word.pluralize}"
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def fault_name(fault)
|
|
69
|
+
fault.to_s.tr("_", " ")
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# ≥ 1.0 success, ≥ 0.7 warning, else error — the same thresholds the
|
|
73
|
+
# dashboard colors every pass ratio with.
|
|
74
|
+
def tone_for(ratio)
|
|
75
|
+
if ratio >= 1.0
|
|
76
|
+
"success"
|
|
77
|
+
elsif ratio >= 0.7
|
|
78
|
+
"warning"
|
|
79
|
+
else
|
|
80
|
+
"error"
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# "openrouter/meta-llama/llama-3.1-8b" → ["llama-3.1-8b", "openrouter/meta-llama"];
|
|
85
|
+
# a bare label reads its provider from the spec.
|
|
86
|
+
def split_label(spec)
|
|
87
|
+
label = spec.label.to_s
|
|
88
|
+
slash = label.rindex("/")
|
|
89
|
+
return [ label, spec.provider.to_s ] if slash.nil? || slash.zero?
|
|
90
|
+
|
|
91
|
+
[ label[(slash + 1)..], label[0...slash] ]
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def short_name(spec)
|
|
95
|
+
split_label(spec).first
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def model_by_label(label)
|
|
99
|
+
@models.find { |spec| spec.label == label } || ModelSpec.new(label: label, provider: "", model: label)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# The judge to name in the header chip and the footer: the judge that
|
|
103
|
+
# ran, the label a rebuilt run recorded, or the rules that scored it.
|
|
104
|
+
def judge_name
|
|
105
|
+
@judge_label || @judge&.label || "rules"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Who picked the best model, the way the dashboard's suite panel reads
|
|
109
|
+
# it: the verdict's judge — unless that is only the framework's
|
|
110
|
+
# pass-rate ranking, which is not a judge.
|
|
111
|
+
def judged_by
|
|
112
|
+
judge = verdict&.dig("judge").presence
|
|
113
|
+
judge && judge != Report::PASS_RATE_JUDGE ? judge : judge_name
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Results per scenario, in run order.
|
|
117
|
+
def scenario_cohorts
|
|
118
|
+
@scenario_cohorts ||= @results.group_by { |result| result.scenario.key }.values
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Consecutive scenarios that share a group, in run order.
|
|
122
|
+
def scenario_groups
|
|
123
|
+
@scenario_groups ||= scenario_cohorts.chunk_while { |a, b| a.first.scenario.group == b.first.scenario.group }.to_a
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def group_name(scenario)
|
|
127
|
+
scenario.group_name.presence || scenario.group.presence
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def anchor(scenario)
|
|
131
|
+
"scenario-#{scenario.key.to_s.gsub(/[^\w-]+/, '-')}"
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def fmt_k(value)
|
|
135
|
+
n = value.to_f
|
|
136
|
+
if n.abs >= 1_000_000
|
|
137
|
+
format("%.1fM", n / 1_000_000)
|
|
138
|
+
elsif n.abs >= 1_000
|
|
139
|
+
format("%.1fK", n / 1_000)
|
|
140
|
+
else
|
|
141
|
+
n.round.to_s
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def fmt_ms(ms)
|
|
146
|
+
return "—" if ms.nil?
|
|
147
|
+
|
|
148
|
+
n = ms.to_f
|
|
149
|
+
return "#{n.round}ms" if n < 1_000
|
|
150
|
+
|
|
151
|
+
"#{format("%.#{n >= 10_000 ? 1 : 2}f", n / 1_000).sub(/\.?0+\z/, '')}s"
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def fmt_cost(value)
|
|
155
|
+
value.nil? ? "—" : format("$%.4f", value)
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def fmt_score(value)
|
|
159
|
+
value.nil? ? "—" : format("%.2f", value)
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def fmt_mean_score(value)
|
|
163
|
+
value.nil? ? "—" : format("%.3f", value)
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# --- page ------------------------------------------------------------
|
|
167
|
+
|
|
168
|
+
def html_title
|
|
169
|
+
"Evaluation — #{plural(scenario_cohorts.size, 'scenario')} × #{plural(@models.size, 'model')}"
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
def html_styles
|
|
173
|
+
[
|
|
174
|
+
DesignTokens.css(scope: ":root", color_scheme: "light"),
|
|
175
|
+
"@media (prefers-color-scheme: dark) {",
|
|
176
|
+
DesignTokens.css(scope: ":root:not(.theme-light)", tokens: DesignTokens::DARK, color_scheme: "dark"),
|
|
177
|
+
"}",
|
|
178
|
+
DesignTokens.css(scope: ":root.theme-dark", tokens: DesignTokens::DARK, color_scheme: "dark"),
|
|
179
|
+
STYLES,
|
|
180
|
+
".mx { grid-template-columns: minmax(240px, 1.6fr) 150px repeat(#{@models.size}, minmax(170px, 1fr)); }",
|
|
181
|
+
".matrix .inner { min-width: #{390 + 185 * @models.size}px; }"
|
|
182
|
+
].join("\n")
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def html_header(title)
|
|
186
|
+
chips = @metadata.to_h.map { |key, value| html_chip(key, value) }
|
|
187
|
+
chips << html_chip("judge", judge_name)
|
|
188
|
+
|
|
189
|
+
<<~HEADER
|
|
190
|
+
<header>
|
|
191
|
+
<h1>#{h(title)}</h1>
|
|
192
|
+
<div class="chips">#{chips.join}</div>
|
|
193
|
+
</header>
|
|
194
|
+
HEADER
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def html_chip(key, value)
|
|
198
|
+
%(<span class="chip"><b>#{h(key)}</b>#{h(value)}</span>)
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def html_stat_tiles
|
|
202
|
+
total = @results.size
|
|
203
|
+
passed = @results.count(&:passed?)
|
|
204
|
+
ratio = total.positive? ? passed.to_f / total : 0.0
|
|
205
|
+
tiles = [
|
|
206
|
+
html_tile("Scenario runs", total, "#{plural(scenario_cohorts.size, 'scenario')} × #{plural(@models.size, 'model')}"),
|
|
207
|
+
html_tile("Pass rate", "#{(ratio * 100).round}%", "#{passed} / #{total} passed", tone: tone_for(ratio)),
|
|
208
|
+
html_tile("Open faults", total - passed, plural(fix_items.size, "fix item")),
|
|
209
|
+
html_tile("Models", @models.size, models_subline)
|
|
210
|
+
]
|
|
211
|
+
%(<section class="stats">#{tiles.join}</section>)
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
def html_tile(label, value, sub, tone: nil)
|
|
215
|
+
%(<div class="tile"><div class="micro">#{h(label)}</div>) +
|
|
216
|
+
%(<div class="value#{" tone-#{tone}" if tone}">#{h(value)}</div><div class="sub">#{h(sub)}</div></div>)
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def models_subline
|
|
220
|
+
if comparing? && winner
|
|
221
|
+
"judge's pick · #{short_name(model_by_label(winner))}"
|
|
222
|
+
else
|
|
223
|
+
@models.map { |spec| short_name(spec) }.join(" · ")
|
|
224
|
+
end
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# --- MODELS panel ----------------------------------------------------
|
|
228
|
+
|
|
229
|
+
def html_models_panel
|
|
230
|
+
blocks = summary_by_model.map { |label, stats| html_model_block(label, stats) }
|
|
231
|
+
verdict_row = verdict ? %(<div class="verdict"><span class="micro sm">Verdict</span>#{h(verdict['rationale'])}</div>) : ""
|
|
232
|
+
|
|
233
|
+
<<~PANEL
|
|
234
|
+
<div class="panel">
|
|
235
|
+
<div class="panel-head"><span class="micro">Models</span><span class="right">judged by #{h(judged_by)}</span></div>
|
|
236
|
+
#{blocks.join}
|
|
237
|
+
#{verdict_row}
|
|
238
|
+
</div>
|
|
239
|
+
PANEL
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
def html_model_block(label, stats)
|
|
243
|
+
short, provider = split_label(model_by_label(label))
|
|
244
|
+
total = stats["scenarios"]
|
|
245
|
+
ratio = total.positive? ? stats["passed"].to_f / total : 0.0
|
|
246
|
+
tone = tone_for(ratio)
|
|
247
|
+
pick = comparing? && winner == label ? %(<span class="badge info xs">judge's pick</span>) : ""
|
|
248
|
+
faults = stats["faults"].map { |fault, count| %(<span class="badge error">#{h(fault_name(fault))} ×#{count}</span>) }
|
|
249
|
+
faults_html = faults.any? ? faults.join : %(<span class="clean">[+] no faults</span>)
|
|
250
|
+
|
|
251
|
+
<<~BLOCK
|
|
252
|
+
<div class="model">
|
|
253
|
+
<div class="line"><span class="name">#{h(short)}</span><span class="provider">#{h(provider)}</span>#{pick}<span class="pass"><span class="bar bar-#{tone}"><span style="width:#{(ratio * 100).round}%"></span></span><span class="ratio tone-#{tone}">#{stats['passed']}/#{total}</span></span></div>
|
|
254
|
+
<div class="stats-line"><span>score <b>#{h(fmt_mean_score(stats['avg_score']))}</b></span><span>latency <b>#{h(fmt_ms(stats['avg_duration_ms']))}</b></span><span class="tok"><span class="in">in</span> #{h(fmt_k(stats['input_tokens']))} · <span class="out">out</span> #{h(fmt_k(stats['output_tokens']))}</span><span>cost <b>#{h(fmt_cost(stats['cost']))}</b></span></div>
|
|
255
|
+
<div class="faults">#{faults_html}</div>
|
|
256
|
+
</div>
|
|
257
|
+
BLOCK
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
# --- WHAT TO FIX -----------------------------------------------------
|
|
261
|
+
|
|
262
|
+
# The section stands even for a run with nothing to fix — the dashboard
|
|
263
|
+
# keeps it too, so a clean run reads as clean rather than as a page
|
|
264
|
+
# missing a section. A report over no results at all has nothing to say.
|
|
265
|
+
def html_fixes
|
|
266
|
+
return "" if @results.empty?
|
|
267
|
+
|
|
268
|
+
items = fix_items
|
|
269
|
+
faulted = @results.reject(&:passed?)
|
|
270
|
+
meta = "#{plural(items.size, 'item')} · #{plural(faulted.size, 'fault')} across " \
|
|
271
|
+
"#{plural(faulted.map { |result| result.scenario.key }.uniq.size, 'scenario')}"
|
|
272
|
+
body =
|
|
273
|
+
if items.any?
|
|
274
|
+
%(<div class="fixes">#{items.map { |item| html_fix_card(item) }.join}</div>)
|
|
275
|
+
else
|
|
276
|
+
%(<div class="nothing">[+] nothing to fix</div>)
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
<<~FIXES
|
|
280
|
+
<section class="section" aria-label="Recommendations">
|
|
281
|
+
<div class="section-head"><span class="micro">What to fix</span><span class="meta">#{h(meta)}</span></div>
|
|
282
|
+
#{body}
|
|
283
|
+
</section>
|
|
284
|
+
FIXES
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
def html_fix_card(item)
|
|
288
|
+
tone = item["kind"] == "instruction" ? "info" : "error"
|
|
289
|
+
glyph = tone == "info" ? "[i]" : "[!]"
|
|
290
|
+
title = fault_name(item["fault"]) + (item["count"].to_i > 1 ? " ×#{item['count']}" : "")
|
|
291
|
+
|
|
292
|
+
parts = [ %(<div class="head"><span class="glyph tone-#{tone}">#{glyph}</span>) +
|
|
293
|
+
%(<span class="badge #{tone}">#{h(title)}</span><span class="scope">#{h(fix_scope(item))}</span></div>) ]
|
|
294
|
+
parts << %(<p>#{h(item['recommendation'])}</p>) if item["recommendation"].present?
|
|
295
|
+
parts << %(<div class="quote">“#{h(item['quote'])}”</div>) if item["quote"].present?
|
|
296
|
+
parts << html_fix_tools(item) if item["tools"].any?
|
|
297
|
+
parts << html_fix_server(item["server"]) if item["server"]
|
|
298
|
+
parts << %(<div class="note">#{h(item['note'])}</div>) if item["note"].present?
|
|
299
|
+
parts << html_fix_action(item["action"]) if item["action"]
|
|
300
|
+
%(<div class="fix">#{parts.join}</div>)
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
def html_fix_tools(item)
|
|
304
|
+
chips = item["tools"].map do |tool|
|
|
305
|
+
note = tool["note"].presence
|
|
306
|
+
%(<span class="tool"><b>#{h(tool['name'])}</b>#{%(<span class="note">#{h(note)}</span>) if note}</span>)
|
|
307
|
+
end
|
|
308
|
+
%(<div class="tools"><span class="micro sm">#{h(item['tools_label'])}</span><div class="list">#{chips.join}</div></div>)
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
# "available · not enabled for Clara", "unknown · not enabled for
|
|
312
|
+
# Clara" — every status but "enabled" leads with the status word, the
|
|
313
|
+
# way the dashboard's fix list reads it.
|
|
314
|
+
def html_fix_server(server)
|
|
315
|
+
badge =
|
|
316
|
+
if server["status"] == "enabled"
|
|
317
|
+
%(<span class="badge success xs">enabled for #{h(@agent_name)}</span>)
|
|
318
|
+
else
|
|
319
|
+
%(<span class="badge warning xs">#{h(server['status'].presence || 'unknown')} · not enabled for #{h(@agent_name)}</span>)
|
|
320
|
+
end
|
|
321
|
+
%(<div class="served"><span>served by</span><b>#{h(server['name'].presence || server['key'])}</b>#{badge}</div>)
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
# With a route the action is a button; without one, the page can only
|
|
325
|
+
# say where in the dashboard the fix lives. The link targets the top
|
|
326
|
+
# window: served in the dashboard's report iframe it would otherwise
|
|
327
|
+
# open the whole dashboard inside the frame.
|
|
328
|
+
def html_fix_action(action)
|
|
329
|
+
button = action["path"].present? ? %(<a class="btn" target="_top" href="#{h(action['path'])}">#{h(action['label'])}</a>) : ""
|
|
330
|
+
%(<div class="action">#{button}<span class="hint">#{h(action['hint'])}</span></div>)
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
# "3 scenarios · both models" — the models are worth naming only on a
|
|
334
|
+
# comparison run; on a single-model run the count says it all.
|
|
335
|
+
def fix_scope(item)
|
|
336
|
+
return "#{item['scenario_keys'].join(', ')} · judge suggestion" if item["kind"] == "instruction"
|
|
337
|
+
|
|
338
|
+
scenarios = plural(item["scenario_keys"].size, "scenario")
|
|
339
|
+
labels = Array(item["models"])
|
|
340
|
+
return scenarios unless comparing? && labels.any?
|
|
341
|
+
|
|
342
|
+
models =
|
|
343
|
+
if labels.size >= @models.size
|
|
344
|
+
@models.size == 2 ? "both models" : "all models"
|
|
345
|
+
else
|
|
346
|
+
labels.map { |label| short_name(model_by_label(label)) }.join(", ")
|
|
347
|
+
end
|
|
348
|
+
"#{scenarios} · #{models}"
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
# --- SCENARIOS matrix ------------------------------------------------
|
|
352
|
+
|
|
353
|
+
def html_matrix
|
|
354
|
+
columns = @models.map do |spec|
|
|
355
|
+
short, provider = split_label(spec)
|
|
356
|
+
%(<span class="col"><span class="name">#{h(short)}</span><span class="provider">#{h(provider)}</span></span>)
|
|
357
|
+
end
|
|
358
|
+
rows = [ %(<div class="mx head"><span class="micro sm">Scenario</span><span class="micro sm">Expects</span>#{columns.join}</div>) ]
|
|
359
|
+
scenario_groups.each do |cohorts|
|
|
360
|
+
rows << html_group_row(cohorts) if group_name(cohorts.first.first.scenario)
|
|
361
|
+
cohorts.each { |cohort| rows << html_scenario_row(cohort) }
|
|
362
|
+
end
|
|
363
|
+
groups = scenario_groups.count { |cohorts| group_name(cohorts.first.first.scenario) }
|
|
364
|
+
meta = plural(scenario_cohorts.size, "scenario")
|
|
365
|
+
meta += " in #{plural(groups, 'group')}" if groups.positive?
|
|
366
|
+
|
|
367
|
+
<<~MATRIX
|
|
368
|
+
<section class="section">
|
|
369
|
+
<div class="section-head"><span class="micro">Scenarios</span><span class="meta">#{h(meta)}</span></div>
|
|
370
|
+
<div class="matrix"><div class="inner">#{rows.join}</div></div>
|
|
371
|
+
</section>
|
|
372
|
+
MATRIX
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
def html_group_row(cohorts)
|
|
376
|
+
name = group_name(cohorts.first.first.scenario)
|
|
377
|
+
passes = @models.map do |spec|
|
|
378
|
+
results = cohorts.filter_map { |cohort| cohort.find { |result| result.label == spec.label } }
|
|
379
|
+
passed = results.count(&:passed?)
|
|
380
|
+
tone =
|
|
381
|
+
if results.empty? then ""
|
|
382
|
+
elsif passed == results.size then " text-success"
|
|
383
|
+
elsif passed.zero? then " text-error"
|
|
384
|
+
else ""
|
|
385
|
+
end
|
|
386
|
+
%(<span class="group-pass#{tone}">#{passed}/#{results.size} passed</span>)
|
|
387
|
+
end
|
|
388
|
+
%(<div class="mx group"><span class="group-name">#{h(name)}</span>) +
|
|
389
|
+
%(<span class="count">#{h(plural(cohorts.size, 'scenario'))}</span>#{passes.join}</div>)
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
def html_scenario_row(cohort)
|
|
393
|
+
scenario = cohort.first.scenario
|
|
394
|
+
expects = scenario.expected_tools.map { |tool| %(<span class="expect">#{h(tool)}</span>) }.join
|
|
395
|
+
cells = @models.map do |spec|
|
|
396
|
+
result = cohort.find { |candidate| candidate.label == spec.label }
|
|
397
|
+
result ? html_result_cell(result) : %(<div class="cell"><div class="top"><span class="muted">—</span></div></div>)
|
|
398
|
+
end
|
|
399
|
+
%(<div class="mx"><div><div class="key"><a href="##{h(anchor(scenario))}">#{h(scenario.key)}</a></div>) +
|
|
400
|
+
%(<div class="prompt">#{h(scenario.prompt)}</div></div><div class="expects">#{expects}</div>#{cells.join}</div>)
|
|
401
|
+
end
|
|
402
|
+
|
|
403
|
+
def html_result_cell(result)
|
|
404
|
+
tone = result.passed? ? "success" : "error"
|
|
405
|
+
glyph = result.passed? ? "[+]" : "[!]"
|
|
406
|
+
fault = result.fault ? %(<span class="f">#{h(fault_name(result.fault))}</span>) : ""
|
|
407
|
+
%(<div class="cell"><div class="top"><span class="g tone-#{tone}">#{glyph}</span>) +
|
|
408
|
+
%(<span class="s tone-#{tone}">#{h(fmt_score(result.score))}</span>#{fault}</div>) +
|
|
409
|
+
%(<div class="calls">#{html_calls(result, empty: 'no tools called')}</div></div>)
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
def html_calls(result, empty:)
|
|
413
|
+
calls = tool_call_labels(result)
|
|
414
|
+
return %(<span>#{h(empty)}</span>) if calls.empty?
|
|
415
|
+
|
|
416
|
+
calls.map { |label, kind| %(<span#{%( class="#{kind}") if kind}>#{h(label)}</span>) }.join
|
|
417
|
+
end
|
|
418
|
+
|
|
419
|
+
# One [label, css class] per distinct call: a call of an expected tool
|
|
420
|
+
# is a hit, an errored call carries ` ✗`, repeats carry ` ×k`.
|
|
421
|
+
def tool_call_labels(result)
|
|
422
|
+
expected = result.scenario.expected_tools
|
|
423
|
+
result.replay.tool_calls.group_by { |call| [ call["name"].to_s, call["error"] ? true : false ] }.map do |(name, errored), calls|
|
|
424
|
+
label = name.dup
|
|
425
|
+
label << " ✗" if errored
|
|
426
|
+
label << " ×#{calls.size}" if calls.size > 1
|
|
427
|
+
kind = errored ? "call-err" : (expected.include?(name) ? "call-hit" : nil)
|
|
428
|
+
[ label, kind ]
|
|
429
|
+
end
|
|
430
|
+
end
|
|
431
|
+
|
|
432
|
+
# --- per-scenario details --------------------------------------------
|
|
433
|
+
|
|
434
|
+
def html_details
|
|
435
|
+
return "" if scenario_cohorts.empty?
|
|
436
|
+
|
|
437
|
+
<<~DETAILS
|
|
438
|
+
<section class="section">
|
|
439
|
+
<div class="section-head"><span class="micro">Details</span><span class="meta">answers and tool calls per scenario</span></div>
|
|
440
|
+
<div class="details">#{scenario_cohorts.map { |cohort| html_scenario_details(cohort) }.join}</div>
|
|
441
|
+
</section>
|
|
442
|
+
DETAILS
|
|
443
|
+
end
|
|
444
|
+
|
|
445
|
+
def html_scenario_details(cohort)
|
|
446
|
+
scenario = cohort.first.scenario
|
|
447
|
+
expects = scenario.expected_tools.any? ? %(<span class="exp">expects <b>#{h(scenario.expected_tools.join(' or '))}</b></span>) : ""
|
|
448
|
+
labels = @models.map(&:label)
|
|
449
|
+
cards = cohort.sort_by { |result| labels.index(result.label) || labels.size }.map { |result| html_result_card(result) }
|
|
450
|
+
|
|
451
|
+
<<~BLOCK
|
|
452
|
+
<div id="#{h(anchor(scenario))}">
|
|
453
|
+
<details>
|
|
454
|
+
<summary><span class="chev">></span><span class="key">#{h(scenario.key)}</span><span class="prompt">#{h(scenario.prompt)}</span>#{expects}</summary>
|
|
455
|
+
<div class="drill">#{cards.join}</div>
|
|
456
|
+
</details>
|
|
457
|
+
</div>
|
|
458
|
+
BLOCK
|
|
459
|
+
end
|
|
460
|
+
|
|
461
|
+
def html_result_card(result)
|
|
462
|
+
short, = split_label(result.spec)
|
|
463
|
+
status =
|
|
464
|
+
if result.passed? then %(<span class="badge success">passed · #{h(fmt_score(result.score))}</span>)
|
|
465
|
+
elsif result.errored? then %(<span class="badge error">errored</span>)
|
|
466
|
+
else %(<span class="badge error">failed · #{h(fmt_score(result.score))}</span>)
|
|
467
|
+
end
|
|
468
|
+
fault = result.fault ? %(<div class="fault-box"><b>[!] #{h(fault_name(result.fault))}</b> — #{h(result.summary)} #{h(result.recommendation)}</div>) : ""
|
|
469
|
+
error = result.replay.error ? %(<div class="error-line">Error: #{h(result.replay.error.to_s.truncate(ERROR_LIMIT))}</div>) : ""
|
|
470
|
+
answer =
|
|
471
|
+
if result.replay.answer.present?
|
|
472
|
+
%(<div class="answer">#{h(result.replay.answer.to_s.truncate(ANSWER_LIMIT))}</div>)
|
|
473
|
+
else
|
|
474
|
+
%(<div class="no-answer">answer not retained for this run</div>)
|
|
475
|
+
end
|
|
476
|
+
|
|
477
|
+
<<~CARD
|
|
478
|
+
<div class="result">
|
|
479
|
+
<div class="head"><span class="name">#{h(short)}</span>#{status}<span class="meta">#{h(result_meta(result))}</span></div>
|
|
480
|
+
<div class="body"><div class="tools-line"><span class="micro sm">Tools</span>#{html_calls(result, empty: 'none called')}</div>#{fault}#{error}#{answer}</div>
|
|
481
|
+
</div>
|
|
482
|
+
CARD
|
|
483
|
+
end
|
|
484
|
+
|
|
485
|
+
def result_meta(result)
|
|
486
|
+
replay = result.replay
|
|
487
|
+
[
|
|
488
|
+
replay.duration_ms && fmt_ms(replay.duration_ms),
|
|
489
|
+
replay.total_tokens.positive? ? "#{fmt_k(replay.total_tokens)} tokens" : nil,
|
|
490
|
+
replay.cost && fmt_cost(replay.cost)
|
|
491
|
+
].compact.join(" · ")
|
|
492
|
+
end
|
|
493
|
+
|
|
494
|
+
# --- footer ----------------------------------------------------------
|
|
495
|
+
|
|
496
|
+
def html_footer
|
|
497
|
+
criteria = criterion_keys.map { |key| key.to_s.tr("_", " ") }.join(" · ")
|
|
498
|
+
spans = [ %(<span class="nowrap">judge #{h(judge_name)}</span>) ]
|
|
499
|
+
spans << %(<span class="criteria">criteria #{h(criteria)}</span>) if criteria.present?
|
|
500
|
+
spans.concat(@metadata.to_h.map { |key, value| %(<span class="nowrap">#{h(key)} #{h(value)}</span>) })
|
|
501
|
+
%(<footer>#{spans.join}</footer>)
|
|
502
|
+
end
|
|
503
|
+
|
|
504
|
+
# Colors only through the token variables; radii 4 badges · 6 chips ·
|
|
505
|
+
# 8 controls · 10 nested panels · 12 cards · 999 bars; no shadows.
|
|
506
|
+
STYLES = <<~CSS.freeze
|
|
507
|
+
* { box-sizing: border-box; }
|
|
508
|
+
html, body { margin: 0; min-height: 100%; }
|
|
509
|
+
body { background: var(--color-background); color: var(--color-text-primary); font-family: var(--font-text); font-size: 13px; line-height: 1.45; -webkit-font-smoothing: antialiased; }
|
|
510
|
+
a { color: var(--color-info); text-decoration: none; }
|
|
511
|
+
a:hover { color: var(--color-info-text); text-decoration: underline; }
|
|
512
|
+
.page { max-width: 1440px; margin: 0 auto; padding: 24px; display: flex; flex-direction: column; gap: 20px; }
|
|
513
|
+
.micro { font-family: var(--font-mono); font-size: 11px; font-weight: 600; letter-spacing: 0.06em; text-transform: uppercase; color: var(--color-text-secondary); }
|
|
514
|
+
.micro.sm { font-size: 10px; color: var(--color-text-muted); }
|
|
515
|
+
.muted { color: var(--color-text-muted); }
|
|
516
|
+
h1 { margin: 0; font-size: 24px; font-weight: 700; letter-spacing: -0.01em; }
|
|
517
|
+
.chips { display: flex; flex-wrap: wrap; gap: 6px; margin-top: 8px; }
|
|
518
|
+
.chip { display: inline-flex; align-items: center; gap: 5px; padding: 2px 8px; border-radius: 6px; background: var(--color-muted); font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
|
|
519
|
+
.chip b { font-weight: 600; color: var(--color-text-secondary); }
|
|
520
|
+
.badge { display: inline-flex; align-items: center; padding: 2px 7px; border-radius: 4px; font-family: var(--font-mono); font-size: 11px; font-weight: 600; white-space: nowrap; }
|
|
521
|
+
.badge.xs { padding: 1px 6px; font-size: 10px; }
|
|
522
|
+
.badge.success { background: var(--color-success-soft); color: var(--color-success-text); }
|
|
523
|
+
.badge.warning { background: var(--color-warning-soft); color: var(--color-warning-text); }
|
|
524
|
+
.badge.error { background: var(--color-error-soft); color: var(--color-error-text); }
|
|
525
|
+
.badge.info { background: var(--color-info-soft); color: var(--color-info-text); }
|
|
526
|
+
.tone-success { color: var(--color-success); }
|
|
527
|
+
.tone-warning { color: var(--color-warning); }
|
|
528
|
+
.tone-error { color: var(--color-error); }
|
|
529
|
+
.tone-info { color: var(--color-info); }
|
|
530
|
+
.text-success { color: var(--color-success-text); }
|
|
531
|
+
.text-error { color: var(--color-error-text); }
|
|
532
|
+
.stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(190px, 1fr)); gap: 16px; }
|
|
533
|
+
.tile { background: var(--color-card); border: 1px solid var(--color-border); border-radius: 12px; padding: 20px; }
|
|
534
|
+
.tile .value { margin-top: 8px; font-family: var(--font-mono); font-size: 32px; font-weight: 700; line-height: 1.1; }
|
|
535
|
+
.tile .sub { margin-top: 8px; font-size: 13px; color: var(--color-text-secondary); }
|
|
536
|
+
.card { background: var(--color-card); border: 1px solid var(--color-border); border-radius: 12px; padding: 16px; display: flex; flex-direction: column; gap: 16px; }
|
|
537
|
+
.section { display: flex; flex-direction: column; gap: 10px; }
|
|
538
|
+
.section-head { display: flex; align-items: baseline; gap: 10px; flex-wrap: wrap; }
|
|
539
|
+
.section-head .meta { font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
|
|
540
|
+
.panel { border: 1px solid var(--color-border-light); border-radius: 10px; overflow: hidden; }
|
|
541
|
+
.panel-head { display: flex; align-items: center; gap: 10px; padding: 8px 12px; background: var(--color-muted); }
|
|
542
|
+
.panel-head .right { margin-left: auto; font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
|
|
543
|
+
.model { padding: 10px 12px; border-top: 1px solid var(--color-border-light); display: flex; flex-direction: column; gap: 6px; }
|
|
544
|
+
.model .line { display: flex; align-items: center; gap: 10px; flex-wrap: wrap; }
|
|
545
|
+
.model .name { font-family: var(--font-mono); font-size: 12px; font-weight: 600; }
|
|
546
|
+
.model .provider { font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
|
|
547
|
+
.model .pass { margin-left: auto; display: flex; align-items: center; gap: 10px; }
|
|
548
|
+
.bar { display: inline-block; width: 120px; height: 6px; border-radius: 999px; background: var(--color-muted); overflow: hidden; }
|
|
549
|
+
.bar span { display: block; height: 100%; border-radius: 999px; }
|
|
550
|
+
.bar-success span { background: var(--color-success); }
|
|
551
|
+
.bar-warning span { background: var(--color-warning); }
|
|
552
|
+
.bar-error span { background: var(--color-error); }
|
|
553
|
+
.ratio { font-family: var(--font-mono); font-size: 12px; font-weight: 600; }
|
|
554
|
+
.stats-line { display: flex; gap: 14px; flex-wrap: wrap; font-family: var(--font-mono); font-size: 11px; color: var(--color-text-secondary); }
|
|
555
|
+
.stats-line b { font-weight: 600; color: var(--color-text-primary); }
|
|
556
|
+
.tok { white-space: nowrap; }
|
|
557
|
+
.tok .in { color: var(--color-token-in); }
|
|
558
|
+
.tok .out { color: var(--color-token-out); }
|
|
559
|
+
.faults { display: flex; gap: 6px; flex-wrap: wrap; }
|
|
560
|
+
.clean { font-family: var(--font-mono); font-size: 11px; color: var(--color-success-text); }
|
|
561
|
+
.verdict { padding: 10px 12px; border-top: 1px solid var(--color-border-light); font-size: 12px; line-height: 18px; color: var(--color-text-cell); }
|
|
562
|
+
.verdict .micro { margin-right: 8px; }
|
|
563
|
+
.fixes { display: grid; grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); gap: 12px; }
|
|
564
|
+
.nothing { border: 1px solid var(--color-border-light); border-radius: 10px; padding: 14px 12px; text-align: center; font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
|
|
565
|
+
.fix { border: 1px solid var(--color-border); border-radius: 10px; padding: 12px 14px; display: flex; flex-direction: column; gap: 10px; min-width: 0; }
|
|
566
|
+
.fix .head { display: flex; align-items: center; gap: 8px; flex-wrap: wrap; }
|
|
567
|
+
.fix .glyph { font-family: var(--font-mono); font-size: 12px; font-weight: 700; }
|
|
568
|
+
.fix .scope { font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
|
|
569
|
+
.fix p { margin: 0; font-size: 13px; line-height: 19px; color: var(--color-text-cell); }
|
|
570
|
+
.fix .quote { background: var(--color-muted); border-radius: 8px; padding: 8px 10px; font-size: 12px; line-height: 18px; color: var(--color-text-cell); font-style: italic; }
|
|
571
|
+
.tools { display: flex; flex-direction: column; gap: 6px; }
|
|
572
|
+
.tools .list { display: flex; flex-wrap: wrap; gap: 6px; }
|
|
573
|
+
.tool { display: inline-flex; align-items: center; gap: 6px; padding: 3px 8px; border-radius: 6px; border: 1px solid var(--color-border); font-family: var(--font-mono); font-size: 11px; max-width: 100%; }
|
|
574
|
+
.tool b { font-weight: 600; color: var(--color-text-primary); }
|
|
575
|
+
.tool .note { color: var(--color-text-muted); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
|
576
|
+
.served { display: flex; align-items: center; gap: 8px; flex-wrap: wrap; font-size: 12px; color: var(--color-text-cell); }
|
|
577
|
+
.served b { font-weight: 600; color: var(--color-text-primary); }
|
|
578
|
+
.fix .note { font-size: 12px; line-height: 18px; color: var(--color-text-secondary); }
|
|
579
|
+
.action { display: flex; align-items: center; gap: 10px; margin-top: auto; padding-top: 2px; flex-wrap: wrap; }
|
|
580
|
+
.btn { display: inline-block; padding: 6px 12px; border-radius: 8px; font-size: 13px; font-weight: 500; color: var(--color-text-cell); border: 1px solid var(--color-border-strong); background: transparent; white-space: nowrap; }
|
|
581
|
+
.btn:hover { background: var(--color-hover); color: var(--color-text-primary); text-decoration: none; }
|
|
582
|
+
.hint { font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); white-space: nowrap; }
|
|
583
|
+
.matrix { border: 1px solid var(--color-border-light); border-radius: 10px; overflow-x: auto; }
|
|
584
|
+
.mx { display: grid; gap: 12px; padding: 10px 12px; border-top: 1px solid var(--color-border-light); }
|
|
585
|
+
.mx.head { padding: 8px 12px; background: var(--color-muted); align-items: end; border-top: 0; }
|
|
586
|
+
.mx.group { padding: 7px 12px; background: var(--color-background); align-items: center; }
|
|
587
|
+
.mx .col { display: flex; flex-direction: column; gap: 1px; min-width: 0; }
|
|
588
|
+
.mx .col .name { font-family: var(--font-mono); font-size: 11px; font-weight: 600; color: var(--color-text-primary); }
|
|
589
|
+
.mx .col .provider { font-family: var(--font-mono); font-size: 10px; color: var(--color-text-muted); }
|
|
590
|
+
.group-name { font-size: 12px; font-weight: 600; }
|
|
591
|
+
.count { font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
|
|
592
|
+
.group-pass { font-family: var(--font-mono); font-size: 11px; font-weight: 600; color: var(--color-text-cell); }
|
|
593
|
+
.key { font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); margin-bottom: 2px; }
|
|
594
|
+
.key a { color: inherit; }
|
|
595
|
+
.prompt { font-size: 13px; line-height: 18px; color: var(--color-text-primary); }
|
|
596
|
+
.expects { display: flex; flex-wrap: wrap; gap: 4px; align-content: flex-start; min-width: 0; }
|
|
597
|
+
.expect { font-family: var(--font-mono); font-size: 11px; padding: 2px 6px; border: 1px solid var(--color-border); border-radius: 4px; color: var(--color-text-cell); white-space: nowrap; }
|
|
598
|
+
.cell { min-width: 0; display: flex; flex-direction: column; gap: 3px; }
|
|
599
|
+
.cell .top { display: flex; align-items: baseline; gap: 6px; flex-wrap: wrap; font-family: var(--font-mono); font-size: 12px; }
|
|
600
|
+
.cell .top .g { font-weight: 700; }
|
|
601
|
+
.cell .top .s { font-weight: 600; }
|
|
602
|
+
.cell .top .f { font-size: 11px; color: var(--color-error-text); }
|
|
603
|
+
.calls { display: flex; flex-wrap: wrap; gap: 2px 8px; font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
|
|
604
|
+
.call-hit { color: var(--color-success-text); font-weight: 600; }
|
|
605
|
+
.call-err { color: var(--color-error); font-weight: 600; }
|
|
606
|
+
.details { display: flex; flex-direction: column; gap: 8px; }
|
|
607
|
+
details { border: 1px solid var(--color-border-light); border-radius: 10px; overflow: hidden; }
|
|
608
|
+
summary { display: flex; align-items: center; gap: 10px; padding: 10px 12px; cursor: pointer; list-style: none; flex-wrap: wrap; }
|
|
609
|
+
summary::-webkit-details-marker { display: none; }
|
|
610
|
+
summary:hover { background: var(--color-hover); }
|
|
611
|
+
summary .chev { font-family: var(--font-mono); font-size: 12px; color: var(--color-text-muted); display: inline-block; transition: transform 0.15s ease; }
|
|
612
|
+
details[open] > summary .chev { transform: rotate(90deg); }
|
|
613
|
+
summary .key { margin: 0; }
|
|
614
|
+
summary .exp { margin-left: auto; font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
|
|
615
|
+
summary .exp b { font-weight: 600; color: var(--color-text-primary); }
|
|
616
|
+
.drill { border-top: 1px solid var(--color-border-light); background: var(--color-background); padding: 12px; display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 12px; }
|
|
617
|
+
.result { background: var(--color-card); border: 1px solid var(--color-border-light); border-radius: 10px; overflow: hidden; min-width: 0; }
|
|
618
|
+
.result .head { display: flex; align-items: center; gap: 8px; padding: 8px 12px; border-bottom: 1px solid var(--color-border-light); flex-wrap: wrap; }
|
|
619
|
+
.result .head .name { font-family: var(--font-mono); font-size: 12px; font-weight: 600; }
|
|
620
|
+
.result .head .meta { margin-left: auto; font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
|
|
621
|
+
.result .body { padding: 10px 12px; display: flex; flex-direction: column; gap: 10px; }
|
|
622
|
+
.tools-line { display: flex; align-items: baseline; gap: 8px; flex-wrap: wrap; font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
|
|
623
|
+
.fault-box { background: var(--color-error-soft); border-radius: 8px; padding: 8px 10px; font-size: 12px; line-height: 18px; color: var(--color-error-text); }
|
|
624
|
+
.fault-box b { font-family: var(--font-mono); font-weight: 700; }
|
|
625
|
+
.error-line { font-family: var(--font-mono); font-size: 11px; color: var(--color-error); overflow-wrap: anywhere; }
|
|
626
|
+
.answer { font-family: var(--font-text); font-size: 13px; line-height: 19px; color: var(--color-text-cell); max-height: 190px; overflow: auto; white-space: pre-wrap; overflow-wrap: anywhere; }
|
|
627
|
+
.no-answer { font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
|
|
628
|
+
footer { display: flex; align-items: center; gap: 16px; flex-wrap: wrap; padding-top: 12px; border-top: 1px solid var(--color-border-light); font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
|
|
629
|
+
footer .criteria { min-width: 0; }
|
|
630
|
+
footer .nowrap { white-space: nowrap; }
|
|
631
|
+
CSS
|
|
632
|
+
end
|
|
633
|
+
end
|
|
634
|
+
end
|