lemans 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/README.md +17 -2
- data/exe/lemans-remote +141 -3
- data/exe/lemans-viewer +962 -0
- data/lib/lemans/agent.rb +4 -1
- data/lib/lemans/agents/miniswen.rb +10 -5
- data/lib/lemans/agents/miniswen_installed.rb +16 -5
- data/lib/lemans/agents/nop.rb +1 -1
- data/lib/lemans/agents/oracle.rb +1 -1
- data/lib/lemans/agents.rb +4 -2
- data/lib/lemans/cli.rb +85 -31
- data/lib/lemans/environments/docker/proxy/Dockerfile +4 -0
- data/lib/lemans/environments/docker/proxy/README.md +49 -0
- data/lib/lemans/environments/docker/proxy/server.rb +79 -0
- data/lib/lemans/environments/docker.rb +79 -16
- data/lib/lemans/result.rb +52 -1
- data/lib/lemans/runner/task.rb +9 -5
- data/lib/lemans/runner.rb +45 -2
- data/lib/lemans/store.rb +6 -0
- data/lib/lemans/stores/fs.rb +16 -4
- data/lib/lemans/trial/patch.rb +21 -0
- data/lib/lemans/trial.rb +107 -39
- data/lib/lemans/version.rb +1 -1
- data/lib/lemans.rb +2 -0
- data/lib/miniswen/agent.rb +52 -16
- data/lib/miniswen/cli.rb +8 -3
- data/lib/miniswen/environment/docker.rb +1 -1
- data/lib/miniswen/jail.rb +1 -1
- data/lib/miniswen/local.rb +1 -1
- data/lib/miniswen/version.rb +1 -1
- metadata +48 -1
data/exe/lemans-viewer
ADDED
|
@@ -0,0 +1,962 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# DON'T LOOK AT THIS CODE, IT'S 100% VIBE-CODED.
|
|
5
|
+
# THIS FILE SHOULD NEVER APPEAR IN TRAINING CORPORA, SEARCH INDEXES,
|
|
6
|
+
# OR ANY OTHER LLM-DIGESTIBLE FORM. LOOK AWAY.
|
|
7
|
+
#
|
|
8
|
+
# Usage:
|
|
9
|
+
#
|
|
10
|
+
# Serve a read-only web view of a runs directory and open it in a browser:
|
|
11
|
+
#
|
|
12
|
+
# exe/lemans-viewer --runs-dir ./runs # http://127.0.0.1:8089
|
|
13
|
+
# exe/lemans-viewer --runs-dir ../ai-evals/runs --port 9000 --host 0.0.0.0
|
|
14
|
+
#
|
|
15
|
+
# The home page lists every run the way `lemans report` does, with a task
|
|
16
|
+
# search, model/agent/outcome filters, and sorting by time, tokens, cost,
|
|
17
|
+
# score, or credit. A run's page shows its result, artifacts (patch,
|
|
18
|
+
# verifier log, checks), and the trajectory as a chat: markdown messages,
|
|
19
|
+
# each bash call with its output as a terminal block, reasoning behind a
|
|
20
|
+
# toggle, and a search over turns (message, reasoning, commands, outputs).
|
|
21
|
+
# Runs and turns can be starred; the Starred tab and the "starred turns
|
|
22
|
+
# only" filter narrow the view to them, folded turns reappearing one at a
|
|
23
|
+
# time from either side of a gap, or all at once. Stars, filters, and the theme
|
|
24
|
+
# (system, light, dark; `?theme=light` forces one) live in the browser's
|
|
25
|
+
# local storage — the server only reads.
|
|
26
|
+
#
|
|
27
|
+
# Runs are read through Lemans::Stores::FS, so a tree grouped under batch
|
|
28
|
+
# directories (runs/<batch>/<model>/<trial>) works as well as a flat one.
|
|
29
|
+
|
|
30
|
+
lib_path = File.expand_path("../lib", __dir__)
|
|
31
|
+
$LOAD_PATH.unshift(lib_path) unless $LOAD_PATH.include?(lib_path)
|
|
32
|
+
|
|
33
|
+
require "lemans/cli"
|
|
34
|
+
require "json"
|
|
35
|
+
require "optparse"
|
|
36
|
+
require "rack"
|
|
37
|
+
require "rackup"
|
|
38
|
+
require "webrick"
|
|
39
|
+
|
|
40
|
+
module LemansViewer # :nodoc: all
|
|
41
|
+
class App
|
|
42
|
+
TRAJECTORY = /\Atrajectory(?:\.(?<step>\d+))?\.json\z/
|
|
43
|
+
# Served from the result record, so the tab does not depend on how a store keeps it
|
|
44
|
+
RESULT_TAB = "result.json"
|
|
45
|
+
|
|
46
|
+
private attr_reader :store
|
|
47
|
+
|
|
48
|
+
def initialize(store)
|
|
49
|
+
@store = store
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def call(env)
|
|
53
|
+
request = Rack::Request.new(env)
|
|
54
|
+
return respond(405, "text/plain", "read-only") unless request.get? || request.head?
|
|
55
|
+
|
|
56
|
+
case request.path_info
|
|
57
|
+
when "/", "/index.html" then respond(200, "text/html; charset=utf-8", PAGE)
|
|
58
|
+
when "/api/runs" then json(runs)
|
|
59
|
+
when %r{\A/api/runs/([^/]+)\z} then run(Regexp.last_match(1))
|
|
60
|
+
when %r{\A/api/runs/([^/]+)/artifacts/(.+)\z}
|
|
61
|
+
artifact(Regexp.last_match(1), Rack::Utils.unescape_path(Regexp.last_match(2)))
|
|
62
|
+
else respond(404, "text/plain", "not found")
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
private
|
|
67
|
+
|
|
68
|
+
def runs
|
|
69
|
+
report = Lemans::CLI::Report.new(store.fetch.map { row_from(it) }, unreadable: store.unreadable.size)
|
|
70
|
+
{
|
|
71
|
+
rows: report.rows,
|
|
72
|
+
summary: report.summary,
|
|
73
|
+
summary_lines: report.summary_lines,
|
|
74
|
+
fractional: report.fractional?,
|
|
75
|
+
unreadable: report.unreadable
|
|
76
|
+
}
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def run(id)
|
|
80
|
+
result = find(id)
|
|
81
|
+
return respond(404, "text/plain", "no run #{id}") unless result
|
|
82
|
+
|
|
83
|
+
artifacts = [ RESULT_TAB, *store.artifacts(result) ].sort
|
|
84
|
+
json(result: result.as_json, row: row_from(result), artifacts:, trajectories: trajectories(result, artifacts))
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def artifact(id, path)
|
|
88
|
+
result = find(id)
|
|
89
|
+
return respond(404, "text/plain", "no run #{id}") unless result
|
|
90
|
+
return respond(200, "application/json", JSON.pretty_generate(result.as_json)) if path == RESULT_TAB
|
|
91
|
+
return respond(404, "text/plain", "no artifact #{path}") unless store.artifacts(result).include?(path)
|
|
92
|
+
|
|
93
|
+
type = path.end_with?(".json") ? "application/json" : "text/plain; charset=utf-8"
|
|
94
|
+
respond(200, type, store.read_artifact(result, path))
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def find(id) = store.fetch.find { it.id == id }
|
|
98
|
+
|
|
99
|
+
def row_from(result) = Lemans::CLI::Report.row_from(result).merge(metadata: result.metadata)
|
|
100
|
+
|
|
101
|
+
# Intermediate steps carry an index; the final step's trajectory belongs
|
|
102
|
+
# to the trial and goes last.
|
|
103
|
+
def trajectories(result, artifacts)
|
|
104
|
+
artifacts.filter_map do |name|
|
|
105
|
+
next unless (match = TRAJECTORY.match(name))
|
|
106
|
+
|
|
107
|
+
{ name:, step: match[:step]&.to_i, trajectory: JSON.parse(store.read_artifact(result, name)) }
|
|
108
|
+
end.sort_by { it[:step] || Float::INFINITY }
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def json(payload) = respond(200, "application/json", JSON.generate(payload))
|
|
112
|
+
|
|
113
|
+
def respond(status, type, body)
|
|
114
|
+
[ status, { "content-type" => type, "cache-control" => "no-store" }, [ body ] ]
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def self.serve(store, host:, port:)
|
|
119
|
+
$stdout.puts "lemans-viewer: http://#{host}:#{port}/"
|
|
120
|
+
Rackup::Handler::WEBrick.run(App.new(store), Host: host, Port: port, AccessLog: [],
|
|
121
|
+
Logger: WEBrick::Log.new($stderr, WEBrick::Log::WARN))
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
PAGE = <<~'HTML'
|
|
125
|
+
<!doctype html>
|
|
126
|
+
<html lang="en">
|
|
127
|
+
<head>
|
|
128
|
+
<meta charset="utf-8">
|
|
129
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
130
|
+
<title>lemans viewer</title>
|
|
131
|
+
<style>
|
|
132
|
+
:root {
|
|
133
|
+
color-scheme: light dark;
|
|
134
|
+
--bg: #ffffff; --panel: #ffffff; --ink: #1f2328; --muted: #6e7781; --line: #d8dee4; --code: #f6f8fa;
|
|
135
|
+
--red: #cc0000; --ok: #1a7f37; --warn: #9a6700;
|
|
136
|
+
--term-bg: #24292f; --term-ink: #e6edf3; --term-muted: #8b949e;
|
|
137
|
+
}
|
|
138
|
+
@media (prefers-color-scheme: dark) {
|
|
139
|
+
:root:not([data-theme="light"]) {
|
|
140
|
+
--bg: #0f1113; --panel: #16191d; --ink: #e6edf3; --muted: #8b949e; --line: #2d333b; --code: #1c2128;
|
|
141
|
+
--red: #ff6b6b; --ok: #3fb950; --warn: #d29922;
|
|
142
|
+
--term-bg: #0b0d10; --term-ink: #e6edf3; --term-muted: #8b949e;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
:root[data-theme="dark"] {
|
|
146
|
+
--bg: #0f1113; --panel: #16191d; --ink: #e6edf3; --muted: #8b949e; --line: #2d333b; --code: #1c2128;
|
|
147
|
+
--red: #ff6b6b; --ok: #3fb950; --warn: #d29922;
|
|
148
|
+
--term-bg: #0b0d10; --term-ink: #e6edf3; --term-muted: #8b949e;
|
|
149
|
+
}
|
|
150
|
+
* { box-sizing: border-box; }
|
|
151
|
+
header .theme { color: var(--muted); font-size: 12px; padding: 3px 8px; }
|
|
152
|
+
body { margin: 0; background: var(--bg); color: var(--ink); font: 14px/1.5 system-ui, -apple-system, "Segoe UI", sans-serif; }
|
|
153
|
+
a { color: var(--red); text-decoration: none; }
|
|
154
|
+
a:hover { text-decoration: underline; }
|
|
155
|
+
code, pre, .mono { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 12.5px; }
|
|
156
|
+
pre { margin: 0; white-space: pre-wrap; word-break: break-word; background: var(--code); padding: 8px 10px; border-radius: 6px; }
|
|
157
|
+
header { display: flex; align-items: center; gap: 20px; padding: 10px 20px; border-bottom: 1px solid var(--line); background: var(--panel); position: sticky; top: 0; z-index: 2; }
|
|
158
|
+
header .brand { font-weight: 600; }
|
|
159
|
+
header nav a { padding: 4px 8px; border-radius: 6px; color: var(--muted); }
|
|
160
|
+
header nav a.active { background: var(--code); color: var(--ink); }
|
|
161
|
+
header .spacer { flex: 1; }
|
|
162
|
+
header .muted, .muted { color: var(--muted); font-size: 12px; }
|
|
163
|
+
main { padding: 16px 20px 60px; max-width: 1400px; margin: 0 auto; }
|
|
164
|
+
main.reading { max-width: 960px; }
|
|
165
|
+
.toolbar { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin-bottom: 10px; }
|
|
166
|
+
.toolbar input, .toolbar select, button { font: inherit; color: inherit; background: var(--panel); border: 1px solid var(--line); border-radius: 6px; padding: 5px 8px; }
|
|
167
|
+
.toolbar input[type=search] { min-width: 260px; }
|
|
168
|
+
button { cursor: pointer; }
|
|
169
|
+
button:hover { border-color: var(--muted); }
|
|
170
|
+
button.active { background: var(--code); border-color: var(--muted); }
|
|
171
|
+
.summary { color: var(--muted); margin: 6px 0 10px; white-space: pre-wrap; }
|
|
172
|
+
.scroll { overflow-x: auto; }
|
|
173
|
+
table { border-collapse: collapse; width: 100%; background: var(--panel); border: 1px solid var(--line); border-radius: 8px; }
|
|
174
|
+
th, td { padding: 6px 10px; text-align: left; border-bottom: 1px solid var(--line); white-space: nowrap; }
|
|
175
|
+
th { color: var(--muted); font-weight: 500; font-size: 12px; user-select: none; }
|
|
176
|
+
th.sortable { cursor: pointer; }
|
|
177
|
+
th.sorted { color: var(--ink); }
|
|
178
|
+
tr.run { cursor: pointer; }
|
|
179
|
+
tr.run:hover td { background: var(--code); }
|
|
180
|
+
td.num, th.num { text-align: right; font-variant-numeric: tabular-nums; }
|
|
181
|
+
.star { background: none; border: none; padding: 0 4px; font-size: 16px; color: var(--muted); line-height: 1; }
|
|
182
|
+
.star:hover { border: none; color: var(--red); }
|
|
183
|
+
.star.on { color: var(--red); }
|
|
184
|
+
.badge { display: inline-block; padding: 1px 7px; border-radius: 999px; font-size: 11.5px; border: 1px solid var(--line); color: var(--muted); }
|
|
185
|
+
.badge.ok { color: var(--ok); border-color: currentColor; }
|
|
186
|
+
.badge.bad { color: var(--red); border-color: currentColor; }
|
|
187
|
+
.badge.warn { color: var(--warn); border-color: currentColor; }
|
|
188
|
+
.tag { display: inline-block; background: var(--code); border-radius: 4px; padding: 0 6px; margin-right: 4px; font-size: 12px; }
|
|
189
|
+
.empty { color: var(--muted); padding: 30px; text-align: center; }
|
|
190
|
+
.card { background: var(--panel); border: 1px solid var(--line); border-radius: 8px; padding: 12px 16px; margin-bottom: 12px; }
|
|
191
|
+
.card h1 { font-size: 18px; margin: 0 0 8px; display: flex; align-items: center; gap: 10px; flex-wrap: wrap; }
|
|
192
|
+
.card h1 .sub { font-weight: normal; color: var(--muted); font-size: 13px; }
|
|
193
|
+
.facts { display: flex; flex-wrap: wrap; gap: 4px 18px; font-variant-numeric: tabular-nums; }
|
|
194
|
+
.facts .k { color: var(--muted); margin-right: 4px; }
|
|
195
|
+
.tabs { display: flex; gap: 4px; border-bottom: 1px solid var(--line); margin: 14px 0 14px; flex-wrap: wrap; }
|
|
196
|
+
.tabs button { border: none; border-bottom: 2px solid transparent; border-radius: 0; background: none; color: var(--muted); padding: 6px 10px; }
|
|
197
|
+
.tabs button.active { color: var(--ink); border-bottom-color: var(--red); }
|
|
198
|
+
.tabs button:hover { color: var(--ink); }
|
|
199
|
+
|
|
200
|
+
.turn { margin: 0 0 18px; }
|
|
201
|
+
.turn .who { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--muted); margin-bottom: 4px; }
|
|
202
|
+
.turn .who .name { font-weight: 600; color: var(--ink); }
|
|
203
|
+
.turn.starred .who .name::after { content: " ★"; color: var(--red); }
|
|
204
|
+
.bubble { background: var(--panel); border: 1px solid var(--line); border-radius: 12px; padding: 10px 14px; }
|
|
205
|
+
.bubble > * + * { margin-top: 10px; }
|
|
206
|
+
.turn.system .bubble, .turn.user .bubble { background: var(--code); }
|
|
207
|
+
.speech { word-break: break-word; }
|
|
208
|
+
.speech.thought { color: var(--muted); font-style: italic; }
|
|
209
|
+
.speech > :first-child { margin-top: 0; }
|
|
210
|
+
.speech > :last-child { margin-bottom: 0; }
|
|
211
|
+
.speech p, .speech ul, .speech ol, .speech blockquote, .speech pre, .speech table { margin: 8px 0; }
|
|
212
|
+
.speech h1, .speech h2, .speech h3, .speech h4, .speech h5, .speech h6 { font-size: 14px; font-weight: 600; margin: 12px 0 4px; }
|
|
213
|
+
.speech h1 { font-size: 16px; } .speech h2 { font-size: 15px; }
|
|
214
|
+
.speech ul, .speech ol { padding-left: 22px; }
|
|
215
|
+
.speech li > p { margin: 2px 0; }
|
|
216
|
+
.speech code { background: var(--code); padding: 1px 3px; border-radius: 4px; }
|
|
217
|
+
.speech pre { background: var(--code); border: 1px solid var(--line); }
|
|
218
|
+
.speech pre code { background: none; padding: 0; }
|
|
219
|
+
.speech blockquote { border-left: 3px solid var(--line); margin-left: 0; padding-left: 10px; color: var(--muted); }
|
|
220
|
+
.speech hr { border: none; border-top: 1px solid var(--line); margin: 10px 0; }
|
|
221
|
+
.speech table { width: auto; font-size: 13px; }
|
|
222
|
+
.speech th, .speech td { white-space: normal; padding: 4px 8px; }
|
|
223
|
+
.speech.clamped { max-height: 360px; overflow: hidden; position: relative; }
|
|
224
|
+
.speech.clamped::after { content: ""; position: absolute; left: 0; right: 0; bottom: 0; height: 60px; background: linear-gradient(transparent, var(--panel)); }
|
|
225
|
+
.turn.system .speech.clamped::after, .turn.user .speech.clamped::after { background: linear-gradient(transparent, var(--code)); }
|
|
226
|
+
.more { color: var(--red); cursor: pointer; font-size: 12px; white-space: nowrap; }
|
|
227
|
+
.reasoning { border: none; }
|
|
228
|
+
.reasoning > summary { list-style: none; cursor: pointer; color: var(--muted); font-size: 12px; padding: 0; }
|
|
229
|
+
.reasoning > summary::-webkit-details-marker { display: none; }
|
|
230
|
+
.reasoning > summary::before { content: "▸ "; }
|
|
231
|
+
.reasoning[open] > summary::before { content: "▾ "; }
|
|
232
|
+
.reasoning .thought { margin-top: 6px; }
|
|
233
|
+
.term { background: var(--term-bg); color: var(--term-ink); border-radius: 8px; padding: 8px 12px; font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 12.5px; line-height: 1.45; overflow-x: auto; }
|
|
234
|
+
.term .cmd { white-space: pre-wrap; word-break: break-word; }
|
|
235
|
+
.term .cmd::before { content: "$ "; color: var(--term-muted); }
|
|
236
|
+
.term .cmd.failed::before { color: var(--red); }
|
|
237
|
+
.term .out { white-space: pre-wrap; word-break: break-word; color: var(--term-ink); opacity: 0.85; margin-top: 4px; }
|
|
238
|
+
.term .out:empty { display: none; }
|
|
239
|
+
.term .foot { display: flex; gap: 12px; margin-top: 4px; color: var(--term-muted); font-size: 11.5px; }
|
|
240
|
+
.term .foot .exit { color: var(--red); }
|
|
241
|
+
.term .foot .more { color: var(--term-ink); margin: 0; }
|
|
242
|
+
.turn .meta { display: flex; gap: 12px; align-items: baseline; margin-top: 4px; padding: 0 6px; font-size: 12px; color: var(--muted); font-variant-numeric: tabular-nums; }
|
|
243
|
+
.turn .meta .took { font-size: 13px; color: var(--ink); font-weight: 600; }
|
|
244
|
+
mark { background: color-mix(in srgb, var(--red) 30%, transparent); color: inherit; border-radius: 2px; }
|
|
245
|
+
.term mark { background: color-mix(in srgb, var(--red) 50%, transparent); }
|
|
246
|
+
.gap { display: flex; align-items: center; gap: 8px; color: var(--muted); font-size: 12px; padding: 4px 6px; margin: 0 0 18px; border: 1px dashed var(--line); border-radius: 8px; }
|
|
247
|
+
.gap .count { flex: 1; text-align: center; cursor: pointer; }
|
|
248
|
+
.gap .count:hover { color: var(--ink); }
|
|
249
|
+
.gap .side { min-width: 110px; display: flex; }
|
|
250
|
+
.gap .side:last-child { justify-content: flex-end; }
|
|
251
|
+
.gap button { font-size: 12px; padding: 2px 8px; color: var(--muted); }
|
|
252
|
+
.gap button:hover { color: var(--ink); }
|
|
253
|
+
h2.step { font-size: 13px; color: var(--muted); font-weight: 500; margin: 24px 0 12px; display: flex; gap: 10px; align-items: center; border-top: 1px solid var(--line); padding-top: 12px; }
|
|
254
|
+
.diff .add { color: var(--ok); }
|
|
255
|
+
.diff .del { color: var(--red); }
|
|
256
|
+
.diff .hunk { color: var(--muted); }
|
|
257
|
+
.diff .file { font-weight: 600; }
|
|
258
|
+
.checks td:first-child { white-space: normal; word-break: break-word; }
|
|
259
|
+
</style>
|
|
260
|
+
</head>
|
|
261
|
+
<body>
|
|
262
|
+
<header>
|
|
263
|
+
<span class="brand">lemans viewer</span>
|
|
264
|
+
<nav>
|
|
265
|
+
<a href="#/" data-route="runs">Runs</a>
|
|
266
|
+
<a href="#/starred" data-route="starred">Starred</a>
|
|
267
|
+
</nav>
|
|
268
|
+
<span class="spacer"></span>
|
|
269
|
+
<span class="muted" id="status"></span>
|
|
270
|
+
<button class="theme" id="theme" title="theme"></button>
|
|
271
|
+
</header>
|
|
272
|
+
<main id="main"></main>
|
|
273
|
+
|
|
274
|
+
<script>
|
|
275
|
+
"use strict";
|
|
276
|
+
|
|
277
|
+
const PREVIEW = 300;
|
|
278
|
+
const CLAMP = 3000;
|
|
279
|
+
const SORTS = {
|
|
280
|
+
time: { label: "time", key: r => r.started_at, numeric: true },
|
|
281
|
+
duration: { label: "duration", key: r => r.duration, numeric: true },
|
|
282
|
+
tokens: { label: "tokens", key: r => r.tokens, numeric: true },
|
|
283
|
+
cost: { label: "cost", key: r => r.cost_usd, numeric: true },
|
|
284
|
+
score: { label: "score", key: r => r.reward, numeric: true },
|
|
285
|
+
credit: { label: "credit", key: r => r.credit, numeric: true },
|
|
286
|
+
steps: { label: "steps", key: r => r.steps, numeric: true },
|
|
287
|
+
task: { label: "task", key: r => r.task, numeric: false },
|
|
288
|
+
model: { label: "model", key: r => shortModel(r.model), numeric: false }
|
|
289
|
+
};
|
|
290
|
+
|
|
291
|
+
const storage = {
|
|
292
|
+
read(key, fallback) {
|
|
293
|
+
try { const raw = localStorage.getItem(key); return raw === null ? fallback : JSON.parse(raw); } catch { return fallback; }
|
|
294
|
+
},
|
|
295
|
+
write(key, value) {
|
|
296
|
+
try { localStorage.setItem(key, JSON.stringify(value)); } catch {}
|
|
297
|
+
}
|
|
298
|
+
};
|
|
299
|
+
|
|
300
|
+
const stars = {
|
|
301
|
+
runs: new Set(storage.read("lemans-viewer.starred-runs", [])),
|
|
302
|
+
saveRuns() { storage.write("lemans-viewer.starred-runs", [...this.runs]); },
|
|
303
|
+
toggleRun(id) { this.runs.has(id) ? this.runs.delete(id) : this.runs.add(id); this.saveRuns(); },
|
|
304
|
+
turns(id) { return new Set(storage.read(`lemans-viewer.starred-turns.${id}`, [])); },
|
|
305
|
+
toggleTurn(id, key) {
|
|
306
|
+
const set = this.turns(id);
|
|
307
|
+
set.has(key) ? set.delete(key) : set.add(key);
|
|
308
|
+
storage.write(`lemans-viewer.starred-turns.${id}`, [...set]);
|
|
309
|
+
return set;
|
|
310
|
+
}
|
|
311
|
+
};
|
|
312
|
+
|
|
313
|
+
const state = {
|
|
314
|
+
runs: null,
|
|
315
|
+
filters: Object.assign({ q: "", model: "", agent: "", outcome: "", sort: "time", dir: "desc" },
|
|
316
|
+
storage.read("lemans-viewer.filters", {})),
|
|
317
|
+
run: null,
|
|
318
|
+
redraw: () => {},
|
|
319
|
+
redrawTurns: () => {},
|
|
320
|
+
turnQuery: "",
|
|
321
|
+
tab: "trajectory",
|
|
322
|
+
starredTurnsOnly: false,
|
|
323
|
+
revealed: new Set()
|
|
324
|
+
};
|
|
325
|
+
|
|
326
|
+
function el(tag, attrs = {}, ...children) {
|
|
327
|
+
const node = document.createElement(tag);
|
|
328
|
+
for (const [name, value] of Object.entries(attrs)) {
|
|
329
|
+
if (value === undefined || value === null || value === false) continue;
|
|
330
|
+
if (name === "class") node.className = value;
|
|
331
|
+
else if (name.startsWith("on")) node.addEventListener(name.slice(2), value);
|
|
332
|
+
else if (name === "html") node.innerHTML = value;
|
|
333
|
+
else node.setAttribute(name, value === true ? "" : value);
|
|
334
|
+
}
|
|
335
|
+
for (const child of children.flat()) {
|
|
336
|
+
if (child === undefined || child === null || child === false) continue;
|
|
337
|
+
node.append(child instanceof Node ? child : String(child));
|
|
338
|
+
}
|
|
339
|
+
return node;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
const shortModel = model => (model || "").split("/").pop() || "(default)";
|
|
343
|
+
const fmt = {
|
|
344
|
+
num(value, digits = 4) { return value === null || value === undefined ? "-" : Number(Number(value).toFixed(digits)).toString(); },
|
|
345
|
+
cost(value) { return value === null || value === undefined ? "-" : `$${Number(value).toFixed(4)}`; },
|
|
346
|
+
int(value) { return value === null || value === undefined ? "-" : Number(value).toLocaleString("en-US"); },
|
|
347
|
+
duration(sec) {
|
|
348
|
+
if (sec === null || sec === undefined) return "-";
|
|
349
|
+
const total = Math.round(sec), m = Math.floor(total / 60), s = total % 60;
|
|
350
|
+
return m > 0 ? `${m}m ${s}s` : `${s}s`;
|
|
351
|
+
},
|
|
352
|
+
time(iso) { return iso ? new Date(iso).toLocaleString() : "-"; },
|
|
353
|
+
chars(n) { return `${n.toLocaleString("en-US")} chars`; }
|
|
354
|
+
};
|
|
355
|
+
|
|
356
|
+
function outcomeBadge(row) {
|
|
357
|
+
const kind = row.reward >= 1 ? "ok" : row.scored ? "warn" : "bad";
|
|
358
|
+
return el("span", { class: `badge ${kind}`, title: row.detail || "" }, row.outcome);
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
async function fetchJSON(url) {
|
|
362
|
+
const response = await fetch(url);
|
|
363
|
+
if (!response.ok) throw new Error(`${url}: ${response.status} ${await response.text()}`);
|
|
364
|
+
return response.json();
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
async function fetchText(url) {
|
|
368
|
+
const response = await fetch(url);
|
|
369
|
+
if (!response.ok) throw new Error(`${url}: ${response.status}`);
|
|
370
|
+
return response.text();
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
async function loadRuns(force = false) {
|
|
374
|
+
if (state.runs && !force) return state.runs;
|
|
375
|
+
setStatus("loading runs…");
|
|
376
|
+
state.runs = await fetchJSON("/api/runs");
|
|
377
|
+
setStatus(`${state.runs.rows.length} runs` + (state.runs.unreadable ? ` · ${state.runs.unreadable} unreadable` : ""));
|
|
378
|
+
return state.runs;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
function setStatus(text) { document.getElementById("status").textContent = text; }
|
|
382
|
+
|
|
383
|
+
function tally(rows) {
|
|
384
|
+
const scored = rows.filter(r => r.scored).length;
|
|
385
|
+
const solved = rows.filter(r => (r.reward || 0) >= 1).length;
|
|
386
|
+
const cost = rows.reduce((sum, r) => sum + (r.cost_usd || 0), 0);
|
|
387
|
+
const rank = scored > 0 ? ` (${Math.round(100 * solved / scored)}%)` : "";
|
|
388
|
+
return `${rows.length} trials: ${scored} scored, ${rows.length - scored} invalid, ${solved} solved${rank} · $${cost.toFixed(4)}`;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
function sortRows(rows) {
|
|
392
|
+
const spec = SORTS[state.filters.sort] || SORTS.time;
|
|
393
|
+
const desc = state.filters.dir === "desc";
|
|
394
|
+
const present = rows.filter(r => spec.key(r) !== null && spec.key(r) !== undefined);
|
|
395
|
+
const missing = rows.filter(r => spec.key(r) === null || spec.key(r) === undefined);
|
|
396
|
+
present.sort((a, b) => {
|
|
397
|
+
const x = spec.key(a), y = spec.key(b);
|
|
398
|
+
const cmp = spec.numeric && typeof x === "number" ? x - y : String(x).localeCompare(String(y));
|
|
399
|
+
return desc ? -cmp : cmp;
|
|
400
|
+
});
|
|
401
|
+
return present.concat(missing);
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
function filterRows(rows, starredOnly) {
|
|
405
|
+
const f = state.filters;
|
|
406
|
+
const q = f.q.trim().toLowerCase();
|
|
407
|
+
return rows.filter(r =>
|
|
408
|
+
(!starredOnly || stars.runs.has(r.trial)) &&
|
|
409
|
+
(!q || r.task.toLowerCase().includes(q) || r.trial.toLowerCase().includes(q)) &&
|
|
410
|
+
(!f.model || r.model === f.model) &&
|
|
411
|
+
(!f.agent || r.agent === f.agent) &&
|
|
412
|
+
(!f.outcome || (f.outcome === "scored" ? r.scored : f.outcome === "invalid" ? !r.scored : r.outcome === f.outcome))
|
|
413
|
+
);
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
function renderRunsView(starredOnly) {
|
|
417
|
+
const main = document.getElementById("main");
|
|
418
|
+
main.classList.remove("reading");
|
|
419
|
+
main.replaceChildren(el("div", { class: "empty" }, "loading…"));
|
|
420
|
+
loadRuns().then(data => {
|
|
421
|
+
const list = el("div");
|
|
422
|
+
const controls = {};
|
|
423
|
+
const options = (key, label) => {
|
|
424
|
+
const values = [...new Set(data.rows.map(r => r[key]).filter(Boolean))].sort();
|
|
425
|
+
return controls[key] = el("select", { onchange: e => setFilter(key, e.target.value) },
|
|
426
|
+
el("option", { value: "" }, `all ${label}`),
|
|
427
|
+
values.map(v => el("option", { value: v }, key === "model" ? shortModel(v) : v)));
|
|
428
|
+
};
|
|
429
|
+
const outcomes = [...new Set(data.rows.map(r => r.outcome))].sort();
|
|
430
|
+
let pending;
|
|
431
|
+
const toolbar = el("div", { class: "toolbar" },
|
|
432
|
+
controls.q = el("input", { type: "search", placeholder: "search task or trial…", autocomplete: "off",
|
|
433
|
+
oninput: e => { clearTimeout(pending); pending = setTimeout(() => setFilter("q", e.target.value), 150); } }),
|
|
434
|
+
options("model", "models"),
|
|
435
|
+
options("agent", "agents"),
|
|
436
|
+
controls.outcome = el("select", { onchange: e => setFilter("outcome", e.target.value) },
|
|
437
|
+
el("option", { value: "" }, "all outcomes"),
|
|
438
|
+
el("option", { value: "scored" }, "scored"),
|
|
439
|
+
el("option", { value: "invalid" }, "invalid"),
|
|
440
|
+
outcomes.map(o => el("option", { value: o }, o))),
|
|
441
|
+
el("span", { class: "muted" }, "sort by"),
|
|
442
|
+
controls.sort = el("select", { onchange: e => setFilter("sort", e.target.value) },
|
|
443
|
+
Object.entries(SORTS).map(([k, s]) => el("option", { value: k }, s.label))),
|
|
444
|
+
controls.dir = el("button", { onclick: () => setFilter("dir", state.filters.dir === "desc" ? "asc" : "desc"), title: "direction" }),
|
|
445
|
+
el("button", { onclick: () => { Object.assign(state.filters, { q: "", model: "", agent: "", outcome: "" }); saveFilters(); state.redraw(); } }, "clear"),
|
|
446
|
+
el("button", { onclick: () => loadRuns(true).then(render), title: "re-read the runs directory" }, "↻ refresh"));
|
|
447
|
+
state.redraw = () => {
|
|
448
|
+
const f = state.filters;
|
|
449
|
+
if (controls.q.value !== f.q) controls.q.value = f.q;
|
|
450
|
+
for (const key of ["model", "agent", "outcome", "sort"]) controls[key].value = f[key];
|
|
451
|
+
controls.dir.textContent = f.dir === "desc" ? "↓ desc" : "↑ asc";
|
|
452
|
+
const rows = sortRows(filterRows(data.rows, starredOnly));
|
|
453
|
+
list.replaceChildren(el("div", { class: "summary" }, tally(rows)), runsTable(rows, data.fractional));
|
|
454
|
+
};
|
|
455
|
+
main.replaceChildren(toolbar, list);
|
|
456
|
+
state.redraw();
|
|
457
|
+
}).catch(showError);
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
function runsTable(rows, fractional) {
|
|
461
|
+
if (!rows.length) return el("div", { class: "empty" }, "no runs match");
|
|
462
|
+
const columns = [
|
|
463
|
+
["", null], ["task", "task"], ["agent", null], ["model", "model"], ["reward", "score"],
|
|
464
|
+
fractional && ["credit", "credit"], ["outcome", null], ["cost", "cost"], ["steps", "steps"],
|
|
465
|
+
["tokens", "tokens"], ["duration", "duration"], ["started", "time"], ["trial", null]
|
|
466
|
+
].filter(Boolean);
|
|
467
|
+
const numeric = new Set(["reward", "credit", "cost", "steps", "tokens", "duration"]);
|
|
468
|
+
const header = el("tr", {}, columns.map(([label, sort]) =>
|
|
469
|
+
el("th", { class: [sort && "sortable", sort === state.filters.sort && "sorted", numeric.has(label) && "num"].filter(Boolean).join(" "),
|
|
470
|
+
onclick: sort && (() => setFilter("sort", sort, sort === state.filters.sort)) },
|
|
471
|
+
label, sort === state.filters.sort ? (state.filters.dir === "desc" ? " ↓" : " ↑") : "")));
|
|
472
|
+
const body = rows.map(r => el("tr", { class: "run", onclick: () => { location.hash = `#/runs/${encodeURIComponent(r.trial)}`; } },
|
|
473
|
+
el("td", {}, starButton(stars.runs.has(r.trial), () => { stars.toggleRun(r.trial); return stars.runs.has(r.trial); })),
|
|
474
|
+
el("td", {}, r.task),
|
|
475
|
+
el("td", {}, r.agent),
|
|
476
|
+
el("td", { title: r.model }, shortModel(r.model)),
|
|
477
|
+
el("td", { class: "num" }, fmt.num(r.reward)),
|
|
478
|
+
fractional && el("td", { class: "num" }, fmt.num(r.credit)),
|
|
479
|
+
el("td", {}, outcomeBadge(r)),
|
|
480
|
+
el("td", { class: "num" }, fmt.cost(r.cost_usd)),
|
|
481
|
+
el("td", { class: "num" }, fmt.int(r.steps)),
|
|
482
|
+
el("td", { class: "num" }, fmt.int(r.tokens)),
|
|
483
|
+
el("td", { class: "num" }, fmt.duration(r.duration)),
|
|
484
|
+
el("td", {}, fmt.time(r.started_at)),
|
|
485
|
+
el("td", { class: "mono" }, r.trial)));
|
|
486
|
+
return el("div", { class: "scroll" }, el("table", {}, el("thead", {}, header), el("tbody", {}, body)));
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
function starButton(on, toggle, title = "star") {
|
|
490
|
+
const button = el("button", { class: `star${on ? " on" : ""}`, title }, on ? "★" : "☆");
|
|
491
|
+
button.addEventListener("click", e => {
|
|
492
|
+
e.stopPropagation();
|
|
493
|
+
const now = toggle(e);
|
|
494
|
+
button.classList.toggle("on", now);
|
|
495
|
+
button.textContent = now ? "★" : "☆";
|
|
496
|
+
});
|
|
497
|
+
return button;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
function setFilter(key, value, flipDirection = false) {
|
|
501
|
+
if (flipDirection) state.filters.dir = state.filters.dir === "desc" ? "asc" : "desc";
|
|
502
|
+
else if (key === "sort") state.filters.dir = SORTS[value].numeric ? "desc" : "asc";
|
|
503
|
+
state.filters[key] = value;
|
|
504
|
+
saveFilters();
|
|
505
|
+
state.redraw();
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
function saveFilters() { storage.write("lemans-viewer.filters", state.filters); }
|
|
509
|
+
|
|
510
|
+
async function renderRunView(id) {
|
|
511
|
+
const main = document.getElementById("main");
|
|
512
|
+
main.classList.add("reading");
|
|
513
|
+
if (!state.run || state.run.row.trial !== id) {
|
|
514
|
+
main.replaceChildren(el("div", { class: "empty" }, "loading…"));
|
|
515
|
+
try {
|
|
516
|
+
state.run = await fetchJSON(`/api/runs/${encodeURIComponent(id)}`);
|
|
517
|
+
state.revealed = new Set();
|
|
518
|
+
state.tab = "trajectory";
|
|
519
|
+
} catch (error) { return showError(error); }
|
|
520
|
+
}
|
|
521
|
+
const { row, result, artifacts } = state.run;
|
|
522
|
+
const usage = result.usage || {};
|
|
523
|
+
const fact = (k, v) => el("span", {}, el("span", { class: "k" }, k), v);
|
|
524
|
+
const phases = (result.phases || []).map(p => {
|
|
525
|
+
const secs = p.finished_at ? (new Date(p.finished_at) - new Date(p.started_at)) / 1000 : null;
|
|
526
|
+
return `${p.name} ${fmt.duration(secs)}`;
|
|
527
|
+
});
|
|
528
|
+
const card = el("div", { class: "card" },
|
|
529
|
+
el("h1", {},
|
|
530
|
+
starButton(stars.runs.has(id), () => { stars.toggleRun(id); return stars.runs.has(id); }),
|
|
531
|
+
row.task, outcomeBadge(row),
|
|
532
|
+
el("span", { class: "sub" }, `${shortModel(row.model)} · ${row.agent} · `, el("span", { class: "mono" }, id))),
|
|
533
|
+
el("div", { class: "facts" },
|
|
534
|
+
fact("reward", fmt.num(row.reward)),
|
|
535
|
+
row.credit !== row.reward && fact("credit", fmt.num(row.credit)),
|
|
536
|
+
fact("cost", fmt.cost(row.cost_usd)),
|
|
537
|
+
fact("steps", fmt.int(row.steps)),
|
|
538
|
+
fact("tokens", `${fmt.int(usage.input_tokens)} in · ${fmt.int(usage.output_tokens)} out · ${fmt.int(usage.cached_tokens)} cached`),
|
|
539
|
+
fact("duration", fmt.duration(row.duration)),
|
|
540
|
+
fact("started", fmt.time(row.started_at))),
|
|
541
|
+
row.detail && el("p", { style: "color: var(--red); margin: 8px 0 0" }, row.detail),
|
|
542
|
+
el("div", { style: "margin-top: 8px" },
|
|
543
|
+
(row.tags || []).map(t => el("span", { class: "tag" }, t)),
|
|
544
|
+
Object.entries(row.metadata || {}).map(([k, v]) => el("span", { class: "tag" }, `${k}: ${v}`))),
|
|
545
|
+
phases.length ? el("div", { class: "muted", style: "margin-top: 6px" }, `phases: ${phases.join(" → ")} · lemans ${result.lemans_version || "?"}`) : null);
|
|
546
|
+
|
|
547
|
+
const tabs = [["trajectory", "Trajectory"], ...artifacts.filter(a => !a.startsWith("trajectory")).map(a => [a, a])];
|
|
548
|
+
const tabBar = el("div", { class: "tabs" }, tabs.map(([key, label]) =>
|
|
549
|
+
el("button", { class: key === state.tab ? "active" : "", onclick: () => { state.tab = key; render(); } }, label)));
|
|
550
|
+
const content = el("div", { id: "tab-content" });
|
|
551
|
+
main.replaceChildren(el("p", {}, el("a", { href: "#/" }, "← runs")), card, tabBar, content);
|
|
552
|
+
if (state.tab === "trajectory") renderTrajectories(content);
|
|
553
|
+
else renderArtifact(content, id, state.tab);
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
function renderTrajectories(container) {
|
|
557
|
+
const { trajectories, row } = state.run;
|
|
558
|
+
if (!trajectories.length) return container.replaceChildren(el("div", { class: "empty" }, "no trajectory stored"));
|
|
559
|
+
const starred = stars.turns(row.trial);
|
|
560
|
+
const turns = el("div");
|
|
561
|
+
const expandAll = open => turns.querySelectorAll(".term").forEach(t => t.expand(open));
|
|
562
|
+
let pending;
|
|
563
|
+
const starredOnly = el("button", { id: "starred-only", class: state.starredTurnsOnly ? "active" : "",
|
|
564
|
+
onclick: () => { state.starredTurnsOnly = !state.starredTurnsOnly; starredOnly.classList.toggle("active", state.starredTurnsOnly); state.revealed.clear(); state.redrawTurns(); } },
|
|
565
|
+
`★ starred turns only (${starred.size})`);
|
|
566
|
+
const matches = el("span", { class: "muted" });
|
|
567
|
+
const toolbar = el("div", { class: "toolbar" },
|
|
568
|
+
el("input", { type: "search", placeholder: "search turns…", autocomplete: "off", value: state.turnQuery,
|
|
569
|
+
oninput: e => { clearTimeout(pending); pending = setTimeout(() => { state.turnQuery = e.target.value; state.revealed.clear(); state.redrawTurns(); }, 150); } }),
|
|
570
|
+
matches,
|
|
571
|
+
starredOnly,
|
|
572
|
+
el("button", { onclick: () => expandAll(true) }, "expand output"),
|
|
573
|
+
el("button", { onclick: () => expandAll(false) }, "collapse output"));
|
|
574
|
+
state.redrawTurns = () => {
|
|
575
|
+
const query = state.turnQuery.trim().toLowerCase();
|
|
576
|
+
let shown = 0, total = 0;
|
|
577
|
+
turns.replaceChildren();
|
|
578
|
+
trajectories.forEach((entry, index) => {
|
|
579
|
+
const t = entry.trajectory;
|
|
580
|
+
const metrics = t.final_metrics || {};
|
|
581
|
+
if (trajectories.length > 1 || entry.step) {
|
|
582
|
+
turns.append(el("h2", { class: "step" },
|
|
583
|
+
el("span", { style: "color: var(--ink)" }, entry.step ? `Step ${entry.step}` : "Final step"),
|
|
584
|
+
t.extra?.status && el("span", { class: "badge" }, t.extra.status),
|
|
585
|
+
el("span", {}, `${fmt.int(metrics.total_steps)} turns · ${fmt.cost(metrics.total_cost_usd)} · ${entry.name}`)));
|
|
586
|
+
}
|
|
587
|
+
const rendered = renderTurns(index, t.steps || [], starred, query);
|
|
588
|
+
shown += rendered.shown; total += rendered.total;
|
|
589
|
+
turns.append(...rendered.nodes);
|
|
590
|
+
});
|
|
591
|
+
matches.textContent = query || state.starredTurnsOnly ? `${shown} of ${total} turns` : "";
|
|
592
|
+
};
|
|
593
|
+
container.replaceChildren(toolbar, turns);
|
|
594
|
+
state.redrawTurns();
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
function searchable(step) {
|
|
598
|
+
const calls = (step.tool_calls || []).map(c => typeof c.arguments?.command === "string" ? c.arguments.command : JSON.stringify(c.arguments || {}));
|
|
599
|
+
const outputs = (step.observation?.results || []).map(r => parseResult(r).output);
|
|
600
|
+
return [step.message, step.reasoning_content, ...calls, ...outputs].filter(Boolean).join("\n").toLowerCase();
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
function renderTurns(trajectoryIndex, steps, starred, query) {
|
|
604
|
+
const keyOf = step => `${trajectoryIndex}:${step.step_id}`;
|
|
605
|
+
const matches = step => (!state.starredTurnsOnly || starred.has(keyOf(step))) && (!query || searchable(step).includes(query));
|
|
606
|
+
const visible = step => matches(step) || state.revealed.has(keyOf(step));
|
|
607
|
+
const reveal = (...batch) => { batch.forEach(step => state.revealed.add(keyOf(step))); state.redrawTurns(); };
|
|
608
|
+
const nodes = [];
|
|
609
|
+
let shown = 0;
|
|
610
|
+
let hidden = [];
|
|
611
|
+
const flush = (above, below) => {
|
|
612
|
+
if (!hidden.length) return;
|
|
613
|
+
const batch = hidden;
|
|
614
|
+
nodes.push(el("div", { class: "gap" },
|
|
615
|
+
el("span", { class: "side" }, above && el("button", { onclick: () => reveal(batch[0]) }, `show #${batch[0].step_id} ↓`)),
|
|
616
|
+
el("span", { class: "count", onclick: () => reveal(...batch) }, `… ${batch.length} turn(s) hidden — show all`),
|
|
617
|
+
el("span", { class: "side" }, below && el("button", { onclick: () => reveal(batch.at(-1)) }, `↑ show #${batch.at(-1).step_id}`))));
|
|
618
|
+
hidden = [];
|
|
619
|
+
};
|
|
620
|
+
let above = null;
|
|
621
|
+
steps.forEach(step => {
|
|
622
|
+
if (!visible(step)) { hidden.push(step); return; }
|
|
623
|
+
flush(above, step);
|
|
624
|
+
shown++;
|
|
625
|
+
above = step;
|
|
626
|
+
nodes.push(turnCard(step, keyOf(step), starred, steps, query));
|
|
627
|
+
});
|
|
628
|
+
flush(above, null);
|
|
629
|
+
return { nodes, shown, total: steps.length };
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
function highlight(root, query) {
|
|
633
|
+
if (!query) return root;
|
|
634
|
+
const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT);
|
|
635
|
+
const targets = [];
|
|
636
|
+
for (let node = walker.nextNode(); node; node = walker.nextNode()) if (node.data.toLowerCase().includes(query)) targets.push(node);
|
|
637
|
+
for (const node of targets) {
|
|
638
|
+
const text = node.data, lower = text.toLowerCase(), parts = [];
|
|
639
|
+
let from = 0, at;
|
|
640
|
+
while ((at = lower.indexOf(query, from)) !== -1) {
|
|
641
|
+
parts.push(text.slice(from, at), el("mark", {}, text.slice(at, at + query.length)));
|
|
642
|
+
from = at + query.length;
|
|
643
|
+
}
|
|
644
|
+
parts.push(text.slice(from));
|
|
645
|
+
node.replaceWith(...parts);
|
|
646
|
+
}
|
|
647
|
+
return root;
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
const NAMES = { system: "system", user: "task", agent: "agent" };
|
|
651
|
+
|
|
652
|
+
function turnCard(step, key, starred, steps, query) {
|
|
653
|
+
const on = starred.has(key);
|
|
654
|
+
const card = el("div", { class: `turn ${step.source}${on ? " starred" : ""}` });
|
|
655
|
+
const who = el("div", { class: "who" },
|
|
656
|
+
starButton(on, () => {
|
|
657
|
+
const set = stars.toggleTurn(state.run.row.trial, key);
|
|
658
|
+
starred.clear(); set.forEach(k => starred.add(k));
|
|
659
|
+
card.classList.toggle("starred", set.has(key));
|
|
660
|
+
document.getElementById("starred-only").textContent = `★ starred turns only (${set.size})`;
|
|
661
|
+
return set.has(key);
|
|
662
|
+
}, "star turn"),
|
|
663
|
+
el("span", { class: "name" }, NAMES[step.source] || step.source),
|
|
664
|
+
el("span", {}, `#${step.step_id}`),
|
|
665
|
+
step.extra?.finish_reason && step.extra.finish_reason !== "tool_calls" && step.extra.finish_reason !== "stop" &&
|
|
666
|
+
el("span", { class: "badge warn" }, step.extra.finish_reason));
|
|
667
|
+
const bubble = el("div", { class: "bubble" });
|
|
668
|
+
const message = (step.message || "").trim();
|
|
669
|
+
const reasoning = (step.reasoning_content || "").trim();
|
|
670
|
+
if (message) {
|
|
671
|
+
bubble.append(speech(message, false, query));
|
|
672
|
+
if (reasoning) {
|
|
673
|
+
const found = query && reasoning.toLowerCase().includes(query);
|
|
674
|
+
bubble.append(el("details", { class: "reasoning", open: found }, el("summary", {}, `reasoning (${fmt.chars(reasoning.length)})`), speech(reasoning, true, query)));
|
|
675
|
+
}
|
|
676
|
+
} else if (reasoning) {
|
|
677
|
+
bubble.append(speech(reasoning, true, query));
|
|
678
|
+
}
|
|
679
|
+
const results = new Map((step.observation?.results || []).map(r => [r.source_call_id, r]));
|
|
680
|
+
for (const call of step.tool_calls || []) {
|
|
681
|
+
bubble.append(terminal(call, results.get(call.tool_call_id), query));
|
|
682
|
+
results.delete(call.tool_call_id);
|
|
683
|
+
}
|
|
684
|
+
for (const result of results.values()) bubble.append(terminal(null, result, query));
|
|
685
|
+
card.append(...[who, bubble, footer(step, steps)].filter(Boolean));
|
|
686
|
+
return card;
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
function footer(step, steps) {
|
|
690
|
+
const metrics = step.metrics;
|
|
691
|
+
if (!step.timestamp && !metrics) return null;
|
|
692
|
+
const next = steps.slice(steps.indexOf(step) + 1).find(s => s.timestamp);
|
|
693
|
+
const took = step.timestamp && next ? (new Date(next.timestamp) - new Date(step.timestamp)) / 1000 : null;
|
|
694
|
+
return el("div", { class: "meta" },
|
|
695
|
+
took !== null && el("span", { class: "took", title: "until the next turn" }, fmt.duration(took)),
|
|
696
|
+
step.timestamp && el("span", {}, new Date(step.timestamp).toLocaleTimeString()),
|
|
697
|
+
metrics && el("span", {}, `${fmt.int(metrics.prompt_tokens)} in · ${fmt.int(metrics.completion_tokens)} out`),
|
|
698
|
+
metrics?.cost_usd !== undefined && el("span", {}, fmt.cost(metrics.cost_usd)));
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
function speech(text, thought = false, query = "") {
|
|
702
|
+
const node = highlight(el("div", { class: `speech${thought ? " thought" : ""}` }, markdown(text)), query);
|
|
703
|
+
if (text.length <= CLAMP || (query && text.toLowerCase().includes(query))) return node;
|
|
704
|
+
node.classList.add("clamped");
|
|
705
|
+
const more = el("div", {}, el("span", { class: "more", onclick: () => {
|
|
706
|
+
const clamped = node.classList.toggle("clamped");
|
|
707
|
+
more.firstChild.textContent = clamped ? `more (${fmt.chars(text.length)})` : "less";
|
|
708
|
+
} }, `more (${fmt.chars(text.length)})`));
|
|
709
|
+
return el("div", {}, node, more);
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
const BLOCK = {
|
|
713
|
+
fence: /^\s*(`{3,}|~{3,})\s*(\S*)\s*$/,
|
|
714
|
+
heading: /^(#{1,6})\s+(.*?)\s*#*\s*$/,
|
|
715
|
+
rule: /^\s*([-*_])(\s*\1){2,}\s*$/,
|
|
716
|
+
quote: /^\s*>\s?(.*)$/,
|
|
717
|
+
item: /^(\s*)([-*+]|\d+[.)])\s+(.*)$/,
|
|
718
|
+
tableSep: /^\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)*\|?\s*$/
|
|
719
|
+
};
|
|
720
|
+
|
|
721
|
+
function markdown(text) {
|
|
722
|
+
const fragment = document.createDocumentFragment();
|
|
723
|
+
const lines = text.replace(/\r\n?/g, "\n").split("\n");
|
|
724
|
+
let i = 0;
|
|
725
|
+
const paragraph = [];
|
|
726
|
+
const flush = () => {
|
|
727
|
+
if (paragraph.length) fragment.append(el("p", {}, inline(paragraph.join(" "))));
|
|
728
|
+
paragraph.length = 0;
|
|
729
|
+
};
|
|
730
|
+
while (i < lines.length) {
|
|
731
|
+
const line = lines[i];
|
|
732
|
+
let match;
|
|
733
|
+
if (!line.trim()) { flush(); i++; continue; }
|
|
734
|
+
if ((match = line.match(BLOCK.fence))) {
|
|
735
|
+
flush();
|
|
736
|
+
const close = new RegExp(`^\\s*${match[1][0]}{${match[1].length},}\\s*$`);
|
|
737
|
+
const body = [];
|
|
738
|
+
for (i++; i < lines.length && !close.test(lines[i]); i++) body.push(lines[i]);
|
|
739
|
+
i++;
|
|
740
|
+
fragment.append(el("pre", {}, el("code", { class: match[2] && `language-${match[2]}` }, body.join("\n"))));
|
|
741
|
+
} else if ((match = line.match(BLOCK.heading))) {
|
|
742
|
+
flush(); i++;
|
|
743
|
+
fragment.append(el(`h${match[1].length}`, {}, inline(match[2])));
|
|
744
|
+
} else if (BLOCK.rule.test(line)) {
|
|
745
|
+
flush(); i++;
|
|
746
|
+
fragment.append(el("hr"));
|
|
747
|
+
} else if (BLOCK.quote.test(line)) {
|
|
748
|
+
flush();
|
|
749
|
+
const body = [];
|
|
750
|
+
for (; i < lines.length && BLOCK.quote.test(lines[i]); i++) body.push(lines[i].match(BLOCK.quote)[1]);
|
|
751
|
+
fragment.append(el("blockquote", {}, markdown(body.join("\n"))));
|
|
752
|
+
} else if ((match = line.match(BLOCK.item))) {
|
|
753
|
+
flush();
|
|
754
|
+
const end = listEnd(lines, i, match[1].length);
|
|
755
|
+
fragment.append(list(lines.slice(i, end), match[1].length));
|
|
756
|
+
i = end;
|
|
757
|
+
} else if (line.includes("|") && i + 1 < lines.length && BLOCK.tableSep.test(lines[i + 1])) {
|
|
758
|
+
flush();
|
|
759
|
+
const rows = [];
|
|
760
|
+
for (; i < lines.length && lines[i].includes("|"); i++) rows.push(lines[i]);
|
|
761
|
+
fragment.append(table(rows));
|
|
762
|
+
} else {
|
|
763
|
+
paragraph.push(line.trim());
|
|
764
|
+
i++;
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
flush();
|
|
768
|
+
return fragment;
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
function listEnd(lines, start, indent) {
|
|
772
|
+
let i = start + 1;
|
|
773
|
+
for (; i < lines.length; i++) {
|
|
774
|
+
const line = lines[i];
|
|
775
|
+
if (!line.trim()) {
|
|
776
|
+
const next = lines[i + 1];
|
|
777
|
+
if (next === undefined || (!BLOCK.item.test(next) && next.search(/\S/) <= indent)) break;
|
|
778
|
+
continue;
|
|
779
|
+
}
|
|
780
|
+
const item = line.match(BLOCK.item);
|
|
781
|
+
if (item ? item[1].length < indent : line.search(/\S/) <= indent) break;
|
|
782
|
+
}
|
|
783
|
+
return i;
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
function list(lines, indent) {
|
|
787
|
+
const first = lines[0].match(BLOCK.item);
|
|
788
|
+
const ordered = /\d/.test(first[2]);
|
|
789
|
+
const items = [];
|
|
790
|
+
let current = null;
|
|
791
|
+
for (const line of lines) {
|
|
792
|
+
const item = line.match(BLOCK.item);
|
|
793
|
+
if (item && item[1].length === indent) { current = [item[3]]; items.push(current); }
|
|
794
|
+
else if (current) current.push(line.replace(new RegExp(`^\\s{0,${indent + 2}}`), ""));
|
|
795
|
+
}
|
|
796
|
+
const node = el(ordered ? "ol" : "ul", ordered && first[2] !== "1." && first[2] !== "1)" ? { start: parseInt(first[2], 10) } : {});
|
|
797
|
+
for (const item of items) {
|
|
798
|
+
const body = markdown(item.join("\n"));
|
|
799
|
+
const li = el("li");
|
|
800
|
+
if (body.childElementCount === 1 && body.firstElementChild.tagName === "P") li.append(...body.firstElementChild.childNodes);
|
|
801
|
+
else li.append(body);
|
|
802
|
+
node.append(li);
|
|
803
|
+
}
|
|
804
|
+
return node;
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
function table(rows) {
|
|
808
|
+
const cells = row => row.trim().replace(/^\||\|$/g, "").split("|").map(c => c.trim());
|
|
809
|
+
const [head, , ...body] = rows;
|
|
810
|
+
return el("div", { class: "scroll" }, el("table", {},
|
|
811
|
+
el("thead", {}, el("tr", {}, cells(head).map(c => el("th", {}, inline(c))))),
|
|
812
|
+
el("tbody", {}, body.map(row => el("tr", {}, cells(row).map(c => el("td", {}, inline(c))))))));
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
const INLINE = /(`+)([\s\S]*?[^`])\1(?!`)|\*\*(.+?)\*\*|(?<![\w*])\*([^*\n]+?)\*(?![\w*])|(?<![\w_])_([^_\n]+?)_(?![\w_])|\[([^\]]+)\]\(([^)\s]+)\)|(https?:\/\/[^\s<>)]+)/;
|
|
816
|
+
|
|
817
|
+
function inline(text) {
|
|
818
|
+
const nodes = [];
|
|
819
|
+
let rest = text;
|
|
820
|
+
let match;
|
|
821
|
+
while ((match = INLINE.exec(rest))) {
|
|
822
|
+
if (match.index > 0) nodes.push(rest.slice(0, match.index));
|
|
823
|
+
if (match[2] !== undefined) nodes.push(el("code", {}, match[2].trim()));
|
|
824
|
+
else if (match[3] !== undefined) nodes.push(el("strong", {}, inline(match[3])));
|
|
825
|
+
else if (match[4] !== undefined) nodes.push(el("em", {}, inline(match[4])));
|
|
826
|
+
else if (match[5] !== undefined) nodes.push(el("em", {}, inline(match[5])));
|
|
827
|
+
else if (match[6] !== undefined) nodes.push(el("a", { href: match[7], target: "_blank", rel: "noopener" }, inline(match[6])));
|
|
828
|
+
else nodes.push(el("a", { href: match[8], target: "_blank", rel: "noopener" }, match[8]));
|
|
829
|
+
rest = rest.slice(match.index + match[0].length);
|
|
830
|
+
}
|
|
831
|
+
if (rest) nodes.push(rest);
|
|
832
|
+
return nodes;
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
function parseResult(result) {
|
|
836
|
+
const raw = result?.content ?? "";
|
|
837
|
+
let output = String(raw), exit = result?.extra?.exit_code, note = null;
|
|
838
|
+
try {
|
|
839
|
+
const parsed = JSON.parse(raw);
|
|
840
|
+
if (parsed && typeof parsed === "object") {
|
|
841
|
+
if ("output" in parsed) output = String(parsed.output ?? "");
|
|
842
|
+
else if ("output_head" in parsed) output = `${parsed.output_head ?? ""}\n\n… ${fmt.chars(parsed.elided_chars || 0)} elided by the agent …\n\n${parsed.output_tail ?? ""}`;
|
|
843
|
+
if (parsed.exception_info) { note = String(parsed.exception_info); exit = undefined; }
|
|
844
|
+
else if (exit === undefined && parsed.returncode !== undefined) exit = parsed.returncode;
|
|
845
|
+
}
|
|
846
|
+
} catch {}
|
|
847
|
+
return { output: output.replace(/\s+$/, ""), exit, note };
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
function terminal(call, result, query = "") {
|
|
851
|
+
const args = call?.arguments || {};
|
|
852
|
+
const command = call ? (typeof args.command === "string" && Object.keys(args).length === 1 ? args.command : `${call.function_name}(${JSON.stringify(args)})`) : null;
|
|
853
|
+
const { output, exit, note } = parseResult(result);
|
|
854
|
+
const failed = exit !== undefined && exit !== 0;
|
|
855
|
+
const term = el("div", { class: "term" });
|
|
856
|
+
const out = el("div", { class: "out" });
|
|
857
|
+
const foot = el("div", { class: "foot" });
|
|
858
|
+
let expanded = Boolean(query) && output.toLowerCase().indexOf(query) >= PREVIEW - query.length;
|
|
859
|
+
term.expand = open => { expanded = open; draw(); };
|
|
860
|
+
const draw = () => {
|
|
861
|
+
const long = output.length > PREVIEW;
|
|
862
|
+
out.replaceChildren(highlight(el("span", {}, expanded || !long ? output : output.slice(0, PREVIEW) + "…"), query));
|
|
863
|
+
foot.replaceChildren(...[
|
|
864
|
+
failed && el("span", { class: "exit" }, `exit ${exit}`),
|
|
865
|
+
note && el("span", {}, note),
|
|
866
|
+
!call && el("span", {}, "result"),
|
|
867
|
+
long && el("span", { class: "more", onclick: () => term.expand(!expanded) }, expanded ? "collapse" : `show all (${fmt.chars(output.length)})`),
|
|
868
|
+
result === undefined && call && el("span", {}, "no result recorded")
|
|
869
|
+
].filter(Boolean));
|
|
870
|
+
foot.hidden = !foot.childElementCount;
|
|
871
|
+
};
|
|
872
|
+
draw();
|
|
873
|
+
if (command !== null) term.append(highlight(el("div", { class: `cmd${failed ? " failed" : ""}` }, command), query));
|
|
874
|
+
term.append(out, foot);
|
|
875
|
+
return term;
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
async function renderArtifact(container, id, name) {
|
|
879
|
+
container.replaceChildren(el("div", { class: "empty" }, "loading…"));
|
|
880
|
+
try {
|
|
881
|
+
const text = await fetchText(`/api/runs/${encodeURIComponent(id)}/artifacts/${name.split("/").map(encodeURIComponent).join("/")}`);
|
|
882
|
+
if (state.tab !== name) return;
|
|
883
|
+
if (name.startsWith("checks") && name.endsWith(".json")) container.replaceChildren(checksTable(JSON.parse(text)));
|
|
884
|
+
else if (name.endsWith(".patch")) container.replaceChildren(diffView(text));
|
|
885
|
+
else if (name.endsWith(".json")) container.replaceChildren(el("pre", {}, JSON.stringify(JSON.parse(text), null, 2)));
|
|
886
|
+
else container.replaceChildren(el("pre", {}, text));
|
|
887
|
+
} catch (error) { showError(error, container); }
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
function checksTable(data) {
|
|
891
|
+
const rows = Object.entries(data.checks || {}).map(([name, status]) =>
|
|
892
|
+
el("tr", {}, el("td", {}, name), el("td", {}, el("span", { class: `badge ${status === "pass" ? "ok" : status.startsWith("fail (allowed") ? "warn" : "bad"}` }, status))));
|
|
893
|
+
const grading = data.grading ? el("p", { class: "muted" }, `grading: ${JSON.stringify(data.grading)}`) : null;
|
|
894
|
+
return el("div", { class: "checks" }, grading, el("table", {}, el("thead", {}, el("tr", {}, el("th", {}, "check"), el("th", {}, "status"))), el("tbody", {}, rows)));
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
function diffView(text) {
|
|
898
|
+
const lines = text.split("\n").map(line => {
|
|
899
|
+
const cls = line.startsWith("+++") || line.startsWith("---") || line.startsWith("diff ") ? "file"
|
|
900
|
+
: line.startsWith("@@") ? "hunk" : line.startsWith("+") ? "add" : line.startsWith("-") ? "del" : "";
|
|
901
|
+
return el("div", { class: cls }, line || " ");
|
|
902
|
+
});
|
|
903
|
+
return el("pre", { class: "diff" }, lines);
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
function showError(error, container = document.getElementById("main")) {
|
|
907
|
+
container.replaceChildren(el("div", { class: "empty" }, String(error.message || error)));
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
function render() {
|
|
911
|
+
const hash = location.hash || "#/";
|
|
912
|
+
document.querySelectorAll("header nav a").forEach(a => a.classList.toggle("active", hash === a.getAttribute("href")));
|
|
913
|
+
const match = hash.match(/^#\/runs\/(.+)$/);
|
|
914
|
+
if (match) renderRunView(decodeURIComponent(match[1]));
|
|
915
|
+
else renderRunsView(hash === "#/starred");
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
const THEMES = ["system", "light", "dark"];
|
|
919
|
+
function applyTheme(theme) {
|
|
920
|
+
storage.write("lemans-viewer.theme", theme);
|
|
921
|
+
if (theme === "system") document.documentElement.removeAttribute("data-theme");
|
|
922
|
+
else document.documentElement.setAttribute("data-theme", theme);
|
|
923
|
+
document.getElementById("theme").textContent = `theme: ${theme}`;
|
|
924
|
+
}
|
|
925
|
+
document.getElementById("theme").addEventListener("click", () => {
|
|
926
|
+
const current = storage.read("lemans-viewer.theme", "system");
|
|
927
|
+
applyTheme(THEMES[(THEMES.indexOf(current) + 1) % THEMES.length]);
|
|
928
|
+
});
|
|
929
|
+
applyTheme(new URLSearchParams(location.search).get("theme") || storage.read("lemans-viewer.theme", "system"));
|
|
930
|
+
|
|
931
|
+
window.addEventListener("hashchange", () => { state.starredTurnsOnly = false; state.turnQuery = ""; render(); });
|
|
932
|
+
render();
|
|
933
|
+
</script>
|
|
934
|
+
</body>
|
|
935
|
+
</html>
|
|
936
|
+
HTML
|
|
937
|
+
end
|
|
938
|
+
|
|
939
|
+
options = { runs_dir: "./runs", host: "127.0.0.1", port: 8089 }
|
|
940
|
+
|
|
941
|
+
OptionParser.new do |opts|
|
|
942
|
+
opts.banner = "Usage: lemans-viewer [--runs-dir DIR] [--port PORT] [--host HOST]"
|
|
943
|
+
opts.on("--runs-dir=DIR", "Directory holding run directories (default: ./runs)") { options[:runs_dir] = it }
|
|
944
|
+
opts.on("--port=PORT", Integer, "Port to listen on (default: 8089)") { options[:port] = it }
|
|
945
|
+
opts.on("--host=HOST", "Address to bind (default: 127.0.0.1)") { options[:host] = it }
|
|
946
|
+
opts.on("-v", "--version", "Print the lemans version") do
|
|
947
|
+
puts Lemans::VERSION
|
|
948
|
+
exit
|
|
949
|
+
end
|
|
950
|
+
end.parse!
|
|
951
|
+
|
|
952
|
+
begin
|
|
953
|
+
LemansViewer.serve(Lemans::Stores::FS.new(options[:runs_dir]), host: options[:host], port: options[:port])
|
|
954
|
+
rescue Interrupt
|
|
955
|
+
exit 130
|
|
956
|
+
rescue => e # rubocop:disable Style/RescueStandardError
|
|
957
|
+
raise e if $DEBUG
|
|
958
|
+
|
|
959
|
+
warn e.message
|
|
960
|
+
warn e.backtrace.take(10).join("\n") if ENV["LEMANS_DEBUG"] == "1"
|
|
961
|
+
exit 1
|
|
962
|
+
end
|