acuvo-code 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +328 -0
- package/ENTERPRISE.md +927 -0
- package/LICENSE +120 -0
- package/README.md +1245 -0
- package/ROADMAP.md +556 -0
- package/bin/acuvo-mcp.mjs +208 -0
- package/bin/acuvo.mjs +3108 -0
- package/lib/acceptance-consent.mjs +168 -0
- package/lib/acceptance.mjs +859 -0
- package/lib/account.mjs +226 -0
- package/lib/acuvo-dir.mjs +72 -0
- package/lib/acuvo-models.mjs +141 -0
- package/lib/apply-patch.mjs +570 -0
- package/lib/ask-user.mjs +173 -0
- package/lib/audit.mjs +530 -0
- package/lib/auto-lease.mjs +174 -0
- package/lib/background.mjs +842 -0
- package/lib/best-of.mjs +334 -0
- package/lib/board.mjs +232 -0
- package/lib/breaker.mjs +93 -0
- package/lib/budget.mjs +1375 -0
- package/lib/builtin-skills.mjs +135 -0
- package/lib/cache-floor.mjs +204 -0
- package/lib/chain.mjs +303 -0
- package/lib/changed-paths.mjs +84 -0
- package/lib/chat.mjs +434 -0
- package/lib/checkpoint.mjs +637 -0
- package/lib/child-lifetime.mjs +71 -0
- package/lib/cli-args.mjs +1255 -0
- package/lib/code-review.mjs +1382 -0
- package/lib/colour.mjs +82 -0
- package/lib/command.mjs +2847 -0
- package/lib/compact.mjs +1151 -0
- package/lib/completion.mjs +515 -0
- package/lib/creative-engines.mjs +779 -0
- package/lib/db-inspect.mjs +1624 -0
- package/lib/delete.mjs +111 -0
- package/lib/design-loop.mjs +570 -0
- package/lib/diff-preview.mjs +1044 -0
- package/lib/doctor.mjs +2139 -0
- package/lib/dropped.mjs +216 -0
- package/lib/edit-diagnostics.mjs +277 -0
- package/lib/edit.mjs +460 -0
- package/lib/env-file.mjs +250 -0
- package/lib/escalate.mjs +702 -0
- package/lib/evaluate.mjs +284 -0
- package/lib/fetch-text.mjs +952 -0
- package/lib/fleet-budget.mjs +256 -0
- package/lib/gh.mjs +1536 -0
- package/lib/git.mjs +1341 -0
- package/lib/github.mjs +261 -0
- package/lib/h2.mjs +194 -0
- package/lib/handoff.mjs +417 -0
- package/lib/hooks.mjs +626 -0
- package/lib/http-probe.mjs +907 -0
- package/lib/image-director.mjs +322 -0
- package/lib/image-edit.mjs +522 -0
- package/lib/imagegen.mjs +998 -0
- package/lib/interrupt.mjs +234 -0
- package/lib/learned.mjs +353 -0
- package/lib/lease-watch.mjs +115 -0
- package/lib/lease.mjs +868 -0
- package/lib/localize.mjs +834 -0
- package/lib/log-tail.mjs +1052 -0
- package/lib/login.mjs +157 -0
- package/lib/lsp.mjs +1613 -0
- package/lib/mcp-consent.mjs +377 -0
- package/lib/mcp-defaults.mjs +780 -0
- package/lib/mcp-server.mjs +1343 -0
- package/lib/mcp.mjs +1263 -0
- package/lib/media.mjs +1283 -0
- package/lib/memory-workspace.mjs +179 -0
- package/lib/model-json.mjs +132 -0
- package/lib/model-tier.mjs +171 -0
- package/lib/model.mjs +1445 -0
- package/lib/parallel.mjs +144 -0
- package/lib/perchance.mjs +210 -0
- package/lib/plan-coherence.mjs +1461 -0
- package/lib/plan-ledger.mjs +981 -0
- package/lib/plan.mjs +461 -0
- package/lib/policy.mjs +783 -0
- package/lib/prefix-order.mjs +38 -0
- package/lib/project-memory.mjs +127 -0
- package/lib/prompt.mjs +109 -0
- package/lib/python.mjs +862 -0
- package/lib/rcfile.mjs +853 -0
- package/lib/read-window.mjs +743 -0
- package/lib/refute-tools.mjs +34 -0
- package/lib/refute.mjs +806 -0
- package/lib/repl-driver.mjs +264 -0
- package/lib/repl.mjs +324 -0
- package/lib/replay.mjs +1218 -0
- package/lib/repo-map.mjs +1101 -0
- package/lib/report.mjs +419 -0
- package/lib/search-rank.mjs +99 -0
- package/lib/search.mjs +659 -0
- package/lib/secret-paths.mjs +54 -0
- package/lib/session.mjs +1017 -0
- package/lib/skills.mjs +703 -0
- package/lib/slash.mjs +356 -0
- package/lib/spawn-argv.mjs +1151 -0
- package/lib/spend.mjs +250 -0
- package/lib/steer.mjs +280 -0
- package/lib/stream.mjs +253 -0
- package/lib/stuck.mjs +712 -0
- package/lib/subagent.mjs +749 -0
- package/lib/terminal-graphics.mjs +171 -0
- package/lib/tool-prefix.mjs +226 -0
- package/lib/tool-shortlist.mjs +162 -0
- package/lib/tools.mjs +2333 -0
- package/lib/tsserver.mjs +423 -0
- package/lib/turn.mjs +5672 -0
- package/lib/untrusted-block.mjs +271 -0
- package/lib/verify-claim.mjs +299 -0
- package/lib/vision.mjs +330 -0
- package/lib/voice-task.mjs +561 -0
- package/lib/warm-provider.mjs +255 -0
- package/lib/websearch.mjs +401 -0
- package/lib/workspace.mjs +928 -0
- package/lib/write-approval.mjs +235 -0
- package/lib/write-many.mjs +162 -0
- package/package.json +62 -0
- package/scripts/bundle.mjs +768 -0
- package/scripts/cache-floor.mjs +176 -0
- package/scripts/machine.mjs +226 -0
- package/scripts/test.mjs +139 -0
- package/skills/accessibility.md +87 -0
- package/skills/acuvo-design-system.md +123 -0
- package/skills/animation.md +84 -0
- package/skills/api-design.md +82 -0
- package/skills/auth-and-sessions.md +78 -0
- package/skills/build-with-a-framework.md +101 -0
- package/skills/colour-and-contrast.md +112 -0
- package/skills/creative-engines.md +81 -0
- package/skills/css-layout.md +85 -0
- package/skills/data-and-charts.md +77 -0
- package/skills/debugging.md +76 -0
- package/skills/designing-by-looking.md +84 -0
- package/skills/error-handling.md +78 -0
- package/skills/forms-and-validation.md +93 -0
- package/skills/nextjs-app-router.md +75 -0
- package/skills/page-composition.md +103 -0
- package/skills/performance.md +77 -0
- package/skills/plan-before-building.md +52 -0
- package/skills/planning-and-delegating.md +72 -0
- package/skills/refactoring.md +70 -0
- package/skills/security-basics.md +76 -0
- package/skills/state-management.md +73 -0
- package/skills/supabase-multitenant.md +72 -0
- package/skills/typescript-strict.md +90 -0
- package/skills/typography.md +135 -0
- package/skills/verify-your-own-work.md +62 -0
- package/skills/web-app-quality.md +62 -0
- package/skills/working-in-the-background.md +64 -0
package/lib/replay.mjs
ADDED
|
@@ -0,0 +1,1218 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ── ⭐⭐ REPLAY — STEP THROUGH A RUN THAT ALREADY HAPPENED ───────────────────
|
|
3
|
+
*
|
|
4
|
+
* "It did something weird on Tuesday" has no answer anywhere in this category.
|
|
5
|
+
* It does here, and it costs nothing, because the two documents that answer it
|
|
6
|
+
* are already being written on every run:
|
|
7
|
+
*
|
|
8
|
+
* · `session.mjs` saves the whole conversation — every assistant thought,
|
|
9
|
+
* every tool call with its arguments, every result — secrets redacted on
|
|
10
|
+
* the way in.
|
|
11
|
+
* · `audit.mjs` appends one line per run — the verdict, the acceptance, the
|
|
12
|
+
* refusals, the money.
|
|
13
|
+
*
|
|
14
|
+
* This module turns those into a TIMELINE: round by round, what it thought,
|
|
15
|
+
* what it called, what came back, where it was refused, how it ended. Plus the
|
|
16
|
+
* two views that make it a tool rather than a log dump — a DIFF between two
|
|
17
|
+
* runs of the same task, and a FILTER for the one question you actually have.
|
|
18
|
+
*
|
|
19
|
+
* ── ⚠️⚠️ THE INVARIANT: REPLAY EXECUTES NOTHING ─────────────────────────────
|
|
20
|
+
*
|
|
21
|
+
* Not a file written, not a command run, not a model called. `session.mjs`
|
|
22
|
+
* holds the same line for resume — its `resumeMessages` returns `replayed:
|
|
23
|
+
* false` and its header explains why a resume that re-executes is "a command
|
|
24
|
+
* run twice by a user who typed it once". A replay that re-executes is worse:
|
|
25
|
+
* the user asked to LOOK at what happened, and looking is the one operation
|
|
26
|
+
* that must be safe on a run you already know went wrong.
|
|
27
|
+
*
|
|
28
|
+
* ⭐ THE INVARIANT IS ENFORCED BY THE IMPORT LIST, NOT BY DISCIPLINE. This file
|
|
29
|
+
* imports one pure function and nothing else — no filesystem, no process, no
|
|
30
|
+
* network, not one builtin. It is structurally incapable of the thing it must
|
|
31
|
+
* not do, and `replay.test.mjs` asserts that from both directions: it replays a
|
|
32
|
+
* record whose calls delete a real file and checks the file survives, AND it
|
|
33
|
+
* reads this source and fails if a single builtin import ever appears.
|
|
34
|
+
*
|
|
35
|
+
* That is why loading and saving live at the EDGES — the caller does the I/O:
|
|
36
|
+
*
|
|
37
|
+
* const loaded = loadSession(root, id); // session.mjs touches disk
|
|
38
|
+
* const replay = replaySession(loaded.session); // this file never does
|
|
39
|
+
*
|
|
40
|
+
* ── ⚠️ VALIDATE THE RECORD, DO NOT TRUST IT ────────────────────────────────
|
|
41
|
+
*
|
|
42
|
+
* A record can be from an older build, half-written by a killed process, or
|
|
43
|
+
* hand-edited by the person debugging with it. Every one of those must produce
|
|
44
|
+
* a sentence, never a stack trace — `plan-ledger.mjs`'s rule, and it applies
|
|
45
|
+
* harder here because the reader of a replay is already having a bad day.
|
|
46
|
+
*
|
|
47
|
+
* The split: IDENTITY fields are strict (a wrong version is a refusal, because
|
|
48
|
+
* misreading a future shape is how a replay quietly describes a different run);
|
|
49
|
+
* SECONDARY fields are tolerant and the repair is WARNED (a hand-edited record
|
|
50
|
+
* missing its `commands` array is still worth reading, and a silent default
|
|
51
|
+
* would be the lie).
|
|
52
|
+
*
|
|
53
|
+
* ── ⚠️ SECRETS ARE REDACTED ON THE WAY IN, AND AGAIN ON THE WAY OUT ────────
|
|
54
|
+
*
|
|
55
|
+
* `session.mjs` scrubs before it writes. This file scrubs again on read, and
|
|
56
|
+
* the belt-and-braces is deliberate: records get hand-edited, older builds had
|
|
57
|
+
* weaker redactors, and a key pasted into a transcript by a user is a shape no
|
|
58
|
+
* writer has yet been asked about. Redaction happens ONCE, here, in the data
|
|
59
|
+
* layer — so every renderer below (timeline, diff, filter) is safe by
|
|
60
|
+
* construction rather than by each remembering to be. There is no opt-out flag,
|
|
61
|
+
* because an opt-out flag is the thing someone eventually passes.
|
|
62
|
+
*
|
|
63
|
+
* ⚠️ AND REDACT BEFORE TRUNCATING, never after. `audit.mjs` names this exact
|
|
64
|
+
* ordering trap: a key just past the clamp is hidden today and printed the
|
|
65
|
+
* first time someone raises the limit, and the test still passes.
|
|
66
|
+
*/
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* ⚠️ THE ONLY IMPORT, AND IT IS A PURE FUNCTION.
|
|
70
|
+
*
|
|
71
|
+
* `redactSecrets` is `session.mjs`'s hardened redactor — the one that already
|
|
72
|
+
* knows `tokenCount = tokens.length` is code and `DB_PASSWORD=hunter2hunter` is
|
|
73
|
+
* not. A second copy here would be the copy that goes stale, and a redactor
|
|
74
|
+
* that has drifted is worse than an obvious hole: it reads as protection.
|
|
75
|
+
*/
|
|
76
|
+
import { redactSecrets } from './session.mjs';
|
|
77
|
+
|
|
78
|
+
/** Bump when the STEP SHAPE changes in a way a consumer must branch on. */
|
|
79
|
+
export const REPLAY_FORMAT_VERSION = 1;
|
|
80
|
+
|
|
81
|
+
/** The only `session.mjs` record version this file claims to understand. */
|
|
82
|
+
export const SUPPORTED_SESSION_VERSION = 1;
|
|
83
|
+
|
|
84
|
+
/** The only `audit.mjs` schema version this file claims to understand. */
|
|
85
|
+
export const SUPPORTED_AUDIT_VERSION = 1;
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Tools that PRODUCE A FILE in the workspace.
|
|
89
|
+
*
|
|
90
|
+
* ⚠️ Kept in step with `tools.mjs`'s `mutated: true` cases, and the membership
|
|
91
|
+
* is not obvious for two of them: `see_page` reads a page and writes a
|
|
92
|
+
* SCREENSHOT, and `speak` writes an audio file. Both are the kind of output a
|
|
93
|
+
* "what did this run write" question is asking about, and both were invisible
|
|
94
|
+
* to the first draft of this list.
|
|
95
|
+
*/
|
|
96
|
+
export const WRITING_TOOLS = new Set([
|
|
97
|
+
'write_file',
|
|
98
|
+
'edit_file',
|
|
99
|
+
'delete_file',
|
|
100
|
+
'make_document',
|
|
101
|
+
'generate_image',
|
|
102
|
+
'speak',
|
|
103
|
+
'see_page',
|
|
104
|
+
]);
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Tools that RUN A PROCESS.
|
|
108
|
+
*
|
|
109
|
+
* ⚠️ `git_commit` is here rather than in the writing set on purpose: it does
|
|
110
|
+
* not write a file the user asked for, it changes repository state by running
|
|
111
|
+
* git — and someone asking "what did this run execute" wants it in the answer.
|
|
112
|
+
* `check_acceptance` is here for the reason `turn.mjs` spells out at length:
|
|
113
|
+
* it spawns through the same gate `run_command` does, and a command does not
|
|
114
|
+
* become less run because a different verb asked for it.
|
|
115
|
+
*/
|
|
116
|
+
export const RUNNING_TOOLS = new Set([
|
|
117
|
+
'run_command',
|
|
118
|
+
'run_program',
|
|
119
|
+
'evaluate',
|
|
120
|
+
'check_acceptance',
|
|
121
|
+
'git_commit',
|
|
122
|
+
// ⚠️ Added with the delivery verbs (2026-08-14). `git_branch` moves HEAD and
|
|
123
|
+
// `git_push` writes to a REMOTE — if anything belongs in the answer to "what
|
|
124
|
+
// did this run execute", it is the one action other people can see.
|
|
125
|
+
'git_branch',
|
|
126
|
+
'git_push',
|
|
127
|
+
]);
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* How much of one step's text is carried in the DATA.
|
|
131
|
+
*
|
|
132
|
+
* ⚠️ The renderer clamps FURTHER, and the hidden-character count it prints is
|
|
133
|
+
* computed from `step.chars` — the ORIGINAL length — never from what survived
|
|
134
|
+
* this clamp. Two clamps in series, each reporting against its own input, is
|
|
135
|
+
* how a 500KB read gets described as "+800 characters".
|
|
136
|
+
*/
|
|
137
|
+
export const MAX_STEP_TEXT = 1_200;
|
|
138
|
+
|
|
139
|
+
/** How much of one step's text the timeline shows by default. */
|
|
140
|
+
export const MAX_RENDER_TEXT = 400;
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* ⚠️ THE ALIGNMENT IS O(n·m). Real runs are tens of steps and a quadratic walk
|
|
144
|
+
* over them is free; a pathological record is not this module's problem to
|
|
145
|
+
* solve heroically, so past this many steps the diff falls back to a positional
|
|
146
|
+
* comparison and SAYS it did. A slow correct answer nobody waits for is not an
|
|
147
|
+
* answer.
|
|
148
|
+
*/
|
|
149
|
+
export const MAX_ALIGNED_STEPS = 600;
|
|
150
|
+
|
|
151
|
+
/* ────────────────────────────────────────────────────────────────────────────
|
|
152
|
+
* PURE HELPERS
|
|
153
|
+
* ──────────────────────────────────────────────────────────────────────────── */
|
|
154
|
+
|
|
155
|
+
const isPlainObject = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Newlines normalised FIRST.
|
|
159
|
+
*
|
|
160
|
+
* ⚠️ Two reasons and both are load-bearing. A stray `\r` inside a rendered
|
|
161
|
+
* timeline moves the cursor to column zero and overwrites the line above it —
|
|
162
|
+
* on Windows, where every recorded command output is CRLF, that silently eats
|
|
163
|
+
* the indentation of the whole report. And the redactor's assignment rule is
|
|
164
|
+
* line-anchored (`^…$` with `/gm`), so a `DB_PASSWORD=…` line ending in `\r`
|
|
165
|
+
* ends with a non-newline character and does not match. Normalising is
|
|
166
|
+
* therefore cosmetic AND a security fix, which is not a coincidence: both bugs
|
|
167
|
+
* come from pretending `\r\n` is one character.
|
|
168
|
+
*/
|
|
169
|
+
function normalizeText(value) {
|
|
170
|
+
if (typeof value === 'string') return value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
|
171
|
+
if (value === undefined || value === null) return '';
|
|
172
|
+
// A multimodal round carries an array of parts. Keep it rather than dropping
|
|
173
|
+
// it — an empty step where content existed is a hole with no marker.
|
|
174
|
+
try {
|
|
175
|
+
return JSON.stringify(value).replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
|
176
|
+
} catch {
|
|
177
|
+
return String(value);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Normalise → redact → clamp, in that order, and report the TRUE length. */
|
|
182
|
+
function scrub(value, max = MAX_STEP_TEXT) {
|
|
183
|
+
const normalized = normalizeText(value);
|
|
184
|
+
const { text } = redactSecrets(normalized);
|
|
185
|
+
const chars = text.length;
|
|
186
|
+
if (chars <= max) return { text, chars };
|
|
187
|
+
return { text: text.slice(0, max), chars };
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Redact every string inside a parsed structure.
|
|
192
|
+
*
|
|
193
|
+
* ⚠️ WHY THE PARSED OBJECT IS REDACTED RATHER THAN THE JSON TEXT REPARSED. A
|
|
194
|
+
* replacement dropped into a JSON string could, in principle, break the quoting
|
|
195
|
+
* and turn a readable argument list into `null`. Walking the parsed value
|
|
196
|
+
* cannot produce invalid JSON because it never produces JSON at all — it edits
|
|
197
|
+
* leaves in place.
|
|
198
|
+
*/
|
|
199
|
+
function redactDeep(value, depth = 0) {
|
|
200
|
+
if (depth > 12) return value;
|
|
201
|
+
if (typeof value === 'string') return redactSecrets(normalizeText(value)).text;
|
|
202
|
+
if (Array.isArray(value)) return value.map((v) => redactDeep(v, depth + 1));
|
|
203
|
+
if (isPlainObject(value)) {
|
|
204
|
+
const out = {};
|
|
205
|
+
for (const [k, v] of Object.entries(value)) out[k] = redactDeep(v, depth + 1);
|
|
206
|
+
return out;
|
|
207
|
+
}
|
|
208
|
+
return value;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* One path spelling, so `lib\a.mjs` and `lib/a.mjs` are one file.
|
|
213
|
+
*
|
|
214
|
+
* ⚠️ NO CASE FOLDING. Windows is case-insensitive and Linux is not, and a
|
|
215
|
+
* replay of a Linux run read on a Windows laptop must not merge `Makefile` and
|
|
216
|
+
* `makefile` into one history. Getting the separator right is unambiguous;
|
|
217
|
+
* getting the case right is a guess about a machine that is not this one.
|
|
218
|
+
*/
|
|
219
|
+
function normalizePath(p) {
|
|
220
|
+
if (typeof p !== 'string' || p === '') return null;
|
|
221
|
+
return p.replace(/\\/g, '/').replace(/^\.\//, '');
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** Which files does this call name? `tools.mjs` spells the argument four ways. */
|
|
225
|
+
function pathsOf(args) {
|
|
226
|
+
if (!isPlainObject(args)) return [];
|
|
227
|
+
const found = [];
|
|
228
|
+
for (const key of ['path', 'file', 'filename', 'target', 'out']) {
|
|
229
|
+
const norm = normalizePath(args[key]);
|
|
230
|
+
if (norm && !found.includes(norm)) found.push(norm);
|
|
231
|
+
}
|
|
232
|
+
if (Array.isArray(args.paths)) {
|
|
233
|
+
for (const p of args.paths) {
|
|
234
|
+
const norm = normalizePath(p);
|
|
235
|
+
if (norm && !found.includes(norm)) found.push(norm);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
return found;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* ⭐ HOW A REFUSAL IS RECOGNISED, AND WHY IT IS NOT A GUESS.
|
|
243
|
+
*
|
|
244
|
+
* `turn.mjs`'s `toolResultText` has exactly one line for a failed tool:
|
|
245
|
+
*
|
|
246
|
+
* if (!result || result.ok !== true) return `${name} failed: ${error}`;
|
|
247
|
+
*
|
|
248
|
+
* So every refusal in every transcript begins with the tool's own name followed
|
|
249
|
+
* by ` failed: `, and nothing else does — a success is rendered by a per-tool
|
|
250
|
+
* formatter that never starts that way. Anchoring on the START of the content
|
|
251
|
+
* (not a search) is what keeps a `read_file` result whose FILE happens to
|
|
252
|
+
* contain that sentence from being reported as a refusal.
|
|
253
|
+
*/
|
|
254
|
+
function readRefusal(text, tool) {
|
|
255
|
+
if (typeof text !== 'string') return null;
|
|
256
|
+
const match = /^([A-Za-z_][A-Za-z0-9_]*) failed: ([\s\S]*)$/.exec(text);
|
|
257
|
+
if (!match) return null;
|
|
258
|
+
// When we know which tool answered, the prefix must be that tool. A record
|
|
259
|
+
// where they disagree is a record to be suspicious of, not to reinterpret.
|
|
260
|
+
if (tool && match[1] !== tool) return null;
|
|
261
|
+
return match[2].trim();
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* ── ⚠️⭐ A COMMAND THAT FAILED IS NOT A TOOL THAT FAILED ────────────────────
|
|
266
|
+
*
|
|
267
|
+
* `command.mjs` states the rule at the point it builds the result: "`ok: true`
|
|
268
|
+
* MEANS THE COMMAND RAN, NOT THAT IT PASSED. Those are different facts and
|
|
269
|
+
* conflating them is exactly how a loop ends up reporting success on a failing
|
|
270
|
+
* test." So a run of `npm test` that exits 1 is a SUCCESSFUL `run_command` —
|
|
271
|
+
* `ok: true`, no refusal, nothing in the transcript starting `run_command
|
|
272
|
+
* failed:`.
|
|
273
|
+
*
|
|
274
|
+
* ⚠️ WHICH MAKES IT THE ENTIRE POINT OF THE DIFF. "It passed once and failed
|
|
275
|
+
* once" is the question this module exists to answer, and until this function
|
|
276
|
+
* existed the two runs differed only in a wall of stdout — reported as
|
|
277
|
+
* "different output", buried among every other line that also differed. The
|
|
278
|
+
* exit code is the one number the reader wants and it was not being read.
|
|
279
|
+
*
|
|
280
|
+
* ⚠️ IT NEVER GUESSES. Unrecognised shape ⇒ `passed: null`, and `null` here
|
|
281
|
+
* means "this result does not say", never "it did not pass". A confidently
|
|
282
|
+
* wrong `false` in a debugging view is worse than an absent field — `report.mjs`
|
|
283
|
+
* makes the same argument for omitting `lines` rather than reporting zero.
|
|
284
|
+
*
|
|
285
|
+
* Two shapes, because there are two renderers:
|
|
286
|
+
* · `formatRunForModel` / `formatProgramRunForModel` emit a human line —
|
|
287
|
+
* `exit code: 1 (3.2s) — FAILED`. Both files, verified, emit it identically.
|
|
288
|
+
* · `evaluate` and the JSON default emit the result object itself, which is
|
|
289
|
+
* structured and needs no parsing at all.
|
|
290
|
+
*/
|
|
291
|
+
function readOutcome(text) {
|
|
292
|
+
const none = { passed: null, exitCode: null, timedOut: false };
|
|
293
|
+
if (typeof text !== 'string' || text === '') return none;
|
|
294
|
+
|
|
295
|
+
// The structured shape first — reading data beats parsing a rendering.
|
|
296
|
+
if (text.startsWith('{')) {
|
|
297
|
+
try {
|
|
298
|
+
const value = JSON.parse(text);
|
|
299
|
+
if (isPlainObject(value) && ('exitCode' in value || 'passed' in value)) {
|
|
300
|
+
return {
|
|
301
|
+
passed: typeof value.passed === 'boolean' ? value.passed : (typeof value.exitCode === 'number' ? value.exitCode === 0 : null),
|
|
302
|
+
exitCode: typeof value.exitCode === 'number' ? value.exitCode : null,
|
|
303
|
+
timedOut: value.timedOut === true,
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
} catch { /* not the structured shape; fall through to the rendered one */ }
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
if (/^TIMED OUT after /m.test(text)) return { passed: false, exitCode: null, timedOut: true };
|
|
310
|
+
|
|
311
|
+
const line = /^exit code: (-?\d+) \([0-9.]+s\)(?: — (PASSED|FAILED))?$/m.exec(text);
|
|
312
|
+
if (!line) return none;
|
|
313
|
+
return {
|
|
314
|
+
exitCode: Number(line[1]),
|
|
315
|
+
passed: line[2] ? line[2] === 'PASSED' : Number(line[1]) === 0,
|
|
316
|
+
timedOut: false,
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/* ────────────────────────────────────────────────────────────────────────────
|
|
321
|
+
* THE AUDIT LINE — the half of the story the session record does not hold
|
|
322
|
+
* ──────────────────────────────────────────────────────────────────────────── */
|
|
323
|
+
|
|
324
|
+
/**
|
|
325
|
+
* Pull the run-level verdict out of one `audit.mjs` JSONL record.
|
|
326
|
+
*
|
|
327
|
+
* ⭐ THE SESSION RECORD AND THE AUDIT LINE ANSWER DIFFERENT QUESTIONS, which is
|
|
328
|
+
* why replaying is better with both. The session holds WHAT HAPPENED, turn by
|
|
329
|
+
* turn. The audit holds WHETHER IT WAS ACCEPTABLE — the declared criterion, the
|
|
330
|
+
* verdict against it, the model chain that actually answered, the money. A
|
|
331
|
+
* timeline that ends "stopped: no-tool-calls" and cannot say the declared
|
|
332
|
+
* `npm test` was never satisfied is describing a success that was not one, and
|
|
333
|
+
* `report.mjs` already documents that exact failure happening in production.
|
|
334
|
+
*
|
|
335
|
+
* @param {unknown} record one parsed line from `.acuvo/audit/YYYY-MM-DD.jsonl`
|
|
336
|
+
* @returns {{ ok: true, at: string|null, acceptance: any, refusals: any[], changes: any[], verification: any, costUsd: number|null, tokens: number|null, model: any, stoppedBecause: string|null }
|
|
337
|
+
* | { ok: false, error: string }}
|
|
338
|
+
*/
|
|
339
|
+
export function auditContext(record) {
|
|
340
|
+
if (!isPlainObject(record)) {
|
|
341
|
+
return { ok: false, error: 'that is not an audit record — pass one parsed line from .acuvo/audit/<date>.jsonl' };
|
|
342
|
+
}
|
|
343
|
+
if (record.v !== SUPPORTED_AUDIT_VERSION) {
|
|
344
|
+
return {
|
|
345
|
+
ok: false,
|
|
346
|
+
error: `audit record schema v${record.v} cannot be read by this build, which understands v${SUPPORTED_AUDIT_VERSION}. The timeline is still readable without it.`,
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
if (!isPlainObject(record.run)) {
|
|
350
|
+
return { ok: false, error: 'this audit record has no `run` block, so there is nothing in it to attach to a timeline' };
|
|
351
|
+
}
|
|
352
|
+
const run = redactDeep(record.run);
|
|
353
|
+
return {
|
|
354
|
+
ok: true,
|
|
355
|
+
at: typeof record.at === 'string' ? record.at : null,
|
|
356
|
+
acceptance: isPlainObject(run.acceptance) ? run.acceptance : null,
|
|
357
|
+
refusals: Array.isArray(run.refusals) ? run.refusals : [],
|
|
358
|
+
changes: Array.isArray(run.changes) ? run.changes : [],
|
|
359
|
+
verification: isPlainObject(run.verification) ? run.verification : null,
|
|
360
|
+
costUsd: typeof run.costUsd === 'number' ? run.costUsd : null,
|
|
361
|
+
tokens: typeof run.tokens === 'number' ? run.tokens : null,
|
|
362
|
+
model: isPlainObject(run.model) ? run.model : null,
|
|
363
|
+
stoppedBecause: typeof run.stoppedBecause === 'string' ? run.stoppedBecause : null,
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
/* ────────────────────────────────────────────────────────────────────────────
|
|
368
|
+
* THE REPLAY
|
|
369
|
+
* ──────────────────────────────────────────────────────────────────────────── */
|
|
370
|
+
|
|
371
|
+
/**
|
|
372
|
+
* @typedef {object} ReplayStep
|
|
373
|
+
* @property {number} n index into `steps` — stable, and what a diff quotes
|
|
374
|
+
* @property {number} round 0 for the prologue, then 1..N, one per assistant turn
|
|
375
|
+
* @property {'system'|'task'|'note'|'gap'|'reasoning'|'call'|'result'|'verdict'} kind
|
|
376
|
+
*/
|
|
377
|
+
|
|
378
|
+
/**
|
|
379
|
+
* Turn a saved session record into an ordered, inert timeline.
|
|
380
|
+
*
|
|
381
|
+
* ⚠️ NOTHING HERE RUNS ANYTHING. See the header. `executed: false` is in the
|
|
382
|
+
* returned object so a caller can ASSERT the property rather than trust this
|
|
383
|
+
* comment — `resumeMessages` carries `replayed: false` for the same reason.
|
|
384
|
+
*
|
|
385
|
+
* @param {unknown} record a record from `loadSession(root, id).session`
|
|
386
|
+
* @param {{ onStep?: (step: ReplayStep) => void, audit?: unknown, maxText?: number }} [opts]
|
|
387
|
+
* @returns {{ ok: true, [k: string]: any } | { ok: false, error: string }}
|
|
388
|
+
*/
|
|
389
|
+
export function replaySession(record, opts = {}) {
|
|
390
|
+
const { onStep = null, audit = undefined, maxText = MAX_STEP_TEXT } = opts ?? {};
|
|
391
|
+
const warnings = [];
|
|
392
|
+
|
|
393
|
+
/* ── identity: strict ─────────────────────────────────────────────────── */
|
|
394
|
+
if (!isPlainObject(record)) {
|
|
395
|
+
const what = record === null ? 'null' : Array.isArray(record) ? 'an array' : typeof record;
|
|
396
|
+
return {
|
|
397
|
+
ok: false,
|
|
398
|
+
error: `there is no run to replay here — got ${what} where a saved session record was expected. `
|
|
399
|
+
+ 'Load one first: `loadSession(root, id).session`, or list what exists with `acuvo --sessions`.',
|
|
400
|
+
};
|
|
401
|
+
}
|
|
402
|
+
if (record.version !== SUPPORTED_SESSION_VERSION) {
|
|
403
|
+
return {
|
|
404
|
+
ok: false,
|
|
405
|
+
error: `this record is session format v${record.version}, and this build reads v${SUPPORTED_SESSION_VERSION}. `
|
|
406
|
+
+ 'It was written by a different version of Acuvo Code — the file is harmless where it is, but nothing here can honestly describe it.',
|
|
407
|
+
};
|
|
408
|
+
}
|
|
409
|
+
if (!Array.isArray(record.messages)) {
|
|
410
|
+
return {
|
|
411
|
+
ok: false,
|
|
412
|
+
error: 'this record has no `messages` array, so there is no conversation to step through. '
|
|
413
|
+
+ 'A run that died before its first round is saved and listable but holds no transcript.',
|
|
414
|
+
};
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/* ── secondary: tolerant, and the repair is said out loud ──────────────── */
|
|
418
|
+
const files = Array.isArray(record.files) ? redactDeep(record.files) : (warnings.push('this record has no `files` list — it was hand-edited or written by an older build; the transcript below is unaffected'), []);
|
|
419
|
+
const commands = Array.isArray(record.commands) ? redactDeep(record.commands) : (warnings.push('this record has no `commands` list — it was hand-edited or written by an older build; the transcript below is unaffected'), []);
|
|
420
|
+
if (!isPlainObject(record.usage)) warnings.push('this record has no `usage` block, so the cost of this run is unknown — not zero');
|
|
421
|
+
if (record.verification !== null && !isPlainObject(record.verification)) {
|
|
422
|
+
warnings.push('this record has no `verification` block, so whether anything was proven is unknown — not "no"');
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
/* ── the audit line, if one was handed in ──────────────────────────────── */
|
|
426
|
+
let auditInfo = null;
|
|
427
|
+
if (audit !== undefined && audit !== null) {
|
|
428
|
+
const ctx = auditContext(audit);
|
|
429
|
+
if (ctx.ok) auditInfo = ctx;
|
|
430
|
+
else warnings.push(`the audit line handed in could not be read (${ctx.error}) — the timeline below is from the session record alone`);
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
/* ── walk ─────────────────────────────────────────────────────────────── */
|
|
434
|
+
const steps = [];
|
|
435
|
+
const messages = record.messages;
|
|
436
|
+
const push = (step) => {
|
|
437
|
+
step.n = steps.length;
|
|
438
|
+
steps.push(step);
|
|
439
|
+
return step;
|
|
440
|
+
};
|
|
441
|
+
|
|
442
|
+
let i = 0;
|
|
443
|
+
while (i < messages.length && messages[i]?.role === 'system') {
|
|
444
|
+
const s = scrub(messages[i].content, maxText);
|
|
445
|
+
push({ kind: 'system', round: 0, text: s.text, chars: s.chars });
|
|
446
|
+
i += 1;
|
|
447
|
+
}
|
|
448
|
+
if (i < messages.length && messages[i]?.role === 'user') {
|
|
449
|
+
const s = scrub(messages[i].content, maxText);
|
|
450
|
+
push({ kind: 'task', round: 0, text: s.text, chars: s.chars });
|
|
451
|
+
i += 1;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* ⚠️ THE HOLES GO IN BEFORE THE BODY, WHERE THE HOLE ACTUALLY IS.
|
|
456
|
+
* `sanitizeMessages` drops the OLDEST droppable groups to fit the size cap
|
|
457
|
+
* and injects its own note at exactly this position; a replay that mentioned
|
|
458
|
+
* the gap in a footer would put the marker somewhere the run never was. A
|
|
459
|
+
* reader scanning a timeline top to bottom has to meet the hole in order or
|
|
460
|
+
* they will reason across it — the same failure the note in `session.mjs`
|
|
461
|
+
* exists to prevent, one layer up.
|
|
462
|
+
*/
|
|
463
|
+
const droppedGroups = Number.isInteger(record.droppedGroups) ? record.droppedGroups : 0;
|
|
464
|
+
const droppedIncomplete = Number.isInteger(record.droppedIncomplete) ? record.droppedIncomplete : 0;
|
|
465
|
+
if (droppedGroups > 0) {
|
|
466
|
+
push({
|
|
467
|
+
kind: 'gap',
|
|
468
|
+
round: 0,
|
|
469
|
+
reason: 'size',
|
|
470
|
+
count: droppedGroups,
|
|
471
|
+
text: `${droppedGroups} earlier round${droppedGroups === 1 ? '' : 's'} of this run were dropped when it was saved, to fit the record size limit. `
|
|
472
|
+
+ 'Their work may be on disk; it is not in this transcript.',
|
|
473
|
+
});
|
|
474
|
+
}
|
|
475
|
+
if (droppedIncomplete > 0) {
|
|
476
|
+
push({
|
|
477
|
+
kind: 'gap',
|
|
478
|
+
round: 0,
|
|
479
|
+
reason: 'incomplete',
|
|
480
|
+
count: droppedIncomplete,
|
|
481
|
+
text: `${droppedIncomplete} incomplete round${droppedIncomplete === 1 ? '' : 's'} were dropped when this run was saved — `
|
|
482
|
+
+ 'a tool call that nothing ever answered, which is a PENDING action rather than a record of one.',
|
|
483
|
+
});
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
/**
|
|
487
|
+
* ⭐ THE CALL INDEX IS BUILT AS WE GO, NOT UP FRONT, and that is what lets an
|
|
488
|
+
* orphan be spotted. A tool reply whose `tool_call_id` no preceding assistant
|
|
489
|
+
* message declared is not a result of this conversation — it is damage. It is
|
|
490
|
+
* kept (deleting evidence from a debugging view is the wrong instinct) and
|
|
491
|
+
* FLAGGED, so nobody reads it as the answer to the call above it.
|
|
492
|
+
*/
|
|
493
|
+
const callsById = new Map();
|
|
494
|
+
let round = 0;
|
|
495
|
+
let orphans = 0;
|
|
496
|
+
|
|
497
|
+
for (; i < messages.length; i += 1) {
|
|
498
|
+
const m = messages[i];
|
|
499
|
+
const role = m?.role;
|
|
500
|
+
|
|
501
|
+
if (role === 'assistant') {
|
|
502
|
+
round += 1;
|
|
503
|
+
const calls = Array.isArray(m.tool_calls) ? m.tool_calls : [];
|
|
504
|
+
const body = scrub(m.content, maxText);
|
|
505
|
+
const isLast = i === messages.length - 1;
|
|
506
|
+
|
|
507
|
+
if (body.chars > 0) {
|
|
508
|
+
// ⭐ The FINAL assistant message with no calls is the ANSWER, and it is
|
|
509
|
+
// a different thing from a mid-run thought. Collapsing the two makes a
|
|
510
|
+
// timeline where the conclusion looks like one more deliberation.
|
|
511
|
+
const kind = isLast && calls.length === 0 ? 'verdict' : 'reasoning';
|
|
512
|
+
push({ kind, round, text: body.text, chars: body.chars });
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
for (const c of calls) {
|
|
516
|
+
const tool = typeof c?.function?.name === 'string' ? c.function.name : 'unknown';
|
|
517
|
+
const raw = typeof c?.function?.arguments === 'string' ? c.function.arguments : '';
|
|
518
|
+
let parsed = null;
|
|
519
|
+
let argsParsed = false;
|
|
520
|
+
try {
|
|
521
|
+
const value = JSON.parse(raw === '' ? '{}' : raw);
|
|
522
|
+
if (isPlainObject(value)) { parsed = redactDeep(value); argsParsed = true; }
|
|
523
|
+
} catch { /* an unparseable call is a fact about the run, not a crash */ }
|
|
524
|
+
|
|
525
|
+
const rawScrub = scrub(raw, maxText);
|
|
526
|
+
push({
|
|
527
|
+
kind: 'call',
|
|
528
|
+
round,
|
|
529
|
+
tool,
|
|
530
|
+
callId: typeof c?.id === 'string' ? c.id : null,
|
|
531
|
+
args: parsed,
|
|
532
|
+
argsParsed,
|
|
533
|
+
argsRaw: rawScrub.text,
|
|
534
|
+
chars: rawScrub.chars,
|
|
535
|
+
// `session.mjs` replaces the arguments of a credential-touching call
|
|
536
|
+
// with `{"withheld":true}`. Surface that as a fact rather than
|
|
537
|
+
// rendering a mysterious one-key object.
|
|
538
|
+
withheld: parsed?.withheld === true,
|
|
539
|
+
paths: pathsOf(parsed),
|
|
540
|
+
writes: WRITING_TOOLS.has(tool),
|
|
541
|
+
runs: RUNNING_TOOLS.has(tool),
|
|
542
|
+
});
|
|
543
|
+
if (typeof c?.id === 'string') callsById.set(c.id, steps[steps.length - 1]);
|
|
544
|
+
}
|
|
545
|
+
continue;
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
if (role === 'tool') {
|
|
549
|
+
const callId = typeof m?.tool_call_id === 'string' ? m.tool_call_id : null;
|
|
550
|
+
const paired = callId ? callsById.get(callId) : undefined;
|
|
551
|
+
const tool = typeof m?.name === 'string' && m.name
|
|
552
|
+
? m.name
|
|
553
|
+
// Older records did not carry `name` on the reply; the call it answers does.
|
|
554
|
+
: (paired?.tool ?? 'unknown');
|
|
555
|
+
const body = scrub(m.content, maxText);
|
|
556
|
+
const refusalText = readRefusal(body.text, tool === 'unknown' ? null : tool);
|
|
557
|
+
const orphan = paired === undefined;
|
|
558
|
+
if (orphan) orphans += 1;
|
|
559
|
+
// ⚠️ Read from the FULL text, not the clamped one — the exit-code line is
|
|
560
|
+
// near the top, but a long stdout ahead of it would push a `TIMED OUT`
|
|
561
|
+
// marker past the clamp and turn a killed command into "does not say".
|
|
562
|
+
const outcome = refusalText === null ? readOutcome(normalizeText(m?.content)) : { passed: false, exitCode: null, timedOut: false };
|
|
563
|
+
|
|
564
|
+
push({
|
|
565
|
+
kind: 'result',
|
|
566
|
+
round: paired ? paired.round : round,
|
|
567
|
+
tool,
|
|
568
|
+
callId,
|
|
569
|
+
// ⭐ `ok` IS ABOUT THE TOOL. `passed` IS ABOUT THE COMMAND. Keeping them
|
|
570
|
+
// apart is `command.mjs`'s rule and this whole file inherits it.
|
|
571
|
+
ok: refusalText === null,
|
|
572
|
+
refusal: refusalText !== null,
|
|
573
|
+
error: refusalText,
|
|
574
|
+
passed: outcome.passed,
|
|
575
|
+
exitCode: outcome.exitCode,
|
|
576
|
+
timedOut: outcome.timedOut,
|
|
577
|
+
text: body.text,
|
|
578
|
+
chars: body.chars,
|
|
579
|
+
orphan,
|
|
580
|
+
writes: WRITING_TOOLS.has(tool),
|
|
581
|
+
runs: RUNNING_TOOLS.has(tool),
|
|
582
|
+
});
|
|
583
|
+
continue;
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
// A `user` message after the opening one: an interactive follow-up, or the
|
|
587
|
+
// marker `sanitizeMessages` injects where it removed rounds.
|
|
588
|
+
const body = scrub(m?.content, maxText);
|
|
589
|
+
push({ kind: 'note', round, role: role ?? 'unknown', text: body.text, chars: body.chars });
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
if (orphans > 0) {
|
|
593
|
+
warnings.push(
|
|
594
|
+
`${orphans} tool result${orphans === 1 ? '' : 's'} in this record answer a call that is not in the transcript (orphan). `
|
|
595
|
+
+ 'The record is damaged or was edited; those results are shown but are not attributed to any call.',
|
|
596
|
+
);
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
const roundsRecorded = Number.isInteger(record.roundsUsed) ? record.roundsUsed : null;
|
|
600
|
+
if (roundsRecorded !== null && roundsRecorded !== round) {
|
|
601
|
+
/**
|
|
602
|
+
* ⚠️⭐ MEASURED ON A REAL RUN, and it is not an error — it is the normal
|
|
603
|
+
* shape. `stoppedBecause: 'no-tool-calls'` means the loop ended on an
|
|
604
|
+
* assistant message that the saved `messages` array does not always carry,
|
|
605
|
+
* so the record counts one more round than the transcript shows. Saying so
|
|
606
|
+
* is the difference between a replay and a replay you can trust: without
|
|
607
|
+
* this line, the final answer is simply missing and nothing says why.
|
|
608
|
+
*/
|
|
609
|
+
warnings.push(
|
|
610
|
+
`the transcript holds ${round} round${round === 1 ? '' : 's'} but the record counts ${roundsRecorded}. `
|
|
611
|
+
+ (roundsRecorded > round
|
|
612
|
+
? "The run's final answer was not saved with the conversation — the timeline ends one step before the run did."
|
|
613
|
+
: 'The record undercounts its own transcript, which means it was edited.'),
|
|
614
|
+
);
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
for (const s of steps) if (onStep) onStep(s);
|
|
618
|
+
|
|
619
|
+
const counts = {
|
|
620
|
+
calls: steps.filter((s) => s.kind === 'call').length,
|
|
621
|
+
results: steps.filter((s) => s.kind === 'result').length,
|
|
622
|
+
refusals: steps.filter((s) => s.kind === 'result' && s.refusal).length,
|
|
623
|
+
writes: steps.filter((s) => s.kind === 'call' && s.writes).length,
|
|
624
|
+
runs: steps.filter((s) => s.kind === 'call' && s.runs).length,
|
|
625
|
+
reasoning: steps.filter((s) => s.kind === 'reasoning').length,
|
|
626
|
+
orphans,
|
|
627
|
+
};
|
|
628
|
+
|
|
629
|
+
const taskStep = steps.find((s) => s.kind === 'task');
|
|
630
|
+
return {
|
|
631
|
+
ok: true,
|
|
632
|
+
formatVersion: REPLAY_FORMAT_VERSION,
|
|
633
|
+
/**
|
|
634
|
+
* ⭐ THE PROPERTY, IN THE DATA. A caller — or a test — can assert on this
|
|
635
|
+
* rather than on a promise made in a comment.
|
|
636
|
+
*/
|
|
637
|
+
executed: false,
|
|
638
|
+
id: typeof record.id === 'string' ? record.id : null,
|
|
639
|
+
savedAt: typeof record.savedAt === 'string' ? record.savedAt : null,
|
|
640
|
+
root: typeof record.root === 'string' ? record.root : null,
|
|
641
|
+
task: scrub(record.task ?? taskStep?.text ?? '', maxText).text,
|
|
642
|
+
model: typeof record.model === 'string' ? record.model : (auditInfo?.model?.answered ?? null),
|
|
643
|
+
rounds: round,
|
|
644
|
+
roundsRecorded,
|
|
645
|
+
steps,
|
|
646
|
+
counts,
|
|
647
|
+
warnings,
|
|
648
|
+
outcome: {
|
|
649
|
+
stoppedBecause: typeof record.stoppedBecause === 'string' ? record.stoppedBecause : null,
|
|
650
|
+
error: record.error ? scrub(record.error, maxText).text : null,
|
|
651
|
+
resumable: record.resumable === true,
|
|
652
|
+
truncated: record.truncated === true,
|
|
653
|
+
verification: isPlainObject(record.verification) ? record.verification : (auditInfo?.verification ?? null),
|
|
654
|
+
files,
|
|
655
|
+
commands,
|
|
656
|
+
costUsd: typeof record.usage?.cost === 'number' ? record.usage.cost : (auditInfo?.costUsd ?? null),
|
|
657
|
+
tokens: typeof record.usage?.total_tokens === 'number' ? record.usage.total_tokens : (auditInfo?.tokens ?? null),
|
|
658
|
+
// Audit-sourced. The transcript's own refusals are `filterSteps(steps, 'refusals')`;
|
|
659
|
+
// these two answer different questions and must not be merged into one number.
|
|
660
|
+
acceptance: auditInfo?.acceptance ?? null,
|
|
661
|
+
refusals: auditInfo?.refusals ?? [],
|
|
662
|
+
modelChain: auditInfo?.model?.chain ?? null,
|
|
663
|
+
},
|
|
664
|
+
};
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
/* ────────────────────────────────────────────────────────────────────────────
|
|
668
|
+
* THE FILTERS — the question you actually have
|
|
669
|
+
* ──────────────────────────────────────────────────────────────────────────── */
|
|
670
|
+
|
|
671
|
+
const FILTERS = ['all', 'refusals', 'writes', 'runs', 'effects', 'reasoning'];
|
|
672
|
+
|
|
673
|
+
/**
|
|
674
|
+
* Narrow a timeline to one question.
|
|
675
|
+
*
|
|
676
|
+
* ⭐ A FILTERED CALL BRINGS ITS RESULT, AND A FILTERED RESULT BRINGS ITS CALL.
|
|
677
|
+
* This is the whole difference between a filter and a grep. "Show me the
|
|
678
|
+
* refusals" and getting `run_command failed: refused — not on the allowlist`
|
|
679
|
+
* with no sight of the command that was refused is a worse answer than the
|
|
680
|
+
* unfiltered log, because it looks complete.
|
|
681
|
+
*
|
|
682
|
+
* ⚠️ AN UNKNOWN FILTER THROWS. The alternative is returning `[]`, which reads
|
|
683
|
+
* exactly like "this run had no refusals" — a typo becoming a clean bill of
|
|
684
|
+
* health is the failure mode this whole package keeps being bitten by.
|
|
685
|
+
*
|
|
686
|
+
* @param {ReplayStep[]} steps
|
|
687
|
+
* @param {'all'|'refusals'|'writes'|'runs'|'effects'|'reasoning'|{file: string}} [spec]
|
|
688
|
+
* @returns {ReplayStep[]}
|
|
689
|
+
*/
|
|
690
|
+
export function filterSteps(steps, spec = 'all') {
|
|
691
|
+
if (!Array.isArray(steps)) return [];
|
|
692
|
+
|
|
693
|
+
if (isPlainObject(spec)) {
|
|
694
|
+
if (typeof spec.file !== 'string' || spec.file === '') {
|
|
695
|
+
throw new Error('filterSteps({ file }) needs a path — for example { file: "lib/turn.mjs" }');
|
|
696
|
+
}
|
|
697
|
+
const want = normalizePath(spec.file);
|
|
698
|
+
const wantedIds = new Set();
|
|
699
|
+
const keep = new Set();
|
|
700
|
+
for (const s of steps) {
|
|
701
|
+
if (s.kind !== 'call') continue;
|
|
702
|
+
if (!s.paths?.includes(want)) continue;
|
|
703
|
+
keep.add(s.n);
|
|
704
|
+
if (s.callId) wantedIds.add(s.callId);
|
|
705
|
+
}
|
|
706
|
+
for (const s of steps) {
|
|
707
|
+
if (s.kind === 'result' && s.callId && wantedIds.has(s.callId)) keep.add(s.n);
|
|
708
|
+
}
|
|
709
|
+
return steps.filter((s) => keep.has(s.n));
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
if (spec === 'all') return steps.slice();
|
|
713
|
+
if (spec === 'reasoning') return steps.filter((s) => s.kind === 'reasoning' || s.kind === 'verdict');
|
|
714
|
+
|
|
715
|
+
if (!FILTERS.includes(spec)) {
|
|
716
|
+
throw new Error(`"${spec}" is not a replay filter. The ones that exist are: ${FILTERS.join(', ')}, or { file: "<path>" }.`);
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
const wantsCall = (s) => {
|
|
720
|
+
if (spec === 'writes') return s.writes === true;
|
|
721
|
+
if (spec === 'runs') return s.runs === true;
|
|
722
|
+
if (spec === 'effects') return s.writes === true || s.runs === true;
|
|
723
|
+
return false;
|
|
724
|
+
};
|
|
725
|
+
|
|
726
|
+
const keep = new Set();
|
|
727
|
+
const wantedIds = new Set();
|
|
728
|
+
|
|
729
|
+
if (spec === 'refusals') {
|
|
730
|
+
for (const s of steps) {
|
|
731
|
+
if (s.kind === 'result' && s.refusal) { keep.add(s.n); if (s.callId) wantedIds.add(s.callId); }
|
|
732
|
+
}
|
|
733
|
+
for (const s of steps) if (s.kind === 'call' && s.callId && wantedIds.has(s.callId)) keep.add(s.n);
|
|
734
|
+
return steps.filter((s) => keep.has(s.n));
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
for (const s of steps) {
|
|
738
|
+
if (s.kind !== 'call' || !wantsCall(s)) continue;
|
|
739
|
+
keep.add(s.n);
|
|
740
|
+
if (s.callId) wantedIds.add(s.callId);
|
|
741
|
+
}
|
|
742
|
+
for (const s of steps) if (s.kind === 'result' && s.callId && wantedIds.has(s.callId)) keep.add(s.n);
|
|
743
|
+
return steps.filter((s) => keep.has(s.n));
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
/* ────────────────────────────────────────────────────────────────────────────
|
|
747
|
+
* THE DIFF — "it passed once and failed once. where did they split?"
|
|
748
|
+
* ──────────────────────────────────────────────────────────────────────────── */
|
|
749
|
+
|
|
750
|
+
/**
|
|
751
|
+
* ⚠️⭐ THE KEY IS THE ACTION, NOT THE PROSE.
|
|
752
|
+
*
|
|
753
|
+
* Two runs of one task never word a thought identically — that is what a
|
|
754
|
+
* sampled model IS. If the alignment keyed on text, every comparison would
|
|
755
|
+
* "diverge" at the first sentence and the feature would be worth nothing. So
|
|
756
|
+
* the alignment key is the SHAPE of the step (which tool, in which direction),
|
|
757
|
+
* and prose differences are reported as differences without being called a
|
|
758
|
+
* divergence.
|
|
759
|
+
*/
|
|
760
|
+
function alignKey(step) {
|
|
761
|
+
if (step.kind === 'call') return `call:${step.tool}`;
|
|
762
|
+
if (step.kind === 'result') return `result:${step.tool}`;
|
|
763
|
+
return step.kind;
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
const isAction = (step) => step.kind === 'call' || step.kind === 'result';
|
|
767
|
+
|
|
768
|
+
/**
|
|
769
|
+
* Longest common subsequence over alignment keys. Small inputs by construction;
|
|
770
|
+
* see MAX_ALIGNED_STEPS for the guard.
|
|
771
|
+
*/
|
|
772
|
+
function alignSteps(a, b) {
|
|
773
|
+
const n = a.length;
|
|
774
|
+
const m = b.length;
|
|
775
|
+
const ka = a.map(alignKey);
|
|
776
|
+
const kb = b.map(alignKey);
|
|
777
|
+
const table = [];
|
|
778
|
+
for (let i = 0; i <= n; i += 1) table.push(new Uint32Array(m + 1));
|
|
779
|
+
for (let i = n - 1; i >= 0; i -= 1) {
|
|
780
|
+
for (let j = m - 1; j >= 0; j -= 1) {
|
|
781
|
+
table[i][j] = ka[i] === kb[j]
|
|
782
|
+
? table[i + 1][j + 1] + 1
|
|
783
|
+
: Math.max(table[i + 1][j], table[i][j + 1]);
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
const pairs = [];
|
|
787
|
+
let i = 0;
|
|
788
|
+
let j = 0;
|
|
789
|
+
while (i < n && j < m) {
|
|
790
|
+
if (ka[i] === kb[j]) { pairs.push([a[i], b[j]]); i += 1; j += 1; continue; }
|
|
791
|
+
// ⚠️ THE TIE GOES TO "A DID SOMETHING EXTRA". It has to go somewhere, and
|
|
792
|
+
// reporting the left run's extra step first keeps the rows in the order a
|
|
793
|
+
// reader scans them: what A did, then what B did instead.
|
|
794
|
+
if (table[i + 1][j] >= table[i][j + 1]) { pairs.push([a[i], null]); i += 1; }
|
|
795
|
+
else { pairs.push([null, b[j]]); j += 1; }
|
|
796
|
+
}
|
|
797
|
+
while (i < n) { pairs.push([a[i], null]); i += 1; }
|
|
798
|
+
while (j < m) { pairs.push([null, b[j]]); j += 1; }
|
|
799
|
+
return pairs;
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
/** Positional fallback for pathological records — see MAX_ALIGNED_STEPS. */
|
|
803
|
+
function alignPositional(a, b) {
|
|
804
|
+
const pairs = [];
|
|
805
|
+
for (let k = 0; k < Math.max(a.length, b.length); k += 1) pairs.push([a[k] ?? null, b[k] ?? null]);
|
|
806
|
+
return pairs;
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
/** What, if anything, is different about an aligned pair? */
|
|
810
|
+
function compare(a, b) {
|
|
811
|
+
if (a.kind === 'call') {
|
|
812
|
+
if (a.argsRaw !== b.argsRaw) {
|
|
813
|
+
return { changed: true, prose: false, why: 'the same tool, called with different arguments' };
|
|
814
|
+
}
|
|
815
|
+
return { changed: false, prose: false, why: null };
|
|
816
|
+
}
|
|
817
|
+
if (a.kind === 'result') {
|
|
818
|
+
if (a.refusal !== b.refusal) {
|
|
819
|
+
const refused = a.refusal ? 'the first' : 'the second';
|
|
820
|
+
return { changed: true, prose: false, why: `the same call came back differently — ${refused} run was refused and the other was not` };
|
|
821
|
+
}
|
|
822
|
+
/**
|
|
823
|
+
* ⭐⭐ THE LINE THE WHOLE FEATURE IS FOR. Same command, same arguments, one
|
|
824
|
+
* green and one red — and because `ok` is true on both sides (the tool
|
|
825
|
+
* worked; the command did not) nothing above this catches it. Reported as
|
|
826
|
+
* its own sentence rather than folded into "different output", where it
|
|
827
|
+
* would sit indistinguishable among a hundred differing stdout lines.
|
|
828
|
+
*
|
|
829
|
+
* ⚠️ `null` IS NOT A VALUE HERE. A result whose shape does not state an
|
|
830
|
+
* outcome must not be compared against one that does — that is how "we
|
|
831
|
+
* could not tell" becomes "it failed".
|
|
832
|
+
*/
|
|
833
|
+
if (a.passed !== null && b.passed !== null && a.passed !== b.passed) {
|
|
834
|
+
const green = a.passed ? 'first' : 'second';
|
|
835
|
+
const red = a.passed ? 'second' : 'first';
|
|
836
|
+
const codes = `exit ${a.exitCode ?? '?'} vs exit ${b.exitCode ?? '?'}`;
|
|
837
|
+
return { changed: true, prose: false, why: `the same command PASSED in the ${green} run and FAILED in the ${red} (${codes})` };
|
|
838
|
+
}
|
|
839
|
+
if (a.timedOut !== b.timedOut) {
|
|
840
|
+
return { changed: true, prose: false, why: `the same command timed out in the ${a.timedOut ? 'first' : 'second'} run and not in the other` };
|
|
841
|
+
}
|
|
842
|
+
/**
|
|
843
|
+
* ⚠️ NO `a.ok !== b.ok` CHECK, AND ITS ABSENCE IS DELIBERATE. `ok` is
|
|
844
|
+
* defined as the complement of `refusal` one function above, so that branch
|
|
845
|
+
* could never be reached — it would be a line that looks like a safety net
|
|
846
|
+
* and catches nothing, which is the defect this codebase has shipped
|
|
847
|
+
* before. Caught here by a mutation test that stayed green.
|
|
848
|
+
*/
|
|
849
|
+
if (a.text !== b.text) return { changed: true, prose: false, why: 'the same call returned different output' };
|
|
850
|
+
return { changed: false, prose: false, why: null };
|
|
851
|
+
}
|
|
852
|
+
if (a.text !== b.text) return { changed: true, prose: true, why: 'different wording, same shape' };
|
|
853
|
+
return { changed: false, prose: false, why: null };
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
/** Accept a raw record or an already-built replay. */
|
|
857
|
+
function asReplay(value, side) {
|
|
858
|
+
if (isPlainObject(value) && value.ok === true && Array.isArray(value.steps)) return value;
|
|
859
|
+
const r = replaySession(value);
|
|
860
|
+
if (r.ok) return r;
|
|
861
|
+
return { ok: false, error: `the ${side} run could not be replayed: ${r.error}` };
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
/**
|
|
865
|
+
* Compare two runs and answer the one question worth asking: where did they
|
|
866
|
+
* stop doing the same thing?
|
|
867
|
+
*
|
|
868
|
+
* @param {unknown} runA a session record, or the result of `replaySession`
|
|
869
|
+
* @param {unknown} runB likewise
|
|
870
|
+
* @returns {{ ok: true, [k: string]: any } | { ok: false, error: string }}
|
|
871
|
+
*/
|
|
872
|
+
export function diffRuns(runA, runB) {
|
|
873
|
+
const a = asReplay(runA, 'first');
|
|
874
|
+
if (!a.ok) return a;
|
|
875
|
+
const b = asReplay(runB, 'second');
|
|
876
|
+
if (!b.ok) return b;
|
|
877
|
+
|
|
878
|
+
const degraded = a.steps.length > MAX_ALIGNED_STEPS || b.steps.length > MAX_ALIGNED_STEPS;
|
|
879
|
+
const pairs = degraded ? alignPositional(a.steps, b.steps) : alignSteps(a.steps, b.steps);
|
|
880
|
+
|
|
881
|
+
const rows = [];
|
|
882
|
+
const summary = { same: 0, changed: 0, onlyA: 0, onlyB: 0 };
|
|
883
|
+
let proseDiffers = false;
|
|
884
|
+
|
|
885
|
+
for (const [sa, sb] of pairs) {
|
|
886
|
+
if (sa && sb) {
|
|
887
|
+
const c = compare(sa, sb);
|
|
888
|
+
if (!c.changed) { rows.push({ kind: 'same', a: sa, b: sb, prose: false, why: null }); summary.same += 1; continue; }
|
|
889
|
+
if (c.prose) proseDiffers = true;
|
|
890
|
+
rows.push({ kind: 'changed', a: sa, b: sb, prose: c.prose, why: c.why });
|
|
891
|
+
summary.changed += 1;
|
|
892
|
+
continue;
|
|
893
|
+
}
|
|
894
|
+
if (sa) { rows.push({ kind: 'only-a', a: sa, b: null, prose: !isAction(sa), why: 'only the first run did this' }); summary.onlyA += 1; continue; }
|
|
895
|
+
rows.push({ kind: 'only-b', a: null, b: sb, prose: !isAction(sb), why: 'only the second run did this' }); summary.onlyB += 1;
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
/**
|
|
899
|
+
* ⚠️ DIVERGENCE IS COMPUTED OVER ACTIONS ONLY. See `alignKey`. A prose-only
|
|
900
|
+
* difference sets `proseDiffers` and nothing else — it is real, it is
|
|
901
|
+
* reported, and it is not the answer to "where did they split".
|
|
902
|
+
*/
|
|
903
|
+
let divergedAt = null;
|
|
904
|
+
let divergence = null;
|
|
905
|
+
for (let k = 0; k < rows.length; k += 1) {
|
|
906
|
+
const row = rows[k];
|
|
907
|
+
if (row.kind === 'same') continue;
|
|
908
|
+
const actionA = row.a && isAction(row.a);
|
|
909
|
+
const actionB = row.b && isAction(row.b);
|
|
910
|
+
if (!actionA && !actionB) continue;
|
|
911
|
+
|
|
912
|
+
divergedAt = k;
|
|
913
|
+
let stepA = row.a;
|
|
914
|
+
let stepB = row.b;
|
|
915
|
+
/**
|
|
916
|
+
* ⭐ A ONE-SIDED ROW IS HALF AN ANSWER. "A called edit_file" is only useful
|
|
917
|
+
* next to "and B called write_file instead", so the opposite side is taken
|
|
918
|
+
* from the nearest following row that supplies it. Without this the most
|
|
919
|
+
* common divergence — two runs choosing different tools — reports one tool
|
|
920
|
+
* and a null.
|
|
921
|
+
*/
|
|
922
|
+
if (row.kind === 'only-a') {
|
|
923
|
+
for (let p = k + 1; p < rows.length; p += 1) {
|
|
924
|
+
if (rows[p].kind === 'only-b' && rows[p].b && isAction(rows[p].b)) { stepB = rows[p].b; break; }
|
|
925
|
+
if (rows[p].kind === 'same') break;
|
|
926
|
+
}
|
|
927
|
+
} else if (row.kind === 'only-b') {
|
|
928
|
+
for (let p = k + 1; p < rows.length; p += 1) {
|
|
929
|
+
if (rows[p].kind === 'only-a' && rows[p].a && isAction(rows[p].a)) { stepA = rows[p].a; break; }
|
|
930
|
+
if (rows[p].kind === 'same') break;
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
divergence = {
|
|
934
|
+
row: k,
|
|
935
|
+
round: stepA?.round ?? stepB?.round ?? null,
|
|
936
|
+
a: stepA,
|
|
937
|
+
b: stepB,
|
|
938
|
+
why: row.why ?? 'the two runs took different actions here',
|
|
939
|
+
};
|
|
940
|
+
break;
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
return {
|
|
944
|
+
ok: true,
|
|
945
|
+
formatVersion: REPLAY_FORMAT_VERSION,
|
|
946
|
+
executed: false,
|
|
947
|
+
degraded,
|
|
948
|
+
sameTask: a.task === b.task,
|
|
949
|
+
a: { id: a.id, task: a.task, model: a.model, rounds: a.rounds, stoppedBecause: a.outcome.stoppedBecause, costUsd: a.outcome.costUsd },
|
|
950
|
+
b: { id: b.id, task: b.task, model: b.model, rounds: b.rounds, stoppedBecause: b.outcome.stoppedBecause, costUsd: b.outcome.costUsd },
|
|
951
|
+
rows,
|
|
952
|
+
summary,
|
|
953
|
+
proseDiffers,
|
|
954
|
+
divergedAt,
|
|
955
|
+
divergence,
|
|
956
|
+
};
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
/* ────────────────────────────────────────────────────────────────────────────
|
|
960
|
+
* RENDERING
|
|
961
|
+
* ──────────────────────────────────────────────────────────────────────────── */
|
|
962
|
+
|
|
963
|
+
const IDENTITY = (t) => t;
|
|
964
|
+
const painter = (paint) => ({
|
|
965
|
+
dim: paint?.dim ?? IDENTITY,
|
|
966
|
+
bold: paint?.bold ?? IDENTITY,
|
|
967
|
+
gold: paint?.gold ?? IDENTITY,
|
|
968
|
+
green: paint?.green ?? IDENTITY,
|
|
969
|
+
red: paint?.red ?? IDENTITY,
|
|
970
|
+
cyan: paint?.cyan ?? IDENTITY,
|
|
971
|
+
});
|
|
972
|
+
|
|
973
|
+
/**
|
|
974
|
+
* Show at most `max` characters, and be honest about the rest.
|
|
975
|
+
*
|
|
976
|
+
* ⚠️ THE HIDDEN COUNT IS AGAINST `chars` — the length BEFORE the data-layer
|
|
977
|
+
* clamp — not against what reached this function. See MAX_STEP_TEXT.
|
|
978
|
+
*/
|
|
979
|
+
function show(step, max) {
|
|
980
|
+
const text = String(step.text ?? '');
|
|
981
|
+
const total = Number.isInteger(step.chars) ? step.chars : text.length;
|
|
982
|
+
/**
|
|
983
|
+
* ⚠️ TRIMMED FOR DISPLAY ONLY, AND `chars` IS UNTOUCHED. Models end almost
|
|
984
|
+
* every message with a blank line or two; rendered verbatim they put a hole
|
|
985
|
+
* between every thought and the call it introduced. Trimming in the DATA
|
|
986
|
+
* would make the reported character count disagree with the record, which is
|
|
987
|
+
* the one thing a replay may never do.
|
|
988
|
+
*/
|
|
989
|
+
if (total <= max) return text.replace(/\s+$/, '');
|
|
990
|
+
return `${text.slice(0, max).replace(/\s+$/, '')}… [+${total - max} chars]`;
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
/** Indent every line of a block, so a multi-line thought stays inside its row. */
|
|
994
|
+
function indent(text, pad) {
|
|
995
|
+
return String(text).split('\n').map((line) => `${pad}${line}`).join('\n');
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
/** A one-line rendering of a call's arguments. */
|
|
999
|
+
function argsLine(step) {
|
|
1000
|
+
if (step.withheld) return '[withheld: this call touched a credential file]';
|
|
1001
|
+
if (!step.argsParsed) return `unparseable arguments: ${step.argsRaw.slice(0, 160)}`;
|
|
1002
|
+
const parts = [];
|
|
1003
|
+
for (const [k, v] of Object.entries(step.args ?? {})) {
|
|
1004
|
+
const value = typeof v === 'string'
|
|
1005
|
+
? (v.length > 60 ? `${JSON.stringify(v.slice(0, 60))}… (${v.length} chars)` : JSON.stringify(v))
|
|
1006
|
+
: JSON.stringify(v);
|
|
1007
|
+
parts.push(`${k}=${value}`);
|
|
1008
|
+
}
|
|
1009
|
+
return parts.join(' ') || '(no arguments)';
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
/**
|
|
1013
|
+
* Render a run as a readable timeline.
|
|
1014
|
+
*
|
|
1015
|
+
* ⚠️ THE "NOTHING WAS RE-RUN" LINE IS IN THE HEADER, NOT THE FOOTER. Someone
|
|
1016
|
+
* reading a timeline of a run that deleted their files needs to know it is a
|
|
1017
|
+
* recording BEFORE they read the deletions, not after.
|
|
1018
|
+
*
|
|
1019
|
+
* @param {any} replay the object `replaySession` returned
|
|
1020
|
+
* @param {{ paint?: any, maxText?: number, filter?: any }} [opts]
|
|
1021
|
+
* @returns {string}
|
|
1022
|
+
*/
|
|
1023
|
+
export function formatTimeline(replay, opts = {}) {
|
|
1024
|
+
const p = painter(opts.paint);
|
|
1025
|
+
const max = opts.maxText ?? MAX_RENDER_TEXT;
|
|
1026
|
+
|
|
1027
|
+
if (!isPlainObject(replay)) return 'nothing to replay — formatTimeline() needs the object replaySession() returned.\n';
|
|
1028
|
+
if (replay.ok !== true) return `${p.red('cannot replay this run')}\n ${String(replay.error ?? 'no reason given')}\n`;
|
|
1029
|
+
|
|
1030
|
+
const out = [];
|
|
1031
|
+
const money = typeof replay.outcome.costUsd === 'number' ? ` · $${replay.outcome.costUsd.toFixed(6)}` : '';
|
|
1032
|
+
out.push('');
|
|
1033
|
+
out.push(`${p.bold('run')} ${replay.id ?? '(no id)'} · ${replay.model ?? 'unknown model'} · ${replay.rounds} round${replay.rounds === 1 ? '' : 's'}${money}`);
|
|
1034
|
+
if (replay.task) out.push(`${p.dim('task')} ${show({ text: replay.task, chars: replay.task.length }, 160)}`);
|
|
1035
|
+
if (replay.root) out.push(`${p.dim('root')} ${replay.root}`);
|
|
1036
|
+
out.push(p.gold('⚠ REPLAY — nothing here was re-run. Every line below is what happened then.'));
|
|
1037
|
+
for (const w of replay.warnings) out.push(`${p.red('⚠')} ${w}`);
|
|
1038
|
+
out.push('');
|
|
1039
|
+
|
|
1040
|
+
const steps = opts.filter === undefined ? replay.steps : filterSteps(replay.steps, opts.filter);
|
|
1041
|
+
if (steps.length === 0) {
|
|
1042
|
+
out.push(' nothing was recorded for this run — it stopped before any of the conversation was saved.');
|
|
1043
|
+
out.push(` it ended: ${replay.outcome.stoppedBecause ?? 'unknown'}`);
|
|
1044
|
+
out.push('');
|
|
1045
|
+
return out.join('\n');
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
let shown = -1;
|
|
1049
|
+
for (const s of steps) {
|
|
1050
|
+
if (s.round !== shown && s.round > 0) {
|
|
1051
|
+
shown = s.round;
|
|
1052
|
+
out.push(p.dim(`── round ${s.round} ${'─'.repeat(Math.max(4, 46 - String(s.round).length))}`));
|
|
1053
|
+
}
|
|
1054
|
+
switch (s.kind) {
|
|
1055
|
+
case 'system':
|
|
1056
|
+
out.push(` ${p.dim('system ')} ${p.dim(`${s.chars} characters of instructions`)}`);
|
|
1057
|
+
break;
|
|
1058
|
+
case 'task':
|
|
1059
|
+
out.push(` ${p.bold('task ')} ${indent(show(s, max), ' ').trimStart()}`);
|
|
1060
|
+
break;
|
|
1061
|
+
case 'gap':
|
|
1062
|
+
out.push(` ${p.red('⚠ gap ')} ${indent(s.text, ' ').trimStart()}`);
|
|
1063
|
+
break;
|
|
1064
|
+
case 'note':
|
|
1065
|
+
out.push(` ${p.dim('note ')} ${indent(show(s, max), ' ').trimStart()}`);
|
|
1066
|
+
break;
|
|
1067
|
+
case 'reasoning':
|
|
1068
|
+
out.push(` ${p.dim('thought ')} ${indent(show(s, max), ' ').trimStart()}`);
|
|
1069
|
+
break;
|
|
1070
|
+
case 'call':
|
|
1071
|
+
out.push(` ${p.cyan('→ call ')} ${p.bold(s.tool)} ${p.dim(argsLine(s))}`);
|
|
1072
|
+
break;
|
|
1073
|
+
case 'result': {
|
|
1074
|
+
/**
|
|
1075
|
+
* ⚠️ A GREEN TICK NEXT TO A FAILING TEST IS THE LIE THIS PACKAGE KEEPS
|
|
1076
|
+
* FIGHTING. The tool succeeded and the command exited 1: three states,
|
|
1077
|
+
* not two, and they get three marks.
|
|
1078
|
+
*/
|
|
1079
|
+
const mark = s.refusal
|
|
1080
|
+
? p.red('✘ refused')
|
|
1081
|
+
: s.passed === false
|
|
1082
|
+
? p.red('✘ FAILED ')
|
|
1083
|
+
: s.passed === true
|
|
1084
|
+
? p.green('✔ passed ')
|
|
1085
|
+
: p.green('✔ result ');
|
|
1086
|
+
out.push(
|
|
1087
|
+
` ${mark} ${s.orphan ? p.red('(ORPHAN — answers no call in this transcript) ') : ''}`
|
|
1088
|
+
+ `${indent(show(s, max), ' ').trimStart()}`,
|
|
1089
|
+
);
|
|
1090
|
+
break;
|
|
1091
|
+
}
|
|
1092
|
+
case 'verdict':
|
|
1093
|
+
out.push(` ${p.bold('answer ')} ${indent(show(s, max), ' ').trimStart()}`);
|
|
1094
|
+
break;
|
|
1095
|
+
default:
|
|
1096
|
+
out.push(` ${s.kind}: ${show(s, max)}`);
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
out.push(p.dim(`── end ${'─'.repeat(44)}`));
|
|
1101
|
+
out.push(` ${p.dim('stopped ')} ${replay.outcome.stoppedBecause ?? 'unknown'}`);
|
|
1102
|
+
if (replay.outcome.error) out.push(` ${p.red('error ')} ${replay.outcome.error}`);
|
|
1103
|
+
const v = replay.outcome.verification;
|
|
1104
|
+
if (v) {
|
|
1105
|
+
out.push(` ${p.dim('verified')} ${v.ran ? 'ran' : 'NOTHING RAN'} · ${v.passed === true ? p.green('passed') : p.red('did not pass')} · ${v.command ?? 'no command'}`);
|
|
1106
|
+
}
|
|
1107
|
+
const acc = replay.outcome.acceptance;
|
|
1108
|
+
if (acc) {
|
|
1109
|
+
const unmet = (acc.unmet ?? []).map((u) => u?.command).filter(Boolean).join(', ');
|
|
1110
|
+
out.push(` ${p.dim('accepted')} ${acc.verdict ?? 'unknown'}${acc.gating ? ' (gating)' : ''}${unmet ? ` · unmet: ${unmet}` : ''}`);
|
|
1111
|
+
}
|
|
1112
|
+
out.push(` ${p.dim('counts ')} ${replay.counts.calls} calls · ${replay.counts.refusals} refused · ${replay.counts.writes} writes · ${replay.counts.runs} runs`);
|
|
1113
|
+
out.push('');
|
|
1114
|
+
return out.join('\n');
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
/** One side of a diff row, rendered short. */
|
|
1118
|
+
function rowLabel(step) {
|
|
1119
|
+
if (!step) return '';
|
|
1120
|
+
if (step.kind === 'call') return `call ${step.tool} ${argsLine(step)}`;
|
|
1121
|
+
if (step.kind === 'result') {
|
|
1122
|
+
const state = step.refusal ? 'refused' : step.passed === false ? 'FAILED' : step.passed === true ? 'passed' : 'result';
|
|
1123
|
+
return `${state} ${step.tool}${step.exitCode === null ? '' : ` (exit ${step.exitCode})`}`;
|
|
1124
|
+
}
|
|
1125
|
+
return step.kind;
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
/**
|
|
1129
|
+
* Render a comparison of two runs.
|
|
1130
|
+
*
|
|
1131
|
+
* ⭐ THE DIVERGENCE IS AT THE TOP. It is the answer; the row list is the
|
|
1132
|
+
* working. A reader who has two runs open already knows they differ.
|
|
1133
|
+
*
|
|
1134
|
+
* @param {any} diff the object `diffRuns` returned
|
|
1135
|
+
* @param {{ paint?: any, rows?: number }} [opts]
|
|
1136
|
+
* @returns {string}
|
|
1137
|
+
*/
|
|
1138
|
+
export function formatDiff(diff, opts = {}) {
|
|
1139
|
+
const p = painter(opts.paint);
|
|
1140
|
+
if (!isPlainObject(diff)) return 'nothing to compare — formatDiff() needs the object diffRuns() returned.\n';
|
|
1141
|
+
if (diff.ok !== true) return `${p.red('cannot compare these runs')}\n ${String(diff.error ?? 'no reason given')}\n`;
|
|
1142
|
+
|
|
1143
|
+
const out = [''];
|
|
1144
|
+
out.push(`${p.bold('diff')} ${diff.a.id ?? '(a)'} ↔ ${diff.b.id ?? '(b)'}`);
|
|
1145
|
+
out.push(diff.sameTask
|
|
1146
|
+
? `${p.dim('task')} the same on both sides`
|
|
1147
|
+
: p.red(`⚠ DIFFERENT TASKS — these two runs were not asked the same thing:\n A: ${diff.a.task}\n B: ${diff.b.task}`));
|
|
1148
|
+
out.push(`${p.dim(' A')} ${diff.a.rounds} rounds · stopped ${diff.a.stoppedBecause ?? 'unknown'}`);
|
|
1149
|
+
out.push(`${p.dim(' B')} ${diff.b.rounds} rounds · stopped ${diff.b.stoppedBecause ?? 'unknown'}`);
|
|
1150
|
+
if (diff.degraded) out.push(p.red('⚠ these runs are too long to align properly; rows below are compared by position, not by shape'));
|
|
1151
|
+
out.push('');
|
|
1152
|
+
|
|
1153
|
+
if (diff.divergence) {
|
|
1154
|
+
const d = diff.divergence;
|
|
1155
|
+
out.push(p.red(`⚠ DIVERGED at round ${d.round ?? '?'} — ${d.why}`));
|
|
1156
|
+
out.push(` A: ${rowLabel(d.a) || '(nothing)'}`);
|
|
1157
|
+
out.push(` B: ${rowLabel(d.b) || '(nothing)'}`);
|
|
1158
|
+
} else {
|
|
1159
|
+
/**
|
|
1160
|
+
* ── ⚠️ THE VERDICT USED TO CONTRADICT ITS OWN SUMMARY ────────────────────
|
|
1161
|
+
*
|
|
1162
|
+
* `divergence` is only set when two ACTIONS disagree AT THE SAME POSITION.
|
|
1163
|
+
* A run where one side simply did MORE has no divergence point at all, so
|
|
1164
|
+
* this printed "both runs took the same actions, in the same order" one line
|
|
1165
|
+
* above a summary reading "0 only in A · 2 only in B".
|
|
1166
|
+
*
|
|
1167
|
+
* ⭐ A reader cannot reconcile those, and when a tool contradicts itself the
|
|
1168
|
+
* honest assumption is that the reassuring half is the wrong one. Observed
|
|
1169
|
+
* on a real pair of runs.
|
|
1170
|
+
*
|
|
1171
|
+
* ⚠️ EXTRA PROSE IS NOT AN EXTRA ACTION, and the distinction is already
|
|
1172
|
+
* carried on every row (`prose: !isAction(step)`). A model that says more
|
|
1173
|
+
* has not done more, so the two cases get different sentences rather than
|
|
1174
|
+
* one hedge covering both.
|
|
1175
|
+
*/
|
|
1176
|
+
const extraRows = (diff.rows ?? []).filter((r) => r.kind === 'only-a' || r.kind === 'only-b');
|
|
1177
|
+
const extraActions = extraRows.filter((r) => !r.prose).length;
|
|
1178
|
+
const extraProse = extraRows.length - extraActions;
|
|
1179
|
+
|
|
1180
|
+
if (extraActions > 0) {
|
|
1181
|
+
const side = diff.summary.onlyA > 0 && diff.summary.onlyB > 0
|
|
1182
|
+
? 'each run'
|
|
1183
|
+
: (diff.summary.onlyA > 0 ? 'the FIRST run' : 'the SECOND run');
|
|
1184
|
+
out.push(p.red(`⚠ NO DIVERGENCE POINT, BUT ${side} DID MORE — ${extraActions} action${extraActions === 1 ? '' : 's'} appear on one side only.`));
|
|
1185
|
+
out.push(p.dim(' Nothing contradicts; one run simply went further. Read the rows marked A-only / B-only.'));
|
|
1186
|
+
} else {
|
|
1187
|
+
out.push(p.green('✔ no divergence — both runs took the same actions, in the same order.'));
|
|
1188
|
+
if (extraProse > 0) {
|
|
1189
|
+
out.push(p.dim(` (${extraProse} extra line${extraProse === 1 ? '' : 's'} of commentary on one side — said more, did not do more)`));
|
|
1190
|
+
}
|
|
1191
|
+
if (diff.proseDiffers) {
|
|
1192
|
+
out.push(p.dim(' (the wording differs, which is what a sampled model does and is not a divergence)'));
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
out.push('');
|
|
1197
|
+
|
|
1198
|
+
const limit = opts.rows ?? 60;
|
|
1199
|
+
let printed = 0;
|
|
1200
|
+
for (const row of diff.rows) {
|
|
1201
|
+
if (row.kind === 'same') continue;
|
|
1202
|
+
if (printed >= limit) { out.push(p.dim(` … ${diff.rows.length - printed} more rows`)); break; }
|
|
1203
|
+
printed += 1;
|
|
1204
|
+
const mark = row.kind === 'changed' ? '≠' : row.kind === 'only-a' ? 'A' : 'B';
|
|
1205
|
+
const paintRow = row.prose ? p.dim : IDENTITY;
|
|
1206
|
+
if (row.kind === 'changed') {
|
|
1207
|
+
out.push(paintRow(` ${mark} ${rowLabel(row.a)}`));
|
|
1208
|
+
out.push(paintRow(` ${' '.repeat(0)}→ ${rowLabel(row.b)} ${p.dim(row.why ?? '')}`));
|
|
1209
|
+
} else {
|
|
1210
|
+
out.push(paintRow(` ${mark} ${rowLabel(row.a ?? row.b)}`));
|
|
1211
|
+
}
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1214
|
+
out.push('');
|
|
1215
|
+
out.push(`${p.dim('summary')} ${diff.summary.same} same · ${diff.summary.changed} changed · ${diff.summary.onlyA} only in A · ${diff.summary.onlyB} only in B`);
|
|
1216
|
+
out.push('');
|
|
1217
|
+
return out.join('\n');
|
|
1218
|
+
}
|