@basein/runner 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/bir-hooks.js +5 -0
- package/dist/bin/bir.js +40 -1
- package/dist/bin/investigate.d.ts +195 -0
- package/dist/bin/investigate.js +580 -0
- package/dist/control/server.js +15 -4
- package/dist/record/recorder.d.ts +6 -0
- package/dist/record/remote-recorder.js +2 -0
- package/dist/replay/controller.d.ts +26 -1
- package/dist/replay/controller.js +78 -20
- package/dist/replay/handover.js +5 -0
- package/dist/replay/plan.d.ts +2 -0
- package/dist/replay/plan.js +53 -6
- package/dist/replay/pricing.d.ts +1 -1
- package/dist/replay/pricing.js +12 -4
- package/dist/replay/tool-error.d.ts +15 -0
- package/dist/replay/tool-error.js +17 -0
- package/dist/replay/types.d.ts +48 -1
- package/dist/util/journal.d.ts +38 -0
- package/dist/util/journal.js +97 -0
- package/dist/util/log.js +4 -0
- package/docs/calculatedReplayGuide.md +34 -0
- package/package.json +1 -1
|
@@ -0,0 +1,580 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `bir investigate` — why did this turn do what it did, and what did it cost?
|
|
3
|
+
* (docs/calculatedReplayGuide.md §9.1)
|
|
4
|
+
*
|
|
5
|
+
* Two sources, merged:
|
|
6
|
+
*
|
|
7
|
+
* 1. The **journal** this directory's `bir-hooks` keeps (util/journal.ts):
|
|
8
|
+
* the run boundaries, the match, every gate decision, the plan's mode and
|
|
9
|
+
* coverage, the replay outcome and the execution report. This is the only
|
|
10
|
+
* record of a *matched* turn, because the service deliberately keeps no
|
|
11
|
+
* run for one.
|
|
12
|
+
* 2. The **service**'s `GET /investigate/:id`: the recording, its scenario,
|
|
13
|
+
* the baseline, the execution ledger and the service's own findings.
|
|
14
|
+
* Owner-only; an admin may look at anyone's.
|
|
15
|
+
*
|
|
16
|
+
* Each side diagnoses what only it can see. The journal knows which gate
|
|
17
|
+
* declined and what mode armed; the service knows the baseline, the ledger and
|
|
18
|
+
* whether a run ever became a recording. Both lists are printed, worst first.
|
|
19
|
+
*/
|
|
20
|
+
const num = (v) => {
|
|
21
|
+
if (v === undefined || v === null || v === "")
|
|
22
|
+
return undefined;
|
|
23
|
+
const n = Number(v);
|
|
24
|
+
return Number.isFinite(n) ? n : undefined;
|
|
25
|
+
};
|
|
26
|
+
const str = (v) => (v === undefined || v === null ? undefined : String(v));
|
|
27
|
+
const bool = (v) => v === true || v === "true" ? true : v === false || v === "false" ? false : undefined;
|
|
28
|
+
const list = (v) => {
|
|
29
|
+
const s = str(v);
|
|
30
|
+
return s ? s.split(",").filter(Boolean) : undefined;
|
|
31
|
+
};
|
|
32
|
+
const INCIDENTS = new Set([
|
|
33
|
+
"replay.arm_failed",
|
|
34
|
+
"replay.derive_failed",
|
|
35
|
+
"replay.compose_failed",
|
|
36
|
+
"replay.flatten_failed",
|
|
37
|
+
"replay.thread_failed",
|
|
38
|
+
"replay.step_failed",
|
|
39
|
+
"replay.failed",
|
|
40
|
+
"replay.diverge",
|
|
41
|
+
"replay.handover",
|
|
42
|
+
"replay.thread_fallback",
|
|
43
|
+
"run.lossy",
|
|
44
|
+
"run.unmeasured",
|
|
45
|
+
"run.prompt_late",
|
|
46
|
+
]);
|
|
47
|
+
/**
|
|
48
|
+
* Group journal entries into turns. A turn opens at `run.start` and closes at
|
|
49
|
+
* `run.finish`; an entry joins the turn whose local run id it names, else the
|
|
50
|
+
* turn whose *matched* run id it names (the replay lines log the service's run),
|
|
51
|
+
* else the latest turn — which is where an older build's `execution.reported`,
|
|
52
|
+
* arriving after `run.finish` with no run id, belongs.
|
|
53
|
+
*/
|
|
54
|
+
export function turnsFromJournal(entries) {
|
|
55
|
+
const turns = [];
|
|
56
|
+
const owner = (e) => {
|
|
57
|
+
const run = str(e.run);
|
|
58
|
+
// The newest turn that is, or matched, this run. Newest wins: once a
|
|
59
|
+
// recording has been matched, `plan.armed run=<its id>` is about the
|
|
60
|
+
// matching turn, not about the recording's own (long finished) turn.
|
|
61
|
+
if (run) {
|
|
62
|
+
for (let i = turns.length - 1; i >= 0; i -= 1) {
|
|
63
|
+
if (turns[i].runId === run || turns[i].matched?.runId === run)
|
|
64
|
+
return turns[i];
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
if (e.event === "execution.reported") {
|
|
68
|
+
for (let i = turns.length - 1; i >= 0; i -= 1) {
|
|
69
|
+
if (turns[i].plan && !turns[i].reported)
|
|
70
|
+
return turns[i];
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return turns[turns.length - 1];
|
|
74
|
+
};
|
|
75
|
+
for (const e of entries) {
|
|
76
|
+
if (e.event === "run.start") {
|
|
77
|
+
const runId = str(e.run) ?? `unknown-${turns.length}`;
|
|
78
|
+
const turn = { runId, sessionId: str(e.sess), startedAt: e.at, incidents: [], events: [e] };
|
|
79
|
+
turns.push(turn);
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
const turn = owner(e);
|
|
83
|
+
if (!turn)
|
|
84
|
+
continue;
|
|
85
|
+
turn.events.push(e);
|
|
86
|
+
switch (e.event) {
|
|
87
|
+
case "run.prompt":
|
|
88
|
+
turn.prompt = str(e.prompt);
|
|
89
|
+
break;
|
|
90
|
+
case "run.matched":
|
|
91
|
+
turn.matched = { runId: str(e.matchedRun) ?? "", scenarioId: str(e.scenario), similarity: num(e.similarity) };
|
|
92
|
+
break;
|
|
93
|
+
case "replay.decision":
|
|
94
|
+
turn.decision = {
|
|
95
|
+
verdict: str(e.verdict) ?? "",
|
|
96
|
+
code: str(e.code),
|
|
97
|
+
why: str(e.why),
|
|
98
|
+
threshold: num(e.threshold),
|
|
99
|
+
};
|
|
100
|
+
break;
|
|
101
|
+
case "plan.armed":
|
|
102
|
+
turn.plan = {
|
|
103
|
+
mode: str(e.mode) ?? "",
|
|
104
|
+
steps: num(e.steps),
|
|
105
|
+
coverage: list(e.coverage),
|
|
106
|
+
tools: list(e.tools),
|
|
107
|
+
scenarioId: str(e.scenario),
|
|
108
|
+
by: str(e.by),
|
|
109
|
+
};
|
|
110
|
+
break;
|
|
111
|
+
case "replay.derived":
|
|
112
|
+
turn.derived = { params: num(e.params), costUsd: num(e.costUsd), source: str(e.source) };
|
|
113
|
+
break;
|
|
114
|
+
case "replay.done":
|
|
115
|
+
turn.done = { outcome: str(e.outcome) ?? "", steps: str(e.steps), ms: num(e.ms) };
|
|
116
|
+
break;
|
|
117
|
+
case "execution.reported":
|
|
118
|
+
turn.reported = {
|
|
119
|
+
outcome: str(e.outcome) ?? "",
|
|
120
|
+
derive: num(e.derive) ?? 0,
|
|
121
|
+
session: num(e.session) ?? 0,
|
|
122
|
+
fallback: num(e.fallback) ?? 0,
|
|
123
|
+
savedUsd: num(e.savedUsd),
|
|
124
|
+
measured: bool(e.measured),
|
|
125
|
+
ticket: str(e.ticket),
|
|
126
|
+
scenarioId: str(e.scenario),
|
|
127
|
+
declined: str(e.declined),
|
|
128
|
+
};
|
|
129
|
+
break;
|
|
130
|
+
case "run.finish":
|
|
131
|
+
turn.finishedAt = e.at;
|
|
132
|
+
turn.finish = {
|
|
133
|
+
steps: num(e.steps),
|
|
134
|
+
durationMs: num(e.durationMs),
|
|
135
|
+
costUsd: num(e.costUsd),
|
|
136
|
+
measured: bool(e.measured),
|
|
137
|
+
recorded: bool(e.recorded),
|
|
138
|
+
};
|
|
139
|
+
break;
|
|
140
|
+
default:
|
|
141
|
+
if (INCIDENTS.has(e.event))
|
|
142
|
+
turn.incidents.push(e);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return turns;
|
|
146
|
+
}
|
|
147
|
+
/** The turn an id names: its own run, the run it matched, its scenario or its ticket. */
|
|
148
|
+
export function findTurn(turns, id) {
|
|
149
|
+
for (let i = turns.length - 1; i >= 0; i -= 1) {
|
|
150
|
+
const t = turns[i];
|
|
151
|
+
if (t.runId === id ||
|
|
152
|
+
t.matched?.runId === id ||
|
|
153
|
+
t.matched?.scenarioId === id ||
|
|
154
|
+
t.plan?.scenarioId === id ||
|
|
155
|
+
t.reported?.scenarioId === id ||
|
|
156
|
+
t.reported?.ticket === id) {
|
|
157
|
+
return t;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
return undefined;
|
|
161
|
+
}
|
|
162
|
+
/** The runner's own gate vocabulary, with the cause and the fix (guide §9). */
|
|
163
|
+
export const DECLINES = {
|
|
164
|
+
replay_disabled: {
|
|
165
|
+
cause: "calculated replay was off in this bir-hooks (BIR_REPLAY=0), or a sub-task was handed out with BIR_SEGMENT_ARM off.",
|
|
166
|
+
fix: "start bir-hooks with BIR_REPLAY=1 (the default); for sub-tasks set BIR_SEGMENT_ARM=1.",
|
|
167
|
+
},
|
|
168
|
+
not_ready: {
|
|
169
|
+
cause: "the matched recording had no ready scenario at that moment.",
|
|
170
|
+
fix: "wait for the calculation, or start one with `bir scenario calc <runId>`, then run the prompt again.",
|
|
171
|
+
},
|
|
172
|
+
similarity: {
|
|
173
|
+
cause: "the prompt was a hit on the recording but below the steering threshold (BIR_REPLAY_MIN_SIMILARITY, default 0.92).",
|
|
174
|
+
fix: "phrase the prompt closer to the recorded one, or lower BIR_REPLAY_MIN_SIMILARITY.",
|
|
175
|
+
},
|
|
176
|
+
coverage: {
|
|
177
|
+
cause: "no step of the scenario could run anywhere: its tools are neither wrapped MCP servers registered in this session nor built-ins.",
|
|
178
|
+
fix: "wrap the servers the scenario uses (`bir install --server <name>`) and start a fresh session; `bir doctor` lists what is registered.",
|
|
179
|
+
},
|
|
180
|
+
known_bad_first_step: {
|
|
181
|
+
cause: "the chain's first step is parked after repeated failures.",
|
|
182
|
+
fix: "wait: the service retries it once an hour for a day, then repairs the plan from the next run of the prompt. To repair now: `bir scenario calc <runId> --force`.",
|
|
183
|
+
},
|
|
184
|
+
nondeterministic_first_step: {
|
|
185
|
+
cause: "the chain's first step needs a judgement the calculation could not write code for, so the plan cannot start.",
|
|
186
|
+
fix: "the agent does this task itself; a model step for such judgements is planned (plan-services.md W3.1).",
|
|
187
|
+
},
|
|
188
|
+
unusable_first_step: {
|
|
189
|
+
cause: "the first step calls a sub-task that is gone, switched off or stale.",
|
|
190
|
+
fix: "enable or recalculate that sub-task, or recalculate this scenario.",
|
|
191
|
+
},
|
|
192
|
+
missing_target: {
|
|
193
|
+
cause: "the prompt names a target the runner could not find where the recording had it.",
|
|
194
|
+
fix: "run from the recording's directory, or name the target as the recording did.",
|
|
195
|
+
},
|
|
196
|
+
no_derive_key: {
|
|
197
|
+
cause: "the scenario has parameters and the runner had no way to derive them: no service session and no ANTHROPIC_API_KEY.",
|
|
198
|
+
fix: "set BIR_AUTH_URL and `bir login` in the terminal that starts bir-hooks, or export ANTHROPIC_API_KEY there.",
|
|
199
|
+
},
|
|
200
|
+
flatten_failed: {
|
|
201
|
+
cause: "the chain could not be flattened — a sub-task it calls no longer resolves.",
|
|
202
|
+
fix: "recalculate the scenario.",
|
|
203
|
+
},
|
|
204
|
+
};
|
|
205
|
+
export const LOW_SAVING_PCT = 0.3;
|
|
206
|
+
const usd = (v) => `$${v.toFixed(4)}`;
|
|
207
|
+
const pct = (v) => `${(v * 100).toFixed(1)}%`;
|
|
208
|
+
export function localFindings(turn) {
|
|
209
|
+
const out = [];
|
|
210
|
+
if (!turn.matched) {
|
|
211
|
+
out.push({
|
|
212
|
+
code: "no_match",
|
|
213
|
+
severity: "info",
|
|
214
|
+
title: "This turn matched no recording, so the agent did the task itself.",
|
|
215
|
+
cause: "The service found no recording whose prompt is similar enough (its SIMILARITY_THRESHOLD). A run that is below the recording threshold is never embedded and can never be matched.",
|
|
216
|
+
fix: "If a recording of this task exists, run the prompt closer to its wording. The service section below says whether this turn became a recording.",
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
else if (turn.decision?.verdict === "no-steer") {
|
|
220
|
+
const code = turn.decision.code ?? "";
|
|
221
|
+
const known = DECLINES[code];
|
|
222
|
+
out.push({
|
|
223
|
+
code: `declined_${code || "unknown"}`,
|
|
224
|
+
severity: code === "not_ready" && !turn.matched.scenarioId ? "info" : "problem",
|
|
225
|
+
title: `Matched ${turn.matched.runId} but declined to steer: ${turn.decision.why ?? code}.`,
|
|
226
|
+
cause: known?.cause ?? "the runner's log line carries the reason text.",
|
|
227
|
+
fix: known?.fix ??
|
|
228
|
+
(turn.matched.scenarioId
|
|
229
|
+
? "read the `replay.decision` line above."
|
|
230
|
+
: "the matched recording has no scenario yet — calculate it, or let the service pick it up after enough hits."),
|
|
231
|
+
evidence: { code, similarity: turn.matched.similarity, threshold: turn.decision.threshold },
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
else if (!turn.plan) {
|
|
235
|
+
out.push({
|
|
236
|
+
code: "not_armed",
|
|
237
|
+
severity: "warn",
|
|
238
|
+
title: `Matched ${turn.matched.runId} but no plan was armed.`,
|
|
239
|
+
cause: "The match arrived but arming failed or timed out before the prompt hook answered (`replay.arm_failed`, or the match budget).",
|
|
240
|
+
fix: "Check the incidents below and `bir doctor`; the service section says whether the scenario is ready.",
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
if (turn.plan) {
|
|
244
|
+
const coverage = turn.plan.coverage ?? [];
|
|
245
|
+
const tools = turn.plan.tools ?? [];
|
|
246
|
+
const live = coverage
|
|
247
|
+
.map((c, i) => (c === "live" ? (tools[i] ?? `step ${i}`) : undefined))
|
|
248
|
+
.filter((t) => Boolean(t));
|
|
249
|
+
if (turn.plan.mode === "steer" && live.length > 0) {
|
|
250
|
+
const lowSaving = savedPct(turn) != null && savedPct(turn) < LOW_SAVING_PCT;
|
|
251
|
+
out.push({
|
|
252
|
+
code: "steer_mode",
|
|
253
|
+
severity: lowSaving ? "warn" : "info",
|
|
254
|
+
title: `The plan armed in steer mode: ${live.length} of ${coverage.length} steps are reachable only inside the session (${[...new Set(live)].join(", ")}).`,
|
|
255
|
+
cause: "In steer mode the runner pins each call's inputs and auto-approves it, but the model still emits every tool call, reads every result and writes the answer — so the whole live turn is still paid for. Direct mode, where the model makes no tool call, needs every step to be a wrapped MCP server tool.",
|
|
256
|
+
fix: "Put the data those tools reach behind an MCP server, wrap it (`bir install --server <name>`), and record the prompt again so its steps are MCP calls.",
|
|
257
|
+
evidence: { coverage, tools },
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
if (turn.plan.mode === "direct") {
|
|
261
|
+
out.push({
|
|
262
|
+
code: "direct_mode",
|
|
263
|
+
severity: "info",
|
|
264
|
+
title: `The plan armed in direct mode: ${turn.plan.steps ?? coverage.length} steps ran on the proxies, no model in the loop.`,
|
|
265
|
+
cause: "Every step is a wrapped MCP tool this session has a proxy for.",
|
|
266
|
+
fix: "",
|
|
267
|
+
});
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
for (const i of turn.incidents) {
|
|
271
|
+
const fields = Object.fromEntries(Object.entries(i).filter(([k]) => k !== "at" && k !== "event"));
|
|
272
|
+
out.push({
|
|
273
|
+
code: i.event.replace(/\./g, "_"),
|
|
274
|
+
severity: i.event.endsWith("_failed") || i.event === "replay.failed" ? "problem" : "warn",
|
|
275
|
+
title: `${i.event}${i.error ? `: ${String(i.error)}` : ""}`,
|
|
276
|
+
cause: Object.entries(fields)
|
|
277
|
+
.map(([k, v]) => `${k}=${String(v)}`)
|
|
278
|
+
.join(" "),
|
|
279
|
+
fix: i.event === "run.unmeasured"
|
|
280
|
+
? "the turn's cost could not be read from the transcript, so its report is unmeasured; run bir-hooks where the Claude Code transcript is readable."
|
|
281
|
+
: i.event === "replay.diverge" || i.event === "replay.handover"
|
|
282
|
+
? "the model left the script or the runner handed the turn back; the service section shows what the fallback cost."
|
|
283
|
+
: "see the guide's troubleshooting table for this line.",
|
|
284
|
+
});
|
|
285
|
+
}
|
|
286
|
+
const r = turn.reported;
|
|
287
|
+
if (r) {
|
|
288
|
+
if (r.measured === false) {
|
|
289
|
+
out.push({
|
|
290
|
+
code: "unmeasured",
|
|
291
|
+
severity: "warn",
|
|
292
|
+
title: "The execution report is unmeasured: the session's tokens could not be read.",
|
|
293
|
+
cause: "Without the transcript delta the report carries the derivation cost alone, and `baseline − cost` overstates the saving.",
|
|
294
|
+
fix: "Run bir-hooks on the machine and user that own the Claude Code transcript; `run.unmeasured` names what was missing.",
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
const p = savedPct(turn);
|
|
298
|
+
const cost = r.derive + r.session + r.fallback;
|
|
299
|
+
if (r.savedUsd != null && r.savedUsd < 0) {
|
|
300
|
+
out.push({
|
|
301
|
+
code: "saving_negative",
|
|
302
|
+
severity: "problem",
|
|
303
|
+
title: `This replay cost more than the agent: ${usd(cost)} against a baseline of ${usd(r.savedUsd + cost)}.`,
|
|
304
|
+
cause: `derive ${usd(r.derive)}, session ${usd(r.session)}, fallback ${usd(r.fallback)}.`,
|
|
305
|
+
fix: "The service findings below name the dominant cost and the fix.",
|
|
306
|
+
evidence: { ...r, costUsd: cost },
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
else if (p != null && p < LOW_SAVING_PCT && r.outcome === "steered_full") {
|
|
310
|
+
out.push({
|
|
311
|
+
code: "saving_low",
|
|
312
|
+
severity: "warn",
|
|
313
|
+
title: `Saved ${pct(p)}: ${usd(r.savedUsd ?? 0)} of a ${usd((r.savedUsd ?? 0) + cost)} baseline.`,
|
|
314
|
+
cause: `The live session cost ${usd(r.session)} of that baseline${turn.plan?.mode === "steer" ? " — steer mode pays for the whole turn." : "."}`,
|
|
315
|
+
fix: turn.plan?.mode === "steer"
|
|
316
|
+
? "Direct mode is the fix: wrap the servers so every step is an MCP call (see steer_mode)."
|
|
317
|
+
: "Compare the baseline with a few unsteered runs (BIR_REPLAY=0); a single cheap source run makes any replay look poor.",
|
|
318
|
+
evidence: { ...r, costUsd: cost, savedPct: p },
|
|
319
|
+
});
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
else if (turn.plan && turn.finishedAt) {
|
|
323
|
+
out.push({
|
|
324
|
+
code: "not_reported",
|
|
325
|
+
severity: "warn",
|
|
326
|
+
title: "A plan armed but no execution report was sent.",
|
|
327
|
+
cause: "The turn ended without `execution.reported`: the report failed to send, or the ticket was missing (an older service).",
|
|
328
|
+
fix: "Look for `recorder.send_failed` in the bir-hooks log; the saving of this turn is not on the ledger.",
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
const rank = { problem: 0, warn: 1, info: 2 };
|
|
332
|
+
return out
|
|
333
|
+
.map((f, i) => ({ f, i }))
|
|
334
|
+
.sort((a, b) => rank[a.f.severity] - rank[b.f.severity] || a.i - b.i)
|
|
335
|
+
.map(({ f }) => f);
|
|
336
|
+
}
|
|
337
|
+
/** `saved ÷ (saved + cost)`: the baseline is recoverable from the report alone. */
|
|
338
|
+
export function savedPct(turn) {
|
|
339
|
+
const r = turn.reported;
|
|
340
|
+
if (!r || r.savedUsd == null)
|
|
341
|
+
return null;
|
|
342
|
+
const baseline = r.savedUsd + r.derive + r.session + r.fallback;
|
|
343
|
+
return baseline > 0 ? r.savedUsd / baseline : null;
|
|
344
|
+
}
|
|
345
|
+
const short = (id) => (id ? id.slice(0, 13) : "—");
|
|
346
|
+
const money = (v) => (v == null ? "—" : usd(v));
|
|
347
|
+
const when = (iso) => (iso ? iso.replace("T", " ").replace(/\.\d+Z$/, "Z") : "—");
|
|
348
|
+
function renderFindings(out, findings, heading) {
|
|
349
|
+
if (findings.length === 0)
|
|
350
|
+
return;
|
|
351
|
+
out();
|
|
352
|
+
out(heading);
|
|
353
|
+
findings.forEach((f, i) => {
|
|
354
|
+
out(` ${i + 1}. [${f.severity}] ${f.title}`);
|
|
355
|
+
if (f.cause)
|
|
356
|
+
out(` why: ${f.cause}`);
|
|
357
|
+
if (f.fix)
|
|
358
|
+
out(` fix: ${f.fix}`);
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
export function renderTurn(out, turn) {
|
|
362
|
+
out(`Turn ${turn.runId} ${when(turn.startedAt)}${turn.sessionId ? ` session ${turn.sessionId.slice(0, 8)}` : ""}`);
|
|
363
|
+
if (turn.prompt)
|
|
364
|
+
out(` prompt ${turn.prompt.replace(/\s+/g, " ").slice(0, 110)}`);
|
|
365
|
+
if (turn.matched) {
|
|
366
|
+
out(` matched ${turn.matched.runId}${turn.matched.similarity != null ? ` (similarity ${turn.matched.similarity.toFixed(3)})` : ""}${turn.matched.scenarioId ? ` → scenario ${turn.matched.scenarioId}` : " → no scenario"}`);
|
|
367
|
+
}
|
|
368
|
+
else {
|
|
369
|
+
out(" matched no recording — the agent ran the task; this turn was recorded");
|
|
370
|
+
}
|
|
371
|
+
if (turn.decision?.verdict === "no-steer") {
|
|
372
|
+
out(` decision declined${turn.decision.code ? ` (${turn.decision.code})` : ""}: ${turn.decision.why ?? ""}`);
|
|
373
|
+
}
|
|
374
|
+
if (turn.plan) {
|
|
375
|
+
const cov = turn.plan.coverage?.map((c, i) => `${turn.plan?.tools?.[i] ?? "?"}:${c}`).join(" ");
|
|
376
|
+
out(` plan ${turn.plan.mode}${turn.plan.by ? ` (by ${turn.plan.by})` : ""}, ${turn.plan.steps ?? "?"} steps${cov ? ` ${cov}` : ""}`);
|
|
377
|
+
}
|
|
378
|
+
if (turn.derived) {
|
|
379
|
+
out(` derived ${turn.derived.params ?? "?"} params by ${turn.derived.source ?? "?"} ${money(turn.derived.costUsd)}`);
|
|
380
|
+
}
|
|
381
|
+
if (turn.done) {
|
|
382
|
+
out(` replay ${turn.done.outcome}${turn.done.steps ? ` ${turn.done.steps} steps` : ""}${turn.done.ms != null ? ` in ${(turn.done.ms / 1000).toFixed(1)} s` : ""}`);
|
|
383
|
+
}
|
|
384
|
+
if (turn.reported) {
|
|
385
|
+
const r = turn.reported;
|
|
386
|
+
const p = savedPct(turn);
|
|
387
|
+
const cost = r.derive + r.session + r.fallback;
|
|
388
|
+
out(` reported ${r.outcome}${r.declined ? ` (${r.declined})` : ""} derive ${usd(r.derive)} session ${usd(r.session)} fallback ${usd(r.fallback)}` +
|
|
389
|
+
(r.savedUsd != null
|
|
390
|
+
? ` → saved ${usd(r.savedUsd)}${p != null ? ` (${pct(p)} of ${usd(r.savedUsd + cost)})` : ""}`
|
|
391
|
+
: "") +
|
|
392
|
+
(r.measured === false ? " UNMEASURED" : ""));
|
|
393
|
+
}
|
|
394
|
+
if (turn.finish) {
|
|
395
|
+
const f = turn.finish;
|
|
396
|
+
out(` finished ${f.steps ?? "?"} steps, ${f.durationMs != null ? `${(f.durationMs / 1000).toFixed(1)} s` : "?"}${f.costUsd != null ? `, cost ${usd(f.costUsd)}` : f.measured === false ? ", cost unmeasured" : ""}, ${f.recorded ? "recorded" : "not recorded"}`);
|
|
397
|
+
}
|
|
398
|
+
else {
|
|
399
|
+
out(" finished (not yet — the turn is still open, or the session ended without its Stop hook)");
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
export function renderService(out, s, viewerIsOwner) {
|
|
403
|
+
out();
|
|
404
|
+
out(`From the service${s.subject ? ` (${s.subject.kind} ${s.subject.id})` : ""}${s.owner ? `, owner ${viewerIsOwner ? "you" : (s.owner.email ?? s.owner.id)}` : ""}:`);
|
|
405
|
+
if (s.run) {
|
|
406
|
+
const r = s.run;
|
|
407
|
+
out(` recording ${r.id} ${r.isRecording ? "listed" : "below threshold"}${r.embedded ? ", embedded" : ", NOT embedded"} ${r.actionCount} tool calls (${r.tools.join(", ") || "none"}) cost ${money(r.originalCostUsd)} hits ${r.iterations}${r.duplicateOf ? ` duplicate of ${r.duplicateOf}` : ""}`);
|
|
408
|
+
if (r.title || r.input)
|
|
409
|
+
out(` "${(r.title ?? r.input).replace(/\s+/g, " ").slice(0, 100)}"`);
|
|
410
|
+
}
|
|
411
|
+
else {
|
|
412
|
+
out(" recording none");
|
|
413
|
+
}
|
|
414
|
+
if (s.scenario) {
|
|
415
|
+
const sc = s.scenario;
|
|
416
|
+
out(` scenario ${sc.id} ${sc.state}${sc.disabledAt ? " (switched off)" : ""} ${sc.steps.length} steps (${[...new Set(sc.steps.map((x) => x.toolName ?? "?"))].join(", ")}) baseline ${money(sc.baseline.costUsd)} from ${sc.baseline.samples} sample(s)${sc.hitCount != null ? ` hits ${sc.hitCount}` : ""}`);
|
|
417
|
+
if (sc.lastDecline)
|
|
418
|
+
out(` last decline ${sc.lastDecline.reason} at ${when(sc.lastDecline.at ?? undefined)}`);
|
|
419
|
+
if (sc.error)
|
|
420
|
+
out(` error ${sc.error.slice(0, 160)}`);
|
|
421
|
+
}
|
|
422
|
+
else {
|
|
423
|
+
out(" scenario none");
|
|
424
|
+
}
|
|
425
|
+
const ex = s.executions ?? [];
|
|
426
|
+
if (ex.length > 0) {
|
|
427
|
+
out(` executions (${ex.length}, newest first)`);
|
|
428
|
+
for (const e of ex.slice(0, 10)) {
|
|
429
|
+
out(` ${when(e.createdAt)} ${short(e.id)} ${e.outcome.padEnd(13)} cost ${usd(e.costUsd)} saved ${money(e.savedUsd)}${e.savedPct != null ? ` (${pct(e.savedPct)})` : ""}${e.measured ? "" : " unmeasured"}`);
|
|
430
|
+
const failed = e.steps?.filter((st) => st.status === "failed") ?? [];
|
|
431
|
+
for (const st of failed)
|
|
432
|
+
out(` step ${st.stepIndex} ${st.toolName ?? ""} failed${st.error ? `: ${st.error.slice(0, 120)}` : ""}`);
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
renderFindings(out, s.findings ?? [], "Service findings");
|
|
436
|
+
}
|
|
437
|
+
// ── the command ───────────────────────────────────────────────────────────────
|
|
438
|
+
async function fetchInvestigation(deps, id, limit) {
|
|
439
|
+
const reply = await deps.service(`/investigate/${id}${limit ? `?limit=${limit}` : ""}`);
|
|
440
|
+
if ("error" in reply)
|
|
441
|
+
return { note: `service not consulted: ${reply.error}` };
|
|
442
|
+
if (reply.status === 404)
|
|
443
|
+
return { note: `the service has nothing for ${id} (not yours, or never reached it)` };
|
|
444
|
+
if (reply.status !== 200)
|
|
445
|
+
return { note: `service answered HTTP ${reply.status}` };
|
|
446
|
+
return { data: reply.body };
|
|
447
|
+
}
|
|
448
|
+
/** Which id the service should be asked about for a turn. */
|
|
449
|
+
function serviceIdFor(turn) {
|
|
450
|
+
return (turn.reported?.ticket ??
|
|
451
|
+
turn.reported?.scenarioId ??
|
|
452
|
+
turn.plan?.scenarioId ??
|
|
453
|
+
turn.matched?.scenarioId ??
|
|
454
|
+
turn.matched?.runId ??
|
|
455
|
+
turn.runId);
|
|
456
|
+
}
|
|
457
|
+
export async function investigateCommand(args, deps) {
|
|
458
|
+
const sub = args.positionals[0];
|
|
459
|
+
const { out } = deps;
|
|
460
|
+
if (sub === "executions") {
|
|
461
|
+
const params = new URLSearchParams();
|
|
462
|
+
if (args.limit)
|
|
463
|
+
params.set("limit", String(args.limit));
|
|
464
|
+
if (args.user)
|
|
465
|
+
params.set("user", args.user);
|
|
466
|
+
const reply = await deps.service(`/investigate/executions${params.size ? `?${params}` : ""}`);
|
|
467
|
+
if ("error" in reply) {
|
|
468
|
+
out(`Could not reach the service: ${reply.error}`);
|
|
469
|
+
return 1;
|
|
470
|
+
}
|
|
471
|
+
if (reply.status === 403) {
|
|
472
|
+
out("Only an admin may look at another account's executions.");
|
|
473
|
+
return 1;
|
|
474
|
+
}
|
|
475
|
+
if (reply.status === 404) {
|
|
476
|
+
out(`No such account: ${args.user}`);
|
|
477
|
+
return 1;
|
|
478
|
+
}
|
|
479
|
+
if (reply.status !== 200) {
|
|
480
|
+
out(`Could not list executions (HTTP ${reply.status}).`);
|
|
481
|
+
return 1;
|
|
482
|
+
}
|
|
483
|
+
const body = reply.body;
|
|
484
|
+
if (args.json) {
|
|
485
|
+
out(JSON.stringify(body, null, 2));
|
|
486
|
+
return 0;
|
|
487
|
+
}
|
|
488
|
+
const rows = body.executions ?? [];
|
|
489
|
+
out(`Executions of ${body.owner?.email ?? body.owner?.id ?? "you"} (${rows.length}, newest first)`);
|
|
490
|
+
if (rows.length === 0)
|
|
491
|
+
out(" none yet");
|
|
492
|
+
for (const e of rows) {
|
|
493
|
+
out(` ${when(e.createdAt)} ${e.id} ${e.outcome.padEnd(13)} cost ${usd(e.costUsd)} saved ${money(e.savedUsd)}${e.savedPct != null ? ` (${pct(e.savedPct)})` : ""}${e.measured ? "" : " unmeasured"}`);
|
|
494
|
+
out(` scenario ${e.scenarioId}${e.runId ? ` run ${e.runId}` : ""}${e.intent ? ` "${e.intent.slice(0, 60)}"` : ""}`);
|
|
495
|
+
}
|
|
496
|
+
out();
|
|
497
|
+
out("Investigate one with `bir investigate <sexec_…>`.");
|
|
498
|
+
return 0;
|
|
499
|
+
}
|
|
500
|
+
const entries = deps.journal(deps.cwd);
|
|
501
|
+
const turns = turnsFromJournal(entries);
|
|
502
|
+
if (sub === "list") {
|
|
503
|
+
const limit = args.limit ?? 20;
|
|
504
|
+
const recent = turns.slice(-limit).reverse();
|
|
505
|
+
if (args.json) {
|
|
506
|
+
out(JSON.stringify(recent, null, 2));
|
|
507
|
+
return 0;
|
|
508
|
+
}
|
|
509
|
+
if (recent.length === 0) {
|
|
510
|
+
out(`No turns in the journal for this directory (${deps.journalPath(deps.cwd)}).`);
|
|
511
|
+
out("Start `bir-hooks` here and run a prompt; every turn is journaled from then on.");
|
|
512
|
+
return 0;
|
|
513
|
+
}
|
|
514
|
+
out(`Recent turns in ${deps.cwd} (newest first)`);
|
|
515
|
+
for (const t of recent) {
|
|
516
|
+
const verdict = !t.matched
|
|
517
|
+
? "recorded"
|
|
518
|
+
: t.decision?.verdict === "no-steer"
|
|
519
|
+
? `declined ${t.decision.code ?? ""}`.trim()
|
|
520
|
+
: t.plan
|
|
521
|
+
? `${t.plan.mode} ${t.done?.outcome ?? t.reported?.outcome ?? "armed"}`
|
|
522
|
+
: "matched";
|
|
523
|
+
const p = savedPct(t);
|
|
524
|
+
const saved = t.reported?.savedUsd != null ? ` saved ${usd(t.reported.savedUsd)}${p != null ? ` (${pct(p)})` : ""}` : "";
|
|
525
|
+
out(` ${when(t.startedAt)} ${t.runId} ${verdict}${saved}`);
|
|
526
|
+
if (t.prompt)
|
|
527
|
+
out(` "${t.prompt.replace(/\s+/g, " ").slice(0, 90)}"`);
|
|
528
|
+
}
|
|
529
|
+
out();
|
|
530
|
+
out("Investigate one with `bir investigate <run_…>`; the newest with `bir investigate`.");
|
|
531
|
+
return 0;
|
|
532
|
+
}
|
|
533
|
+
// One subject: an id, or the newest turn.
|
|
534
|
+
const id = sub;
|
|
535
|
+
let turn;
|
|
536
|
+
if (id) {
|
|
537
|
+
turn = findTurn(turns, id);
|
|
538
|
+
}
|
|
539
|
+
else {
|
|
540
|
+
turn = turns[turns.length - 1];
|
|
541
|
+
if (!turn) {
|
|
542
|
+
out(`No turns in the journal for this directory (${deps.journalPath(deps.cwd)}).`);
|
|
543
|
+
out("Start `bir-hooks` here and run a prompt, or name an id: `bir investigate <run_|scn_|sexec_ id>`.");
|
|
544
|
+
return 1;
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
if (id && !turn && !/^(run_|scn_|sexec_)/.test(id)) {
|
|
548
|
+
out(`Not an id this command knows: ${id}. Expected run_…, scn_… or sexec_…, or a subcommand (list, executions).`);
|
|
549
|
+
return 2;
|
|
550
|
+
}
|
|
551
|
+
const findings = turn ? localFindings(turn) : [];
|
|
552
|
+
const serviceId = turn ? serviceIdFor(turn) : id;
|
|
553
|
+
const service = await fetchInvestigation(deps, serviceId, args.limit);
|
|
554
|
+
// A matched turn's own run id is unknown to the service; when the scenario
|
|
555
|
+
// lookup fails, the matched run is still worth a try.
|
|
556
|
+
const fallbackId = turn && service.note && turn.matched?.runId && serviceId !== turn.matched.runId ? turn.matched.runId : undefined;
|
|
557
|
+
const second = fallbackId ? await fetchInvestigation(deps, fallbackId, args.limit) : undefined;
|
|
558
|
+
const chosen = service.data ? service : second?.data ? second : service;
|
|
559
|
+
if (args.json) {
|
|
560
|
+
out(JSON.stringify({ turn: turn ?? null, findings, service: chosen.data ?? null, note: chosen.note ?? null }, null, 2));
|
|
561
|
+
return 0;
|
|
562
|
+
}
|
|
563
|
+
if (turn) {
|
|
564
|
+
renderTurn(out, turn);
|
|
565
|
+
renderFindings(out, findings, "Runner findings");
|
|
566
|
+
}
|
|
567
|
+
else {
|
|
568
|
+
out(`Nothing in this directory's journal for ${id}; asking the service.`);
|
|
569
|
+
}
|
|
570
|
+
if (chosen.data) {
|
|
571
|
+
const viewerIsOwner = Boolean(deps.viewerEmail) && chosen.data.owner?.email === deps.viewerEmail;
|
|
572
|
+
renderService(out, chosen.data, viewerIsOwner);
|
|
573
|
+
}
|
|
574
|
+
else {
|
|
575
|
+
out();
|
|
576
|
+
out(`Service: ${chosen.note ?? "no data"}`);
|
|
577
|
+
}
|
|
578
|
+
return 0;
|
|
579
|
+
}
|
|
580
|
+
//# sourceMappingURL=investigate.js.map
|
package/dist/control/server.js
CHANGED
|
@@ -36,6 +36,7 @@ import { NullRecorder, isIntentMatcher, isMatchAware, isRunCreationAware, isScen
|
|
|
36
36
|
import { ReplayController, POLL_HOLD_MS, } from "../replay/controller.js";
|
|
37
37
|
import { calculateCostUsd } from "../replay/pricing.js";
|
|
38
38
|
import { logDetail, logLine, errText } from "../util/log.js";
|
|
39
|
+
import { journal } from "../util/journal.js";
|
|
39
40
|
import { packageVersion } from "../util/version.js";
|
|
40
41
|
/** How long a `/tool/post` waits for the proxy's own report before recording its own view. */
|
|
41
42
|
const PROXY_REPORT_GRACE_MS = 1_500;
|
|
@@ -388,6 +389,10 @@ export class ControlServer {
|
|
|
388
389
|
};
|
|
389
390
|
session.run = run;
|
|
390
391
|
logLine("run.start", { run: runId, sess: session.sessionId, tier: "bound" });
|
|
392
|
+
// Journal-only: the prompt is what `bir investigate` lists a turn by. Never
|
|
393
|
+
// on stderr, and never more than a preview.
|
|
394
|
+
if (input)
|
|
395
|
+
journal("run.prompt", { run: runId, prompt: input.slice(0, 200) });
|
|
391
396
|
// Started here so the round trip overlaps whatever the caller does next;
|
|
392
397
|
// `onPrompt` awaits the same memoized promise under its own budget.
|
|
393
398
|
void this.watchForMatch(session, run);
|
|
@@ -521,6 +526,8 @@ export class ControlServer {
|
|
|
521
526
|
run: run.runId,
|
|
522
527
|
steps: run.ordering.next,
|
|
523
528
|
durationMs,
|
|
529
|
+
costUsd: cost.measured ? cost.usd.toFixed(4) : undefined,
|
|
530
|
+
measured: cost.measured,
|
|
524
531
|
lossy: this.lossy,
|
|
525
532
|
recorded: run.recording,
|
|
526
533
|
});
|
|
@@ -603,6 +610,7 @@ export class ControlServer {
|
|
|
603
610
|
durationMs: Math.max(0, windowMs),
|
|
604
611
|
prompt: run.input || undefined,
|
|
605
612
|
siblings: states,
|
|
613
|
+
runId: run.runId,
|
|
606
614
|
});
|
|
607
615
|
if (!report)
|
|
608
616
|
return;
|
|
@@ -936,11 +944,14 @@ export class ControlServer {
|
|
|
936
944
|
continue;
|
|
937
945
|
let position;
|
|
938
946
|
if (!state.plan) {
|
|
939
|
-
// Only a
|
|
940
|
-
//
|
|
941
|
-
// decline leaves the recording
|
|
942
|
-
|
|
947
|
+
// Only a first step the plan cannot start on — parked, or a judgement
|
|
948
|
+
// — means the turn is committed to doing this recording's work itself
|
|
949
|
+
// from position 0 (R-HIT-14); every other decline leaves the recording
|
|
950
|
+
// untouched.
|
|
951
|
+
if (state.declined === "known_bad_first_step" ||
|
|
952
|
+
state.declined === "nondeterministic_first_step") {
|
|
943
953
|
position = 0;
|
|
954
|
+
}
|
|
944
955
|
}
|
|
945
956
|
else if (state.handover) {
|
|
946
957
|
position = state.handover.stepIndex;
|
|
@@ -216,6 +216,12 @@ export declare function isIntentMatcher(r: Recorder): r is Recorder & IntentMatc
|
|
|
216
216
|
*/
|
|
217
217
|
export interface ExecutionReport {
|
|
218
218
|
scenarioId: string;
|
|
219
|
+
/**
|
|
220
|
+
* The control server's run id for the turn this report closes. Journal-only:
|
|
221
|
+
* it lets `bir investigate` join the report to the turn's other lines. Never
|
|
222
|
+
* sent to the service, which has no row for a matched turn.
|
|
223
|
+
*/
|
|
224
|
+
runId?: string;
|
|
219
225
|
/** The match's claim token. It *is* the execution row's id, so a doubled report books once. */
|
|
220
226
|
ticket?: string;
|
|
221
227
|
outcome: "steered_full" | "diverged" | "not_steered" | "failed" | "fell_back";
|
|
@@ -219,7 +219,9 @@ export class RemoteRecorder {
|
|
|
219
219
|
const r = (body ?? {});
|
|
220
220
|
const failed = report.steps?.filter((s) => s.status === "failed").length ?? 0;
|
|
221
221
|
logLine("execution.reported", {
|
|
222
|
+
run: report.runId,
|
|
222
223
|
scenario: report.scenarioId,
|
|
224
|
+
ticket: report.ticket,
|
|
223
225
|
outcome: report.outcome,
|
|
224
226
|
derive: report.deriveCostUsd.toFixed(4),
|
|
225
227
|
session: report.sessionCostUsd.toFixed(4),
|