@basein/runner 0.3.15 → 0.3.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/bin/bir-scenario.js +8 -5
- package/dist/bin/bir.js +2 -1
- package/dist/bin/investigate.d.ts +71 -0
- package/dist/bin/investigate.js +145 -13
- package/dist/bin/scenario-edit.d.ts +45 -1
- package/dist/bin/scenario-edit.js +134 -14
- package/dist/control/server.d.ts +16 -0
- package/dist/control/server.js +156 -7
- package/dist/control/transcript.d.ts +18 -0
- package/dist/control/transcript.js +55 -0
- package/dist/record/recorder.d.ts +47 -4
- package/dist/record/remote-recorder.d.ts +5 -4
- package/dist/record/remote-recorder.js +23 -1
- package/dist/record/switchable-recorder.d.ts +1 -5
- package/dist/replay/bundle.d.ts +10 -0
- package/dist/replay/bundle.js +23 -2
- package/dist/replay/controller.d.ts +133 -0
- package/dist/replay/controller.js +496 -59
- package/dist/replay/flatten.d.ts +39 -3
- package/dist/replay/flatten.js +42 -2
- package/dist/replay/handover.d.ts +39 -1
- package/dist/replay/handover.js +81 -2
- package/dist/replay/logic.d.ts +8 -0
- package/dist/replay/logic.js +19 -0
- package/dist/replay/plan.d.ts +165 -4
- package/dist/replay/plan.js +488 -25
- package/dist/replay/source-run.d.ts +14 -1
- package/dist/replay/source-run.js +25 -2
- package/dist/replay/targets.js +8 -1
- package/dist/replay/types.d.ts +52 -3
- package/dist/replay/types.js +14 -0
- package/dist/util/service-headers.d.ts +7 -1
- package/dist/util/service-headers.js +7 -1
- package/docs/BaseInstRunner.md +3 -1
- package/docs/calculatedReplay.md +117 -6
- package/docs/calculatedReplayGuide.md +41 -3
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -393,6 +393,12 @@ which `bir setup` turns on (`bir scenario editing on` in a project wired by
|
|
|
393
393
|
`bir install`). Step by step, with real output:
|
|
394
394
|
[the guide's §9.2](docs/calculatedReplayGuide.md).
|
|
395
395
|
|
|
396
|
+
The service can correct a plan itself too, after the agent did a planned task
|
|
397
|
+
again its own way (built 2026-10-23, not yet released). `bir scenario edits`
|
|
398
|
+
lists such a correction beside the hand edits, under its `scor_` id. Undoing it
|
|
399
|
+
puts back the whole plan from before, and the plan is then held until you
|
|
400
|
+
choose "Use it again" on the recording page.
|
|
401
|
+
|
|
396
402
|
```bash
|
|
397
403
|
bir scenario show scn_… --step 3 # read one step (scenarioId and chainRevision first)
|
|
398
404
|
bir scenario check scn_… --step 3 --input-logic step3.js # try a change; nothing is saved
|
|
@@ -400,6 +406,7 @@ bir scenario edit scn_… --step 3 --input-logic step3.js --note "why" # the
|
|
|
400
406
|
bir scenario edits scn_… # the history, newest first
|
|
401
407
|
bir scenario undo scn_… # take back the newest edit; again to go further back
|
|
402
408
|
bir scenario undo scn_… --step 3 # take back step 3's newest edit only
|
|
409
|
+
bir scenario undo scn_… --edit scor_… # take back a correction the service made
|
|
403
410
|
bir scenario calc scn_… --force --discard-edits # recalculate this plan, dropping its hand edits
|
|
404
411
|
bir scenario editing on # let Claude's `bir` tools change scenarios here (setup did)
|
|
405
412
|
```
|
package/dist/bin/bir-scenario.js
CHANGED
|
@@ -497,7 +497,9 @@ const SCENARIO_TOOLS = [
|
|
|
497
497
|
changes: false,
|
|
498
498
|
service: true,
|
|
499
499
|
description: "List a calculated scenario's hand edits, newest first — step edits and parameter kind changes: step or " +
|
|
500
|
-
"parameter, note, revision, and whether each was forced, undone or replaced by a recalculation
|
|
500
|
+
"parameter, note, revision, and whether each was forced, undone or replaced by a recalculation — and the " +
|
|
501
|
+
"corrections the service made from a run the agent redid (scor_ ids): what each added, tested or changed, " +
|
|
502
|
+
"and whether it is used.",
|
|
501
503
|
inputSchema: {
|
|
502
504
|
type: "object",
|
|
503
505
|
properties: { target: TARGET },
|
|
@@ -646,14 +648,15 @@ const SCENARIO_TOOLS = [
|
|
|
646
648
|
"edit. Without `edit`, the newest edit that can be undone (of `step` or `param`, when given); calling it again goes " +
|
|
647
649
|
"one further back. The service checks a step's undo like a save and refuses it when a later step or the final " +
|
|
648
650
|
"answer would break — `force` with a `note` undoes it anyway. To redo, undo the undo: pass its id as `edit` (the " +
|
|
649
|
-
"answer names it)."
|
|
651
|
+
"answer names it). A scor_ id takes back a correction the service made: the plan goes back exactly as it was " +
|
|
652
|
+
"before it, and is held — not used — until it is corrected again or let go on the recording page.",
|
|
650
653
|
inputSchema: {
|
|
651
654
|
type: "object",
|
|
652
655
|
properties: {
|
|
653
656
|
target: TARGET,
|
|
654
657
|
edit: {
|
|
655
658
|
type: "string",
|
|
656
|
-
description: "The sedit_ or
|
|
659
|
+
description: "The sedit_, pedit_ or scor_ id to undo, as scenario_edits lists it. Not with `step` or `param`.",
|
|
657
660
|
},
|
|
658
661
|
step: {
|
|
659
662
|
...STEP,
|
|
@@ -679,8 +682,8 @@ const SCENARIO_TOOLS = [
|
|
|
679
682
|
throw new BadArguments("`edit`, `step` and `param` each pick what to undo; give one of them");
|
|
680
683
|
}
|
|
681
684
|
if (a.edit !== undefined && a.edit !== null) {
|
|
682
|
-
if (typeof a.edit !== "string" || !/^(sedit|pedit)_[\w-]+$/.test(a.edit)) {
|
|
683
|
-
throw new BadArguments("`edit` must be a sedit_… or
|
|
685
|
+
if (typeof a.edit !== "string" || !/^(sedit|pedit|scor)_[\w-]+$/.test(a.edit)) {
|
|
686
|
+
throw new BadArguments("`edit` must be a sedit_…, pedit_… or scor_… id");
|
|
684
687
|
}
|
|
685
688
|
if (a.step !== undefined && a.step !== null) {
|
|
686
689
|
throw new BadArguments("`edit` names one entry and `step` picks one; give one of them");
|
package/dist/bin/bir.js
CHANGED
|
@@ -172,7 +172,8 @@ Options:
|
|
|
172
172
|
--unfreeze scenario check|edit: ask to remove that mark; the check decides
|
|
173
173
|
--note <text> scenario edit|undo: why, kept in the history (required with --force)
|
|
174
174
|
--revision <n> scenario edit|undo: only if the plan is still at this chainRevision
|
|
175
|
-
--edit <id> scenario undo: the sedit_ or
|
|
175
|
+
--edit <id> scenario undo: the sedit_, pedit_ or scor_ id to undo (default: the newest
|
|
176
|
+
that can be); a scor_ id puts back the plan from before that correction
|
|
176
177
|
--param <name> scenario edit: the parameter whose kind changes · scenario undo: its newest change
|
|
177
178
|
--kind <kind> scenario edit --param: target (a prompt must name it) or setting (its
|
|
178
179
|
recorded value is used when a prompt names none; needs --note)
|
|
@@ -78,6 +78,12 @@ export interface Turn {
|
|
|
78
78
|
ticket?: string;
|
|
79
79
|
scenarioId?: string;
|
|
80
80
|
declined?: string;
|
|
81
|
+
/**
|
|
82
|
+
* The plan ran to its end and the agent redid the task: the repeat run it
|
|
83
|
+
* was recorded as, and the agent's own calls in it (autocorrect.md AC1).
|
|
84
|
+
*/
|
|
85
|
+
redoneBy?: string;
|
|
86
|
+
ownCalls?: number;
|
|
81
87
|
};
|
|
82
88
|
finish?: {
|
|
83
89
|
steps?: number;
|
|
@@ -86,6 +92,16 @@ export interface Turn {
|
|
|
86
92
|
measured?: boolean;
|
|
87
93
|
recorded?: boolean;
|
|
88
94
|
};
|
|
95
|
+
/**
|
|
96
|
+
* The model's own work in a matched turn, recorded as the recording's newest
|
|
97
|
+
* run (`run.repeat`): its run id, the recording it repeats, and why — the
|
|
98
|
+
* plan did not run, or `redone` after it ran to its end.
|
|
99
|
+
*/
|
|
100
|
+
repeat?: {
|
|
101
|
+
runId?: string;
|
|
102
|
+
of?: string;
|
|
103
|
+
reason?: string;
|
|
104
|
+
};
|
|
89
105
|
/** `replay.*_failed`, `replay.arm_failed`, `replay.diverge`, `replay.handover` … */
|
|
90
106
|
incidents: JournalEntry[];
|
|
91
107
|
events: JournalEntry[];
|
|
@@ -138,6 +154,31 @@ export interface InvestigateArgs {
|
|
|
138
154
|
user?: string;
|
|
139
155
|
limit?: number;
|
|
140
156
|
}
|
|
157
|
+
/**
|
|
158
|
+
* The service's verdict on a turn whose plan the agent redid (its
|
|
159
|
+
* autocorrect.md AC3): did the agent reject the result, why, and what it left.
|
|
160
|
+
*/
|
|
161
|
+
interface ServiceVerdict {
|
|
162
|
+
accepted?: boolean;
|
|
163
|
+
kind?: string;
|
|
164
|
+
why?: string;
|
|
165
|
+
rejectedSteps?: number[];
|
|
166
|
+
leftInPlace?: string[];
|
|
167
|
+
condition?: string | null;
|
|
168
|
+
}
|
|
169
|
+
/** A correction of the plan, in brief (AC6, AC7). */
|
|
170
|
+
interface ServiceCorrection {
|
|
171
|
+
id: string;
|
|
172
|
+
state?: string;
|
|
173
|
+
createdAt?: string;
|
|
174
|
+
usedAt?: string | null;
|
|
175
|
+
summary?: string | null;
|
|
176
|
+
added?: number[];
|
|
177
|
+
tested?: number[];
|
|
178
|
+
changed?: number[];
|
|
179
|
+
answerChanged?: boolean;
|
|
180
|
+
problems?: string[];
|
|
181
|
+
}
|
|
141
182
|
interface ServiceInvestigation {
|
|
142
183
|
subject?: {
|
|
143
184
|
kind: string;
|
|
@@ -159,6 +200,14 @@ interface ServiceInvestigation {
|
|
|
159
200
|
isRecording: boolean;
|
|
160
201
|
embedded: boolean;
|
|
161
202
|
duplicateOf: string | null;
|
|
203
|
+
/** A repeat: the agent doing that run's task again itself (newestRun.md). */
|
|
204
|
+
repeatOfRunId?: string | null;
|
|
205
|
+
/** Why: the plan did not run, or `redone` after it ran to its end. */
|
|
206
|
+
repeatReason?: string | null;
|
|
207
|
+
/** On a redo: the plan's execution it redid (autocorrect.md AC1). */
|
|
208
|
+
executionId?: string | null;
|
|
209
|
+
/** On a recording: the agent's newest run of its task. */
|
|
210
|
+
newestRunId?: string | null;
|
|
162
211
|
} | null;
|
|
163
212
|
scenario?: {
|
|
164
213
|
id: string;
|
|
@@ -167,6 +216,14 @@ interface ServiceInvestigation {
|
|
|
167
216
|
stepIndex: number;
|
|
168
217
|
toolName: string | null;
|
|
169
218
|
failureCount: number;
|
|
219
|
+
/** Runs only when its test says so (AC5). */
|
|
220
|
+
when?: {
|
|
221
|
+
otherwise?: string;
|
|
222
|
+
text?: string | null;
|
|
223
|
+
} | null;
|
|
224
|
+
/** Added, tested or changed by a correction (AC6). */
|
|
225
|
+
correctionId?: string | null;
|
|
226
|
+
correctionKind?: string | null;
|
|
170
227
|
}>;
|
|
171
228
|
baseline: {
|
|
172
229
|
costUsd: number | null;
|
|
@@ -180,6 +237,14 @@ interface ServiceInvestigation {
|
|
|
180
237
|
hitCount: number | null;
|
|
181
238
|
disabledAt: string | null;
|
|
182
239
|
error: string | null;
|
|
240
|
+
/** Not handed out until it is corrected or let go (AC4). */
|
|
241
|
+
held?: {
|
|
242
|
+
at?: string | null;
|
|
243
|
+
reason?: string | null;
|
|
244
|
+
executionId?: string | null;
|
|
245
|
+
runId?: string | null;
|
|
246
|
+
} | null;
|
|
247
|
+
corrections?: ServiceCorrection[];
|
|
183
248
|
} | null;
|
|
184
249
|
executions?: Array<{
|
|
185
250
|
id: string;
|
|
@@ -200,6 +265,12 @@ interface ServiceInvestigation {
|
|
|
200
265
|
status: string;
|
|
201
266
|
error?: string;
|
|
202
267
|
}> | null;
|
|
268
|
+
/** The repeat the agent redid the task in, and its own calls there (AC1). */
|
|
269
|
+
redoneByRunId?: string | null;
|
|
270
|
+
redoneOwnCalls?: number | null;
|
|
271
|
+
/** The service's verdict on that redo, and where it stands (AC3). */
|
|
272
|
+
verdict?: ServiceVerdict | null;
|
|
273
|
+
verdictState?: string | null;
|
|
203
274
|
}>;
|
|
204
275
|
findings?: Finding[];
|
|
205
276
|
}
|
package/dist/bin/investigate.js
CHANGED
|
@@ -65,8 +65,23 @@ export function turnsFromJournal(entries) {
|
|
|
65
65
|
// matching turn, not about the recording's own (long finished) turn.
|
|
66
66
|
if (run) {
|
|
67
67
|
for (let i = turns.length - 1; i >= 0; i -= 1) {
|
|
68
|
-
|
|
68
|
+
// A turn recorded as a repeat goes on under the repeat's own run id.
|
|
69
|
+
if (turns[i].runId === run || turns[i].matched?.runId === run || turns[i].repeat?.runId === run) {
|
|
69
70
|
return turns[i];
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
// The repeat's own line names a run no turn has yet: it belongs to the
|
|
75
|
+
// newest turn that matched the recording it repeats (`of`) and opened no
|
|
76
|
+
// repeat so far — never simply the newest turn, which with two sessions in
|
|
77
|
+
// one project may be the other session's.
|
|
78
|
+
if (e.event === "run.repeat") {
|
|
79
|
+
const of = str(e.of);
|
|
80
|
+
if (of) {
|
|
81
|
+
for (let i = turns.length - 1; i >= 0; i -= 1) {
|
|
82
|
+
if (turns[i].matched?.runId === of && !turns[i].repeat)
|
|
83
|
+
return turns[i];
|
|
84
|
+
}
|
|
70
85
|
}
|
|
71
86
|
}
|
|
72
87
|
if (e.event === "execution.reported") {
|
|
@@ -143,8 +158,13 @@ export function turnsFromJournal(entries) {
|
|
|
143
158
|
ticket: str(e.ticket),
|
|
144
159
|
scenarioId: str(e.scenario),
|
|
145
160
|
declined: str(e.declined),
|
|
161
|
+
redoneBy: str(e.redoneBy),
|
|
162
|
+
ownCalls: num(e.ownCalls),
|
|
146
163
|
};
|
|
147
164
|
break;
|
|
165
|
+
case "run.repeat":
|
|
166
|
+
turn.repeat = { runId: str(e.run), of: str(e.of), reason: str(e.reason) };
|
|
167
|
+
break;
|
|
148
168
|
case "run.finish":
|
|
149
169
|
turn.finishedAt = e.at;
|
|
150
170
|
turn.finish = {
|
|
@@ -171,7 +191,10 @@ export function findTurn(turns, id) {
|
|
|
171
191
|
t.matched?.scenarioId === id ||
|
|
172
192
|
t.plan?.scenarioId === id ||
|
|
173
193
|
t.reported?.scenarioId === id ||
|
|
174
|
-
t.reported?.ticket === id
|
|
194
|
+
t.reported?.ticket === id ||
|
|
195
|
+
// The repeat the turn was recorded as (newestRun.md), a redo included.
|
|
196
|
+
t.repeat?.runId === id ||
|
|
197
|
+
t.reported?.redoneBy === id) {
|
|
175
198
|
return t;
|
|
176
199
|
}
|
|
177
200
|
}
|
|
@@ -228,6 +251,10 @@ export const DECLINES = {
|
|
|
228
251
|
cause: "the service's answer to the match landed too late: after the prompt hook had stopped waiting for it (3 s, BIR_MATCH_BUDGET_MS); while prompts were not waiting on the match at all — after one slow answer they stop waiting for a few minutes (`replay.match_gate state=holding`); or with the hook not waiting from the start, because a failure to reach the service was still on record (a `service.degraded` line, perhaps hours before, and no `service.recovered` until after this prompt) or the match request's first attempt failed. The agent did the task itself.",
|
|
229
252
|
fix: "nothing in the prompt has to change: run it again a few minutes later. A `service.recovered` line just after this turn's `run.start` means an old failure was still on record, and the next prompt waits again. If it keeps happening without one, the service answers matches slowly: `bir doctor` shows its state, and `/health.matchGate` when prompts wait again.",
|
|
230
253
|
},
|
|
254
|
+
held: {
|
|
255
|
+
cause: "the service matched the prompt and handed over no plan: it holds the plan, because after the plan's last run the agent did the task again its own way. The service judges that redo first, and then keeps the plan back until its owner decides.",
|
|
256
|
+
fix: "open the recording page: read the verdict, then use the correction if there is one, or choose \"Use it again\". `bir investigate <scnId>` shows the hold, the verdict and the newest correction. Until then the agent does the task.",
|
|
257
|
+
},
|
|
231
258
|
runner_update_required: {
|
|
232
259
|
cause: "the service matched the prompt and handed over no plan: this runner is older than its plans need.",
|
|
233
260
|
fix: "update the runner (`bir update`), then run `bir setup` in each project that uses it; `bir doctor` says when the service is satisfied.",
|
|
@@ -236,7 +263,7 @@ export const DECLINES = {
|
|
|
236
263
|
cause: "the chain could not be flattened — a sub-task it calls no longer resolves.",
|
|
237
264
|
fix: "recalculate the scenario.",
|
|
238
265
|
},
|
|
239
|
-
// The
|
|
266
|
+
// The gates of a host that is not Claude Code (a LangChain application).
|
|
240
267
|
host_records_only: {
|
|
241
268
|
cause: "the runner in that application's folder is older than replay for LangChain needs, so the LangChain package only records.",
|
|
242
269
|
fix: "nothing to do while replay for LangChain is not released. After that: update the runner. Until then the model does the task, and the run counts as a sample of the usual cost.",
|
|
@@ -245,6 +272,10 @@ export const DECLINES = {
|
|
|
245
272
|
cause: "the plan names a tool that the agent which asked does not have.",
|
|
246
273
|
fix: "give the agent that tool, or delete the recording so the task is recorded again; `bir scenario show <scnId>` lists the plan's tools.",
|
|
247
274
|
},
|
|
275
|
+
host_tests_unsupported: {
|
|
276
|
+
cause: "the plan has steps that run only when a test says so (an \"if … otherwise\"), and the LangChain package steps through a plan call by call without reading those tests yet, so the runner did not hand it the plan.",
|
|
277
|
+
fix: "nothing to change in the prompt: the model does the task, and the run counts as a sample of the usual cost. A plan without tests still runs there.",
|
|
278
|
+
},
|
|
248
279
|
};
|
|
249
280
|
/** Whether a turn ran in a host that is not Claude Code (see {@link Turn.host}). */
|
|
250
281
|
function otherHost(turn) {
|
|
@@ -337,7 +368,8 @@ export function localFindings(turn) {
|
|
|
337
368
|
const reading = readingDecline(code, turn.decision.why ?? "");
|
|
338
369
|
out.push({
|
|
339
370
|
code: `declined_${code || "unknown"}`,
|
|
340
|
-
|
|
371
|
+
// A held plan is not broken: it waits for its owner's decision.
|
|
372
|
+
severity: code === "not_ready" && !turn.matched.scenarioId ? "info" : code === "held" ? "warn" : "problem",
|
|
341
373
|
title: `Matched ${turn.matched.runId} but declined to steer: ${turn.decision.why ?? code}.`,
|
|
342
374
|
cause: reading?.cause ?? known?.cause ?? "the runner's log line carries the reason text.",
|
|
343
375
|
fix: reading?.fix ??
|
|
@@ -413,16 +445,27 @@ export function localFindings(turn) {
|
|
|
413
445
|
? otherHost(turn)
|
|
414
446
|
? HOST_UNMEASURED
|
|
415
447
|
: "the turn's cost could not be read from the transcript, or the runner has no price for the model: update it. Its report is unmeasured; run bir-hooks where the Claude Code transcript is readable."
|
|
416
|
-
: i.event === "replay.
|
|
417
|
-
? "
|
|
418
|
-
: i.event === "
|
|
419
|
-
? "the
|
|
420
|
-
: i.event === "run.
|
|
421
|
-
? "the
|
|
422
|
-
:
|
|
448
|
+
: i.event === "replay.handover" && i.kind === "other_branch"
|
|
449
|
+
? "this request was the other half of the task the plan holds (a step's test said so), and the agent did it; that run is how the service learns the other half."
|
|
450
|
+
: i.event === "replay.diverge" || i.event === "replay.handover"
|
|
451
|
+
? "the model left the script or the runner handed the turn back; the service section shows what the fallback cost."
|
|
452
|
+
: i.event === "run.incomplete"
|
|
453
|
+
? "the recording is not the whole run, so it is never calculated and no longer catches its prompt; run the task again to record it whole."
|
|
454
|
+
: i.event === "run.host_failed"
|
|
455
|
+
? "the application's run failed; its recording is closed with that error and never becomes a plan."
|
|
456
|
+
: "see the guide's troubleshooting table for this line.",
|
|
423
457
|
});
|
|
424
458
|
}
|
|
425
459
|
const r = turn.reported;
|
|
460
|
+
if (r?.redoneBy) {
|
|
461
|
+
out.push({
|
|
462
|
+
code: "redone",
|
|
463
|
+
severity: "warn",
|
|
464
|
+
title: `The plan ran to its end, and the agent did not take its result: it did the task again (${r.redoneBy}${r.ownCalls != null ? `, ${r.ownCalls} calls of its own` : ""}).`,
|
|
465
|
+
cause: "The service keeps that run as the recording's newest run, with the plan's own calls first. It judges the turn — did the agent reject the result, and why — and holds the plan while it does, so a wrong half is not done again; a correction of the plan may follow.",
|
|
466
|
+
fix: `\`bir investigate ${r.redoneBy}\` shows the redo; the service section below shows the verdict, the hold and any correction (\`bir scenario edits <scnId>\` lists corrections, \`bir scenario undo <scnId> --edit scor_…\` takes one back).`,
|
|
467
|
+
});
|
|
468
|
+
}
|
|
426
469
|
if (r) {
|
|
427
470
|
if (r.measured === false) {
|
|
428
471
|
out.push(otherHost(turn)
|
|
@@ -542,6 +585,9 @@ export function renderTurn(out, turn) {
|
|
|
542
585
|
if (turn.done) {
|
|
543
586
|
out(` replay ${turn.done.outcome}${turn.done.steps ? ` ${turn.done.steps} steps` : ""}${turn.done.ms != null ? ` in ${(turn.done.ms / 1000).toFixed(1)} s` : ""}`);
|
|
544
587
|
}
|
|
588
|
+
if (turn.repeat?.runId) {
|
|
589
|
+
out(` repeat ${turn.repeat.runId}${turn.repeat.of ? ` — the newest run of ${turn.repeat.of}` : ""}${turn.repeat.reason ? ` (${turn.repeat.reason})` : ""}`);
|
|
590
|
+
}
|
|
545
591
|
if (turn.reported) {
|
|
546
592
|
const r = turn.reported;
|
|
547
593
|
const p = savedPct(turn);
|
|
@@ -550,7 +596,9 @@ export function renderTurn(out, turn) {
|
|
|
550
596
|
(r.savedUsd != null
|
|
551
597
|
? ` → saved ${usd(r.savedUsd)}${p != null ? ` (${pct(p)} of ${usd(r.savedUsd + cost)})` : ""}`
|
|
552
598
|
: "") +
|
|
553
|
-
(r.measured === false ? " UNMEASURED" : "")
|
|
599
|
+
(r.measured === false ? " UNMEASURED" : "") +
|
|
600
|
+
// The agent did not take the plan's result (autocorrect.md AC1).
|
|
601
|
+
(r.redoneBy ? ` redone by ${r.redoneBy}${r.ownCalls != null ? ` (${r.ownCalls} calls of its own)` : ""}` : ""));
|
|
554
602
|
}
|
|
555
603
|
if (turn.finish) {
|
|
556
604
|
const f = turn.finish;
|
|
@@ -563,11 +611,23 @@ export function renderTurn(out, turn) {
|
|
|
563
611
|
export function renderService(out, s, viewerIsOwner) {
|
|
564
612
|
out();
|
|
565
613
|
out(`From the service${s.subject ? ` (${s.subject.kind} ${s.subject.id})` : ""}${s.owner ? `, owner ${viewerIsOwner ? "you" : (s.owner.email ?? s.owner.id)}` : ""}:`);
|
|
566
|
-
if (s.run) {
|
|
614
|
+
if (s.run?.repeatOfRunId) {
|
|
615
|
+
// A repeat is evidence, never a recording (newestRun.md N4): it is not
|
|
616
|
+
// listed, not embedded and never calculated, so none of those words — and
|
|
617
|
+
// no "calculate it" — belongs to it.
|
|
618
|
+
const r = s.run;
|
|
619
|
+
out(` repeat ${r.id} the newest run of ${r.repeatOfRunId}${r.repeatReason ? ` (${r.repeatReason})` : ""} ${r.actionCount} tool calls of the agent's own (${r.tools.join(", ") || "none"}) cost ${money(r.originalCostUsd)}`);
|
|
620
|
+
if (r.executionId)
|
|
621
|
+
out(` redid the plan's run ${r.executionId}`);
|
|
622
|
+
out(" a repeat is evidence about its recording's task, not a recording of its own: no plan is ever made from it");
|
|
623
|
+
}
|
|
624
|
+
else if (s.run) {
|
|
567
625
|
const r = s.run;
|
|
568
626
|
out(` recording ${r.id} ${r.isRecording ? "listed" : "below threshold"}${r.embedded ? ", embedded" : ", NOT embedded"} ${r.actionCount} tool calls (${r.tools.join(", ") || "none"}) cost ${money(r.originalCostUsd)} hits ${r.iterations}${r.duplicateOf ? ` duplicate of ${r.duplicateOf}` : ""}`);
|
|
569
627
|
if (r.title || r.input)
|
|
570
628
|
out(` "${(r.title ?? r.input).replace(/\s+/g, " ").slice(0, 100)}"`);
|
|
629
|
+
if (r.newestRunId)
|
|
630
|
+
out(` newest run of its task ${r.newestRunId}`);
|
|
571
631
|
}
|
|
572
632
|
else {
|
|
573
633
|
out(" recording none");
|
|
@@ -575,6 +635,14 @@ export function renderService(out, s, viewerIsOwner) {
|
|
|
575
635
|
if (s.scenario) {
|
|
576
636
|
const sc = s.scenario;
|
|
577
637
|
out(` scenario ${sc.id} ${sc.state}${sc.disabledAt ? " (switched off)" : ""} ${sc.steps.length} steps (${[...new Set(sc.steps.map((x) => x.toolName ?? "?"))].join(", ")}) baseline ${money(sc.baseline.costUsd)} from ${sc.baseline.samples} sample(s)${sc.hitCount != null ? ` hits ${sc.hitCount}` : ""}`);
|
|
638
|
+
if (sc.held) {
|
|
639
|
+
out(` HELD since ${when(sc.held.at ?? undefined)}${sc.held.reason ? ` (${sc.held.reason})` : ""}: not handed out — the agent does the task until the plan is corrected or let go${sc.held.runId ? `; the redo is ${sc.held.runId}` : ""}`);
|
|
640
|
+
}
|
|
641
|
+
for (const line of stepLines(sc.steps))
|
|
642
|
+
out(` ${line}`);
|
|
643
|
+
const correction = newestCorrection(sc.corrections);
|
|
644
|
+
if (correction)
|
|
645
|
+
out(` ${correctionLine(correction)}`);
|
|
578
646
|
if (sc.lastDecline)
|
|
579
647
|
out(` last decline ${sc.lastDecline.reason} at ${when(sc.lastDecline.at ?? undefined)}`);
|
|
580
648
|
if (sc.error)
|
|
@@ -591,10 +659,74 @@ export function renderService(out, s, viewerIsOwner) {
|
|
|
591
659
|
const failed = e.steps?.filter((st) => st.status === "failed") ?? [];
|
|
592
660
|
for (const st of failed)
|
|
593
661
|
out(` step ${st.stepIndex} ${st.toolName ?? ""} failed${st.error ? `: ${st.error.slice(0, 120)}` : ""}`);
|
|
662
|
+
const notTaken = e.steps?.filter((st) => st.status === "not_taken") ?? [];
|
|
663
|
+
if (notTaken.length > 0)
|
|
664
|
+
out(` not needed: step${notTaken.length === 1 ? "" : "s"} ${notTaken.map((st) => st.stepIndex).join(", ")}`);
|
|
665
|
+
if (e.redoneByRunId) {
|
|
666
|
+
out(` redone by ${e.redoneByRunId}${e.redoneOwnCalls != null ? ` (${e.redoneOwnCalls} calls of the agent's own)` : ""}`);
|
|
667
|
+
}
|
|
668
|
+
for (const line of verdictLines(e.verdict, e.verdictState))
|
|
669
|
+
out(` ${line}`);
|
|
594
670
|
}
|
|
595
671
|
}
|
|
596
672
|
renderFindings(out, s.findings ?? [], "Service findings");
|
|
597
673
|
}
|
|
674
|
+
/** The steps that run only under a test, or that a correction made, one line each (AC5, AC6). */
|
|
675
|
+
function stepLines(steps) {
|
|
676
|
+
const lines = [];
|
|
677
|
+
for (const st of steps) {
|
|
678
|
+
const parts = [];
|
|
679
|
+
if (st.when) {
|
|
680
|
+
const otherwise = st.when.otherwise === "skip" ? "otherwise skip" : st.when.otherwise === "hand_over" ? "otherwise hand over" : "";
|
|
681
|
+
parts.push(`only when ${st.when.text ?? "its test says so"}${otherwise ? `, ${otherwise}` : ""}`);
|
|
682
|
+
}
|
|
683
|
+
if (st.correctionId)
|
|
684
|
+
parts.push(`${st.correctionKind ?? "changed"} by correction ${st.correctionId}`);
|
|
685
|
+
if (parts.length > 0)
|
|
686
|
+
lines.push(`step ${st.stepIndex} ${st.toolName ?? "(a sub-task)"}: ${parts.join("; ")}`);
|
|
687
|
+
}
|
|
688
|
+
return lines;
|
|
689
|
+
}
|
|
690
|
+
/** The newest correction of a plan, by when it was made. */
|
|
691
|
+
function newestCorrection(corrections) {
|
|
692
|
+
return [...(corrections ?? [])]
|
|
693
|
+
.filter((c) => typeof c?.id === "string")
|
|
694
|
+
.sort((x, y) => ((x.createdAt ?? "") < (y.createdAt ?? "") ? 1 : (x.createdAt ?? "") > (y.createdAt ?? "") ? -1 : 0))[0];
|
|
695
|
+
}
|
|
696
|
+
function correctionLine(c) {
|
|
697
|
+
const what = [];
|
|
698
|
+
if (c.added?.length)
|
|
699
|
+
what.push(`${c.added.length} step${c.added.length === 1 ? "" : "s"} added`);
|
|
700
|
+
if (c.tested?.length)
|
|
701
|
+
what.push(`tests on ${c.tested.length} step${c.tested.length === 1 ? "" : "s"}`);
|
|
702
|
+
if (c.changed?.length)
|
|
703
|
+
what.push(`${c.changed.length} step${c.changed.length === 1 ? "" : "s"} changed`);
|
|
704
|
+
if (c.answerChanged)
|
|
705
|
+
what.push("the answer changed");
|
|
706
|
+
const state = c.state === "proposed"
|
|
707
|
+
? 'shown, not used yet ("Use the correction" on the recording page)'
|
|
708
|
+
: c.state === "used"
|
|
709
|
+
? `used${c.usedAt ? ` ${when(c.usedAt)}` : ""}`
|
|
710
|
+
: c.state === "failed"
|
|
711
|
+
? `not used: its check failed${c.problems?.[0] ? ` — ${c.problems[0].slice(0, 120)}` : ""}`
|
|
712
|
+
: (c.state ?? "");
|
|
713
|
+
return `correction ${c.id} ${state}${what.length ? ` (${what.join(", ")})` : ""}${c.summary ? ` ${c.summary.replace(/\s+/g, " ").slice(0, 120)}` : ""}`;
|
|
714
|
+
}
|
|
715
|
+
/** The service's verdict on a redone turn, in words (AC3). */
|
|
716
|
+
function verdictLines(verdict, state) {
|
|
717
|
+
if (!verdict) {
|
|
718
|
+
return state && state !== "done" ? [`verdict ${state === "waiting" ? "waiting (the service judges the turn when the redo has finished)" : state}`] : [];
|
|
719
|
+
}
|
|
720
|
+
const lines = [`verdict ${verdict.kind ?? (verdict.accepted ? "accepted" : "?")}${verdict.why ? `: ${verdict.why}` : ""}`];
|
|
721
|
+
if (verdict.rejectedSteps?.length)
|
|
722
|
+
lines.push(`rejected step${verdict.rejectedSteps.length === 1 ? "" : "s"} ${verdict.rejectedSteps.join(", ")}`);
|
|
723
|
+
if (verdict.condition)
|
|
724
|
+
lines.push(`the test it needs: ${verdict.condition}`);
|
|
725
|
+
if (verdict.leftInPlace?.length) {
|
|
726
|
+
lines.push(`left in place (remove by hand if it should not be there): ${verdict.leftInPlace.join("; ")}`);
|
|
727
|
+
}
|
|
728
|
+
return lines;
|
|
729
|
+
}
|
|
598
730
|
// ── the command ───────────────────────────────────────────────────────────────
|
|
599
731
|
async function fetchInvestigation(deps, id, limit, clock) {
|
|
600
732
|
const reply = await clock.within(deps.service(`/investigate/${id}${limit ? `?limit=${limit}` : ""}`));
|
|
@@ -138,6 +138,37 @@ export interface ParamEditEntry {
|
|
|
138
138
|
/** A recalculation dropped the parameter; the entry is history now. */
|
|
139
139
|
replacedAt: string | null;
|
|
140
140
|
}
|
|
141
|
+
/**
|
|
142
|
+
* A correction the service made from a run the agent redid (its
|
|
143
|
+
* autocorrect.md AC6): the plan changed as a whole — steps added, tests put
|
|
144
|
+
* on steps, a step's code or the answer's changed — and checked by code over
|
|
145
|
+
* the recording and the redone turn before it was kept. `bir scenario undo
|
|
146
|
+
* --edit scor_…` puts the plan back as it was before it, and holds it.
|
|
147
|
+
*/
|
|
148
|
+
export interface CorrectionEntry {
|
|
149
|
+
id: string;
|
|
150
|
+
scenarioId: string;
|
|
151
|
+
/**
|
|
152
|
+
* `proposed`: made and checked, shown, not used yet (SCENARIO_AUTOCORRECT=observe);
|
|
153
|
+
* `used`: the plan runs it; `failed`: its check failed, nothing was kept;
|
|
154
|
+
* `undone`: taken back; `replaced`: a recalculation replaced the plan since.
|
|
155
|
+
*/
|
|
156
|
+
state: "proposed" | "used" | "failed" | "undone" | "replaced" | string;
|
|
157
|
+
createdAt: string;
|
|
158
|
+
usedAt?: string | null;
|
|
159
|
+
undoneAt?: string | null;
|
|
160
|
+
executionId?: string | null;
|
|
161
|
+
redoneRunId?: string | null;
|
|
162
|
+
verdictKind?: string | null;
|
|
163
|
+
summary?: string | null;
|
|
164
|
+
added?: number[];
|
|
165
|
+
tested?: number[];
|
|
166
|
+
changed?: number[];
|
|
167
|
+
answerChanged?: boolean;
|
|
168
|
+
problems?: string[];
|
|
169
|
+
chainRevisionBefore?: number | null;
|
|
170
|
+
chainRevisionAfter?: number | null;
|
|
171
|
+
}
|
|
141
172
|
export type ServiceReply = {
|
|
142
173
|
status: number;
|
|
143
174
|
body: unknown;
|
|
@@ -166,7 +197,10 @@ export interface ScenarioEditArgs {
|
|
|
166
197
|
freeze: boolean;
|
|
167
198
|
unfreeze: boolean;
|
|
168
199
|
note?: string;
|
|
169
|
-
/**
|
|
200
|
+
/**
|
|
201
|
+
* `undo --edit`: the `sedit_` (or `pedit_`) id to put back, or a `scor_`
|
|
202
|
+
* correction to take back whole. `undo --step` is {@link step}.
|
|
203
|
+
*/
|
|
170
204
|
edit?: string;
|
|
171
205
|
/** `--revision`: save only if the plan is still at this `chainRevision`. */
|
|
172
206
|
revision?: number;
|
|
@@ -222,6 +256,10 @@ export declare function renderReport(out: (line?: string) => void, r: EditReport
|
|
|
222
256
|
* short — a window the service cut itself, or a small call — is printed whole.
|
|
223
257
|
*/
|
|
224
258
|
export declare function around(text: string, at: number): string;
|
|
259
|
+
/** `[11,12,13,15]` → "11–13, 15": the step lists of a correction. */
|
|
260
|
+
export declare function stepRanges(steps: readonly number[] | undefined): string;
|
|
261
|
+
/** The second line of a correction in `edits`: what it changed, from which run, and what became of it. */
|
|
262
|
+
export declare function correctionSummary(c: CorrectionEntry): string;
|
|
225
263
|
/**
|
|
226
264
|
* What `undo` without `--edit` takes back (D9): the newest **edit** — never an
|
|
227
265
|
* undo entry — that nothing has put back and no recalculation replaced; of
|
|
@@ -239,5 +277,11 @@ export declare function around(text: string, at: number): string;
|
|
|
239
277
|
export declare function newestUndoable(edits: EditEntry[], step?: number): EditEntry | undefined;
|
|
240
278
|
/** {@link newestUndoable} for kind changes: the newest that stands, of `param` when it is given. */
|
|
241
279
|
export declare function newestUndoableParam(paramEdits: ParamEditEntry[], param?: string): ParamEditEntry | undefined;
|
|
280
|
+
/**
|
|
281
|
+
* The correction `undo` without `--edit` takes back, when it is the newest
|
|
282
|
+
* change of all: one in use, not undone. A plan-wide change, so never one
|
|
283
|
+
* when `--step` or `--param` narrows the undo to a step or a parameter.
|
|
284
|
+
*/
|
|
285
|
+
export declare function newestUndoableCorrection(corrections: CorrectionEntry[]): CorrectionEntry | undefined;
|
|
242
286
|
export {};
|
|
243
287
|
//# sourceMappingURL=scenario-edit.d.ts.map
|