@basein/runner 0.3.18 → 0.3.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/bir-hooks.d.ts +11 -8
- package/dist/bin/bir-hooks.js +11 -8
- package/dist/bin/bir.js +20 -7
- package/dist/bin/investigate.d.ts +9 -0
- package/dist/bin/investigate.js +26 -12
- package/dist/bin/scenario-edit.js +10 -0
- package/dist/control/recorder-main.d.ts +15 -0
- package/dist/control/recorder-main.js +33 -8
- package/dist/control/server.d.ts +27 -11
- package/dist/control/server.js +100 -20
- package/dist/record/recorder.d.ts +10 -2
- package/dist/replay/condition-words.d.ts +28 -0
- package/dist/replay/condition-words.js +59 -0
- package/dist/replay/controller.d.ts +45 -6
- package/dist/replay/controller.js +181 -50
- package/dist/replay/handover.d.ts +7 -0
- package/dist/replay/handover.js +89 -51
- package/dist/replay/plan.d.ts +11 -4
- package/dist/replay/plan.js +28 -8
- package/dist/replay/types.d.ts +19 -3
- package/docs/calculatedReplay.md +45 -5
- package/docs/calculatedReplayGuide.md +13 -4
- package/package.json +1 -1
- package/production.md +21 -7
package/dist/bin/bir-hooks.d.ts
CHANGED
|
@@ -101,15 +101,18 @@
|
|
|
101
101
|
* Separate from BIR_INTENT_MATCHES_PER_TURN: arming
|
|
102
102
|
* is a claim on the turn, counting hits is not
|
|
103
103
|
* BIR_INTENT_MATCH_BUDGET_MS one intent probe's own ceiling (default 4000).
|
|
104
|
-
* Not a wait: PreToolUse
|
|
105
|
-
*
|
|
106
|
-
*
|
|
104
|
+
* Not a wait of its own: PreToolUse waits for a
|
|
105
|
+
* probe only while arming is on (the default from
|
|
106
|
+
* 0.3.19; off with BIR_SEGMENT_ARM=0 or
|
|
107
|
+
* BIR_INTENT_ARM=0), and then at most
|
|
107
108
|
* BIR_INTENT_ARM_BUDGET_MS (default 500)
|
|
108
|
-
* BIR_SEGMENT_ARM 1
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
* (
|
|
109
|
+
* BIR_SEGMENT_ARM unset or 1 (the default from 0.3.19): a segment
|
|
110
|
+
* the service hands out runs mid-task — one found
|
|
111
|
+
* by repeated work, or a half after a plan's
|
|
112
|
+
* decision — and so may a whole plan whose intent
|
|
113
|
+
* a step matches (BIR_INTENT_ARM follows it unless
|
|
114
|
+
* set). 0 is observe-only: what would have armed
|
|
115
|
+
* is logged and nothing is replaced
|
|
113
116
|
* BIR_REPLAY_BUDGET_MS whole-plan ceiling, direct mode (unset: the
|
|
114
117
|
* controller's 100000, BIR_REPLAY_PLAN_BUDGET_MS)
|
|
115
118
|
* BIR_STEP_TIMEOUT_MS one direct tools/call (default 60000)
|
package/dist/bin/bir-hooks.js
CHANGED
|
@@ -101,15 +101,18 @@
|
|
|
101
101
|
* Separate from BIR_INTENT_MATCHES_PER_TURN: arming
|
|
102
102
|
* is a claim on the turn, counting hits is not
|
|
103
103
|
* BIR_INTENT_MATCH_BUDGET_MS one intent probe's own ceiling (default 4000).
|
|
104
|
-
* Not a wait: PreToolUse
|
|
105
|
-
*
|
|
106
|
-
*
|
|
104
|
+
* Not a wait of its own: PreToolUse waits for a
|
|
105
|
+
* probe only while arming is on (the default from
|
|
106
|
+
* 0.3.19; off with BIR_SEGMENT_ARM=0 or
|
|
107
|
+
* BIR_INTENT_ARM=0), and then at most
|
|
107
108
|
* BIR_INTENT_ARM_BUDGET_MS (default 500)
|
|
108
|
-
* BIR_SEGMENT_ARM 1
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
* (
|
|
109
|
+
* BIR_SEGMENT_ARM unset or 1 (the default from 0.3.19): a segment
|
|
110
|
+
* the service hands out runs mid-task — one found
|
|
111
|
+
* by repeated work, or a half after a plan's
|
|
112
|
+
* decision — and so may a whole plan whose intent
|
|
113
|
+
* a step matches (BIR_INTENT_ARM follows it unless
|
|
114
|
+
* set). 0 is observe-only: what would have armed
|
|
115
|
+
* is logged and nothing is replaced
|
|
113
116
|
* BIR_REPLAY_BUDGET_MS whole-plan ceiling, direct mode (unset: the
|
|
114
117
|
* controller's 100000, BIR_REPLAY_PLAN_BUDGET_MS)
|
|
115
118
|
* BIR_STEP_TIMEOUT_MS one direct tools/call (default 60000)
|
package/dist/bin/bir.js
CHANGED
|
@@ -3118,13 +3118,20 @@ async function doctor(args) {
|
|
|
3118
3118
|
"For headless runs (`claude -p`) allow them in permissions.allow or --allowedTools, or set " +
|
|
3119
3119
|
"BIR_CORRELATION_DECISION=allow where the recorder starts");
|
|
3120
3120
|
}
|
|
3121
|
-
// Whether a
|
|
3122
|
-
//
|
|
3123
|
-
//
|
|
3124
|
-
//
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
|
|
3121
|
+
// Whether a segment the service hands out may run mid-task (segmented.md
|
|
3122
|
+
// R-LIFE-8). On by default from 0.3.19 (the service's segmentsbreakdown.md
|
|
3123
|
+
// SB-21); either state must be visible without reading a log, and an older
|
|
3124
|
+
// recorder still running here keeps the old default until it is replaced.
|
|
3125
|
+
// A segment starts only where the probe is waited on, so `intentArm: false`
|
|
3126
|
+
// (BIR_INTENT_ARM=0) starts nothing mid-task, segments included; a
|
|
3127
|
+
// recorder older than that field says nothing of it.
|
|
3128
|
+
notes.push(health.segmentArm === true && health.intentArm !== false
|
|
3129
|
+
? "segments run mid-task — recurring sub-tasks, and the halves after a plan's decision — and so may whole plans (the default; BIR_SEGMENT_ARM=0 turns this off)"
|
|
3130
|
+
: health.segmentArm === true
|
|
3131
|
+
? "segments are observed only — nothing starts mid-task (BIR_INTENT_ARM=0 where the recorder started)"
|
|
3132
|
+
: theirs && theirs !== VERSION
|
|
3133
|
+
? `segments are observed only — this recorder (${theirs}) is older than this bir, which runs them by default: \`bir up --restart\``
|
|
3134
|
+
: `segments are observed only — nothing is replaced (BIR_SEGMENT_ARM=0 where the recorder started)${health.intentArm === true ? "; whole plans may still start mid-task (BIR_INTENT_ARM=1)" : ""}`);
|
|
3128
3135
|
const sessions = health.sessions ?? [];
|
|
3129
3136
|
const probes = sessions.reduce((n, s) => n + (s.intent?.probes ?? 0), 0);
|
|
3130
3137
|
const armed = sessions.reduce((n, s) => n + (s.intent?.armed ?? 0), 0);
|
|
@@ -3714,6 +3721,12 @@ async function replayCommand(args) {
|
|
|
3714
3721
|
}
|
|
3715
3722
|
out(`response ${JSON.stringify(result.responseModel ?? {})}`);
|
|
3716
3723
|
out(`steps ${result.steps ?? 0}${result.partial ? " (partial — budget reached)" : ""}`);
|
|
3724
|
+
// A plan's decision (the service's segmentsbreakdown.md SB-6): the steps
|
|
3725
|
+
// from there are the half the recording took, so the run stops before them.
|
|
3726
|
+
if (result.stopped?.reason === "decision") {
|
|
3727
|
+
out(`stopped before step ${result.stopped.step}${result.stopped.tool ? ` (${result.stopped.tool})` : ""}: ` +
|
|
3728
|
+
"the plan's decision, which the agent makes — the steps from there did not run");
|
|
3729
|
+
}
|
|
3717
3730
|
return 0;
|
|
3718
3731
|
}
|
|
3719
3732
|
// ── wrap ───────────────────────────────────────────────────────────────────
|
|
@@ -245,6 +245,13 @@ interface ServiceInvestigation {
|
|
|
245
245
|
runId?: string | null;
|
|
246
246
|
} | null;
|
|
247
247
|
corrections?: ServiceCorrection[];
|
|
248
|
+
/**
|
|
249
|
+
* The step the plan stops in front of so that the agent decides, and the
|
|
250
|
+
* condition in words (the service's segmentsbreakdown.md SB-6). Null when
|
|
251
|
+
* the plan has none; absent from an older service.
|
|
252
|
+
*/
|
|
253
|
+
decisionStep?: number | null;
|
|
254
|
+
decisionText?: string | null;
|
|
248
255
|
} | null;
|
|
249
256
|
executions?: Array<{
|
|
250
257
|
id: string;
|
|
@@ -265,6 +272,8 @@ interface ServiceInvestigation {
|
|
|
265
272
|
status: string;
|
|
266
273
|
error?: string;
|
|
267
274
|
}> | null;
|
|
275
|
+
/** Where a `fell_back` run handed over, and why (`decision`, …): null for any other run. */
|
|
276
|
+
fallbackKind?: string | null;
|
|
268
277
|
/** The repeat the agent redid the task in, and its own calls there (AC1). */
|
|
269
278
|
redoneByRunId?: string | null;
|
|
270
279
|
redoneOwnCalls?: number | null;
|
package/dist/bin/investigate.js
CHANGED
|
@@ -203,8 +203,8 @@ export function findTurn(turns, id) {
|
|
|
203
203
|
/** The runner's own gate vocabulary, with the cause and the fix (guide §9). */
|
|
204
204
|
export const DECLINES = {
|
|
205
205
|
replay_disabled: {
|
|
206
|
-
cause: "calculated replay was off in this bir-hooks (BIR_REPLAY=0), or a sub-task was handed out
|
|
207
|
-
fix: "start bir-hooks with BIR_REPLAY=1 (the default); for sub-tasks
|
|
206
|
+
cause: "calculated replay was off in this bir-hooks (BIR_REPLAY=0), or a sub-task was handed out to a recorder that only observes sub-tasks (BIR_SEGMENT_ARM=0, or a recorder older than 0.3.19).",
|
|
207
|
+
fix: "start bir-hooks with BIR_REPLAY=1 (the default); for sub-tasks leave BIR_SEGMENT_ARM unset (on from 0.3.19), then `bir up --restart`.",
|
|
208
208
|
},
|
|
209
209
|
not_ready: {
|
|
210
210
|
cause: "the matched recording had no ready scenario at that moment.",
|
|
@@ -434,9 +434,11 @@ export function localFindings(turn) {
|
|
|
434
434
|
}
|
|
435
435
|
for (const i of turn.incidents) {
|
|
436
436
|
const fields = Object.fromEntries(Object.entries(i).filter(([k]) => k !== "at" && k !== "event"));
|
|
437
|
+
// A stop at the plan's decision is the plan working as split: no fault.
|
|
438
|
+
const decisionStop = i.event === "replay.handover" && i.kind === "decision";
|
|
437
439
|
out.push({
|
|
438
440
|
code: i.event.replace(/\./g, "_"),
|
|
439
|
-
severity: i.event.endsWith("_failed") || i.event === "replay.failed" ? "problem" : "warn",
|
|
441
|
+
severity: i.event.endsWith("_failed") || i.event === "replay.failed" ? "problem" : decisionStop ? "info" : "warn",
|
|
440
442
|
title: `${i.event}${i.error ? `: ${String(i.error)}` : ""}`,
|
|
441
443
|
cause: Object.entries(fields)
|
|
442
444
|
.map(([k, v]) => `${k}=${String(v)}`)
|
|
@@ -447,13 +449,15 @@ export function localFindings(turn) {
|
|
|
447
449
|
: "the turn's cost could not be read from the transcript, or the runner has no price for the model: update it. Its report is unmeasured; run bir-hooks where the Claude Code transcript is readable."
|
|
448
450
|
: i.event === "replay.handover" && i.kind === "other_branch"
|
|
449
451
|
? "this request was the other half of the task the plan holds (a step's test said so), and the agent did it; that run is how the service learns the other half."
|
|
450
|
-
:
|
|
451
|
-
? "the
|
|
452
|
-
: i.event === "
|
|
453
|
-
? "the
|
|
454
|
-
: i.event === "run.
|
|
455
|
-
? "the
|
|
456
|
-
:
|
|
452
|
+
: decisionStop
|
|
453
|
+
? "nothing to fix: the plan stopped at its decision, as planned. It ran the steps before it, and the agent chose the way. A segment of the half it chose may have run next (a `replay.intent_armed` line with kind segment); a `run.fragment` line with decision=true is the agent's own work, from which the service makes the other half."
|
|
454
|
+
: i.event === "replay.diverge" || i.event === "replay.handover"
|
|
455
|
+
? "the model left the script or the runner handed the turn back; the service section shows what the fallback cost."
|
|
456
|
+
: i.event === "run.incomplete"
|
|
457
|
+
? "the recording is not the whole run, so it is never calculated and no longer catches its prompt; run the task again to record it whole."
|
|
458
|
+
: i.event === "run.host_failed"
|
|
459
|
+
? "the application's run failed; its recording is closed with that error and never becomes a plan."
|
|
460
|
+
: "see the guide's troubleshooting table for this line.",
|
|
457
461
|
});
|
|
458
462
|
}
|
|
459
463
|
const r = turn.reported;
|
|
@@ -536,6 +540,12 @@ export function savedPct(turn) {
|
|
|
536
540
|
return baseline > 0 ? r.savedUsd / baseline : null;
|
|
537
541
|
}
|
|
538
542
|
const short = (id) => (id ? id.slice(0, 13) : "—");
|
|
543
|
+
/**
|
|
544
|
+
* An execution's outcome, and where it handed the task over when the service
|
|
545
|
+
* says (`fell_back · decision`). The service sends `fallbackKind: null` for a
|
|
546
|
+
* run that did not hand over, and an older one sends nothing.
|
|
547
|
+
*/
|
|
548
|
+
const outcomeOf = (e) => typeof e.fallbackKind === "string" && e.fallbackKind !== "" ? `${e.outcome} · ${e.fallbackKind}` : e.outcome;
|
|
539
549
|
/** 4624 → "4,624", whatever the machine's locale. */
|
|
540
550
|
const count = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ",");
|
|
541
551
|
const money = (v) => (v == null ? "—" : usd(v));
|
|
@@ -638,6 +648,10 @@ export function renderService(out, s, viewerIsOwner) {
|
|
|
638
648
|
if (sc.held) {
|
|
639
649
|
out(` HELD since ${when(sc.held.at ?? undefined)}${sc.held.reason ? ` (${sc.held.reason})` : ""}: not handed out — the agent does the task until the plan is corrected or let go${sc.held.runId ? `; the redo is ${sc.held.runId}` : ""}`);
|
|
640
650
|
}
|
|
651
|
+
if (typeof sc.decisionStep === "number") {
|
|
652
|
+
const text = typeof sc.decisionText === "string" ? sc.decisionText.replace(/\s+/g, " ").trim() : "";
|
|
653
|
+
out(` decision at step ${sc.decisionStep}: the plan stops in front of it, and the agent decides${text ? ` (${text.length > 160 ? `${text.slice(0, 160)}…` : text})` : ""}`);
|
|
654
|
+
}
|
|
641
655
|
for (const line of stepLines(sc.steps))
|
|
642
656
|
out(` ${line}`);
|
|
643
657
|
const correction = newestCorrection(sc.corrections);
|
|
@@ -655,7 +669,7 @@ export function renderService(out, s, viewerIsOwner) {
|
|
|
655
669
|
if (ex.length > 0) {
|
|
656
670
|
out(` executions (${ex.length}, newest first)`);
|
|
657
671
|
for (const e of ex.slice(0, 10)) {
|
|
658
|
-
out(` ${when(e.createdAt)} ${short(e.id)} ${e.
|
|
672
|
+
out(` ${when(e.createdAt)} ${short(e.id)} ${outcomeOf(e).padEnd(13)} cost ${usd(e.costUsd)} saved ${money(e.savedUsd)}${e.savedPct != null ? ` (${pct(e.savedPct)})` : ""}${e.measured ? "" : " unmeasured"}`);
|
|
659
673
|
const failed = e.steps?.filter((st) => st.status === "failed") ?? [];
|
|
660
674
|
for (const st of failed)
|
|
661
675
|
out(` step ${st.stepIndex} ${st.toolName ?? ""} failed${st.error ? `: ${st.error.slice(0, 120)}` : ""}`);
|
|
@@ -790,7 +804,7 @@ export async function investigateCommand(args, deps) {
|
|
|
790
804
|
if (rows.length === 0)
|
|
791
805
|
out(" none yet");
|
|
792
806
|
for (const e of rows) {
|
|
793
|
-
out(` ${when(e.createdAt)} ${e.id} ${e.
|
|
807
|
+
out(` ${when(e.createdAt)} ${e.id} ${outcomeOf(e).padEnd(13)} cost ${usd(e.costUsd)} saved ${money(e.savedUsd)}${e.savedPct != null ? ` (${pct(e.savedPct)})` : ""}${e.measured ? "" : " unmeasured"}`);
|
|
794
808
|
out(` scenario ${e.scenarioId}${e.runId ? ` run ${e.runId}` : ""}${e.intent ? ` "${e.intent.slice(0, 60)}"` : ""}`);
|
|
795
809
|
}
|
|
796
810
|
out();
|
|
@@ -1231,6 +1231,16 @@ function explain(reply, args, deps, e) {
|
|
|
1231
1231
|
// newer entry puts the step back as this one left it, and then this one
|
|
1232
1232
|
// can be undone. It used to loop, when undo only toggled (C23).
|
|
1233
1233
|
const latest = typeof body.latestEditId === "string" ? body.latestEditId : "<the newer edit>";
|
|
1234
|
+
if (latest.startsWith("ssplit_")) {
|
|
1235
|
+
// The plan was split at its decision since (the service's
|
|
1236
|
+
// segmentsbreakdown.md). No `--edit` takes a split back: its Undo is
|
|
1237
|
+
// on the recording page's Split card, and only then can this go.
|
|
1238
|
+
const what = aboutCorrection ? "correction" : "edit";
|
|
1239
|
+
out(`${e.verb} (not_latest_edit): the plan was split at its decision after that ${what} (${latest}). ` +
|
|
1240
|
+
`It cannot be taken back while the split is in use: undo the split first (Undo on the Split card of the plan's recording page), then this ${what}:`);
|
|
1241
|
+
out(` bir scenario undo ${target} --edit ${e.editId ?? "<this edit>"}`);
|
|
1242
|
+
return 1;
|
|
1243
|
+
}
|
|
1234
1244
|
out(aboutCorrection
|
|
1235
1245
|
? `${e.verb} (not_latest_edit): the plan has changed since that correction — ${latest} changed it after. Undo the newer one first, then this one:`
|
|
1236
1246
|
: `${e.verb} (not_latest_edit): the ${aboutParam ? "parameter" : "step"} has changed since that edit — ${latest} changed it after. Undo the newer one first, then this one:`);
|
|
@@ -52,6 +52,21 @@ export declare function derivationKey(daemon?: boolean): {
|
|
|
52
52
|
apiKey?: string;
|
|
53
53
|
source?: string;
|
|
54
54
|
};
|
|
55
|
+
/**
|
|
56
|
+
* `BIR_SEGMENT_ARM`: whether a segment the service hands out may run mid-task.
|
|
57
|
+
* On unless it is `0`, from runner 0.3.19 (the service's segmentsbreakdown.md
|
|
58
|
+
* SB-21): segments found by repeated work, and the halves after a plan's
|
|
59
|
+
* decision. `0` is observe-only: what would have armed is logged, and nothing
|
|
60
|
+
* is replaced. While it is on, whole plans may start mid-task by intent too:
|
|
61
|
+
* `BIR_INTENT_ARM` follows it unless that is set.
|
|
62
|
+
*/
|
|
63
|
+
export declare function segmentArmFromEnv(env?: NodeJS.ProcessEnv): boolean;
|
|
64
|
+
/**
|
|
65
|
+
* What the `replay.enabled` startup line says segments do. A segment starts
|
|
66
|
+
* only where the probe is waited on, so `BIR_INTENT_ARM=0` starts nothing
|
|
67
|
+
* mid-task, segments included — the rule `bir doctor` words from /health.
|
|
68
|
+
*/
|
|
69
|
+
export declare function segmentsAtStartup(segmentArm: boolean, env?: NodeJS.ProcessEnv): string;
|
|
55
70
|
/** Where the two replay switches came from, for `/health` and `bir doctor`. */
|
|
56
71
|
export type ReplaySource = "env" | "sidecar" | "default";
|
|
57
72
|
/** What a recorder still serves (K4, `ControlServer.activity()`). */
|
|
@@ -152,6 +152,27 @@ export function derivationKey(daemon = process.env[DAEMON_ENV] === "1") {
|
|
|
152
152
|
}
|
|
153
153
|
return {};
|
|
154
154
|
}
|
|
155
|
+
/**
|
|
156
|
+
* `BIR_SEGMENT_ARM`: whether a segment the service hands out may run mid-task.
|
|
157
|
+
* On unless it is `0`, from runner 0.3.19 (the service's segmentsbreakdown.md
|
|
158
|
+
* SB-21): segments found by repeated work, and the halves after a plan's
|
|
159
|
+
* decision. `0` is observe-only: what would have armed is logged, and nothing
|
|
160
|
+
* is replaced. While it is on, whole plans may start mid-task by intent too:
|
|
161
|
+
* `BIR_INTENT_ARM` follows it unless that is set.
|
|
162
|
+
*/
|
|
163
|
+
export function segmentArmFromEnv(env = process.env) {
|
|
164
|
+
return env.BIR_SEGMENT_ARM !== "0";
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* What the `replay.enabled` startup line says segments do. A segment starts
|
|
168
|
+
* only where the probe is waited on, so `BIR_INTENT_ARM=0` starts nothing
|
|
169
|
+
* mid-task, segments included — the rule `bir doctor` words from /health.
|
|
170
|
+
*/
|
|
171
|
+
export function segmentsAtStartup(segmentArm, env = process.env) {
|
|
172
|
+
if (!segmentArm)
|
|
173
|
+
return "observed only (BIR_SEGMENT_ARM=0)";
|
|
174
|
+
return env.BIR_INTENT_ARM === "0" ? "observed only (BIR_INTENT_ARM=0)" : "run mid-task";
|
|
175
|
+
}
|
|
155
176
|
/**
|
|
156
177
|
* Assemble the replay configuration (docs/calculatedReplay.md §5.3 of the guide).
|
|
157
178
|
*
|
|
@@ -204,19 +225,21 @@ function buildReplayOptions(keeper, policy, key) {
|
|
|
204
225
|
intentMatch: {
|
|
205
226
|
enabled: process.env.BIR_INTENT_MATCH !== "0",
|
|
206
227
|
// The probe's own ceiling, not a wait on the hook's path: `PreToolUse`
|
|
207
|
-
//
|
|
208
|
-
//
|
|
228
|
+
// waits for it only while arming is on — the default, unless
|
|
229
|
+
// BIR_SEGMENT_ARM=0 or BIR_INTENT_ARM=0 — and then at most
|
|
209
230
|
// BIR_INTENT_ARM_BUDGET_MS (500 ms), never while the service is failing
|
|
210
|
-
// (control/server.ts)
|
|
211
|
-
//
|
|
212
|
-
// 2 500 ms
|
|
231
|
+
// (control/server.ts); otherwise it answers without it (hit counting
|
|
232
|
+
// only). 4 000 leaves a request whose best segment lands in the
|
|
233
|
+
// not-clear band its one live question under the service's own 2 500 ms
|
|
234
|
+
// attempt (segmented.md R-HIT-11).
|
|
213
235
|
budgetMs: positiveInt(process.env.BIR_INTENT_MATCH_BUDGET_MS, 4_000),
|
|
214
236
|
maxPerTurn: positiveInt(process.env.BIR_INTENT_MATCHES_PER_TURN, 3),
|
|
215
237
|
maxRequestsPerTurn: positiveInt(process.env.BIR_INTENT_REQUESTS_PER_TURN, 40),
|
|
216
238
|
},
|
|
217
|
-
//
|
|
218
|
-
//
|
|
219
|
-
|
|
239
|
+
// On by default from 0.3.19 (the service's segmentsbreakdown.md SB-21): a
|
|
240
|
+
// plan that stops at its decision hands the rest to segments of each half,
|
|
241
|
+
// and they run only mid-task. BIR_SEGMENT_ARM=0 brings back observe-only.
|
|
242
|
+
segmentArm: segmentArmFromEnv(),
|
|
220
243
|
// Both read live, never captured (C5). The recorder signs in after it
|
|
221
244
|
// listens, refreshes the token as the session outlives it, and may sign
|
|
222
245
|
// in or out while it runs; replay reads whatever holds *now*. The old
|
|
@@ -247,6 +270,8 @@ function buildReplayOptions(keeper, policy, key) {
|
|
|
247
270
|
? "the service"
|
|
248
271
|
: "recorded sample values — sign in, or a scenario with a target will not run"),
|
|
249
272
|
intentMatch: opts.intentMatch?.enabled ? "on" : "off",
|
|
273
|
+
// Whether a segment the service hands out runs, or is only logged.
|
|
274
|
+
segments: segmentsAtStartup(opts.segmentArm === true),
|
|
250
275
|
source,
|
|
251
276
|
why: "a prompt that matches a calculated scenario runs it: the scenario's own steps run without a " +
|
|
252
277
|
"permission prompt of their own (never in plan mode); every other tool call is asked about as " +
|
package/dist/control/server.d.ts
CHANGED
|
@@ -276,12 +276,13 @@ export interface ControlServerOptions {
|
|
|
276
276
|
* (fallbk.md §Runner 4).
|
|
277
277
|
*
|
|
278
278
|
* Arming there means *waiting* there, on every unsteered tool call, for a
|
|
279
|
-
* service round trip
|
|
280
|
-
*
|
|
281
|
-
*
|
|
282
|
-
*
|
|
283
|
-
*
|
|
284
|
-
*
|
|
279
|
+
* service round trip. It waits (at most `BIR_INTENT_ARM_BUDGET_MS`, 500 ms,
|
|
280
|
+
* and never while the service is failing) when segment arming is on — the
|
|
281
|
+
* recorder's default from 0.3.19, `BIR_SEGMENT_ARM` unset or `1` — or when
|
|
282
|
+
* `BIR_INTENT_ARM=1` asks for whole-scenario intent arming. `BIR_INTENT_ARM=0`
|
|
283
|
+
* never waits. Otherwise, as in a server built with neither (a test, an
|
|
284
|
+
* embedder), the probe is sent without waiting, for hit counting only, and
|
|
285
|
+
* nothing arms mid-turn.
|
|
285
286
|
*/
|
|
286
287
|
intentArm?: boolean;
|
|
287
288
|
/**
|
|
@@ -1154,14 +1155,19 @@ export declare class ControlServer {
|
|
|
1154
1155
|
*/
|
|
1155
1156
|
private excludeRunsFor;
|
|
1156
1157
|
/**
|
|
1157
|
-
* Per
|
|
1158
|
-
* (segmented.md R-OUT-6).
|
|
1158
|
+
* Per plan of this turn, its recording and how far the plan got in its own
|
|
1159
|
+
* chain (segmented.md R-OUT-6).
|
|
1159
1160
|
*
|
|
1160
1161
|
* The server uses it to refuse a segment that starts *before* where this turn
|
|
1161
1162
|
* already is: re-running steps the turn has just done is worse than not
|
|
1162
|
-
* replaying at all. The
|
|
1163
|
-
*
|
|
1164
|
-
*
|
|
1163
|
+
* replaying at all. The position is the plan's own: the step it handed over
|
|
1164
|
+
* at (its decision included), or its last step when it ran to its end. The
|
|
1165
|
+
* server maps it onto the recording — a segment's first position, a
|
|
1166
|
+
* decision's step before it — and takes the largest per recording (the
|
|
1167
|
+
* service's segmentsbreakdown.md D18). One entry per plan, so that two plans
|
|
1168
|
+
* of one recording, whose own positions are on different scales, never
|
|
1169
|
+
* hide each other. A plan declined for anything but a known-bad first step
|
|
1170
|
+
* reports nothing — it ran none of the recording, so it blocks none of it.
|
|
1165
1171
|
*
|
|
1166
1172
|
* Rebuilt on every probe rather than cached: a plan that hands over between
|
|
1167
1173
|
* two calls changes the answer.
|
|
@@ -1212,6 +1218,16 @@ export declare class ControlServer {
|
|
|
1212
1218
|
* whose intent-armed plan handed over is recording the model's work anyway.
|
|
1213
1219
|
*/
|
|
1214
1220
|
private noteHandover;
|
|
1221
|
+
/**
|
|
1222
|
+
* Drop a fragment scheduled at a decision (the service's
|
|
1223
|
+
* segmentsbreakdown.md SB-16) when a plan the intent check armed takes the
|
|
1224
|
+
* agent's next call: what follows is that plan's half, not the other one,
|
|
1225
|
+
* and recorded as the other half it would become a wrong segment C. Cleared
|
|
1226
|
+
* rather than ended, so a later hand-over of that plan schedules its own
|
|
1227
|
+
* fragment, as any plan's does in a turn with none. Any other fragment is
|
|
1228
|
+
* left as it is.
|
|
1229
|
+
*/
|
|
1230
|
+
private dropDecisionFragment;
|
|
1215
1231
|
/**
|
|
1216
1232
|
* Open the fragment run and record from this call on (fallbk.md D7).
|
|
1217
1233
|
*
|
package/dist/control/server.js
CHANGED
|
@@ -946,7 +946,9 @@ export class ControlServer {
|
|
|
946
946
|
lastStepHit: s.run.intent.lastStepHit ?? null,
|
|
947
947
|
}
|
|
948
948
|
: null,
|
|
949
|
-
fragment: s.run?.fragment
|
|
949
|
+
fragment: s.run?.fragment
|
|
950
|
+
? { state: s.run.fragment.state, ...(s.run.fragment.decision ? { decision: true } : {}) }
|
|
951
|
+
: null,
|
|
950
952
|
// The service's offer to record the turn as a recording's newest run,
|
|
951
953
|
// and what became of it (newestRun.md).
|
|
952
954
|
repeat: s.run?.repeat ? { state: s.run.repeat.state, of: s.run.repeat.runId } : null,
|
|
@@ -964,6 +966,10 @@ export class ControlServer {
|
|
|
964
966
|
// be invisible: an operator has to be able to see which of the two this
|
|
965
967
|
// machine is doing without reading a log file.
|
|
966
968
|
segmentArm: this.replay.segmentArm,
|
|
969
|
+
// And whether a step's intent may start a plan in its own PreToolUse,
|
|
970
|
+
// which means waiting for the probe there (it follows segmentArm unless
|
|
971
|
+
// BIR_INTENT_ARM says otherwise). For `bir doctor`.
|
|
972
|
+
intentArm: this.intentArming(),
|
|
967
973
|
authUrl: resolveAuthUrl() || null,
|
|
968
974
|
sessionId: this.sessionId,
|
|
969
975
|
pid: process.pid,
|
|
@@ -2095,6 +2101,12 @@ export class ControlServer {
|
|
|
2095
2101
|
if (!otherHost(session))
|
|
2096
2102
|
state.reportTool = () => this.reportToolServed();
|
|
2097
2103
|
this.declineWithoutDeliveryTool(state);
|
|
2104
|
+
// A plan that stops at its decision is billed only up to the stop (the
|
|
2105
|
+
// service's segmentsbreakdown.md SB-26): its hand-over takes this mark. A
|
|
2106
|
+
// plan of the prompt without one keeps the window it always had.
|
|
2107
|
+
if (!otherHost(session) && this.replay.stopsAtDecision(state)) {
|
|
2108
|
+
state.markUsage = () => markTranscriptUsage(session.transcriptPath);
|
|
2109
|
+
}
|
|
2098
2110
|
// A steer plan with a shell or file-write step can only be followed if the
|
|
2099
2111
|
// directive shows its computed arguments (the pin guard holds those steps
|
|
2100
2112
|
// to their exact shape), so give its parameters a moment to settle —
|
|
@@ -3021,11 +3033,21 @@ export class ControlServer {
|
|
|
3021
3033
|
// that finished in three steps would be billed for everything the agent
|
|
3022
3034
|
// did afterwards, and its saving would read as a loss.
|
|
3023
3035
|
let end = nextMark;
|
|
3036
|
+
// A plan of the prompt stopped at its decision after a step ran: its
|
|
3037
|
+
// window ends at the hand-over (the service's segmentsbreakdown.md
|
|
3038
|
+
// SB-26), and the agent's decision and the half it took are not on its
|
|
3039
|
+
// bill. Every other plan of the prompt keeps its window: the answer is
|
|
3040
|
+
// still the plan's work.
|
|
3041
|
+
const atDecision = state.handover?.kind === "decision" && state.retiredMark !== undefined;
|
|
3024
3042
|
if (state.armedBy === "intent") {
|
|
3025
3043
|
windowCost = this.costSince(session, state.usageMark);
|
|
3026
3044
|
windowMs = (state.retiredAt ?? next?.armedAt ?? Date.now()) - state.armedAt;
|
|
3027
3045
|
end = state.retiredMark ?? nextMark;
|
|
3028
3046
|
}
|
|
3047
|
+
else if (atDecision) {
|
|
3048
|
+
windowMs = (state.retiredAt ?? Date.now()) - run.startedAtMs;
|
|
3049
|
+
end = state.retiredMark;
|
|
3050
|
+
}
|
|
3029
3051
|
else if (next) {
|
|
3030
3052
|
windowMs = next.armedAt - run.startedAtMs;
|
|
3031
3053
|
}
|
|
@@ -3047,11 +3069,15 @@ export class ControlServer {
|
|
|
3047
3069
|
});
|
|
3048
3070
|
if (!report)
|
|
3049
3071
|
return;
|
|
3050
|
-
if (state.armedBy === "intent") {
|
|
3072
|
+
if (state.armedBy === "intent" || atDecision) {
|
|
3051
3073
|
logDetail("replay.done", {
|
|
3052
3074
|
scenario: state.scenarioId ?? undefined,
|
|
3053
3075
|
windowMs: Math.max(0, windowMs),
|
|
3054
|
-
billedTo:
|
|
3076
|
+
billedTo: atDecision
|
|
3077
|
+
? "its decision"
|
|
3078
|
+
: state.retiredMark !== undefined
|
|
3079
|
+
? "its own retire"
|
|
3080
|
+
: "the next arm or the turn",
|
|
3055
3081
|
});
|
|
3056
3082
|
}
|
|
3057
3083
|
// The plan ran to its end, and no redo of the model's own was recorded:
|
|
@@ -3696,8 +3722,12 @@ export class ControlServer {
|
|
|
3696
3722
|
? undefined
|
|
3697
3723
|
: await this.tryIntentMatch(session, run, toolName, intent, modelArgs);
|
|
3698
3724
|
if (armed) {
|
|
3699
|
-
if (!overran)
|
|
3725
|
+
if (!overran) {
|
|
3726
|
+
// The plan answers in this call's place: the agent's work from here
|
|
3727
|
+
// is that plan's, never the other half after a decision (SB-16).
|
|
3728
|
+
this.dropDecisionFragment(run, run.replay);
|
|
3700
3729
|
return armed;
|
|
3730
|
+
}
|
|
3701
3731
|
const late = run.replay;
|
|
3702
3732
|
if (late && late.armedBy === "intent" && late.plan && !late.retired) {
|
|
3703
3733
|
logLine("replay.decision", {
|
|
@@ -3714,9 +3744,14 @@ export class ControlServer {
|
|
|
3714
3744
|
// 3. No scenario took over: the model's own work after a hand-over is
|
|
3715
3745
|
// recorded as a fragment, starting with this call.
|
|
3716
3746
|
// Not for the call an intent match just armed a plan beside: the plan
|
|
3717
|
-
// runs next, and the fragment opens on the model's own work after it
|
|
3718
|
-
|
|
3719
|
-
|
|
3747
|
+
// runs next, and the fragment opens on the model's own work after it
|
|
3748
|
+
// — unless it was scheduled at a decision, whose fragment is the
|
|
3749
|
+
// other half only when the agent's own call opens it (SB-16).
|
|
3750
|
+
if (run.fragment?.state === "due") {
|
|
3751
|
+
if (!run.pendingDirective)
|
|
3752
|
+
await this.openFragment(session, run, reasoning, toolName, modelArgs);
|
|
3753
|
+
else
|
|
3754
|
+
this.dropDecisionFragment(run, run.pendingDirective.state);
|
|
3720
3755
|
}
|
|
3721
3756
|
}
|
|
3722
3757
|
// The arguments that will actually run — which is what must be correlated and
|
|
@@ -4314,14 +4349,19 @@ export class ControlServer {
|
|
|
4314
4349
|
return [...out];
|
|
4315
4350
|
}
|
|
4316
4351
|
/**
|
|
4317
|
-
* Per
|
|
4318
|
-
* (segmented.md R-OUT-6).
|
|
4352
|
+
* Per plan of this turn, its recording and how far the plan got in its own
|
|
4353
|
+
* chain (segmented.md R-OUT-6).
|
|
4319
4354
|
*
|
|
4320
4355
|
* The server uses it to refuse a segment that starts *before* where this turn
|
|
4321
4356
|
* already is: re-running steps the turn has just done is worse than not
|
|
4322
|
-
* replaying at all. The
|
|
4323
|
-
*
|
|
4324
|
-
*
|
|
4357
|
+
* replaying at all. The position is the plan's own: the step it handed over
|
|
4358
|
+
* at (its decision included), or its last step when it ran to its end. The
|
|
4359
|
+
* server maps it onto the recording — a segment's first position, a
|
|
4360
|
+
* decision's step before it — and takes the largest per recording (the
|
|
4361
|
+
* service's segmentsbreakdown.md D18). One entry per plan, so that two plans
|
|
4362
|
+
* of one recording, whose own positions are on different scales, never
|
|
4363
|
+
* hide each other. A plan declined for anything but a known-bad first step
|
|
4364
|
+
* reports nothing — it ran none of the recording, so it blocks none of it.
|
|
4325
4365
|
*
|
|
4326
4366
|
* Rebuilt on every probe rather than cached: a plan that hands over between
|
|
4327
4367
|
* two calls changes the answer.
|
|
@@ -4354,15 +4394,15 @@ export class ControlServer {
|
|
|
4354
4394
|
}
|
|
4355
4395
|
if (position === undefined)
|
|
4356
4396
|
continue;
|
|
4357
|
-
const seen = best.get(
|
|
4397
|
+
const seen = best.get(state.scenarioId);
|
|
4358
4398
|
if (!seen || position > seen.position) {
|
|
4359
|
-
best.set(
|
|
4399
|
+
best.set(state.scenarioId, { runId: recording, position });
|
|
4360
4400
|
}
|
|
4361
4401
|
}
|
|
4362
|
-
return [...best].map(([
|
|
4363
|
-
runId,
|
|
4402
|
+
return [...best].map(([scenarioId, v]) => ({
|
|
4403
|
+
runId: v.runId,
|
|
4364
4404
|
position: v.position,
|
|
4365
|
-
scenarioId
|
|
4405
|
+
scenarioId,
|
|
4366
4406
|
}));
|
|
4367
4407
|
}
|
|
4368
4408
|
/**
|
|
@@ -4446,7 +4486,10 @@ export class ControlServer {
|
|
|
4446
4486
|
stepPosition: run.ordering.next,
|
|
4447
4487
|
excludeRuns: this.excludeRunsFor(run),
|
|
4448
4488
|
ranThrough: this.ranThroughFor(run),
|
|
4449
|
-
|
|
4489
|
+
// Segments are asked for only where one can arm: a probe that is
|
|
4490
|
+
// waited on, in Claude Code (below). Any other probe gets the
|
|
4491
|
+
// service's `segmentWouldArm` instead, logged as observe-only.
|
|
4492
|
+
...(this.replay.segmentArm && this.intentArming() && !otherHost(session) ? { acceptSegments: true } : {}),
|
|
4450
4493
|
supportsCalls: true,
|
|
4451
4494
|
// Left out for Claude Code: a missing framework means Claude Code to
|
|
4452
4495
|
// the service, which matches a step only inside its own host.
|
|
@@ -4534,6 +4577,9 @@ export class ControlServer {
|
|
|
4534
4577
|
// the jail it may not leave (replay/builtins.ts). The session's own cwd
|
|
4535
4578
|
// from the hook payloads, never the daemon's.
|
|
4536
4579
|
state.sessionCwd = session.cwd ?? this.opts.cwd;
|
|
4580
|
+
// The request this plan serves, for a note at its decision (SB-18): the
|
|
4581
|
+
// turn's prompt. `arm` was given this step's intent instead.
|
|
4582
|
+
state.request = run.input || undefined;
|
|
4537
4583
|
this.declineWithoutDeliveryTool(state);
|
|
4538
4584
|
if (!state.plan) {
|
|
4539
4585
|
// A declined intent hit reports nothing: the turn's cost is not a
|
|
@@ -4699,7 +4745,35 @@ export class ControlServer {
|
|
|
4699
4745
|
const h = state?.handover;
|
|
4700
4746
|
if (!h || !state?.scenarioId)
|
|
4701
4747
|
return;
|
|
4702
|
-
run.fragment = {
|
|
4748
|
+
run.fragment = {
|
|
4749
|
+
scenarioId: state.scenarioId,
|
|
4750
|
+
stepIndex: h.stepIndex,
|
|
4751
|
+
state: "due",
|
|
4752
|
+
...(h.kind === "decision" ? { decision: true } : {}),
|
|
4753
|
+
};
|
|
4754
|
+
}
|
|
4755
|
+
/**
|
|
4756
|
+
* Drop a fragment scheduled at a decision (the service's
|
|
4757
|
+
* segmentsbreakdown.md SB-16) when a plan the intent check armed takes the
|
|
4758
|
+
* agent's next call: what follows is that plan's half, not the other one,
|
|
4759
|
+
* and recorded as the other half it would become a wrong segment C. Cleared
|
|
4760
|
+
* rather than ended, so a later hand-over of that plan schedules its own
|
|
4761
|
+
* fragment, as any plan's does in a turn with none. Any other fragment is
|
|
4762
|
+
* left as it is.
|
|
4763
|
+
*/
|
|
4764
|
+
dropDecisionFragment(run, by) {
|
|
4765
|
+
const f = run.fragment;
|
|
4766
|
+
if (!f || f.state !== "due" || !f.decision)
|
|
4767
|
+
return;
|
|
4768
|
+
run.fragment = undefined;
|
|
4769
|
+
logLine("run.fragment_dropped", {
|
|
4770
|
+
run: run.runId,
|
|
4771
|
+
scenario: f.scenarioId,
|
|
4772
|
+
step: f.stepIndex,
|
|
4773
|
+
by: by?.scenarioId ?? undefined,
|
|
4774
|
+
kind: by?.kind,
|
|
4775
|
+
why: "a plan the intent check armed takes the agent's next step after the decision — that work is not the other half",
|
|
4776
|
+
});
|
|
4703
4777
|
}
|
|
4704
4778
|
/**
|
|
4705
4779
|
* Open the fragment run and record from this call on (fallbk.md D7).
|
|
@@ -4719,9 +4793,14 @@ export class ControlServer {
|
|
|
4719
4793
|
return;
|
|
4720
4794
|
const input = reasoning.trim() || `${toolName} ${serializeCapped(redact(args))}`.slice(0, 2_000);
|
|
4721
4795
|
const fallbackOf = { scenarioId: f.scenarioId, stepIndex: f.stepIndex };
|
|
4796
|
+
// The agent's own call after a plan's decision (SB-16): this runner's word
|
|
4797
|
+
// that the work that follows is the agent's choice, not a plan's half.
|
|
4798
|
+
// The service reads it beside its own mark of the fragment (the
|
|
4799
|
+
// service's segmentsbreakdown.md D14).
|
|
4800
|
+
const decisionOf = f.decision ? { scenarioId: f.scenarioId, stepIndex: f.stepIndex } : undefined;
|
|
4722
4801
|
// The fragment is a run of the same host as the turn it continues.
|
|
4723
4802
|
const framework = frameworkOf(session);
|
|
4724
|
-
const id = this.recorder.startRun(input, { ...this.runMetadata(session), fallbackOf }, framework ? { fallbackOf, framework } : { fallbackOf });
|
|
4803
|
+
const id = this.recorder.startRun(input, { ...this.runMetadata(session), fallbackOf, ...(decisionOf ? { decisionOf } : {}) }, framework ? { fallbackOf, framework } : { fallbackOf });
|
|
4725
4804
|
for (const c of run.correlations.values()) {
|
|
4726
4805
|
c.settled = true;
|
|
4727
4806
|
if (c.fallback)
|
|
@@ -4738,6 +4817,7 @@ export class ControlServer {
|
|
|
4738
4817
|
run: id,
|
|
4739
4818
|
scenario: f.scenarioId,
|
|
4740
4819
|
step: f.stepIndex,
|
|
4820
|
+
decision: f.decision || undefined,
|
|
4741
4821
|
why: "recording the agent's own work after the scenario handed over",
|
|
4742
4822
|
});
|
|
4743
4823
|
if (!isRunCreationAware(this.recorder))
|