@basein/runner 0.3.18 → 0.3.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/bir-hooks.d.ts +11 -8
- package/dist/bin/bir-hooks.js +11 -8
- package/dist/bin/bir.js +20 -7
- package/dist/bin/investigate.d.ts +9 -0
- package/dist/bin/investigate.js +31 -12
- package/dist/bin/scenario-edit.js +10 -0
- package/dist/control/recorder-main.d.ts +15 -0
- package/dist/control/recorder-main.js +33 -8
- package/dist/control/server.d.ts +91 -11
- package/dist/control/server.js +736 -49
- package/dist/record/recorder.d.ts +47 -4
- package/dist/record/recorder.js +3 -0
- package/dist/record/remote-recorder.d.ts +9 -1
- package/dist/record/remote-recorder.js +30 -0
- package/dist/record/switchable-recorder.js +2 -0
- package/dist/replay/condition-words.d.ts +28 -0
- package/dist/replay/condition-words.js +59 -0
- package/dist/replay/controller.d.ts +45 -6
- package/dist/replay/controller.js +184 -51
- package/dist/replay/derive.d.ts +10 -0
- package/dist/replay/derive.js +3 -0
- package/dist/replay/handover.d.ts +7 -0
- package/dist/replay/handover.js +89 -51
- package/dist/replay/plan.d.ts +11 -4
- package/dist/replay/plan.js +28 -8
- package/dist/replay/reads-only.d.ts +29 -0
- package/dist/replay/reads-only.js +261 -0
- package/dist/replay/types.d.ts +19 -3
- package/docs/calculatedReplay.md +45 -5
- package/docs/calculatedReplayGuide.md +16 -4
- package/package.json +1 -1
- package/production.md +34 -7
package/dist/bin/bir-hooks.d.ts
CHANGED
|
@@ -101,15 +101,18 @@
|
|
|
101
101
|
* Separate from BIR_INTENT_MATCHES_PER_TURN: arming
|
|
102
102
|
* is a claim on the turn, counting hits is not
|
|
103
103
|
* BIR_INTENT_MATCH_BUDGET_MS one intent probe's own ceiling (default 4000).
|
|
104
|
-
* Not a wait: PreToolUse
|
|
105
|
-
*
|
|
106
|
-
*
|
|
104
|
+
* Not a wait of its own: PreToolUse waits for a
|
|
105
|
+
* probe only while arming is on (the default from
|
|
106
|
+
* 0.3.19; off with BIR_SEGMENT_ARM=0 or
|
|
107
|
+
* BIR_INTENT_ARM=0), and then at most
|
|
107
108
|
* BIR_INTENT_ARM_BUDGET_MS (default 500)
|
|
108
|
-
* BIR_SEGMENT_ARM 1
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
* (
|
|
109
|
+
* BIR_SEGMENT_ARM unset or 1 (the default from 0.3.19): a segment
|
|
110
|
+
* the service hands out runs mid-task — one found
|
|
111
|
+
* by repeated work, or a half after a plan's
|
|
112
|
+
* decision — and so may a whole plan whose intent
|
|
113
|
+
* a step matches (BIR_INTENT_ARM follows it unless
|
|
114
|
+
* set). 0 is observe-only: what would have armed
|
|
115
|
+
* is logged and nothing is replaced
|
|
113
116
|
* BIR_REPLAY_BUDGET_MS whole-plan ceiling, direct mode (unset: the
|
|
114
117
|
* controller's 100000, BIR_REPLAY_PLAN_BUDGET_MS)
|
|
115
118
|
* BIR_STEP_TIMEOUT_MS one direct tools/call (default 60000)
|
package/dist/bin/bir-hooks.js
CHANGED
|
@@ -101,15 +101,18 @@
|
|
|
101
101
|
* Separate from BIR_INTENT_MATCHES_PER_TURN: arming
|
|
102
102
|
* is a claim on the turn, counting hits is not
|
|
103
103
|
* BIR_INTENT_MATCH_BUDGET_MS one intent probe's own ceiling (default 4000).
|
|
104
|
-
* Not a wait: PreToolUse
|
|
105
|
-
*
|
|
106
|
-
*
|
|
104
|
+
* Not a wait of its own: PreToolUse waits for a
|
|
105
|
+
* probe only while arming is on (the default from
|
|
106
|
+
* 0.3.19; off with BIR_SEGMENT_ARM=0 or
|
|
107
|
+
* BIR_INTENT_ARM=0), and then at most
|
|
107
108
|
* BIR_INTENT_ARM_BUDGET_MS (default 500)
|
|
108
|
-
* BIR_SEGMENT_ARM 1
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
* (
|
|
109
|
+
* BIR_SEGMENT_ARM unset or 1 (the default from 0.3.19): a segment
|
|
110
|
+
* the service hands out runs mid-task — one found
|
|
111
|
+
* by repeated work, or a half after a plan's
|
|
112
|
+
* decision — and so may a whole plan whose intent
|
|
113
|
+
* a step matches (BIR_INTENT_ARM follows it unless
|
|
114
|
+
* set). 0 is observe-only: what would have armed
|
|
115
|
+
* is logged and nothing is replaced
|
|
113
116
|
* BIR_REPLAY_BUDGET_MS whole-plan ceiling, direct mode (unset: the
|
|
114
117
|
* controller's 100000, BIR_REPLAY_PLAN_BUDGET_MS)
|
|
115
118
|
* BIR_STEP_TIMEOUT_MS one direct tools/call (default 60000)
|
package/dist/bin/bir.js
CHANGED
|
@@ -3118,13 +3118,20 @@ async function doctor(args) {
|
|
|
3118
3118
|
"For headless runs (`claude -p`) allow them in permissions.allow or --allowedTools, or set " +
|
|
3119
3119
|
"BIR_CORRELATION_DECISION=allow where the recorder starts");
|
|
3120
3120
|
}
|
|
3121
|
-
// Whether a
|
|
3122
|
-
//
|
|
3123
|
-
//
|
|
3124
|
-
//
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
|
|
3121
|
+
// Whether a segment the service hands out may run mid-task (segmented.md
|
|
3122
|
+
// R-LIFE-8). On by default from 0.3.19 (the service's segmentsbreakdown.md
|
|
3123
|
+
// SB-21); either state must be visible without reading a log, and an older
|
|
3124
|
+
// recorder still running here keeps the old default until it is replaced.
|
|
3125
|
+
// A segment starts only where the probe is waited on, so `intentArm: false`
|
|
3126
|
+
// (BIR_INTENT_ARM=0) starts nothing mid-task, segments included; a
|
|
3127
|
+
// recorder older than that field says nothing of it.
|
|
3128
|
+
notes.push(health.segmentArm === true && health.intentArm !== false
|
|
3129
|
+
? "segments run mid-task — recurring sub-tasks, and the halves after a plan's decision — and so may whole plans (the default; BIR_SEGMENT_ARM=0 turns this off)"
|
|
3130
|
+
: health.segmentArm === true
|
|
3131
|
+
? "segments are observed only — nothing starts mid-task (BIR_INTENT_ARM=0 where the recorder started)"
|
|
3132
|
+
: theirs && theirs !== VERSION
|
|
3133
|
+
? `segments are observed only — this recorder (${theirs}) is older than this bir, which runs them by default: \`bir up --restart\``
|
|
3134
|
+
: `segments are observed only — nothing is replaced (BIR_SEGMENT_ARM=0 where the recorder started)${health.intentArm === true ? "; whole plans may still start mid-task (BIR_INTENT_ARM=1)" : ""}`);
|
|
3128
3135
|
const sessions = health.sessions ?? [];
|
|
3129
3136
|
const probes = sessions.reduce((n, s) => n + (s.intent?.probes ?? 0), 0);
|
|
3130
3137
|
const armed = sessions.reduce((n, s) => n + (s.intent?.armed ?? 0), 0);
|
|
@@ -3714,6 +3721,12 @@ async function replayCommand(args) {
|
|
|
3714
3721
|
}
|
|
3715
3722
|
out(`response ${JSON.stringify(result.responseModel ?? {})}`);
|
|
3716
3723
|
out(`steps ${result.steps ?? 0}${result.partial ? " (partial — budget reached)" : ""}`);
|
|
3724
|
+
// A plan's decision (the service's segmentsbreakdown.md SB-6): the steps
|
|
3725
|
+
// from there are the half the recording took, so the run stops before them.
|
|
3726
|
+
if (result.stopped?.reason === "decision") {
|
|
3727
|
+
out(`stopped before step ${result.stopped.step}${result.stopped.tool ? ` (${result.stopped.tool})` : ""}: ` +
|
|
3728
|
+
"the plan's decision, which the agent makes — the steps from there did not run");
|
|
3729
|
+
}
|
|
3717
3730
|
return 0;
|
|
3718
3731
|
}
|
|
3719
3732
|
// ── wrap ───────────────────────────────────────────────────────────────────
|
|
@@ -245,6 +245,13 @@ interface ServiceInvestigation {
|
|
|
245
245
|
runId?: string | null;
|
|
246
246
|
} | null;
|
|
247
247
|
corrections?: ServiceCorrection[];
|
|
248
|
+
/**
|
|
249
|
+
* The step the plan stops in front of so that the agent decides, and the
|
|
250
|
+
* condition in words (the service's segmentsbreakdown.md SB-6). Null when
|
|
251
|
+
* the plan has none; absent from an older service.
|
|
252
|
+
*/
|
|
253
|
+
decisionStep?: number | null;
|
|
254
|
+
decisionText?: string | null;
|
|
248
255
|
} | null;
|
|
249
256
|
executions?: Array<{
|
|
250
257
|
id: string;
|
|
@@ -265,6 +272,8 @@ interface ServiceInvestigation {
|
|
|
265
272
|
status: string;
|
|
266
273
|
error?: string;
|
|
267
274
|
}> | null;
|
|
275
|
+
/** Where a `fell_back` run handed over, and why (`decision`, …): null for any other run. */
|
|
276
|
+
fallbackKind?: string | null;
|
|
268
277
|
/** The repeat the agent redid the task in, and its own calls there (AC1). */
|
|
269
278
|
redoneByRunId?: string | null;
|
|
270
279
|
redoneOwnCalls?: number | null;
|
package/dist/bin/investigate.js
CHANGED
|
@@ -203,8 +203,8 @@ export function findTurn(turns, id) {
|
|
|
203
203
|
/** The runner's own gate vocabulary, with the cause and the fix (guide §9). */
|
|
204
204
|
export const DECLINES = {
|
|
205
205
|
replay_disabled: {
|
|
206
|
-
cause: "calculated replay was off in this bir-hooks (BIR_REPLAY=0), or a sub-task was handed out
|
|
207
|
-
fix: "start bir-hooks with BIR_REPLAY=1 (the default); for sub-tasks
|
|
206
|
+
cause: "calculated replay was off in this bir-hooks (BIR_REPLAY=0), or a sub-task was handed out to a recorder that only observes sub-tasks (BIR_SEGMENT_ARM=0, or a recorder older than 0.3.19).",
|
|
207
|
+
fix: "start bir-hooks with BIR_REPLAY=1 (the default); for sub-tasks leave BIR_SEGMENT_ARM unset (on from 0.3.19), then `bir up --restart`.",
|
|
208
208
|
},
|
|
209
209
|
not_ready: {
|
|
210
210
|
cause: "the matched recording had no ready scenario at that moment.",
|
|
@@ -246,6 +246,11 @@ export const DECLINES = {
|
|
|
246
246
|
cause: "the plan was not ready before the hook that would have announced it had to answer: the prompt hook waits 7.5 s in all, match included, and a tool call that starts a plan mid-task waits 1.5 s. What arming waits for is the reading of the prompt's values (the derivation). A plan that was not announced is never run.",
|
|
247
247
|
fix: "run the prompt again. If it keeps happening the reading is slow: the recorder's log (`bir status` names the file) shows when its `replay.derived` line landed, and a dry replay shows what the service's reading took, under `reading` (`bir scenario replay <scnId> --prompt \"…\" --dry --json`). BIR_HOOK_DERIVE_BUDGET_MS does not help here, because the hook's own limit was reached first. To read values on this machine instead of asking the service, set BIR_DERIVE_API_KEY where Claude Code starts.",
|
|
248
248
|
},
|
|
249
|
+
// Nor this one: the control server's own rule (the service's offer1.md O8).
|
|
250
|
+
would_repeat: {
|
|
251
|
+
cause: "a plan the intent check matched would have started after the agent's call, not in its place, and its steps call that same tool: it would have made the call a second time — an e-mail sent twice (issue fx2). Never twice: it was not started, and the agent went on.",
|
|
252
|
+
fix: "nothing to fix in the plan. A scenario starts in the call's place only when the agent's call is its first step, value for value; one recorded in the order the agent works in (the agent's own run after a decision becomes one) is what starts there.",
|
|
253
|
+
},
|
|
249
254
|
// Nor is this one: the match itself was late, so nothing was armed at all.
|
|
250
255
|
match_late: {
|
|
251
256
|
cause: "the service's answer to the match landed too late: after the prompt hook had stopped waiting for it (3 s, BIR_MATCH_BUDGET_MS); while prompts were not waiting on the match at all — after one slow answer they stop waiting for a few minutes (`replay.match_gate state=holding`); or with the hook not waiting from the start, because a failure to reach the service was still on record (a `service.degraded` line, perhaps hours before, and no `service.recovered` until after this prompt) or the match request's first attempt failed. The agent did the task itself.",
|
|
@@ -434,9 +439,11 @@ export function localFindings(turn) {
|
|
|
434
439
|
}
|
|
435
440
|
for (const i of turn.incidents) {
|
|
436
441
|
const fields = Object.fromEntries(Object.entries(i).filter(([k]) => k !== "at" && k !== "event"));
|
|
442
|
+
// A stop at the plan's decision is the plan working as split: no fault.
|
|
443
|
+
const decisionStop = i.event === "replay.handover" && i.kind === "decision";
|
|
437
444
|
out.push({
|
|
438
445
|
code: i.event.replace(/\./g, "_"),
|
|
439
|
-
severity: i.event.endsWith("_failed") || i.event === "replay.failed" ? "problem" : "warn",
|
|
446
|
+
severity: i.event.endsWith("_failed") || i.event === "replay.failed" ? "problem" : decisionStop ? "info" : "warn",
|
|
440
447
|
title: `${i.event}${i.error ? `: ${String(i.error)}` : ""}`,
|
|
441
448
|
cause: Object.entries(fields)
|
|
442
449
|
.map(([k, v]) => `${k}=${String(v)}`)
|
|
@@ -447,13 +454,15 @@ export function localFindings(turn) {
|
|
|
447
454
|
: "the turn's cost could not be read from the transcript, or the runner has no price for the model: update it. Its report is unmeasured; run bir-hooks where the Claude Code transcript is readable."
|
|
448
455
|
: i.event === "replay.handover" && i.kind === "other_branch"
|
|
449
456
|
? "this request was the other half of the task the plan holds (a step's test said so), and the agent did it; that run is how the service learns the other half."
|
|
450
|
-
:
|
|
451
|
-
? "the
|
|
452
|
-
: i.event === "
|
|
453
|
-
? "the
|
|
454
|
-
: i.event === "run.
|
|
455
|
-
? "the
|
|
456
|
-
:
|
|
457
|
+
: decisionStop
|
|
458
|
+
? "nothing to fix: the plan stopped at its decision, as planned. It ran the steps before it, and the agent chose the way. A segment of the half it chose may have run next (a `replay.intent_armed` line with kind segment); a `run.fragment` line with decision=true is the agent's own work, from which the service makes the other half."
|
|
459
|
+
: i.event === "replay.diverge" || i.event === "replay.handover"
|
|
460
|
+
? "the model left the script or the runner handed the turn back; the service section shows what the fallback cost."
|
|
461
|
+
: i.event === "run.incomplete"
|
|
462
|
+
? "the recording is not the whole run, so it is never calculated and no longer catches its prompt; run the task again to record it whole."
|
|
463
|
+
: i.event === "run.host_failed"
|
|
464
|
+
? "the application's run failed; its recording is closed with that error and never becomes a plan."
|
|
465
|
+
: "see the guide's troubleshooting table for this line.",
|
|
457
466
|
});
|
|
458
467
|
}
|
|
459
468
|
const r = turn.reported;
|
|
@@ -536,6 +545,12 @@ export function savedPct(turn) {
|
|
|
536
545
|
return baseline > 0 ? r.savedUsd / baseline : null;
|
|
537
546
|
}
|
|
538
547
|
const short = (id) => (id ? id.slice(0, 13) : "—");
|
|
548
|
+
/**
|
|
549
|
+
* An execution's outcome, and where it handed the task over when the service
|
|
550
|
+
* says (`fell_back · decision`). The service sends `fallbackKind: null` for a
|
|
551
|
+
* run that did not hand over, and an older one sends nothing.
|
|
552
|
+
*/
|
|
553
|
+
const outcomeOf = (e) => typeof e.fallbackKind === "string" && e.fallbackKind !== "" ? `${e.outcome} · ${e.fallbackKind}` : e.outcome;
|
|
539
554
|
/** 4624 → "4,624", whatever the machine's locale. */
|
|
540
555
|
const count = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ",");
|
|
541
556
|
const money = (v) => (v == null ? "—" : usd(v));
|
|
@@ -638,6 +653,10 @@ export function renderService(out, s, viewerIsOwner) {
|
|
|
638
653
|
if (sc.held) {
|
|
639
654
|
out(` HELD since ${when(sc.held.at ?? undefined)}${sc.held.reason ? ` (${sc.held.reason})` : ""}: not handed out — the agent does the task until the plan is corrected or let go${sc.held.runId ? `; the redo is ${sc.held.runId}` : ""}`);
|
|
640
655
|
}
|
|
656
|
+
if (typeof sc.decisionStep === "number") {
|
|
657
|
+
const text = typeof sc.decisionText === "string" ? sc.decisionText.replace(/\s+/g, " ").trim() : "";
|
|
658
|
+
out(` decision at step ${sc.decisionStep}: the plan stops in front of it, and the agent decides${text ? ` (${text.length > 160 ? `${text.slice(0, 160)}…` : text})` : ""}`);
|
|
659
|
+
}
|
|
641
660
|
for (const line of stepLines(sc.steps))
|
|
642
661
|
out(` ${line}`);
|
|
643
662
|
const correction = newestCorrection(sc.corrections);
|
|
@@ -655,7 +674,7 @@ export function renderService(out, s, viewerIsOwner) {
|
|
|
655
674
|
if (ex.length > 0) {
|
|
656
675
|
out(` executions (${ex.length}, newest first)`);
|
|
657
676
|
for (const e of ex.slice(0, 10)) {
|
|
658
|
-
out(` ${when(e.createdAt)} ${short(e.id)} ${e.
|
|
677
|
+
out(` ${when(e.createdAt)} ${short(e.id)} ${outcomeOf(e).padEnd(13)} cost ${usd(e.costUsd)} saved ${money(e.savedUsd)}${e.savedPct != null ? ` (${pct(e.savedPct)})` : ""}${e.measured ? "" : " unmeasured"}`);
|
|
659
678
|
const failed = e.steps?.filter((st) => st.status === "failed") ?? [];
|
|
660
679
|
for (const st of failed)
|
|
661
680
|
out(` step ${st.stepIndex} ${st.toolName ?? ""} failed${st.error ? `: ${st.error.slice(0, 120)}` : ""}`);
|
|
@@ -790,7 +809,7 @@ export async function investigateCommand(args, deps) {
|
|
|
790
809
|
if (rows.length === 0)
|
|
791
810
|
out(" none yet");
|
|
792
811
|
for (const e of rows) {
|
|
793
|
-
out(` ${when(e.createdAt)} ${e.id} ${e.
|
|
812
|
+
out(` ${when(e.createdAt)} ${e.id} ${outcomeOf(e).padEnd(13)} cost ${usd(e.costUsd)} saved ${money(e.savedUsd)}${e.savedPct != null ? ` (${pct(e.savedPct)})` : ""}${e.measured ? "" : " unmeasured"}`);
|
|
794
813
|
out(` scenario ${e.scenarioId}${e.runId ? ` run ${e.runId}` : ""}${e.intent ? ` "${e.intent.slice(0, 60)}"` : ""}`);
|
|
795
814
|
}
|
|
796
815
|
out();
|
|
@@ -1231,6 +1231,16 @@ function explain(reply, args, deps, e) {
|
|
|
1231
1231
|
// newer entry puts the step back as this one left it, and then this one
|
|
1232
1232
|
// can be undone. It used to loop, when undo only toggled (C23).
|
|
1233
1233
|
const latest = typeof body.latestEditId === "string" ? body.latestEditId : "<the newer edit>";
|
|
1234
|
+
if (latest.startsWith("ssplit_")) {
|
|
1235
|
+
// The plan was split at its decision since (the service's
|
|
1236
|
+
// segmentsbreakdown.md). No `--edit` takes a split back: its Undo is
|
|
1237
|
+
// on the recording page's Split card, and only then can this go.
|
|
1238
|
+
const what = aboutCorrection ? "correction" : "edit";
|
|
1239
|
+
out(`${e.verb} (not_latest_edit): the plan was split at its decision after that ${what} (${latest}). ` +
|
|
1240
|
+
`It cannot be taken back while the split is in use: undo the split first (Undo on the Split card of the plan's recording page), then this ${what}:`);
|
|
1241
|
+
out(` bir scenario undo ${target} --edit ${e.editId ?? "<this edit>"}`);
|
|
1242
|
+
return 1;
|
|
1243
|
+
}
|
|
1234
1244
|
out(aboutCorrection
|
|
1235
1245
|
? `${e.verb} (not_latest_edit): the plan has changed since that correction — ${latest} changed it after. Undo the newer one first, then this one:`
|
|
1236
1246
|
: `${e.verb} (not_latest_edit): the ${aboutParam ? "parameter" : "step"} has changed since that edit — ${latest} changed it after. Undo the newer one first, then this one:`);
|
|
@@ -52,6 +52,21 @@ export declare function derivationKey(daemon?: boolean): {
|
|
|
52
52
|
apiKey?: string;
|
|
53
53
|
source?: string;
|
|
54
54
|
};
|
|
55
|
+
/**
|
|
56
|
+
* `BIR_SEGMENT_ARM`: whether a segment the service hands out may run mid-task.
|
|
57
|
+
* On unless it is `0`, from runner 0.3.19 (the service's segmentsbreakdown.md
|
|
58
|
+
* SB-21): segments found by repeated work, and the halves after a plan's
|
|
59
|
+
* decision. `0` is observe-only: what would have armed is logged, and nothing
|
|
60
|
+
* is replaced. While it is on, whole plans may start mid-task by intent too:
|
|
61
|
+
* `BIR_INTENT_ARM` follows it unless that is set.
|
|
62
|
+
*/
|
|
63
|
+
export declare function segmentArmFromEnv(env?: NodeJS.ProcessEnv): boolean;
|
|
64
|
+
/**
|
|
65
|
+
* What the `replay.enabled` startup line says segments do. A segment starts
|
|
66
|
+
* only where the probe is waited on, so `BIR_INTENT_ARM=0` starts nothing
|
|
67
|
+
* mid-task, segments included — the rule `bir doctor` words from /health.
|
|
68
|
+
*/
|
|
69
|
+
export declare function segmentsAtStartup(segmentArm: boolean, env?: NodeJS.ProcessEnv): string;
|
|
55
70
|
/** Where the two replay switches came from, for `/health` and `bir doctor`. */
|
|
56
71
|
export type ReplaySource = "env" | "sidecar" | "default";
|
|
57
72
|
/** What a recorder still serves (K4, `ControlServer.activity()`). */
|
|
@@ -152,6 +152,27 @@ export function derivationKey(daemon = process.env[DAEMON_ENV] === "1") {
|
|
|
152
152
|
}
|
|
153
153
|
return {};
|
|
154
154
|
}
|
|
155
|
+
/**
|
|
156
|
+
* `BIR_SEGMENT_ARM`: whether a segment the service hands out may run mid-task.
|
|
157
|
+
* On unless it is `0`, from runner 0.3.19 (the service's segmentsbreakdown.md
|
|
158
|
+
* SB-21): segments found by repeated work, and the halves after a plan's
|
|
159
|
+
* decision. `0` is observe-only: what would have armed is logged, and nothing
|
|
160
|
+
* is replaced. While it is on, whole plans may start mid-task by intent too:
|
|
161
|
+
* `BIR_INTENT_ARM` follows it unless that is set.
|
|
162
|
+
*/
|
|
163
|
+
export function segmentArmFromEnv(env = process.env) {
|
|
164
|
+
return env.BIR_SEGMENT_ARM !== "0";
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* What the `replay.enabled` startup line says segments do. A segment starts
|
|
168
|
+
* only where the probe is waited on, so `BIR_INTENT_ARM=0` starts nothing
|
|
169
|
+
* mid-task, segments included — the rule `bir doctor` words from /health.
|
|
170
|
+
*/
|
|
171
|
+
export function segmentsAtStartup(segmentArm, env = process.env) {
|
|
172
|
+
if (!segmentArm)
|
|
173
|
+
return "observed only (BIR_SEGMENT_ARM=0)";
|
|
174
|
+
return env.BIR_INTENT_ARM === "0" ? "observed only (BIR_INTENT_ARM=0)" : "run mid-task";
|
|
175
|
+
}
|
|
155
176
|
/**
|
|
156
177
|
* Assemble the replay configuration (docs/calculatedReplay.md §5.3 of the guide).
|
|
157
178
|
*
|
|
@@ -204,19 +225,21 @@ function buildReplayOptions(keeper, policy, key) {
|
|
|
204
225
|
intentMatch: {
|
|
205
226
|
enabled: process.env.BIR_INTENT_MATCH !== "0",
|
|
206
227
|
// The probe's own ceiling, not a wait on the hook's path: `PreToolUse`
|
|
207
|
-
//
|
|
208
|
-
//
|
|
228
|
+
// waits for it only while arming is on — the default, unless
|
|
229
|
+
// BIR_SEGMENT_ARM=0 or BIR_INTENT_ARM=0 — and then at most
|
|
209
230
|
// BIR_INTENT_ARM_BUDGET_MS (500 ms), never while the service is failing
|
|
210
|
-
// (control/server.ts)
|
|
211
|
-
//
|
|
212
|
-
// 2 500 ms
|
|
231
|
+
// (control/server.ts); otherwise it answers without it (hit counting
|
|
232
|
+
// only). 4 000 leaves a request whose best segment lands in the
|
|
233
|
+
// not-clear band its one live question under the service's own 2 500 ms
|
|
234
|
+
// attempt (segmented.md R-HIT-11).
|
|
213
235
|
budgetMs: positiveInt(process.env.BIR_INTENT_MATCH_BUDGET_MS, 4_000),
|
|
214
236
|
maxPerTurn: positiveInt(process.env.BIR_INTENT_MATCHES_PER_TURN, 3),
|
|
215
237
|
maxRequestsPerTurn: positiveInt(process.env.BIR_INTENT_REQUESTS_PER_TURN, 40),
|
|
216
238
|
},
|
|
217
|
-
//
|
|
218
|
-
//
|
|
219
|
-
|
|
239
|
+
// On by default from 0.3.19 (the service's segmentsbreakdown.md SB-21): a
|
|
240
|
+
// plan that stops at its decision hands the rest to segments of each half,
|
|
241
|
+
// and they run only mid-task. BIR_SEGMENT_ARM=0 brings back observe-only.
|
|
242
|
+
segmentArm: segmentArmFromEnv(),
|
|
220
243
|
// Both read live, never captured (C5). The recorder signs in after it
|
|
221
244
|
// listens, refreshes the token as the session outlives it, and may sign
|
|
222
245
|
// in or out while it runs; replay reads whatever holds *now*. The old
|
|
@@ -247,6 +270,8 @@ function buildReplayOptions(keeper, policy, key) {
|
|
|
247
270
|
? "the service"
|
|
248
271
|
: "recorded sample values — sign in, or a scenario with a target will not run"),
|
|
249
272
|
intentMatch: opts.intentMatch?.enabled ? "on" : "off",
|
|
273
|
+
// Whether a segment the service hands out runs, or is only logged.
|
|
274
|
+
segments: segmentsAtStartup(opts.segmentArm === true),
|
|
250
275
|
source,
|
|
251
276
|
why: "a prompt that matches a calculated scenario runs it: the scenario's own steps run without a " +
|
|
252
277
|
"permission prompt of their own (never in plan mode); every other tool call is asked about as " +
|
package/dist/control/server.d.ts
CHANGED
|
@@ -276,12 +276,13 @@ export interface ControlServerOptions {
|
|
|
276
276
|
* (fallbk.md §Runner 4).
|
|
277
277
|
*
|
|
278
278
|
* Arming there means *waiting* there, on every unsteered tool call, for a
|
|
279
|
-
* service round trip
|
|
280
|
-
*
|
|
281
|
-
*
|
|
282
|
-
*
|
|
283
|
-
*
|
|
284
|
-
*
|
|
279
|
+
* service round trip. It waits (at most `BIR_INTENT_ARM_BUDGET_MS`, 500 ms,
|
|
280
|
+
* and never while the service is failing) when segment arming is on — the
|
|
281
|
+
* recorder's default from 0.3.19, `BIR_SEGMENT_ARM` unset or `1` — or when
|
|
282
|
+
* `BIR_INTENT_ARM=1` asks for whole-scenario intent arming. `BIR_INTENT_ARM=0`
|
|
283
|
+
* never waits. Otherwise, as in a server built with neither (a test, an
|
|
284
|
+
* embedder), the probe is sent without waiting, for hit counting only, and
|
|
285
|
+
* nothing arms mid-turn.
|
|
285
286
|
*/
|
|
286
287
|
intentArm?: boolean;
|
|
287
288
|
/**
|
|
@@ -1154,14 +1155,19 @@ export declare class ControlServer {
|
|
|
1154
1155
|
*/
|
|
1155
1156
|
private excludeRunsFor;
|
|
1156
1157
|
/**
|
|
1157
|
-
* Per
|
|
1158
|
-
* (segmented.md R-OUT-6).
|
|
1158
|
+
* Per plan of this turn, its recording and how far the plan got in its own
|
|
1159
|
+
* chain (segmented.md R-OUT-6).
|
|
1159
1160
|
*
|
|
1160
1161
|
* The server uses it to refuse a segment that starts *before* where this turn
|
|
1161
1162
|
* already is: re-running steps the turn has just done is worse than not
|
|
1162
|
-
* replaying at all. The
|
|
1163
|
-
*
|
|
1164
|
-
*
|
|
1163
|
+
* replaying at all. The position is the plan's own: the step it handed over
|
|
1164
|
+
* at (its decision included), or its last step when it ran to its end. The
|
|
1165
|
+
* server maps it onto the recording — a segment's first position, a
|
|
1166
|
+
* decision's step before it — and takes the largest per recording (the
|
|
1167
|
+
* service's segmentsbreakdown.md D18). One entry per plan, so that two plans
|
|
1168
|
+
* of one recording, whose own positions are on different scales, never
|
|
1169
|
+
* hide each other. A plan declined for anything but a known-bad first step
|
|
1170
|
+
* reports nothing — it ran none of the recording, so it blocks none of it.
|
|
1165
1171
|
*
|
|
1166
1172
|
* Rebuilt on every probe rather than cached: a plan that hands over between
|
|
1167
1173
|
* two calls changes the answer.
|
|
@@ -1196,6 +1202,70 @@ export declare class ControlServer {
|
|
|
1196
1202
|
* computed is "no" — the directive then rides on the call's PostToolUse.
|
|
1197
1203
|
*/
|
|
1198
1204
|
private isFirstStep;
|
|
1205
|
+
/**
|
|
1206
|
+
* The agent's call against `state`'s first step, value for value (the
|
|
1207
|
+
* service's offer1.md O5): `hit` when every argument is the step's own,
|
|
1208
|
+
* `missing` when the only differences are values the plan does not have
|
|
1209
|
+
* (a target the reading left empty, which the agent's call may hold), and
|
|
1210
|
+
* `other` otherwise.
|
|
1211
|
+
*/
|
|
1212
|
+
private firstStepVerdict;
|
|
1213
|
+
/**
|
|
1214
|
+
* Whether `state`'s plan calls `toolName` in a step it would run: what makes
|
|
1215
|
+
* a plan started after a call of that tool make it a second time (the
|
|
1216
|
+
* service's offer1.md O8). A tool that only reads never counts — reading
|
|
1217
|
+
* twice changes nothing — by the service's own rule ({@link readsOnly}): a
|
|
1218
|
+
* `crm_search_customers` mid-task never keeps a plan that searches later from
|
|
1219
|
+
* starting.
|
|
1220
|
+
*/
|
|
1221
|
+
private planCallsTool;
|
|
1222
|
+
/**
|
|
1223
|
+
* A plan of this turn stopped at its decision: ask the service, once per
|
|
1224
|
+
* plan, which calculated scenarios can follow it, and arm each in the
|
|
1225
|
+
* background while the agent reads the note and decides (offer1.md O3, O4).
|
|
1226
|
+
* Called wherever a hand-over is first seen; a second call for the same plan
|
|
1227
|
+
* does nothing.
|
|
1228
|
+
*/
|
|
1229
|
+
private noteDecision;
|
|
1230
|
+
/**
|
|
1231
|
+
* Ask for a decision's next scenarios (offer1.md O3): each is checkable as
|
|
1232
|
+
* soon as the service answers, and armed in the background, each on its own
|
|
1233
|
+
* (O4). Resolves when the service has answered, not when the arms have.
|
|
1234
|
+
* Never rejects.
|
|
1235
|
+
*/
|
|
1236
|
+
private fetchNext;
|
|
1237
|
+
/**
|
|
1238
|
+
* Arm one next scenario in the background (offer1.md O4): its values read
|
|
1239
|
+
* from the turn's request and its results so far — or, once more, with the
|
|
1240
|
+
* agent's own call (O5) — and settled, so a call is compared with the first
|
|
1241
|
+
* step as it would really run. Never rejects.
|
|
1242
|
+
*/
|
|
1243
|
+
private armNext;
|
|
1244
|
+
/**
|
|
1245
|
+
* The agent's call after a decision, against the scenarios that can follow
|
|
1246
|
+
* it (offer1.md O5, O6). A hit — the call is a scenario's first step, value
|
|
1247
|
+
* for value — is answered in the call's place with that scenario's directive:
|
|
1248
|
+
* the agent calls `run_scenario`, and the scenario runs every step, this one
|
|
1249
|
+
* included, once. Undefined otherwise: the call runs as the agent wrote it.
|
|
1250
|
+
*/
|
|
1251
|
+
private tryNextScenarios;
|
|
1252
|
+
/** A next scenario's hit: bookkeeping as for any plan armed mid-task, then its directive in the call's place. */
|
|
1253
|
+
private nextHit;
|
|
1254
|
+
/**
|
|
1255
|
+
* A next scenario hit after the agent's own calls — looks — opened the
|
|
1256
|
+
* fragment of its work after the decision: the fragment ends here, with what
|
|
1257
|
+
* the agent did itself, and nothing more of this turn is recorded into it.
|
|
1258
|
+
* The scenario's own calls, pinned in steer mode or run in direct mode, are
|
|
1259
|
+
* not the agent's work, and would make the fragment a copy of that scenario
|
|
1260
|
+
* (offer1.md O7). A fragment still due is dropped instead
|
|
1261
|
+
* ({@link dropDecisionFragment}).
|
|
1262
|
+
*/
|
|
1263
|
+
private closeDecisionFragment;
|
|
1264
|
+
/**
|
|
1265
|
+
* End this turn's next scenarios (offer1.md O6): every one but the hit is
|
|
1266
|
+
* never announced, so never runs and is never reported.
|
|
1267
|
+
*/
|
|
1268
|
+
private endNext;
|
|
1199
1269
|
/**
|
|
1200
1270
|
* Whether an intent probe may arm a plan in its own `PreToolUse` — and so
|
|
1201
1271
|
* whether that hook waits for it. See {@link ControlServerOptions.intentArm}.
|
|
@@ -1212,6 +1282,16 @@ export declare class ControlServer {
|
|
|
1212
1282
|
* whose intent-armed plan handed over is recording the model's work anyway.
|
|
1213
1283
|
*/
|
|
1214
1284
|
private noteHandover;
|
|
1285
|
+
/**
|
|
1286
|
+
* Drop a fragment scheduled at a decision (the service's
|
|
1287
|
+
* segmentsbreakdown.md SB-16) when a plan the intent check armed takes the
|
|
1288
|
+
* agent's next call: what follows is that plan's half, not the other one,
|
|
1289
|
+
* and recorded as the other half it would become a wrong segment C. Cleared
|
|
1290
|
+
* rather than ended, so a later hand-over of that plan schedules its own
|
|
1291
|
+
* fragment, as any plan's does in a turn with none. Any other fragment is
|
|
1292
|
+
* left as it is.
|
|
1293
|
+
*/
|
|
1294
|
+
private dropDecisionFragment;
|
|
1215
1295
|
/**
|
|
1216
1296
|
* Open the fragment run and record from this call on (fallbk.md D7).
|
|
1217
1297
|
*
|