@basein/runner 0.2.12 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/README.md +540 -369
  2. package/dist/auth/client.d.ts +65 -3
  3. package/dist/auth/client.js +305 -53
  4. package/dist/bin/bir-hooks.d.ts +72 -31
  5. package/dist/bin/bir-hooks.js +176 -284
  6. package/dist/bin/bir-proxy.d.ts +17 -2
  7. package/dist/bin/bir-proxy.js +222 -28
  8. package/dist/bin/bir-scenario.d.ts +61 -21
  9. package/dist/bin/bir-scenario.js +593 -122
  10. package/dist/bin/bir.d.ts +12 -6
  11. package/dist/bin/bir.js +2935 -489
  12. package/dist/bin/investigate.js +40 -15
  13. package/dist/bin/scenario-edit.d.ts +7 -1
  14. package/dist/bin/scenario-edit.js +43 -21
  15. package/dist/bin/scenario-liveness.d.ts +194 -0
  16. package/dist/bin/scenario-liveness.js +459 -0
  17. package/dist/bin/scenario-stderr.d.ts +43 -0
  18. package/dist/bin/scenario-stderr.js +76 -0
  19. package/dist/bin/service-read.d.ts +88 -0
  20. package/dist/bin/service-read.js +128 -0
  21. package/dist/bin/setup.d.ts +19 -0
  22. package/dist/bin/setup.js +146 -20
  23. package/dist/config/adapters/claude-code.d.ts +92 -14
  24. package/dist/config/adapters/claude-code.js +291 -101
  25. package/dist/config/adapters/generic.d.ts +4 -1
  26. package/dist/config/adapters/generic.js +13 -14
  27. package/dist/config/generate.d.ts +392 -14
  28. package/dist/config/generate.js +666 -19
  29. package/dist/config/mcp-approval.d.ts +76 -0
  30. package/dist/config/mcp-approval.js +210 -0
  31. package/dist/config/node-path.d.ts +42 -0
  32. package/dist/config/node-path.js +102 -0
  33. package/dist/config/overrides.d.ts +62 -0
  34. package/dist/config/overrides.js +136 -0
  35. package/dist/config/resolve.d.ts +70 -8
  36. package/dist/config/resolve.js +186 -15
  37. package/dist/control/client.d.ts +22 -0
  38. package/dist/control/client.js +26 -7
  39. package/dist/control/correlation.d.ts +23 -5
  40. package/dist/control/correlation.js +0 -0
  41. package/dist/control/daemon.d.ts +400 -18
  42. package/dist/control/daemon.js +1105 -102
  43. package/dist/control/discovery.d.ts +77 -5
  44. package/dist/control/discovery.js +186 -11
  45. package/dist/control/ensure-hook.d.ts +61 -0
  46. package/dist/control/ensure-hook.js +160 -8
  47. package/dist/control/paths.d.ts +26 -0
  48. package/dist/control/paths.js +34 -0
  49. package/dist/control/pin-guard.d.ts +83 -0
  50. package/dist/control/pin-guard.js +534 -0
  51. package/dist/control/recorder-keeper.d.ts +229 -0
  52. package/dist/control/recorder-keeper.js +721 -0
  53. package/dist/control/recorder-main.d.ts +73 -0
  54. package/dist/control/recorder-main.js +778 -0
  55. package/dist/control/server.d.ts +707 -31
  56. package/dist/control/server.js +2787 -302
  57. package/dist/control/transcript.d.ts +21 -1
  58. package/dist/control/transcript.js +378 -112
  59. package/dist/index.d.ts +5 -1
  60. package/dist/index.js +5 -1
  61. package/dist/jsonrpc/framing.d.ts +76 -12
  62. package/dist/jsonrpc/framing.js +216 -59
  63. package/dist/proxy/intercept.d.ts +49 -5
  64. package/dist/proxy/intercept.js +188 -59
  65. package/dist/proxy/relay.d.ts +16 -0
  66. package/dist/proxy/relay.js +77 -5
  67. package/dist/proxy/revive.d.ts +122 -0
  68. package/dist/proxy/revive.js +251 -0
  69. package/dist/proxy/session.d.ts +222 -14
  70. package/dist/proxy/session.js +708 -99
  71. package/dist/record/recorder.d.ts +51 -3
  72. package/dist/record/recorder.js +3 -0
  73. package/dist/record/redact.d.ts +103 -9
  74. package/dist/record/redact.js +659 -30
  75. package/dist/record/remote-recorder.d.ts +327 -32
  76. package/dist/record/remote-recorder.js +1517 -187
  77. package/dist/record/switchable-recorder.d.ts +126 -0
  78. package/dist/record/switchable-recorder.js +351 -0
  79. package/dist/record/truncate.d.ts +40 -1
  80. package/dist/record/truncate.js +111 -4
  81. package/dist/replay/bash-channel.d.ts +35 -0
  82. package/dist/replay/bash-channel.js +133 -0
  83. package/dist/replay/bundle.d.ts +10 -0
  84. package/dist/replay/bundle.js +10 -0
  85. package/dist/replay/controller.d.ts +369 -20
  86. package/dist/replay/controller.js +1176 -107
  87. package/dist/replay/derive.d.ts +1 -0
  88. package/dist/replay/derive.js +81 -24
  89. package/dist/replay/executor.d.ts +51 -16
  90. package/dist/replay/executor.js +114 -41
  91. package/dist/replay/handover.d.ts +54 -1
  92. package/dist/replay/handover.js +55 -5
  93. package/dist/replay/logic.d.ts +62 -10
  94. package/dist/replay/logic.js +140 -22
  95. package/dist/replay/plan.d.ts +201 -27
  96. package/dist/replay/plan.js +581 -141
  97. package/dist/replay/safety.d.ts +72 -0
  98. package/dist/replay/safety.js +303 -0
  99. package/dist/replay/source-run.d.ts +10 -0
  100. package/dist/replay/source-run.js +80 -35
  101. package/dist/replay/targets.d.ts +15 -1
  102. package/dist/replay/targets.js +62 -4
  103. package/dist/replay/tool-error.d.ts +15 -1
  104. package/dist/replay/tool-error.js +33 -1
  105. package/dist/replay/types.d.ts +7 -4
  106. package/dist/upstream/client.d.ts +56 -3
  107. package/dist/upstream/client.js +61 -4
  108. package/dist/upstream/headers.d.ts +43 -0
  109. package/dist/upstream/headers.js +90 -0
  110. package/dist/upstream/http-client.d.ts +213 -22
  111. package/dist/upstream/http-client.js +1034 -168
  112. package/dist/upstream/http-transport.d.ts +143 -0
  113. package/dist/upstream/http-transport.js +391 -0
  114. package/dist/upstream/lazy-client.d.ts +8 -0
  115. package/dist/upstream/lazy-client.js +29 -1
  116. package/dist/upstream/sse.d.ts +78 -0
  117. package/dist/upstream/sse.js +189 -0
  118. package/dist/upstream/stdio-client.d.ts +61 -1
  119. package/dist/upstream/stdio-client.js +356 -33
  120. package/dist/util/atomic-write.d.ts +29 -0
  121. package/dist/util/atomic-write.js +96 -0
  122. package/dist/util/journal.d.ts +15 -0
  123. package/dist/util/journal.js +48 -3
  124. package/dist/util/log.d.ts +10 -0
  125. package/dist/util/log.js +28 -1
  126. package/dist/util/off-switch.d.ts +46 -0
  127. package/dist/util/off-switch.js +91 -0
  128. package/dist/util/service-health.d.ts +133 -0
  129. package/dist/util/service-health.js +217 -0
  130. package/docs/BaseInstRunner.md +694 -621
  131. package/docs/calculatedReplay.md +1625 -1304
  132. package/docs/calculatedReplayGuide.md +1212 -1087
  133. package/docs/installRun.md +1013 -772
  134. package/docs/loginWeb.md +627 -607
  135. package/docs/mcpmark.md +49 -16
  136. package/docs/my-first-sample.md +569 -545
  137. package/docs/quickstart.md +384 -311
  138. package/docs/t-bench.md +406 -394
  139. package/package.json +58 -57
  140. package/production.md +1371 -0
  141. package/scripts/install.ps1 +97 -6
  142. package/scripts/install.sh +104 -7
@@ -16,34 +16,155 @@ import { isHousekeepingCall } from "../record/housekeeping.js";
16
16
  import { redact } from "../record/redact.js";
17
17
  import { serializeCapped } from "../record/truncate.js";
18
18
  import { logDetail, logLine, errText } from "../util/log.js";
19
- import { MAX_REPLAY_REASON } from "./bundle.js";
19
+ import { serviceHealth, serviceTimeoutMs, timeoutSignal, } from "../util/service-health.js";
20
+ import { bashCommandFor } from "./bash-channel.js";
21
+ import { BASH_CHANNEL_MAX, MAX_REPLAY_REASON, truncate } from "./bundle.js";
20
22
  import { coverageOf, modeFor, reachOf } from "./coverage.js";
21
23
  import { deriveParameters, } from "./derive.js";
22
- import { buildHandoverNote } from "./handover.js";
23
- import { ProxyWorkQueue } from "./executor.js";
24
- import { ScenarioReplayPlan } from "./plan.js";
24
+ import { buildHandoverNote, MAX_HANDOVER_NOTE, } from "./handover.js";
25
+ import { NO_PROXY, NoSessionProxyError, ProxyWorkQueue, QUEUE_CLOSED } from "./executor.js";
26
+ import { logicTimeoutMs, MAX_LOGIC_TIMEOUT_MS } from "./logic.js";
27
+ import { PREVIEW_BUDGET_MS, ScenarioReplayPlan, StepRunError, } from "./plan.js";
25
28
  import { PRICING_VERSION } from "./pricing.js";
29
+ import { hasShellStep, sinkOf, sinkOfPlan, unsafeParameter } from "./safety.js";
26
30
  import { SourceRunOutputs } from "./source-run.js";
27
31
  import { clampToFrameStart, flattenChain, } from "./flatten.js";
28
32
  import { blockingTargets } from "./targets.js";
29
- import { toolResultError } from "./tool-error.js";
33
+ import { isNoResult, toolResultError } from "./tool-error.js";
30
34
  import { OUTCOME_RANK, hasTargets, isReadyScenario, } from "./types.js";
31
35
  /** The first-party tool a `direct` plan is delivered through (§6.3). */
32
36
  export const DIRECT_TOOL_NAME = "mcp__bir__run_scenario";
33
37
  /** How long a `/proxy/poll` is held open before it answers empty. */
34
38
  export const POLL_HOLD_MS = 25_000;
39
+ /**
40
+ * The longest a prompt ever waits for a steer plan's parameters so that its
41
+ * directive can preview them ({@link ReplayController.settleForPreview}).
42
+ * `UserPromptSubmit` is on the user's critical path: a preview is worth a
43
+ * little of it, never the derivation's whole budget.
44
+ */
45
+ export const PREVIEW_SETTLE_MAX_MS = 1_200;
46
+ /**
47
+ * Whether the pin guard holds a call to this tool to the step's exact shape —
48
+ * the same command head, the same file (control/pin-guard.ts) — so that a call
49
+ * written without the directive's preview is declined rather than pinned.
50
+ */
51
+ function pinnedToShape(toolName) {
52
+ const name = toolName ?? "";
53
+ return !parseQualifiedName(name) && sinkOf(name) !== undefined;
54
+ }
35
55
  export const DEFAULT_BUDGETS = {
36
- matchMs: 2_500,
56
+ // The longest a prompt waits for the service to say whether this is a known
57
+ // task; a match that has not answered by then is a turn run normally. The
58
+ // control server stops waiting at all for a while after one that answered
59
+ // slowly (`BIR_MATCH_SLOW_MS`). `BIR_MATCH_BUDGET_MS` in the recorder.
60
+ matchMs: 3_000,
37
61
  deriveMs: 8_000,
38
- planMs: 120_000,
62
+ planMs: 100_000,
39
63
  stepMs: 60_000,
64
+ hookDeriveMs: 4_000,
65
+ preToolMs: 9_000,
40
66
  };
67
+ /**
68
+ * The least time a dispatched step is given, however little the run has left.
69
+ * A step sent with 50 ms to answer is a step that may or may not have run.
70
+ */
71
+ const MIN_STEP_MS = 1_000;
72
+ /**
73
+ * The control server answers a `PreToolUse` by this, whatever replay is doing
74
+ * (its `HOOK_DEADLINE_MS` for `/tool/pre`). A deployment's `hookDeadlinesMs`
75
+ * can only shorten it; the server passes the one it keeps as `answerBy`
76
+ * ({@link PreToolOptions}).
77
+ */
78
+ const PRETOOL_ROUTE_DEADLINE_MS = 12_000;
79
+ /** Logic, the bundle and the answer's own trip: work past the last step's start. */
80
+ const PRETOOL_MARGIN_MS = 1_000;
81
+ /**
82
+ * The most {@link ReplayBudgets.preToolMs} may be: the last step started at the
83
+ * budget's end still gets its {@link MIN_STEP_MS} and the margin before the
84
+ * route's deadline answers the host. Past it, a divergence kept starting steps
85
+ * — writes, unattended — after the model had been told to carry on itself.
86
+ */
87
+ export const MAX_PRETOOL_REPLAY_MS = PRETOOL_ROUTE_DEADLINE_MS - MIN_STEP_MS - PRETOOL_MARGIN_MS;
88
+ /**
89
+ * The longest a direct run may take by its budgets, end to end.
90
+ *
91
+ * `bir-scenario` gives `/scenario/run` 290 s (`BIR_SCENARIO_RUN_TIMEOUT_MS`),
92
+ * just under Claude Code's 5-minute idle timeout for an MCP call, and a run
93
+ * that answers after its client stopped waiting is a replay thrown away with
94
+ * its steps already done. So the budgets are held to 270 s, and the defaults
95
+ * sit far below it: 8 s of derivation + 100 s of plan (more only when an
96
+ * operator sets `BIR_REPLAY_BUDGET_MS` or `BIR_REPLAY_PLAN_BUDGET_MS`) + 12 s
97
+ * of slack (one last step's 1 s, six 1 s logic bodies, a 5 s margin —
98
+ * {@link clampDirectRun}) = 120 s.
99
+ *
100
+ * Only `/scenario/run` is held to it. `bir replay` (`runAdHoc`) answers the
101
+ * operator's own command, which no MCP client times out, and keeps the budgets
102
+ * the environment asked for.
103
+ */
104
+ export const DIRECT_RUN_CEILING_MS = 270_000;
105
+ /** Transport, the bundle and redaction: work a direct run does outside every budget. */
106
+ const DIRECT_RUN_MARGIN_MS = 5_000;
107
+ /**
108
+ * Logic bodies a direct run can still evaluate once its plan deadline has
109
+ * passed: `paramsLogic` before the plan (after the derivation's own wait), and
110
+ * for the step started just before the deadline its parameter mapping and
111
+ * input logic, its output logic and result mapping, then `responseParamsLogic`.
112
+ * Each is bounded by `logicTimeoutMs()` (logic.ts).
113
+ */
114
+ const LOGIC_BODIES_OUTSIDE_PLAN = 6;
115
+ /** A derivation wait longer than this is not a budget a direct run can keep. */
116
+ const MAX_DIRECT_DERIVE_MS = 60_000;
117
+ /** However a direct run is squeezed, its plan keeps at least this. */
118
+ const MIN_DIRECT_PLAN_MS = 10_000;
119
+ /**
120
+ * The direct-run budgets, held under {@link DIRECT_RUN_CEILING_MS}.
121
+ *
122
+ * The worst case is `deriveMs + planMs` plus what can happen past the plan's
123
+ * deadline: one last step's {@link MIN_STEP_MS}, {@link LOGIC_BODIES_OUTSIDE_PLAN}
124
+ * logic bodies, and the margin. `stepMs` never adds to it — each step is
125
+ * clipped to what the plan has left. Budgets that would pass the ceiling are
126
+ * brought under it — the derivation wait first to {@link MAX_DIRECT_DERIVE_MS},
127
+ * then the plan — and `clamped` says what changed, for the log.
128
+ */
129
+ export function clampDirectRun(budgets, logicMs = logicTimeoutMs()) {
130
+ const outside = MIN_STEP_MS + LOGIC_BODIES_OUTSIDE_PLAN * logicMs + DIRECT_RUN_MARGIN_MS;
131
+ if (budgets.deriveMs + budgets.planMs + outside <= DIRECT_RUN_CEILING_MS)
132
+ return { budgets };
133
+ const deriveMs = Math.min(budgets.deriveMs, MAX_DIRECT_DERIVE_MS);
134
+ const planMs = Math.max(MIN_DIRECT_PLAN_MS, Math.min(budgets.planMs, DIRECT_RUN_CEILING_MS - deriveMs - outside));
135
+ const clamped = {};
136
+ if (deriveMs !== budgets.deriveMs)
137
+ clamped.deriveMs = [budgets.deriveMs, deriveMs];
138
+ if (planMs !== budgets.planMs)
139
+ clamped.planMs = [budgets.planMs, planMs];
140
+ return { budgets: { ...budgets, deriveMs, planMs }, clamped };
141
+ }
142
+ /** The `why` of a {@link RunArmedResult} with nothing armed to run. */
143
+ export const RUN_NO_PLAN = "no_plan";
144
+ /** The hand-over shape of a {@link RunArmedResult}, or undefined when there is no note to hand over. */
145
+ function handedOver(note) {
146
+ return note ? { ok: false, why: note, note: true } : undefined;
147
+ }
41
148
  export const DEFAULT_INTENT_MATCH = {
42
149
  enabled: true,
43
150
  budgetMs: 4_000,
44
151
  maxPerTurn: 3,
45
152
  maxRequestsPerTurn: 40,
46
153
  };
154
+ /**
155
+ * How long a call waits for the previous pin to thread before it is judged to
156
+ * be running beside it: the control server's hook-view fallback
157
+ * (`PROXY_REPORT_GRACE_MS`, 1.5 s after `PostToolUse`) plus slack. Bounded by
158
+ * the hook's own deadline as well.
159
+ */
160
+ const PIN_THREAD_WAIT_MS = 1_750;
161
+ /**
162
+ * How long a call waits for a pin the host has *not* reported finished, once
163
+ * the control server is known to report completions: only the time for a
164
+ * `PostToolUse` racing this `PreToolUse` to land. Anything longer is a call
165
+ * running beside it, and waiting on that could stall the turn for minutes.
166
+ */
167
+ const HOST_RACE_MS = 200;
47
168
  /** How many step verdicts one report carries. A chain longer than this is not
48
169
  * a chain anybody is reading step by step, and the report has to stay small. */
49
170
  const MAX_STEP_RESULTS = 200;
@@ -52,14 +173,78 @@ const MAX_STEP_ERROR_CHARS = 500;
52
173
  export class ReplayController {
53
174
  enabled;
54
175
  work = new ProxyWorkQueue();
176
+ /** The budgets every path runs on, a direct run's held under {@link DIRECT_RUN_CEILING_MS}. */
55
177
  budgets;
178
+ /**
179
+ * `bir replay`'s derivation wait and plan budget: what the environment asked
180
+ * for, unclamped — no MCP client times that run out (see
181
+ * {@link DIRECT_RUN_CEILING_MS}).
182
+ */
183
+ adHocBudgets;
56
184
  intentMatch;
57
185
  opts;
186
+ /** See {@link ReplayOptions.recordedFallback}. */
187
+ recordedFallback;
188
+ /** Calls waiting for a plan's pins to change (see `awaitPins`). */
189
+ pinWaiters = new WeakMap();
190
+ /**
191
+ * Set the first time the control server reports a host completion
192
+ * ({@link hostCompleted}). From then on a pin the host has not finished is a
193
+ * call really running beside the next one, and nothing waits for it.
194
+ */
195
+ hostSignals = false;
58
196
  constructor(opts) {
59
197
  this.opts = opts;
60
198
  this.enabled = opts.enabled;
61
- this.budgets = { ...DEFAULT_BUDGETS, ...(opts.budgets ?? {}) };
199
+ const asked = {
200
+ ...DEFAULT_BUDGETS,
201
+ planMs: positiveEnv("BIR_REPLAY_PLAN_BUDGET_MS") ?? DEFAULT_BUDGETS.planMs,
202
+ hookDeriveMs: positiveEnv("BIR_HOOK_DERIVE_BUDGET_MS") ?? DEFAULT_BUDGETS.hookDeriveMs,
203
+ preToolMs: positiveEnv("BIR_PRETOOL_REPLAY_BUDGET_MS") ?? DEFAULT_BUDGETS.preToolMs,
204
+ ...(opts.budgets ?? {}),
205
+ };
206
+ // A PreToolUse's replay work past the route's deadline is steps started
207
+ // after the host was told to go ahead without them: held under it.
208
+ if (Number.isNaN(asked.preToolMs))
209
+ asked.preToolMs = DEFAULT_BUDGETS.preToolMs;
210
+ if (asked.preToolMs > MAX_PRETOOL_REPLAY_MS) {
211
+ const from = asked.preToolMs;
212
+ asked.preToolMs = MAX_PRETOOL_REPLAY_MS;
213
+ logLine("replay.budget_clamped", {
214
+ budget: "BIR_PRETOOL_REPLAY_BUDGET_MS",
215
+ from,
216
+ to: MAX_PRETOOL_REPLAY_MS,
217
+ why: `PreToolUse is answered by ${PRETOOL_ROUTE_DEADLINE_MS} ms; a divergence step started later ` +
218
+ "would run after the model was told to carry on itself",
219
+ });
220
+ }
221
+ // A budget set past what a direct run can keep is brought back under it,
222
+ // and said so once: the alternative is a run its client has already
223
+ // abandoned, with its steps done and its answer thrown away.
224
+ const logicAsked = positiveEnv("BIR_LOGIC_TIMEOUT_MS");
225
+ if (logicAsked !== undefined && logicAsked > MAX_LOGIC_TIMEOUT_MS) {
226
+ logLine("replay.budget_clamped", {
227
+ budget: "BIR_LOGIC_TIMEOUT_MS",
228
+ from: logicAsked,
229
+ to: MAX_LOGIC_TIMEOUT_MS,
230
+ why: "a logic body holds up every hook of the project while it runs",
231
+ });
232
+ }
233
+ const direct = clampDirectRun(asked);
234
+ if (direct.clamped) {
235
+ logLine("replay.budget_clamped", {
236
+ deriveMs: direct.clamped.deriveMs?.join("→"),
237
+ planMs: direct.clamped.planMs?.join("→"),
238
+ ceilingMs: DIRECT_RUN_CEILING_MS,
239
+ why: "a direct run must answer before run_scenario's client gives up on it (290 s); " +
240
+ "`bir replay` keeps the budget asked for",
241
+ });
242
+ }
243
+ this.budgets = direct.budgets;
244
+ this.adHocBudgets = { deriveMs: asked.deriveMs, planMs: asked.planMs };
62
245
  this.intentMatch = { ...DEFAULT_INTENT_MATCH, ...(opts.intentMatch ?? {}) };
246
+ this.recordedFallback =
247
+ opts.recordedFallback ?? process.env.BIR_REPLAY_RECORDED_OUTPUTS === "1";
63
248
  }
64
249
  /**
65
250
  * Whether a handed-out segment may actually arm (segmented.md R-OUT-10).
@@ -83,6 +268,13 @@ export class ReplayController {
83
268
  * covers both "no match" and "too slow", which are the same thing from here.
84
269
  */
85
270
  async awaitMatch(pending) {
271
+ // The match is the service's answer, so this is a service wait on the hot
272
+ // path (C1): none at all while the service is cooling down. An overrun is
273
+ // not reported from here — the recorder's own request reports its timeout
274
+ // or failure when it settles, and it is the one that knows which it was.
275
+ pending.catch(() => undefined);
276
+ if (!serviceHealth.canWait())
277
+ return null;
86
278
  let timer;
87
279
  const budget = new Promise((resolve) => {
88
280
  timer = setTimeout(() => resolve(null), this.budgets.matchMs);
@@ -235,7 +427,30 @@ export class ReplayController {
235
427
  const mode = modeFor(planned.map((e) => e.step), wrapped, this.opts.allowServers);
236
428
  if (mode === "none")
237
429
  return decline("no step is executable", "coverage");
238
- const derivation = this.startDerivation(scenario, prompt, state, ctx);
430
+ // Tool results feed derivation as context, and they are whatever the agent
431
+ // last read — a fetched page, an email. A plan that hands its parameters
432
+ // to a shell or a database must not take them from there: a value lifted
433
+ // from a page and templated into a command is the page running a command
434
+ // (safety.ts). Every `command` sink, not only a shell: `mcp__pg__query`
435
+ // runs its input unattended just the same. A tool merely *named* for a
436
+ // query (a search `*_query`) loses the results too, and a scenario whose
437
+ // target only a result names declines more often — one replay, never a
438
+ // command nobody asked for.
439
+ let deriveCtx = ctx;
440
+ if (ctx.recentResults?.length && sinkOfPlan(planned) === "command") {
441
+ deriveCtx = { liveCall: ctx.liveCall };
442
+ logLine("replay.derive_context", {
443
+ scenario: scenario.id,
444
+ shell: hasShellStep(planned) || undefined,
445
+ why: "a step's input is run by a shell or a database (or its tool is named for one, such as *_query) — " +
446
+ "recent tool results are not read for its parameters, so a target only they name is not found",
447
+ });
448
+ }
449
+ const derivation = this.startDerivation(scenario, prompt, state, deriveCtx);
450
+ // Whether the derivation has answered yet — read, never waited on, when the
451
+ // directive is written (below).
452
+ let deriveAnswered = false;
453
+ derivation.then(() => (deriveAnswered = true), () => (deriveAnswered = true));
239
454
  const params = derivation.then((r) => r.params);
240
455
  // `params` is awaited by the plan, and by gate 7 below when there is a
241
456
  // target. Without this, the losing race in gate 7 leaves `derivation`
@@ -254,22 +469,22 @@ export class ReplayController {
254
469
  if (hasTargets(scenario.paramsObject)) {
255
470
  // Of the targets this answer left unnamed, the ones this plan could act
256
471
  // on the recorded value of. Empty means the chain finds them for itself.
257
- const blocking = (missing) => blockingTargets(missing, scenario.paramsObject, planned);
258
- let result;
259
- try {
260
- result = await this.withBudget(derivation, this.budgets.deriveMs, "derivation");
261
- }
262
- catch (err) {
263
- // Out of budget, or the call threw. The turn has not said what this task
264
- // is to act on, so every target is unnamed (R-PARAM-3).
472
+ const blocking = (missing) => blockingTargets(missing, scenario.paramsObject, planned, scenario.paramsLogic);
473
+ const budget = Math.min(this.budgets.deriveMs, this.budgets.hookDeriveMs);
474
+ const result = await this.waitDerivation(state, derivation, budget);
475
+ if (result === undefined) {
476
+ // Out of budget, or the service is cooling down and was not waited on.
477
+ // The turn has not said what this task is to act on, so every target is
478
+ // unnamed (R-PARAM-3).
479
+ const error = `derivation did not answer within ${budget}ms`;
265
480
  const blocked = blocking(targetKeys(scenario.paramsObject));
266
481
  if (blocked.length > 0) {
267
- return decline(`target ${blocked[0]} not found — ${errText(err)}`, "missing_target");
482
+ return decline(`target ${blocked[0]} not found — ${error}`, "missing_target");
268
483
  }
269
484
  logLine("replay.targets_unread", {
270
485
  scenario: scenario.id,
271
486
  why: "the derivation did not answer, and this chain takes no target from the caller",
272
- error: errText(err),
487
+ error,
273
488
  });
274
489
  }
275
490
  if (result && !result.derived) {
@@ -301,6 +516,26 @@ export class ReplayController {
301
516
  why: "the chain computes these from its own steps; none is read from the caller",
302
517
  });
303
518
  }
519
+ // The values are in hand: one this plan must not template into its
520
+ // command (safety.ts) declines here, before any directive is injected,
521
+ // instead of arming a plan whose first call can only be denied with a
522
+ // hand-over — a denied call and a model round trip, for nothing. The
523
+ // same check still runs where the values settle later (`readyFor`).
524
+ // It travels as `missing_target`: the value read for a target could not
525
+ // be used, and no new code reaches a service that may not know it.
526
+ const sink = sinkOfPlan(planned);
527
+ if (result?.derived && sink) {
528
+ const bad = unsafeParameter(result.params, scenario.paramsObject, sink);
529
+ if (bad) {
530
+ logLine("replay.unsafe_param", {
531
+ scenario: scenario.id,
532
+ param: bad.key,
533
+ sink,
534
+ why: "a value read for this request would be templated into a command unattended — declined before arming",
535
+ });
536
+ return decline(`the value read for ${bad.key} is not safe to put into a command unattended`, "missing_target");
537
+ }
538
+ }
304
539
  }
305
540
  // A plan armed by intent runs inside a task the agent is already doing, so
306
541
  // its directive and its bundles say "continue", never "answer the user"
@@ -313,6 +548,21 @@ export class ReplayController {
313
548
  entries,
314
549
  subTask: armedBy === "intent",
315
550
  });
551
+ if (mode === "steer") {
552
+ // A steer directive shows the arguments the steps will run with, so the
553
+ // call the model writes is one the pin guard accepts (plan.ts,
554
+ // `steeringDirective`). That needs the parameters settled — which costs
555
+ // nothing when the derivation has already answered: gate 7 waited for
556
+ // it, or there was nothing to derive. One turn of the loop lets an answer
557
+ // that is already in hand through; one still on its way is not waited
558
+ // for here, and the directive simply leaves the arguments to the system,
559
+ // as it always did. A `paramsLogic` that throws is not judged here: the
560
+ // first call that needs the parameters meets the same failure, as before.
561
+ if (!deriveAnswered)
562
+ await new Promise((resolve) => setImmediate(resolve));
563
+ if (deriveAnswered)
564
+ await state.plan.ready().catch(() => undefined);
565
+ }
316
566
  state.mode = mode;
317
567
  state.stepsPlanned = entries.length;
318
568
  // Why the plan stops, when a call is what stops it: the note names the
@@ -321,7 +571,9 @@ export class ReplayController {
321
571
  typeof stopAt === "number" && flat.stopAt !== undefined && stopAt <= flat.stopAt
322
572
  ? flat.stopReason
323
573
  : undefined;
324
- if (this.opts.authUrl && this.opts.authToken) {
574
+ // Recorded outputs are read only when they may stand in (opt-in), so a
575
+ // plan never fetches a run it will not show.
576
+ if (this.recordedFallback && this.opts.authUrl && this.opts.authToken) {
325
577
  state.sourceRun = new SourceRunOutputs({
326
578
  baseUrl: this.opts.authUrl,
327
579
  runId: scenario.runId,
@@ -351,12 +603,75 @@ export class ReplayController {
351
603
  });
352
604
  return state;
353
605
  }
354
- /** The directive to inject via `additionalContext`, or undefined when declined. */
355
- directiveFor(state) {
606
+ /**
607
+ * The directive to inject via `additionalContext`, or undefined when declined.
608
+ *
609
+ * A steer directive previews each step's arguments (plan.ts,
610
+ * `previewArguments`), which can take up to {@link PREVIEW_BUDGET_MS}.
611
+ * `previewMs` — the time the caller's hook has left — lowers that bound,
612
+ * never raises it; under ~10 ms nothing is previewed and the directive keeps
613
+ * its old wording.
614
+ */
615
+ directiveFor(state, opts = {}) {
616
+ const asked = opts.previewMs;
356
617
  return state.plan?.steeringDirective(DIRECT_TOOL_NAME, {
357
618
  subTask: state.armedBy === "intent",
619
+ // NaN is a caller's arithmetic gone wrong: no time known, no preview.
620
+ previewBudgetMs: typeof asked === "number"
621
+ ? Math.max(0, Math.min(Number.isNaN(asked) ? 0 : asked, PREVIEW_BUDGET_MS))
622
+ : undefined,
358
623
  });
359
624
  }
625
+ /**
626
+ * Give a steer plan's parameters a short, bounded chance to settle before its
627
+ * directive is written, so {@link directiveFor} can show the arguments the
628
+ * steps run with. Resolves true when they are settled.
629
+ *
630
+ * Without it a derivation the service answers — nearly every one, since a
631
+ * signed-in runner has no key of its own — has never answered by the time
632
+ * `arm` returns (it waits one turn of the loop, on purpose), the directive
633
+ * says "the system supplies the exact arguments", the model writes a guessed
634
+ * command, the pin guard declines it, and the plan retires having run
635
+ * nothing.
636
+ *
637
+ * Only for the steps the pin guard holds to their exact shape — a shell
638
+ * command, a file write (control/pin-guard.ts) — and never longer than
639
+ * {@link PREVIEW_SETTLE_MAX_MS} or `budgetMs`, whichever is less: the caller
640
+ * passes what its prompt hook has left, less what the preview itself needs.
641
+ * Not at all while the service is cooling down. A late answer is not an
642
+ * overrun of anything, so it marks nothing against the service. `arm`
643
+ * itself still never waits (integ-replay R2).
644
+ */
645
+ async settleForPreview(state, budgetMs) {
646
+ const plan = state.plan;
647
+ if (!plan || state.retired || state.mode !== "steer")
648
+ return false;
649
+ if (plan.parametersSettled)
650
+ return true;
651
+ const planned = plan.allEntries().slice(0, plan.stepCount);
652
+ if (!planned.some((e) => pinnedToShape(e.step.toolName)))
653
+ return false;
654
+ if (state.deriveOnService && !serviceHealth.canWait())
655
+ return false;
656
+ const ms = Math.min(Number.isFinite(budgetMs) ? budgetMs : 0, PREVIEW_SETTLE_MAX_MS);
657
+ if (ms <= 0)
658
+ return false;
659
+ let timer;
660
+ try {
661
+ await Promise.race([
662
+ plan.ready().catch(() => undefined),
663
+ new Promise((resolve) => {
664
+ timer = setTimeout(resolve, ms);
665
+ timer.unref?.();
666
+ }),
667
+ ]);
668
+ }
669
+ finally {
670
+ if (timer)
671
+ clearTimeout(timer);
672
+ }
673
+ return plan.parametersSettled;
674
+ }
360
675
  /**
361
676
  * A hand-over note not yet delivered, claimed for delivery (fallbk.md D4).
362
677
  * Returns it once and marks it delivered; undefined when there is none.
@@ -379,9 +694,12 @@ export class ReplayController {
379
694
  */
380
695
  deliverInstead(text, toolName) {
381
696
  if (toolName === "Bash") {
382
- const delimiter = "BIR_EOF";
383
- const safe = text.split(delimiter).join("BIR_EOF_");
384
- return { kind: "bash", command: `cat <<'${delimiter}'\n${safe}\n${delimiter}` };
697
+ // A short command, whatever the text's size (bash-channel.ts): a long
698
+ // heredoc is cut by the shell, and a long output by the host.
699
+ const command = bashCommandFor(truncate(text, BASH_CHANNEL_MAX));
700
+ if (command)
701
+ return { kind: "bash", command };
702
+ // The text could not be staged: the denial reason still carries it.
385
703
  }
386
704
  return { kind: "deny", reason: text };
387
705
  }
@@ -391,31 +709,43 @@ export class ReplayController {
391
709
  * Three tiers (§8): pin the expected call; on divergence execute the remainder
392
710
  * and hand it back through the best channel available; if even that fails,
393
711
  * abort to an ordinary turn.
712
+ *
713
+ * `hook` is optional and only ever tightens the work: `answerBy` lowers the
714
+ * deadline to the caller's own, and `signal` stops a divergence from
715
+ * starting another step once the host has been answered without it.
394
716
  */
395
717
  async preTool(state, toolName, toolUseId,
396
718
  /** The call's `tool_input`: whether a `Bash` call is only `bir` is read from it (D13). */
397
- toolInput) {
719
+ toolInput, hook = {}) {
398
720
  const plan = state.plan;
399
721
  if (!plan || state.retired)
400
722
  return { kind: "passthrough" };
401
723
  // The delivery vehicle for a direct plan is not a step in it.
402
- if (this.isDirectTool(toolName))
724
+ if (this.isDirectTool(toolName)) {
725
+ state.directRequested = true;
403
726
  return { kind: "passthrough" };
404
- try {
405
- await this.withBudget(plan.ready(), this.budgets.deriveMs, "derivation");
406
727
  }
407
- catch (err) {
408
- logLine("replay.derive_failed", {
728
+ // A direct run is executing this plan right now: another call beside it is
729
+ // the model's own, and a divergence would drive the same cursor twice.
730
+ if (state.running)
731
+ return { kind: "passthrough" };
732
+ // The model asked for `run_scenario` and the run never started. The usual
733
+ // reason is the user refusing that call — and executing the plan anyway,
734
+ // through a divergence, would override that refusal with the very steps it
735
+ // refused. The model does the task; the plan stays idle.
736
+ if (state.mode === "direct" && state.directRequested) {
737
+ logDetail("replay.direct_not_run", {
409
738
  scenario: state.scenarioId ?? undefined,
410
- why: "continuing with the scenario's recorded sample values",
411
- error: errText(err),
739
+ tool: toolName,
740
+ why: "run_scenario was called but never ran (refused, or it never reached bir) — nothing is executed in its place",
412
741
  });
742
+ return { kind: "passthrough" };
413
743
  }
414
744
  // HOUSEKEEPING IS NOT DIVERGENCE. See `record/housekeeping.ts`. Asked
415
745
  // before the pin, not after it: a `Bash` step is pinned by name, and a
416
746
  // `bir scenario show` the model runs while a Bash step is expected must
417
- // run as `bir`, never be swapped for the plan's command (D13).
418
- const expected = plan.expectedTool();
747
+ // run as `bir`, never be swapped for the plan's command (D13). And before
748
+ // the wait for the parameters, which a `ToolSearch` has no use for.
419
749
  if (isHousekeepingCall(toolName, toolInput)) {
420
750
  logDetail("replay.housekeeping", {
421
751
  tool: toolName,
@@ -424,8 +754,68 @@ export class ReplayController {
424
754
  });
425
755
  return { kind: "passthrough" };
426
756
  }
757
+ // Everything below — the wait for the parameters, and a divergence that
758
+ // executes the rest of the plan — happens inside this one hook, which the
759
+ // host waits on for a bounded time. One deadline covers all of it — and
760
+ // never reaches past the caller's own answer (`hook.answerBy`), less the
761
+ // last step's minimum and a margin.
762
+ // The parameters may be waited on until the margin before that answer; a
763
+ // step may only *start* until its minimum before it as well.
764
+ const answerBy = typeof hook.answerBy === "number" && Number.isFinite(hook.answerBy)
765
+ ? hook.answerBy - PRETOOL_MARGIN_MS
766
+ : Number.POSITIVE_INFINITY;
767
+ const deadline = Math.min(Date.now() + this.budgets.preToolMs, answerBy - MIN_STEP_MS);
768
+ const ready = await this.readyFor(state, plan, Math.max(0, Math.min(this.budgets.deriveMs, this.budgets.hookDeriveMs, answerBy - Date.now())));
769
+ // Another call may have retired the plan while this one waited — or the
770
+ // host may have been answered without this call's action.
771
+ if (state.retired || hook.signal?.aborted)
772
+ return { kind: "passthrough" };
773
+ if (!ready.ok) {
774
+ // Nothing may run on parameters that are unknown, unprocessed or unsafe.
775
+ // The model does the task itself — told so in place of this call.
776
+ const step = plan.currentStep();
777
+ if (step && ready.blame) {
778
+ this.recordStep(state, { step, outcome: "failed", stage: ready.stage, error: ready.error });
779
+ }
780
+ this.handOver(state, {
781
+ kind: "step_failed",
782
+ step,
783
+ entry: plan.currentEntry(),
784
+ position: plan.currentStepIndex,
785
+ error: ready.error,
786
+ delivery: "above",
787
+ stopReason: "not_run",
788
+ });
789
+ const note = this.takeNote(state);
790
+ return note ? this.deliverInstead(note, toolName) : { kind: "abort" };
791
+ }
792
+ // ONE CALL AT A TIME. A pin moves the cursor only when its output threads,
793
+ // which for a wrapped step is the proxy's report — landing either side of
794
+ // `PostToolUse`, or up to the control server's 1.5 s fallback after it
795
+ // when no report comes (a standalone proxy). A call arriving before that
796
+ // would be pinned to the very same step (it would run twice,
797
+ // auto-approved), or would diverge and re-execute the step still in
798
+ // flight. So it first waits, briefly and within this hook's deadline, for
799
+ // the pin to thread: the model's *next* call after a finished one is the
800
+ // ordinary sequential case, and must be judged against the moved cursor —
801
+ // never handed over as "concurrent". Only a pin still out after that is a
802
+ // call really running beside this one, and then the plan hands over, and
803
+ // this call carries the note that says so.
804
+ if (state.pinned.size > 0 && !state.pinned.has(toolUseId)) {
805
+ // With the plan's last step in flight there is nothing left to pin twice:
806
+ // this call is the model's own whatever that step does, so it never waits.
807
+ const lastInFlight = [...state.pinned.values()].some((p) => p.stepIndex + 1 >= plan.stepCount);
808
+ if (!lastInFlight)
809
+ await this.awaitPins(state, toolUseId, deadline);
810
+ if (state.retired || hook.signal?.aborted)
811
+ return { kind: "passthrough" };
812
+ if (state.pinned.size > 0 && !state.pinned.has(toolUseId)) {
813
+ return this.concurrentCall(state, plan, toolName);
814
+ }
815
+ }
427
816
  // A direct plan does not steer individual calls: the model was asked for one
428
817
  // tool call and made a different one. That is divergence.
818
+ const expected = plan.expectedTool();
429
819
  if (state.mode === "steer" && toolName === expected && toolUseId) {
430
820
  try {
431
821
  const step = plan.currentStep();
@@ -474,7 +864,140 @@ export class ReplayController {
474
864
  return { kind: "abort" };
475
865
  }
476
866
  }
477
- return await this.diverge(state, toolName, `expected ${expected ?? "no more tools"}, model called ${toolName}`);
867
+ return await this.diverge(state, toolName, `expected ${expected ?? "no more tools"}, model called ${toolName}`, deadline, hook.signal);
868
+ }
869
+ /**
870
+ * A call arrived while another pinned call's output has not threaded yet
871
+ * (see `preTool`). The plan stops here rather than pin a step twice.
872
+ */
873
+ concurrentCall(state, plan, toolName) {
874
+ const outstanding = [...state.pinned.values()][0];
875
+ const next = (outstanding?.stepIndex ?? plan.currentStepIndex) + 1;
876
+ if (next >= plan.stepCount) {
877
+ // The call in flight is the plan's last step: nothing is left to pin
878
+ // twice, and this call is the model's own.
879
+ logDetail("replay.concurrent_call", {
880
+ scenario: state.scenarioId ?? undefined,
881
+ tool: toolName,
882
+ why: "the last step is in flight — this call is the model's own",
883
+ });
884
+ return { kind: "passthrough" };
885
+ }
886
+ logLine("replay.concurrent_call", {
887
+ scenario: state.scenarioId ?? undefined,
888
+ tool: toolName,
889
+ inFlight: outstanding ? `${outstanding.stepIndex}:${outstanding.toolName}` : undefined,
890
+ why: "a second call arrived while the pinned step was still running — a plan runs one call at a time, so the rest is handed over",
891
+ });
892
+ // No outcome of its own: the model, not the plan, does the rest, so this
893
+ // is no divergence saving. `handOver` books `fell_back` if earlier steps
894
+ // threaded and `failed` if none did — and the pins still in flight stay
895
+ // readable, so a step among them that finishes is still counted.
896
+ //
897
+ // The pin in flight is *not* counted as run. It may be running, or it may
898
+ // never run at all — the user's own deny rule or PreToolUse hook blocked
899
+ // it, and no PostToolUse will ever come. The note says exactly that,
900
+ // rather than "Steps 1–N ran … do not repeat them" about a call that may
901
+ // not have happened.
902
+ const inFlight = new Map(state.pinned);
903
+ this.handOver(state, {
904
+ kind: "step_failed",
905
+ step: plan.allSteps()[next],
906
+ entry: plan.allEntries()[next],
907
+ position: next,
908
+ delivery: "above",
909
+ stopReason: "concurrent",
910
+ ran: outstanding ? outstanding.stepIndex : plan.currentStepIndex,
911
+ inFlight: outstanding
912
+ ? { stepIndex: outstanding.stepIndex, toolName: outstanding.toolName }
913
+ : undefined,
914
+ });
915
+ if (inFlight.size > 0)
916
+ state.lingering = inFlight;
917
+ const note = this.takeNote(state);
918
+ return note ? this.deliverInstead(note, toolName) : { kind: "passthrough" };
919
+ }
920
+ /**
921
+ * Wait until the plan's outstanding pins thread, or until waiting is no
922
+ * longer worth it (see `preTool`, ONE CALL AT A TIME).
923
+ *
924
+ * Always bounded by the hook's `deadline`. Within it:
925
+ * - every pin finished by the host ({@link hostCompleted}): until
926
+ * {@link PIN_THREAD_WAIT_MS} after the last of them finished — the
927
+ * server's fallback has threaded it by then, if anything will;
928
+ * - the server is known to report host completions and a pin is not
929
+ * finished: a call really running beside this one, which may run for
930
+ * minutes, so only {@link HOST_RACE_MS} in case its completion is racing
931
+ * this hook;
932
+ * - otherwise (a server that reports nothing): {@link PIN_THREAD_WAIT_MS}.
933
+ */
934
+ async awaitPins(state, toolUseId, deadline) {
935
+ const startedAt = Date.now();
936
+ const outstanding = () => [...state.pinned.entries()].filter(([id]) => id !== toolUseId).map(([, pin]) => pin);
937
+ for (;;) {
938
+ if (state.retired)
939
+ return;
940
+ const pins = outstanding();
941
+ if (pins.length === 0)
942
+ return;
943
+ const hostDone = pins.map((p) => p.hostDoneAt);
944
+ const finished = hostDone.every((t) => t !== undefined);
945
+ const until = Math.min(deadline, finished
946
+ ? Math.max(...hostDone) + PIN_THREAD_WAIT_MS
947
+ : startedAt + (this.hostSignals ? HOST_RACE_MS : PIN_THREAD_WAIT_MS));
948
+ const left = until - Date.now();
949
+ if (left <= 0) {
950
+ logDetail("replay.pin_wait", {
951
+ scenario: state.scenarioId ?? undefined,
952
+ waitedMs: Date.now() - startedAt,
953
+ why: "the previous pinned step has not threaded — judged a call beside it",
954
+ });
955
+ return;
956
+ }
957
+ await this.pinChange(state, left);
958
+ }
959
+ }
960
+ /** Resolves on the next change to a plan's pins, or after `ms`. */
961
+ pinChange(state, ms) {
962
+ return new Promise((resolve) => {
963
+ let waiters = this.pinWaiters.get(state);
964
+ if (!waiters) {
965
+ waiters = new Set();
966
+ this.pinWaiters.set(state, waiters);
967
+ }
968
+ const set = waiters;
969
+ const done = () => {
970
+ clearTimeout(timer);
971
+ set.delete(done);
972
+ resolve();
973
+ };
974
+ const timer = setTimeout(done, Math.max(0, ms));
975
+ timer.unref?.();
976
+ set.add(done);
977
+ });
978
+ }
979
+ /** Wake every call waiting on this plan's pins. */
980
+ notifyPins(state) {
981
+ const waiters = this.pinWaiters.get(state);
982
+ if (!waiters || waiters.size === 0)
983
+ return;
984
+ for (const wake of [...waiters])
985
+ wake();
986
+ }
987
+ /**
988
+ * The host finished a call (`PostToolUse` / `PostToolUseFailure`) — whether
989
+ * or not its output has threaded yet. The control server calls this for
990
+ * every pinned call; it is what tells the model's next, sequential call
991
+ * apart from one running beside a pin (see `awaitPins`). Optional: without
992
+ * it every such call waits the bounded {@link PIN_THREAD_WAIT_MS}.
993
+ */
994
+ hostCompleted(state, toolUseId) {
995
+ const pin = state?.pinned.get(toolUseId);
996
+ this.hostSignals = true;
997
+ if (!state || !pin || pin.hostDoneAt !== undefined)
998
+ return;
999
+ pin.hostDoneAt = Date.now();
1000
+ this.notifyPins(state);
478
1001
  }
479
1002
  /**
480
1003
  * `PostToolUse` for a call this plan pinned.
@@ -483,17 +1006,72 @@ export class ReplayController {
483
1006
  * wrapped MCP step that is the proxy's whole `CallToolResult`, which is why the
484
1007
  * server threads those from `/proxy/step` rather than from the hook (§7.2).
485
1008
  * Returns true when the plan is now complete.
1009
+ *
1010
+ * `failed` is the host's verdict that the call failed — `PostToolUseFailure`,
1011
+ * or a proxy report with `isError` — with its error text when there is one.
1012
+ * It outranks anything read from `output`: a built-in's response is Claude
1013
+ * Code's rendering and is never judged here, so for an `Edit` whose
1014
+ * `old_string` moved since the recording this is the only word there is. A
1015
+ * step that failed must never thread as if it worked: the next step would be
1016
+ * pinned, and auto-approved, on top of it.
1017
+ *
1018
+ * Not every failure is one. A call its proxy *abandoned* — the upstream
1019
+ * closed, restarted or was cancelled mid-call, reported with `isError` and
1020
+ * no result — may well have done its work, and so may one the host says
1021
+ * timed out or was interrupted: the note says "may or may not have
1022
+ * completed", never "failed", or the model runs a write a second time
1023
+ * ({@link pinFailure}). `how` lets the caller say so outright.
486
1024
  */
487
- postTool(state, toolUseId, output) {
1025
+ postTool(state, toolUseId, output, failed, how = {}) {
488
1026
  const plan = state.plan;
1027
+ if (plan && state.retired) {
1028
+ this.lateThread(state, plan, toolUseId, output, failed, how);
1029
+ return false;
1030
+ }
489
1031
  const pin = state.pinned.get(toolUseId);
490
1032
  if (!plan || !pin || state.retired)
491
1033
  return false;
492
1034
  state.pinned.delete(toolUseId);
1035
+ // A call waiting for this pin (`awaitPins`) re-reads the cursor once this
1036
+ // synchronous method has finished moving it.
1037
+ this.notifyPins(state);
493
1038
  // The pin's own step, not `currentStep()`: `applyOutput` moves the cursor, so
494
1039
  // reading it after the call would judge the *next* step, and reading it
495
1040
  // before assumes no other pin ever moved it.
496
1041
  const step = plan.allSteps()[pin.stepIndex];
1042
+ // The output threads into the step at the cursor. One pin at a time
1043
+ // (`preTool`) keeps that the pin's own step; if it ever is not, threading
1044
+ // would feed one step's output to another's logic, so the plan stops.
1045
+ //
1046
+ // A proxy-threaded call that reports *no result* was sent and abandoned —
1047
+ // its upstream died mid-call (tool-error.ts, `isNoResult`). It may or may
1048
+ // not have done its work, and nothing may be pinned on top of it.
1049
+ const failure = pinFailure(pin, output, failed, how);
1050
+ const hostError = failure?.error;
1051
+ if (step && (hostError !== undefined || plan.currentStepIndex !== pin.stepIndex)) {
1052
+ const error = hostError ??
1053
+ `the plan's cursor is at step ${plan.currentStepIndex}, not the pinned step ${pin.stepIndex}`;
1054
+ const info = {
1055
+ step,
1056
+ outcome: "failed",
1057
+ stage: "tool_call",
1058
+ error,
1059
+ ms: Date.now() - pin.pinnedAt,
1060
+ };
1061
+ this.logStep(info);
1062
+ if (hostError !== undefined)
1063
+ this.recordStep(state, info);
1064
+ this.handOver(state, {
1065
+ kind: "step_failed",
1066
+ step,
1067
+ entry: plan.allEntries()[pin.stepIndex],
1068
+ position: pin.stepIndex,
1069
+ error,
1070
+ delivery: "above",
1071
+ stopReason: failure?.reason,
1072
+ });
1073
+ return false;
1074
+ }
497
1075
  let derivedKeys = [];
498
1076
  try {
499
1077
  derivedKeys = plan.applyOutput(output);
@@ -604,38 +1182,132 @@ export class ReplayController {
604
1182
  }
605
1183
  return done;
606
1184
  }
1185
+ /**
1186
+ * The result of a call pinned under a plan that has since handed over
1187
+ * because another call ran beside it (`concurrentCall`). Nothing threads —
1188
+ * the plan is retired and the model is doing the rest — but the step's
1189
+ * verdict is recorded, and a step that did its work is counted: the outcome
1190
+ * moves from `failed` to `fell_back` when it is the first that did.
1191
+ */
1192
+ lateThread(state, plan, toolUseId, output, failed, how = {}) {
1193
+ const pin = state.lingering?.get(toolUseId);
1194
+ if (!pin)
1195
+ return;
1196
+ state.lingering.delete(toolUseId);
1197
+ const step = plan.allSteps()[pin.stepIndex];
1198
+ if (!step)
1199
+ return;
1200
+ const ms = Date.now() - pin.pinnedAt;
1201
+ // Read the same way as a live pin's (`pinFailure`). A call that was sent
1202
+ // and not seen through says so in its verdict: it may have done its work.
1203
+ const failure = pinFailure(pin, output, failed, how);
1204
+ const error = failure
1205
+ ? failure.reason === "uncertain"
1206
+ ? `${failure.error} — it was sent, and may or may not have completed`
1207
+ : failure.error
1208
+ : toolResultError(output);
1209
+ if (error !== undefined) {
1210
+ const info = { step, outcome: "failed", stage: "tool_call", error, ms };
1211
+ this.logStep(info);
1212
+ this.recordStep(state, info);
1213
+ return;
1214
+ }
1215
+ this.recordStep(state, { step, outcome: "executed", ms });
1216
+ state.stepsPinned += 1;
1217
+ this.upgrade(state, "fell_back");
1218
+ logDetail("replay.thread_late", {
1219
+ scenario: state.scenarioId ?? undefined,
1220
+ n: pin.stepIndex,
1221
+ why: "a step pinned before the hand-over finished — counted, not threaded",
1222
+ });
1223
+ }
607
1224
  /** Whether a pinned call's output is threaded from the proxy's report (§7.2). */
608
1225
  threadsFromProxy(state, toolUseId) {
609
- return state.pinned.get(toolUseId)?.reach === "direct";
1226
+ const pin = state.pinned.get(toolUseId) ?? state.lingering?.get(toolUseId);
1227
+ return pin?.reach === "direct";
610
1228
  }
611
- /** True when this call belongs to the active plan. */
1229
+ /**
1230
+ * True when this call belongs to the plan: pinned by it and not yet
1231
+ * threaded — including a pin still in flight when the plan handed over, so
1232
+ * its result still reaches `postTool` (see `lateThread`).
1233
+ */
612
1234
  isPinned(state, toolUseId) {
613
- return state.pinned.has(toolUseId);
1235
+ return state.pinned.has(toolUseId) || state.lingering?.has(toolUseId) === true;
614
1236
  }
615
1237
  /**
616
1238
  * `POST /scenario/run` — the whole of a `direct` plan, executed here.
617
1239
  *
618
1240
  * Every step runs through the proxy that already owns its upstream, so the
619
1241
  * model spends nothing beyond the turn that reads the results.
1242
+ *
1243
+ * `signal` aborts when the caller is gone — `bir-scenario`'s request was
1244
+ * cancelled (Esc) or timed out — so no further step is started (C4). The
1245
+ * worst case this can take is `deriveMs + planMs` plus one step's
1246
+ * `MIN_STEP_MS`, six logic bodies at their full limit and a 5 s margin —
1247
+ * 120 s by default, more only for a plan budget an operator raised
1248
+ * ({@link clampDirectRun}) — which `bir-scenario`'s own timeout (290 s) must
1249
+ * exceed, and which is held under {@link DIRECT_RUN_CEILING_MS} whatever the
1250
+ * environment asks for. That bound counts every body at its limit; a run
1251
+ * that does not stall ends well inside its plan budget.
1252
+ *
1253
+ * One run per plan: a second call while the first is running gets the first
1254
+ * one's answer, never a second execution of the same steps.
620
1255
  */
621
- async runArmed(state) {
1256
+ async runArmed(state, signal) {
622
1257
  const plan = state.plan;
623
1258
  if (!plan || state.retired)
624
- return { ok: false, why: "no_plan" };
625
- try {
626
- await this.withBudget(plan.ready(), this.budgets.deriveMs, "derivation");
627
- }
628
- catch (err) {
629
- logLine("replay.derive_failed", {
1259
+ return { ok: false, why: RUN_NO_PLAN };
1260
+ if (state.running)
1261
+ return state.running;
1262
+ const running = this.runArmedOnce(state, plan, signal).catch((err) => {
1263
+ // Never a throw at a hook: a broken run is the model's task again.
1264
+ logLine("replay.failed", {
630
1265
  scenario: state.scenarioId ?? undefined,
631
- why: "continuing with the scenario's recorded sample values",
1266
+ why: "the run threw — continuing as a normal run",
632
1267
  error: errText(err),
633
1268
  });
1269
+ this.retire(state, "failed");
1270
+ return { ok: false, why: "the scenario could not be run" };
1271
+ });
1272
+ state.running = running;
1273
+ return running;
1274
+ }
1275
+ async runArmedOnce(state, plan, signal) {
1276
+ // `/scenario/run` has no hook timeout, so this is the one place the full
1277
+ // derivation budget is waited — and still not while the service is
1278
+ // cooling down.
1279
+ const ready = await this.readyFor(state, plan, this.budgets.deriveMs);
1280
+ if (state.retired)
1281
+ return { ok: false, why: RUN_NO_PLAN };
1282
+ if (!ready.ok) {
1283
+ // Nothing may run on parameters that are unknown, unprocessed or unsafe.
1284
+ // The tool answers with an error that says so, which is what a model
1285
+ // reacts to best: it does the task itself.
1286
+ const step = plan.currentStep();
1287
+ if (step && ready.blame) {
1288
+ this.recordStep(state, {
1289
+ step,
1290
+ entry: plan.currentEntry(),
1291
+ outcome: "failed",
1292
+ stage: ready.stage,
1293
+ error: ready.error,
1294
+ });
1295
+ }
1296
+ this.handOver(state, {
1297
+ kind: "step_failed",
1298
+ step,
1299
+ entry: plan.currentEntry(),
1300
+ position: plan.currentStepIndex,
1301
+ error: ready.error,
1302
+ delivery: "none",
1303
+ stopReason: "not_run",
1304
+ });
1305
+ return handedOver(this.takeNote(state) ?? state.handover?.note) ?? { ok: false, why: ready.error };
634
1306
  }
635
1307
  const deadline = Date.now() + this.budgets.planMs;
636
1308
  let result;
637
1309
  try {
638
- result = await plan.runToCompletion(this.executeStep(state), state.sourceRun ? (entry) => state.sourceRun.outputFor(entry) : undefined, this.observeStep(state), deadline, MAX_REPLAY_REASON, { stopOnFailure: true });
1310
+ result = await plan.runToCompletion(this.executeStep(state, { deadline }), this.recordedSource(state), this.observeStep(state), deadline, MAX_REPLAY_REASON, { stopOnFailure: true, recordedFallback: this.recordedFallback, signal });
639
1311
  }
640
1312
  catch (err) {
641
1313
  // The scenario's own logic failed. Retire and let the model do the work.
@@ -647,15 +1319,33 @@ export class ReplayController {
647
1319
  this.retire(state, "failed");
648
1320
  return { ok: false, why: "not_ready" };
649
1321
  }
650
- // The plan stopped part-way (fallbk.md §Runner 3): a step broke, or the
651
- // next one is parked. The steps that ran are real, so their bundle goes to
652
- // the model — behind a note saying the rest of the task is its own.
1322
+ // The plan stopped part-way (fallbk.md §Runner 3): a step broke or produced
1323
+ // no result, the next one is parked, the budget ran out, or the caller left.
1324
+ // The steps that ran are real, so their bundle goes to the model — behind a
1325
+ // note saying the rest of the task is its own.
653
1326
  if (result.stopped) {
654
1327
  const stop = result.stopped;
655
- const ranOk = result.executed + result.recorded - result.errored;
1328
+ // Only live steps did work here. A recorded stand-in is last week's
1329
+ // output (BIR_REPLAY_RECORDED_OUTPUTS): it books nothing, and a run of
1330
+ // them alone is a baseline sample, not a hand-over "saving".
1331
+ const ranOk = result.executed - result.errored;
656
1332
  state.stepsPinned += Math.max(0, ranOk);
657
1333
  const position = stop.flatIndex ?? plan.stepCount;
658
1334
  const step = plan.allSteps()[position];
1335
+ // Cut short by the clock, nothing failed: the steps that ran did their
1336
+ // work and none is blamed. A divergence for the ledger — a partial
1337
+ // saving, no fallback step — as a budget stop always was.
1338
+ if (stop.reason === "budget" && ranOk > 0)
1339
+ this.upgrade(state, "diverged");
1340
+ // Cut short by the *caller* — Esc on run_scenario, or its client timing
1341
+ // out — is no saving at all: the turn it ended cost next to nothing and
1342
+ // says nothing about what the task costs. `handOver` books `fell_back`
1343
+ // (steps did run) or `failed`, and the report is not a measurement.
1344
+ if (stop.reason === "cancelled")
1345
+ state.callerCancelled = true;
1346
+ // Nothing ran *here*: there is no bundle, only the task — a recorded
1347
+ // output is not a result that happened (plan.ts, DriveOptions).
1348
+ const live = result.executed > 0;
659
1349
  this.handOver(state, {
660
1350
  // The plan only knows it stopped in front of its end; which kind of
661
1351
  // stop that is — parked, a judgement, a call that cannot run — is
@@ -665,13 +1355,36 @@ export class ReplayController {
665
1355
  entry: plan.allEntries()[position],
666
1356
  position,
667
1357
  error: stop.error,
668
- delivery: "below",
1358
+ // A cancelled run's note is kept for the model's next call, alone —
1359
+ // the bundle went to a caller who is gone — so it points at side
1360
+ // effects in place, never at results "below" that nobody receives.
1361
+ delivery: live && stop.reason !== "cancelled" ? "below" : "none",
1362
+ stopReason: stop.reason,
1363
+ ran: live ? undefined : 0,
1364
+ // An output-logic failure stops *after* its step's tool ran: the step
1365
+ // is done, its result is in the bundle, and it must not be run again.
1366
+ ranThisStep: stop.stage === "tool_output_logic",
669
1367
  });
1368
+ if (stop.reason === "cancelled") {
1369
+ // The caller is gone and reads nothing. The note stays undelivered: a
1370
+ // client that timed out hands the task back to the model in this same
1371
+ // turn, and the model's next call is where it learns which steps
1372
+ // already ran — before it runs them a second time.
1373
+ logLine("replay.cancelled", {
1374
+ scenario: state.scenarioId ?? undefined,
1375
+ ran: `${result.executed}/${plan.stepCount}`,
1376
+ why: "the caller went away — no further step was started",
1377
+ });
1378
+ const note = state.handover?.note ?? "";
1379
+ return live
1380
+ ? { ok: true, text: `${note}\n\n${result.text}`, responseModel: {}, steps: result.executed + result.recorded, partial: true }
1381
+ : (handedOver(note) ?? { ok: false, why: note });
1382
+ }
670
1383
  const note = this.takeNote(state) ?? state.handover?.note ?? "";
671
- if (result.executed + result.recorded === 0) {
672
- // Nothing ran: there is no bundle, only the task. The tool answers with
673
- // an error that says so, which is what a model reacts to best.
674
- return { ok: false, why: note };
1384
+ if (!live) {
1385
+ // Nothing ran: the tool answers with an error that says so, which is
1386
+ // what a model reacts to best.
1387
+ return handedOver(note) ?? { ok: false, why: note };
675
1388
  }
676
1389
  return {
677
1390
  ok: true,
@@ -681,7 +1394,24 @@ export class ReplayController {
681
1394
  partial: result.partial,
682
1395
  };
683
1396
  }
684
- state.stepsPinned += result.executed + result.recorded;
1397
+ if (result.executed === 0) {
1398
+ // Every step stood on its recorded output (BIR_REPLAY_RECORDED_OUTPUTS):
1399
+ // nothing happened here, and an answer built on last week's outputs is
1400
+ // not a replay. The model does the task.
1401
+ this.handOver(state, {
1402
+ kind: "step_failed",
1403
+ step: plan.allSteps()[0],
1404
+ entry: plan.allEntries()[0],
1405
+ position: 0,
1406
+ error: "none of the scenario's steps could run here",
1407
+ delivery: "none",
1408
+ stopReason: "not_run",
1409
+ ran: 0,
1410
+ });
1411
+ return handedOver(this.takeNote(state) ?? state.handover?.note) ?? { ok: false, why: "" };
1412
+ }
1413
+ // Live steps only, as above: a recorded stand-in did no work here.
1414
+ state.stepsPinned += result.executed;
685
1415
  let responseModel = {};
686
1416
  try {
687
1417
  responseModel = plan.responseModel();
@@ -692,9 +1422,11 @@ export class ReplayController {
692
1422
  // A plan whose tools errored ran to the end and did nothing. `steered_full`
693
1423
  // would book the largest saving available for it; `failed` books a baseline
694
1424
  // sample, which is what a replay that produced nothing is worth (mcpmark.md §13).
1425
+ // A step that stood on a recorded output did not happen here either, so a
1426
+ // plan with one is at best a divergence.
695
1427
  this.upgrade(state, result.errored > 0
696
1428
  ? "failed"
697
- : result.partial || result.skipped > 0
1429
+ : result.partial || result.skipped > 0 || result.recorded > 0
698
1430
  ? "diverged"
699
1431
  : "steered_full");
700
1432
  // `runToCompletion` returned without a stop: the plan ran to its end, so
@@ -757,7 +1489,9 @@ export class ReplayController {
757
1489
  durationMs: d.durationMs,
758
1490
  stepsPlanned: state.stepsPlanned,
759
1491
  stepsPinned: state.stepsPinned,
760
- measured: d.measured,
1492
+ // A run its caller cancelled ended the turn early: its cost is no
1493
+ // measurement of the task, whatever the transcript says.
1494
+ measured: d.measured && state.callerCancelled !== true,
761
1495
  pricingVersion: PRICING_VERSION,
762
1496
  prompt: d.prompt,
763
1497
  steps,
@@ -886,6 +1620,7 @@ export class ReplayController {
886
1620
  mode,
887
1621
  entries: flat.entries,
888
1622
  stopAt: flat.stopAt,
1623
+ fallbackTargets: "samples",
889
1624
  });
890
1625
  state.plan = plan;
891
1626
  state.flatStop = flat.stopReason;
@@ -904,15 +1639,19 @@ export class ReplayController {
904
1639
  coverage: coverageOf(scenario.steps, wrapped, this.opts.allowServers).join(","),
905
1640
  why: "bir replay — no match, an operator asked for it",
906
1641
  });
907
- try {
908
- await this.withBudget(plan.ready(), this.budgets.deriveMs, "derivation");
909
- }
910
- catch (err) {
911
- logLine("replay.derive_failed", { scenario: scenario.id, error: errText(err) });
912
- }
1642
+ // An operator's run keeps its own contract (R-CALL-37): out of time, the
1643
+ // targets too take their samples. What it never does is run on `{}`.
1644
+ const ready = await this.readyFor(state, plan, this.adHocBudgets.deriveMs, { adHoc: true });
1645
+ if (!ready.ok)
1646
+ return { ok: false, why: ready.error };
913
1647
  const trace = [];
1648
+ const deadline = Date.now() + this.adHocBudgets.planMs;
914
1649
  try {
915
- const result = await plan.runToCompletion(this.executeStep(state), state.sourceRun ? (entry) => state.sourceRun.outputFor(entry) : undefined, (info) => {
1650
+ const result = await plan.runToCompletion(this.executeStep(state, { deadline, adHoc: true }),
1651
+ // The operator reads a per-step trace that labels every stand-in, so
1652
+ // `bir replay` keeps serving recorded outputs for steps that cannot be
1653
+ // sent from here — and still stops at one that was sent and failed.
1654
+ this.recordedSource(state, true), (info) => {
916
1655
  this.observeStep(state)(info);
917
1656
  trace.push({
918
1657
  step: info.step.stepIndex,
@@ -921,7 +1660,7 @@ export class ReplayController {
921
1660
  outcome: info.outcome,
922
1661
  ms: info.ms,
923
1662
  });
924
- }, Date.now() + this.budgets.planMs);
1663
+ }, deadline, MAX_REPLAY_REASON, { recordedFallback: true });
925
1664
  let responseModel = {};
926
1665
  try {
927
1666
  responseModel = plan.responseModel();
@@ -929,6 +1668,8 @@ export class ReplayController {
929
1668
  catch (err) {
930
1669
  logDetail("replay.response_model_failed", { error: errText(err) });
931
1670
  }
1671
+ const stop = result.stopped;
1672
+ const stoppedStep = stop ? plan.allSteps()[stop.flatIndex ?? plan.stepCount] : undefined;
932
1673
  return {
933
1674
  ok: true,
934
1675
  text: result.text,
@@ -936,6 +1677,14 @@ export class ReplayController {
936
1677
  steps: result.executed + result.recorded,
937
1678
  partial: result.partial,
938
1679
  trace,
1680
+ stopped: stop
1681
+ ? {
1682
+ step: stop.stepIndex,
1683
+ tool: stoppedStep?.toolName ?? "",
1684
+ reason: stop.reason ?? stop.stage ?? stop.kind,
1685
+ error: stop.error,
1686
+ }
1687
+ : undefined,
939
1688
  };
940
1689
  }
941
1690
  catch (err) {
@@ -970,17 +1719,26 @@ export class ReplayController {
970
1719
  }
971
1720
  startDerivation(scenario, prompt, state, ctx = {}) {
972
1721
  const derive = this.opts.deriveImpl ?? deriveParameters;
1722
+ const apiKey = this.opts.apiKey ?? process.env.ANTHROPIC_API_KEY;
1723
+ const service = this.deriveService(scenario.id);
1724
+ const hasSchema = Object.keys(scenario.paramsObject ?? {}).length > 0;
1725
+ state.deriveOnService = !apiKey && service !== undefined && hasSchema;
973
1726
  return derive({
974
1727
  prompt,
975
1728
  intent: scenario.intent ?? "",
976
1729
  paramsObject: scenario.paramsObject,
977
- apiKey: this.opts.apiKey ?? process.env.ANTHROPIC_API_KEY,
1730
+ apiKey,
978
1731
  // Where to ask when there is no key here, which is the ordinary case
979
1732
  // (segmented.md R-PARAM-5).
980
- service: this.deriveService(scenario.id),
1733
+ service,
981
1734
  fetchImpl: this.opts.fetchImpl,
982
1735
  liveCall: ctx.liveCall,
983
1736
  recentResults: ctx.recentResults,
1737
+ // The request itself ends too. Without this a hung service or model held
1738
+ // the socket for up to 300 s after every wait on it had given up. Long
1739
+ // enough to cover both waits that can use its answer (the arm and the
1740
+ // first tool call); an answer after that is read by nobody.
1741
+ signal: timeoutSignal(Math.max(serviceTimeoutMs(), this.budgets.deriveMs)),
984
1742
  })
985
1743
  .then((r) => {
986
1744
  state.deriveCostUsd += r.costUsd;
@@ -1031,8 +1789,14 @@ export class ReplayController {
1031
1789
  *
1032
1790
  * The plan is retired first: whatever happens next, this turn neither steers
1033
1791
  * nor re-injects again.
1792
+ *
1793
+ * This runs inside the diverging call's `PreToolUse`, so `deadline` bounds
1794
+ * it: no step is started after it, and none is waited on past it. The bundle
1795
+ * then says exactly what ran, and hands the rest over.
1034
1796
  */
1035
- async diverge(state, toolName, why) {
1797
+ async diverge(state, toolName, why, deadline,
1798
+ /** Aborted once the host has its answer: no further step is started. */
1799
+ signal) {
1036
1800
  const plan = state.plan;
1037
1801
  if (!plan)
1038
1802
  return { kind: "passthrough" };
@@ -1044,11 +1808,19 @@ export class ReplayController {
1044
1808
  why,
1045
1809
  });
1046
1810
  this.retire(state, undefined);
1811
+ // The bundle, and the note in front of it, must fit the channel it goes
1812
+ // out on whole: the host cuts a long Bash output in the middle.
1813
+ const channelMax = toolName === "Bash" ? BASH_CHANNEL_MAX : MAX_REPLAY_REASON;
1047
1814
  let composed;
1048
1815
  try {
1049
- composed = await plan.composeBundle(MAX_REPLAY_REASON, this.executeStep(state), state.sourceRun ? (step) => state.sourceRun.outputFor(step) : undefined, this.observeStep(state),
1050
- // A parked step ends the bundle short of the task (fallbk.md D3).
1051
- { handover: plan.stopsEarly() });
1816
+ composed = await plan.composeBundle(Math.max(2_000, channelMax - MAX_HANDOVER_NOTE - 2), this.executeStep(state, { deadline }), this.recordedSource(state), this.observeStep(state), {
1817
+ // A parked step ends the bundle short of the task (fallbk.md D3).
1818
+ handover: plan.stopsEarly(),
1819
+ stopOnFailure: true,
1820
+ recordedFallback: this.recordedFallback,
1821
+ deadline,
1822
+ signal,
1823
+ });
1052
1824
  }
1053
1825
  catch (err) {
1054
1826
  this.upgrade(state, "failed");
@@ -1059,18 +1831,34 @@ export class ReplayController {
1059
1831
  });
1060
1832
  return { kind: "abort" };
1061
1833
  }
1062
- // Same rule as `runArmed`: a recovered chain whose tools errored is `failed`.
1063
- this.upgrade(state, composed.errored > 0 ? "failed" : "diverged");
1064
- state.stepsPinned += composed.executed + composed.recorded;
1834
+ const stop = composed.stopped;
1835
+ // Same rule as `runArmed`: a recovered chain whose tools errored is
1836
+ // `failed`, and so is one that stopped at a step that broke — its logic
1837
+ // threw, or it was sent and failed or never answered. A step that simply
1838
+ // could not be sent from here, or the hook's clock, is still a divergence.
1839
+ const broke = composed.errored > 0 ||
1840
+ (stop !== undefined &&
1841
+ stop.reason !== "not_run" &&
1842
+ stop.reason !== "budget" &&
1843
+ stop.reason !== "cancelled");
1844
+ // And a divergence is booked only when something ran under the plan — a
1845
+ // step pinned before it, or one executed now. One that stopped in front of
1846
+ // its first step (a built-in, which only the session runs) ran nothing and
1847
+ // delivered nothing: no saving, a baseline sample, as `handOver` books it.
1848
+ // Recorded stand-ins are no work here (BIR_REPLAY_RECORDED_OUTPUTS).
1849
+ const didWork = composed.executed > 0 || state.stepsPinned > 0;
1850
+ this.upgrade(state, broke || !didWork ? "failed" : "diverged");
1851
+ state.stepsPinned += composed.executed;
1065
1852
  // The headline number of this whole design: recovering from a divergence
1066
1853
  // costs nothing, because the proxies were already connected.
1067
1854
  logLine("replay.compose", {
1068
1855
  scenario: state.scenarioId ?? undefined,
1069
- remaining: composed.executed + composed.recorded + composed.skipped,
1856
+ remaining: plan.stepCount - plan.currentStepIndex,
1070
1857
  executed: composed.executed,
1071
1858
  recorded: composed.recorded,
1072
1859
  skipped: composed.skipped,
1073
1860
  errored: composed.errored || undefined,
1861
+ stopped: stop ? `${stop.flatIndex ?? "?"}:${stop.reason ?? stop.stage ?? stop.kind}` : undefined,
1074
1862
  bytes: composed.text.length,
1075
1863
  costUsd: state.fallbackCostUsd.toFixed(2),
1076
1864
  });
@@ -1081,15 +1869,26 @@ export class ReplayController {
1081
1869
  outcome: state.outcome,
1082
1870
  ms: Date.now() - state.armedAt,
1083
1871
  });
1084
- if (composed.executed + composed.recorded === 0) {
1085
- // Nothing survived; a bundle of nothing is worse than no bundle.
1872
+ if (composed.executed === 0) {
1873
+ // Nothing ran here: a bundle of nothing is worse than no bundle, and one
1874
+ // of recorded outputs only is last week's work. The model's own call goes
1875
+ // ahead — unless a step was *sent* and never answered. That one may have
1876
+ // done its work, and the model must know before it does the task itself.
1877
+ if (stop?.reason === "uncertain" && stop.flatIndex !== undefined) {
1878
+ const note = this.stopNote(state, plan, stop, "none");
1879
+ return this.deliverInstead(note, toolName);
1880
+ }
1086
1881
  return { kind: "abort" };
1087
1882
  }
1088
- // The bundle stops in front of a parked step: say so ahead of it, and the
1089
- // model carries on from there. The outcome stays `diverged` — the model
1090
- // went off-script first — so the report carries no fallback fields.
1883
+ // The bundle stops short of the task: say so ahead of it — which step, and
1884
+ // what may be assumed about it — and the model carries on from there. The
1885
+ // outcome stays as judged above, so the report carries no fallback fields.
1091
1886
  let text = composed.text;
1092
- if (plan.stopsEarly()) {
1887
+ if (stop && stop.kind === "step_failed" && stop.flatIndex !== undefined) {
1888
+ const note = this.stopNote(state, plan, stop, "below");
1889
+ text = `${note}\n\n${composed.text}`;
1890
+ }
1891
+ else if (plan.stopsEarly()) {
1093
1892
  const parked = plan.stopStep();
1094
1893
  const stopKind = this.stopKindOf(state, plan.stopEntry());
1095
1894
  state.handover = {
@@ -1102,6 +1901,7 @@ export class ReplayController {
1102
1901
  entry: plan.stopEntry(),
1103
1902
  position: plan.stepCount,
1104
1903
  delivery: "below",
1904
+ executedAbove: plan.currentStepIndex,
1105
1905
  }),
1106
1906
  delivered: true,
1107
1907
  };
@@ -1109,11 +1909,46 @@ export class ReplayController {
1109
1909
  }
1110
1910
  // Bash-clean delivery: the model reaches for `Bash` when it cannot call a
1111
1911
  // scripted tool, and a genuine command output is trusted where a `deny`
1112
- // reason is read as adversarial interception. Neutralise the delimiter first.
1912
+ // reason is read as adversarial interception.
1113
1913
  const action = this.deliverInstead(text, toolName);
1114
1914
  logDetail("replay.inject", { channel: action.kind, bytes: text.length });
1115
1915
  return action;
1116
1916
  }
1917
+ /**
1918
+ * The note for a divergence bundle that stopped at a step, recorded as this
1919
+ * plan's hand-over (already delivered — it goes out with the bundle), so the
1920
+ * model's own work from here is recorded as a fragment.
1921
+ */
1922
+ stopNote(state, plan, stop, delivery) {
1923
+ const position = stop.flatIndex ?? plan.currentStepIndex;
1924
+ const step = plan.allSteps()[position];
1925
+ const entry = plan.allEntries()[position];
1926
+ const note = this.noteFor(state, {
1927
+ kind: stop.kind,
1928
+ step,
1929
+ entry,
1930
+ position,
1931
+ error: stop.error,
1932
+ delivery,
1933
+ stopReason: stop.reason,
1934
+ // The steps before the cursor ran in the live session — the model's own
1935
+ // pinned calls, just above. With no bundle (`none`) they are the only
1936
+ // steps that ran; "no step ran" would invite the model to run them again.
1937
+ ran: delivery === "none" ? plan.currentStepIndex : undefined,
1938
+ executedAbove: plan.currentStepIndex,
1939
+ // An output-logic failure stops after its step's tool ran (plan.ts).
1940
+ ranThisStep: stop.stage === "tool_output_logic",
1941
+ });
1942
+ state.handover ??= {
1943
+ kind: stop.kind,
1944
+ stepIndex: callerStepIndexOf(entry) ?? step?.stepIndex ?? position,
1945
+ toolName: step?.toolName ?? undefined,
1946
+ error: stop.error,
1947
+ note,
1948
+ delivered: true,
1949
+ };
1950
+ return note;
1951
+ }
1117
1952
  /**
1118
1953
  * Hand the rest of the task to the model (fallbk.md D4/D5).
1119
1954
  *
@@ -1155,7 +1990,13 @@ export class ReplayController {
1155
1990
  ? "the next step has failed too often — the agent continues from here"
1156
1991
  : h.kind === "nondeterministic_step"
1157
1992
  ? "the next step needs a judgement the plan cannot compute — the agent makes it"
1158
- : "a step broke — the agent continues from here",
1993
+ : h.stopReason === "concurrent"
1994
+ ? "a second call arrived while the pinned step was still running — the agent continues from here"
1995
+ : h.stopReason === "budget" || h.stopReason === "cancelled"
1996
+ ? "the run was cut short before this step — the agent continues from here"
1997
+ : h.stopReason === "not_run" || h.stopReason === "uncertain"
1998
+ ? "a step produced no result here — the agent continues from here"
1999
+ : "a step broke — the agent continues from here",
1159
2000
  });
1160
2001
  }
1161
2002
  /**
@@ -1171,7 +2012,7 @@ export class ReplayController {
1171
2012
  }
1172
2013
  noteFor(state, h) {
1173
2014
  const plan = state.plan;
1174
- const executed = h.ranThisStep ? h.position + 1 : h.position;
2015
+ const executed = h.ran ?? (h.ranThisStep ? h.position + 1 : h.position);
1175
2016
  let params;
1176
2017
  try {
1177
2018
  params = plan?.parameters;
@@ -1186,6 +2027,7 @@ export class ReplayController {
1186
2027
  stepToolName: h.step?.toolName ?? undefined,
1187
2028
  failureCount: h.step?.failureCount,
1188
2029
  error: h.error,
2030
+ stopReason: h.stopReason,
1189
2031
  unusable: state.flatStop
1190
2032
  ? {
1191
2033
  reason: state.flatStop.reason,
@@ -1197,6 +2039,9 @@ export class ReplayController {
1197
2039
  ownerScenarioId: h.entry && h.entry.depth > 0 ? h.entry.scenarioId : undefined,
1198
2040
  inSubTask: h.entry && h.entry.depth > 0 ? (h.entry.frame.callIntent ?? null) : null,
1199
2041
  executed: Math.max(0, Math.min(executed, plan?.totalSteps ?? executed)),
2042
+ executedAbove: h.executedAbove,
2043
+ inFlight: h.inFlight,
2044
+ ranThisStep: h.ranThisStep,
1200
2045
  totalSteps: plan?.totalSteps ?? state.stepsPlanned,
1201
2046
  remainingTools: plan ? plan.toolsFrom(h.ranThisStep ? h.position + 1 : h.position) : [],
1202
2047
  params,
@@ -1207,22 +2052,173 @@ export class ReplayController {
1207
2052
  * The executor handed to a plan: dispatch a step to the proxy that owns its
1208
2053
  * upstream.
1209
2054
  *
1210
- * Rejecting means "could not be run **here**" — which is the only condition
1211
- * under which a recorded output may stand in. A tool that ran and failed
1212
- * resolves with its failure as the response, exactly as it would in a session.
2055
+ * A tool that ran and failed resolves with its failure as the response,
2056
+ * exactly as it would in a session. A rejection is a {@link StepRunError}
2057
+ * saying what happened to the call — not sent, sent and unanswered, or
2058
+ * refused — because the model is told differently about each (plan.ts).
2059
+ *
2060
+ * `deadline` is the whole run's: no step is waited on past it (but each gets
2061
+ * at least {@link MIN_STEP_MS}). `adHoc` is `bir replay`, which the operator
2062
+ * asked for by id and which the allowlist does not gate.
1213
2063
  */
1214
- executeStep(state) {
2064
+ executeStep(state, opts = {}) {
1215
2065
  return async (step, input) => {
1216
- const mcp = parseQualifiedName(step.toolName ?? "");
2066
+ const name = step.toolName ?? "";
2067
+ const mcp = parseQualifiedName(name);
1217
2068
  if (!mcp) {
1218
- throw new Error(`${step.toolName} is not an MCP tool — it can only run in the session`);
2069
+ throw new StepRunError(`${name || "this step"} is a built-in tool, which only runs in the session`, "not_run");
2070
+ }
2071
+ // `BIR_REPLAY_ALLOW_SERVERS` is about exactly this: a call bir makes by
2072
+ // itself, with nobody approving it. A steered plan's divergence recovery
2073
+ // reaches here too, for steps that were only ever to be steered in the
2074
+ // session — where the user's own permission rules apply.
2075
+ const allow = this.opts.allowServers;
2076
+ if (!opts.adHoc && allow && !allow.has(mcp.serverName)) {
2077
+ throw new StepRunError(`${mcp.serverName} is not on BIR_REPLAY_ALLOW_SERVERS, so bir does not call it unattended`, "not_run");
2078
+ }
2079
+ const left = opts.deadline !== undefined ? opts.deadline - Date.now() : Number.POSITIVE_INFINITY;
2080
+ const timeoutMs = Math.max(MIN_STEP_MS, Math.min(this.budgets.stepMs, left));
2081
+ let result;
2082
+ try {
2083
+ result = await this.work.call(mcp.serverName, mcp.toolName, input, timeoutMs, {
2084
+ sessionId: state.sessionId,
2085
+ });
2086
+ }
2087
+ catch (err) {
2088
+ throw classifyWorkError(err, mcp.serverName);
2089
+ }
2090
+ // A proxy that answers the work with no result at all took it and lost
2091
+ // it (tool-error.ts, `isNoResult`): sent, and may or may not have run.
2092
+ // Threaded as `null`, it would read as a success.
2093
+ if (result === undefined || result === null) {
2094
+ throw new StepRunError(`the ${mcp.serverName} proxy answered with no result — the call may or may not have run`, "uncertain");
1219
2095
  }
1220
- const result = await this.work.call(mcp.serverName, mcp.toolName, input, this.budgets.stepMs, { sessionId: state.sessionId });
1221
2096
  // Serialize exactly as the proxy records it, or `toolOutputLogic` — which
1222
2097
  // was authored against that shape — silently derives nothing (§7.2).
1223
2098
  return serializeCapped(redact(result));
1224
2099
  };
1225
2100
  }
2101
+ /** Where recorded outputs come from, when they may stand in at all. */
2102
+ recordedSource(state, force = false) {
2103
+ if (!force && !this.recordedFallback)
2104
+ return undefined;
2105
+ const source = state.sourceRun;
2106
+ return source ? (entry) => source.outputFor(entry) : undefined;
2107
+ }
2108
+ /**
2109
+ * Wait for a plan's parameters within `budgetMs`, then judge whether the plan
2110
+ * may run on them. Never throws.
2111
+ *
2112
+ * Out of time, the parameters settle without the derivation
2113
+ * (`useFallbackParams`: settings at their samples, targets null) — never on
2114
+ * `{}` — and the plan runs only if it needs no target it was not given. A
2115
+ * `paramsLogic` that throws, or a value too dangerous to template into a
2116
+ * command (safety.ts), stops it. `adHoc` keeps `bir replay`'s contract: the
2117
+ * operator named the scenario, so neither judgement applies.
2118
+ */
2119
+ async readyFor(state, plan, budgetMs, opts = {}) {
2120
+ const logicFailed = (err) => ({
2121
+ ok: false,
2122
+ error: `the scenario's parameter logic failed: ${errText(err)}`,
2123
+ stage: "params_logic",
2124
+ // The scenario's own logic is broken: the step it stopped in front of
2125
+ // carries the verdict, so the recording page says why it never runs.
2126
+ blame: true,
2127
+ });
2128
+ if (!plan.parametersSettled) {
2129
+ let failure;
2130
+ const settled = plan.ready().then(() => true, (err) => {
2131
+ failure = err;
2132
+ return true;
2133
+ });
2134
+ const waited = await this.waitDerivation(state, settled, budgetMs);
2135
+ if (failure !== undefined)
2136
+ return logicFailed(failure);
2137
+ if (waited === undefined) {
2138
+ logLine("replay.derive_timeout", {
2139
+ scenario: state.scenarioId ?? undefined,
2140
+ waitedMs: budgetMs,
2141
+ why: opts.adHoc
2142
+ ? "bir replay — every parameter takes its recorded sample"
2143
+ : "settings take their recorded samples, targets stay unnamed — the plan runs only if it needs none",
2144
+ });
2145
+ try {
2146
+ plan.useFallbackParams();
2147
+ }
2148
+ catch (err) {
2149
+ return logicFailed(err);
2150
+ }
2151
+ if (!opts.adHoc) {
2152
+ const schema = plan.paramsSchema;
2153
+ const planned = plan.allEntries().slice(0, plan.stepCount);
2154
+ const blocked = blockingTargets(targetKeys(schema), schema, planned, plan.paramsLogic);
2155
+ if (blocked.length > 0) {
2156
+ return {
2157
+ ok: false,
2158
+ error: `the value of ${blocked[0]} could not be read from this request in time`,
2159
+ stage: "derive_params",
2160
+ blame: false,
2161
+ };
2162
+ }
2163
+ }
2164
+ }
2165
+ }
2166
+ const sink = opts.adHoc ? undefined : sinkOfPlan(plan.allEntries().slice(0, plan.stepCount));
2167
+ if (sink) {
2168
+ const bad = unsafeParameter(plan.derivedValues, plan.paramsSchema, sink);
2169
+ if (bad) {
2170
+ logLine("replay.unsafe_param", {
2171
+ scenario: state.scenarioId ?? undefined,
2172
+ param: bad.key,
2173
+ sink,
2174
+ why: "a value read for this request would be templated into a command unattended — the agent does the task instead",
2175
+ });
2176
+ return {
2177
+ ok: false,
2178
+ error: `the value read for ${bad.key} contains characters that are not safe to put into a command unattended`,
2179
+ stage: "derive_params",
2180
+ blame: false,
2181
+ };
2182
+ }
2183
+ }
2184
+ return { ok: true };
2185
+ }
2186
+ /**
2187
+ * Wait for something the derivation decides, within `budgetMs`. Resolves
2188
+ * undefined when the time ran out.
2189
+ *
2190
+ * When the service is the one deriving, this is a service wait on a hot path
2191
+ * (C1): it does not wait at all while the service is cooling down, and an
2192
+ * overrun is reported to the breaker so the next hook does not wait either.
2193
+ * Even a zero wait lets an answer that has already arrived through.
2194
+ */
2195
+ async waitDerivation(state, pending, budgetMs) {
2196
+ const onService = state.deriveOnService === true;
2197
+ const budget = onService && !serviceHealth.canWait() ? 0 : budgetMs;
2198
+ let timer;
2199
+ const late = Symbol("late");
2200
+ try {
2201
+ const winner = await Promise.race([
2202
+ pending,
2203
+ new Promise((resolve) => {
2204
+ timer = setTimeout(() => resolve(late), Math.max(0, budget));
2205
+ timer.unref?.();
2206
+ }),
2207
+ ]);
2208
+ if (winner !== late)
2209
+ return winner;
2210
+ if (onService && budget > 0)
2211
+ serviceHealth.slow("derive");
2212
+ return undefined;
2213
+ }
2214
+ catch {
2215
+ return undefined;
2216
+ }
2217
+ finally {
2218
+ if (timer)
2219
+ clearTimeout(timer);
2220
+ }
2221
+ }
1226
2222
  /**
1227
2223
  * Where a pinned step's output will arrive from — the proxy's report, or the
1228
2224
  * hook's `tool_response` (§7.2).
@@ -1312,7 +2308,7 @@ export class ReplayController {
1312
2308
  logLine("replay.step_skipped", {
1313
2309
  n: info.step.stepIndex,
1314
2310
  tool: info.step.toolName,
1315
- why: "its tool could not run here and it has no recorded output",
2311
+ why: "its call produced no result here — the plan stops at it",
1316
2312
  error: info.error,
1317
2313
  });
1318
2314
  return;
@@ -1358,17 +2354,8 @@ export class ReplayController {
1358
2354
  }
1359
2355
  state.retired = true;
1360
2356
  state.pinned.clear();
1361
- }
1362
- withBudget(p, ms, what) {
1363
- let timer;
1364
- const budget = new Promise((_, reject) => {
1365
- timer = setTimeout(() => reject(new Error(`${what} exceeded ${ms}ms`)), ms);
1366
- timer.unref?.();
1367
- });
1368
- return Promise.race([p, budget]).finally(() => {
1369
- if (timer)
1370
- clearTimeout(timer);
1371
- });
2357
+ // A call waiting for a pin to thread has nothing left to wait for.
2358
+ this.notifyPins(state);
1372
2359
  }
1373
2360
  }
1374
2361
  /** Every target of a schema. A parameter with no kind is one (R-PARAM-3). */
@@ -1377,6 +2364,88 @@ function targetKeys(schema) {
1377
2364
  return [];
1378
2365
  return Object.keys(schema).filter((k) => schema[k]?.kind !== "setting");
1379
2366
  }
2367
+ /** A positive number from the environment, or undefined. */
2368
+ function positiveEnv(name) {
2369
+ const n = Number(process.env[name]);
2370
+ return Number.isFinite(n) && n > 0 ? n : undefined;
2371
+ }
2372
+ /**
2373
+ * What a rejected dispatch means for the step (plan.ts, `StepRejection`).
2374
+ *
2375
+ * Read from the message, because that is all the proxy sends back. Only the
2376
+ * cases that are certain are named: nothing was sent (no proxy, no proxy that
2377
+ * is this session's, the queue closed before any proxy took the work, the
2378
+ * upstream already gone), or the upstream answered with a JSON-RPC error.
2379
+ * Everything else — a timeout, an
2380
+ * aborted wait, a connection that dropped mid-call, a shutdown while a proxy
2381
+ * held the work (`QUEUE_CLOSED_IN_FLIGHT`), an error nobody recognises — was
2382
+ * sent and may have done its work, and is said to be exactly that. Guessing
2383
+ * "failed" or "not run" there is how a model is invited to run a write twice.
2384
+ */
2385
+ function classifyWorkError(err, serverName) {
2386
+ // Proxies are here, but none is this session's: not sent, and not the
2387
+ // scenario's fault — recorded as not run, never as a failed step.
2388
+ if (err instanceof NoSessionProxyError)
2389
+ return new StepRunError(err.message, "not_run");
2390
+ const message = errText(err);
2391
+ if (message === NO_PROXY) {
2392
+ return new StepRunError(`no ${serverName} proxy of this session is connected to bir (${NO_PROXY})`, "not_run");
2393
+ }
2394
+ if (message === QUEUE_CLOSED || /\bupstream is not available\b/i.test(message)) {
2395
+ return new StepRunError(message, "not_run");
2396
+ }
2397
+ if (/^MCP error -?\d+/i.test(message))
2398
+ return new StepRunError(message, "failed");
2399
+ return new StepRunError(message, "uncertain");
2400
+ }
2401
+ /**
2402
+ * What the proxy says of a call its upstream was already gone for: it was
2403
+ * never sent (relay.ts, `upstream unavailable`; executor.ts's wording too).
2404
+ */
2405
+ const NOT_SENT_TEXT = /\bupstream (?:is not |un)available\b/i;
2406
+ /**
2407
+ * What a host or a proxy says of a call that was sent and not seen through: a
2408
+ * timeout, an interrupt, a cancellation, a dropped connection. Its work may
2409
+ * have happened — the upstream does not stop because the waiting did.
2410
+ */
2411
+ const UNCERTAIN_TEXT = /\b(?:timed? ?out|timeout|interrupted|cancell?ed|aborted|disconnected|connection (?:closed|reset|lost|dropped)|upstream (?:closed|restarted)|relay stopped|socket hang up|ECONNRESET|EPIPE)\b/i;
2412
+ /**
2413
+ * How a pinned call ended when it did not simply work: the error to record
2414
+ * and, when it is not an ordinary failure, the stop reason the note words it
2415
+ * by (handover.ts). Undefined when neither the caller nor the output says
2416
+ * anything went wrong — the output's own tool error is judged after.
2417
+ *
2418
+ * - `how` from the caller wins.
2419
+ * - A proxy-threaded call with *no result* was abandoned by its proxy
2420
+ * (intercept.ts `onAbandoned`: `isError` and a reason, nothing else) — sent
2421
+ * and unanswered, so `uncertain`, unless the reason says it was never sent.
2422
+ * - A failure whose text is a timeout, an interrupt or a dropped connection
2423
+ * is `uncertain` too. Anything else is a failure: the tool answered one.
2424
+ */
2425
+ function pinFailure(pin, output, failed, how) {
2426
+ const noResult = pin.reach === "direct" && isNoResult(output);
2427
+ const text = failed?.trim();
2428
+ if (failed === undefined && !noResult && !how.uncertain && !how.notSent)
2429
+ return undefined;
2430
+ const reason = how.notSent
2431
+ ? "not_run"
2432
+ : how.uncertain
2433
+ ? "uncertain"
2434
+ : noResult
2435
+ ? text && NOT_SENT_TEXT.test(text)
2436
+ ? "not_run"
2437
+ : "uncertain"
2438
+ : text && UNCERTAIN_TEXT.test(text)
2439
+ ? "uncertain"
2440
+ : undefined;
2441
+ const error = text ||
2442
+ (reason === "not_run"
2443
+ ? "the call was never sent — its server was not available"
2444
+ : reason === "uncertain" || noResult
2445
+ ? "the call produced no result — its server stopped answering mid-call"
2446
+ : "the tool reported an error");
2447
+ return { error, reason };
2448
+ }
1380
2449
  /**
1381
2450
  * The index a hand-over reports, in the *caller's* chain (R-CALL-32).
1382
2451
  *