@basein/runner 0.2.11 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +540 -350
- package/dist/auth/client.d.ts +65 -3
- package/dist/auth/client.js +305 -53
- package/dist/bin/bir-hooks.d.ts +72 -31
- package/dist/bin/bir-hooks.js +176 -284
- package/dist/bin/bir-proxy.d.ts +17 -2
- package/dist/bin/bir-proxy.js +222 -28
- package/dist/bin/bir-scenario.d.ts +61 -10
- package/dist/bin/bir-scenario.js +704 -135
- package/dist/bin/bir.d.ts +14 -8
- package/dist/bin/bir.js +2937 -488
- package/dist/bin/investigate.js +41 -16
- package/dist/bin/scenario-edit.d.ts +43 -10
- package/dist/bin/scenario-edit.js +281 -80
- package/dist/bin/scenario-liveness.d.ts +194 -0
- package/dist/bin/scenario-liveness.js +459 -0
- package/dist/bin/scenario-stderr.d.ts +43 -0
- package/dist/bin/scenario-stderr.js +76 -0
- package/dist/bin/service-read.d.ts +88 -0
- package/dist/bin/service-read.js +128 -0
- package/dist/bin/setup.d.ts +19 -0
- package/dist/bin/setup.js +146 -20
- package/dist/config/adapters/claude-code.d.ts +92 -14
- package/dist/config/adapters/claude-code.js +291 -101
- package/dist/config/adapters/generic.d.ts +4 -1
- package/dist/config/adapters/generic.js +13 -14
- package/dist/config/generate.d.ts +392 -14
- package/dist/config/generate.js +666 -19
- package/dist/config/mcp-approval.d.ts +76 -0
- package/dist/config/mcp-approval.js +210 -0
- package/dist/config/node-path.d.ts +42 -0
- package/dist/config/node-path.js +102 -0
- package/dist/config/overrides.d.ts +62 -0
- package/dist/config/overrides.js +136 -0
- package/dist/config/resolve.d.ts +70 -8
- package/dist/config/resolve.js +186 -15
- package/dist/control/client.d.ts +22 -0
- package/dist/control/client.js +26 -7
- package/dist/control/correlation.d.ts +23 -5
- package/dist/control/correlation.js +0 -0
- package/dist/control/daemon.d.ts +400 -18
- package/dist/control/daemon.js +1105 -102
- package/dist/control/discovery.d.ts +77 -5
- package/dist/control/discovery.js +186 -11
- package/dist/control/ensure-hook.d.ts +61 -0
- package/dist/control/ensure-hook.js +160 -8
- package/dist/control/paths.d.ts +26 -0
- package/dist/control/paths.js +34 -0
- package/dist/control/pin-guard.d.ts +83 -0
- package/dist/control/pin-guard.js +534 -0
- package/dist/control/recorder-keeper.d.ts +229 -0
- package/dist/control/recorder-keeper.js +721 -0
- package/dist/control/recorder-main.d.ts +73 -0
- package/dist/control/recorder-main.js +778 -0
- package/dist/control/server.d.ts +707 -31
- package/dist/control/server.js +2803 -306
- package/dist/control/transcript.d.ts +21 -1
- package/dist/control/transcript.js +378 -112
- package/dist/index.d.ts +5 -1
- package/dist/index.js +5 -1
- package/dist/jsonrpc/framing.d.ts +76 -12
- package/dist/jsonrpc/framing.js +216 -59
- package/dist/proxy/intercept.d.ts +49 -5
- package/dist/proxy/intercept.js +188 -59
- package/dist/proxy/relay.d.ts +16 -0
- package/dist/proxy/relay.js +77 -5
- package/dist/proxy/revive.d.ts +122 -0
- package/dist/proxy/revive.js +251 -0
- package/dist/proxy/session.d.ts +222 -14
- package/dist/proxy/session.js +708 -99
- package/dist/record/housekeeping.d.ts +71 -0
- package/dist/record/housekeeping.js +417 -0
- package/dist/record/recorder.d.ts +51 -3
- package/dist/record/recorder.js +3 -0
- package/dist/record/redact.d.ts +103 -9
- package/dist/record/redact.js +659 -30
- package/dist/record/remote-recorder.d.ts +327 -32
- package/dist/record/remote-recorder.js +1517 -187
- package/dist/record/switchable-recorder.d.ts +126 -0
- package/dist/record/switchable-recorder.js +351 -0
- package/dist/record/truncate.d.ts +40 -1
- package/dist/record/truncate.js +111 -4
- package/dist/replay/bash-channel.d.ts +35 -0
- package/dist/replay/bash-channel.js +133 -0
- package/dist/replay/bundle.d.ts +10 -0
- package/dist/replay/bundle.js +10 -0
- package/dist/replay/controller.d.ts +371 -20
- package/dist/replay/controller.js +1185 -111
- package/dist/replay/derive.d.ts +1 -0
- package/dist/replay/derive.js +81 -24
- package/dist/replay/executor.d.ts +51 -16
- package/dist/replay/executor.js +114 -41
- package/dist/replay/handover.d.ts +54 -1
- package/dist/replay/handover.js +55 -5
- package/dist/replay/logic.d.ts +62 -10
- package/dist/replay/logic.js +140 -22
- package/dist/replay/plan.d.ts +201 -27
- package/dist/replay/plan.js +581 -141
- package/dist/replay/safety.d.ts +72 -0
- package/dist/replay/safety.js +303 -0
- package/dist/replay/source-run.d.ts +10 -0
- package/dist/replay/source-run.js +80 -35
- package/dist/replay/targets.d.ts +15 -1
- package/dist/replay/targets.js +62 -4
- package/dist/replay/tool-error.d.ts +15 -1
- package/dist/replay/tool-error.js +33 -1
- package/dist/replay/types.d.ts +7 -4
- package/dist/upstream/client.d.ts +56 -3
- package/dist/upstream/client.js +61 -4
- package/dist/upstream/headers.d.ts +43 -0
- package/dist/upstream/headers.js +90 -0
- package/dist/upstream/http-client.d.ts +213 -22
- package/dist/upstream/http-client.js +1034 -168
- package/dist/upstream/http-transport.d.ts +143 -0
- package/dist/upstream/http-transport.js +391 -0
- package/dist/upstream/lazy-client.d.ts +8 -0
- package/dist/upstream/lazy-client.js +29 -1
- package/dist/upstream/sse.d.ts +78 -0
- package/dist/upstream/sse.js +189 -0
- package/dist/upstream/stdio-client.d.ts +61 -1
- package/dist/upstream/stdio-client.js +356 -33
- package/dist/util/atomic-write.d.ts +29 -0
- package/dist/util/atomic-write.js +96 -0
- package/dist/util/journal.d.ts +15 -0
- package/dist/util/journal.js +48 -3
- package/dist/util/log.d.ts +10 -0
- package/dist/util/log.js +28 -1
- package/dist/util/off-switch.d.ts +46 -0
- package/dist/util/off-switch.js +91 -0
- package/dist/util/service-health.d.ts +133 -0
- package/dist/util/service-health.js +217 -0
- package/docs/BaseInstRunner.md +694 -621
- package/docs/calculatedReplay.md +1625 -1244
- package/docs/calculatedReplayGuide.md +1212 -879
- package/docs/installRun.md +1013 -772
- package/docs/loginWeb.md +627 -607
- package/docs/mcpmark.md +49 -16
- package/docs/my-first-sample.md +569 -545
- package/docs/quickstart.md +384 -271
- package/docs/t-bench.md +406 -394
- package/package.json +58 -57
- package/production.md +1371 -0
- package/scripts/install.ps1 +97 -6
- package/scripts/install.sh +104 -7
|
@@ -12,38 +12,159 @@
|
|
|
12
12
|
* ignore. Nothing throws at a hook.
|
|
13
13
|
*/
|
|
14
14
|
import { parseQualifiedName } from "../control/correlation.js";
|
|
15
|
-
import {
|
|
15
|
+
import { isHousekeepingCall } from "../record/housekeeping.js";
|
|
16
16
|
import { redact } from "../record/redact.js";
|
|
17
17
|
import { serializeCapped } from "../record/truncate.js";
|
|
18
18
|
import { logDetail, logLine, errText } from "../util/log.js";
|
|
19
|
-
import {
|
|
19
|
+
import { serviceHealth, serviceTimeoutMs, timeoutSignal, } from "../util/service-health.js";
|
|
20
|
+
import { bashCommandFor } from "./bash-channel.js";
|
|
21
|
+
import { BASH_CHANNEL_MAX, MAX_REPLAY_REASON, truncate } from "./bundle.js";
|
|
20
22
|
import { coverageOf, modeFor, reachOf } from "./coverage.js";
|
|
21
23
|
import { deriveParameters, } from "./derive.js";
|
|
22
|
-
import { buildHandoverNote } from "./handover.js";
|
|
23
|
-
import { ProxyWorkQueue } from "./executor.js";
|
|
24
|
-
import {
|
|
24
|
+
import { buildHandoverNote, MAX_HANDOVER_NOTE, } from "./handover.js";
|
|
25
|
+
import { NO_PROXY, NoSessionProxyError, ProxyWorkQueue, QUEUE_CLOSED } from "./executor.js";
|
|
26
|
+
import { logicTimeoutMs, MAX_LOGIC_TIMEOUT_MS } from "./logic.js";
|
|
27
|
+
import { PREVIEW_BUDGET_MS, ScenarioReplayPlan, StepRunError, } from "./plan.js";
|
|
25
28
|
import { PRICING_VERSION } from "./pricing.js";
|
|
29
|
+
import { hasShellStep, sinkOf, sinkOfPlan, unsafeParameter } from "./safety.js";
|
|
26
30
|
import { SourceRunOutputs } from "./source-run.js";
|
|
27
31
|
import { clampToFrameStart, flattenChain, } from "./flatten.js";
|
|
28
32
|
import { blockingTargets } from "./targets.js";
|
|
29
|
-
import { toolResultError } from "./tool-error.js";
|
|
33
|
+
import { isNoResult, toolResultError } from "./tool-error.js";
|
|
30
34
|
import { OUTCOME_RANK, hasTargets, isReadyScenario, } from "./types.js";
|
|
31
35
|
/** The first-party tool a `direct` plan is delivered through (§6.3). */
|
|
32
36
|
export const DIRECT_TOOL_NAME = "mcp__bir__run_scenario";
|
|
33
37
|
/** How long a `/proxy/poll` is held open before it answers empty. */
|
|
34
38
|
export const POLL_HOLD_MS = 25_000;
|
|
39
|
+
/**
|
|
40
|
+
* The longest a prompt ever waits for a steer plan's parameters so that its
|
|
41
|
+
* directive can preview them ({@link ReplayController.settleForPreview}).
|
|
42
|
+
* `UserPromptSubmit` is on the user's critical path: a preview is worth a
|
|
43
|
+
* little of it, never the derivation's whole budget.
|
|
44
|
+
*/
|
|
45
|
+
export const PREVIEW_SETTLE_MAX_MS = 1_200;
|
|
46
|
+
/**
|
|
47
|
+
* Whether the pin guard holds a call to this tool to the step's exact shape —
|
|
48
|
+
* the same command head, the same file (control/pin-guard.ts) — so that a call
|
|
49
|
+
* written without the directive's preview is declined rather than pinned.
|
|
50
|
+
*/
|
|
51
|
+
function pinnedToShape(toolName) {
|
|
52
|
+
const name = toolName ?? "";
|
|
53
|
+
return !parseQualifiedName(name) && sinkOf(name) !== undefined;
|
|
54
|
+
}
|
|
35
55
|
export const DEFAULT_BUDGETS = {
|
|
36
|
-
|
|
56
|
+
// The longest a prompt waits for the service to say whether this is a known
|
|
57
|
+
// task; a match that has not answered by then is a turn run normally. The
|
|
58
|
+
// control server stops waiting at all for a while after one that answered
|
|
59
|
+
// slowly (`BIR_MATCH_SLOW_MS`). `BIR_MATCH_BUDGET_MS` in the recorder.
|
|
60
|
+
matchMs: 1_500,
|
|
37
61
|
deriveMs: 8_000,
|
|
38
|
-
planMs:
|
|
62
|
+
planMs: 100_000,
|
|
39
63
|
stepMs: 60_000,
|
|
64
|
+
hookDeriveMs: 4_000,
|
|
65
|
+
preToolMs: 9_000,
|
|
40
66
|
};
|
|
67
|
+
/**
|
|
68
|
+
* The least time a dispatched step is given, however little the run has left.
|
|
69
|
+
* A step sent with 50 ms to answer is a step that may or may not have run.
|
|
70
|
+
*/
|
|
71
|
+
const MIN_STEP_MS = 1_000;
|
|
72
|
+
/**
|
|
73
|
+
* The control server answers a `PreToolUse` by this, whatever replay is doing
|
|
74
|
+
* (its `HOOK_DEADLINE_MS` for `/tool/pre`). A deployment's `hookDeadlinesMs`
|
|
75
|
+
* can only shorten it; the server passes the one it keeps as `answerBy`
|
|
76
|
+
* ({@link PreToolOptions}).
|
|
77
|
+
*/
|
|
78
|
+
const PRETOOL_ROUTE_DEADLINE_MS = 12_000;
|
|
79
|
+
/** Logic, the bundle and the answer's own trip: work past the last step's start. */
|
|
80
|
+
const PRETOOL_MARGIN_MS = 1_000;
|
|
81
|
+
/**
|
|
82
|
+
* The most {@link ReplayBudgets.preToolMs} may be: the last step started at the
|
|
83
|
+
* budget's end still gets its {@link MIN_STEP_MS} and the margin before the
|
|
84
|
+
* route's deadline answers the host. Past it, a divergence kept starting steps
|
|
85
|
+
* — writes, unattended — after the model had been told to carry on itself.
|
|
86
|
+
*/
|
|
87
|
+
export const MAX_PRETOOL_REPLAY_MS = PRETOOL_ROUTE_DEADLINE_MS - MIN_STEP_MS - PRETOOL_MARGIN_MS;
|
|
88
|
+
/**
|
|
89
|
+
* The longest a direct run may take by its budgets, end to end.
|
|
90
|
+
*
|
|
91
|
+
* `bir-scenario` gives `/scenario/run` 290 s (`BIR_SCENARIO_RUN_TIMEOUT_MS`),
|
|
92
|
+
* just under Claude Code's 5-minute idle timeout for an MCP call, and a run
|
|
93
|
+
* that answers after its client stopped waiting is a replay thrown away with
|
|
94
|
+
* its steps already done. So the budgets are held to 270 s, and the defaults
|
|
95
|
+
* sit far below it: 8 s of derivation + 100 s of plan (more only when an
|
|
96
|
+
* operator sets `BIR_REPLAY_BUDGET_MS` or `BIR_REPLAY_PLAN_BUDGET_MS`) + 12 s
|
|
97
|
+
* of slack (one last step's 1 s, six 1 s logic bodies, a 5 s margin —
|
|
98
|
+
* {@link clampDirectRun}) = 120 s.
|
|
99
|
+
*
|
|
100
|
+
* Only `/scenario/run` is held to it. `bir replay` (`runAdHoc`) answers the
|
|
101
|
+
* operator's own command, which no MCP client times out, and keeps the budgets
|
|
102
|
+
* the environment asked for.
|
|
103
|
+
*/
|
|
104
|
+
export const DIRECT_RUN_CEILING_MS = 270_000;
|
|
105
|
+
/** Transport, the bundle and redaction: work a direct run does outside every budget. */
|
|
106
|
+
const DIRECT_RUN_MARGIN_MS = 5_000;
|
|
107
|
+
/**
|
|
108
|
+
* Logic bodies a direct run can still evaluate once its plan deadline has
|
|
109
|
+
* passed: `paramsLogic` before the plan (after the derivation's own wait), and
|
|
110
|
+
* for the step started just before the deadline its parameter mapping and
|
|
111
|
+
* input logic, its output logic and result mapping, then `responseParamsLogic`.
|
|
112
|
+
* Each is bounded by `logicTimeoutMs()` (logic.ts).
|
|
113
|
+
*/
|
|
114
|
+
const LOGIC_BODIES_OUTSIDE_PLAN = 6;
|
|
115
|
+
/** A derivation wait longer than this is not a budget a direct run can keep. */
|
|
116
|
+
const MAX_DIRECT_DERIVE_MS = 60_000;
|
|
117
|
+
/** However a direct run is squeezed, its plan keeps at least this. */
|
|
118
|
+
const MIN_DIRECT_PLAN_MS = 10_000;
|
|
119
|
+
/**
|
|
120
|
+
* The direct-run budgets, held under {@link DIRECT_RUN_CEILING_MS}.
|
|
121
|
+
*
|
|
122
|
+
* The worst case is `deriveMs + planMs` plus what can happen past the plan's
|
|
123
|
+
* deadline: one last step's {@link MIN_STEP_MS}, {@link LOGIC_BODIES_OUTSIDE_PLAN}
|
|
124
|
+
* logic bodies, and the margin. `stepMs` never adds to it — each step is
|
|
125
|
+
* clipped to what the plan has left. Budgets that would pass the ceiling are
|
|
126
|
+
* brought under it — the derivation wait first to {@link MAX_DIRECT_DERIVE_MS},
|
|
127
|
+
* then the plan — and `clamped` says what changed, for the log.
|
|
128
|
+
*/
|
|
129
|
+
export function clampDirectRun(budgets, logicMs = logicTimeoutMs()) {
|
|
130
|
+
const outside = MIN_STEP_MS + LOGIC_BODIES_OUTSIDE_PLAN * logicMs + DIRECT_RUN_MARGIN_MS;
|
|
131
|
+
if (budgets.deriveMs + budgets.planMs + outside <= DIRECT_RUN_CEILING_MS)
|
|
132
|
+
return { budgets };
|
|
133
|
+
const deriveMs = Math.min(budgets.deriveMs, MAX_DIRECT_DERIVE_MS);
|
|
134
|
+
const planMs = Math.max(MIN_DIRECT_PLAN_MS, Math.min(budgets.planMs, DIRECT_RUN_CEILING_MS - deriveMs - outside));
|
|
135
|
+
const clamped = {};
|
|
136
|
+
if (deriveMs !== budgets.deriveMs)
|
|
137
|
+
clamped.deriveMs = [budgets.deriveMs, deriveMs];
|
|
138
|
+
if (planMs !== budgets.planMs)
|
|
139
|
+
clamped.planMs = [budgets.planMs, planMs];
|
|
140
|
+
return { budgets: { ...budgets, deriveMs, planMs }, clamped };
|
|
141
|
+
}
|
|
142
|
+
/** The `why` of a {@link RunArmedResult} with nothing armed to run. */
|
|
143
|
+
export const RUN_NO_PLAN = "no_plan";
|
|
144
|
+
/** The hand-over shape of a {@link RunArmedResult}, or undefined when there is no note to hand over. */
|
|
145
|
+
function handedOver(note) {
|
|
146
|
+
return note ? { ok: false, why: note, note: true } : undefined;
|
|
147
|
+
}
|
|
41
148
|
export const DEFAULT_INTENT_MATCH = {
|
|
42
149
|
enabled: true,
|
|
43
150
|
budgetMs: 4_000,
|
|
44
151
|
maxPerTurn: 3,
|
|
45
152
|
maxRequestsPerTurn: 40,
|
|
46
153
|
};
|
|
154
|
+
/**
|
|
155
|
+
* How long a call waits for the previous pin to thread before it is judged to
|
|
156
|
+
* be running beside it: the control server's hook-view fallback
|
|
157
|
+
* (`PROXY_REPORT_GRACE_MS`, 1.5 s after `PostToolUse`) plus slack. Bounded by
|
|
158
|
+
* the hook's own deadline as well.
|
|
159
|
+
*/
|
|
160
|
+
const PIN_THREAD_WAIT_MS = 1_750;
|
|
161
|
+
/**
|
|
162
|
+
* How long a call waits for a pin the host has *not* reported finished, once
|
|
163
|
+
* the control server is known to report completions: only the time for a
|
|
164
|
+
* `PostToolUse` racing this `PreToolUse` to land. Anything longer is a call
|
|
165
|
+
* running beside it, and waiting on that could stall the turn for minutes.
|
|
166
|
+
*/
|
|
167
|
+
const HOST_RACE_MS = 200;
|
|
47
168
|
/** How many step verdicts one report carries. A chain longer than this is not
|
|
48
169
|
* a chain anybody is reading step by step, and the report has to stay small. */
|
|
49
170
|
const MAX_STEP_RESULTS = 200;
|
|
@@ -52,14 +173,78 @@ const MAX_STEP_ERROR_CHARS = 500;
|
|
|
52
173
|
export class ReplayController {
|
|
53
174
|
enabled;
|
|
54
175
|
work = new ProxyWorkQueue();
|
|
176
|
+
/** The budgets every path runs on, a direct run's held under {@link DIRECT_RUN_CEILING_MS}. */
|
|
55
177
|
budgets;
|
|
178
|
+
/**
|
|
179
|
+
* `bir replay`'s derivation wait and plan budget: what the environment asked
|
|
180
|
+
* for, unclamped — no MCP client times that run out (see
|
|
181
|
+
* {@link DIRECT_RUN_CEILING_MS}).
|
|
182
|
+
*/
|
|
183
|
+
adHocBudgets;
|
|
56
184
|
intentMatch;
|
|
57
185
|
opts;
|
|
186
|
+
/** See {@link ReplayOptions.recordedFallback}. */
|
|
187
|
+
recordedFallback;
|
|
188
|
+
/** Calls waiting for a plan's pins to change (see `awaitPins`). */
|
|
189
|
+
pinWaiters = new WeakMap();
|
|
190
|
+
/**
|
|
191
|
+
* Set the first time the control server reports a host completion
|
|
192
|
+
* ({@link hostCompleted}). From then on a pin the host has not finished is a
|
|
193
|
+
* call really running beside the next one, and nothing waits for it.
|
|
194
|
+
*/
|
|
195
|
+
hostSignals = false;
|
|
58
196
|
constructor(opts) {
|
|
59
197
|
this.opts = opts;
|
|
60
198
|
this.enabled = opts.enabled;
|
|
61
|
-
|
|
199
|
+
const asked = {
|
|
200
|
+
...DEFAULT_BUDGETS,
|
|
201
|
+
planMs: positiveEnv("BIR_REPLAY_PLAN_BUDGET_MS") ?? DEFAULT_BUDGETS.planMs,
|
|
202
|
+
hookDeriveMs: positiveEnv("BIR_HOOK_DERIVE_BUDGET_MS") ?? DEFAULT_BUDGETS.hookDeriveMs,
|
|
203
|
+
preToolMs: positiveEnv("BIR_PRETOOL_REPLAY_BUDGET_MS") ?? DEFAULT_BUDGETS.preToolMs,
|
|
204
|
+
...(opts.budgets ?? {}),
|
|
205
|
+
};
|
|
206
|
+
// A PreToolUse's replay work past the route's deadline is steps started
|
|
207
|
+
// after the host was told to go ahead without them: held under it.
|
|
208
|
+
if (Number.isNaN(asked.preToolMs))
|
|
209
|
+
asked.preToolMs = DEFAULT_BUDGETS.preToolMs;
|
|
210
|
+
if (asked.preToolMs > MAX_PRETOOL_REPLAY_MS) {
|
|
211
|
+
const from = asked.preToolMs;
|
|
212
|
+
asked.preToolMs = MAX_PRETOOL_REPLAY_MS;
|
|
213
|
+
logLine("replay.budget_clamped", {
|
|
214
|
+
budget: "BIR_PRETOOL_REPLAY_BUDGET_MS",
|
|
215
|
+
from,
|
|
216
|
+
to: MAX_PRETOOL_REPLAY_MS,
|
|
217
|
+
why: `PreToolUse is answered by ${PRETOOL_ROUTE_DEADLINE_MS} ms; a divergence step started later ` +
|
|
218
|
+
"would run after the model was told to carry on itself",
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
// A budget set past what a direct run can keep is brought back under it,
|
|
222
|
+
// and said so once: the alternative is a run its client has already
|
|
223
|
+
// abandoned, with its steps done and its answer thrown away.
|
|
224
|
+
const logicAsked = positiveEnv("BIR_LOGIC_TIMEOUT_MS");
|
|
225
|
+
if (logicAsked !== undefined && logicAsked > MAX_LOGIC_TIMEOUT_MS) {
|
|
226
|
+
logLine("replay.budget_clamped", {
|
|
227
|
+
budget: "BIR_LOGIC_TIMEOUT_MS",
|
|
228
|
+
from: logicAsked,
|
|
229
|
+
to: MAX_LOGIC_TIMEOUT_MS,
|
|
230
|
+
why: "a logic body holds up every hook of the project while it runs",
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
const direct = clampDirectRun(asked);
|
|
234
|
+
if (direct.clamped) {
|
|
235
|
+
logLine("replay.budget_clamped", {
|
|
236
|
+
deriveMs: direct.clamped.deriveMs?.join("→"),
|
|
237
|
+
planMs: direct.clamped.planMs?.join("→"),
|
|
238
|
+
ceilingMs: DIRECT_RUN_CEILING_MS,
|
|
239
|
+
why: "a direct run must answer before run_scenario's client gives up on it (290 s); " +
|
|
240
|
+
"`bir replay` keeps the budget asked for",
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
this.budgets = direct.budgets;
|
|
244
|
+
this.adHocBudgets = { deriveMs: asked.deriveMs, planMs: asked.planMs };
|
|
62
245
|
this.intentMatch = { ...DEFAULT_INTENT_MATCH, ...(opts.intentMatch ?? {}) };
|
|
246
|
+
this.recordedFallback =
|
|
247
|
+
opts.recordedFallback ?? process.env.BIR_REPLAY_RECORDED_OUTPUTS === "1";
|
|
63
248
|
}
|
|
64
249
|
/**
|
|
65
250
|
* Whether a handed-out segment may actually arm (segmented.md R-OUT-10).
|
|
@@ -83,6 +268,13 @@ export class ReplayController {
|
|
|
83
268
|
* covers both "no match" and "too slow", which are the same thing from here.
|
|
84
269
|
*/
|
|
85
270
|
async awaitMatch(pending) {
|
|
271
|
+
// The match is the service's answer, so this is a service wait on the hot
|
|
272
|
+
// path (C1): none at all while the service is cooling down. An overrun is
|
|
273
|
+
// not reported from here — the recorder's own request reports its timeout
|
|
274
|
+
// or failure when it settles, and it is the one that knows which it was.
|
|
275
|
+
pending.catch(() => undefined);
|
|
276
|
+
if (!serviceHealth.canWait())
|
|
277
|
+
return null;
|
|
86
278
|
let timer;
|
|
87
279
|
const budget = new Promise((resolve) => {
|
|
88
280
|
timer = setTimeout(() => resolve(null), this.budgets.matchMs);
|
|
@@ -235,7 +427,30 @@ export class ReplayController {
|
|
|
235
427
|
const mode = modeFor(planned.map((e) => e.step), wrapped, this.opts.allowServers);
|
|
236
428
|
if (mode === "none")
|
|
237
429
|
return decline("no step is executable", "coverage");
|
|
238
|
-
|
|
430
|
+
// Tool results feed derivation as context, and they are whatever the agent
|
|
431
|
+
// last read — a fetched page, an email. A plan that hands its parameters
|
|
432
|
+
// to a shell or a database must not take them from there: a value lifted
|
|
433
|
+
// from a page and templated into a command is the page running a command
|
|
434
|
+
// (safety.ts). Every `command` sink, not only a shell: `mcp__pg__query`
|
|
435
|
+
// runs its input unattended just the same. A tool merely *named* for a
|
|
436
|
+
// query (a search `*_query`) loses the results too, and a scenario whose
|
|
437
|
+
// target only a result names declines more often — one replay, never a
|
|
438
|
+
// command nobody asked for.
|
|
439
|
+
let deriveCtx = ctx;
|
|
440
|
+
if (ctx.recentResults?.length && sinkOfPlan(planned) === "command") {
|
|
441
|
+
deriveCtx = { liveCall: ctx.liveCall };
|
|
442
|
+
logLine("replay.derive_context", {
|
|
443
|
+
scenario: scenario.id,
|
|
444
|
+
shell: hasShellStep(planned) || undefined,
|
|
445
|
+
why: "a step's input is run by a shell or a database (or its tool is named for one, such as *_query) — " +
|
|
446
|
+
"recent tool results are not read for its parameters, so a target only they name is not found",
|
|
447
|
+
});
|
|
448
|
+
}
|
|
449
|
+
const derivation = this.startDerivation(scenario, prompt, state, deriveCtx);
|
|
450
|
+
// Whether the derivation has answered yet — read, never waited on, when the
|
|
451
|
+
// directive is written (below).
|
|
452
|
+
let deriveAnswered = false;
|
|
453
|
+
derivation.then(() => (deriveAnswered = true), () => (deriveAnswered = true));
|
|
239
454
|
const params = derivation.then((r) => r.params);
|
|
240
455
|
// `params` is awaited by the plan, and by gate 7 below when there is a
|
|
241
456
|
// target. Without this, the losing race in gate 7 leaves `derivation`
|
|
@@ -254,22 +469,22 @@ export class ReplayController {
|
|
|
254
469
|
if (hasTargets(scenario.paramsObject)) {
|
|
255
470
|
// Of the targets this answer left unnamed, the ones this plan could act
|
|
256
471
|
// on the recorded value of. Empty means the chain finds them for itself.
|
|
257
|
-
const blocking = (missing) => blockingTargets(missing, scenario.paramsObject, planned);
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
//
|
|
264
|
-
|
|
472
|
+
const blocking = (missing) => blockingTargets(missing, scenario.paramsObject, planned, scenario.paramsLogic);
|
|
473
|
+
const budget = Math.min(this.budgets.deriveMs, this.budgets.hookDeriveMs);
|
|
474
|
+
const result = await this.waitDerivation(state, derivation, budget);
|
|
475
|
+
if (result === undefined) {
|
|
476
|
+
// Out of budget, or the service is cooling down and was not waited on.
|
|
477
|
+
// The turn has not said what this task is to act on, so every target is
|
|
478
|
+
// unnamed (R-PARAM-3).
|
|
479
|
+
const error = `derivation did not answer within ${budget}ms`;
|
|
265
480
|
const blocked = blocking(targetKeys(scenario.paramsObject));
|
|
266
481
|
if (blocked.length > 0) {
|
|
267
|
-
return decline(`target ${blocked[0]} not found — ${
|
|
482
|
+
return decline(`target ${blocked[0]} not found — ${error}`, "missing_target");
|
|
268
483
|
}
|
|
269
484
|
logLine("replay.targets_unread", {
|
|
270
485
|
scenario: scenario.id,
|
|
271
486
|
why: "the derivation did not answer, and this chain takes no target from the caller",
|
|
272
|
-
error
|
|
487
|
+
error,
|
|
273
488
|
});
|
|
274
489
|
}
|
|
275
490
|
if (result && !result.derived) {
|
|
@@ -301,6 +516,26 @@ export class ReplayController {
|
|
|
301
516
|
why: "the chain computes these from its own steps; none is read from the caller",
|
|
302
517
|
});
|
|
303
518
|
}
|
|
519
|
+
// The values are in hand: one this plan must not template into its
|
|
520
|
+
// command (safety.ts) declines here, before any directive is injected,
|
|
521
|
+
// instead of arming a plan whose first call can only be denied with a
|
|
522
|
+
// hand-over — a denied call and a model round trip, for nothing. The
|
|
523
|
+
// same check still runs where the values settle later (`readyFor`).
|
|
524
|
+
// It travels as `missing_target`: the value read for a target could not
|
|
525
|
+
// be used, and no new code reaches a service that may not know it.
|
|
526
|
+
const sink = sinkOfPlan(planned);
|
|
527
|
+
if (result?.derived && sink) {
|
|
528
|
+
const bad = unsafeParameter(result.params, scenario.paramsObject, sink);
|
|
529
|
+
if (bad) {
|
|
530
|
+
logLine("replay.unsafe_param", {
|
|
531
|
+
scenario: scenario.id,
|
|
532
|
+
param: bad.key,
|
|
533
|
+
sink,
|
|
534
|
+
why: "a value read for this request would be templated into a command unattended — declined before arming",
|
|
535
|
+
});
|
|
536
|
+
return decline(`the value read for ${bad.key} is not safe to put into a command unattended`, "missing_target");
|
|
537
|
+
}
|
|
538
|
+
}
|
|
304
539
|
}
|
|
305
540
|
// A plan armed by intent runs inside a task the agent is already doing, so
|
|
306
541
|
// its directive and its bundles say "continue", never "answer the user"
|
|
@@ -313,6 +548,21 @@ export class ReplayController {
|
|
|
313
548
|
entries,
|
|
314
549
|
subTask: armedBy === "intent",
|
|
315
550
|
});
|
|
551
|
+
if (mode === "steer") {
|
|
552
|
+
// A steer directive shows the arguments the steps will run with, so the
|
|
553
|
+
// call the model writes is one the pin guard accepts (plan.ts,
|
|
554
|
+
// `steeringDirective`). That needs the parameters settled — which costs
|
|
555
|
+
// nothing when the derivation has already answered: gate 7 waited for
|
|
556
|
+
// it, or there was nothing to derive. One turn of the loop lets an answer
|
|
557
|
+
// that is already in hand through; one still on its way is not waited
|
|
558
|
+
// for here, and the directive simply leaves the arguments to the system,
|
|
559
|
+
// as it always did. A `paramsLogic` that throws is not judged here: the
|
|
560
|
+
// first call that needs the parameters meets the same failure, as before.
|
|
561
|
+
if (!deriveAnswered)
|
|
562
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
563
|
+
if (deriveAnswered)
|
|
564
|
+
await state.plan.ready().catch(() => undefined);
|
|
565
|
+
}
|
|
316
566
|
state.mode = mode;
|
|
317
567
|
state.stepsPlanned = entries.length;
|
|
318
568
|
// Why the plan stops, when a call is what stops it: the note names the
|
|
@@ -321,7 +571,9 @@ export class ReplayController {
|
|
|
321
571
|
typeof stopAt === "number" && flat.stopAt !== undefined && stopAt <= flat.stopAt
|
|
322
572
|
? flat.stopReason
|
|
323
573
|
: undefined;
|
|
324
|
-
|
|
574
|
+
// Recorded outputs are read only when they may stand in (opt-in), so a
|
|
575
|
+
// plan never fetches a run it will not show.
|
|
576
|
+
if (this.recordedFallback && this.opts.authUrl && this.opts.authToken) {
|
|
325
577
|
state.sourceRun = new SourceRunOutputs({
|
|
326
578
|
baseUrl: this.opts.authUrl,
|
|
327
579
|
runId: scenario.runId,
|
|
@@ -351,12 +603,75 @@ export class ReplayController {
|
|
|
351
603
|
});
|
|
352
604
|
return state;
|
|
353
605
|
}
|
|
354
|
-
/**
|
|
355
|
-
|
|
606
|
+
/**
|
|
607
|
+
* The directive to inject via `additionalContext`, or undefined when declined.
|
|
608
|
+
*
|
|
609
|
+
* A steer directive previews each step's arguments (plan.ts,
|
|
610
|
+
* `previewArguments`), which can take up to {@link PREVIEW_BUDGET_MS}.
|
|
611
|
+
* `previewMs` — the time the caller's hook has left — lowers that bound,
|
|
612
|
+
* never raises it; under ~10 ms nothing is previewed and the directive keeps
|
|
613
|
+
* its old wording.
|
|
614
|
+
*/
|
|
615
|
+
directiveFor(state, opts = {}) {
|
|
616
|
+
const asked = opts.previewMs;
|
|
356
617
|
return state.plan?.steeringDirective(DIRECT_TOOL_NAME, {
|
|
357
618
|
subTask: state.armedBy === "intent",
|
|
619
|
+
// NaN is a caller's arithmetic gone wrong: no time known, no preview.
|
|
620
|
+
previewBudgetMs: typeof asked === "number"
|
|
621
|
+
? Math.max(0, Math.min(Number.isNaN(asked) ? 0 : asked, PREVIEW_BUDGET_MS))
|
|
622
|
+
: undefined,
|
|
358
623
|
});
|
|
359
624
|
}
|
|
625
|
+
/**
|
|
626
|
+
* Give a steer plan's parameters a short, bounded chance to settle before its
|
|
627
|
+
* directive is written, so {@link directiveFor} can show the arguments the
|
|
628
|
+
* steps run with. Resolves true when they are settled.
|
|
629
|
+
*
|
|
630
|
+
* Without it a derivation the service answers — nearly every one, since a
|
|
631
|
+
* signed-in runner has no key of its own — has never answered by the time
|
|
632
|
+
* `arm` returns (it waits one turn of the loop, on purpose), the directive
|
|
633
|
+
* says "the system supplies the exact arguments", the model writes a guessed
|
|
634
|
+
* command, the pin guard declines it, and the plan retires having run
|
|
635
|
+
* nothing.
|
|
636
|
+
*
|
|
637
|
+
* Only for the steps the pin guard holds to their exact shape — a shell
|
|
638
|
+
* command, a file write (control/pin-guard.ts) — and never longer than
|
|
639
|
+
* {@link PREVIEW_SETTLE_MAX_MS} or `budgetMs`, whichever is less: the caller
|
|
640
|
+
* passes what its prompt hook has left, less what the preview itself needs.
|
|
641
|
+
* Not at all while the service is cooling down. A late answer is not an
|
|
642
|
+
* overrun of anything, so it marks nothing against the service. `arm`
|
|
643
|
+
* itself still never waits (integ-replay R2).
|
|
644
|
+
*/
|
|
645
|
+
async settleForPreview(state, budgetMs) {
|
|
646
|
+
const plan = state.plan;
|
|
647
|
+
if (!plan || state.retired || state.mode !== "steer")
|
|
648
|
+
return false;
|
|
649
|
+
if (plan.parametersSettled)
|
|
650
|
+
return true;
|
|
651
|
+
const planned = plan.allEntries().slice(0, plan.stepCount);
|
|
652
|
+
if (!planned.some((e) => pinnedToShape(e.step.toolName)))
|
|
653
|
+
return false;
|
|
654
|
+
if (state.deriveOnService && !serviceHealth.canWait())
|
|
655
|
+
return false;
|
|
656
|
+
const ms = Math.min(Number.isFinite(budgetMs) ? budgetMs : 0, PREVIEW_SETTLE_MAX_MS);
|
|
657
|
+
if (ms <= 0)
|
|
658
|
+
return false;
|
|
659
|
+
let timer;
|
|
660
|
+
try {
|
|
661
|
+
await Promise.race([
|
|
662
|
+
plan.ready().catch(() => undefined),
|
|
663
|
+
new Promise((resolve) => {
|
|
664
|
+
timer = setTimeout(resolve, ms);
|
|
665
|
+
timer.unref?.();
|
|
666
|
+
}),
|
|
667
|
+
]);
|
|
668
|
+
}
|
|
669
|
+
finally {
|
|
670
|
+
if (timer)
|
|
671
|
+
clearTimeout(timer);
|
|
672
|
+
}
|
|
673
|
+
return plan.parametersSettled;
|
|
674
|
+
}
|
|
360
675
|
/**
|
|
361
676
|
* A hand-over note not yet delivered, claimed for delivery (fallbk.md D4).
|
|
362
677
|
* Returns it once and marks it delivered; undefined when there is none.
|
|
@@ -379,9 +694,12 @@ export class ReplayController {
|
|
|
379
694
|
*/
|
|
380
695
|
deliverInstead(text, toolName) {
|
|
381
696
|
if (toolName === "Bash") {
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
697
|
+
// A short command, whatever the text's size (bash-channel.ts): a long
|
|
698
|
+
// heredoc is cut by the shell, and a long output by the host.
|
|
699
|
+
const command = bashCommandFor(truncate(text, BASH_CHANNEL_MAX));
|
|
700
|
+
if (command)
|
|
701
|
+
return { kind: "bash", command };
|
|
702
|
+
// The text could not be staged: the denial reason still carries it.
|
|
385
703
|
}
|
|
386
704
|
return { kind: "deny", reason: text };
|
|
387
705
|
}
|
|
@@ -391,23 +709,109 @@ export class ReplayController {
|
|
|
391
709
|
* Three tiers (§8): pin the expected call; on divergence execute the remainder
|
|
392
710
|
* and hand it back through the best channel available; if even that fails,
|
|
393
711
|
* abort to an ordinary turn.
|
|
712
|
+
*
|
|
713
|
+
* `hook` is optional and only ever tightens the work: `answerBy` lowers the
|
|
714
|
+
* deadline to the caller's own, and `signal` stops a divergence from
|
|
715
|
+
* starting another step once the host has been answered without it.
|
|
394
716
|
*/
|
|
395
|
-
async preTool(state, toolName, toolUseId
|
|
717
|
+
async preTool(state, toolName, toolUseId,
|
|
718
|
+
/** The call's `tool_input`: whether a `Bash` call is only `bir` is read from it (D13). */
|
|
719
|
+
toolInput, hook = {}) {
|
|
396
720
|
const plan = state.plan;
|
|
397
721
|
if (!plan || state.retired)
|
|
398
722
|
return { kind: "passthrough" };
|
|
399
723
|
// The delivery vehicle for a direct plan is not a step in it.
|
|
400
|
-
if (this.isDirectTool(toolName))
|
|
724
|
+
if (this.isDirectTool(toolName)) {
|
|
725
|
+
state.directRequested = true;
|
|
401
726
|
return { kind: "passthrough" };
|
|
402
|
-
try {
|
|
403
|
-
await this.withBudget(plan.ready(), this.budgets.deriveMs, "derivation");
|
|
404
727
|
}
|
|
405
|
-
|
|
406
|
-
|
|
728
|
+
// A direct run is executing this plan right now: another call beside it is
|
|
729
|
+
// the model's own, and a divergence would drive the same cursor twice.
|
|
730
|
+
if (state.running)
|
|
731
|
+
return { kind: "passthrough" };
|
|
732
|
+
// The model asked for `run_scenario` and the run never started. The usual
|
|
733
|
+
// reason is the user refusing that call — and executing the plan anyway,
|
|
734
|
+
// through a divergence, would override that refusal with the very steps it
|
|
735
|
+
// refused. The model does the task; the plan stays idle.
|
|
736
|
+
if (state.mode === "direct" && state.directRequested) {
|
|
737
|
+
logDetail("replay.direct_not_run", {
|
|
407
738
|
scenario: state.scenarioId ?? undefined,
|
|
408
|
-
|
|
409
|
-
|
|
739
|
+
tool: toolName,
|
|
740
|
+
why: "run_scenario was called but never ran (refused, or it never reached bir) — nothing is executed in its place",
|
|
741
|
+
});
|
|
742
|
+
return { kind: "passthrough" };
|
|
743
|
+
}
|
|
744
|
+
// HOUSEKEEPING IS NOT DIVERGENCE. See `record/housekeeping.ts`. Asked
|
|
745
|
+
// before the pin, not after it: a `Bash` step is pinned by name, and a
|
|
746
|
+
// `bir scenario show` the model runs while a Bash step is expected must
|
|
747
|
+
// run as `bir`, never be swapped for the plan's command (D13). And before
|
|
748
|
+
// the wait for the parameters, which a `ToolSearch` has no use for.
|
|
749
|
+
if (isHousekeepingCall(toolName, toolInput)) {
|
|
750
|
+
logDetail("replay.housekeeping", {
|
|
751
|
+
tool: toolName,
|
|
752
|
+
step: `${plan.currentStepIndex}/${plan.stepCount}`,
|
|
753
|
+
why: "host bookkeeping, not task work — plan stays armed",
|
|
410
754
|
});
|
|
755
|
+
return { kind: "passthrough" };
|
|
756
|
+
}
|
|
757
|
+
// Everything below — the wait for the parameters, and a divergence that
|
|
758
|
+
// executes the rest of the plan — happens inside this one hook, which the
|
|
759
|
+
// host waits on for a bounded time. One deadline covers all of it — and
|
|
760
|
+
// never reaches past the caller's own answer (`hook.answerBy`), less the
|
|
761
|
+
// last step's minimum and a margin.
|
|
762
|
+
// The parameters may be waited on until the margin before that answer; a
|
|
763
|
+
// step may only *start* until its minimum before it as well.
|
|
764
|
+
const answerBy = typeof hook.answerBy === "number" && Number.isFinite(hook.answerBy)
|
|
765
|
+
? hook.answerBy - PRETOOL_MARGIN_MS
|
|
766
|
+
: Number.POSITIVE_INFINITY;
|
|
767
|
+
const deadline = Math.min(Date.now() + this.budgets.preToolMs, answerBy - MIN_STEP_MS);
|
|
768
|
+
const ready = await this.readyFor(state, plan, Math.max(0, Math.min(this.budgets.deriveMs, this.budgets.hookDeriveMs, answerBy - Date.now())));
|
|
769
|
+
// Another call may have retired the plan while this one waited — or the
|
|
770
|
+
// host may have been answered without this call's action.
|
|
771
|
+
if (state.retired || hook.signal?.aborted)
|
|
772
|
+
return { kind: "passthrough" };
|
|
773
|
+
if (!ready.ok) {
|
|
774
|
+
// Nothing may run on parameters that are unknown, unprocessed or unsafe.
|
|
775
|
+
// The model does the task itself — told so in place of this call.
|
|
776
|
+
const step = plan.currentStep();
|
|
777
|
+
if (step && ready.blame) {
|
|
778
|
+
this.recordStep(state, { step, outcome: "failed", stage: ready.stage, error: ready.error });
|
|
779
|
+
}
|
|
780
|
+
this.handOver(state, {
|
|
781
|
+
kind: "step_failed",
|
|
782
|
+
step,
|
|
783
|
+
entry: plan.currentEntry(),
|
|
784
|
+
position: plan.currentStepIndex,
|
|
785
|
+
error: ready.error,
|
|
786
|
+
delivery: "above",
|
|
787
|
+
stopReason: "not_run",
|
|
788
|
+
});
|
|
789
|
+
const note = this.takeNote(state);
|
|
790
|
+
return note ? this.deliverInstead(note, toolName) : { kind: "abort" };
|
|
791
|
+
}
|
|
792
|
+
// ONE CALL AT A TIME. A pin moves the cursor only when its output threads,
|
|
793
|
+
// which for a wrapped step is the proxy's report — landing either side of
|
|
794
|
+
// `PostToolUse`, or up to the control server's 1.5 s fallback after it
|
|
795
|
+
// when no report comes (a standalone proxy). A call arriving before that
|
|
796
|
+
// would be pinned to the very same step (it would run twice,
|
|
797
|
+
// auto-approved), or would diverge and re-execute the step still in
|
|
798
|
+
// flight. So it first waits, briefly and within this hook's deadline, for
|
|
799
|
+
// the pin to thread: the model's *next* call after a finished one is the
|
|
800
|
+
// ordinary sequential case, and must be judged against the moved cursor —
|
|
801
|
+
// never handed over as "concurrent". Only a pin still out after that is a
|
|
802
|
+
// call really running beside this one, and then the plan hands over, and
|
|
803
|
+
// this call carries the note that says so.
|
|
804
|
+
if (state.pinned.size > 0 && !state.pinned.has(toolUseId)) {
|
|
805
|
+
// With the plan's last step in flight there is nothing left to pin twice:
|
|
806
|
+
// this call is the model's own whatever that step does, so it never waits.
|
|
807
|
+
const lastInFlight = [...state.pinned.values()].some((p) => p.stepIndex + 1 >= plan.stepCount);
|
|
808
|
+
if (!lastInFlight)
|
|
809
|
+
await this.awaitPins(state, toolUseId, deadline);
|
|
810
|
+
if (state.retired || hook.signal?.aborted)
|
|
811
|
+
return { kind: "passthrough" };
|
|
812
|
+
if (state.pinned.size > 0 && !state.pinned.has(toolUseId)) {
|
|
813
|
+
return this.concurrentCall(state, plan, toolName);
|
|
814
|
+
}
|
|
411
815
|
}
|
|
412
816
|
// A direct plan does not steer individual calls: the model was asked for one
|
|
413
817
|
// tool call and made a different one. That is divergence.
|
|
@@ -460,16 +864,140 @@ export class ReplayController {
|
|
|
460
864
|
return { kind: "abort" };
|
|
461
865
|
}
|
|
462
866
|
}
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
867
|
+
return await this.diverge(state, toolName, `expected ${expected ?? "no more tools"}, model called ${toolName}`, deadline, hook.signal);
|
|
868
|
+
}
|
|
869
|
+
/**
|
|
870
|
+
* A call arrived while another pinned call's output has not threaded yet
|
|
871
|
+
* (see `preTool`). The plan stops here rather than pin a step twice.
|
|
872
|
+
*/
|
|
873
|
+
concurrentCall(state, plan, toolName) {
|
|
874
|
+
const outstanding = [...state.pinned.values()][0];
|
|
875
|
+
const next = (outstanding?.stepIndex ?? plan.currentStepIndex) + 1;
|
|
876
|
+
if (next >= plan.stepCount) {
|
|
877
|
+
// The call in flight is the plan's last step: nothing is left to pin
|
|
878
|
+
// twice, and this call is the model's own.
|
|
879
|
+
logDetail("replay.concurrent_call", {
|
|
880
|
+
scenario: state.scenarioId ?? undefined,
|
|
466
881
|
tool: toolName,
|
|
467
|
-
|
|
468
|
-
why: "host bookkeeping, not task work — plan stays armed",
|
|
882
|
+
why: "the last step is in flight — this call is the model's own",
|
|
469
883
|
});
|
|
470
884
|
return { kind: "passthrough" };
|
|
471
885
|
}
|
|
472
|
-
|
|
886
|
+
logLine("replay.concurrent_call", {
|
|
887
|
+
scenario: state.scenarioId ?? undefined,
|
|
888
|
+
tool: toolName,
|
|
889
|
+
inFlight: outstanding ? `${outstanding.stepIndex}:${outstanding.toolName}` : undefined,
|
|
890
|
+
why: "a second call arrived while the pinned step was still running — a plan runs one call at a time, so the rest is handed over",
|
|
891
|
+
});
|
|
892
|
+
// No outcome of its own: the model, not the plan, does the rest, so this
|
|
893
|
+
// is no divergence saving. `handOver` books `fell_back` if earlier steps
|
|
894
|
+
// threaded and `failed` if none did — and the pins still in flight stay
|
|
895
|
+
// readable, so a step among them that finishes is still counted.
|
|
896
|
+
//
|
|
897
|
+
// The pin in flight is *not* counted as run. It may be running, or it may
|
|
898
|
+
// never run at all — the user's own deny rule or PreToolUse hook blocked
|
|
899
|
+
// it, and no PostToolUse will ever come. The note says exactly that,
|
|
900
|
+
// rather than "Steps 1–N ran … do not repeat them" about a call that may
|
|
901
|
+
// not have happened.
|
|
902
|
+
const inFlight = new Map(state.pinned);
|
|
903
|
+
this.handOver(state, {
|
|
904
|
+
kind: "step_failed",
|
|
905
|
+
step: plan.allSteps()[next],
|
|
906
|
+
entry: plan.allEntries()[next],
|
|
907
|
+
position: next,
|
|
908
|
+
delivery: "above",
|
|
909
|
+
stopReason: "concurrent",
|
|
910
|
+
ran: outstanding ? outstanding.stepIndex : plan.currentStepIndex,
|
|
911
|
+
inFlight: outstanding
|
|
912
|
+
? { stepIndex: outstanding.stepIndex, toolName: outstanding.toolName }
|
|
913
|
+
: undefined,
|
|
914
|
+
});
|
|
915
|
+
if (inFlight.size > 0)
|
|
916
|
+
state.lingering = inFlight;
|
|
917
|
+
const note = this.takeNote(state);
|
|
918
|
+
return note ? this.deliverInstead(note, toolName) : { kind: "passthrough" };
|
|
919
|
+
}
|
|
920
|
+
/**
|
|
921
|
+
* Wait until the plan's outstanding pins thread, or until waiting is no
|
|
922
|
+
* longer worth it (see `preTool`, ONE CALL AT A TIME).
|
|
923
|
+
*
|
|
924
|
+
* Always bounded by the hook's `deadline`. Within it:
|
|
925
|
+
* - every pin finished by the host ({@link hostCompleted}): until
|
|
926
|
+
* {@link PIN_THREAD_WAIT_MS} after the last of them finished — the
|
|
927
|
+
* server's fallback has threaded it by then, if anything will;
|
|
928
|
+
* - the server is known to report host completions and a pin is not
|
|
929
|
+
* finished: a call really running beside this one, which may run for
|
|
930
|
+
* minutes, so only {@link HOST_RACE_MS} in case its completion is racing
|
|
931
|
+
* this hook;
|
|
932
|
+
* - otherwise (a server that reports nothing): {@link PIN_THREAD_WAIT_MS}.
|
|
933
|
+
*/
|
|
934
|
+
async awaitPins(state, toolUseId, deadline) {
|
|
935
|
+
const startedAt = Date.now();
|
|
936
|
+
const outstanding = () => [...state.pinned.entries()].filter(([id]) => id !== toolUseId).map(([, pin]) => pin);
|
|
937
|
+
for (;;) {
|
|
938
|
+
if (state.retired)
|
|
939
|
+
return;
|
|
940
|
+
const pins = outstanding();
|
|
941
|
+
if (pins.length === 0)
|
|
942
|
+
return;
|
|
943
|
+
const hostDone = pins.map((p) => p.hostDoneAt);
|
|
944
|
+
const finished = hostDone.every((t) => t !== undefined);
|
|
945
|
+
const until = Math.min(deadline, finished
|
|
946
|
+
? Math.max(...hostDone) + PIN_THREAD_WAIT_MS
|
|
947
|
+
: startedAt + (this.hostSignals ? HOST_RACE_MS : PIN_THREAD_WAIT_MS));
|
|
948
|
+
const left = until - Date.now();
|
|
949
|
+
if (left <= 0) {
|
|
950
|
+
logDetail("replay.pin_wait", {
|
|
951
|
+
scenario: state.scenarioId ?? undefined,
|
|
952
|
+
waitedMs: Date.now() - startedAt,
|
|
953
|
+
why: "the previous pinned step has not threaded — judged a call beside it",
|
|
954
|
+
});
|
|
955
|
+
return;
|
|
956
|
+
}
|
|
957
|
+
await this.pinChange(state, left);
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
/** Resolves on the next change to a plan's pins, or after `ms`. */
|
|
961
|
+
pinChange(state, ms) {
|
|
962
|
+
return new Promise((resolve) => {
|
|
963
|
+
let waiters = this.pinWaiters.get(state);
|
|
964
|
+
if (!waiters) {
|
|
965
|
+
waiters = new Set();
|
|
966
|
+
this.pinWaiters.set(state, waiters);
|
|
967
|
+
}
|
|
968
|
+
const set = waiters;
|
|
969
|
+
const done = () => {
|
|
970
|
+
clearTimeout(timer);
|
|
971
|
+
set.delete(done);
|
|
972
|
+
resolve();
|
|
973
|
+
};
|
|
974
|
+
const timer = setTimeout(done, Math.max(0, ms));
|
|
975
|
+
timer.unref?.();
|
|
976
|
+
set.add(done);
|
|
977
|
+
});
|
|
978
|
+
}
|
|
979
|
+
/** Wake every call waiting on this plan's pins. */
|
|
980
|
+
notifyPins(state) {
|
|
981
|
+
const waiters = this.pinWaiters.get(state);
|
|
982
|
+
if (!waiters || waiters.size === 0)
|
|
983
|
+
return;
|
|
984
|
+
for (const wake of [...waiters])
|
|
985
|
+
wake();
|
|
986
|
+
}
|
|
987
|
+
/**
|
|
988
|
+
* The host finished a call (`PostToolUse` / `PostToolUseFailure`) — whether
|
|
989
|
+
* or not its output has threaded yet. The control server calls this for
|
|
990
|
+
* every pinned call; it is what tells the model's next, sequential call
|
|
991
|
+
* apart from one running beside a pin (see `awaitPins`). Optional: without
|
|
992
|
+
* it every such call waits the bounded {@link PIN_THREAD_WAIT_MS}.
|
|
993
|
+
*/
|
|
994
|
+
hostCompleted(state, toolUseId) {
|
|
995
|
+
const pin = state?.pinned.get(toolUseId);
|
|
996
|
+
this.hostSignals = true;
|
|
997
|
+
if (!state || !pin || pin.hostDoneAt !== undefined)
|
|
998
|
+
return;
|
|
999
|
+
pin.hostDoneAt = Date.now();
|
|
1000
|
+
this.notifyPins(state);
|
|
473
1001
|
}
|
|
474
1002
|
/**
|
|
475
1003
|
* `PostToolUse` for a call this plan pinned.
|
|
@@ -478,17 +1006,72 @@ export class ReplayController {
|
|
|
478
1006
|
* wrapped MCP step that is the proxy's whole `CallToolResult`, which is why the
|
|
479
1007
|
* server threads those from `/proxy/step` rather than from the hook (§7.2).
|
|
480
1008
|
* Returns true when the plan is now complete.
|
|
1009
|
+
*
|
|
1010
|
+
* `failed` is the host's verdict that the call failed — `PostToolUseFailure`,
|
|
1011
|
+
* or a proxy report with `isError` — with its error text when there is one.
|
|
1012
|
+
* It outranks anything read from `output`: a built-in's response is Claude
|
|
1013
|
+
* Code's rendering and is never judged here, so for an `Edit` whose
|
|
1014
|
+
* `old_string` moved since the recording this is the only word there is. A
|
|
1015
|
+
* step that failed must never thread as if it worked: the next step would be
|
|
1016
|
+
* pinned, and auto-approved, on top of it.
|
|
1017
|
+
*
|
|
1018
|
+
* Not every failure is one. A call its proxy *abandoned* — the upstream
|
|
1019
|
+
* closed, restarted or was cancelled mid-call, reported with `isError` and
|
|
1020
|
+
* no result — may well have done its work, and so may one the host says
|
|
1021
|
+
* timed out or was interrupted: the note says "may or may not have
|
|
1022
|
+
* completed", never "failed", or the model runs a write a second time
|
|
1023
|
+
* ({@link pinFailure}). `how` lets the caller say so outright.
|
|
481
1024
|
*/
|
|
482
|
-
postTool(state, toolUseId, output) {
|
|
1025
|
+
postTool(state, toolUseId, output, failed, how = {}) {
|
|
483
1026
|
const plan = state.plan;
|
|
1027
|
+
if (plan && state.retired) {
|
|
1028
|
+
this.lateThread(state, plan, toolUseId, output, failed, how);
|
|
1029
|
+
return false;
|
|
1030
|
+
}
|
|
484
1031
|
const pin = state.pinned.get(toolUseId);
|
|
485
1032
|
if (!plan || !pin || state.retired)
|
|
486
1033
|
return false;
|
|
487
1034
|
state.pinned.delete(toolUseId);
|
|
1035
|
+
// A call waiting for this pin (`awaitPins`) re-reads the cursor once this
|
|
1036
|
+
// synchronous method has finished moving it.
|
|
1037
|
+
this.notifyPins(state);
|
|
488
1038
|
// The pin's own step, not `currentStep()`: `applyOutput` moves the cursor, so
|
|
489
1039
|
// reading it after the call would judge the *next* step, and reading it
|
|
490
1040
|
// before assumes no other pin ever moved it.
|
|
491
1041
|
const step = plan.allSteps()[pin.stepIndex];
|
|
1042
|
+
// The output threads into the step at the cursor. One pin at a time
|
|
1043
|
+
// (`preTool`) keeps that the pin's own step; if it ever is not, threading
|
|
1044
|
+
// would feed one step's output to another's logic, so the plan stops.
|
|
1045
|
+
//
|
|
1046
|
+
// A proxy-threaded call that reports *no result* was sent and abandoned —
|
|
1047
|
+
// its upstream died mid-call (tool-error.ts, `isNoResult`). It may or may
|
|
1048
|
+
// not have done its work, and nothing may be pinned on top of it.
|
|
1049
|
+
const failure = pinFailure(pin, output, failed, how);
|
|
1050
|
+
const hostError = failure?.error;
|
|
1051
|
+
if (step && (hostError !== undefined || plan.currentStepIndex !== pin.stepIndex)) {
|
|
1052
|
+
const error = hostError ??
|
|
1053
|
+
`the plan's cursor is at step ${plan.currentStepIndex}, not the pinned step ${pin.stepIndex}`;
|
|
1054
|
+
const info = {
|
|
1055
|
+
step,
|
|
1056
|
+
outcome: "failed",
|
|
1057
|
+
stage: "tool_call",
|
|
1058
|
+
error,
|
|
1059
|
+
ms: Date.now() - pin.pinnedAt,
|
|
1060
|
+
};
|
|
1061
|
+
this.logStep(info);
|
|
1062
|
+
if (hostError !== undefined)
|
|
1063
|
+
this.recordStep(state, info);
|
|
1064
|
+
this.handOver(state, {
|
|
1065
|
+
kind: "step_failed",
|
|
1066
|
+
step,
|
|
1067
|
+
entry: plan.allEntries()[pin.stepIndex],
|
|
1068
|
+
position: pin.stepIndex,
|
|
1069
|
+
error,
|
|
1070
|
+
delivery: "above",
|
|
1071
|
+
stopReason: failure?.reason,
|
|
1072
|
+
});
|
|
1073
|
+
return false;
|
|
1074
|
+
}
|
|
492
1075
|
let derivedKeys = [];
|
|
493
1076
|
try {
|
|
494
1077
|
derivedKeys = plan.applyOutput(output);
|
|
@@ -599,38 +1182,132 @@ export class ReplayController {
|
|
|
599
1182
|
}
|
|
600
1183
|
return done;
|
|
601
1184
|
}
|
|
1185
|
+
/**
|
|
1186
|
+
* The result of a call pinned under a plan that has since handed over
|
|
1187
|
+
* because another call ran beside it (`concurrentCall`). Nothing threads —
|
|
1188
|
+
* the plan is retired and the model is doing the rest — but the step's
|
|
1189
|
+
* verdict is recorded, and a step that did its work is counted: the outcome
|
|
1190
|
+
* moves from `failed` to `fell_back` when it is the first that did.
|
|
1191
|
+
*/
|
|
1192
|
+
lateThread(state, plan, toolUseId, output, failed, how = {}) {
|
|
1193
|
+
const pin = state.lingering?.get(toolUseId);
|
|
1194
|
+
if (!pin)
|
|
1195
|
+
return;
|
|
1196
|
+
state.lingering.delete(toolUseId);
|
|
1197
|
+
const step = plan.allSteps()[pin.stepIndex];
|
|
1198
|
+
if (!step)
|
|
1199
|
+
return;
|
|
1200
|
+
const ms = Date.now() - pin.pinnedAt;
|
|
1201
|
+
// Read the same way as a live pin's (`pinFailure`). A call that was sent
|
|
1202
|
+
// and not seen through says so in its verdict: it may have done its work.
|
|
1203
|
+
const failure = pinFailure(pin, output, failed, how);
|
|
1204
|
+
const error = failure
|
|
1205
|
+
? failure.reason === "uncertain"
|
|
1206
|
+
? `${failure.error} — it was sent, and may or may not have completed`
|
|
1207
|
+
: failure.error
|
|
1208
|
+
: toolResultError(output);
|
|
1209
|
+
if (error !== undefined) {
|
|
1210
|
+
const info = { step, outcome: "failed", stage: "tool_call", error, ms };
|
|
1211
|
+
this.logStep(info);
|
|
1212
|
+
this.recordStep(state, info);
|
|
1213
|
+
return;
|
|
1214
|
+
}
|
|
1215
|
+
this.recordStep(state, { step, outcome: "executed", ms });
|
|
1216
|
+
state.stepsPinned += 1;
|
|
1217
|
+
this.upgrade(state, "fell_back");
|
|
1218
|
+
logDetail("replay.thread_late", {
|
|
1219
|
+
scenario: state.scenarioId ?? undefined,
|
|
1220
|
+
n: pin.stepIndex,
|
|
1221
|
+
why: "a step pinned before the hand-over finished — counted, not threaded",
|
|
1222
|
+
});
|
|
1223
|
+
}
|
|
602
1224
|
/** Whether a pinned call's output is threaded from the proxy's report (§7.2). */
|
|
603
1225
|
threadsFromProxy(state, toolUseId) {
|
|
604
|
-
|
|
1226
|
+
const pin = state.pinned.get(toolUseId) ?? state.lingering?.get(toolUseId);
|
|
1227
|
+
return pin?.reach === "direct";
|
|
605
1228
|
}
|
|
606
|
-
/**
|
|
1229
|
+
/**
|
|
1230
|
+
* True when this call belongs to the plan: pinned by it and not yet
|
|
1231
|
+
* threaded — including a pin still in flight when the plan handed over, so
|
|
1232
|
+
* its result still reaches `postTool` (see `lateThread`).
|
|
1233
|
+
*/
|
|
607
1234
|
isPinned(state, toolUseId) {
|
|
608
|
-
return state.pinned.has(toolUseId);
|
|
1235
|
+
return state.pinned.has(toolUseId) || state.lingering?.has(toolUseId) === true;
|
|
609
1236
|
}
|
|
610
1237
|
/**
|
|
611
1238
|
* `POST /scenario/run` — the whole of a `direct` plan, executed here.
|
|
612
1239
|
*
|
|
613
1240
|
* Every step runs through the proxy that already owns its upstream, so the
|
|
614
1241
|
* model spends nothing beyond the turn that reads the results.
|
|
1242
|
+
*
|
|
1243
|
+
* `signal` aborts when the caller is gone — `bir-scenario`'s request was
|
|
1244
|
+
* cancelled (Esc) or timed out — so no further step is started (C4). The
|
|
1245
|
+
* worst case this can take is `deriveMs + planMs` plus one step's
|
|
1246
|
+
* `MIN_STEP_MS`, six logic bodies at their full limit and a 5 s margin —
|
|
1247
|
+
* 120 s by default, more only for a plan budget an operator raised
|
|
1248
|
+
* ({@link clampDirectRun}) — which `bir-scenario`'s own timeout (290 s) must
|
|
1249
|
+
* exceed, and which is held under {@link DIRECT_RUN_CEILING_MS} whatever the
|
|
1250
|
+
* environment asks for. That bound counts every body at its limit; a run
|
|
1251
|
+
* that does not stall ends well inside its plan budget.
|
|
1252
|
+
*
|
|
1253
|
+
* One run per plan: a second call while the first is running gets the first
|
|
1254
|
+
* one's answer, never a second execution of the same steps.
|
|
615
1255
|
*/
|
|
616
|
-
async runArmed(state) {
|
|
1256
|
+
async runArmed(state, signal) {
|
|
617
1257
|
const plan = state.plan;
|
|
618
1258
|
if (!plan || state.retired)
|
|
619
|
-
return { ok: false, why:
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
logLine("replay.
|
|
1259
|
+
return { ok: false, why: RUN_NO_PLAN };
|
|
1260
|
+
if (state.running)
|
|
1261
|
+
return state.running;
|
|
1262
|
+
const running = this.runArmedOnce(state, plan, signal).catch((err) => {
|
|
1263
|
+
// Never a throw at a hook: a broken run is the model's task again.
|
|
1264
|
+
logLine("replay.failed", {
|
|
625
1265
|
scenario: state.scenarioId ?? undefined,
|
|
626
|
-
why: "
|
|
1266
|
+
why: "the run threw — continuing as a normal run",
|
|
627
1267
|
error: errText(err),
|
|
628
1268
|
});
|
|
1269
|
+
this.retire(state, "failed");
|
|
1270
|
+
return { ok: false, why: "the scenario could not be run" };
|
|
1271
|
+
});
|
|
1272
|
+
state.running = running;
|
|
1273
|
+
return running;
|
|
1274
|
+
}
|
|
1275
|
+
async runArmedOnce(state, plan, signal) {
|
|
1276
|
+
// `/scenario/run` has no hook timeout, so this is the one place the full
|
|
1277
|
+
// derivation budget is waited — and still not while the service is
|
|
1278
|
+
// cooling down.
|
|
1279
|
+
const ready = await this.readyFor(state, plan, this.budgets.deriveMs);
|
|
1280
|
+
if (state.retired)
|
|
1281
|
+
return { ok: false, why: RUN_NO_PLAN };
|
|
1282
|
+
if (!ready.ok) {
|
|
1283
|
+
// Nothing may run on parameters that are unknown, unprocessed or unsafe.
|
|
1284
|
+
// The tool answers with an error that says so, which is what a model
|
|
1285
|
+
// reacts to best: it does the task itself.
|
|
1286
|
+
const step = plan.currentStep();
|
|
1287
|
+
if (step && ready.blame) {
|
|
1288
|
+
this.recordStep(state, {
|
|
1289
|
+
step,
|
|
1290
|
+
entry: plan.currentEntry(),
|
|
1291
|
+
outcome: "failed",
|
|
1292
|
+
stage: ready.stage,
|
|
1293
|
+
error: ready.error,
|
|
1294
|
+
});
|
|
1295
|
+
}
|
|
1296
|
+
this.handOver(state, {
|
|
1297
|
+
kind: "step_failed",
|
|
1298
|
+
step,
|
|
1299
|
+
entry: plan.currentEntry(),
|
|
1300
|
+
position: plan.currentStepIndex,
|
|
1301
|
+
error: ready.error,
|
|
1302
|
+
delivery: "none",
|
|
1303
|
+
stopReason: "not_run",
|
|
1304
|
+
});
|
|
1305
|
+
return handedOver(this.takeNote(state) ?? state.handover?.note) ?? { ok: false, why: ready.error };
|
|
629
1306
|
}
|
|
630
1307
|
const deadline = Date.now() + this.budgets.planMs;
|
|
631
1308
|
let result;
|
|
632
1309
|
try {
|
|
633
|
-
result = await plan.runToCompletion(this.executeStep(state
|
|
1310
|
+
result = await plan.runToCompletion(this.executeStep(state, { deadline }), this.recordedSource(state), this.observeStep(state), deadline, MAX_REPLAY_REASON, { stopOnFailure: true, recordedFallback: this.recordedFallback, signal });
|
|
634
1311
|
}
|
|
635
1312
|
catch (err) {
|
|
636
1313
|
// The scenario's own logic failed. Retire and let the model do the work.
|
|
@@ -642,15 +1319,33 @@ export class ReplayController {
|
|
|
642
1319
|
this.retire(state, "failed");
|
|
643
1320
|
return { ok: false, why: "not_ready" };
|
|
644
1321
|
}
|
|
645
|
-
// The plan stopped part-way (fallbk.md §Runner 3): a step broke
|
|
646
|
-
// next one is parked
|
|
647
|
-
//
|
|
1322
|
+
// The plan stopped part-way (fallbk.md §Runner 3): a step broke or produced
|
|
1323
|
+
// no result, the next one is parked, the budget ran out, or the caller left.
|
|
1324
|
+
// The steps that ran are real, so their bundle goes to the model — behind a
|
|
1325
|
+
// note saying the rest of the task is its own.
|
|
648
1326
|
if (result.stopped) {
|
|
649
1327
|
const stop = result.stopped;
|
|
650
|
-
|
|
1328
|
+
// Only live steps did work here. A recorded stand-in is last week's
|
|
1329
|
+
// output (BIR_REPLAY_RECORDED_OUTPUTS): it books nothing, and a run of
|
|
1330
|
+
// them alone is a baseline sample, not a hand-over "saving".
|
|
1331
|
+
const ranOk = result.executed - result.errored;
|
|
651
1332
|
state.stepsPinned += Math.max(0, ranOk);
|
|
652
1333
|
const position = stop.flatIndex ?? plan.stepCount;
|
|
653
1334
|
const step = plan.allSteps()[position];
|
|
1335
|
+
// Cut short by the clock, nothing failed: the steps that ran did their
|
|
1336
|
+
// work and none is blamed. A divergence for the ledger — a partial
|
|
1337
|
+
// saving, no fallback step — as a budget stop always was.
|
|
1338
|
+
if (stop.reason === "budget" && ranOk > 0)
|
|
1339
|
+
this.upgrade(state, "diverged");
|
|
1340
|
+
// Cut short by the *caller* — Esc on run_scenario, or its client timing
|
|
1341
|
+
// out — is no saving at all: the turn it ended cost next to nothing and
|
|
1342
|
+
// says nothing about what the task costs. `handOver` books `fell_back`
|
|
1343
|
+
// (steps did run) or `failed`, and the report is not a measurement.
|
|
1344
|
+
if (stop.reason === "cancelled")
|
|
1345
|
+
state.callerCancelled = true;
|
|
1346
|
+
// Nothing ran *here*: there is no bundle, only the task — a recorded
|
|
1347
|
+
// output is not a result that happened (plan.ts, DriveOptions).
|
|
1348
|
+
const live = result.executed > 0;
|
|
654
1349
|
this.handOver(state, {
|
|
655
1350
|
// The plan only knows it stopped in front of its end; which kind of
|
|
656
1351
|
// stop that is — parked, a judgement, a call that cannot run — is
|
|
@@ -660,13 +1355,36 @@ export class ReplayController {
|
|
|
660
1355
|
entry: plan.allEntries()[position],
|
|
661
1356
|
position,
|
|
662
1357
|
error: stop.error,
|
|
663
|
-
|
|
1358
|
+
// A cancelled run's note is kept for the model's next call, alone —
|
|
1359
|
+
// the bundle went to a caller who is gone — so it points at side
|
|
1360
|
+
// effects in place, never at results "below" that nobody receives.
|
|
1361
|
+
delivery: live && stop.reason !== "cancelled" ? "below" : "none",
|
|
1362
|
+
stopReason: stop.reason,
|
|
1363
|
+
ran: live ? undefined : 0,
|
|
1364
|
+
// An output-logic failure stops *after* its step's tool ran: the step
|
|
1365
|
+
// is done, its result is in the bundle, and it must not be run again.
|
|
1366
|
+
ranThisStep: stop.stage === "tool_output_logic",
|
|
664
1367
|
});
|
|
1368
|
+
if (stop.reason === "cancelled") {
|
|
1369
|
+
// The caller is gone and reads nothing. The note stays undelivered: a
|
|
1370
|
+
// client that timed out hands the task back to the model in this same
|
|
1371
|
+
// turn, and the model's next call is where it learns which steps
|
|
1372
|
+
// already ran — before it runs them a second time.
|
|
1373
|
+
logLine("replay.cancelled", {
|
|
1374
|
+
scenario: state.scenarioId ?? undefined,
|
|
1375
|
+
ran: `${result.executed}/${plan.stepCount}`,
|
|
1376
|
+
why: "the caller went away — no further step was started",
|
|
1377
|
+
});
|
|
1378
|
+
const note = state.handover?.note ?? "";
|
|
1379
|
+
return live
|
|
1380
|
+
? { ok: true, text: `${note}\n\n${result.text}`, responseModel: {}, steps: result.executed + result.recorded, partial: true }
|
|
1381
|
+
: (handedOver(note) ?? { ok: false, why: note });
|
|
1382
|
+
}
|
|
665
1383
|
const note = this.takeNote(state) ?? state.handover?.note ?? "";
|
|
666
|
-
if (
|
|
667
|
-
// Nothing ran:
|
|
668
|
-
//
|
|
669
|
-
return { ok: false, why: note };
|
|
1384
|
+
if (!live) {
|
|
1385
|
+
// Nothing ran: the tool answers with an error that says so, which is
|
|
1386
|
+
// what a model reacts to best.
|
|
1387
|
+
return handedOver(note) ?? { ok: false, why: note };
|
|
670
1388
|
}
|
|
671
1389
|
return {
|
|
672
1390
|
ok: true,
|
|
@@ -676,7 +1394,24 @@ export class ReplayController {
|
|
|
676
1394
|
partial: result.partial,
|
|
677
1395
|
};
|
|
678
1396
|
}
|
|
679
|
-
|
|
1397
|
+
if (result.executed === 0) {
|
|
1398
|
+
// Every step stood on its recorded output (BIR_REPLAY_RECORDED_OUTPUTS):
|
|
1399
|
+
// nothing happened here, and an answer built on last week's outputs is
|
|
1400
|
+
// not a replay. The model does the task.
|
|
1401
|
+
this.handOver(state, {
|
|
1402
|
+
kind: "step_failed",
|
|
1403
|
+
step: plan.allSteps()[0],
|
|
1404
|
+
entry: plan.allEntries()[0],
|
|
1405
|
+
position: 0,
|
|
1406
|
+
error: "none of the scenario's steps could run here",
|
|
1407
|
+
delivery: "none",
|
|
1408
|
+
stopReason: "not_run",
|
|
1409
|
+
ran: 0,
|
|
1410
|
+
});
|
|
1411
|
+
return handedOver(this.takeNote(state) ?? state.handover?.note) ?? { ok: false, why: "" };
|
|
1412
|
+
}
|
|
1413
|
+
// Live steps only, as above: a recorded stand-in did no work here.
|
|
1414
|
+
state.stepsPinned += result.executed;
|
|
680
1415
|
let responseModel = {};
|
|
681
1416
|
try {
|
|
682
1417
|
responseModel = plan.responseModel();
|
|
@@ -687,9 +1422,11 @@ export class ReplayController {
|
|
|
687
1422
|
// A plan whose tools errored ran to the end and did nothing. `steered_full`
|
|
688
1423
|
// would book the largest saving available for it; `failed` books a baseline
|
|
689
1424
|
// sample, which is what a replay that produced nothing is worth (mcpmark.md §13).
|
|
1425
|
+
// A step that stood on a recorded output did not happen here either, so a
|
|
1426
|
+
// plan with one is at best a divergence.
|
|
690
1427
|
this.upgrade(state, result.errored > 0
|
|
691
1428
|
? "failed"
|
|
692
|
-
: result.partial || result.skipped > 0
|
|
1429
|
+
: result.partial || result.skipped > 0 || result.recorded > 0
|
|
693
1430
|
? "diverged"
|
|
694
1431
|
: "steered_full");
|
|
695
1432
|
// `runToCompletion` returned without a stop: the plan ran to its end, so
|
|
@@ -752,7 +1489,9 @@ export class ReplayController {
|
|
|
752
1489
|
durationMs: d.durationMs,
|
|
753
1490
|
stepsPlanned: state.stepsPlanned,
|
|
754
1491
|
stepsPinned: state.stepsPinned,
|
|
755
|
-
|
|
1492
|
+
// A run its caller cancelled ended the turn early: its cost is no
|
|
1493
|
+
// measurement of the task, whatever the transcript says.
|
|
1494
|
+
measured: d.measured && state.callerCancelled !== true,
|
|
756
1495
|
pricingVersion: PRICING_VERSION,
|
|
757
1496
|
prompt: d.prompt,
|
|
758
1497
|
steps,
|
|
@@ -881,6 +1620,7 @@ export class ReplayController {
|
|
|
881
1620
|
mode,
|
|
882
1621
|
entries: flat.entries,
|
|
883
1622
|
stopAt: flat.stopAt,
|
|
1623
|
+
fallbackTargets: "samples",
|
|
884
1624
|
});
|
|
885
1625
|
state.plan = plan;
|
|
886
1626
|
state.flatStop = flat.stopReason;
|
|
@@ -899,15 +1639,19 @@ export class ReplayController {
|
|
|
899
1639
|
coverage: coverageOf(scenario.steps, wrapped, this.opts.allowServers).join(","),
|
|
900
1640
|
why: "bir replay — no match, an operator asked for it",
|
|
901
1641
|
});
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
}
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
}
|
|
1642
|
+
// An operator's run keeps its own contract (R-CALL-37): out of time, the
|
|
1643
|
+
// targets too take their samples. What it never does is run on `{}`.
|
|
1644
|
+
const ready = await this.readyFor(state, plan, this.adHocBudgets.deriveMs, { adHoc: true });
|
|
1645
|
+
if (!ready.ok)
|
|
1646
|
+
return { ok: false, why: ready.error };
|
|
908
1647
|
const trace = [];
|
|
1648
|
+
const deadline = Date.now() + this.adHocBudgets.planMs;
|
|
909
1649
|
try {
|
|
910
|
-
const result = await plan.runToCompletion(this.executeStep(state
|
|
1650
|
+
const result = await plan.runToCompletion(this.executeStep(state, { deadline, adHoc: true }),
|
|
1651
|
+
// The operator reads a per-step trace that labels every stand-in, so
|
|
1652
|
+
// `bir replay` keeps serving recorded outputs for steps that cannot be
|
|
1653
|
+
// sent from here — and still stops at one that was sent and failed.
|
|
1654
|
+
this.recordedSource(state, true), (info) => {
|
|
911
1655
|
this.observeStep(state)(info);
|
|
912
1656
|
trace.push({
|
|
913
1657
|
step: info.step.stepIndex,
|
|
@@ -916,7 +1660,7 @@ export class ReplayController {
|
|
|
916
1660
|
outcome: info.outcome,
|
|
917
1661
|
ms: info.ms,
|
|
918
1662
|
});
|
|
919
|
-
},
|
|
1663
|
+
}, deadline, MAX_REPLAY_REASON, { recordedFallback: true });
|
|
920
1664
|
let responseModel = {};
|
|
921
1665
|
try {
|
|
922
1666
|
responseModel = plan.responseModel();
|
|
@@ -924,6 +1668,8 @@ export class ReplayController {
|
|
|
924
1668
|
catch (err) {
|
|
925
1669
|
logDetail("replay.response_model_failed", { error: errText(err) });
|
|
926
1670
|
}
|
|
1671
|
+
const stop = result.stopped;
|
|
1672
|
+
const stoppedStep = stop ? plan.allSteps()[stop.flatIndex ?? plan.stepCount] : undefined;
|
|
927
1673
|
return {
|
|
928
1674
|
ok: true,
|
|
929
1675
|
text: result.text,
|
|
@@ -931,6 +1677,14 @@ export class ReplayController {
|
|
|
931
1677
|
steps: result.executed + result.recorded,
|
|
932
1678
|
partial: result.partial,
|
|
933
1679
|
trace,
|
|
1680
|
+
stopped: stop
|
|
1681
|
+
? {
|
|
1682
|
+
step: stop.stepIndex,
|
|
1683
|
+
tool: stoppedStep?.toolName ?? "",
|
|
1684
|
+
reason: stop.reason ?? stop.stage ?? stop.kind,
|
|
1685
|
+
error: stop.error,
|
|
1686
|
+
}
|
|
1687
|
+
: undefined,
|
|
934
1688
|
};
|
|
935
1689
|
}
|
|
936
1690
|
catch (err) {
|
|
@@ -965,17 +1719,26 @@ export class ReplayController {
|
|
|
965
1719
|
}
|
|
966
1720
|
startDerivation(scenario, prompt, state, ctx = {}) {
|
|
967
1721
|
const derive = this.opts.deriveImpl ?? deriveParameters;
|
|
1722
|
+
const apiKey = this.opts.apiKey ?? process.env.ANTHROPIC_API_KEY;
|
|
1723
|
+
const service = this.deriveService(scenario.id);
|
|
1724
|
+
const hasSchema = Object.keys(scenario.paramsObject ?? {}).length > 0;
|
|
1725
|
+
state.deriveOnService = !apiKey && service !== undefined && hasSchema;
|
|
968
1726
|
return derive({
|
|
969
1727
|
prompt,
|
|
970
1728
|
intent: scenario.intent ?? "",
|
|
971
1729
|
paramsObject: scenario.paramsObject,
|
|
972
|
-
apiKey
|
|
1730
|
+
apiKey,
|
|
973
1731
|
// Where to ask when there is no key here, which is the ordinary case
|
|
974
1732
|
// (segmented.md R-PARAM-5).
|
|
975
|
-
service
|
|
1733
|
+
service,
|
|
976
1734
|
fetchImpl: this.opts.fetchImpl,
|
|
977
1735
|
liveCall: ctx.liveCall,
|
|
978
1736
|
recentResults: ctx.recentResults,
|
|
1737
|
+
// The request itself ends too. Without this a hung service or model held
|
|
1738
|
+
// the socket for up to 300 s after every wait on it had given up. Long
|
|
1739
|
+
// enough to cover both waits that can use its answer (the arm and the
|
|
1740
|
+
// first tool call); an answer after that is read by nobody.
|
|
1741
|
+
signal: timeoutSignal(Math.max(serviceTimeoutMs(), this.budgets.deriveMs)),
|
|
979
1742
|
})
|
|
980
1743
|
.then((r) => {
|
|
981
1744
|
state.deriveCostUsd += r.costUsd;
|
|
@@ -1026,8 +1789,14 @@ export class ReplayController {
|
|
|
1026
1789
|
*
|
|
1027
1790
|
* The plan is retired first: whatever happens next, this turn neither steers
|
|
1028
1791
|
* nor re-injects again.
|
|
1792
|
+
*
|
|
1793
|
+
* This runs inside the diverging call's `PreToolUse`, so `deadline` bounds
|
|
1794
|
+
* it: no step is started after it, and none is waited on past it. The bundle
|
|
1795
|
+
* then says exactly what ran, and hands the rest over.
|
|
1029
1796
|
*/
|
|
1030
|
-
async diverge(state, toolName, why
|
|
1797
|
+
async diverge(state, toolName, why, deadline,
|
|
1798
|
+
/** Aborted once the host has its answer: no further step is started. */
|
|
1799
|
+
signal) {
|
|
1031
1800
|
const plan = state.plan;
|
|
1032
1801
|
if (!plan)
|
|
1033
1802
|
return { kind: "passthrough" };
|
|
@@ -1039,11 +1808,19 @@ export class ReplayController {
|
|
|
1039
1808
|
why,
|
|
1040
1809
|
});
|
|
1041
1810
|
this.retire(state, undefined);
|
|
1811
|
+
// The bundle, and the note in front of it, must fit the channel it goes
|
|
1812
|
+
// out on whole: the host cuts a long Bash output in the middle.
|
|
1813
|
+
const channelMax = toolName === "Bash" ? BASH_CHANNEL_MAX : MAX_REPLAY_REASON;
|
|
1042
1814
|
let composed;
|
|
1043
1815
|
try {
|
|
1044
|
-
composed = await plan.composeBundle(
|
|
1045
|
-
|
|
1046
|
-
|
|
1816
|
+
composed = await plan.composeBundle(Math.max(2_000, channelMax - MAX_HANDOVER_NOTE - 2), this.executeStep(state, { deadline }), this.recordedSource(state), this.observeStep(state), {
|
|
1817
|
+
// A parked step ends the bundle short of the task (fallbk.md D3).
|
|
1818
|
+
handover: plan.stopsEarly(),
|
|
1819
|
+
stopOnFailure: true,
|
|
1820
|
+
recordedFallback: this.recordedFallback,
|
|
1821
|
+
deadline,
|
|
1822
|
+
signal,
|
|
1823
|
+
});
|
|
1047
1824
|
}
|
|
1048
1825
|
catch (err) {
|
|
1049
1826
|
this.upgrade(state, "failed");
|
|
@@ -1054,18 +1831,34 @@ export class ReplayController {
|
|
|
1054
1831
|
});
|
|
1055
1832
|
return { kind: "abort" };
|
|
1056
1833
|
}
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1834
|
+
const stop = composed.stopped;
|
|
1835
|
+
// Same rule as `runArmed`: a recovered chain whose tools errored is
|
|
1836
|
+
// `failed`, and so is one that stopped at a step that broke — its logic
|
|
1837
|
+
// threw, or it was sent and failed or never answered. A step that simply
|
|
1838
|
+
// could not be sent from here, or the hook's clock, is still a divergence.
|
|
1839
|
+
const broke = composed.errored > 0 ||
|
|
1840
|
+
(stop !== undefined &&
|
|
1841
|
+
stop.reason !== "not_run" &&
|
|
1842
|
+
stop.reason !== "budget" &&
|
|
1843
|
+
stop.reason !== "cancelled");
|
|
1844
|
+
// And a divergence is booked only when something ran under the plan — a
|
|
1845
|
+
// step pinned before it, or one executed now. One that stopped in front of
|
|
1846
|
+
// its first step (a built-in, which only the session runs) ran nothing and
|
|
1847
|
+
// delivered nothing: no saving, a baseline sample, as `handOver` books it.
|
|
1848
|
+
// Recorded stand-ins are no work here (BIR_REPLAY_RECORDED_OUTPUTS).
|
|
1849
|
+
const didWork = composed.executed > 0 || state.stepsPinned > 0;
|
|
1850
|
+
this.upgrade(state, broke || !didWork ? "failed" : "diverged");
|
|
1851
|
+
state.stepsPinned += composed.executed;
|
|
1060
1852
|
// The headline number of this whole design: recovering from a divergence
|
|
1061
1853
|
// costs nothing, because the proxies were already connected.
|
|
1062
1854
|
logLine("replay.compose", {
|
|
1063
1855
|
scenario: state.scenarioId ?? undefined,
|
|
1064
|
-
remaining:
|
|
1856
|
+
remaining: plan.stepCount - plan.currentStepIndex,
|
|
1065
1857
|
executed: composed.executed,
|
|
1066
1858
|
recorded: composed.recorded,
|
|
1067
1859
|
skipped: composed.skipped,
|
|
1068
1860
|
errored: composed.errored || undefined,
|
|
1861
|
+
stopped: stop ? `${stop.flatIndex ?? "?"}:${stop.reason ?? stop.stage ?? stop.kind}` : undefined,
|
|
1069
1862
|
bytes: composed.text.length,
|
|
1070
1863
|
costUsd: state.fallbackCostUsd.toFixed(2),
|
|
1071
1864
|
});
|
|
@@ -1076,15 +1869,26 @@ export class ReplayController {
|
|
|
1076
1869
|
outcome: state.outcome,
|
|
1077
1870
|
ms: Date.now() - state.armedAt,
|
|
1078
1871
|
});
|
|
1079
|
-
if (composed.executed
|
|
1080
|
-
// Nothing
|
|
1872
|
+
if (composed.executed === 0) {
|
|
1873
|
+
// Nothing ran here: a bundle of nothing is worse than no bundle, and one
|
|
1874
|
+
// of recorded outputs only is last week's work. The model's own call goes
|
|
1875
|
+
// ahead — unless a step was *sent* and never answered. That one may have
|
|
1876
|
+
// done its work, and the model must know before it does the task itself.
|
|
1877
|
+
if (stop?.reason === "uncertain" && stop.flatIndex !== undefined) {
|
|
1878
|
+
const note = this.stopNote(state, plan, stop, "none");
|
|
1879
|
+
return this.deliverInstead(note, toolName);
|
|
1880
|
+
}
|
|
1081
1881
|
return { kind: "abort" };
|
|
1082
1882
|
}
|
|
1083
|
-
// The bundle stops
|
|
1084
|
-
// model carries on from there. The
|
|
1085
|
-
//
|
|
1883
|
+
// The bundle stops short of the task: say so ahead of it — which step, and
|
|
1884
|
+
// what may be assumed about it — and the model carries on from there. The
|
|
1885
|
+
// outcome stays as judged above, so the report carries no fallback fields.
|
|
1086
1886
|
let text = composed.text;
|
|
1087
|
-
if (
|
|
1887
|
+
if (stop && stop.kind === "step_failed" && stop.flatIndex !== undefined) {
|
|
1888
|
+
const note = this.stopNote(state, plan, stop, "below");
|
|
1889
|
+
text = `${note}\n\n${composed.text}`;
|
|
1890
|
+
}
|
|
1891
|
+
else if (plan.stopsEarly()) {
|
|
1088
1892
|
const parked = plan.stopStep();
|
|
1089
1893
|
const stopKind = this.stopKindOf(state, plan.stopEntry());
|
|
1090
1894
|
state.handover = {
|
|
@@ -1097,6 +1901,7 @@ export class ReplayController {
|
|
|
1097
1901
|
entry: plan.stopEntry(),
|
|
1098
1902
|
position: plan.stepCount,
|
|
1099
1903
|
delivery: "below",
|
|
1904
|
+
executedAbove: plan.currentStepIndex,
|
|
1100
1905
|
}),
|
|
1101
1906
|
delivered: true,
|
|
1102
1907
|
};
|
|
@@ -1104,11 +1909,46 @@ export class ReplayController {
|
|
|
1104
1909
|
}
|
|
1105
1910
|
// Bash-clean delivery: the model reaches for `Bash` when it cannot call a
|
|
1106
1911
|
// scripted tool, and a genuine command output is trusted where a `deny`
|
|
1107
|
-
// reason is read as adversarial interception.
|
|
1912
|
+
// reason is read as adversarial interception.
|
|
1108
1913
|
const action = this.deliverInstead(text, toolName);
|
|
1109
1914
|
logDetail("replay.inject", { channel: action.kind, bytes: text.length });
|
|
1110
1915
|
return action;
|
|
1111
1916
|
}
|
|
1917
|
+
/**
|
|
1918
|
+
* The note for a divergence bundle that stopped at a step, recorded as this
|
|
1919
|
+
* plan's hand-over (already delivered — it goes out with the bundle), so the
|
|
1920
|
+
* model's own work from here is recorded as a fragment.
|
|
1921
|
+
*/
|
|
1922
|
+
stopNote(state, plan, stop, delivery) {
|
|
1923
|
+
const position = stop.flatIndex ?? plan.currentStepIndex;
|
|
1924
|
+
const step = plan.allSteps()[position];
|
|
1925
|
+
const entry = plan.allEntries()[position];
|
|
1926
|
+
const note = this.noteFor(state, {
|
|
1927
|
+
kind: stop.kind,
|
|
1928
|
+
step,
|
|
1929
|
+
entry,
|
|
1930
|
+
position,
|
|
1931
|
+
error: stop.error,
|
|
1932
|
+
delivery,
|
|
1933
|
+
stopReason: stop.reason,
|
|
1934
|
+
// The steps before the cursor ran in the live session — the model's own
|
|
1935
|
+
// pinned calls, just above. With no bundle (`none`) they are the only
|
|
1936
|
+
// steps that ran; "no step ran" would invite the model to run them again.
|
|
1937
|
+
ran: delivery === "none" ? plan.currentStepIndex : undefined,
|
|
1938
|
+
executedAbove: plan.currentStepIndex,
|
|
1939
|
+
// An output-logic failure stops after its step's tool ran (plan.ts).
|
|
1940
|
+
ranThisStep: stop.stage === "tool_output_logic",
|
|
1941
|
+
});
|
|
1942
|
+
state.handover ??= {
|
|
1943
|
+
kind: stop.kind,
|
|
1944
|
+
stepIndex: callerStepIndexOf(entry) ?? step?.stepIndex ?? position,
|
|
1945
|
+
toolName: step?.toolName ?? undefined,
|
|
1946
|
+
error: stop.error,
|
|
1947
|
+
note,
|
|
1948
|
+
delivered: true,
|
|
1949
|
+
};
|
|
1950
|
+
return note;
|
|
1951
|
+
}
|
|
1112
1952
|
/**
|
|
1113
1953
|
* Hand the rest of the task to the model (fallbk.md D4/D5).
|
|
1114
1954
|
*
|
|
@@ -1150,7 +1990,13 @@ export class ReplayController {
|
|
|
1150
1990
|
? "the next step has failed too often — the agent continues from here"
|
|
1151
1991
|
: h.kind === "nondeterministic_step"
|
|
1152
1992
|
? "the next step needs a judgement the plan cannot compute — the agent makes it"
|
|
1153
|
-
:
|
|
1993
|
+
: h.stopReason === "concurrent"
|
|
1994
|
+
? "a second call arrived while the pinned step was still running — the agent continues from here"
|
|
1995
|
+
: h.stopReason === "budget" || h.stopReason === "cancelled"
|
|
1996
|
+
? "the run was cut short before this step — the agent continues from here"
|
|
1997
|
+
: h.stopReason === "not_run" || h.stopReason === "uncertain"
|
|
1998
|
+
? "a step produced no result here — the agent continues from here"
|
|
1999
|
+
: "a step broke — the agent continues from here",
|
|
1154
2000
|
});
|
|
1155
2001
|
}
|
|
1156
2002
|
/**
|
|
@@ -1166,7 +2012,7 @@ export class ReplayController {
|
|
|
1166
2012
|
}
|
|
1167
2013
|
noteFor(state, h) {
|
|
1168
2014
|
const plan = state.plan;
|
|
1169
|
-
const executed = h.ranThisStep ? h.position + 1 : h.position;
|
|
2015
|
+
const executed = h.ran ?? (h.ranThisStep ? h.position + 1 : h.position);
|
|
1170
2016
|
let params;
|
|
1171
2017
|
try {
|
|
1172
2018
|
params = plan?.parameters;
|
|
@@ -1181,6 +2027,7 @@ export class ReplayController {
|
|
|
1181
2027
|
stepToolName: h.step?.toolName ?? undefined,
|
|
1182
2028
|
failureCount: h.step?.failureCount,
|
|
1183
2029
|
error: h.error,
|
|
2030
|
+
stopReason: h.stopReason,
|
|
1184
2031
|
unusable: state.flatStop
|
|
1185
2032
|
? {
|
|
1186
2033
|
reason: state.flatStop.reason,
|
|
@@ -1192,6 +2039,9 @@ export class ReplayController {
|
|
|
1192
2039
|
ownerScenarioId: h.entry && h.entry.depth > 0 ? h.entry.scenarioId : undefined,
|
|
1193
2040
|
inSubTask: h.entry && h.entry.depth > 0 ? (h.entry.frame.callIntent ?? null) : null,
|
|
1194
2041
|
executed: Math.max(0, Math.min(executed, plan?.totalSteps ?? executed)),
|
|
2042
|
+
executedAbove: h.executedAbove,
|
|
2043
|
+
inFlight: h.inFlight,
|
|
2044
|
+
ranThisStep: h.ranThisStep,
|
|
1195
2045
|
totalSteps: plan?.totalSteps ?? state.stepsPlanned,
|
|
1196
2046
|
remainingTools: plan ? plan.toolsFrom(h.ranThisStep ? h.position + 1 : h.position) : [],
|
|
1197
2047
|
params,
|
|
@@ -1202,22 +2052,173 @@ export class ReplayController {
|
|
|
1202
2052
|
* The executor handed to a plan: dispatch a step to the proxy that owns its
|
|
1203
2053
|
* upstream.
|
|
1204
2054
|
*
|
|
1205
|
-
*
|
|
1206
|
-
*
|
|
1207
|
-
*
|
|
2055
|
+
* A tool that ran and failed resolves with its failure as the response,
|
|
2056
|
+
* exactly as it would in a session. A rejection is a {@link StepRunError}
|
|
2057
|
+
* saying what happened to the call — not sent, sent and unanswered, or
|
|
2058
|
+
* refused — because the model is told differently about each (plan.ts).
|
|
2059
|
+
*
|
|
2060
|
+
* `deadline` is the whole run's: no step is waited on past it (but each gets
|
|
2061
|
+
* at least {@link MIN_STEP_MS}). `adHoc` is `bir replay`, which the operator
|
|
2062
|
+
* asked for by id and which the allowlist does not gate.
|
|
1208
2063
|
*/
|
|
1209
|
-
executeStep(state) {
|
|
2064
|
+
executeStep(state, opts = {}) {
|
|
1210
2065
|
return async (step, input) => {
|
|
1211
|
-
const
|
|
2066
|
+
const name = step.toolName ?? "";
|
|
2067
|
+
const mcp = parseQualifiedName(name);
|
|
1212
2068
|
if (!mcp) {
|
|
1213
|
-
throw new
|
|
2069
|
+
throw new StepRunError(`${name || "this step"} is a built-in tool, which only runs in the session`, "not_run");
|
|
2070
|
+
}
|
|
2071
|
+
// `BIR_REPLAY_ALLOW_SERVERS` is about exactly this: a call bir makes by
|
|
2072
|
+
// itself, with nobody approving it. A steered plan's divergence recovery
|
|
2073
|
+
// reaches here too, for steps that were only ever to be steered in the
|
|
2074
|
+
// session — where the user's own permission rules apply.
|
|
2075
|
+
const allow = this.opts.allowServers;
|
|
2076
|
+
if (!opts.adHoc && allow && !allow.has(mcp.serverName)) {
|
|
2077
|
+
throw new StepRunError(`${mcp.serverName} is not on BIR_REPLAY_ALLOW_SERVERS, so bir does not call it unattended`, "not_run");
|
|
2078
|
+
}
|
|
2079
|
+
const left = opts.deadline !== undefined ? opts.deadline - Date.now() : Number.POSITIVE_INFINITY;
|
|
2080
|
+
const timeoutMs = Math.max(MIN_STEP_MS, Math.min(this.budgets.stepMs, left));
|
|
2081
|
+
let result;
|
|
2082
|
+
try {
|
|
2083
|
+
result = await this.work.call(mcp.serverName, mcp.toolName, input, timeoutMs, {
|
|
2084
|
+
sessionId: state.sessionId,
|
|
2085
|
+
});
|
|
2086
|
+
}
|
|
2087
|
+
catch (err) {
|
|
2088
|
+
throw classifyWorkError(err, mcp.serverName);
|
|
2089
|
+
}
|
|
2090
|
+
// A proxy that answers the work with no result at all took it and lost
|
|
2091
|
+
// it (tool-error.ts, `isNoResult`): sent, and may or may not have run.
|
|
2092
|
+
// Threaded as `null`, it would read as a success.
|
|
2093
|
+
if (result === undefined || result === null) {
|
|
2094
|
+
throw new StepRunError(`the ${mcp.serverName} proxy answered with no result — the call may or may not have run`, "uncertain");
|
|
1214
2095
|
}
|
|
1215
|
-
const result = await this.work.call(mcp.serverName, mcp.toolName, input, this.budgets.stepMs, { sessionId: state.sessionId });
|
|
1216
2096
|
// Serialize exactly as the proxy records it, or `toolOutputLogic` — which
|
|
1217
2097
|
// was authored against that shape — silently derives nothing (§7.2).
|
|
1218
2098
|
return serializeCapped(redact(result));
|
|
1219
2099
|
};
|
|
1220
2100
|
}
|
|
2101
|
+
/** Where recorded outputs come from, when they may stand in at all. */
|
|
2102
|
+
recordedSource(state, force = false) {
|
|
2103
|
+
if (!force && !this.recordedFallback)
|
|
2104
|
+
return undefined;
|
|
2105
|
+
const source = state.sourceRun;
|
|
2106
|
+
return source ? (entry) => source.outputFor(entry) : undefined;
|
|
2107
|
+
}
|
|
2108
|
+
/**
|
|
2109
|
+
* Wait for a plan's parameters within `budgetMs`, then judge whether the plan
|
|
2110
|
+
* may run on them. Never throws.
|
|
2111
|
+
*
|
|
2112
|
+
* Out of time, the parameters settle without the derivation
|
|
2113
|
+
* (`useFallbackParams`: settings at their samples, targets null) — never on
|
|
2114
|
+
* `{}` — and the plan runs only if it needs no target it was not given. A
|
|
2115
|
+
* `paramsLogic` that throws, or a value too dangerous to template into a
|
|
2116
|
+
* command (safety.ts), stops it. `adHoc` keeps `bir replay`'s contract: the
|
|
2117
|
+
* operator named the scenario, so neither judgement applies.
|
|
2118
|
+
*/
|
|
2119
|
+
async readyFor(state, plan, budgetMs, opts = {}) {
|
|
2120
|
+
const logicFailed = (err) => ({
|
|
2121
|
+
ok: false,
|
|
2122
|
+
error: `the scenario's parameter logic failed: ${errText(err)}`,
|
|
2123
|
+
stage: "params_logic",
|
|
2124
|
+
// The scenario's own logic is broken: the step it stopped in front of
|
|
2125
|
+
// carries the verdict, so the recording page says why it never runs.
|
|
2126
|
+
blame: true,
|
|
2127
|
+
});
|
|
2128
|
+
if (!plan.parametersSettled) {
|
|
2129
|
+
let failure;
|
|
2130
|
+
const settled = plan.ready().then(() => true, (err) => {
|
|
2131
|
+
failure = err;
|
|
2132
|
+
return true;
|
|
2133
|
+
});
|
|
2134
|
+
const waited = await this.waitDerivation(state, settled, budgetMs);
|
|
2135
|
+
if (failure !== undefined)
|
|
2136
|
+
return logicFailed(failure);
|
|
2137
|
+
if (waited === undefined) {
|
|
2138
|
+
logLine("replay.derive_timeout", {
|
|
2139
|
+
scenario: state.scenarioId ?? undefined,
|
|
2140
|
+
waitedMs: budgetMs,
|
|
2141
|
+
why: opts.adHoc
|
|
2142
|
+
? "bir replay — every parameter takes its recorded sample"
|
|
2143
|
+
: "settings take their recorded samples, targets stay unnamed — the plan runs only if it needs none",
|
|
2144
|
+
});
|
|
2145
|
+
try {
|
|
2146
|
+
plan.useFallbackParams();
|
|
2147
|
+
}
|
|
2148
|
+
catch (err) {
|
|
2149
|
+
return logicFailed(err);
|
|
2150
|
+
}
|
|
2151
|
+
if (!opts.adHoc) {
|
|
2152
|
+
const schema = plan.paramsSchema;
|
|
2153
|
+
const planned = plan.allEntries().slice(0, plan.stepCount);
|
|
2154
|
+
const blocked = blockingTargets(targetKeys(schema), schema, planned, plan.paramsLogic);
|
|
2155
|
+
if (blocked.length > 0) {
|
|
2156
|
+
return {
|
|
2157
|
+
ok: false,
|
|
2158
|
+
error: `the value of ${blocked[0]} could not be read from this request in time`,
|
|
2159
|
+
stage: "derive_params",
|
|
2160
|
+
blame: false,
|
|
2161
|
+
};
|
|
2162
|
+
}
|
|
2163
|
+
}
|
|
2164
|
+
}
|
|
2165
|
+
}
|
|
2166
|
+
const sink = opts.adHoc ? undefined : sinkOfPlan(plan.allEntries().slice(0, plan.stepCount));
|
|
2167
|
+
if (sink) {
|
|
2168
|
+
const bad = unsafeParameter(plan.derivedValues, plan.paramsSchema, sink);
|
|
2169
|
+
if (bad) {
|
|
2170
|
+
logLine("replay.unsafe_param", {
|
|
2171
|
+
scenario: state.scenarioId ?? undefined,
|
|
2172
|
+
param: bad.key,
|
|
2173
|
+
sink,
|
|
2174
|
+
why: "a value read for this request would be templated into a command unattended — the agent does the task instead",
|
|
2175
|
+
});
|
|
2176
|
+
return {
|
|
2177
|
+
ok: false,
|
|
2178
|
+
error: `the value read for ${bad.key} contains characters that are not safe to put into a command unattended`,
|
|
2179
|
+
stage: "derive_params",
|
|
2180
|
+
blame: false,
|
|
2181
|
+
};
|
|
2182
|
+
}
|
|
2183
|
+
}
|
|
2184
|
+
return { ok: true };
|
|
2185
|
+
}
|
|
2186
|
+
/**
|
|
2187
|
+
* Wait for something the derivation decides, within `budgetMs`. Resolves
|
|
2188
|
+
* undefined when the time ran out.
|
|
2189
|
+
*
|
|
2190
|
+
* When the service is the one deriving, this is a service wait on a hot path
|
|
2191
|
+
* (C1): it does not wait at all while the service is cooling down, and an
|
|
2192
|
+
* overrun is reported to the breaker so the next hook does not wait either.
|
|
2193
|
+
* Even a zero wait lets an answer that has already arrived through.
|
|
2194
|
+
*/
|
|
2195
|
+
async waitDerivation(state, pending, budgetMs) {
|
|
2196
|
+
const onService = state.deriveOnService === true;
|
|
2197
|
+
const budget = onService && !serviceHealth.canWait() ? 0 : budgetMs;
|
|
2198
|
+
let timer;
|
|
2199
|
+
const late = Symbol("late");
|
|
2200
|
+
try {
|
|
2201
|
+
const winner = await Promise.race([
|
|
2202
|
+
pending,
|
|
2203
|
+
new Promise((resolve) => {
|
|
2204
|
+
timer = setTimeout(() => resolve(late), Math.max(0, budget));
|
|
2205
|
+
timer.unref?.();
|
|
2206
|
+
}),
|
|
2207
|
+
]);
|
|
2208
|
+
if (winner !== late)
|
|
2209
|
+
return winner;
|
|
2210
|
+
if (onService && budget > 0)
|
|
2211
|
+
serviceHealth.slow("derive");
|
|
2212
|
+
return undefined;
|
|
2213
|
+
}
|
|
2214
|
+
catch {
|
|
2215
|
+
return undefined;
|
|
2216
|
+
}
|
|
2217
|
+
finally {
|
|
2218
|
+
if (timer)
|
|
2219
|
+
clearTimeout(timer);
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
1221
2222
|
/**
|
|
1222
2223
|
* Where a pinned step's output will arrive from — the proxy's report, or the
|
|
1223
2224
|
* hook's `tool_response` (§7.2).
|
|
@@ -1307,7 +2308,7 @@ export class ReplayController {
|
|
|
1307
2308
|
logLine("replay.step_skipped", {
|
|
1308
2309
|
n: info.step.stepIndex,
|
|
1309
2310
|
tool: info.step.toolName,
|
|
1310
|
-
why: "its
|
|
2311
|
+
why: "its call produced no result here — the plan stops at it",
|
|
1311
2312
|
error: info.error,
|
|
1312
2313
|
});
|
|
1313
2314
|
return;
|
|
@@ -1353,17 +2354,8 @@ export class ReplayController {
|
|
|
1353
2354
|
}
|
|
1354
2355
|
state.retired = true;
|
|
1355
2356
|
state.pinned.clear();
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
let timer;
|
|
1359
|
-
const budget = new Promise((_, reject) => {
|
|
1360
|
-
timer = setTimeout(() => reject(new Error(`${what} exceeded ${ms}ms`)), ms);
|
|
1361
|
-
timer.unref?.();
|
|
1362
|
-
});
|
|
1363
|
-
return Promise.race([p, budget]).finally(() => {
|
|
1364
|
-
if (timer)
|
|
1365
|
-
clearTimeout(timer);
|
|
1366
|
-
});
|
|
2357
|
+
// A call waiting for a pin to thread has nothing left to wait for.
|
|
2358
|
+
this.notifyPins(state);
|
|
1367
2359
|
}
|
|
1368
2360
|
}
|
|
1369
2361
|
/** Every target of a schema. A parameter with no kind is one (R-PARAM-3). */
|
|
@@ -1372,6 +2364,88 @@ function targetKeys(schema) {
|
|
|
1372
2364
|
return [];
|
|
1373
2365
|
return Object.keys(schema).filter((k) => schema[k]?.kind !== "setting");
|
|
1374
2366
|
}
|
|
2367
|
+
/** A positive number from the environment, or undefined. */
|
|
2368
|
+
function positiveEnv(name) {
|
|
2369
|
+
const n = Number(process.env[name]);
|
|
2370
|
+
return Number.isFinite(n) && n > 0 ? n : undefined;
|
|
2371
|
+
}
|
|
2372
|
+
/**
|
|
2373
|
+
* What a rejected dispatch means for the step (plan.ts, `StepRejection`).
|
|
2374
|
+
*
|
|
2375
|
+
* Read from the message, because that is all the proxy sends back. Only the
|
|
2376
|
+
* cases that are certain are named: nothing was sent (no proxy, no proxy that
|
|
2377
|
+
* is this session's, the queue closed before any proxy took the work, the
|
|
2378
|
+
* upstream already gone), or the upstream answered with a JSON-RPC error.
|
|
2379
|
+
* Everything else — a timeout, an
|
|
2380
|
+
* aborted wait, a connection that dropped mid-call, a shutdown while a proxy
|
|
2381
|
+
* held the work (`QUEUE_CLOSED_IN_FLIGHT`), an error nobody recognises — was
|
|
2382
|
+
* sent and may have done its work, and is said to be exactly that. Guessing
|
|
2383
|
+
* "failed" or "not run" there is how a model is invited to run a write twice.
|
|
2384
|
+
*/
|
|
2385
|
+
function classifyWorkError(err, serverName) {
|
|
2386
|
+
// Proxies are here, but none is this session's: not sent, and not the
|
|
2387
|
+
// scenario's fault — recorded as not run, never as a failed step.
|
|
2388
|
+
if (err instanceof NoSessionProxyError)
|
|
2389
|
+
return new StepRunError(err.message, "not_run");
|
|
2390
|
+
const message = errText(err);
|
|
2391
|
+
if (message === NO_PROXY) {
|
|
2392
|
+
return new StepRunError(`no ${serverName} proxy of this session is connected to bir (${NO_PROXY})`, "not_run");
|
|
2393
|
+
}
|
|
2394
|
+
if (message === QUEUE_CLOSED || /\bupstream is not available\b/i.test(message)) {
|
|
2395
|
+
return new StepRunError(message, "not_run");
|
|
2396
|
+
}
|
|
2397
|
+
if (/^MCP error -?\d+/i.test(message))
|
|
2398
|
+
return new StepRunError(message, "failed");
|
|
2399
|
+
return new StepRunError(message, "uncertain");
|
|
2400
|
+
}
|
|
2401
|
+
/**
|
|
2402
|
+
* What the proxy says of a call its upstream was already gone for: it was
|
|
2403
|
+
* never sent (relay.ts, `upstream unavailable`; executor.ts's wording too).
|
|
2404
|
+
*/
|
|
2405
|
+
const NOT_SENT_TEXT = /\bupstream (?:is not |un)available\b/i;
|
|
2406
|
+
/**
|
|
2407
|
+
* What a host or a proxy says of a call that was sent and not seen through: a
|
|
2408
|
+
* timeout, an interrupt, a cancellation, a dropped connection. Its work may
|
|
2409
|
+
* have happened — the upstream does not stop because the waiting did.
|
|
2410
|
+
*/
|
|
2411
|
+
const UNCERTAIN_TEXT = /\b(?:timed? ?out|timeout|interrupted|cancell?ed|aborted|disconnected|connection (?:closed|reset|lost|dropped)|upstream (?:closed|restarted)|relay stopped|socket hang up|ECONNRESET|EPIPE)\b/i;
|
|
2412
|
+
/**
|
|
2413
|
+
* How a pinned call ended when it did not simply work: the error to record
|
|
2414
|
+
* and, when it is not an ordinary failure, the stop reason the note words it
|
|
2415
|
+
* by (handover.ts). Undefined when neither the caller nor the output says
|
|
2416
|
+
* anything went wrong — the output's own tool error is judged after.
|
|
2417
|
+
*
|
|
2418
|
+
* - `how` from the caller wins.
|
|
2419
|
+
* - A proxy-threaded call with *no result* was abandoned by its proxy
|
|
2420
|
+
* (intercept.ts `onAbandoned`: `isError` and a reason, nothing else) — sent
|
|
2421
|
+
* and unanswered, so `uncertain`, unless the reason says it was never sent.
|
|
2422
|
+
* - A failure whose text is a timeout, an interrupt or a dropped connection
|
|
2423
|
+
* is `uncertain` too. Anything else is a failure: the tool answered one.
|
|
2424
|
+
*/
|
|
2425
|
+
function pinFailure(pin, output, failed, how) {
|
|
2426
|
+
const noResult = pin.reach === "direct" && isNoResult(output);
|
|
2427
|
+
const text = failed?.trim();
|
|
2428
|
+
if (failed === undefined && !noResult && !how.uncertain && !how.notSent)
|
|
2429
|
+
return undefined;
|
|
2430
|
+
const reason = how.notSent
|
|
2431
|
+
? "not_run"
|
|
2432
|
+
: how.uncertain
|
|
2433
|
+
? "uncertain"
|
|
2434
|
+
: noResult
|
|
2435
|
+
? text && NOT_SENT_TEXT.test(text)
|
|
2436
|
+
? "not_run"
|
|
2437
|
+
: "uncertain"
|
|
2438
|
+
: text && UNCERTAIN_TEXT.test(text)
|
|
2439
|
+
? "uncertain"
|
|
2440
|
+
: undefined;
|
|
2441
|
+
const error = text ||
|
|
2442
|
+
(reason === "not_run"
|
|
2443
|
+
? "the call was never sent — its server was not available"
|
|
2444
|
+
: reason === "uncertain" || noResult
|
|
2445
|
+
? "the call produced no result — its server stopped answering mid-call"
|
|
2446
|
+
: "the tool reported an error");
|
|
2447
|
+
return { error, reason };
|
|
2448
|
+
}
|
|
1375
2449
|
/**
|
|
1376
2450
|
* The index a hand-over reports, in the *caller's* chain (R-CALL-32).
|
|
1377
2451
|
*
|