@basein/runner 0.2.12 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +540 -369
- package/dist/auth/client.d.ts +65 -3
- package/dist/auth/client.js +305 -53
- package/dist/bin/bir-hooks.d.ts +72 -31
- package/dist/bin/bir-hooks.js +176 -284
- package/dist/bin/bir-proxy.d.ts +17 -2
- package/dist/bin/bir-proxy.js +222 -28
- package/dist/bin/bir-scenario.d.ts +61 -21
- package/dist/bin/bir-scenario.js +593 -122
- package/dist/bin/bir.d.ts +12 -6
- package/dist/bin/bir.js +2935 -489
- package/dist/bin/investigate.js +40 -15
- package/dist/bin/scenario-edit.d.ts +7 -1
- package/dist/bin/scenario-edit.js +43 -21
- package/dist/bin/scenario-liveness.d.ts +194 -0
- package/dist/bin/scenario-liveness.js +459 -0
- package/dist/bin/scenario-stderr.d.ts +43 -0
- package/dist/bin/scenario-stderr.js +76 -0
- package/dist/bin/service-read.d.ts +88 -0
- package/dist/bin/service-read.js +128 -0
- package/dist/bin/setup.d.ts +19 -0
- package/dist/bin/setup.js +146 -20
- package/dist/config/adapters/claude-code.d.ts +92 -14
- package/dist/config/adapters/claude-code.js +291 -101
- package/dist/config/adapters/generic.d.ts +4 -1
- package/dist/config/adapters/generic.js +13 -14
- package/dist/config/generate.d.ts +392 -14
- package/dist/config/generate.js +666 -19
- package/dist/config/mcp-approval.d.ts +76 -0
- package/dist/config/mcp-approval.js +210 -0
- package/dist/config/node-path.d.ts +42 -0
- package/dist/config/node-path.js +102 -0
- package/dist/config/overrides.d.ts +62 -0
- package/dist/config/overrides.js +136 -0
- package/dist/config/resolve.d.ts +70 -8
- package/dist/config/resolve.js +186 -15
- package/dist/control/client.d.ts +22 -0
- package/dist/control/client.js +26 -7
- package/dist/control/correlation.d.ts +23 -5
- package/dist/control/correlation.js +0 -0
- package/dist/control/daemon.d.ts +400 -18
- package/dist/control/daemon.js +1105 -102
- package/dist/control/discovery.d.ts +77 -5
- package/dist/control/discovery.js +186 -11
- package/dist/control/ensure-hook.d.ts +61 -0
- package/dist/control/ensure-hook.js +160 -8
- package/dist/control/paths.d.ts +26 -0
- package/dist/control/paths.js +34 -0
- package/dist/control/pin-guard.d.ts +83 -0
- package/dist/control/pin-guard.js +534 -0
- package/dist/control/recorder-keeper.d.ts +229 -0
- package/dist/control/recorder-keeper.js +721 -0
- package/dist/control/recorder-main.d.ts +73 -0
- package/dist/control/recorder-main.js +778 -0
- package/dist/control/server.d.ts +707 -31
- package/dist/control/server.js +2786 -302
- package/dist/control/transcript.d.ts +21 -1
- package/dist/control/transcript.js +378 -112
- package/dist/index.d.ts +5 -1
- package/dist/index.js +5 -1
- package/dist/jsonrpc/framing.d.ts +76 -12
- package/dist/jsonrpc/framing.js +216 -59
- package/dist/proxy/intercept.d.ts +49 -5
- package/dist/proxy/intercept.js +188 -59
- package/dist/proxy/relay.d.ts +16 -0
- package/dist/proxy/relay.js +77 -5
- package/dist/proxy/revive.d.ts +122 -0
- package/dist/proxy/revive.js +251 -0
- package/dist/proxy/session.d.ts +222 -14
- package/dist/proxy/session.js +708 -99
- package/dist/record/recorder.d.ts +51 -3
- package/dist/record/recorder.js +3 -0
- package/dist/record/redact.d.ts +103 -9
- package/dist/record/redact.js +659 -30
- package/dist/record/remote-recorder.d.ts +327 -32
- package/dist/record/remote-recorder.js +1517 -187
- package/dist/record/switchable-recorder.d.ts +126 -0
- package/dist/record/switchable-recorder.js +351 -0
- package/dist/record/truncate.d.ts +40 -1
- package/dist/record/truncate.js +111 -4
- package/dist/replay/bash-channel.d.ts +35 -0
- package/dist/replay/bash-channel.js +133 -0
- package/dist/replay/bundle.d.ts +10 -0
- package/dist/replay/bundle.js +10 -0
- package/dist/replay/controller.d.ts +369 -20
- package/dist/replay/controller.js +1176 -107
- package/dist/replay/derive.d.ts +1 -0
- package/dist/replay/derive.js +81 -24
- package/dist/replay/executor.d.ts +51 -16
- package/dist/replay/executor.js +114 -41
- package/dist/replay/handover.d.ts +54 -1
- package/dist/replay/handover.js +55 -5
- package/dist/replay/logic.d.ts +62 -10
- package/dist/replay/logic.js +140 -22
- package/dist/replay/plan.d.ts +201 -27
- package/dist/replay/plan.js +581 -141
- package/dist/replay/safety.d.ts +72 -0
- package/dist/replay/safety.js +303 -0
- package/dist/replay/source-run.d.ts +10 -0
- package/dist/replay/source-run.js +80 -35
- package/dist/replay/targets.d.ts +15 -1
- package/dist/replay/targets.js +62 -4
- package/dist/replay/tool-error.d.ts +15 -1
- package/dist/replay/tool-error.js +33 -1
- package/dist/replay/types.d.ts +7 -4
- package/dist/upstream/client.d.ts +56 -3
- package/dist/upstream/client.js +61 -4
- package/dist/upstream/headers.d.ts +43 -0
- package/dist/upstream/headers.js +90 -0
- package/dist/upstream/http-client.d.ts +213 -22
- package/dist/upstream/http-client.js +1034 -168
- package/dist/upstream/http-transport.d.ts +143 -0
- package/dist/upstream/http-transport.js +391 -0
- package/dist/upstream/lazy-client.d.ts +8 -0
- package/dist/upstream/lazy-client.js +29 -1
- package/dist/upstream/sse.d.ts +78 -0
- package/dist/upstream/sse.js +189 -0
- package/dist/upstream/stdio-client.d.ts +61 -1
- package/dist/upstream/stdio-client.js +356 -33
- package/dist/util/atomic-write.d.ts +29 -0
- package/dist/util/atomic-write.js +96 -0
- package/dist/util/journal.d.ts +15 -0
- package/dist/util/journal.js +48 -3
- package/dist/util/log.d.ts +10 -0
- package/dist/util/log.js +28 -1
- package/dist/util/off-switch.d.ts +46 -0
- package/dist/util/off-switch.js +91 -0
- package/dist/util/service-health.d.ts +133 -0
- package/dist/util/service-health.js +217 -0
- package/docs/BaseInstRunner.md +694 -621
- package/docs/calculatedReplay.md +1625 -1304
- package/docs/calculatedReplayGuide.md +1212 -1087
- package/docs/installRun.md +1013 -772
- package/docs/loginWeb.md +627 -607
- package/docs/mcpmark.md +49 -16
- package/docs/my-first-sample.md +569 -545
- package/docs/quickstart.md +384 -311
- package/docs/t-bench.md +406 -394
- package/package.json +58 -57
- package/production.md +1371 -0
- package/scripts/install.ps1 +97 -6
- package/scripts/install.sh +104 -7
package/dist/replay/plan.js
CHANGED
|
@@ -20,10 +20,174 @@
|
|
|
20
20
|
* step to know its reach, so it can choose between the proxy, a recorded
|
|
21
21
|
* output, and skipping.
|
|
22
22
|
*/
|
|
23
|
-
import { evalParamMapLogic, evalParamsLogic, evalResponseParamsLogic, evalResultMapLogic, evalToolInputLogic, evalToolOutputLogic, } from "./logic.js";
|
|
23
|
+
import { evalParamMapLogic, evalParamsLogic, evalResponseParamsLogic, evalResultMapLogic, evalToolInputLogic, evalToolOutputLogic, LogicTimeoutError, } from "./logic.js";
|
|
24
24
|
import { flatEntriesOf } from "./flatten.js";
|
|
25
25
|
import { assembleBundle, bundleInput, MAX_REPLAY_REASON } from "./bundle.js";
|
|
26
|
+
import { redact, redactText } from "../record/redact.js";
|
|
27
|
+
import { sinkOfPlan, unsafeParameter } from "./safety.js";
|
|
28
|
+
import { errText } from "../util/log.js";
|
|
29
|
+
import { awaitWithin } from "../util/service-health.js";
|
|
26
30
|
import { isMissingToolError, toolResultError } from "./tool-error.js";
|
|
31
|
+
/** A rejection from an {@link ExecuteStep} that says what happened to the call. */
|
|
32
|
+
export class StepRunError extends Error {
|
|
33
|
+
rejection;
|
|
34
|
+
constructor(message, rejection) {
|
|
35
|
+
super(message);
|
|
36
|
+
this.name = "StepRunError";
|
|
37
|
+
this.rejection = rejection;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
/** What a rejection means for the step — see {@link ExecuteStep}. */
|
|
41
|
+
export function rejectionOf(err) {
|
|
42
|
+
return err instanceof StepRunError ? err.rejection : "not_run";
|
|
43
|
+
}
|
|
44
|
+
/** A step's reasoning, as a directive shows it: one line, this long at most. */
|
|
45
|
+
const MAX_REASONING_CHARS = 120;
|
|
46
|
+
/** Past this, the directive's step list drops the reasoning and names tools only. */
|
|
47
|
+
const MAX_DIRECTIVE_STEPS_CHARS = 2_000;
|
|
48
|
+
function clipReasoning(reasoning) {
|
|
49
|
+
const first = (reasoning ?? "").trim().split(/\r?\n/, 1)[0]?.trim() ?? "";
|
|
50
|
+
return first.length > MAX_REASONING_CHARS ? `${first.slice(0, MAX_REASONING_CHARS)}…` : first;
|
|
51
|
+
}
|
|
52
|
+
/** A step's computed arguments, as a steer directive shows them: this long at most. */
|
|
53
|
+
const MAX_DIRECTIVE_ARGS_CHARS = 240;
|
|
54
|
+
/**
|
|
55
|
+
* One step's input logic, evaluated only to show its arguments, runs no longer
|
|
56
|
+
* than this (nor than `logicTimeoutMs()`).
|
|
57
|
+
*
|
|
58
|
+
* The limit is wall-clock time (a `vm` timeout), so it must tolerate a busy
|
|
59
|
+
* machine: with ten CPU-bound processes on eight cores, a trivial
|
|
60
|
+
* `return { command: 'ls ' + parameters.x }` took up to ~170 ms, and a 50 ms
|
|
61
|
+
* limit stopped more than half of them — ending the preview, and every
|
|
62
|
+
* argument with it, whenever the machine was loaded. At 250 ms none was
|
|
63
|
+
* stopped. A body stopped at this limit really is slow, and ends the preview.
|
|
64
|
+
*/
|
|
65
|
+
export const PREVIEW_LOGIC_MS = 250;
|
|
66
|
+
/**
|
|
67
|
+
* All of a directive's argument previews together: it is written inside the
|
|
68
|
+
* prompt hook, so this is the most the preview adds to it, a stopped body
|
|
69
|
+
* included — each body's limit is clipped to what is left of it. A caller with
|
|
70
|
+
* less time left passes less (`steeringDirective`'s `previewBudgetMs`).
|
|
71
|
+
*/
|
|
72
|
+
export const PREVIEW_BUDGET_MS = 500;
|
|
73
|
+
/** Below this much of the preview budget left, no further body is started. */
|
|
74
|
+
const MIN_PREVIEW_BODY_MS = 10;
|
|
75
|
+
/** Tools whose whole call is a `command`: the directive shows that, not a JSON object. */
|
|
76
|
+
const SHELL_TOOLS = new Set(["Bash", "PowerShell"]);
|
|
77
|
+
/**
|
|
78
|
+
* `target`, reporting any read of it. A step whose input logic reads what
|
|
79
|
+
* earlier steps accumulated cannot have its arguments known before they run.
|
|
80
|
+
* Writes land on `target`, which is a copy.
|
|
81
|
+
*/
|
|
82
|
+
function watchReads(target, onRead) {
|
|
83
|
+
return new Proxy(target, {
|
|
84
|
+
get(t, key, receiver) {
|
|
85
|
+
onRead();
|
|
86
|
+
return Reflect.get(t, key, receiver);
|
|
87
|
+
},
|
|
88
|
+
has(t, key) {
|
|
89
|
+
onRead();
|
|
90
|
+
return Reflect.has(t, key);
|
|
91
|
+
},
|
|
92
|
+
ownKeys(t) {
|
|
93
|
+
onRead();
|
|
94
|
+
return Reflect.ownKeys(t);
|
|
95
|
+
},
|
|
96
|
+
getOwnPropertyDescriptor(t, key) {
|
|
97
|
+
onRead();
|
|
98
|
+
return Reflect.getOwnPropertyDescriptor(t, key);
|
|
99
|
+
},
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Whether `value` holds `target` itself, anywhere a render would reach — a
|
|
104
|
+
* body that passes the accumulation on (`return { ctx: respParams }`) without
|
|
105
|
+
* reading it. Too deep or too wide to walk, the answer is yes: a preview that
|
|
106
|
+
* cannot tell shows nothing.
|
|
107
|
+
*/
|
|
108
|
+
function holds(value, target, depth = 0, seen = new Set()) {
|
|
109
|
+
if (value === target)
|
|
110
|
+
return true;
|
|
111
|
+
if (!value || typeof value !== "object")
|
|
112
|
+
return false;
|
|
113
|
+
if (depth >= MAX_HOLDS_DEPTH || seen.size >= MAX_HOLDS_NODES)
|
|
114
|
+
return true;
|
|
115
|
+
if (seen.has(value))
|
|
116
|
+
return false;
|
|
117
|
+
seen.add(value);
|
|
118
|
+
for (const v of Object.values(value))
|
|
119
|
+
if (holds(v, target, depth + 1, seen))
|
|
120
|
+
return true;
|
|
121
|
+
return false;
|
|
122
|
+
}
|
|
123
|
+
const MAX_HOLDS_DEPTH = 8;
|
|
124
|
+
const MAX_HOLDS_NODES = 1_000;
|
|
125
|
+
/** A deep copy for a look-ahead evaluation, or undefined when the value cannot be copied. */
|
|
126
|
+
function copyOf(value) {
|
|
127
|
+
try {
|
|
128
|
+
return structuredClone(value);
|
|
129
|
+
}
|
|
130
|
+
catch {
|
|
131
|
+
return undefined;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* One step's computed arguments as a steer directive shows them, or undefined
|
|
136
|
+
* when there is nothing useful to show.
|
|
137
|
+
*
|
|
138
|
+
* Data, and shaped to read as data: a shell step's command, anything else as
|
|
139
|
+
* its JSON. A command is shown WHOLE — one line, within the limit — or not at
|
|
140
|
+
* all: the model is told to pass what it sees, and a call the pin does not
|
|
141
|
+
* take (a hook that failed open, a plan already retired, a step copied out of
|
|
142
|
+
* order) runs as the model wrote it — `cat > config.yaml <<'EOF'` cut from its
|
|
143
|
+
* body empties the file. (One line also keeps a heredoc from spilling the
|
|
144
|
+
* body of a script into the prompt.) Secrets are redacted before anything is
|
|
145
|
+
* laid out — a key the redactor knows (`password`, `token`) and a
|
|
146
|
+
* secret-shaped value alike. A
|
|
147
|
+
* value too long for the line is elided whole (`"body":…`) rather than cut: a
|
|
148
|
+
* cut path or URL copied into the call would name a different target, where
|
|
149
|
+
* an elided one leaves the model to write its own and the pin to supply it.
|
|
150
|
+
*/
|
|
151
|
+
function renderArguments(toolName, input) {
|
|
152
|
+
if (SHELL_TOOLS.has(toolName)) {
|
|
153
|
+
const raw = typeof input.command === "string" ? input.command.trim() : "";
|
|
154
|
+
if (!raw || /[\r\n]/.test(raw))
|
|
155
|
+
return undefined;
|
|
156
|
+
const shown = redactText(raw);
|
|
157
|
+
if (/[\r\n]/.test(shown) || shown.length > MAX_DIRECTIVE_ARGS_CHARS)
|
|
158
|
+
return undefined;
|
|
159
|
+
return `command: ${shown}`;
|
|
160
|
+
}
|
|
161
|
+
const clean = redact(input);
|
|
162
|
+
if (!clean || typeof clean !== "object" || Array.isArray(clean))
|
|
163
|
+
return undefined;
|
|
164
|
+
const parts = [];
|
|
165
|
+
let used = 2;
|
|
166
|
+
let shortened = false;
|
|
167
|
+
for (const [key, value] of Object.entries(clean)) {
|
|
168
|
+
let json;
|
|
169
|
+
try {
|
|
170
|
+
json = JSON.stringify(value);
|
|
171
|
+
}
|
|
172
|
+
catch {
|
|
173
|
+
json = undefined;
|
|
174
|
+
}
|
|
175
|
+
if (json === undefined)
|
|
176
|
+
continue;
|
|
177
|
+
const name = JSON.stringify(key);
|
|
178
|
+
const whole = `${name}:${redactText(json)}`;
|
|
179
|
+
const part = used + whole.length + 1 <= MAX_DIRECTIVE_ARGS_CHARS ? whole : `${name}:…`;
|
|
180
|
+
if (part !== whole)
|
|
181
|
+
shortened = true;
|
|
182
|
+
if (used + part.length + 1 > MAX_DIRECTIVE_ARGS_CHARS) {
|
|
183
|
+
shortened = true;
|
|
184
|
+
break;
|
|
185
|
+
}
|
|
186
|
+
parts.push(part);
|
|
187
|
+
used += part.length + 1;
|
|
188
|
+
}
|
|
189
|
+
return `arguments: {${parts.join(",")}}${shortened ? " (shortened)" : ""}`;
|
|
190
|
+
}
|
|
27
191
|
/**
|
|
28
192
|
* The recorded values for the *settings* a mapping did not name (R-CALL-21).
|
|
29
193
|
*
|
|
@@ -79,6 +243,11 @@ export class ScenarioReplayPlan {
|
|
|
79
243
|
entries;
|
|
80
244
|
paramsPromise;
|
|
81
245
|
params = {};
|
|
246
|
+
/** The derivation's own answer, before `paramsLogic`. What safety.ts judges. */
|
|
247
|
+
derivedParams = {};
|
|
248
|
+
/** Set once the parameters are final — derived, or settled without the derivation. */
|
|
249
|
+
paramsSettled = false;
|
|
250
|
+
fallbackTargets;
|
|
82
251
|
respParams;
|
|
83
252
|
stepIndex = 0;
|
|
84
253
|
readyPromise;
|
|
@@ -94,6 +263,7 @@ export class ScenarioReplayPlan {
|
|
|
94
263
|
this.entries = opts.entries ?? flatEntriesOf(opts.scenario);
|
|
95
264
|
this.mode = opts.mode;
|
|
96
265
|
this.paramsPromise = opts.params;
|
|
266
|
+
this.fallbackTargets = opts.fallbackTargets ?? "null";
|
|
97
267
|
this.respParams = { ...(opts.respParamsInit ?? {}) };
|
|
98
268
|
// The caller's own frame accumulates into the plan's `respParams`, so
|
|
99
269
|
// `accumulated`, `responseModel` and every depth-0 step see one object.
|
|
@@ -179,26 +349,77 @@ export class ScenarioReplayPlan {
|
|
|
179
349
|
}
|
|
180
350
|
/**
|
|
181
351
|
* Await the derivation, then apply `paramsLogic` — once, however many callers
|
|
182
|
-
* race here. A rejected derivation
|
|
183
|
-
*
|
|
184
|
-
*
|
|
352
|
+
* race here. A rejected derivation settles on the fallback values
|
|
353
|
+
* ({@link useFallbackParams}) rather than throwing.
|
|
354
|
+
*
|
|
355
|
+
* Rejects only when `paramsLogic` throws. That is real: the scenario's own
|
|
356
|
+
* logic is broken, and the caller must hand over rather than run steps on
|
|
357
|
+
* parameters nobody processed.
|
|
185
358
|
*/
|
|
186
359
|
ready() {
|
|
360
|
+
if (this.paramsSettled)
|
|
361
|
+
return Promise.resolve();
|
|
187
362
|
if (!this.readyPromise) {
|
|
188
363
|
this.readyPromise = this.paramsPromise
|
|
189
|
-
.
|
|
364
|
+
.then((derived) => derived, () => undefined)
|
|
190
365
|
.then((derived) => {
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
}
|
|
197
|
-
this.params = params;
|
|
366
|
+
// Settled already, without the derivation: a late answer must not
|
|
367
|
+
// change the parameters under steps that ran on the others.
|
|
368
|
+
if (this.paramsSettled)
|
|
369
|
+
return;
|
|
370
|
+
this.settleParams(derived ?? this.fallbackValues());
|
|
198
371
|
});
|
|
199
372
|
}
|
|
200
373
|
return this.readyPromise;
|
|
201
374
|
}
|
|
375
|
+
/**
|
|
376
|
+
* Settle the parameters without waiting any longer for the derivation: every
|
|
377
|
+
* *setting* at its recorded sample, every *target* `null` (R-PARAM-3), then
|
|
378
|
+
* `paramsLogic`. What the caller does when the derivation's budget runs out.
|
|
379
|
+
*
|
|
380
|
+
* Never `{}`. Running on an empty object put `undefined` into every argument
|
|
381
|
+
* — `{"command":"query undefined"}` — and a steered step was auto-approved
|
|
382
|
+
* with it. A target stays null here exactly as it does when the derivation
|
|
383
|
+
* answers without one, and the caller decides whether the plan can run
|
|
384
|
+
* without it (targets.ts).
|
|
385
|
+
*
|
|
386
|
+
* A derivation that answers later is ignored. Throws what `paramsLogic`
|
|
387
|
+
* throws.
|
|
388
|
+
*/
|
|
389
|
+
useFallbackParams() {
|
|
390
|
+
if (this.paramsSettled)
|
|
391
|
+
return;
|
|
392
|
+
this.settleParams(this.fallbackValues());
|
|
393
|
+
}
|
|
394
|
+
/** Whether the parameters are final. */
|
|
395
|
+
get parametersSettled() {
|
|
396
|
+
return this.paramsSettled;
|
|
397
|
+
}
|
|
398
|
+
/** The derivation's own values, before `paramsLogic`. Empty until settled. */
|
|
399
|
+
get derivedValues() {
|
|
400
|
+
return this.derivedParams;
|
|
401
|
+
}
|
|
402
|
+
fallbackValues() {
|
|
403
|
+
const out = {};
|
|
404
|
+
const schema = this.scenario.paramsObject;
|
|
405
|
+
if (!schema || typeof schema !== "object")
|
|
406
|
+
return out;
|
|
407
|
+
for (const [key, p] of Object.entries(schema)) {
|
|
408
|
+
out[key] =
|
|
409
|
+
p?.kind === "setting" || this.fallbackTargets === "samples" ? p?.sampleValue : null;
|
|
410
|
+
}
|
|
411
|
+
return out;
|
|
412
|
+
}
|
|
413
|
+
settleParams(derived) {
|
|
414
|
+
const own = { ...derived };
|
|
415
|
+
let params = { ...derived };
|
|
416
|
+
if (this.scenario.paramsLogic) {
|
|
417
|
+
params = evalParamsLogic(this.scenario.paramsLogic, params, this.intent);
|
|
418
|
+
}
|
|
419
|
+
this.derivedParams = own;
|
|
420
|
+
this.params = params;
|
|
421
|
+
this.paramsSettled = true;
|
|
422
|
+
}
|
|
202
423
|
/** 0-based index of the step awaiting execution (== completed step count). */
|
|
203
424
|
get currentStepIndex() {
|
|
204
425
|
return this.stepIndex;
|
|
@@ -243,6 +464,14 @@ export class ScenarioReplayPlan {
|
|
|
243
464
|
get accumulated() {
|
|
244
465
|
return this.respParams;
|
|
245
466
|
}
|
|
467
|
+
/** The scenario's parameter schema — what the target and safety checks judge. */
|
|
468
|
+
get paramsSchema() {
|
|
469
|
+
return this.scenario.paramsObject ?? null;
|
|
470
|
+
}
|
|
471
|
+
/** The scenario's `paramsLogic`, which targets.ts reads as a consumer. */
|
|
472
|
+
get paramsLogic() {
|
|
473
|
+
return this.scenario.paramsLogic ?? null;
|
|
474
|
+
}
|
|
246
475
|
/** The step awaiting execution, or undefined when the plan is done. */
|
|
247
476
|
currentStep() {
|
|
248
477
|
return this.currentEntry()?.step;
|
|
@@ -317,6 +546,81 @@ export class ScenarioReplayPlan {
|
|
|
317
546
|
return { ...this.respParams };
|
|
318
547
|
return evalResponseParamsLogic(this.scenario.responseParamsLogic, this.respParams, this.params, this.intent);
|
|
319
548
|
}
|
|
549
|
+
/**
|
|
550
|
+
* The arguments each planned step will run with, where they are known before
|
|
551
|
+
* any step runs — for the steer directive (see {@link steeringDirective}).
|
|
552
|
+
*
|
|
553
|
+
* Known means: the parameters are settled, the step is the caller's own
|
|
554
|
+
* (a called segment's parameters are only built when it is entered), and
|
|
555
|
+
* its input logic neither reads nor passes on anything an earlier step
|
|
556
|
+
* accumulates — or it is the step at the cursor, whose accumulation is
|
|
557
|
+
* already final. Everything else is undefined, and the directive leaves
|
|
558
|
+
* those steps' arguments to the system.
|
|
559
|
+
*
|
|
560
|
+
* A look-ahead only: every body runs on copies, under a short limit, and a
|
|
561
|
+
* body that throws or runs long just leaves its step without arguments —
|
|
562
|
+
* the pin, not this, is what runs the step. The whole preview takes no more
|
|
563
|
+
* than `budgetMs` ({@link PREVIEW_BUDGET_MS}), a stopped body included; each
|
|
564
|
+
* body gets at most `bodyMs` ({@link PREVIEW_LOGIC_MS}) of it.
|
|
565
|
+
*/
|
|
566
|
+
previewArguments(planned, budgetMs = PREVIEW_BUDGET_MS, bodyMs = PREVIEW_LOGIC_MS) {
|
|
567
|
+
const out = planned.map(() => undefined);
|
|
568
|
+
if (this.mode !== "steer" || !this.paramsSettled)
|
|
569
|
+
return out;
|
|
570
|
+
// NaN, a negative or a sliver: no preview, and the directive's old wording.
|
|
571
|
+
if (!(budgetMs >= MIN_PREVIEW_BODY_MS) || !(bodyMs >= 1))
|
|
572
|
+
return out;
|
|
573
|
+
const deadline = Date.now() + budgetMs;
|
|
574
|
+
// A value the pin would refuse to template into a command is never shown
|
|
575
|
+
// in one either: the model could run what the plan would not.
|
|
576
|
+
const sink = sinkOfPlan(planned);
|
|
577
|
+
if (sink && unsafeParameter(this.derivedParams, this.scenario.paramsObject, sink))
|
|
578
|
+
return out;
|
|
579
|
+
for (let i = this.stepIndex; i < planned.length; i += 1) {
|
|
580
|
+
const left = deadline - Date.now();
|
|
581
|
+
if (left < MIN_PREVIEW_BODY_MS)
|
|
582
|
+
break;
|
|
583
|
+
const entry = planned[i];
|
|
584
|
+
if (entry.depth !== 0)
|
|
585
|
+
continue;
|
|
586
|
+
const params = copyOf(this.params);
|
|
587
|
+
const accumulated = copyOf(this.respParams);
|
|
588
|
+
if (!params || !accumulated)
|
|
589
|
+
break;
|
|
590
|
+
let readsEarlier = false;
|
|
591
|
+
const resp = i === this.stepIndex
|
|
592
|
+
? accumulated
|
|
593
|
+
: watchReads(accumulated, () => {
|
|
594
|
+
readsEarlier = true;
|
|
595
|
+
});
|
|
596
|
+
let input;
|
|
597
|
+
try {
|
|
598
|
+
input = evalToolInputLogic(entry.step.toolInputLogic ?? "return {};", params, entry.frame.intent, resp, { timeoutMs: Math.min(bodyMs, left) });
|
|
599
|
+
}
|
|
600
|
+
catch (err) {
|
|
601
|
+
// A body that throws (often on an earlier result it does not have
|
|
602
|
+
// yet) only leaves its own step bare. One that had to be stopped
|
|
603
|
+
// either ran long — this scenario's logic is not worth running ahead —
|
|
604
|
+
// or used up what was left of the budget: either way no more of it is.
|
|
605
|
+
if (err instanceof LogicTimeoutError)
|
|
606
|
+
break;
|
|
607
|
+
continue;
|
|
608
|
+
}
|
|
609
|
+
// Reading the accumulation, or passing it on whole (`{ ctx: respParams }`,
|
|
610
|
+
// which reads nothing yet runs with every earlier result): not known yet.
|
|
611
|
+
// Checked before rendering, which would walk into the watched copy.
|
|
612
|
+
try {
|
|
613
|
+
if (readsEarlier || (resp !== accumulated && holds(input, resp)))
|
|
614
|
+
continue;
|
|
615
|
+
out[i] = renderArguments(entry.step.toolName ?? "", input);
|
|
616
|
+
}
|
|
617
|
+
catch {
|
|
618
|
+
// A value that throws when read (a getter the body returned): nothing
|
|
619
|
+
// to show for this step, and never a directive that fails to render.
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
return out;
|
|
623
|
+
}
|
|
320
624
|
/**
|
|
321
625
|
* The directive injected via `UserPromptSubmit`'s `additionalContext`.
|
|
322
626
|
*
|
|
@@ -324,6 +628,20 @@ export class ScenarioReplayPlan {
|
|
|
324
628
|
* whole sequence server-side. In `steer` mode it names the sequence so the
|
|
325
629
|
* model initiates the expected calls — the arguments are supplied by the
|
|
326
630
|
* system, so the model need not compute them.
|
|
631
|
+
*
|
|
632
|
+
* WHY A STEER DIRECTIVE SHOWS ARGUMENTS. The pin replaces the model's
|
|
633
|
+
* arguments with the step's only when the call is plausibly the step
|
|
634
|
+
* (control/pin-guard.ts): the same command head, the same file, an argument
|
|
635
|
+
* in common. Told merely that "the system supplies the arguments", a model
|
|
636
|
+
* writes a placeholder call — `ls`, `{"query":"..."}` — that the guard rightly
|
|
637
|
+
* declines, and the plan retires having run nothing. So where a step's
|
|
638
|
+
* arguments are already known they are shown under it, as data, so that the
|
|
639
|
+
* call the model writes, and the user sees, is the call that runs.
|
|
640
|
+
*
|
|
641
|
+
* `previewBudgetMs` / `previewBodyMs` bound that preview (default
|
|
642
|
+
* {@link PREVIEW_BUDGET_MS} / {@link PREVIEW_LOGIC_MS}): a caller with less
|
|
643
|
+
* of its hook's time left passes less, and under
|
|
644
|
+
* {@link MIN_PREVIEW_BODY_MS} there is no preview at all.
|
|
327
645
|
*/
|
|
328
646
|
steeringDirective(directToolName = "mcp__bir__run_scenario", opts) {
|
|
329
647
|
// A sub-task plan (segmented.md R-OUT-13) never tells the agent to stop
|
|
@@ -332,10 +650,39 @@ export class ScenarioReplayPlan {
|
|
|
332
650
|
// The intent is a paragraph the analyser wrote; it is context, not an
|
|
333
651
|
// instruction, so it is clipped rather than dumped whole into the prompt.
|
|
334
652
|
const intent = this.intent.length > 240 ? `${this.intent.slice(0, 240)}…` : this.intent;
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
653
|
+
// Each step's reasoning is another session's narration ("I'll now drop the
|
|
654
|
+
// temp table…"), injected into this one. It is a label here, not a
|
|
655
|
+
// paragraph: its first line, clipped. A chain long enough that even the
|
|
656
|
+
// labels would run to kilobytes lists its tools alone — every tool always,
|
|
657
|
+
// because in steer mode the list *is* the instruction.
|
|
658
|
+
const planned = this.entries.slice(0, this.stopAt);
|
|
659
|
+
const args = this.previewArguments(planned, opts?.previewBudgetMs, opts?.previewBodyMs);
|
|
660
|
+
const listing = (withWhy, argsBefore) => planned.map((e, i) => {
|
|
661
|
+
const why = withWhy ? clipReasoning(e.step.reasoning) : "";
|
|
662
|
+
const head = ` ${i + 1}. ${e.step.toolName ?? ""}${why ? ` — ${why}` : ""}`;
|
|
663
|
+
const shown = i < argsBefore ? args[i] : undefined;
|
|
664
|
+
return shown ? `${head}\n ${shown}` : head;
|
|
338
665
|
});
|
|
666
|
+
const fits = (ls) => ls.join("\n").length <= MAX_DIRECTIVE_STEPS_CHARS;
|
|
667
|
+
// Within the same limit as ever, in order of use to the pin: every step's
|
|
668
|
+
// arguments, then the earliest steps' (the first call is the one a plan
|
|
669
|
+
// most often dies on), then labels, then bare names.
|
|
670
|
+
const lastArgs = args.reduce((n, a, i) => (a ? i + 1 : n), 0);
|
|
671
|
+
const candidates = [[true, lastArgs], [false, lastArgs]];
|
|
672
|
+
for (let k = lastArgs - 1; k > 0; k -= 1)
|
|
673
|
+
candidates.push([false, k]);
|
|
674
|
+
candidates.push([true, 0]);
|
|
675
|
+
let lines = listing(false, 0);
|
|
676
|
+
let argsShown = 0;
|
|
677
|
+
for (const [withWhy, before] of candidates) {
|
|
678
|
+
const ls = listing(withWhy, before);
|
|
679
|
+
if (!fits(ls))
|
|
680
|
+
continue;
|
|
681
|
+
lines = ls;
|
|
682
|
+
argsShown = before;
|
|
683
|
+
break;
|
|
684
|
+
}
|
|
685
|
+
const anyArgs = args.slice(0, argsShown).some((a) => a !== undefined);
|
|
339
686
|
// A parked step (fallbk.md D3): the model is told up front that the plan is
|
|
340
687
|
// partial, so the hand-over that follows is expected rather than a surprise.
|
|
341
688
|
const tail = this.stopsEarly()
|
|
@@ -370,15 +717,20 @@ export class ScenarioReplayPlan {
|
|
|
370
717
|
...lines,
|
|
371
718
|
...(intent ? ["", `Context: ${intent}`] : []),
|
|
372
719
|
"",
|
|
373
|
-
|
|
720
|
+
...(anyArgs
|
|
721
|
+
? [
|
|
722
|
+
"The system supplies the exact arguments for each call. Where a step shows",
|
|
723
|
+
"its arguments above, they are the ones it runs with: pass them as shown.",
|
|
724
|
+
]
|
|
725
|
+
: ["The system supplies the exact arguments for each call — you do not need to"]),
|
|
374
726
|
...(subTask
|
|
375
727
|
? [
|
|
376
|
-
"compute them. Do not call any other tools until this sequence is complete
|
|
728
|
+
`${anyArgs ? "" : "compute them. "}Do not call any other tools until this sequence is complete.`,
|
|
377
729
|
"These steps are part of your task: when they are done, continue your task,",
|
|
378
730
|
"calling tools as needed.",
|
|
379
731
|
]
|
|
380
732
|
: [
|
|
381
|
-
"compute them. Do not call any other tools until this sequence is complete
|
|
733
|
+
`${anyArgs ? "" : "compute them. "}Do not call any other tools until this sequence is complete,`,
|
|
382
734
|
"then answer the user's request from the tool results.",
|
|
383
735
|
]),
|
|
384
736
|
...tail,
|
|
@@ -391,19 +743,20 @@ export class ScenarioReplayPlan {
|
|
|
391
743
|
* live session already ran — with side effects — every step before the one it
|
|
392
744
|
* diverged on, so re-running them would be wrong.
|
|
393
745
|
*
|
|
394
|
-
*
|
|
395
|
-
*
|
|
396
|
-
*
|
|
397
|
-
*
|
|
398
|
-
*
|
|
399
|
-
*
|
|
400
|
-
*
|
|
746
|
+
* A step whose call produced no result ends the bundle there
|
|
747
|
+
* ({@link DriveOptions}): later steps would compute on a result that does not
|
|
748
|
+
* exist, and nothing may be shown in its place. The stop is returned, and the
|
|
749
|
+
* bundle says the task is not finished. With `stopOnFailure` a logic throw or
|
|
750
|
+
* a tool error ends it the same way; without it they keep their old contract
|
|
751
|
+
* (a throw propagates, an error is threaded and counted).
|
|
752
|
+
*
|
|
753
|
+
* `deadline` bounds the whole bundle: this runs inside a `PreToolUse` hook
|
|
754
|
+
* with a timeout of its own, and a step started after the host stopped
|
|
755
|
+
* waiting is a side effect nobody is told about.
|
|
401
756
|
*/
|
|
402
757
|
async composeBundle(maxChars, execute, recordedOutputFor, onStep, opts = {}) {
|
|
403
|
-
//
|
|
404
|
-
// the caller decides to keep steering afterwards.
|
|
405
|
-
// One copy per frame, for the same reason the plan copied its own: composing
|
|
406
|
-
// must not corrupt the live accumulation if the caller keeps steering.
|
|
758
|
+
// One copy per frame: composing must not corrupt the live plan's
|
|
759
|
+
// accumulation if the caller decides to keep steering afterwards.
|
|
407
760
|
const resp = new Map();
|
|
408
761
|
if (this.entries[0])
|
|
409
762
|
resp.set(this.rootFrame().id, { ...this.respParams });
|
|
@@ -412,61 +765,102 @@ export class ScenarioReplayPlan {
|
|
|
412
765
|
let recordedCount = 0;
|
|
413
766
|
let skipped = 0;
|
|
414
767
|
let errored = 0;
|
|
415
|
-
|
|
768
|
+
let stopped;
|
|
769
|
+
const stopOnFailure = opts.stopOnFailure === true;
|
|
770
|
+
for (let flatIndex = this.stepIndex; flatIndex < this.stopAt; flatIndex += 1) {
|
|
771
|
+
const entry = this.entries[flatIndex];
|
|
416
772
|
const step = entry.step;
|
|
773
|
+
const before = this.stopBefore(entry, flatIndex, opts.signal, opts.deadline);
|
|
774
|
+
if (before) {
|
|
775
|
+
stopped = before;
|
|
776
|
+
break;
|
|
777
|
+
}
|
|
417
778
|
const startedAt = Date.now();
|
|
418
|
-
|
|
419
|
-
this.enterFrame(entry, resp);
|
|
420
|
-
const respParams = resp.get(entry.frame.id) ?? {};
|
|
421
|
-
return evalToolInputLogic(step.toolInputLogic ?? "return {};", this.paramsFor(entry), entry.frame.intent, respParams);
|
|
422
|
-
});
|
|
423
|
-
let response;
|
|
424
|
-
let recorded = false;
|
|
779
|
+
let computed;
|
|
425
780
|
try {
|
|
426
|
-
|
|
427
|
-
|
|
781
|
+
computed = reportingStage(onStep, entry, "tool_input_logic", {}, startedAt, () => {
|
|
782
|
+
this.enterFrame(entry, resp);
|
|
783
|
+
const respParams = resp.get(entry.frame.id) ?? {};
|
|
784
|
+
return evalToolInputLogic(step.toolInputLogic ?? "return {};", this.paramsFor(entry), entry.frame.intent, respParams);
|
|
785
|
+
});
|
|
428
786
|
}
|
|
429
787
|
catch (err) {
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
:
|
|
434
|
-
|
|
435
|
-
|
|
788
|
+
if (!stopOnFailure)
|
|
789
|
+
throw err;
|
|
790
|
+
stopped = {
|
|
791
|
+
kind: "step_failed",
|
|
792
|
+
stepIndex: step.stepIndex,
|
|
793
|
+
flatIndex,
|
|
794
|
+
stage: "tool_input_logic",
|
|
795
|
+
error: errText(err),
|
|
796
|
+
};
|
|
797
|
+
break;
|
|
798
|
+
}
|
|
799
|
+
const call = await this.callStep(entry, flatIndex, computed, startedAt, execute, recordedOutputFor, onStep, opts, opts.deadline);
|
|
800
|
+
if (call.stopped) {
|
|
801
|
+
skipped += 1;
|
|
802
|
+
stopped = call.stopped;
|
|
803
|
+
break;
|
|
804
|
+
}
|
|
805
|
+
const { response, recorded } = call;
|
|
806
|
+
if (recorded)
|
|
807
|
+
recordedCount += 1;
|
|
808
|
+
else
|
|
809
|
+
executed += 1;
|
|
810
|
+
if (stopOnFailure && !recorded) {
|
|
811
|
+
// Judged before the output logic, which was written against a success
|
|
812
|
+
// and would only throw a second, less useful error on this one.
|
|
813
|
+
const toolError = toolResultError(response);
|
|
814
|
+
if (toolError) {
|
|
815
|
+
errored += 1;
|
|
436
816
|
onStep?.({
|
|
437
817
|
step,
|
|
438
818
|
entry,
|
|
439
819
|
input: computed,
|
|
440
|
-
outcome: "
|
|
441
|
-
|
|
442
|
-
|
|
820
|
+
outcome: "failed",
|
|
821
|
+
stage: "tool_call",
|
|
822
|
+
error: toolError,
|
|
823
|
+
toolMissing: isMissingToolError(toolError) || undefined,
|
|
443
824
|
ms: Date.now() - startedAt,
|
|
444
825
|
});
|
|
445
|
-
|
|
826
|
+
entries.push({ toolName: step.toolName ?? "", input: bundleInput(computed), response, recorded });
|
|
827
|
+
stopped = {
|
|
828
|
+
kind: "step_failed",
|
|
829
|
+
stepIndex: step.stepIndex,
|
|
830
|
+
flatIndex,
|
|
831
|
+
stage: "tool_call",
|
|
832
|
+
error: toolError,
|
|
833
|
+
};
|
|
834
|
+
break;
|
|
446
835
|
}
|
|
447
|
-
response = fallback;
|
|
448
|
-
recorded = true;
|
|
449
|
-
recordedCount += 1;
|
|
450
|
-
onStep?.({
|
|
451
|
-
step,
|
|
452
|
-
entry,
|
|
453
|
-
input: computed,
|
|
454
|
-
outcome: "recorded",
|
|
455
|
-
error,
|
|
456
|
-
toolMissing: isMissingToolError(error) || undefined,
|
|
457
|
-
ms: Date.now() - startedAt,
|
|
458
|
-
});
|
|
459
836
|
}
|
|
460
837
|
// Thread the output for later steps' inputs. A recorded output threads
|
|
461
|
-
// too: `toolOutputLogic` was authored against exactly this shape
|
|
462
|
-
// stale value beats a missing one downstream.
|
|
838
|
+
// too: `toolOutputLogic` was authored against exactly this shape.
|
|
463
839
|
let derivedKeys = [];
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
840
|
+
try {
|
|
841
|
+
reportingStage(onStep, entry, "tool_output_logic", computed, startedAt, () => {
|
|
842
|
+
if (step.toolOutputLogic) {
|
|
843
|
+
const derived = evalToolOutputLogic(step.toolOutputLogic, response, this.paramsFor(entry), entry.frame.intent, resp.get(entry.frame.id) ?? {});
|
|
844
|
+
derivedKeys = Object.keys(derived);
|
|
845
|
+
Object.assign(resp.get(entry.frame.id) ?? {}, derived);
|
|
846
|
+
}
|
|
847
|
+
this.leaveFrame(entry, resp);
|
|
848
|
+
});
|
|
849
|
+
}
|
|
850
|
+
catch (err) {
|
|
851
|
+
if (!stopOnFailure)
|
|
852
|
+
throw err;
|
|
853
|
+
// The tool ran; its output is real and belongs in front of the model.
|
|
854
|
+
entries.push({ toolName: step.toolName ?? "", input: bundleInput(computed), response, recorded });
|
|
855
|
+
stopped = {
|
|
856
|
+
kind: "step_failed",
|
|
857
|
+
stepIndex: step.stepIndex,
|
|
858
|
+
flatIndex,
|
|
859
|
+
stage: "tool_output_logic",
|
|
860
|
+
error: errText(err),
|
|
861
|
+
};
|
|
862
|
+
break;
|
|
468
863
|
}
|
|
469
|
-
this.leaveFrame(entry, resp);
|
|
470
864
|
if (!recorded) {
|
|
471
865
|
// A step's verdict is its output, not its resolution: a tool that ran
|
|
472
866
|
// and answered "Error: …" resolves like any other, and a chain of eight
|
|
@@ -486,22 +880,20 @@ export class ScenarioReplayPlan {
|
|
|
486
880
|
ms: Date.now() - startedAt,
|
|
487
881
|
});
|
|
488
882
|
}
|
|
489
|
-
entries.push({
|
|
490
|
-
toolName: step.toolName ?? "",
|
|
491
|
-
input: bundleInput(computed),
|
|
492
|
-
response,
|
|
493
|
-
recorded,
|
|
494
|
-
});
|
|
883
|
+
entries.push({ toolName: step.toolName ?? "", input: bundleInput(computed), response, recorded });
|
|
495
884
|
}
|
|
496
885
|
return {
|
|
497
886
|
text: assembleBundle(entries, maxChars, {
|
|
498
|
-
|
|
887
|
+
// A bundle that stops short of the task — in front of a parked step, or
|
|
888
|
+
// at a step that broke — must never say "the work is done".
|
|
889
|
+
handover: opts.handover === true || stopped !== undefined,
|
|
499
890
|
subTask: this.subTask,
|
|
500
891
|
}),
|
|
501
892
|
executed,
|
|
502
893
|
recorded: recordedCount,
|
|
503
894
|
skipped,
|
|
504
895
|
errored,
|
|
896
|
+
stopped,
|
|
505
897
|
};
|
|
506
898
|
}
|
|
507
899
|
/**
|
|
@@ -510,8 +902,12 @@ export class ScenarioReplayPlan {
|
|
|
510
902
|
* This is `direct` mode's driver, and unlike {@link composeBundle} it mutates
|
|
511
903
|
* the plan — the steps really did run, so the cursor and `respParams` must say
|
|
512
904
|
* so, which is what lets the outcome be `steered_full` and `stepsPinned` be
|
|
513
|
-
* truthful.
|
|
514
|
-
*
|
|
905
|
+
* truthful.
|
|
906
|
+
*
|
|
907
|
+
* A step whose call produced no result stops the run there, with the cursor
|
|
908
|
+
* on it ({@link DriveOptions}). So does the deadline, checked before each
|
|
909
|
+
* step, and the caller's `signal`. A step already sent is never abandoned:
|
|
910
|
+
* its result arrives (or its own timeout does) and is reported as it is.
|
|
515
911
|
*
|
|
516
912
|
* `maxChars` caps the bundle exactly as a divergence bundle is capped. The
|
|
517
913
|
* direct path once passed `Number.MAX_SAFE_INTEGER` here — fine at 10 KB, and
|
|
@@ -525,19 +921,24 @@ export class ScenarioReplayPlan {
|
|
|
525
921
|
let skipped = 0;
|
|
526
922
|
let errored = 0;
|
|
527
923
|
let partial = false;
|
|
924
|
+
let aborted = false;
|
|
528
925
|
let stopped;
|
|
529
926
|
const stopOnFailure = opts.stopOnFailure === true;
|
|
530
|
-
const errText = (err) => (err instanceof Error ? err.message : String(err));
|
|
531
927
|
while (!this.isDone()) {
|
|
532
|
-
if (deadline !== undefined && Date.now() >= deadline) {
|
|
533
|
-
partial = true;
|
|
534
|
-
break;
|
|
535
|
-
}
|
|
536
928
|
const entry = this.entries[this.stepIndex];
|
|
537
929
|
const step = entry.step;
|
|
538
930
|
// Where this step sits in the flat list — what a stop reports. Taken now,
|
|
539
931
|
// because the cursor has moved on by the time some stops are written.
|
|
540
932
|
const flatIndex = this.stepIndex;
|
|
933
|
+
const before = this.stopBefore(entry, flatIndex, opts.signal, deadline);
|
|
934
|
+
if (before) {
|
|
935
|
+
stopped = before;
|
|
936
|
+
if (before.reason === "budget")
|
|
937
|
+
partial = true;
|
|
938
|
+
else
|
|
939
|
+
aborted = true;
|
|
940
|
+
break;
|
|
941
|
+
}
|
|
541
942
|
const startedAt = Date.now();
|
|
542
943
|
let computed;
|
|
543
944
|
try {
|
|
@@ -555,48 +956,17 @@ export class ScenarioReplayPlan {
|
|
|
555
956
|
};
|
|
556
957
|
break;
|
|
557
958
|
}
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
959
|
+
const call = await this.callStep(entry, flatIndex, computed, startedAt, execute, recordedOutputFor, onStep, opts, deadline);
|
|
960
|
+
if (call.stopped) {
|
|
961
|
+
skipped += 1;
|
|
962
|
+
stopped = call.stopped;
|
|
963
|
+
break;
|
|
563
964
|
}
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
const fallback = (typeof step.recordedOutput === "string" && step.recordedOutput.length > 0
|
|
567
|
-
? step.recordedOutput
|
|
568
|
-
: undefined) ?? (await recordedOutputFor?.(entry).catch(() => undefined));
|
|
569
|
-
if (!fallback) {
|
|
570
|
-
skipped += 1;
|
|
571
|
-
onStep?.({
|
|
572
|
-
step,
|
|
573
|
-
entry,
|
|
574
|
-
input: computed,
|
|
575
|
-
outcome: "skipped",
|
|
576
|
-
error,
|
|
577
|
-
toolMissing: isMissingToolError(error) || undefined,
|
|
578
|
-
ms: Date.now() - startedAt,
|
|
579
|
-
});
|
|
580
|
-
// Advance past a step that cannot run, without threading anything.
|
|
581
|
-
// The frame is still left behind it: a sub-task whose last step could
|
|
582
|
-
// not run here still hands back whatever its earlier steps emitted.
|
|
583
|
-
this.leaveFrame(entry);
|
|
584
|
-
this.stepIndex += 1;
|
|
585
|
-
continue;
|
|
586
|
-
}
|
|
587
|
-
response = fallback;
|
|
588
|
-
recorded = true;
|
|
965
|
+
const { response, recorded } = call;
|
|
966
|
+
if (recorded)
|
|
589
967
|
recordedCount += 1;
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
entry,
|
|
593
|
-
input: computed,
|
|
594
|
-
outcome: "recorded",
|
|
595
|
-
error,
|
|
596
|
-
toolMissing: isMissingToolError(error) || undefined,
|
|
597
|
-
ms: Date.now() - startedAt,
|
|
598
|
-
});
|
|
599
|
-
}
|
|
968
|
+
else
|
|
969
|
+
executed += 1;
|
|
600
970
|
if (stopOnFailure && !recorded) {
|
|
601
971
|
// Judged before the output logic, which was written against a success
|
|
602
972
|
// and would only throw a second, less useful error on this one.
|
|
@@ -613,12 +983,7 @@ export class ScenarioReplayPlan {
|
|
|
613
983
|
toolMissing: isMissingToolError(toolError) || undefined,
|
|
614
984
|
ms: Date.now() - startedAt,
|
|
615
985
|
});
|
|
616
|
-
entries.push({
|
|
617
|
-
toolName: step.toolName ?? "",
|
|
618
|
-
input: bundleInput(computed),
|
|
619
|
-
response,
|
|
620
|
-
recorded,
|
|
621
|
-
});
|
|
986
|
+
entries.push({ toolName: step.toolName ?? "", input: bundleInput(computed), response, recorded });
|
|
622
987
|
this.stepIndex += 1;
|
|
623
988
|
stopped = {
|
|
624
989
|
kind: "step_failed",
|
|
@@ -638,12 +1003,7 @@ export class ScenarioReplayPlan {
|
|
|
638
1003
|
if (!stopOnFailure)
|
|
639
1004
|
throw err;
|
|
640
1005
|
// The tool ran; its output is real and belongs in front of the model.
|
|
641
|
-
entries.push({
|
|
642
|
-
toolName: step.toolName ?? "",
|
|
643
|
-
input: bundleInput(computed),
|
|
644
|
-
response,
|
|
645
|
-
recorded,
|
|
646
|
-
});
|
|
1006
|
+
entries.push({ toolName: step.toolName ?? "", input: bundleInput(computed), response, recorded });
|
|
647
1007
|
stopped = {
|
|
648
1008
|
kind: "step_failed",
|
|
649
1009
|
stepIndex: step.stepIndex,
|
|
@@ -672,15 +1032,10 @@ export class ScenarioReplayPlan {
|
|
|
672
1032
|
ms: Date.now() - startedAt,
|
|
673
1033
|
});
|
|
674
1034
|
}
|
|
675
|
-
entries.push({
|
|
676
|
-
toolName: step.toolName ?? "",
|
|
677
|
-
input: bundleInput(computed),
|
|
678
|
-
response,
|
|
679
|
-
recorded,
|
|
680
|
-
});
|
|
1035
|
+
entries.push({ toolName: step.toolName ?? "", input: bundleInput(computed), response, recorded });
|
|
681
1036
|
}
|
|
682
1037
|
// Every planned step ran and the plan ends in front of a parked one.
|
|
683
|
-
if (!stopped &&
|
|
1038
|
+
if (!stopped && this.stopsEarly() && this.isDone()) {
|
|
684
1039
|
// `known_bad_step` here means "in front of the plan's end": the
|
|
685
1040
|
// controller reads the served reason and names the kind (D8).
|
|
686
1041
|
stopped = {
|
|
@@ -700,7 +1055,92 @@ export class ScenarioReplayPlan {
|
|
|
700
1055
|
errored,
|
|
701
1056
|
partial,
|
|
702
1057
|
stopped,
|
|
1058
|
+
aborted: aborted || undefined,
|
|
703
1059
|
};
|
|
704
1060
|
}
|
|
1061
|
+
/**
|
|
1062
|
+
* A reason not to *start* the next step: the caller is gone, or the time is
|
|
1063
|
+
* up. Undefined means go ahead.
|
|
1064
|
+
*/
|
|
1065
|
+
stopBefore(entry, flatIndex, signal, deadline) {
|
|
1066
|
+
if (signal?.aborted) {
|
|
1067
|
+
return { kind: "step_failed", stepIndex: entry.step.stepIndex, flatIndex, reason: "cancelled" };
|
|
1068
|
+
}
|
|
1069
|
+
if (deadline !== undefined && Date.now() >= deadline) {
|
|
1070
|
+
return { kind: "step_failed", stepIndex: entry.step.stepIndex, flatIndex, reason: "budget" };
|
|
1071
|
+
}
|
|
1072
|
+
return undefined;
|
|
1073
|
+
}
|
|
1074
|
+
/**
|
|
1075
|
+
* Call one step's tool. Resolves with its result, with the recorded output
|
|
1076
|
+
* that may stand in for it (only under `recordedFallback`, only for a step
|
|
1077
|
+
* that could not be sent), or with the stop a missing result means.
|
|
1078
|
+
*/
|
|
1079
|
+
async callStep(entry, flatIndex, computed, startedAt, execute, recordedOutputFor, onStep, opts,
|
|
1080
|
+
/** The driver's deadline: a recorded-output lookup is not waited on past it. */
|
|
1081
|
+
deadline) {
|
|
1082
|
+
const step = entry.step;
|
|
1083
|
+
try {
|
|
1084
|
+
return { response: await execute(step, computed), recorded: false };
|
|
1085
|
+
}
|
|
1086
|
+
catch (err) {
|
|
1087
|
+
const error = errText(err);
|
|
1088
|
+
const reason = rejectionOf(err);
|
|
1089
|
+
const standIn = opts.recordedFallback === true && reason === "not_run"
|
|
1090
|
+
? ((typeof step.recordedOutput === "string" && step.recordedOutput.length > 0
|
|
1091
|
+
? step.recordedOutput
|
|
1092
|
+
: undefined) ?? (await this.recordedWithin(recordedOutputFor, entry, deadline)))
|
|
1093
|
+
: undefined;
|
|
1094
|
+
onStep?.({
|
|
1095
|
+
step,
|
|
1096
|
+
entry,
|
|
1097
|
+
input: computed,
|
|
1098
|
+
outcome: standIn !== undefined ? "recorded" : "skipped",
|
|
1099
|
+
error,
|
|
1100
|
+
toolMissing: isMissingToolError(error) || undefined,
|
|
1101
|
+
ms: Date.now() - startedAt,
|
|
1102
|
+
});
|
|
1103
|
+
if (standIn !== undefined)
|
|
1104
|
+
return { response: standIn, recorded: true };
|
|
1105
|
+
return {
|
|
1106
|
+
stopped: {
|
|
1107
|
+
kind: "step_failed",
|
|
1108
|
+
stepIndex: step.stepIndex,
|
|
1109
|
+
flatIndex,
|
|
1110
|
+
stage: "tool_call",
|
|
1111
|
+
error,
|
|
1112
|
+
reason,
|
|
1113
|
+
},
|
|
1114
|
+
};
|
|
1115
|
+
}
|
|
1116
|
+
}
|
|
1117
|
+
/**
|
|
1118
|
+
* A step's recorded output from the source run, waited on no longer than the
|
|
1119
|
+
* driver's deadline.
|
|
1120
|
+
*
|
|
1121
|
+
* The lookup is a service fetch (source-run.ts) of up to its own timeout,
|
|
1122
|
+
* and a divergence runs it inside a `PreToolUse` whose deadline the hook
|
|
1123
|
+
* must keep (C1, C9): so it is a hot-path wait on the service, through the
|
|
1124
|
+
* breaker, and a lookup that has not answered in time is no recorded output.
|
|
1125
|
+
* A sliver of time left is not worth a request, nor a verdict on the service.
|
|
1126
|
+
*/
|
|
1127
|
+
async recordedWithin(recordedOutputFor, entry, deadline) {
|
|
1128
|
+
if (!recordedOutputFor)
|
|
1129
|
+
return undefined;
|
|
1130
|
+
if (deadline !== undefined && deadline - Date.now() < MIN_RECORDED_LOOKUP_MS)
|
|
1131
|
+
return undefined;
|
|
1132
|
+
let pending;
|
|
1133
|
+
try {
|
|
1134
|
+
pending = recordedOutputFor(entry);
|
|
1135
|
+
}
|
|
1136
|
+
catch {
|
|
1137
|
+
return undefined;
|
|
1138
|
+
}
|
|
1139
|
+
if (deadline === undefined)
|
|
1140
|
+
return pending.catch(() => undefined);
|
|
1141
|
+
return awaitWithin(pending, Math.max(0, deadline - Date.now()), "source_run");
|
|
1142
|
+
}
|
|
705
1143
|
}
|
|
1144
|
+
/** Below this much time left, a recorded-output lookup is not attempted. */
|
|
1145
|
+
const MIN_RECORDED_LOOKUP_MS = 250;
|
|
706
1146
|
//# sourceMappingURL=plan.js.map
|