@basein/runner 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/bin/bir-hooks.d.ts +13 -1
- package/dist/bin/bir-hooks.js +13 -1
- package/dist/control/recorder-main.d.ts +3 -3
- package/dist/control/recorder-main.js +16 -3
- package/dist/control/server.js +17 -4
- package/dist/replay/builtins.d.ts +120 -0
- package/dist/replay/builtins.js +641 -0
- package/dist/replay/controller.d.ts +47 -0
- package/dist/replay/controller.js +63 -6
- package/dist/replay/coverage.d.ts +20 -7
- package/dist/replay/coverage.js +27 -10
- package/docs/calculatedReplay.md +15 -6
- package/docs/calculatedReplayGuide.md +6 -4
- package/package.json +1 -1
- package/production.md +1373 -1371
|
@@ -196,6 +196,23 @@ export interface ReplayOptions {
|
|
|
196
196
|
segmentArm?: boolean;
|
|
197
197
|
/** `BIR_REPLAY_ALLOW_SERVERS`; undefined means every wrapped server. */
|
|
198
198
|
allowServers?: ReadonlySet<string>;
|
|
199
|
+
/**
|
|
200
|
+
* `BIR_BUILTIN_TOOLS` — the built-ins the runner may execute itself, from
|
|
201
|
+
* the session's own project directory (replay/builtins.ts), or stand in
|
|
202
|
+
* for with a recorded output under `BIR_REPLAY_RECORDED_OUTPUTS=1`.
|
|
203
|
+
* Omitted, the environment decides; the default is the read-only set
|
|
204
|
+
* (`Read,Glob,Grep`). `Edit`, `Write` and `Bash` run only when listed
|
|
205
|
+
* explicitly — and `Bash` additionally only for commands matching
|
|
206
|
+
* {@link ReplayOptions.bashAllow}.
|
|
207
|
+
*/
|
|
208
|
+
builtinTools?: ReadonlySet<string>;
|
|
209
|
+
/**
|
|
210
|
+
* `BIR_BASH_ALLOW` — command prefixes a runner-side `Bash` step may run
|
|
211
|
+
* (`*` for all). Undefined means `Bash` never executes here, even when
|
|
212
|
+
* allow-listed: runner-side execution bypasses the host's permission
|
|
213
|
+
* prompts, so the shell needs its own explicit opt-in.
|
|
214
|
+
*/
|
|
215
|
+
bashAllow?: readonly string[];
|
|
199
216
|
budgets?: Partial<ReplayBudgets>;
|
|
200
217
|
/**
|
|
201
218
|
* The service: the lazy source-run fetch, and the parameter derivation this
|
|
@@ -331,6 +348,14 @@ export interface ReplayState {
|
|
|
331
348
|
* replay`, which has no session and takes the newest proxy.
|
|
332
349
|
*/
|
|
333
350
|
sessionId?: string;
|
|
351
|
+
/**
|
|
352
|
+
* That session's project directory, set by the control server from the
|
|
353
|
+
* hook payloads — never the daemon's own cwd, which is `~/.baseinstrunner`.
|
|
354
|
+
* It is where an allow-listed built-in step executes and the jail it may
|
|
355
|
+
* not leave (replay/builtins.ts). Absent — `bir replay`, an older server —
|
|
356
|
+
* no built-in executes: the step is `not_run`, exactly as before.
|
|
357
|
+
*/
|
|
358
|
+
sessionCwd?: string;
|
|
334
359
|
/**
|
|
335
360
|
* Which kind of row was handed out (segmented.md R-OUT-7). A `segment` is a
|
|
336
361
|
* named sub-task of a recording rather than a whole task, and it is judged
|
|
@@ -447,6 +472,10 @@ export declare class ReplayController {
|
|
|
447
472
|
private readonly opts;
|
|
448
473
|
/** See {@link ReplayOptions.recordedFallback}. */
|
|
449
474
|
private readonly recordedFallback;
|
|
475
|
+
/** See {@link ReplayOptions.builtinTools}. */
|
|
476
|
+
private readonly builtinTools;
|
|
477
|
+
/** See {@link ReplayOptions.bashAllow}. */
|
|
478
|
+
private readonly bashAllow?;
|
|
450
479
|
/** Calls waiting for a plan's pins to change (see `awaitPins`). */
|
|
451
480
|
private readonly pinWaiters;
|
|
452
481
|
/**
|
|
@@ -762,6 +791,19 @@ export declare class ReplayController {
|
|
|
762
791
|
* asked for by id and which the allowlist does not gate.
|
|
763
792
|
*/
|
|
764
793
|
private executeStep;
|
|
794
|
+
/**
|
|
795
|
+
* Run one allow-listed built-in step on the runner, inside `cwd`.
|
|
796
|
+
*
|
|
797
|
+
* The result mimics the hook's view of the tool, because that is what was
|
|
798
|
+
* recorded for built-in steps and what their `toolOutputLogic` was authored
|
|
799
|
+
* against (control/server.ts, `onToolPost`) — and it is serialized and
|
|
800
|
+
* redacted through the exact pipeline every recorded payload takes, so
|
|
801
|
+
* `Bash` output cannot carry a secret the recording would have stripped.
|
|
802
|
+
*
|
|
803
|
+
* Failures keep {@link ExecuteStep}'s contract: policy (`not_run`), a step
|
|
804
|
+
* cut off by its clock (`uncertain`), a step that ran and broke (`failed`).
|
|
805
|
+
*/
|
|
806
|
+
private executeBuiltin;
|
|
765
807
|
/** Where recorded outputs come from, when they may stand in at all. */
|
|
766
808
|
private recordedSource;
|
|
767
809
|
/**
|
|
@@ -797,6 +839,11 @@ export declare class ReplayController {
|
|
|
797
839
|
* wrapped server the allowlist excludes is still proxied, still reported, and
|
|
798
840
|
* still recorded in the proxy's serialization. Threading such a step from the
|
|
799
841
|
* hook's differently-shaped view would derive nothing at all.
|
|
842
|
+
*
|
|
843
|
+
* The built-in allow-list is deliberately NOT passed here for the same
|
|
844
|
+
* reason in the other direction: a *steered* built-in is never proxied,
|
|
845
|
+
* whatever `BIR_BUILTIN_TOOLS` says, so its output can only ever arrive
|
|
846
|
+
* from the hook.
|
|
800
847
|
*/
|
|
801
848
|
private reachFor;
|
|
802
849
|
/**
|
|
@@ -18,6 +18,7 @@ import { serializeCapped } from "../record/truncate.js";
|
|
|
18
18
|
import { logDetail, logLine, errText } from "../util/log.js";
|
|
19
19
|
import { serviceHealth, serviceTimeoutMs, timeoutSignal, } from "../util/service-health.js";
|
|
20
20
|
import { bashCommandFor } from "./bash-channel.js";
|
|
21
|
+
import { BuiltinNotRunnable, BuiltinTimeout, parseBashAllow, parseBuiltinTools, runBuiltin, } from "./builtins.js";
|
|
21
22
|
import { BASH_CHANNEL_MAX, MAX_REPLAY_REASON, truncate } from "./bundle.js";
|
|
22
23
|
import { coverageOf, modeFor, reachOf } from "./coverage.js";
|
|
23
24
|
import { deriveParameters, } from "./derive.js";
|
|
@@ -57,7 +58,7 @@ export const DEFAULT_BUDGETS = {
|
|
|
57
58
|
// task; a match that has not answered by then is a turn run normally. The
|
|
58
59
|
// control server stops waiting at all for a while after one that answered
|
|
59
60
|
// slowly (`BIR_MATCH_SLOW_MS`). `BIR_MATCH_BUDGET_MS` in the recorder.
|
|
60
|
-
matchMs:
|
|
61
|
+
matchMs: 3_000,
|
|
61
62
|
deriveMs: 8_000,
|
|
62
63
|
planMs: 100_000,
|
|
63
64
|
stepMs: 60_000,
|
|
@@ -185,6 +186,10 @@ export class ReplayController {
|
|
|
185
186
|
opts;
|
|
186
187
|
/** See {@link ReplayOptions.recordedFallback}. */
|
|
187
188
|
recordedFallback;
|
|
189
|
+
/** See {@link ReplayOptions.builtinTools}. */
|
|
190
|
+
builtinTools;
|
|
191
|
+
/** See {@link ReplayOptions.bashAllow}. */
|
|
192
|
+
bashAllow;
|
|
188
193
|
/** Calls waiting for a plan's pins to change (see `awaitPins`). */
|
|
189
194
|
pinWaiters = new WeakMap();
|
|
190
195
|
/**
|
|
@@ -245,6 +250,8 @@ export class ReplayController {
|
|
|
245
250
|
this.intentMatch = { ...DEFAULT_INTENT_MATCH, ...(opts.intentMatch ?? {}) };
|
|
246
251
|
this.recordedFallback =
|
|
247
252
|
opts.recordedFallback ?? process.env.BIR_REPLAY_RECORDED_OUTPUTS === "1";
|
|
253
|
+
this.builtinTools = opts.builtinTools ?? parseBuiltinTools(process.env.BIR_BUILTIN_TOOLS);
|
|
254
|
+
this.bashAllow = opts.bashAllow ?? parseBashAllow(process.env.BIR_BASH_ALLOW);
|
|
248
255
|
}
|
|
249
256
|
/**
|
|
250
257
|
* Whether a handed-out segment may actually arm (segmented.md R-OUT-10).
|
|
@@ -423,8 +430,11 @@ export class ReplayController {
|
|
|
423
430
|
: "known_bad_first_step");
|
|
424
431
|
}
|
|
425
432
|
const planned = stopAt === undefined ? entries : entries.slice(0, stopAt);
|
|
426
|
-
// Gate 6 — tool coverage, over the steps that will actually run.
|
|
427
|
-
|
|
433
|
+
// Gate 6 — tool coverage, over the steps that will actually run. A
|
|
434
|
+
// built-in on `BIR_BUILTIN_TOOLS` counts as reachable: the runner
|
|
435
|
+
// executes it itself (replay/builtins.ts), so a chain that starts with a
|
|
436
|
+
// `Read` no longer forces steer mode.
|
|
437
|
+
const mode = modeFor(planned.map((e) => e.step), wrapped, this.opts.allowServers, this.builtinTools);
|
|
428
438
|
if (mode === "none")
|
|
429
439
|
return decline("no step is executable", "coverage");
|
|
430
440
|
// Tool results feed derivation as context, and they are whatever the agent
|
|
@@ -598,7 +608,7 @@ export class ReplayController {
|
|
|
598
608
|
unusable: flat.stopReason?.reason,
|
|
599
609
|
// Which steps the service told this plan to stop in front of, and why.
|
|
600
610
|
stops: stopLines(entries),
|
|
601
|
-
coverage: coverageOf(entries.map((e) => e.step), wrapped, this.opts.allowServers).join(","),
|
|
611
|
+
coverage: coverageOf(entries.map((e) => e.step), wrapped, this.opts.allowServers, this.builtinTools).join(","),
|
|
602
612
|
tools: entries.map((e) => e.step.toolName ?? "").join(","),
|
|
603
613
|
});
|
|
604
614
|
return state;
|
|
@@ -1565,7 +1575,7 @@ export class ReplayController {
|
|
|
1565
1575
|
}
|
|
1566
1576
|
if (flat.entries.length === 0)
|
|
1567
1577
|
return { ok: false, why: "the chain has no runnable step" };
|
|
1568
|
-
const mode = modeFor(flat.entries.map((e) => e.step), wrapped, this.opts.allowServers);
|
|
1578
|
+
const mode = modeFor(flat.entries.map((e) => e.step), wrapped, this.opts.allowServers, this.builtinTools);
|
|
1569
1579
|
if (mode === "none")
|
|
1570
1580
|
return { ok: false, why: "no step is executable" };
|
|
1571
1581
|
const state = {
|
|
@@ -1636,7 +1646,7 @@ export class ReplayController {
|
|
|
1636
1646
|
scenario: scenario.id,
|
|
1637
1647
|
mode,
|
|
1638
1648
|
steps: scenario.steps.length,
|
|
1639
|
-
coverage: coverageOf(scenario.steps, wrapped, this.opts.allowServers).join(","),
|
|
1649
|
+
coverage: coverageOf(scenario.steps, wrapped, this.opts.allowServers, this.builtinTools).join(","),
|
|
1640
1650
|
why: "bir replay — no match, an operator asked for it",
|
|
1641
1651
|
});
|
|
1642
1652
|
// An operator's run keeps its own contract (R-CALL-37): out of time, the
|
|
@@ -2066,6 +2076,16 @@ export class ReplayController {
|
|
|
2066
2076
|
const name = step.toolName ?? "";
|
|
2067
2077
|
const mcp = parseQualifiedName(name);
|
|
2068
2078
|
if (!mcp) {
|
|
2079
|
+
// An allow-listed built-in runs *here*, against the session's own
|
|
2080
|
+
// project directory (replay/builtins.ts). Everything else about it —
|
|
2081
|
+
// no session cwd (`bir replay`, an older server), off the list, no
|
|
2082
|
+
// runner-side implementation, a path that leaves the project, `Bash`
|
|
2083
|
+
// without `BIR_BASH_ALLOW` — stays the old contract: `not_run`, so
|
|
2084
|
+
// the session can do the step itself or its recorded output can
|
|
2085
|
+
// stand in (`BIR_REPLAY_RECORDED_OUTPUTS=1`).
|
|
2086
|
+
if (this.builtinTools.has(name) && state.sessionCwd) {
|
|
2087
|
+
return this.executeBuiltin(name, input, state.sessionCwd, opts.deadline);
|
|
2088
|
+
}
|
|
2069
2089
|
throw new StepRunError(`${name || "this step"} is a built-in tool, which only runs in the session`, "not_run");
|
|
2070
2090
|
}
|
|
2071
2091
|
// `BIR_REPLAY_ALLOW_SERVERS` is about exactly this: a call bir makes by
|
|
@@ -2098,6 +2118,38 @@ export class ReplayController {
|
|
|
2098
2118
|
return serializeCapped(redact(result));
|
|
2099
2119
|
};
|
|
2100
2120
|
}
|
|
2121
|
+
/**
|
|
2122
|
+
* Run one allow-listed built-in step on the runner, inside `cwd`.
|
|
2123
|
+
*
|
|
2124
|
+
* The result mimics the hook's view of the tool, because that is what was
|
|
2125
|
+
* recorded for built-in steps and what their `toolOutputLogic` was authored
|
|
2126
|
+
* against (control/server.ts, `onToolPost`) — and it is serialized and
|
|
2127
|
+
* redacted through the exact pipeline every recorded payload takes, so
|
|
2128
|
+
* `Bash` output cannot carry a secret the recording would have stripped.
|
|
2129
|
+
*
|
|
2130
|
+
* Failures keep {@link ExecuteStep}'s contract: policy (`not_run`), a step
|
|
2131
|
+
* cut off by its clock (`uncertain`), a step that ran and broke (`failed`).
|
|
2132
|
+
*/
|
|
2133
|
+
async executeBuiltin(name, input, cwd, deadline) {
|
|
2134
|
+
const left = deadline !== undefined ? deadline - Date.now() : Number.POSITIVE_INFINITY;
|
|
2135
|
+
const timeoutMs = Math.max(MIN_STEP_MS, Math.min(this.budgets.stepMs, left));
|
|
2136
|
+
// The audit line precedes the action, always (§13.2, mitigation 4).
|
|
2137
|
+
logDetail("replay.builtin", { tool: name, cwd });
|
|
2138
|
+
let result;
|
|
2139
|
+
try {
|
|
2140
|
+
result = await runBuiltin(name, input, { cwd, timeoutMs, bashAllow: this.bashAllow });
|
|
2141
|
+
}
|
|
2142
|
+
catch (err) {
|
|
2143
|
+
if (err instanceof BuiltinNotRunnable)
|
|
2144
|
+
throw new StepRunError(err.message, "not_run");
|
|
2145
|
+
if (err instanceof BuiltinTimeout)
|
|
2146
|
+
throw new StepRunError(err.message, "uncertain");
|
|
2147
|
+
// It ran and broke — a missing file, a non-unique `old_string`, a
|
|
2148
|
+
// non-zero exit. The step failed here, now; never threaded as success.
|
|
2149
|
+
throw new StepRunError(errText(err), "failed");
|
|
2150
|
+
}
|
|
2151
|
+
return serializeCapped(redact(result));
|
|
2152
|
+
}
|
|
2101
2153
|
/** Where recorded outputs come from, when they may stand in at all. */
|
|
2102
2154
|
recordedSource(state, force = false) {
|
|
2103
2155
|
if (!force && !this.recordedFallback)
|
|
@@ -2230,6 +2282,11 @@ export class ReplayController {
|
|
|
2230
2282
|
* wrapped server the allowlist excludes is still proxied, still reported, and
|
|
2231
2283
|
* still recorded in the proxy's serialization. Threading such a step from the
|
|
2232
2284
|
* hook's differently-shaped view would derive nothing at all.
|
|
2285
|
+
*
|
|
2286
|
+
* The built-in allow-list is deliberately NOT passed here for the same
|
|
2287
|
+
* reason in the other direction: a *steered* built-in is never proxied,
|
|
2288
|
+
* whatever `BIR_BUILTIN_TOOLS` says, so its output can only ever arrive
|
|
2289
|
+
* from the hook.
|
|
2233
2290
|
*/
|
|
2234
2291
|
reachFor(state, step) {
|
|
2235
2292
|
return reachOf(step.toolName ?? "", state.wrapped);
|
|
@@ -8,9 +8,17 @@
|
|
|
8
8
|
* only happen inside the live session.
|
|
9
9
|
*
|
|
10
10
|
* `live` covers three genuinely different cases and treats them identically,
|
|
11
|
-
* correctly: a built-in (`Bash`, `Read`, `Edit`, …)
|
|
12
|
-
* not to wrap, and `claude-in-chrome`
|
|
13
|
-
* no config file, and therefore can
|
|
11
|
+
* correctly: a built-in (`Bash`, `Read`, `Edit`, …) outside the built-in
|
|
12
|
+
* allow-list, an MCP server the user chose not to wrap, and `claude-in-chrome`
|
|
13
|
+
* — which is `scope: "dynamic"`, appears in no config file, and therefore can
|
|
14
|
+
* never be wrapped at all.
|
|
15
|
+
*
|
|
16
|
+
* A built-in *on* the allow-list (`BIR_BUILTIN_TOOLS`, default `Read,Glob,
|
|
17
|
+
* Grep`) is `direct`: the runner executes it itself (replay/builtins.ts), or —
|
|
18
|
+
* under `BIR_REPLAY_RECORDED_OUTPUTS=1` — serves its recorded output. The
|
|
19
|
+
* callers that ask a *different* question, "will a proxy report this call"
|
|
20
|
+
* (`ReplayController.reachFor`), deliberately pass no allow-list: a built-in
|
|
21
|
+
* is never proxied, whoever executes it.
|
|
14
22
|
*/
|
|
15
23
|
import type { SerializedScenarioStep } from "./types.js";
|
|
16
24
|
/** Where a step's tool can actually be executed. */
|
|
@@ -19,18 +27,23 @@ export type StepReach = "direct" | "live";
|
|
|
19
27
|
export type ReplayMode = "direct" | "steer" | "none";
|
|
20
28
|
/**
|
|
21
29
|
* `direct` when the tool is `mcp__<server>__<tool>` for a server this control
|
|
22
|
-
* plane has a proxy for
|
|
30
|
+
* plane has a proxy for, or a built-in on `builtins`; `live` otherwise.
|
|
23
31
|
*
|
|
24
32
|
* `allow` is the `BIR_REPLAY_ALLOW_SERVERS` allowlist (§13.2, mitigation 2): a
|
|
25
33
|
* wrapped server outside it is deliberately demoted to `live`, so it can still be
|
|
26
34
|
* steered inside the session — where the user's own permission rules apply — but
|
|
27
35
|
* is never called unattended by us.
|
|
36
|
+
*
|
|
37
|
+
* `builtins` is the `BIR_BUILTIN_TOOLS` allow-list (replay/builtins.ts): the
|
|
38
|
+
* built-ins the runner may execute itself, or stand in for with a recorded
|
|
39
|
+
* output. Omitted means none — the old contract — which is what every caller
|
|
40
|
+
* that is not the mode decision wants.
|
|
28
41
|
*/
|
|
29
|
-
export declare function reachOf(toolName: string, wrapped: ReadonlySet<string>, allow?: ReadonlySet<string>): StepReach;
|
|
42
|
+
export declare function reachOf(toolName: string, wrapped: ReadonlySet<string>, allow?: ReadonlySet<string>, builtins?: ReadonlySet<string>): StepReach;
|
|
30
43
|
/** The mode for a whole scenario: `direct` only when *every* step is reachable. */
|
|
31
|
-
export declare function modeFor(steps: readonly SerializedScenarioStep[], wrapped: ReadonlySet<string>, allow?: ReadonlySet<string>): ReplayMode;
|
|
44
|
+
export declare function modeFor(steps: readonly SerializedScenarioStep[], wrapped: ReadonlySet<string>, allow?: ReadonlySet<string>, builtins?: ReadonlySet<string>): ReplayMode;
|
|
32
45
|
/** Per-step reach, in step order — what `plan.armed` logs and `bir doctor` shows. */
|
|
33
|
-
export declare function coverageOf(steps: readonly SerializedScenarioStep[], wrapped: ReadonlySet<string>, allow?: ReadonlySet<string>): StepReach[];
|
|
46
|
+
export declare function coverageOf(steps: readonly SerializedScenarioStep[], wrapped: ReadonlySet<string>, allow?: ReadonlySet<string>, builtins?: ReadonlySet<string>): StepReach[];
|
|
34
47
|
/**
|
|
35
48
|
* Parse `BIR_REPLAY_ALLOW_SERVERS`. Unset (or empty) means *every wrapped
|
|
36
49
|
* server*, which is why the return is `undefined` rather than an empty set — an
|
package/dist/replay/coverage.js
CHANGED
|
@@ -8,37 +8,54 @@
|
|
|
8
8
|
* only happen inside the live session.
|
|
9
9
|
*
|
|
10
10
|
* `live` covers three genuinely different cases and treats them identically,
|
|
11
|
-
* correctly: a built-in (`Bash`, `Read`, `Edit`, …)
|
|
12
|
-
* not to wrap, and `claude-in-chrome`
|
|
13
|
-
* no config file, and therefore can
|
|
11
|
+
* correctly: a built-in (`Bash`, `Read`, `Edit`, …) outside the built-in
|
|
12
|
+
* allow-list, an MCP server the user chose not to wrap, and `claude-in-chrome`
|
|
13
|
+
* — which is `scope: "dynamic"`, appears in no config file, and therefore can
|
|
14
|
+
* never be wrapped at all.
|
|
15
|
+
*
|
|
16
|
+
* A built-in *on* the allow-list (`BIR_BUILTIN_TOOLS`, default `Read,Glob,
|
|
17
|
+
* Grep`) is `direct`: the runner executes it itself (replay/builtins.ts), or —
|
|
18
|
+
* under `BIR_REPLAY_RECORDED_OUTPUTS=1` — serves its recorded output. The
|
|
19
|
+
* callers that ask a *different* question, "will a proxy report this call"
|
|
20
|
+
* (`ReplayController.reachFor`), deliberately pass no allow-list: a built-in
|
|
21
|
+
* is never proxied, whoever executes it.
|
|
14
22
|
*/
|
|
15
23
|
import { parseQualifiedName } from "../control/correlation.js";
|
|
16
24
|
/**
|
|
17
25
|
* `direct` when the tool is `mcp__<server>__<tool>` for a server this control
|
|
18
|
-
* plane has a proxy for
|
|
26
|
+
* plane has a proxy for, or a built-in on `builtins`; `live` otherwise.
|
|
19
27
|
*
|
|
20
28
|
* `allow` is the `BIR_REPLAY_ALLOW_SERVERS` allowlist (§13.2, mitigation 2): a
|
|
21
29
|
* wrapped server outside it is deliberately demoted to `live`, so it can still be
|
|
22
30
|
* steered inside the session — where the user's own permission rules apply — but
|
|
23
31
|
* is never called unattended by us.
|
|
32
|
+
*
|
|
33
|
+
* `builtins` is the `BIR_BUILTIN_TOOLS` allow-list (replay/builtins.ts): the
|
|
34
|
+
* built-ins the runner may execute itself, or stand in for with a recorded
|
|
35
|
+
* output. Omitted means none — the old contract — which is what every caller
|
|
36
|
+
* that is not the mode decision wants.
|
|
24
37
|
*/
|
|
25
|
-
export function reachOf(toolName, wrapped, allow) {
|
|
38
|
+
export function reachOf(toolName, wrapped, allow, builtins) {
|
|
26
39
|
const mcp = parseQualifiedName(toolName);
|
|
27
|
-
if (!mcp
|
|
40
|
+
if (!mcp)
|
|
41
|
+
return builtins?.has(toolName) ? "direct" : "live";
|
|
42
|
+
if (!wrapped.has(mcp.serverName))
|
|
28
43
|
return "live";
|
|
29
44
|
if (allow && !allow.has(mcp.serverName))
|
|
30
45
|
return "live";
|
|
31
46
|
return "direct";
|
|
32
47
|
}
|
|
33
48
|
/** The mode for a whole scenario: `direct` only when *every* step is reachable. */
|
|
34
|
-
export function modeFor(steps, wrapped, allow) {
|
|
49
|
+
export function modeFor(steps, wrapped, allow, builtins) {
|
|
35
50
|
if (steps.length === 0)
|
|
36
51
|
return "none";
|
|
37
|
-
return steps.every((s) => reachOf(s.toolName ?? "", wrapped, allow) === "direct")
|
|
52
|
+
return steps.every((s) => reachOf(s.toolName ?? "", wrapped, allow, builtins) === "direct")
|
|
53
|
+
? "direct"
|
|
54
|
+
: "steer";
|
|
38
55
|
}
|
|
39
56
|
/** Per-step reach, in step order — what `plan.armed` logs and `bir doctor` shows. */
|
|
40
|
-
export function coverageOf(steps, wrapped, allow) {
|
|
41
|
-
return steps.map((s) => reachOf(s.toolName ?? "", wrapped, allow));
|
|
57
|
+
export function coverageOf(steps, wrapped, allow, builtins) {
|
|
58
|
+
return steps.map((s) => reachOf(s.toolName ?? "", wrapped, allow, builtins));
|
|
42
59
|
}
|
|
43
60
|
/**
|
|
44
61
|
* Parse `BIR_REPLAY_ALLOW_SERVERS`. Unset (or empty) means *every wrapped
|
package/docs/calculatedReplay.md
CHANGED
|
@@ -45,7 +45,7 @@ this design has a ladder whose cheapest rung is free — and the more of a stack
|
|
|
45
45
|
|---|---|---|
|
|
46
46
|
| Runs MCP tools itself | No (`replay.md` §12: *"rrepeat never executes tools — not even MCP"*) | **Yes**, on the connection it already owns |
|
|
47
47
|
| Divergence fallback | A second Agent SDK session — the dominant cost | Direct `tools/call` through the proxies — **$0** |
|
|
48
|
-
| Reaches built-ins (`Bash`, `Read`) | Only by steering the live session |
|
|
48
|
+
| Reaches built-ins (`Bash`, `Read`) | Only by steering the live session | Read-only ones (`Read`, `Glob`, `Grep`) run on the runner itself, jailed to the project (`BIR_BUILTIN_TOOLS`); `Edit`/`Write`/`Bash` opt-in, else steered |
|
|
49
49
|
| Needs `@anthropic-ai/*` at runtime | Yes | No (§16.3) |
|
|
50
50
|
|
|
51
51
|
---
|
|
@@ -243,15 +243,24 @@ export function modeFor(steps: SerializedScenarioStep[], wrapped: ReadonlySet<st
|
|
|
243
243
|
|
|
244
244
|
| Every step | Mode | Where the tools run | Model spend |
|
|
245
245
|
|---|---|---|---|
|
|
246
|
-
| a wrapped MCP tool | **`direct`** (§6) | The proxies, on connections already open | one Haiku derivation + one turn that reads the results |
|
|
246
|
+
| a wrapped MCP tool, or a built-in on `BIR_BUILTIN_TOOLS` | **`direct`** (§6) | The proxies, on connections already open; allow-listed built-ins on the runner itself, jailed to the session's project directory (replay/builtins.ts) | one Haiku derivation + one turn that reads the results |
|
|
247
247
|
| anything else, or mixed | **`steer`** (§7) | The live session, inputs pinned | one Haiku derivation + the live turn |
|
|
248
248
|
| — | `none` | nowhere | declines at gate 5 |
|
|
249
249
|
|
|
250
250
|
`live` covers three genuinely different cases and treats them identically, correctly: a built-in
|
|
251
|
-
|
|
251
|
+
off the `BIR_BUILTIN_TOOLS` allow-list (by default everything but `Read`, `Glob`, `Grep` — so
|
|
252
|
+
`Bash`, `Edit`, `Write`, `Task`, `WebFetch`), an MCP server the user chose not to wrap, and
|
|
252
253
|
`claude-in-chrome` — which is `scope: "dynamic"`, appears in no config file, and therefore can never
|
|
253
254
|
be wrapped (v1 design §1). None of them can be reached except through the session.
|
|
254
255
|
|
|
256
|
+
A built-in **on** the allow-list runs on the runner, in the session's own project directory and
|
|
257
|
+
confined to it — a path that resolves outside the project, symlinks included, is refused and the
|
|
258
|
+
step is `not_run`, the clean hand-over. `Bash` needs a second opt-in on top (`BIR_BASH_ALLOW`,
|
|
259
|
+
command prefixes), because a runner-side step never reaches the host's permission prompts. The
|
|
260
|
+
results mimic the hook's view of each tool, which is what the recorder stored for built-in steps
|
|
261
|
+
and what their `toolOutputLogic` was authored against; they pass through the same
|
|
262
|
+
`serializeCapped(redact(…))` as every recorded payload.
|
|
263
|
+
|
|
255
264
|
A **mixed** scenario in `steer` mode still gets direct execution for its wrapped steps once it
|
|
256
265
|
diverges (§8), so coverage is not all-or-nothing at the divergence boundary — only at the arm-time
|
|
257
266
|
mode choice.
|
|
@@ -760,7 +769,7 @@ The hook is on the user's critical path. Every await here is a stall the user wa
|
|
|
760
769
|
|
|
761
770
|
| Moment | Route | Hook timeout | Answered by | What may be awaited | Budget |
|
|
762
771
|
|---|---|---|---|---|---|
|
|
763
|
-
| `UserPromptSubmit` | `/session/prompt` | 10 s | 7 s | the match; for a scenario with a target, its derivation; a steer plan's argument preview | match `BIR_MATCH_BUDGET_MS` = **
|
|
772
|
+
| `UserPromptSubmit` | `/session/prompt` | 10 s | 7 s | the match; for a scenario with a target, its derivation; a steer plan's argument preview | match `BIR_MATCH_BUDGET_MS` = **3000**; match and arming together ≤ **6 s**; derivation ≤ `BIR_HOOK_DERIVE_BUDGET_MS` = **4000** |
|
|
764
773
|
| `PreToolUse` | `/tool/pre` | 15 s | 12 s | `plan.ready()` (derivation); on a divergence, the remaining steps | derivation ≤ min(`BIR_DERIVE_BUDGET_MS`, `BIR_HOOK_DERIVE_BUDGET_MS`) = **4000**; all replay work ≤ `BIR_PRETOOL_REPLAY_BUDGET_MS` = **9000** (held at ≤ 10 000) |
|
|
765
774
|
| `run_scenario` | `/scenario/run` | `bir-scenario`'s own, **290 s** (`BIR_SCENARIO_RUN_TIMEOUT_MS`) | — | `ready()` + N steps | derivation `BIR_DERIVE_BUDGET_MS` = **8000**; plan `BIR_REPLAY_PLAN_BUDGET_MS` = **100000** (`BIR_REPLAY_BUDGET_MS` overrides it where set); per step `BIR_STEP_TIMEOUT_MS` = **60000**, clipped to what the plan has left |
|
|
766
775
|
| `PostToolUse` | `/tool/post` | 5 s | 2 s | nothing | — |
|
|
@@ -794,9 +803,9 @@ minutes, while its steps keep starting. `bir replay` is not held to the ceiling.
|
|
|
794
803
|
|
|
795
804
|
**Budget expiry is never an error.** Match budget expired → no plan, ordinary turn, one
|
|
796
805
|
`replay.decision` line (`match_timeout`). **The match gate:** a match that answered slower than
|
|
797
|
-
`BIR_MATCH_SLOW_MS` (**
|
|
806
|
+
`BIR_MATCH_SLOW_MS` (**2000**), or not at all, also makes prompts stop waiting on the match for 5
|
|
798
807
|
minutes, doubling up to 30 while the service stays slow; a fast match ends the hold
|
|
799
|
-
(`replay.match_gate`). A healthy service answers in 300–450 ms, so past
|
|
808
|
+
(`replay.match_gate`). A healthy service answers in 300–450 ms, so past 2 s waiting would add
|
|
800
809
|
that much to every prompt. The match is still asked meanwhile, so a hit still stops the recording. Derivation budget expired →
|
|
801
810
|
settings take their recorded `sampleValue`s and targets stay `null`, never `{}`; a plan that needs a
|
|
802
811
|
missing target, or whose `paramsLogic` throws, hands over instead of running (in steer mode the
|
|
@@ -345,14 +345,16 @@ so there is exactly one switch and it cannot get out of step with itself.
|
|
|
345
345
|
| `BIR_MIN_STEER_SIMILARITY` | `0.92` | Below this a match is detected but not replayed (§8) |
|
|
346
346
|
| `BIR_DERIVE_API_KEY` | *(unset)* | **Not required.** Derivation — reading what this request acts on — is done by the service on its key for a signed-in runner. Set this to keep the reading on this machine instead: the prompt then never leaves it, and it is one round trip faster. Signed out *and* unset, only a scenario with nothing to work out replays. A recorder running in the background (started by the SessionStart hook, `bir up` or `bir setup`) ignores `ANTHROPIC_API_KEY` (usually Claude Code's own, and billed) unless `BIR_USE_ANTHROPIC_API_KEY=1`; a `bir-hooks` you run in a terminal uses it |
|
|
347
347
|
| `BIR_DERIVE_MODEL` | `claude-haiku-4-5-20251001` | The derivation model, when this machine does the reading |
|
|
348
|
-
| `BIR_MATCH_BUDGET_MS` | `
|
|
348
|
+
| `BIR_MATCH_BUDGET_MS` | `3000` | How long the prompt hook waits for a match before giving up. After a match slower than `BIR_MATCH_SLOW_MS` (`2000`), prompts stop waiting for one for 5 minutes, doubling up to 30 while the service stays slow |
|
|
349
349
|
| `BIR_DERIVE_BUDGET_MS` | `8000` | How long `run_scenario` waits for parameters |
|
|
350
350
|
| `BIR_HOOK_DERIVE_BUDGET_MS` | `4000` | How long a hook waits for them: the prompt, for a scenario with a target, and the first `PreToolUse` (never longer than `BIR_DERIVE_BUDGET_MS`) |
|
|
351
351
|
| `BIR_PRETOOL_REPLAY_BUDGET_MS` | `9000` | All replay work inside one `PreToolUse`, a divergence's steps included. At most 10 000 |
|
|
352
352
|
| `BIR_REPLAY_PLAN_BUDGET_MS` | `100000` | Whole-plan ceiling for direct execution; `BIR_REPLAY_BUDGET_MS`, where set, overrides it. Past about 110 000 a long run may be moved to the background by Claude Code. Derivation plus plan is held under 270 s |
|
|
353
353
|
| `BIR_STEP_TIMEOUT_MS` | `60000` | Per-step ceiling for one direct `tools/call`, clipped to what the plan has left |
|
|
354
354
|
| `BIR_LOGIC_TIMEOUT_MS` | `1000` | How long one scenario logic body may run before it is stopped. At most 10 000 |
|
|
355
|
-
| `BIR_REPLAY_RECORDED_OUTPUTS` | *(unset)* | `1` lets a step that could not be sent (a built-in
|
|
355
|
+
| `BIR_REPLAY_RECORDED_OUTPUTS` | *(unset)* | `1` lets a step that could not be sent (a built-in off `BIR_BUILTIN_TOOLS`, no proxy, a server off the allowlist) stand on its recorded output instead of stopping the plan. Never for a timeout or an upstream error |
|
|
356
|
+
| `BIR_BUILTIN_TOOLS` | `Read,Glob,Grep` | Built-ins the runner executes itself during a replay, in the session's own project directory and confined to it. A chain whose only non-MCP steps are on this list arms `direct`. Add `Edit,Write,Bash` only deliberately — a runner-side step never reaches the host's permission prompts. `none` restores the old behaviour: every built-in hands the turn back |
|
|
357
|
+
| `BIR_BASH_ALLOW` | *(unset)* | Command prefixes a runner-side `Bash` step may run (`*` for all). Unset, `Bash` never executes on the runner, even when `BIR_BUILTIN_TOOLS` lists it |
|
|
356
358
|
| `BIR_INTENT_ARM` | *(unset)* | `1` lets a match found mid-turn arm a plan in the calling tool's place, which means waiting for it (`BIR_INTENT_ARM_BUDGET_MS`, 500); `0` never waits; unset follows `BIR_SEGMENT_ARM` |
|
|
357
359
|
| `BIR_VERBOSE` | *(unset)* | `1` adds per-step `replay.*` detail lines |
|
|
358
360
|
|
|
@@ -548,12 +550,12 @@ first prompt that states the task plainly.
|
|
|
548
550
|
|---|---|---|
|
|
549
551
|
| No `run.matched`, ever | Prompt below the server's `SIMILARITY_THRESHOLD` | Rephrase closer, or lower it server-side. Confirm the original run is in the list — a sub-`RECORDING_MIN_ACTIONS` run is never embedded |
|
|
550
552
|
| `run.matched` but no `plan.armed` | A gate declined | Read the `replay.decision` line's `why=`. Ranked by frequency: replay off — `bir replay off`, `BIR_REPLAY=0`, or never switched on here (a `bir install` without `--replay`; `bir replay status` says which) · scenario not `ready` · similarity below threshold · no step is executable |
|
|
551
|
-
| `replay.decision … code=match_timeout` or `code=service_unhealthy` on many prompts | The service answered the match slower than `BIR_MATCH_BUDGET_MS` (
|
|
553
|
+
| `replay.decision … code=match_timeout` or `code=service_unhealthy` on many prompts | The service answered the match slower than `BIR_MATCH_BUDGET_MS` (3 s), or is failing and in its cool-down | Nothing breaks: those turns run normally, and after a slow match prompts stop waiting on it for a while by themselves (`replay.match_gate`). `bir doctor` shows the service's state. Raise `BIR_MATCH_BUDGET_MS` only if the service is reliably just over it — every prompt pays it |
|
|
552
554
|
| `replay.decision … code=no_delivery_tool` | A direct plan, and no running `bir` server has told the recorder it serves `run_scenario`: the server is missing, disabled or failed in `/mcp`, was not added (a subdirectory or worktree of a repository), or the session was started with `BIR_OFF=1` | `/mcp` shows whether `bir` is connected. `bir install --replay` adds it, in the project's own directory; then start a new session |
|
|
553
555
|
| `replay.vetoed … the model's first call was …, not run_scenario` | A direct plan, and the model reached for another tool first — usually because it has no `run_scenario` | As for `no_delivery_tool`. Nothing ran; the model did the task |
|
|
554
556
|
| `replay.pin_declined` on most steered turns | The model's calls to the expected tool are not the step — another command, another file — so nothing is pinned; if it then moves on, `replay.pin_abandoned` retires the plan | Harmless for those calls, which ran as the model wrote them. Persistent declines on one step usually mean its arguments cannot be shown in the directive (they depend on an earlier step's output, or the command runs to several lines), and the model writes its own |
|
|
555
557
|
| `why="no step is executable"` | None of the scenario's tools is a wrapped MCP server and none is a built-in reachable in this session | Wrap the servers the scenario uses (`bir install --server …`) and recalculate |
|
|
556
|
-
| `plan.armed mode=steer` where you expected `direct` | At least one step is a built-in, an unwrapped MCP server, or `claude-in-chrome` | `bir status` shows what is wrapped. `claude-in-chrome` is `scope: "dynamic"` and can never be wrapped |
|
|
558
|
+
| `plan.armed mode=steer` where you expected `direct` | At least one step is a built-in off `BIR_BUILTIN_TOOLS` (default `Read,Glob,Grep` — so usually `Edit`, `Write` or `Bash`), an unwrapped MCP server, or `claude-in-chrome` | `bir status` shows what is wrapped; the `plan.armed` line's `coverage=` names the `live` step. Widen `BIR_BUILTIN_TOOLS` (and `BIR_BASH_ALLOW`) only deliberately. `claude-in-chrome` is `scope: "dynamic"` and can never be wrapped |
|
|
557
559
|
| Model ignores the directive and diverges every turn | Steering is advisory — the model chooses; `bir` only pins arguments | Expected occasionally. Persistent divergence usually means the scenario's tools do not fit the live task; check `intent` |
|
|
558
560
|
| A `replay.thread` line with no `emitted=` (under `BIR_VERBOSE=1`) | `toolOutputLogic` derived nothing — usually the output shape changed since recording | Dry-replay (§3.1). If the dry run emits and the live run does not, the tool's output shape has drifted; recalculate |
|
|
559
561
|
| `replay.thread_fallback` on a wrapped step | The proxy did not report within the grace window | Check that proxy is alive (`bir doctor`). Repeats mean a slow or dying upstream |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@basein/runner",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.2",
|
|
4
4
|
"description": "A recording MCP proxy: sits between any MCP client and its MCP servers, executes each call on the client's behalf, and records the run as a reusable BaseIn scenario.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|