@basein/runner 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/bin/bir-hooks.d.ts +12 -0
- package/dist/bin/bir-hooks.js +12 -0
- package/dist/control/recorder-main.js +13 -0
- package/dist/control/server.js +12 -0
- package/dist/replay/builtins.d.ts +135 -0
- package/dist/replay/builtins.js +674 -0
- package/dist/replay/controller.d.ts +56 -0
- package/dist/replay/controller.js +72 -5
- package/dist/replay/coverage.d.ts +20 -7
- package/dist/replay/coverage.js +27 -10
- package/docs/calculatedReplay.md +15 -3
- package/docs/calculatedReplayGuide.md +4 -2
- package/package.json +1 -1
- package/production.md +2 -0
|
@@ -196,6 +196,23 @@ export interface ReplayOptions {
|
|
|
196
196
|
segmentArm?: boolean;
|
|
197
197
|
/** `BIR_REPLAY_ALLOW_SERVERS`; undefined means every wrapped server. */
|
|
198
198
|
allowServers?: ReadonlySet<string>;
|
|
199
|
+
/**
|
|
200
|
+
* `BIR_BUILTIN_TOOLS` — the built-ins the runner may execute itself, from
|
|
201
|
+
* the session's own project directory (replay/builtins.ts), or stand in
|
|
202
|
+
* for with a recorded output under `BIR_REPLAY_RECORDED_OUTPUTS=1`.
|
|
203
|
+
* Omitted, the environment decides; the default is the read-only set
|
|
204
|
+
* (`Read,Glob,Grep`). `Edit`, `Write` and `Bash` run only when listed
|
|
205
|
+
* explicitly — and `Bash` additionally only for commands matching
|
|
206
|
+
* {@link ReplayOptions.bashAllow}.
|
|
207
|
+
*/
|
|
208
|
+
builtinTools?: ReadonlySet<string>;
|
|
209
|
+
/**
|
|
210
|
+
* `BIR_BASH_ALLOW` — command prefixes a runner-side `Bash` step may run
|
|
211
|
+
* (`*` for all). Undefined means `Bash` never executes here, even when
|
|
212
|
+
* allow-listed: runner-side execution bypasses the host's permission
|
|
213
|
+
* prompts, so the shell needs its own explicit opt-in.
|
|
214
|
+
*/
|
|
215
|
+
bashAllow?: readonly string[];
|
|
199
216
|
budgets?: Partial<ReplayBudgets>;
|
|
200
217
|
/**
|
|
201
218
|
* The service: the lazy source-run fetch, and the parameter derivation this
|
|
@@ -331,6 +348,23 @@ export interface ReplayState {
|
|
|
331
348
|
* replay`, which has no session and takes the newest proxy.
|
|
332
349
|
*/
|
|
333
350
|
sessionId?: string;
|
|
351
|
+
/**
|
|
352
|
+
* That session's project directory, set by the control server from the
|
|
353
|
+
* hook payloads — never the daemon's own cwd, which is `~/.baseinstrunner`.
|
|
354
|
+
* It is where an allow-listed built-in step executes and the jail it may
|
|
355
|
+
* not leave (replay/builtins.ts). Absent — `bir replay`, an older server —
|
|
356
|
+
* no built-in executes: the step is `not_run`, exactly as before.
|
|
357
|
+
*/
|
|
358
|
+
sessionCwd?: string;
|
|
359
|
+
/**
|
|
360
|
+
* Where this plan's last runner-side `Bash` step left its shell. The
|
|
361
|
+
* host's own semantics, which the recorded chain was written against: the
|
|
362
|
+
* working directory persists between Bash calls (variables and functions
|
|
363
|
+
* do not), so a chain of `cd sub` then `cat file` replays as it ran.
|
|
364
|
+
* Starts at {@link sessionCwd}; never read for `Read`/`Edit`/`Write`
|
|
365
|
+
* paths, which resolve against the project directory as the host's do.
|
|
366
|
+
*/
|
|
367
|
+
bashCwd?: string;
|
|
334
368
|
/**
|
|
335
369
|
* Which kind of row was handed out (segmented.md R-OUT-7). A `segment` is a
|
|
336
370
|
* named sub-task of a recording rather than a whole task, and it is judged
|
|
@@ -447,6 +481,10 @@ export declare class ReplayController {
|
|
|
447
481
|
private readonly opts;
|
|
448
482
|
/** See {@link ReplayOptions.recordedFallback}. */
|
|
449
483
|
private readonly recordedFallback;
|
|
484
|
+
/** See {@link ReplayOptions.builtinTools}. */
|
|
485
|
+
private readonly builtinTools;
|
|
486
|
+
/** See {@link ReplayOptions.bashAllow}. */
|
|
487
|
+
private readonly bashAllow?;
|
|
450
488
|
/** Calls waiting for a plan's pins to change (see `awaitPins`). */
|
|
451
489
|
private readonly pinWaiters;
|
|
452
490
|
/**
|
|
@@ -762,6 +800,19 @@ export declare class ReplayController {
|
|
|
762
800
|
* asked for by id and which the allowlist does not gate.
|
|
763
801
|
*/
|
|
764
802
|
private executeStep;
|
|
803
|
+
/**
|
|
804
|
+
* Run one allow-listed built-in step on the runner, inside `cwd`.
|
|
805
|
+
*
|
|
806
|
+
* The result mimics the hook's view of the tool, because that is what was
|
|
807
|
+
* recorded for built-in steps and what their `toolOutputLogic` was authored
|
|
808
|
+
* against (control/server.ts, `onToolPost`) — and it is serialized and
|
|
809
|
+
* redacted through the exact pipeline every recorded payload takes, so
|
|
810
|
+
* `Bash` output cannot carry a secret the recording would have stripped.
|
|
811
|
+
*
|
|
812
|
+
* Failures keep {@link ExecuteStep}'s contract: policy (`not_run`), a step
|
|
813
|
+
* cut off by its clock (`uncertain`), a step that ran and broke (`failed`).
|
|
814
|
+
*/
|
|
815
|
+
private executeBuiltin;
|
|
765
816
|
/** Where recorded outputs come from, when they may stand in at all. */
|
|
766
817
|
private recordedSource;
|
|
767
818
|
/**
|
|
@@ -797,6 +848,11 @@ export declare class ReplayController {
|
|
|
797
848
|
* wrapped server the allowlist excludes is still proxied, still reported, and
|
|
798
849
|
* still recorded in the proxy's serialization. Threading such a step from the
|
|
799
850
|
* hook's differently-shaped view would derive nothing at all.
|
|
851
|
+
*
|
|
852
|
+
* The built-in allow-list is deliberately NOT passed here for the same
|
|
853
|
+
* reason in the other direction: a *steered* built-in is never proxied,
|
|
854
|
+
* whatever `BIR_BUILTIN_TOOLS` says, so its output can only ever arrive
|
|
855
|
+
* from the hook.
|
|
800
856
|
*/
|
|
801
857
|
private reachFor;
|
|
802
858
|
/**
|
|
@@ -18,6 +18,7 @@ import { serializeCapped } from "../record/truncate.js";
|
|
|
18
18
|
import { logDetail, logLine, errText } from "../util/log.js";
|
|
19
19
|
import { serviceHealth, serviceTimeoutMs, timeoutSignal, } from "../util/service-health.js";
|
|
20
20
|
import { bashCommandFor } from "./bash-channel.js";
|
|
21
|
+
import { BuiltinNotRunnable, BuiltinTimeout, parseBashAllow, parseBuiltinTools, runBuiltin, } from "./builtins.js";
|
|
21
22
|
import { BASH_CHANNEL_MAX, MAX_REPLAY_REASON, truncate } from "./bundle.js";
|
|
22
23
|
import { coverageOf, modeFor, reachOf } from "./coverage.js";
|
|
23
24
|
import { deriveParameters, } from "./derive.js";
|
|
@@ -185,6 +186,10 @@ export class ReplayController {
|
|
|
185
186
|
opts;
|
|
186
187
|
/** See {@link ReplayOptions.recordedFallback}. */
|
|
187
188
|
recordedFallback;
|
|
189
|
+
/** See {@link ReplayOptions.builtinTools}. */
|
|
190
|
+
builtinTools;
|
|
191
|
+
/** See {@link ReplayOptions.bashAllow}. */
|
|
192
|
+
bashAllow;
|
|
188
193
|
/** Calls waiting for a plan's pins to change (see `awaitPins`). */
|
|
189
194
|
pinWaiters = new WeakMap();
|
|
190
195
|
/**
|
|
@@ -245,6 +250,8 @@ export class ReplayController {
|
|
|
245
250
|
this.intentMatch = { ...DEFAULT_INTENT_MATCH, ...(opts.intentMatch ?? {}) };
|
|
246
251
|
this.recordedFallback =
|
|
247
252
|
opts.recordedFallback ?? process.env.BIR_REPLAY_RECORDED_OUTPUTS === "1";
|
|
253
|
+
this.builtinTools = opts.builtinTools ?? parseBuiltinTools(process.env.BIR_BUILTIN_TOOLS);
|
|
254
|
+
this.bashAllow = opts.bashAllow ?? parseBashAllow(process.env.BIR_BASH_ALLOW);
|
|
248
255
|
}
|
|
249
256
|
/**
|
|
250
257
|
* Whether a handed-out segment may actually arm (segmented.md R-OUT-10).
|
|
@@ -423,8 +430,11 @@ export class ReplayController {
|
|
|
423
430
|
: "known_bad_first_step");
|
|
424
431
|
}
|
|
425
432
|
const planned = stopAt === undefined ? entries : entries.slice(0, stopAt);
|
|
426
|
-
// Gate 6 — tool coverage, over the steps that will actually run.
|
|
427
|
-
|
|
433
|
+
// Gate 6 — tool coverage, over the steps that will actually run. A
|
|
434
|
+
// built-in on `BIR_BUILTIN_TOOLS` counts as reachable: the runner
|
|
435
|
+
// executes it itself (replay/builtins.ts), so a chain that starts with a
|
|
436
|
+
// `Read` no longer forces steer mode.
|
|
437
|
+
const mode = modeFor(planned.map((e) => e.step), wrapped, this.opts.allowServers, this.builtinTools);
|
|
428
438
|
if (mode === "none")
|
|
429
439
|
return decline("no step is executable", "coverage");
|
|
430
440
|
// Tool results feed derivation as context, and they are whatever the agent
|
|
@@ -598,7 +608,7 @@ export class ReplayController {
|
|
|
598
608
|
unusable: flat.stopReason?.reason,
|
|
599
609
|
// Which steps the service told this plan to stop in front of, and why.
|
|
600
610
|
stops: stopLines(entries),
|
|
601
|
-
coverage: coverageOf(entries.map((e) => e.step), wrapped, this.opts.allowServers).join(","),
|
|
611
|
+
coverage: coverageOf(entries.map((e) => e.step), wrapped, this.opts.allowServers, this.builtinTools).join(","),
|
|
602
612
|
tools: entries.map((e) => e.step.toolName ?? "").join(","),
|
|
603
613
|
});
|
|
604
614
|
return state;
|
|
@@ -1565,7 +1575,7 @@ export class ReplayController {
|
|
|
1565
1575
|
}
|
|
1566
1576
|
if (flat.entries.length === 0)
|
|
1567
1577
|
return { ok: false, why: "the chain has no runnable step" };
|
|
1568
|
-
const mode = modeFor(flat.entries.map((e) => e.step), wrapped, this.opts.allowServers);
|
|
1578
|
+
const mode = modeFor(flat.entries.map((e) => e.step), wrapped, this.opts.allowServers, this.builtinTools);
|
|
1569
1579
|
if (mode === "none")
|
|
1570
1580
|
return { ok: false, why: "no step is executable" };
|
|
1571
1581
|
const state = {
|
|
@@ -1636,7 +1646,7 @@ export class ReplayController {
|
|
|
1636
1646
|
scenario: scenario.id,
|
|
1637
1647
|
mode,
|
|
1638
1648
|
steps: scenario.steps.length,
|
|
1639
|
-
coverage: coverageOf(scenario.steps, wrapped, this.opts.allowServers).join(","),
|
|
1649
|
+
coverage: coverageOf(scenario.steps, wrapped, this.opts.allowServers, this.builtinTools).join(","),
|
|
1640
1650
|
why: "bir replay — no match, an operator asked for it",
|
|
1641
1651
|
});
|
|
1642
1652
|
// An operator's run keeps its own contract (R-CALL-37): out of time, the
|
|
@@ -2066,6 +2076,16 @@ export class ReplayController {
|
|
|
2066
2076
|
const name = step.toolName ?? "";
|
|
2067
2077
|
const mcp = parseQualifiedName(name);
|
|
2068
2078
|
if (!mcp) {
|
|
2079
|
+
// An allow-listed built-in runs *here*, against the session's own
|
|
2080
|
+
// project directory (replay/builtins.ts). Everything else about it —
|
|
2081
|
+
// no session cwd (`bir replay`, an older server), off the list, no
|
|
2082
|
+
// runner-side implementation, a path that leaves the project, `Bash`
|
|
2083
|
+
// without `BIR_BASH_ALLOW` — stays the old contract: `not_run`, so
|
|
2084
|
+
// the session can do the step itself or its recorded output can
|
|
2085
|
+
// stand in (`BIR_REPLAY_RECORDED_OUTPUTS=1`).
|
|
2086
|
+
if (this.builtinTools.has(name) && state.sessionCwd) {
|
|
2087
|
+
return this.executeBuiltin(name, input, state, opts.deadline);
|
|
2088
|
+
}
|
|
2069
2089
|
throw new StepRunError(`${name || "this step"} is a built-in tool, which only runs in the session`, "not_run");
|
|
2070
2090
|
}
|
|
2071
2091
|
// `BIR_REPLAY_ALLOW_SERVERS` is about exactly this: a call bir makes by
|
|
@@ -2098,6 +2118,48 @@ export class ReplayController {
|
|
|
2098
2118
|
return serializeCapped(redact(result));
|
|
2099
2119
|
};
|
|
2100
2120
|
}
|
|
2121
|
+
/**
|
|
2122
|
+
* Run one allow-listed built-in step on the runner, inside `cwd`.
|
|
2123
|
+
*
|
|
2124
|
+
* The result mimics the hook's view of the tool, because that is what was
|
|
2125
|
+
* recorded for built-in steps and what their `toolOutputLogic` was authored
|
|
2126
|
+
* against (control/server.ts, `onToolPost`) — and it is serialized and
|
|
2127
|
+
* redacted through the exact pipeline every recorded payload takes, so
|
|
2128
|
+
* `Bash` output cannot carry a secret the recording would have stripped.
|
|
2129
|
+
*
|
|
2130
|
+
* Failures keep {@link ExecuteStep}'s contract: policy (`not_run`), a step
|
|
2131
|
+
* cut off by its clock (`uncertain`), a step that ran and broke (`failed`).
|
|
2132
|
+
*/
|
|
2133
|
+
async executeBuiltin(name, input, state, deadline) {
|
|
2134
|
+
const cwd = state.sessionCwd; // the caller executes only with one
|
|
2135
|
+
const left = deadline !== undefined ? deadline - Date.now() : Number.POSITIVE_INFINITY;
|
|
2136
|
+
const timeoutMs = Math.max(MIN_STEP_MS, Math.min(this.budgets.stepMs, left));
|
|
2137
|
+
// The audit line precedes the action, always (§13.2, mitigation 4).
|
|
2138
|
+
logDetail("replay.builtin", { tool: name, cwd });
|
|
2139
|
+
let result;
|
|
2140
|
+
try {
|
|
2141
|
+
result = await runBuiltin(name, input, {
|
|
2142
|
+
cwd,
|
|
2143
|
+
timeoutMs,
|
|
2144
|
+
bashAllow: this.bashAllow,
|
|
2145
|
+
// The plan's Bash steps share a working directory, as the host's do.
|
|
2146
|
+
bashCwd: state.bashCwd,
|
|
2147
|
+
onBashCwd: (pwd) => {
|
|
2148
|
+
state.bashCwd = pwd;
|
|
2149
|
+
},
|
|
2150
|
+
});
|
|
2151
|
+
}
|
|
2152
|
+
catch (err) {
|
|
2153
|
+
if (err instanceof BuiltinNotRunnable)
|
|
2154
|
+
throw new StepRunError(err.message, "not_run");
|
|
2155
|
+
if (err instanceof BuiltinTimeout)
|
|
2156
|
+
throw new StepRunError(err.message, "uncertain");
|
|
2157
|
+
// It ran and broke — a missing file, a non-unique `old_string`, a
|
|
2158
|
+
// non-zero exit. The step failed here, now; never threaded as success.
|
|
2159
|
+
throw new StepRunError(errText(err), "failed");
|
|
2160
|
+
}
|
|
2161
|
+
return serializeCapped(redact(result));
|
|
2162
|
+
}
|
|
2101
2163
|
/** Where recorded outputs come from, when they may stand in at all. */
|
|
2102
2164
|
recordedSource(state, force = false) {
|
|
2103
2165
|
if (!force && !this.recordedFallback)
|
|
@@ -2230,6 +2292,11 @@ export class ReplayController {
|
|
|
2230
2292
|
* wrapped server the allowlist excludes is still proxied, still reported, and
|
|
2231
2293
|
* still recorded in the proxy's serialization. Threading such a step from the
|
|
2232
2294
|
* hook's differently-shaped view would derive nothing at all.
|
|
2295
|
+
*
|
|
2296
|
+
* The built-in allow-list is deliberately NOT passed here for the same
|
|
2297
|
+
* reason in the other direction: a *steered* built-in is never proxied,
|
|
2298
|
+
* whatever `BIR_BUILTIN_TOOLS` says, so its output can only ever arrive
|
|
2299
|
+
* from the hook.
|
|
2233
2300
|
*/
|
|
2234
2301
|
reachFor(state, step) {
|
|
2235
2302
|
return reachOf(step.toolName ?? "", state.wrapped);
|
|
@@ -8,9 +8,17 @@
|
|
|
8
8
|
* only happen inside the live session.
|
|
9
9
|
*
|
|
10
10
|
* `live` covers three genuinely different cases and treats them identically,
|
|
11
|
-
* correctly: a built-in (`Bash`, `Read`, `Edit`, …)
|
|
12
|
-
* not to wrap, and `claude-in-chrome`
|
|
13
|
-
* no config file, and therefore can
|
|
11
|
+
* correctly: a built-in (`Bash`, `Read`, `Edit`, …) outside the built-in
|
|
12
|
+
* allow-list, an MCP server the user chose not to wrap, and `claude-in-chrome`
|
|
13
|
+
* — which is `scope: "dynamic"`, appears in no config file, and therefore can
|
|
14
|
+
* never be wrapped at all.
|
|
15
|
+
*
|
|
16
|
+
* A built-in *on* the allow-list (`BIR_BUILTIN_TOOLS`, default `Read,Glob,
|
|
17
|
+
* Grep`) is `direct`: the runner executes it itself (replay/builtins.ts), or —
|
|
18
|
+
* under `BIR_REPLAY_RECORDED_OUTPUTS=1` — serves its recorded output. The
|
|
19
|
+
* callers that ask a *different* question, "will a proxy report this call"
|
|
20
|
+
* (`ReplayController.reachFor`), deliberately pass no allow-list: a built-in
|
|
21
|
+
* is never proxied, whoever executes it.
|
|
14
22
|
*/
|
|
15
23
|
import type { SerializedScenarioStep } from "./types.js";
|
|
16
24
|
/** Where a step's tool can actually be executed. */
|
|
@@ -19,18 +27,23 @@ export type StepReach = "direct" | "live";
|
|
|
19
27
|
export type ReplayMode = "direct" | "steer" | "none";
|
|
20
28
|
/**
|
|
21
29
|
* `direct` when the tool is `mcp__<server>__<tool>` for a server this control
|
|
22
|
-
* plane has a proxy for
|
|
30
|
+
* plane has a proxy for, or a built-in on `builtins`; `live` otherwise.
|
|
23
31
|
*
|
|
24
32
|
* `allow` is the `BIR_REPLAY_ALLOW_SERVERS` allowlist (§13.2, mitigation 2): a
|
|
25
33
|
* wrapped server outside it is deliberately demoted to `live`, so it can still be
|
|
26
34
|
* steered inside the session — where the user's own permission rules apply — but
|
|
27
35
|
* is never called unattended by us.
|
|
36
|
+
*
|
|
37
|
+
* `builtins` is the `BIR_BUILTIN_TOOLS` allow-list (replay/builtins.ts): the
|
|
38
|
+
* built-ins the runner may execute itself, or stand in for with a recorded
|
|
39
|
+
* output. Omitted means none — the old contract — which is what every caller
|
|
40
|
+
* that is not the mode decision wants.
|
|
28
41
|
*/
|
|
29
|
-
export declare function reachOf(toolName: string, wrapped: ReadonlySet<string>, allow?: ReadonlySet<string>): StepReach;
|
|
42
|
+
export declare function reachOf(toolName: string, wrapped: ReadonlySet<string>, allow?: ReadonlySet<string>, builtins?: ReadonlySet<string>): StepReach;
|
|
30
43
|
/** The mode for a whole scenario: `direct` only when *every* step is reachable. */
|
|
31
|
-
export declare function modeFor(steps: readonly SerializedScenarioStep[], wrapped: ReadonlySet<string>, allow?: ReadonlySet<string>): ReplayMode;
|
|
44
|
+
export declare function modeFor(steps: readonly SerializedScenarioStep[], wrapped: ReadonlySet<string>, allow?: ReadonlySet<string>, builtins?: ReadonlySet<string>): ReplayMode;
|
|
32
45
|
/** Per-step reach, in step order — what `plan.armed` logs and `bir doctor` shows. */
|
|
33
|
-
export declare function coverageOf(steps: readonly SerializedScenarioStep[], wrapped: ReadonlySet<string>, allow?: ReadonlySet<string>): StepReach[];
|
|
46
|
+
export declare function coverageOf(steps: readonly SerializedScenarioStep[], wrapped: ReadonlySet<string>, allow?: ReadonlySet<string>, builtins?: ReadonlySet<string>): StepReach[];
|
|
34
47
|
/**
|
|
35
48
|
* Parse `BIR_REPLAY_ALLOW_SERVERS`. Unset (or empty) means *every wrapped
|
|
36
49
|
* server*, which is why the return is `undefined` rather than an empty set — an
|
package/dist/replay/coverage.js
CHANGED
|
@@ -8,37 +8,54 @@
|
|
|
8
8
|
* only happen inside the live session.
|
|
9
9
|
*
|
|
10
10
|
* `live` covers three genuinely different cases and treats them identically,
|
|
11
|
-
* correctly: a built-in (`Bash`, `Read`, `Edit`, …)
|
|
12
|
-
* not to wrap, and `claude-in-chrome`
|
|
13
|
-
* no config file, and therefore can
|
|
11
|
+
* correctly: a built-in (`Bash`, `Read`, `Edit`, …) outside the built-in
|
|
12
|
+
* allow-list, an MCP server the user chose not to wrap, and `claude-in-chrome`
|
|
13
|
+
* — which is `scope: "dynamic"`, appears in no config file, and therefore can
|
|
14
|
+
* never be wrapped at all.
|
|
15
|
+
*
|
|
16
|
+
* A built-in *on* the allow-list (`BIR_BUILTIN_TOOLS`, default `Read,Glob,
|
|
17
|
+
* Grep`) is `direct`: the runner executes it itself (replay/builtins.ts), or —
|
|
18
|
+
* under `BIR_REPLAY_RECORDED_OUTPUTS=1` — serves its recorded output. The
|
|
19
|
+
* callers that ask a *different* question, "will a proxy report this call"
|
|
20
|
+
* (`ReplayController.reachFor`), deliberately pass no allow-list: a built-in
|
|
21
|
+
* is never proxied, whoever executes it.
|
|
14
22
|
*/
|
|
15
23
|
import { parseQualifiedName } from "../control/correlation.js";
|
|
16
24
|
/**
|
|
17
25
|
* `direct` when the tool is `mcp__<server>__<tool>` for a server this control
|
|
18
|
-
* plane has a proxy for
|
|
26
|
+
* plane has a proxy for, or a built-in on `builtins`; `live` otherwise.
|
|
19
27
|
*
|
|
20
28
|
* `allow` is the `BIR_REPLAY_ALLOW_SERVERS` allowlist (§13.2, mitigation 2): a
|
|
21
29
|
* wrapped server outside it is deliberately demoted to `live`, so it can still be
|
|
22
30
|
* steered inside the session — where the user's own permission rules apply — but
|
|
23
31
|
* is never called unattended by us.
|
|
32
|
+
*
|
|
33
|
+
* `builtins` is the `BIR_BUILTIN_TOOLS` allow-list (replay/builtins.ts): the
|
|
34
|
+
* built-ins the runner may execute itself, or stand in for with a recorded
|
|
35
|
+
* output. Omitted means none — the old contract — which is what every caller
|
|
36
|
+
* that is not the mode decision wants.
|
|
24
37
|
*/
|
|
25
|
-
export function reachOf(toolName, wrapped, allow) {
|
|
38
|
+
export function reachOf(toolName, wrapped, allow, builtins) {
|
|
26
39
|
const mcp = parseQualifiedName(toolName);
|
|
27
|
-
if (!mcp
|
|
40
|
+
if (!mcp)
|
|
41
|
+
return builtins?.has(toolName) ? "direct" : "live";
|
|
42
|
+
if (!wrapped.has(mcp.serverName))
|
|
28
43
|
return "live";
|
|
29
44
|
if (allow && !allow.has(mcp.serverName))
|
|
30
45
|
return "live";
|
|
31
46
|
return "direct";
|
|
32
47
|
}
|
|
33
48
|
/** The mode for a whole scenario: `direct` only when *every* step is reachable. */
|
|
34
|
-
export function modeFor(steps, wrapped, allow) {
|
|
49
|
+
export function modeFor(steps, wrapped, allow, builtins) {
|
|
35
50
|
if (steps.length === 0)
|
|
36
51
|
return "none";
|
|
37
|
-
return steps.every((s) => reachOf(s.toolName ?? "", wrapped, allow) === "direct")
|
|
52
|
+
return steps.every((s) => reachOf(s.toolName ?? "", wrapped, allow, builtins) === "direct")
|
|
53
|
+
? "direct"
|
|
54
|
+
: "steer";
|
|
38
55
|
}
|
|
39
56
|
/** Per-step reach, in step order — what `plan.armed` logs and `bir doctor` shows. */
|
|
40
|
-
export function coverageOf(steps, wrapped, allow) {
|
|
41
|
-
return steps.map((s) => reachOf(s.toolName ?? "", wrapped, allow));
|
|
57
|
+
export function coverageOf(steps, wrapped, allow, builtins) {
|
|
58
|
+
return steps.map((s) => reachOf(s.toolName ?? "", wrapped, allow, builtins));
|
|
42
59
|
}
|
|
43
60
|
/**
|
|
44
61
|
* Parse `BIR_REPLAY_ALLOW_SERVERS`. Unset (or empty) means *every wrapped
|
package/docs/calculatedReplay.md
CHANGED
|
@@ -45,7 +45,7 @@ this design has a ladder whose cheapest rung is free — and the more of a stack
|
|
|
45
45
|
|---|---|---|
|
|
46
46
|
| Runs MCP tools itself | No (`replay.md` §12: *"rrepeat never executes tools — not even MCP"*) | **Yes**, on the connection it already owns |
|
|
47
47
|
| Divergence fallback | A second Agent SDK session — the dominant cost | Direct `tools/call` through the proxies — **$0** |
|
|
48
|
-
| Reaches built-ins (`Bash`, `Read`) | Only by steering the live session |
|
|
48
|
+
| Reaches built-ins (`Bash`, `Read`) | Only by steering the live session | Read-only ones (`Read`, `Glob`, `Grep`) run on the runner itself, jailed to the project (`BIR_BUILTIN_TOOLS`); `Edit`/`Write`/`Bash` opt-in, else steered |
|
|
49
49
|
| Needs `@anthropic-ai/*` at runtime | Yes | No (§16.3) |
|
|
50
50
|
|
|
51
51
|
---
|
|
@@ -243,15 +243,27 @@ export function modeFor(steps: SerializedScenarioStep[], wrapped: ReadonlySet<st
|
|
|
243
243
|
|
|
244
244
|
| Every step | Mode | Where the tools run | Model spend |
|
|
245
245
|
|---|---|---|---|
|
|
246
|
-
| a wrapped MCP tool | **`direct`** (§6) | The proxies, on connections already open | one Haiku derivation + one turn that reads the results |
|
|
246
|
+
| a wrapped MCP tool, or a built-in on `BIR_BUILTIN_TOOLS` | **`direct`** (§6) | The proxies, on connections already open; allow-listed built-ins on the runner itself, jailed to the session's project directory (replay/builtins.ts) | one Haiku derivation + one turn that reads the results |
|
|
247
247
|
| anything else, or mixed | **`steer`** (§7) | The live session, inputs pinned | one Haiku derivation + the live turn |
|
|
248
248
|
| — | `none` | nowhere | declines at gate 5 |
|
|
249
249
|
|
|
250
250
|
`live` covers three genuinely different cases and treats them identically, correctly: a built-in
|
|
251
|
-
|
|
251
|
+
off the `BIR_BUILTIN_TOOLS` allow-list (by default everything but `Read`, `Glob`, `Grep` — so
|
|
252
|
+
`Bash`, `Edit`, `Write`, `Task`, `WebFetch`), an MCP server the user chose not to wrap, and
|
|
252
253
|
`claude-in-chrome` — which is `scope: "dynamic"`, appears in no config file, and therefore can never
|
|
253
254
|
be wrapped (v1 design §1). None of them can be reached except through the session.
|
|
254
255
|
|
|
256
|
+
A built-in **on** the allow-list runs on the runner, in the session's own project directory and
|
|
257
|
+
confined to it — a path that resolves outside the project, symlinks included, is refused and the
|
|
258
|
+
step is `not_run`, the clean hand-over. `Bash` needs a second opt-in on top (`BIR_BASH_ALLOW`,
|
|
259
|
+
command prefixes), because a runner-side step never reaches the host's permission prompts. The
|
|
260
|
+
results mimic the hook's view of each tool, which is what the recorder stored for built-in steps
|
|
261
|
+
and what their `toolOutputLogic` was authored against; they pass through the same
|
|
262
|
+
`serializeCapped(redact(…))` as every recorded payload. Bash keeps the host's own semantics a
|
|
263
|
+
recorded chain was written against: each call is a fresh `-l` shell (variables and functions do
|
|
264
|
+
not persist), while the **working directory carries over** from the plan's previous Bash step —
|
|
265
|
+
`cd sub` then `cat file` replays as it ran.
|
|
266
|
+
|
|
255
267
|
A **mixed** scenario in `steer` mode still gets direct execution for its wrapped steps once it
|
|
256
268
|
diverges (§8), so coverage is not all-or-nothing at the divergence boundary — only at the arm-time
|
|
257
269
|
mode choice.
|
|
@@ -352,7 +352,9 @@ so there is exactly one switch and it cannot get out of step with itself.
|
|
|
352
352
|
| `BIR_REPLAY_PLAN_BUDGET_MS` | `100000` | Whole-plan ceiling for direct execution; `BIR_REPLAY_BUDGET_MS`, where set, overrides it. Past about 110 000 a long run may be moved to the background by Claude Code. Derivation plus plan is held under 270 s |
|
|
353
353
|
| `BIR_STEP_TIMEOUT_MS` | `60000` | Per-step ceiling for one direct `tools/call`, clipped to what the plan has left |
|
|
354
354
|
| `BIR_LOGIC_TIMEOUT_MS` | `1000` | How long one scenario logic body may run before it is stopped. At most 10 000 |
|
|
355
|
-
| `BIR_REPLAY_RECORDED_OUTPUTS` | *(unset)* | `1` lets a step that could not be sent (a built-in
|
|
355
|
+
| `BIR_REPLAY_RECORDED_OUTPUTS` | *(unset)* | `1` lets a step that could not be sent (a built-in off `BIR_BUILTIN_TOOLS`, no proxy, a server off the allowlist) stand on its recorded output instead of stopping the plan. Never for a timeout or an upstream error |
|
|
356
|
+
| `BIR_BUILTIN_TOOLS` | `Read,Glob,Grep` | Built-ins the runner executes itself during a replay, in the session's own project directory and confined to it. A chain whose only non-MCP steps are on this list arms `direct`. Add `Edit,Write,Bash` only deliberately — a runner-side step never reaches the host's permission prompts. `none` restores the old behaviour: every built-in hands the turn back |
|
|
357
|
+
| `BIR_BASH_ALLOW` | *(unset)* | Command prefixes a runner-side `Bash` step may run (`*` for all). Unset, `Bash` never executes on the runner, even when `BIR_BUILTIN_TOOLS` lists it. A plan's Bash steps share a working directory, exactly as the host's own Bash tool does — variables and functions do not persist, also as the host's |
|
|
356
358
|
| `BIR_INTENT_ARM` | *(unset)* | `1` lets a match found mid-turn arm a plan in the calling tool's place, which means waiting for it (`BIR_INTENT_ARM_BUDGET_MS`, 500); `0` never waits; unset follows `BIR_SEGMENT_ARM` |
|
|
357
359
|
| `BIR_VERBOSE` | *(unset)* | `1` adds per-step `replay.*` detail lines |
|
|
358
360
|
|
|
@@ -553,7 +555,7 @@ first prompt that states the task plainly.
|
|
|
553
555
|
| `replay.vetoed … the model's first call was …, not run_scenario` | A direct plan, and the model reached for another tool first — usually because it has no `run_scenario` | As for `no_delivery_tool`. Nothing ran; the model did the task |
|
|
554
556
|
| `replay.pin_declined` on most steered turns | The model's calls to the expected tool are not the step — another command, another file — so nothing is pinned; if it then moves on, `replay.pin_abandoned` retires the plan | Harmless for those calls, which ran as the model wrote them. Persistent declines on one step usually mean its arguments cannot be shown in the directive (they depend on an earlier step's output, or the command runs to several lines), and the model writes its own |
|
|
555
557
|
| `why="no step is executable"` | None of the scenario's tools is a wrapped MCP server and none is a built-in reachable in this session | Wrap the servers the scenario uses (`bir install --server …`) and recalculate |
|
|
556
|
-
| `plan.armed mode=steer` where you expected `direct` | At least one step is a built-in, an unwrapped MCP server, or `claude-in-chrome` | `bir status` shows what is wrapped. `claude-in-chrome` is `scope: "dynamic"` and can never be wrapped |
|
|
558
|
+
| `plan.armed mode=steer` where you expected `direct` | At least one step is a built-in off `BIR_BUILTIN_TOOLS` (default `Read,Glob,Grep` — so usually `Edit`, `Write` or `Bash`), an unwrapped MCP server, or `claude-in-chrome` | `bir status` shows what is wrapped; the `plan.armed` line's `coverage=` names the `live` step. Widen `BIR_BUILTIN_TOOLS` (and `BIR_BASH_ALLOW`) only deliberately. `claude-in-chrome` is `scope: "dynamic"` and can never be wrapped |
|
|
557
559
|
| Model ignores the directive and diverges every turn | Steering is advisory — the model chooses; `bir` only pins arguments | Expected occasionally. Persistent divergence usually means the scenario's tools do not fit the live task; check `intent` |
|
|
558
560
|
| A `replay.thread` line with no `emitted=` (under `BIR_VERBOSE=1`) | `toolOutputLogic` derived nothing — usually the output shape changed since recording | Dry-replay (§3.1). If the dry run emits and the live run does not, the tool's output shape has drifted; recalculate |
|
|
559
561
|
| `replay.thread_fallback` on a wrapped step | The proxy did not report within the grace window | Check that proxy is alive (`bir doctor`). Repeats mean a slow or dying upstream |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@basein/runner",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.3",
|
|
4
4
|
"description": "A recording MCP proxy: sits between any MCP client and its MCP servers, executes each call on the client's behalf, and records the run as a reusable BaseIn scenario.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/production.md
CHANGED
|
@@ -992,6 +992,8 @@ Defaults were checked in `src/`.
|
|
|
992
992
|
| `BIR_STEP_TIMEOUT_MS` | recorder | 60000 | One direct `tools/call`. |
|
|
993
993
|
| `BIR_LOGIC_TIMEOUT_MS` | recorder | 1000 (capped at 10000) | Synchronous time limit for one scenario logic body. |
|
|
994
994
|
| `BIR_REPLAY_RECORDED_OUTPUTS` | recorder | unset | `1`: a step that could not be sent may use its recorded output, labelled as such. |
|
|
995
|
+
| `BIR_BUILTIN_TOOLS` | recorder | `Read,Glob,Grep` | Built-ins the runner executes itself during a replay, in the session's project directory and confined to it. Add `Edit,Write,Bash` deliberately — runner-side steps bypass the host's permission prompts. `none` turns it off. |
|
|
996
|
+
| `BIR_BASH_ALLOW` | recorder | unset | Command prefixes a runner-side `Bash` step may run (`*` for all). Unset, `Bash` never executes on the runner even when listed. A plan's Bash steps share a working directory, as the host's do; variables do not persist, also as the host's. |
|
|
995
997
|
| `BIR_DERIVE_RECENT_RESULTS` | recorder | 5 | How many of this prompt's tool results derivation may read (`0`: none). Never for plans with a command sink. |
|
|
996
998
|
| `BIR_INTENT_MATCH` | recorder | on | `0` turns mid-turn intent probes off. |
|
|
997
999
|
| `BIR_INTENT_MATCH_BUDGET_MS` | recorder | 4000 | One probe's own ceiling. It is not a hook wait. |
|