@basein/runner 0.2.8 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -21
- package/dist/auth/client.d.ts +40 -1
- package/dist/auth/client.js +77 -9
- package/dist/bin/bir-hooks.d.ts +18 -3
- package/dist/bin/bir-hooks.js +124 -38
- package/dist/bin/bir.d.ts +2 -0
- package/dist/bin/bir.js +362 -39
- package/dist/bin/setup.d.ts +72 -0
- package/dist/bin/setup.js +286 -0
- package/dist/config/adapters/claude-code.d.ts +90 -4
- package/dist/config/adapters/claude-code.js +164 -16
- package/dist/config/generate.d.ts +93 -1
- package/dist/config/generate.js +90 -3
- package/dist/control/client.d.ts +5 -0
- package/dist/control/client.js +8 -0
- package/dist/control/daemon.d.ts +116 -0
- package/dist/control/daemon.js +339 -0
- package/dist/control/discovery.d.ts +26 -0
- package/dist/control/discovery.js +41 -9
- package/dist/control/ensure-hook.d.ts +39 -0
- package/dist/control/ensure-hook.js +98 -0
- package/dist/control/paths.d.ts +14 -0
- package/dist/control/paths.js +20 -0
- package/dist/control/server.d.ts +28 -0
- package/dist/control/server.js +15 -2
- package/dist/proxy/session.d.ts +8 -1
- package/dist/proxy/session.js +28 -6
- package/docs/calculatedReplayGuide.md +157 -70
- package/docs/installRun.md +457 -111
- package/docs/loginWeb.md +1 -1
- package/docs/quickstart.md +193 -158
- package/package.json +2 -1
- package/scripts/install.ps1 +669 -0
- package/scripts/install.sh +586 -0
package/dist/control/server.d.ts
CHANGED
|
@@ -53,6 +53,14 @@ export interface ControlServerOptions {
|
|
|
53
53
|
cwd: string;
|
|
54
54
|
/** Preferred port; 0 (or a busy port) falls back to an ephemeral one. */
|
|
55
55
|
port?: number;
|
|
56
|
+
/**
|
|
57
|
+
* Whether a busy `port` may fall back to an ephemeral one. Default true — a
|
|
58
|
+
* recorder in a terminal is better on some port than not at all. A background
|
|
59
|
+
* recorder whose port is written into a project's hooks passes false: on a
|
|
60
|
+
* fallback port it would answer SessionStart and then every other hook would
|
|
61
|
+
* post into the void, which is worse than exiting with a reason.
|
|
62
|
+
*/
|
|
63
|
+
portFallback?: boolean;
|
|
56
64
|
/** Loopback bearer token. Generated when omitted. */
|
|
57
65
|
token?: string;
|
|
58
66
|
/** Config keys of the servers a proxy wraps. Grows as proxies register. */
|
|
@@ -90,6 +98,26 @@ export interface ControlServerOptions {
|
|
|
90
98
|
* recording and nothing else happens.
|
|
91
99
|
*/
|
|
92
100
|
replay?: ReplayOptions;
|
|
101
|
+
/**
|
|
102
|
+
* Where the replay switches came from — the environment, the project's
|
|
103
|
+
* stored policy, or the defaults. Reported on `/health` so `bir doctor` can
|
|
104
|
+
* say which, because "the allow-list I set is gone" has exactly one cause.
|
|
105
|
+
*/
|
|
106
|
+
replaySource?: "env" | "sidecar" | "default";
|
|
107
|
+
/**
|
|
108
|
+
* Whose account the recordings land in, for `/health` and `bir doctor`. "The
|
|
109
|
+
* run is not in Recordings" is, more often than not, "it is in somebody
|
|
110
|
+
* else's", and nothing said which until now.
|
|
111
|
+
*/
|
|
112
|
+
account?: string;
|
|
113
|
+
/**
|
|
114
|
+
* What 'POST /control/stop' does after answering. 'bir-hooks' passes its own
|
|
115
|
+
* shutdown; a test passes nothing and the route merely says it would. The
|
|
116
|
+
* route exists so a background recorder can be stopped the way Ctrl-C stops
|
|
117
|
+
* one in a terminal — finishing the run and draining the queue — rather than
|
|
118
|
+
* killed, which on Windows is the only other option.
|
|
119
|
+
*/
|
|
120
|
+
onStopRequested?: () => void;
|
|
93
121
|
}
|
|
94
122
|
export interface ControlServerAddress {
|
|
95
123
|
url: string;
|
package/dist/control/server.js
CHANGED
|
@@ -38,6 +38,7 @@ import { calculateCostUsd } from "../replay/pricing.js";
|
|
|
38
38
|
import { logDetail, logLine, errText } from "../util/log.js";
|
|
39
39
|
import { journal } from "../util/journal.js";
|
|
40
40
|
import { packageVersion } from "../util/version.js";
|
|
41
|
+
import { resolveAuthUrl } from "../auth/client.js";
|
|
41
42
|
/** How long a `/tool/post` waits for the proxy's own report before recording its own view. */
|
|
42
43
|
const PROXY_REPORT_GRACE_MS = 1_500;
|
|
43
44
|
/** One MCP call, as seen from up to two sides. */
|
|
@@ -127,7 +128,7 @@ export class ControlServer {
|
|
|
127
128
|
resolve(this.address);
|
|
128
129
|
});
|
|
129
130
|
};
|
|
130
|
-
bind(preferred, preferred !== 0);
|
|
131
|
+
bind(preferred, preferred !== 0 && this.opts.portFallback !== false);
|
|
131
132
|
});
|
|
132
133
|
}
|
|
133
134
|
async close() {
|
|
@@ -150,6 +151,10 @@ export class ControlServer {
|
|
|
150
151
|
const sessions = [...this.sessions.values()].map((s) => ({
|
|
151
152
|
sessionId: s.sessionId,
|
|
152
153
|
runId: s.run?.runId,
|
|
154
|
+
// Mid-run right now. What 'ensureDaemon' reads before it dares to restart
|
|
155
|
+
// an out-of-date recorder: a restart between turns costs nothing, one
|
|
156
|
+
// during a turn loses the turn.
|
|
157
|
+
active: Boolean(s.run) && !s.run.finished,
|
|
153
158
|
steps: s.run?.ordering.next ?? 0,
|
|
154
159
|
recording: s.run?.recording ?? false,
|
|
155
160
|
replay: s.run?.replay
|
|
@@ -187,12 +192,13 @@ export class ControlServer {
|
|
|
187
192
|
tier: "bound",
|
|
188
193
|
// The authoritative answer to "is anything actually being saved?".
|
|
189
194
|
recording: this.opts.recording ?? !(this.recorder instanceof NullRecorder),
|
|
195
|
+
account: this.opts.account ?? null,
|
|
190
196
|
// Whether a handed-out segment may actually run mid-task, or whether the
|
|
191
197
|
// runner is only watching (R-OUT-10, R-LIFE-8). Observe-only must never
|
|
192
198
|
// be invisible: an operator has to be able to see which of the two this
|
|
193
199
|
// machine is doing without reading a log file.
|
|
194
200
|
segmentArm: this.replay.segmentArm,
|
|
195
|
-
authUrl:
|
|
201
|
+
authUrl: resolveAuthUrl() || null,
|
|
196
202
|
sessionId: this.sessionId,
|
|
197
203
|
pid: process.pid,
|
|
198
204
|
cwd: this.opts.cwd,
|
|
@@ -210,6 +216,7 @@ export class ControlServer {
|
|
|
210
216
|
// (docs/calculatedReplay.md §13.2, mitigation 3).
|
|
211
217
|
replay: {
|
|
212
218
|
enabled: this.replay.enabled,
|
|
219
|
+
source: this.opts.replaySource ?? null,
|
|
213
220
|
minSimilarity: this.opts.replay?.minSimilarity ?? null,
|
|
214
221
|
allowServers: this.opts.replay?.allowServers ? [...this.opts.replay.allowServers] : null,
|
|
215
222
|
deriveKey: Boolean(this.opts.replay?.apiKey ?? process.env.ANTHROPIC_API_KEY),
|
|
@@ -248,6 +255,12 @@ export class ControlServer {
|
|
|
248
255
|
}
|
|
249
256
|
const body = await this.readJson(req);
|
|
250
257
|
switch (route) {
|
|
258
|
+
case "/control/stop":
|
|
259
|
+
// Answer first, then leave: the caller is polling for the pid to go.
|
|
260
|
+
this.send(res, 200, { ok: Boolean(this.opts.onStopRequested) });
|
|
261
|
+
if (this.opts.onStopRequested)
|
|
262
|
+
setImmediate(() => this.opts.onStopRequested?.());
|
|
263
|
+
return;
|
|
251
264
|
case "/session/start":
|
|
252
265
|
this.send(res, 200, this.onSessionStart(body));
|
|
253
266
|
return;
|
package/dist/proxy/session.d.ts
CHANGED
|
@@ -25,7 +25,14 @@ import type { UpstreamClient } from "../upstream/client.js";
|
|
|
25
25
|
import type { ProxyStepReport } from "../control/correlation.js";
|
|
26
26
|
import { type Recorder } from "../record/recorder.js";
|
|
27
27
|
/** How long a proxy waits for a control server before falling to Tier 2. */
|
|
28
|
-
|
|
28
|
+
/**
|
|
29
|
+
* Ten seconds, not five: the recorder is now started by the SessionStart hook,
|
|
30
|
+
* which fires while the host is also spawning this proxy, and a cold start
|
|
31
|
+
* (sign-in refresh, the service's /health probe, then listen) has been
|
|
32
|
+
* measured at one to four seconds. Steps are buffered meanwhile, never
|
|
33
|
+
* dropped, and a host session is never waiting on this.
|
|
34
|
+
*/
|
|
35
|
+
export declare const DISCOVERY_WINDOW_MS = 10000;
|
|
29
36
|
export type Tier = "bound" | "standalone" | "pending";
|
|
30
37
|
export interface ProxySessionOptions {
|
|
31
38
|
serverName: string;
|
package/dist/proxy/session.js
CHANGED
|
@@ -26,7 +26,7 @@ import { hostname } from "node:os";
|
|
|
26
26
|
import { ControlClient } from "../control/client.js";
|
|
27
27
|
import { resolveControl } from "../control/discovery.js";
|
|
28
28
|
import { qualifyToolName } from "../control/correlation.js";
|
|
29
|
-
import { authenticate } from "../auth/client.js";
|
|
29
|
+
import { authenticate, resolveAuthUrl } from "../auth/client.js";
|
|
30
30
|
import { NullRecorder } from "../record/recorder.js";
|
|
31
31
|
import { RemoteRecorder } from "../record/remote-recorder.js";
|
|
32
32
|
import { StepQueue } from "../record/queue.js";
|
|
@@ -36,7 +36,14 @@ import { redact } from "../record/redact.js";
|
|
|
36
36
|
import { logLine, logDetail, errText } from "../util/log.js";
|
|
37
37
|
import { packageVersion } from "../util/version.js";
|
|
38
38
|
/** How long a proxy waits for a control server before falling to Tier 2. */
|
|
39
|
-
|
|
39
|
+
/**
|
|
40
|
+
* Ten seconds, not five: the recorder is now started by the SessionStart hook,
|
|
41
|
+
* which fires while the host is also spawning this proxy, and a cold start
|
|
42
|
+
* (sign-in refresh, the service's /health probe, then listen) has been
|
|
43
|
+
* measured at one to four seconds. Steps are buffered meanwhile, never
|
|
44
|
+
* dropped, and a host session is never waiting on this.
|
|
45
|
+
*/
|
|
46
|
+
export const DISCOVERY_WINDOW_MS = 10_000;
|
|
40
47
|
export class ProxySession {
|
|
41
48
|
serverName;
|
|
42
49
|
/** This proxy process, as the control server tells proxies apart (replay/executor.ts). */
|
|
@@ -98,6 +105,7 @@ export class ProxySession {
|
|
|
98
105
|
return;
|
|
99
106
|
}
|
|
100
107
|
const window = this.opts.discoveryWindowMs ?? DISCOVERY_WINDOW_MS;
|
|
108
|
+
const deadline = Date.now() + window;
|
|
101
109
|
const found = await resolveControl(this.opts.cwd, window);
|
|
102
110
|
if (!found?.url) {
|
|
103
111
|
await this.becomeStandalone(`no control server within ${window}ms`);
|
|
@@ -108,12 +116,24 @@ export class ProxySession {
|
|
|
108
116
|
startedAt: this.startedAt,
|
|
109
117
|
pid: process.pid,
|
|
110
118
|
});
|
|
111
|
-
const
|
|
119
|
+
const info = {
|
|
112
120
|
serverName: this.serverName,
|
|
113
121
|
pid: process.pid,
|
|
114
122
|
cwd: this.opts.cwd,
|
|
115
123
|
version: packageVersion(),
|
|
116
|
-
}
|
|
124
|
+
};
|
|
125
|
+
// A control server that is still coming up — the SessionStart hook is
|
|
126
|
+
// starting it while the host spawns us — answers nothing for a moment.
|
|
127
|
+
// Keep trying inside the same window rather than settle for Tier 2 on the
|
|
128
|
+
// first refusal.
|
|
129
|
+
let registered = await client.register(info);
|
|
130
|
+
while (!registered && Date.now() < deadline) {
|
|
131
|
+
await new Promise((resolve) => {
|
|
132
|
+
const t = setTimeout(resolve, 250);
|
|
133
|
+
t.unref?.();
|
|
134
|
+
});
|
|
135
|
+
registered = await client.register(info);
|
|
136
|
+
}
|
|
117
137
|
if (!registered) {
|
|
118
138
|
await this.becomeStandalone("control server did not answer /proxy/register");
|
|
119
139
|
return;
|
|
@@ -216,9 +236,11 @@ export class ProxySession {
|
|
|
216
236
|
if (this.opts.recorderFactory) {
|
|
217
237
|
return (await this.opts.recorderFactory()) ?? new NullRecorder();
|
|
218
238
|
}
|
|
219
|
-
|
|
239
|
+
// BIR_AUTH_URL, or the address 'bir setup' stored — the same lookup the
|
|
240
|
+
// control server and the CLI make, so no two processes disagree about it.
|
|
241
|
+
const baseUrl = resolveAuthUrl();
|
|
220
242
|
if (!baseUrl) {
|
|
221
|
-
logLine("recorder.disabled", { why: "
|
|
243
|
+
logLine("recorder.disabled", { why: "no service address — run 'bir setup', or set BIR_AUTH_URL" });
|
|
222
244
|
return new NullRecorder();
|
|
223
245
|
}
|
|
224
246
|
// A proxy runs inside the host's process tree with its stdio bound to the
|
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
> one says *what to type*, in order, from an empty machine to a prompt that answers itself.
|
|
5
5
|
>
|
|
6
6
|
> **Read this first.** Replay executes tool calls with arguments a scenario computed, and it does so
|
|
7
|
-
> **without the permission prompts you would normally see** (design §13.2).
|
|
8
|
-
>
|
|
7
|
+
> **without the permission prompts you would normally see** (design §13.2). `bir setup` (§1) turns it
|
|
8
|
+
> on. Read §5 before you leave it on; `bir replay off` turns it off, and §2–§4 work either way.
|
|
9
9
|
|
|
10
10
|
---
|
|
11
11
|
|
|
@@ -29,23 +29,40 @@ end-to-end smoke test.
|
|
|
29
29
|
|
|
30
30
|
| # | You need | Check it with |
|
|
31
31
|
|---|---|---|
|
|
32
|
-
| 1 | A BaseIn service you can reach | `curl -fsS "
|
|
32
|
+
| 1 | A BaseIn service you can reach | `bir status` — its `BaseIn :` line is the address `bir setup` stored; `curl -fsS "<that address>/health"` proves it answers |
|
|
33
33
|
| 2 | Anthropic configured **on the server** — calculation and derivation both need it | a `503 anthropic_not_configured` from `/calculate` means it is not |
|
|
34
34
|
| 3 | `SIMILARITY_DETECTION_ENABLED=true` on the server (default) | otherwise no prompt ever matches |
|
|
35
35
|
| 4 | BaseInstRunnerMCP built and installed in your project | `bir status` |
|
|
36
|
-
| 5 | Tier 1 — the hooks wired and
|
|
36
|
+
| 5 | Tier 1 — the hooks wired and the recorder running | `bir doctor`; the SessionStart hook starts it |
|
|
37
37
|
| 6 | *(replay only)* signed in, so the service can read what each request acts on | `bir doctor` — `derive=the service`; an `ANTHROPIC_API_KEY` here replaces it, see §5.3 |
|
|
38
38
|
|
|
39
39
|
Tier 1 is not optional for replay. A match is a match on **the prompt**, and a standalone proxy never
|
|
40
|
-
sees one (design §1.1). If `bir doctor` says `tier: standalone`, replay cannot
|
|
41
|
-
honest ceiling rather than a bug.
|
|
40
|
+
sees one (design §1.1). If `bir doctor` says `Recording tier : standalone (Tier 2)`, replay cannot
|
|
41
|
+
arm, and that is the honest ceiling rather than a bug.
|
|
42
|
+
|
|
43
|
+
One line, pasted into a terminal opened in the project you start Claude Code in. The console's *Set
|
|
44
|
+
up the runner* page shows it with a one-time setup token filled in. Pasted in the home folder or a
|
|
45
|
+
drive root it stops before downloading anything — `cd` to the project and paste the same line again:
|
|
46
|
+
|
|
47
|
+
```powershell
|
|
48
|
+
$env:BIR_SETUP_TOKEN="<token>"; irm https://api.bi2202.com/install.ps1 | iex # Windows (PowerShell)
|
|
49
|
+
```
|
|
42
50
|
|
|
43
51
|
```bash
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
52
|
+
curl -fsSL https://api.bi2202.com/install.sh | BIR_SETUP_TOKEN="<token>" sh # macOS / Linux
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
From cmd.exe: `powershell -NoProfile -Command "$env:BIR_SETUP_TOKEN='<token>'; irm https://api.bi2202.com/install.ps1 | iex"`.
|
|
56
|
+
|
|
57
|
+
It installs Node and Claude Code if they are missing, installs the package, and runs `bir setup`:
|
|
58
|
+
sign-in, the service address stored in `~/.baseinstrunner/config.json`, the project's MCP servers
|
|
59
|
+
wrapped, the hooks wired into `.claude/settings.local.json`, the scenario server added, the recorder
|
|
60
|
+
started in the background. Nothing stays open: the SessionStart hook starts the recorder whenever
|
|
61
|
+
it is missing, and its audit lines go to `~/.baseinstrunner/logs/<key>.log`. Re-running the line
|
|
62
|
+
upgrades. With the package already installed, the same thing is:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
bir setup --auth-url https://api.bi2202.com # without a token it signs in through the browser
|
|
49
66
|
```
|
|
50
67
|
|
|
51
68
|
---
|
|
@@ -81,12 +98,17 @@ bir scenario list # or: GET /recordings/runs
|
|
|
81
98
|
```
|
|
82
99
|
|
|
83
100
|
```
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
run_a812…
|
|
87
|
-
|
|
101
|
+
run_5f3a… Fleet risk sweep
|
|
102
|
+
iterations=3 steps=14 scenario=scn_9c1b… (ready)
|
|
103
|
+
run_a812… Rewrite the billing README
|
|
104
|
+
iterations=1 steps=9 scenario=none — run `bir scenario calc <runId>`
|
|
105
|
+
run_c004… Check open PRs for stale reviews
|
|
106
|
+
iterations=1 steps=6 scenario=scn_44d0… (calculating)
|
|
88
107
|
```
|
|
89
108
|
|
|
109
|
+
Ids are printed in full on their own line (shortened here), so they can be copied straight into
|
|
110
|
+
`calc` and `replay`.
|
|
111
|
+
|
|
90
112
|
---
|
|
91
113
|
|
|
92
114
|
## 3. Calculate the scenario
|
|
@@ -100,17 +122,20 @@ bir scenario calc run_5f3a… # POST /recordings/runs/:runId/calculat
|
|
|
100
122
|
bir scenario show run_5f3a… # GET /recordings/runs/:runId/scenario (poll)
|
|
101
123
|
```
|
|
102
124
|
|
|
103
|
-
Raw, if you prefer — the
|
|
125
|
+
Raw, if you prefer — the service address lives in `~/.baseinstrunner/config.json` (`BIR_AUTH_URL`
|
|
126
|
+
is normally unset after `bir setup`) and the token in `~/.baseinstrunner/credentials.json`:
|
|
104
127
|
|
|
105
128
|
```bash
|
|
129
|
+
BASEIN=$(node -p "require(require('os').homedir()+'/.baseinstrunner/config.json').authUrl")
|
|
106
130
|
TOKEN=$(node -p "require(require('os').homedir()+'/.baseinstrunner/credentials.json').accessToken")
|
|
107
|
-
curl -fsS -X POST "$
|
|
131
|
+
curl -fsS -X POST "$BASEIN/recordings/runs/run_5f3a…/calculate" \
|
|
108
132
|
-H "authorization: Bearer $TOKEN" -H 'content-type: application/json' -d '{}'
|
|
109
133
|
```
|
|
110
134
|
|
|
111
135
|
```powershell
|
|
136
|
+
$b = (Get-Content "$HOME\.baseinstrunner\config.json" | ConvertFrom-Json).authUrl
|
|
112
137
|
$t = (Get-Content "$HOME\.baseinstrunner\credentials.json" | ConvertFrom-Json).accessToken
|
|
113
|
-
Invoke-RestMethod -Method Post -Uri "$
|
|
138
|
+
Invoke-RestMethod -Method Post -Uri "$b/recordings/runs/run_5f3a…/calculate" `
|
|
114
139
|
-Headers @{ authorization = "Bearer $t" } -ContentType 'application/json' -Body '{}'
|
|
115
140
|
```
|
|
116
141
|
|
|
@@ -139,7 +164,7 @@ bir scenario replay scn_9c1b… --prompt "which devices in the fleet look risky
|
|
|
139
164
|
params { "fleetId": "eu-west-1", "riskThreshold": 0.8 }
|
|
140
165
|
step 0 mcp__chrome-devtools__navigate_page { "url": "https://fleet.internal/eu-west-1" }
|
|
141
166
|
step 1 mcp__chrome-devtools__take_snapshot {}
|
|
142
|
-
→ emitted: devices
|
|
167
|
+
→ emitted: devices
|
|
143
168
|
step 2 mcp__fleet-api__device_detail { "id": "dev_88f1" }
|
|
144
169
|
→ emitted: highestRiskDeviceId, riskScore
|
|
145
170
|
response { "fleet": "eu-west-1", "atRisk": 3, "worst": "dev_88f1" }
|
|
@@ -151,8 +176,13 @@ Read it for three things:
|
|
|
151
176
|
1. **Did the parameters come out right** for a prompt phrased differently from the original?
|
|
152
177
|
2. **Do the step inputs look computed**, or are they the original run's constants copied through? A
|
|
153
178
|
scenario whose inputs never vary with the parameters is a recording, not a scenario.
|
|
154
|
-
3. **Is `emitted` non-empty** for steps that later steps depend on?
|
|
155
|
-
|
|
179
|
+
3. **Is `emitted` non-empty** for steps that later steps depend on? `emitted` lists key names only;
|
|
180
|
+
`→ emitted: (nothing — toolOutputLogic derived no keys)` means every downstream input reading
|
|
181
|
+
that step will be wrong.
|
|
182
|
+
|
|
183
|
+
A `↑ not named by the prompt:` line under `params` lists targets the dry run filled from the
|
|
184
|
+
recording. A live turn finds those in an earlier step, or does not run — so a green trace with
|
|
185
|
+
that line is not yet evidence that a session would steer.
|
|
156
186
|
|
|
157
187
|
A dry replay is recorded as `outcome: 'dry'` and is excluded from savings by construction — it costs
|
|
158
188
|
you a Haiku call and nothing else.
|
|
@@ -175,7 +205,7 @@ you: can you check the eu-west fleet and tell me which boxes are in trouble?
|
|
|
175
205
|
That line is the whole feature working up to the point of replay. If it does not appear, the prompt
|
|
176
206
|
did not clear the server's `SIMILARITY_THRESHOLD` (default 0.9) — see §8.
|
|
177
207
|
|
|
178
|
-
**With replay off (
|
|
208
|
+
**With replay off (`bir replay off`), this is where it stops.** The server keeps its canonical run, bumps
|
|
179
209
|
its `iterations`, this turn is not recorded, and the model answers the ordinary way. Nothing is lost
|
|
180
210
|
and nothing is skipped.
|
|
181
211
|
|
|
@@ -198,24 +228,33 @@ When replay arms:
|
|
|
198
228
|
That is what makes replay fast, and it is the whole risk. Two things follow:
|
|
199
229
|
|
|
200
230
|
1. Only enable it in a project whose scenarios you have dry-replayed and read (§3.1).
|
|
201
|
-
2.
|
|
231
|
+
2. Run `bir replay allow <servers>` (§5.3). An allowlist of the servers you are comfortable having
|
|
202
232
|
called unattended is cheap insurance, and it is the difference between "replay reads a dashboard"
|
|
203
233
|
and "replay can do anything any wrapped server can do".
|
|
204
234
|
|
|
205
235
|
### 5.2 Enable it
|
|
206
236
|
|
|
237
|
+
`bir setup` already did this: it installed the scenario server and pre-approved it, so Claude Code
|
|
238
|
+
asks no question about it. By hand:
|
|
239
|
+
|
|
207
240
|
```bash
|
|
208
|
-
bir install --replay # adds the `bir` MCP server
|
|
209
|
-
|
|
210
|
-
bir-hooks # restart it
|
|
241
|
+
bir install --replay # adds the `bir` MCP server (pre-approved)
|
|
242
|
+
bir up --restart # the recorder reads the change when it restarts
|
|
211
243
|
```
|
|
212
244
|
|
|
213
|
-
`bir install
|
|
214
|
-
|
|
245
|
+
A flag-less `bir install` keeps the invocation the project was set up with — `bir setup` installs
|
|
246
|
+
with `--global` from an installed package and `--local` from a checkout — so this downgrades
|
|
247
|
+
nothing. `bir install --replay` does exactly two things beyond a normal install, and
|
|
248
|
+
`bir uninstall --replay` takes the scenario server out again and leaves everything else as it is:
|
|
249
|
+
|
|
250
|
+
- registers a first-party MCP server under the key **`bir`** in the project's `.mcp.json` (never
|
|
251
|
+
`~/.claude.json`), exposing one tool, `mcp__bir__run_scenario` — the channel a fully-wrapped
|
|
252
|
+
scenario's results come back through;
|
|
253
|
+
- pre-approves that server in `.claude/settings.local.json` (`enabledMcpjsonServers`), because a
|
|
254
|
+
"No" to Claude Code's approval dialog leaves a direct replay with nowhere to deliver its results.
|
|
215
255
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
- raises `UserPromptSubmit`'s hook timeout from 5 s to 15 s, so the match round trip fits.
|
|
256
|
+
The `UserPromptSubmit` hook's 15 s timeout, which the match round trip needs, is not one of them:
|
|
257
|
+
every install writes it, replay or not.
|
|
219
258
|
|
|
220
259
|
The `bir` tool is visible to the model in every session in this project. That is a real cost — one
|
|
221
260
|
more tool in the list, a little context per turn — and it is why replay is a separate flag rather
|
|
@@ -228,23 +267,48 @@ bir doctor
|
|
|
228
267
|
```
|
|
229
268
|
|
|
230
269
|
```
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
270
|
+
Replay : ON servers=chrome-devtools,fleet-api minSimilarity=0.92 source=sidecar
|
|
271
|
+
derive=the service (no key needed here)
|
|
272
|
+
Wrapped in config : chrome-devtools, fleet-api
|
|
273
|
+
Registered proxies: chrome-devtools, fleet-api
|
|
274
|
+
Control server : http://127.0.0.1:53411
|
|
275
|
+
BaseIn (shell) : https://api.bi2202.com (answers /health)
|
|
276
|
+
Recording tier : bound (Tier 1)
|
|
277
|
+
Recording : yes
|
|
278
|
+
|
|
279
|
+
· recording to https://api.bi2202.com as you@example.com
|
|
280
|
+
· the recorder runs in the background (pid 41820); its audit log is ~/.baseinstrunner/logs/<key>.log
|
|
281
|
+
· recurring sub-tasks are observed only — nothing is replaced (BIR_SEGMENT_ARM unset)
|
|
282
|
+
|
|
283
|
+
No problems found.
|
|
236
284
|
```
|
|
237
285
|
|
|
286
|
+
`BaseIn (shell)` is the address this terminal's `bir` commands would use — `BIR_AUTH_URL` if it
|
|
287
|
+
is set, else the one `bir setup` stored — and `recording to` is the one the running recorder
|
|
288
|
+
uses, with the account it is signed in as. The two normally agree.
|
|
289
|
+
|
|
238
290
|
### 5.3 Environment
|
|
239
291
|
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
292
|
+
The first three switches are stored per project, and that is where to set them:
|
|
293
|
+
|
|
294
|
+
```bash
|
|
295
|
+
bir replay allow chrome-devtools,fleet-api # the servers direct execution may call; `allow all` clears it
|
|
296
|
+
bir replay off # and back with `bir replay on`
|
|
297
|
+
bir replay status # what is stored for this project
|
|
298
|
+
bir up --restart # a running recorder reads them when it restarts
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
They are kept in `~/.baseinstrunner/installed.json`. A recorder started by the SessionStart hook has
|
|
302
|
+
Claude Code's environment, not your terminal's, and reads them from there. The environment variables
|
|
303
|
+
below override them wherever the recorder sees them; `bir doctor` says which is in effect
|
|
304
|
+
(`source=sidecar` or `source=env`). The proxies need no replay configuration of their own — each one
|
|
305
|
+
is told whether to open its work loop when it registers, so there is exactly one switch and it cannot
|
|
306
|
+
get out of step with itself.
|
|
243
307
|
|
|
244
308
|
| Variable | Default | What it does |
|
|
245
309
|
|---|---|---|
|
|
246
310
|
| `BIR_REPLAY` | *(on)* | `0` disables replay. Unset or any other value keeps it on; nothing below matters while it is `0` |
|
|
247
|
-
| `BIR_REPLAY_ALLOW_SERVERS` | *(all wrapped)* | Comma-separated server keys eligible for **direct** execution.
|
|
311
|
+
| `BIR_REPLAY_ALLOW_SERVERS` | *(the stored list, else all wrapped)* | Comma-separated server keys eligible for **direct** execution. Overrides `bir replay allow` |
|
|
248
312
|
| `BIR_MIN_STEER_SIMILARITY` | `0.92` | Below this a match is detected but not replayed (§8) |
|
|
249
313
|
| `ANTHROPIC_API_KEY` | *(unset)* | **Not required.** Derivation — reading what this request acts on — is done by the service on its key for a signed-in runner. Set this to keep the reading on this machine instead: the prompt then never leaves it, and it is one round trip faster. Signed out *and* unset, only a scenario with nothing to work out replays |
|
|
250
314
|
| `BIR_DERIVE_MODEL` | `claude-haiku-4-5-20251001` | The derivation model, when this machine does the reading |
|
|
@@ -254,10 +318,14 @@ there is exactly one switch and it cannot get out of step with itself.
|
|
|
254
318
|
| `BIR_STEP_TIMEOUT_MS` | `60000` | Per-step ceiling for one direct `tools/call` |
|
|
255
319
|
| `BIR_VERBOSE` | *(unset)* | `1` adds per-step `replay.*` detail lines |
|
|
256
320
|
|
|
257
|
-
**No API key is a supported configuration, not a broken one.**
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
321
|
+
**No API key is a supported configuration, not a broken one.** Signed in, the service reads the
|
|
322
|
+
prompt on its own key and the replay is the same replay. Signed out *and* without a key, nothing
|
|
323
|
+
can read the prompt: a *setting* takes the value it had in the recorded run, so a scenario whose
|
|
324
|
+
parameters are all settings — a dashboard sweep, a fixed report — is still a complete, free replay;
|
|
325
|
+
but a scenario with a *target* (the thing the task acts on: "check *eu-west*") is declined rather
|
|
326
|
+
than run on last week's value, and the agent does the task normally. `bir doctor` names the state
|
|
327
|
+
you are in: `derive=this machine (ANTHROPIC_API_KEY)`, `derive=the service (no key needed here)`,
|
|
328
|
+
or `derive=recorded sample values — scenarios with a target will NOT run`.
|
|
261
329
|
|
|
262
330
|
---
|
|
263
331
|
|
|
@@ -271,12 +339,12 @@ Ask the matching question again, with replay on.
|
|
|
271
339
|
[bir] … run.matched run=run_… matchedRun=run_5f3a… similarity=0.94
|
|
272
340
|
[bir] … plan.armed scenario=scn_9c1b… mode=direct steps=3
|
|
273
341
|
tools="mcp__chrome-devtools__navigate_page,mcp__chrome-devtools__take_snapshot,mcp__fleet-api__device_detail"
|
|
274
|
-
[bir] … replay.derived params=2 costUsd=0.0011
|
|
342
|
+
[bir] … replay.derived scenario=scn_9c1b… params=2 costUsd=0.0011 source=the service
|
|
275
343
|
[bir] … replay.step n=0 tool=navigate_page server=chrome-devtools ms=612 ok=true
|
|
276
344
|
[bir] … replay.step n=1 tool=take_snapshot server=chrome-devtools ms=1104 ok=true emitted=devices
|
|
277
345
|
[bir] … replay.step n=2 tool=device_detail server=fleet-api ms=210 ok=true emitted=highestRiskDeviceId,riskScore
|
|
278
346
|
[bir] … replay.done scenario=scn_9c1b… mode=direct steps=3/3 outcome=steered_full ms=2766
|
|
279
|
-
[bir] … execution.reported scenario=scn_9c1b… outcome=steered_full derive=0.0011 session=0.0083 fallback=0 savedUsd=0.114
|
|
347
|
+
[bir] … execution.reported scenario=scn_9c1b… outcome=steered_full derive=0.0011 session=0.0083 fallback=0.0000 savedUsd=0.114
|
|
280
348
|
```
|
|
281
349
|
|
|
282
350
|
In the session you will see one tool call, `mcp__bir__run_scenario`, returning the composed results.
|
|
@@ -284,31 +352,35 @@ The model reads them and answers. It never emitted the three underlying calls.
|
|
|
284
352
|
|
|
285
353
|
### 6.2 A mixed scenario — steer
|
|
286
354
|
|
|
287
|
-
The model emits each call itself; `bir` overrides the arguments.
|
|
355
|
+
The model emits each call itself; `bir` overrides the arguments. The per-step `replay.pin` and
|
|
356
|
+
`replay.thread` lines are detail lines, printed only under `BIR_VERBOSE=1`; without it a steer run
|
|
357
|
+
shows `plan.armed`, `replay.done`, and any `replay.thread_fallback` between them.
|
|
288
358
|
|
|
289
359
|
```
|
|
290
360
|
[bir] … plan.armed scenario=scn_44d0… mode=steer steps=4
|
|
291
361
|
[bir] … replay.pin n=0 tool=mcp__github__list_pull_requests
|
|
292
|
-
[bir] … replay.thread n=0 via=proxy emitted=prNumbers
|
|
362
|
+
[bir] … replay.thread n=0 via=proxy emitted=prNumbers done=false
|
|
293
363
|
[bir] … replay.pin n=1 tool=Read
|
|
294
|
-
[bir] … replay.thread n=1 via=hook emitted=reviewers
|
|
364
|
+
[bir] … replay.thread n=1 via=hook emitted=reviewers done=false
|
|
295
365
|
[bir] … replay.done scenario=scn_44d0… mode=steer steps=4/4 outcome=steered_full
|
|
296
366
|
```
|
|
297
367
|
|
|
298
368
|
`via=proxy` versus `via=hook` matters: a wrapped MCP step's output is threaded from the **proxy's**
|
|
299
369
|
report, because that is the shape the scenario's logic was written against (design §7.2). A
|
|
300
|
-
`
|
|
301
|
-
|
|
370
|
+
`replay.thread_fallback run=… tool=… why="no proxy report — threading the hook's view; shapes may differ"`
|
|
371
|
+
line — printed whether or not you are verbose — means the proxy did not report within its grace
|
|
372
|
+
window and the hook's differently-shaped view was used instead. Expect derived values to be empty,
|
|
373
|
+
and treat repeats as a bug.
|
|
302
374
|
|
|
303
375
|
### 6.3 The model goes off-script
|
|
304
376
|
|
|
305
377
|
```
|
|
306
378
|
[bir] … replay.diverge scenario=scn_44d0… expected=Read called=Bash step=2/4
|
|
307
|
-
[bir] … replay.compose remaining=2
|
|
308
|
-
[bir] … replay.done
|
|
379
|
+
[bir] … replay.compose scenario=scn_44d0… remaining=2 executed=1 recorded=1 skipped=0 bytes=8412 costUsd=0.00
|
|
380
|
+
[bir] … replay.done scenario=scn_44d0… mode=steer steps=2/4 outcome=diverged
|
|
309
381
|
```
|
|
310
382
|
|
|
311
|
-
`costUsd=0` on the compose line is the point of the whole design: the remaining steps ran through
|
|
383
|
+
`costUsd=0.00` on the compose line is the point of the whole design: the remaining steps ran through
|
|
312
384
|
proxies that were already connected, so recovering from a divergence cost no tokens at all.
|
|
313
385
|
|
|
314
386
|
### 6.4 The audit vocabulary
|
|
@@ -321,13 +393,16 @@ Everything replay decides is one line. `grep` for these:
|
|
|
321
393
|
| `replay.decision … verdict=no-steer` | Matched, and declined. `why=` says which gate |
|
|
322
394
|
| `plan.armed` | A plan exists. Carries mode, step count and the ordered tool names — **before** anything runs |
|
|
323
395
|
| `replay.derived` / `replay.derive_failed` | Parameters resolved, or fell back to sample values |
|
|
324
|
-
| `replay.pin` / `replay.thread` | One steered step's input pinned / output threaded |
|
|
396
|
+
| `replay.pin` / `replay.thread` | One steered step's input pinned / output threaded — detail lines, only under `BIR_VERBOSE=1` |
|
|
397
|
+
| `replay.thread_fallback` | A wrapped step's output was threaded from the hook's view because the proxy did not report in time |
|
|
325
398
|
| `replay.step` | One direct step executed |
|
|
326
399
|
| `replay.diverge` / `replay.compose` | Off-script, and the recovery |
|
|
327
400
|
| `replay.done` | Final outcome, steps completed, elapsed |
|
|
328
401
|
| `execution.reported` | The ticket was redeemed and the saving booked |
|
|
329
402
|
|
|
330
|
-
|
|
403
|
+
A recorder started in the background — by the SessionStart hook, `bir up` or `bir setup` — keeps
|
|
404
|
+
them in `~/.baseinstrunner/logs/<key>.log`; `bir status` names the file. One started in a terminal
|
|
405
|
+
writes them to that terminal; keep them yourself: `bir-hooks 2>&1 | tee -a ~/.baseinstrunner/audit.log`.
|
|
331
406
|
|
|
332
407
|
---
|
|
333
408
|
|
|
@@ -381,19 +456,22 @@ first prompt that states the task plainly.
|
|
|
381
456
|
| Symptom | Cause | Fix |
|
|
382
457
|
|---|---|---|
|
|
383
458
|
| No `run.matched`, ever | Prompt below the server's `SIMILARITY_THRESHOLD` | Rephrase closer, or lower it server-side. Confirm the original run is in the list — a sub-`RECORDING_MIN_ACTIONS` run is never embedded |
|
|
384
|
-
| `run.matched` but no `plan.armed` | A gate declined | Read the `replay.decision` line's `why=`. Ranked by frequency: `BIR_REPLAY=0` · scenario not `ready` · similarity below threshold · no step is executable |
|
|
459
|
+
| `run.matched` but no `plan.armed` | A gate declined | Read the `replay.decision` line's `why=`. Ranked by frequency: `bir replay off` or `BIR_REPLAY=0` · scenario not `ready` · similarity below threshold · no step is executable |
|
|
385
460
|
| `why="no step is executable"` | None of the scenario's tools is a wrapped MCP server and none is a built-in reachable in this session | Wrap the servers the scenario uses (`bir install --server …`) and recalculate |
|
|
386
461
|
| `plan.armed mode=steer` where you expected `direct` | At least one step is a built-in, an unwrapped MCP server, or `claude-in-chrome` | `bir status` shows what is wrapped. `claude-in-chrome` is `scope: "dynamic"` and can never be wrapped |
|
|
387
462
|
| Model ignores the directive and diverges every turn | Steering is advisory — the model chooses; `bir` only pins arguments | Expected occasionally. Persistent divergence usually means the scenario's tools do not fit the live task; check `intent` |
|
|
388
|
-
| `replay.thread
|
|
389
|
-
| `
|
|
463
|
+
| A `replay.thread` line with no `emitted=` (under `BIR_VERBOSE=1`) | `toolOutputLogic` derived nothing — usually the output shape changed since recording | Dry-replay (§3.1). If the dry run emits and the live run does not, the tool's output shape has drifted; recalculate |
|
|
464
|
+
| `replay.thread_fallback` on a wrapped step | The proxy did not report within the grace window | Check that proxy is alive (`bir doctor`). Repeats mean a slow or dying upstream |
|
|
390
465
|
| `run_scenario` returns "no scenario is armed" | The model called it on a turn with no plan | Harmless. It happens when the model remembers the tool from an earlier turn |
|
|
391
|
-
| `
|
|
466
|
+
| `recorder.send_failed … error="HTTP 409 … invalid_ticket"` | The execution report's ticket was already redeemed, or belongs to another scenario | Harmless if the body says `duplicate: true`. Otherwise a rollover raced `SessionEnd`; the server books once |
|
|
392
467
|
| `savedUsd` looks impossible | Pricing drift between the runner's table and the server's | Compare `PRICING_VERSION` on both sides. Design §11.3 documents a known live drift between two existing copies |
|
|
393
|
-
| The hand-over note says a step "did not run: its input needs a judgement" | The service marked the step non-deterministic when it calculated the plan: the generated logic could not compute its input from the prompt or from earlier outputs, so the recorded value was a copy (plan-services
|
|
394
|
-
| A parked step ran again after an hour, or a plan was recalculated by itself | The service's grace period (D8): a parked step is served with `stop: null` once an hour for a day and a success clears its count; if it keeps failing, or its tool is gone, the next repeat of the prompt is recorded in full and the plan is calculated again from it | Nothing: that is the repair. `bir investigate`
|
|
468
|
+
| The hand-over note says a step "did not run: its input needs a judgement" | The service marked the step non-deterministic when it calculated the plan: the generated logic could not compute its input from the prompt or from earlier outputs, so the recorded value was a copy (the service's plan-services design, D8 kind 3 — it lives in the BaseIn repository, not this one) | Nothing on the runner. The agent makes that choice and finishes the task; a model step for such judgements is planned (W3.1). The journal's `plan.armed` line lists it under `stops=` |
|
|
469
|
+
| A parked step ran again after an hour, or a plan was recalculated by itself | The service's grace period (D8): a parked step is served with `stop: null` once an hour for a day and a success clears its count; if it keeps failing, or its tool is gone, the next repeat of the prompt is recorded in full and the plan is calculated again from it | Nothing: that is the repair. `bir investigate` says which it is — a step still being retried, a parked step whose repair is due, or a plan being calculated again. To repair now by hand: `bir scenario calc <runId> --force` |
|
|
395
470
|
| `sessionCostUsd` grows every prompt in a session | The per-run usage watermark is missing or not taken | Design §11.4. Reports without a mark must carry `measured: false` |
|
|
396
471
|
| Everything works, nothing is saved | A replayed turn is deliberately not recorded (design §12) | Correct. The matched run stays canonical and its `iterations` is bumped |
|
|
472
|
+
| `bir doctor`: the SessionStart hook points at a Node that is gone | The hook names Node by absolute path. That Node was uninstalled, moved, or was a version manager's per-shell copy; the hook then fails on every session, no recorder is started, and runs record Tier 2 | `bir setup` here again — it rewrites the hook to the Node that runs it |
|
|
473
|
+
| `control.port_busy` in the log; the recorder is not running | Another program holds this project's port, or the system excludes it (on Windows, a range Hyper-V or WSL reserved). A background recorder with a recorded port does not fall back to another, because every hook but SessionStart names this one; only a `bir-hooks` you run by hand in a terminal still does | `bir setup --port <another>` in this directory — the fix the log line itself names. It rewrites the hook URLs and the scenario server's address together and restarts the recorder |
|
|
474
|
+
| The recorder was started by the hook, and `bir doctor` says `servers=(all wrapped)` although you set `BIR_REPLAY_ALLOW_SERVERS` | The hook starts the recorder with Claude Code's environment, not your terminal's; a variable set in one terminal is not seen | `bir replay allow <a,b>` — stored for the project, read by every recorder — then `bir up --restart` |
|
|
397
475
|
|
|
398
476
|
---
|
|
399
477
|
|
|
@@ -401,8 +479,9 @@ first prompt that states the task plainly.
|
|
|
401
479
|
|
|
402
480
|
Every audit line that explains a turn is also kept in a **journal**, one JSON
|
|
403
481
|
object per line, under `~/.baseinstrunner/control/journal/` (one file per
|
|
404
|
-
directory, rotated at 4 MB).
|
|
405
|
-
|
|
482
|
+
directory, rotated at 4 MB). The recorder names it at start — the `journal=`
|
|
483
|
+
field of its `control.listening` line, in the audit log for a background
|
|
484
|
+
recorder. Ask the journal and the service together:
|
|
406
485
|
|
|
407
486
|
```
|
|
408
487
|
bir investigate # the newest turn in this directory
|
|
@@ -446,8 +525,8 @@ tested; everything this guide describes is code you can run.
|
|
|
446
525
|
| **R7** | Usage watermark, reporting, ticket, doctor | [control/transcript.ts](../src/control/transcript.ts), [remote-recorder.ts](../src/record/remote-recorder.ts) |
|
|
447
526
|
|
|
448
527
|
```bash
|
|
449
|
-
npm test
|
|
450
|
-
npm run build &&
|
|
528
|
+
npm test # ~400 tests, incl. test/replay.test.ts and test/replay-server.test.ts
|
|
529
|
+
npm run build && npm run pretest && npm run test:smoke # the smoke test needs the test build too
|
|
451
530
|
```
|
|
452
531
|
|
|
453
532
|
The smoke test is the one worth running after any change to the seams. It starts a stand-in BaseIn
|
|
@@ -473,10 +552,18 @@ Three things the implementation settled that the design left open:
|
|
|
473
552
|
Three levels, least to most.
|
|
474
553
|
|
|
475
554
|
```bash
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
555
|
+
bir replay off # stop arming. Matches still detected; recording still stops on a match.
|
|
556
|
+
# Stored for this project; `bir up --restart` applies it, `bir replay on` undoes it
|
|
557
|
+
export BIR_REPLAY=0 # the same, wherever the recorder is started from (the SessionStart hook
|
|
558
|
+
# gives it Claude Code's environment, not your terminal's)
|
|
559
|
+
bir uninstall --replay # remove the `bir` MCP server only; proxies, hooks and the prompt-hook timeout stay
|
|
560
|
+
bir uninstall # remove everything: proxies, hooks, the pre-approval
|
|
479
561
|
```
|
|
480
562
|
|
|
481
|
-
|
|
563
|
+
`bir uninstall` restores a file byte-for-byte when nothing else edited it since the install, and
|
|
564
|
+
repairs it entry by entry otherwise. A file the install created — usually
|
|
565
|
+
`.claude/settings.local.json` — stays behind with our entries removed, and the line that hid it in
|
|
566
|
+
`.git/info/exclude` goes with them.
|
|
567
|
+
|
|
568
|
+
With replay off the system is exactly v1 again: it recognises a repeated prompt, declines to
|
|
482
569
|
record it a second time, and lets the model do the work.
|