@basein/runner 0.3.18 → 0.3.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/bir-hooks.d.ts +11 -8
- package/dist/bin/bir-hooks.js +11 -8
- package/dist/bin/bir.js +20 -7
- package/dist/bin/investigate.d.ts +9 -0
- package/dist/bin/investigate.js +31 -12
- package/dist/bin/scenario-edit.js +10 -0
- package/dist/control/recorder-main.d.ts +15 -0
- package/dist/control/recorder-main.js +33 -8
- package/dist/control/server.d.ts +91 -11
- package/dist/control/server.js +736 -49
- package/dist/record/recorder.d.ts +47 -4
- package/dist/record/recorder.js +3 -0
- package/dist/record/remote-recorder.d.ts +9 -1
- package/dist/record/remote-recorder.js +30 -0
- package/dist/record/switchable-recorder.js +2 -0
- package/dist/replay/condition-words.d.ts +28 -0
- package/dist/replay/condition-words.js +59 -0
- package/dist/replay/controller.d.ts +45 -6
- package/dist/replay/controller.js +184 -51
- package/dist/replay/derive.d.ts +10 -0
- package/dist/replay/derive.js +3 -0
- package/dist/replay/handover.d.ts +7 -0
- package/dist/replay/handover.js +89 -51
- package/dist/replay/plan.d.ts +11 -4
- package/dist/replay/plan.js +28 -8
- package/dist/replay/reads-only.d.ts +29 -0
- package/dist/replay/reads-only.js +261 -0
- package/dist/replay/types.d.ts +19 -3
- package/docs/calculatedReplay.md +45 -5
- package/docs/calculatedReplayGuide.md +16 -4
- package/package.json +1 -1
- package/production.md +34 -7
package/dist/replay/plan.d.ts
CHANGED
|
@@ -469,6 +469,12 @@ export declare class ScenarioReplayPlan {
|
|
|
469
469
|
stopStep(): SerializedScenarioStep | undefined;
|
|
470
470
|
/** The entry the plan stops in front of, with the frame that owns it. */
|
|
471
471
|
stopEntry(): FlatEntry | undefined;
|
|
472
|
+
/**
|
|
473
|
+
* The flat position of the chain's decision stop (the service's
|
|
474
|
+
* segmentsbreakdown.md SB-18), or -1 when it has none. The steps from there
|
|
475
|
+
* on are the half the recording took, whatever stops the plan first.
|
|
476
|
+
*/
|
|
477
|
+
decisionIndex(): number;
|
|
472
478
|
/** Every entry in order — for the report, and for the owning scenario ids. */
|
|
473
479
|
allEntries(): readonly FlatEntry[];
|
|
474
480
|
/** The entry awaiting execution, or undefined when the plan is done. */
|
|
@@ -539,11 +545,12 @@ export declare class ScenarioReplayPlan {
|
|
|
539
545
|
text: string | null;
|
|
540
546
|
} | undefined;
|
|
541
547
|
/**
|
|
542
|
-
* The tool names of the steps from flat position `from` to the end
|
|
543
|
-
* that runs only under a test marked so — for a
|
|
544
|
-
* planned steps", which must not read as steps
|
|
548
|
+
* The tool names of the steps from flat position `from` to the end (or to
|
|
549
|
+
* `until`, not included), each that runs only under a test marked so — for a
|
|
550
|
+
* hand-over note's "remaining planned steps", which must not read as steps
|
|
551
|
+
* this request needs.
|
|
545
552
|
*/
|
|
546
|
-
remainingFrom(from: number): string[];
|
|
553
|
+
remainingFrom(from: number, until?: number): string[];
|
|
547
554
|
/** The steps the plan's own cursor has left out so far, in order (AC5). */
|
|
548
555
|
leftOut(): readonly LeftOut[];
|
|
549
556
|
/** Whether any planned step runs only under a test. */
|
package/dist/replay/plan.js
CHANGED
|
@@ -700,6 +700,14 @@ export class ScenarioReplayPlan {
|
|
|
700
700
|
stopEntry() {
|
|
701
701
|
return this.stopsEarly() ? this.entries[this.stopAt] : undefined;
|
|
702
702
|
}
|
|
703
|
+
/**
|
|
704
|
+
* The flat position of the chain's decision stop (the service's
|
|
705
|
+
* segmentsbreakdown.md SB-18), or -1 when it has none. The steps from there
|
|
706
|
+
* on are the half the recording took, whatever stops the plan first.
|
|
707
|
+
*/
|
|
708
|
+
decisionIndex() {
|
|
709
|
+
return this.entries.findIndex((e) => e.step.stop?.kind === "decision");
|
|
710
|
+
}
|
|
703
711
|
/** Every entry in order — for the report, and for the owning scenario ids. */
|
|
704
712
|
allEntries() {
|
|
705
713
|
return this.entries;
|
|
@@ -853,12 +861,13 @@ export class ScenarioReplayPlan {
|
|
|
853
861
|
return undefined;
|
|
854
862
|
}
|
|
855
863
|
/**
|
|
856
|
-
* The tool names of the steps from flat position `from` to the end
|
|
857
|
-
* that runs only under a test marked so — for a
|
|
858
|
-
* planned steps", which must not read as steps
|
|
864
|
+
* The tool names of the steps from flat position `from` to the end (or to
|
|
865
|
+
* `until`, not included), each that runs only under a test marked so — for a
|
|
866
|
+
* hand-over note's "remaining planned steps", which must not read as steps
|
|
867
|
+
* this request needs.
|
|
859
868
|
*/
|
|
860
|
-
remainingFrom(from) {
|
|
861
|
-
return this.entries.slice(Math.max(0, from)).map((e) => {
|
|
869
|
+
remainingFrom(from, until) {
|
|
870
|
+
return this.entries.slice(Math.max(0, from), until).map((e) => {
|
|
862
871
|
const name = e.step.toolName ?? "";
|
|
863
872
|
const t = this.onlyWhen(e);
|
|
864
873
|
return t ? `${name} (only when ${t.text ?? "a request needs it"})` : name;
|
|
@@ -1279,10 +1288,21 @@ export class ScenarioReplayPlan {
|
|
|
1279
1288
|
break;
|
|
1280
1289
|
}
|
|
1281
1290
|
const anyArgs = args.slice(0, argsShown).some((a) => a !== undefined);
|
|
1291
|
+
// A plan whose chain holds a decision (the service's segmentsbreakdown.md
|
|
1292
|
+
// SB-18) never shows its intent, even when it stops earlier for another
|
|
1293
|
+
// reason: the intent names the recorded task, the half the recording took
|
|
1294
|
+
// included, and which way applies is the agent's choice.
|
|
1295
|
+
const atDecision = this.stopStep()?.stop?.kind === "decision";
|
|
1296
|
+
const shownIntent = this.decisionIndex() >= 0 ? "" : intent;
|
|
1282
1297
|
// A parked step (fallbk.md D3): the model is told up front that the plan is
|
|
1283
1298
|
// partial, so the hand-over that follows is expected rather than a surprise.
|
|
1284
1299
|
const tail = this.stopsEarly()
|
|
1285
|
-
? [
|
|
1300
|
+
? [
|
|
1301
|
+
"",
|
|
1302
|
+
atDecision
|
|
1303
|
+
? "The system will then hand the task back to you at a decision."
|
|
1304
|
+
: "The system will then hand the rest of the task back to you.",
|
|
1305
|
+
]
|
|
1286
1306
|
: [];
|
|
1287
1307
|
if (this.mode === "direct") {
|
|
1288
1308
|
// The sequence is listed even though the model does not call it itself:
|
|
@@ -1291,7 +1311,7 @@ export class ScenarioReplayPlan {
|
|
|
1291
1311
|
// place an operator watching the session can see what a replay is doing.
|
|
1292
1312
|
return [
|
|
1293
1313
|
"[BaseInstRunner calculated replay] A known-good tool sequence was recorded",
|
|
1294
|
-
`for this request${
|
|
1314
|
+
`for this request${shownIntent ? ` (intent: ${shownIntent})` : ""}.`,
|
|
1295
1315
|
"",
|
|
1296
1316
|
anyTest
|
|
1297
1317
|
? 'It runs these tools, in order, and returns their results. A step marked "only when" runs only when this request needs it, and the results say which ones did not:'
|
|
@@ -1313,7 +1333,7 @@ export class ScenarioReplayPlan {
|
|
|
1313
1333
|
"recorded for this request. Call exactly these tools, in order, one at a time:",
|
|
1314
1334
|
"",
|
|
1315
1335
|
...lines,
|
|
1316
|
-
...(
|
|
1336
|
+
...(shownIntent ? ["", `Context: ${shownIntent}`] : []),
|
|
1317
1337
|
...(anyTest
|
|
1318
1338
|
? [
|
|
1319
1339
|
"",
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whether a tool call may change something, judged by its name alone — the
|
|
3
|
+
* runner's copy of the service's own rule (BaseIn src/scenarios/correctionCheck.ts,
|
|
4
|
+
* `readsOnly`, `commandReadsOnly`, `callMayChange`), kept word for word so the
|
|
5
|
+
* two never disagree about the same call.
|
|
6
|
+
*
|
|
7
|
+
* Used where a call after a plan's decision is the agent starting a way of its
|
|
8
|
+
* own (the service's offer1.md O7), and where a plan started after a call
|
|
9
|
+
* would make it a second time (O8). When in doubt, a call is a change: that
|
|
10
|
+
* only ends a set of scenarios early or keeps a plan from starting, never lets
|
|
11
|
+
* a second write through.
|
|
12
|
+
*/
|
|
13
|
+
/**
|
|
14
|
+
* A tool that only reads: Claude Code's read-only tools, then the name — a
|
|
15
|
+
* reading word and no writing word (`calendar_search_events`, `getJiraIssue`;
|
|
16
|
+
* `run_sql` no: `run` writes). A name with both (`get_or_create_contact`)
|
|
17
|
+
* changes something.
|
|
18
|
+
*/
|
|
19
|
+
export declare function readsOnly(toolName: string): boolean;
|
|
20
|
+
/** Whether a shell command only looks: every part of it is a looking command, and nothing is written to a file. */
|
|
21
|
+
export declare function commandReadsOnly(command: string): boolean;
|
|
22
|
+
/**
|
|
23
|
+
* Whether one call may change something. A Bash call changes nothing only
|
|
24
|
+
* when its command is there and only looks; any other tool is a change unless
|
|
25
|
+
* {@link readsOnly} says it only reads — a tool it cannot tell about counts as
|
|
26
|
+
* a change. `input` is the call's arguments, as an object or as JSON text.
|
|
27
|
+
*/
|
|
28
|
+
export declare function callMayChange(toolName: string, input: unknown): boolean;
|
|
29
|
+
//# sourceMappingURL=reads-only.d.ts.map
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whether a tool call may change something, judged by its name alone — the
|
|
3
|
+
* runner's copy of the service's own rule (BaseIn src/scenarios/correctionCheck.ts,
|
|
4
|
+
* `readsOnly`, `commandReadsOnly`, `callMayChange`), kept word for word so the
|
|
5
|
+
* two never disagree about the same call.
|
|
6
|
+
*
|
|
7
|
+
* Used where a call after a plan's decision is the agent starting a way of its
|
|
8
|
+
* own (the service's offer1.md O7), and where a plan started after a call
|
|
9
|
+
* would make it a second time (O8). When in doubt, a call is a change: that
|
|
10
|
+
* only ends a set of scenarios early or keeps a plan from starting, never lets
|
|
11
|
+
* a second write through.
|
|
12
|
+
*/
|
|
13
|
+
/** Built-in tools that only read (Claude Code's). */
|
|
14
|
+
const READ_TOOLS = new Set([
|
|
15
|
+
"Read",
|
|
16
|
+
"Grep",
|
|
17
|
+
"Glob",
|
|
18
|
+
"LS",
|
|
19
|
+
"WebFetch",
|
|
20
|
+
"WebSearch",
|
|
21
|
+
"NotebookRead",
|
|
22
|
+
"TodoWrite",
|
|
23
|
+
"TaskOutput",
|
|
24
|
+
"BashOutput",
|
|
25
|
+
]);
|
|
26
|
+
/** Built-in tools that change files or run commands (Claude Code's). */
|
|
27
|
+
const CHANGING_BUILTINS = new Set([
|
|
28
|
+
"Write",
|
|
29
|
+
"Edit",
|
|
30
|
+
"MultiEdit",
|
|
31
|
+
"NotebookEdit",
|
|
32
|
+
"Bash",
|
|
33
|
+
"KillShell",
|
|
34
|
+
]);
|
|
35
|
+
// Not `query`, `select` or `check`: a database query, a selected option, a
|
|
36
|
+
// check-in can change something.
|
|
37
|
+
const READ_WORDS = new Set([
|
|
38
|
+
"list",
|
|
39
|
+
"get",
|
|
40
|
+
"search",
|
|
41
|
+
"find",
|
|
42
|
+
"read",
|
|
43
|
+
"fetch",
|
|
44
|
+
"lookup",
|
|
45
|
+
"count",
|
|
46
|
+
"view",
|
|
47
|
+
"show",
|
|
48
|
+
"describe",
|
|
49
|
+
"retrieve",
|
|
50
|
+
"browse",
|
|
51
|
+
"peek",
|
|
52
|
+
"inspect",
|
|
53
|
+
"snapshot",
|
|
54
|
+
]);
|
|
55
|
+
/**
|
|
56
|
+
* A word that may change something: a name holding one writes, whatever else
|
|
57
|
+
* it says (`get_or_create_contact`, `find_and_replace`).
|
|
58
|
+
*/
|
|
59
|
+
const WRITE_WORDS = new Set([
|
|
60
|
+
"create",
|
|
61
|
+
"add",
|
|
62
|
+
"set",
|
|
63
|
+
"update",
|
|
64
|
+
"upsert",
|
|
65
|
+
"insert",
|
|
66
|
+
"put",
|
|
67
|
+
"post",
|
|
68
|
+
"send",
|
|
69
|
+
"delete",
|
|
70
|
+
"remove",
|
|
71
|
+
"mark",
|
|
72
|
+
"replace",
|
|
73
|
+
"archive",
|
|
74
|
+
"move",
|
|
75
|
+
"assign",
|
|
76
|
+
"book",
|
|
77
|
+
"cancel",
|
|
78
|
+
"write",
|
|
79
|
+
"edit",
|
|
80
|
+
"modify",
|
|
81
|
+
"save",
|
|
82
|
+
"submit",
|
|
83
|
+
"approve",
|
|
84
|
+
"reject",
|
|
85
|
+
"close",
|
|
86
|
+
"open",
|
|
87
|
+
"merge",
|
|
88
|
+
"push",
|
|
89
|
+
"publish",
|
|
90
|
+
"upload",
|
|
91
|
+
"invite",
|
|
92
|
+
"schedule",
|
|
93
|
+
"reply",
|
|
94
|
+
"forward",
|
|
95
|
+
"transfer",
|
|
96
|
+
"pay",
|
|
97
|
+
"refund",
|
|
98
|
+
"notify",
|
|
99
|
+
"run",
|
|
100
|
+
"execute",
|
|
101
|
+
"exec",
|
|
102
|
+
"apply",
|
|
103
|
+
"reset",
|
|
104
|
+
"start",
|
|
105
|
+
"stop",
|
|
106
|
+
"kill",
|
|
107
|
+
"enable",
|
|
108
|
+
"disable",
|
|
109
|
+
"grant",
|
|
110
|
+
"revoke",
|
|
111
|
+
"rename",
|
|
112
|
+
"tag",
|
|
113
|
+
"label",
|
|
114
|
+
"comment",
|
|
115
|
+
]);
|
|
116
|
+
/** A tool name's words: `getJiraIssue` → get, jira, issue; `calendar_search_events` → calendar, search, events. */
|
|
117
|
+
function nameWords(name) {
|
|
118
|
+
return name
|
|
119
|
+
.replace(/([a-z\d])([A-Z])/g, "$1 $2")
|
|
120
|
+
.replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
|
|
121
|
+
.toLowerCase()
|
|
122
|
+
.split(/[^a-z\d]+/)
|
|
123
|
+
.filter(Boolean);
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* A tool that only reads: Claude Code's read-only tools, then the name — a
|
|
127
|
+
* reading word and no writing word (`calendar_search_events`, `getJiraIssue`;
|
|
128
|
+
* `run_sql` no: `run` writes). A name with both (`get_or_create_contact`)
|
|
129
|
+
* changes something.
|
|
130
|
+
*/
|
|
131
|
+
export function readsOnly(toolName) {
|
|
132
|
+
if (READ_TOOLS.has(toolName))
|
|
133
|
+
return true;
|
|
134
|
+
if (CHANGING_BUILTINS.has(toolName))
|
|
135
|
+
return false;
|
|
136
|
+
const words = nameWords(toolName.replace(/^mcp__.+?__/, ""));
|
|
137
|
+
if (words.some((w) => WRITE_WORDS.has(w)))
|
|
138
|
+
return false;
|
|
139
|
+
return words.some((w) => READ_WORDS.has(w));
|
|
140
|
+
}
|
|
141
|
+
/** Commands that only look (`ls`, `git status`, `grep …`). */
|
|
142
|
+
const READ_COMMANDS = new Set([
|
|
143
|
+
"ls",
|
|
144
|
+
"dir",
|
|
145
|
+
"cat",
|
|
146
|
+
"type",
|
|
147
|
+
"head",
|
|
148
|
+
"tail",
|
|
149
|
+
"less",
|
|
150
|
+
"more",
|
|
151
|
+
"grep",
|
|
152
|
+
"rg",
|
|
153
|
+
"egrep",
|
|
154
|
+
"fgrep",
|
|
155
|
+
"find",
|
|
156
|
+
"wc",
|
|
157
|
+
"pwd",
|
|
158
|
+
"echo",
|
|
159
|
+
"printf",
|
|
160
|
+
"date",
|
|
161
|
+
"whoami",
|
|
162
|
+
"which",
|
|
163
|
+
"where",
|
|
164
|
+
"env",
|
|
165
|
+
"printenv",
|
|
166
|
+
"stat",
|
|
167
|
+
"file",
|
|
168
|
+
"du",
|
|
169
|
+
"df",
|
|
170
|
+
"tree",
|
|
171
|
+
"jq",
|
|
172
|
+
"sort",
|
|
173
|
+
"uniq",
|
|
174
|
+
"cut",
|
|
175
|
+
"awk",
|
|
176
|
+
"diff",
|
|
177
|
+
"cmp",
|
|
178
|
+
"basename",
|
|
179
|
+
"dirname",
|
|
180
|
+
"realpath",
|
|
181
|
+
"readlink",
|
|
182
|
+
"true",
|
|
183
|
+
"false",
|
|
184
|
+
"test",
|
|
185
|
+
"sleep",
|
|
186
|
+
"ps",
|
|
187
|
+
"uname",
|
|
188
|
+
"hostname",
|
|
189
|
+
"id",
|
|
190
|
+
"nproc",
|
|
191
|
+
"free",
|
|
192
|
+
"uptime",
|
|
193
|
+
"node --version",
|
|
194
|
+
"npm --version",
|
|
195
|
+
]);
|
|
196
|
+
const READ_GIT = new Set([
|
|
197
|
+
"status",
|
|
198
|
+
"log",
|
|
199
|
+
"diff",
|
|
200
|
+
"show",
|
|
201
|
+
"branch",
|
|
202
|
+
"rev-parse",
|
|
203
|
+
"ls-files",
|
|
204
|
+
"blame",
|
|
205
|
+
"remote",
|
|
206
|
+
"describe",
|
|
207
|
+
"config --get",
|
|
208
|
+
"grep",
|
|
209
|
+
"shortlog",
|
|
210
|
+
"tag --list",
|
|
211
|
+
]);
|
|
212
|
+
/** Whether a shell command only looks: every part of it is a looking command, and nothing is written to a file. */
|
|
213
|
+
export function commandReadsOnly(command) {
|
|
214
|
+
const text = command.trim();
|
|
215
|
+
if (!text)
|
|
216
|
+
return true;
|
|
217
|
+
// A redirect into a file (not into /dev/null, not 2>&1), tee, or an in-place edit changes something.
|
|
218
|
+
if (/(^|[^0-9&])>{1,2}\s*(?!&|\/dev\/null)\S/.test(text) || /\btee\b/.test(text) || /\bsed\b[^|;&]*\s-i\b/.test(text)) {
|
|
219
|
+
return false;
|
|
220
|
+
}
|
|
221
|
+
if (/\bfind\b[^|;&]*\s-(delete|exec|execdir|ok)\b/.test(text))
|
|
222
|
+
return false;
|
|
223
|
+
const parts = text
|
|
224
|
+
.split(/\|\||&&|[|;\n]/)
|
|
225
|
+
.map((p) => p.trim())
|
|
226
|
+
.filter(Boolean);
|
|
227
|
+
return parts.every((part) => {
|
|
228
|
+
const words = part.replace(/^(\w+=\S+\s+)+/, "").split(/\s+/);
|
|
229
|
+
const cmd = (words[0] ?? "").replace(/^.*[\\/]/, "");
|
|
230
|
+
if (cmd === "git")
|
|
231
|
+
return READ_GIT.has(words[1] ?? "") || READ_GIT.has(`${words[1] ?? ""} ${words[2] ?? ""}`);
|
|
232
|
+
if (cmd === "sed" || cmd === "awk")
|
|
233
|
+
return !/\s-i\b/.test(part);
|
|
234
|
+
return READ_COMMANDS.has(cmd) || READ_COMMANDS.has(`${cmd} ${words[1] ?? ""}`);
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Whether one call may change something. A Bash call changes nothing only
|
|
239
|
+
* when its command is there and only looks; any other tool is a change unless
|
|
240
|
+
* {@link readsOnly} says it only reads — a tool it cannot tell about counts as
|
|
241
|
+
* a change. `input` is the call's arguments, as an object or as JSON text.
|
|
242
|
+
*/
|
|
243
|
+
export function callMayChange(toolName, input) {
|
|
244
|
+
if (toolName === "Bash") {
|
|
245
|
+
let value = input;
|
|
246
|
+
if (typeof value === "string") {
|
|
247
|
+
try {
|
|
248
|
+
value = JSON.parse(value);
|
|
249
|
+
}
|
|
250
|
+
catch {
|
|
251
|
+
value = undefined;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
const command = value && typeof value === "object" && typeof value.command === "string"
|
|
255
|
+
? value.command
|
|
256
|
+
: "";
|
|
257
|
+
return command.trim() === "" || !commandReadsOnly(command);
|
|
258
|
+
}
|
|
259
|
+
return !readsOnly(toolName);
|
|
260
|
+
}
|
|
261
|
+
//# sourceMappingURL=reads-only.js.map
|
package/dist/replay/types.d.ts
CHANGED
|
@@ -47,9 +47,18 @@ export interface StepStop {
|
|
|
47
47
|
* `parked`: the step has failed too often and its retry is not due, or its
|
|
48
48
|
* grace has ended. `nondeterministic`: the calculation could not write code
|
|
49
49
|
* that computes this step's input — a judgement — so the agent makes it.
|
|
50
|
+
* `decision`: the task has an "if … otherwise" here (the service's
|
|
51
|
+
* segmentsbreakdown.md SB-6). The plan runs only the steps before it and
|
|
52
|
+
* hands the task back, and the agent decides which way applies. Not a
|
|
53
|
+
* failure: it counts nothing against the step. Runners up to 0.3.18 read it
|
|
54
|
+
* as an unknown kind and stop here too, as `known_bad_step`.
|
|
55
|
+
*/
|
|
56
|
+
kind: "parked" | "nondeterministic" | "decision";
|
|
57
|
+
/**
|
|
58
|
+
* For `nondeterministic`: which check the step failed at calculation. For
|
|
59
|
+
* `decision`: the condition in words. It holds the recorded run's values,
|
|
60
|
+
* so it is never shown to the agent (it would hint at the recorded half).
|
|
50
61
|
*/
|
|
51
|
-
kind: "parked" | "nondeterministic";
|
|
52
|
-
/** For `nondeterministic`: which check the step failed at calculation. */
|
|
53
62
|
why?: string;
|
|
54
63
|
/** For `parked` inside its grace: when the step may be tried again. */
|
|
55
64
|
retryAt?: string;
|
|
@@ -252,7 +261,14 @@ export type FallbackKind =
|
|
|
252
261
|
* (autocorrect.md AC5): this request is the other half of an "if …
|
|
253
262
|
* otherwise", and the agent does it. No fault of the plan; counts nothing.
|
|
254
263
|
*/
|
|
255
|
-
| "other_branch"
|
|
264
|
+
| "other_branch"
|
|
265
|
+
/**
|
|
266
|
+
* The plan ran the steps before its decision and stopped in front of it
|
|
267
|
+
* (the service's segmentsbreakdown.md SB-17): the agent decides which way
|
|
268
|
+
* applies and does the rest. No fault of the plan; it counts nothing, and
|
|
269
|
+
* the plan is billed only up to the stop (SB-26).
|
|
270
|
+
*/
|
|
271
|
+
| "decision";
|
|
256
272
|
/**
|
|
257
273
|
* Upgrade order. `not_steered` is the floor — the control group, and the safe
|
|
258
274
|
* answer if a turn dies mid-flight — and each later decision point can only move
|
package/docs/calculatedReplay.md
CHANGED
|
@@ -748,6 +748,30 @@ about it:
|
|
|
748
748
|
It ends with the remaining planned steps and the parameters derived for this request, within 4 000
|
|
749
749
|
characters.
|
|
750
750
|
|
|
751
|
+
**A plan's decision (0.3.19)** has a note of its own. The service serves a step with
|
|
752
|
+
`stop: {kind: "decision"}` where the task has an "if … otherwise" (its segmentsbreakdown.md). The
|
|
753
|
+
plan runs only the steps before it, in steer and direct mode and when a divergence's bundle runs up
|
|
754
|
+
to it, and hands the task back. The note opens *"[BaseInstRunner calculated replay] The calculated
|
|
755
|
+
scenario for this request ran the steps before a decision, and stopped there."*, says which steps
|
|
756
|
+
ran as above, and ends *"This request asks for a decision: “…”. Decide from the results above
|
|
757
|
+
which way applies, then continue the task, calling tools as needed."* (*below* in a direct run's
|
|
758
|
+
answer, *above and below* after a divergence; *"This task has a decision to make here."* when the
|
|
759
|
+
request has no such sentence). The quote is the live request's own sentences with a word for a
|
|
760
|
+
condition, found with the service's word list (`if`, `otherwise`, `else`, `unless`, `only when`,
|
|
761
|
+
`only if`, `in case`, `whether`, and the Hebrew ones), at most 600 characters
|
|
762
|
+
([`condition-words.ts`](../src/replay/condition-words.ts)). The note has no intent line, no
|
|
763
|
+
remaining planned steps and no parameters, and never the condition the service keeps (`stop.why`):
|
|
764
|
+
each names, or hints at, the half the recording took. The directive at the prompt leaves the intent
|
|
765
|
+
out for the same reason, and says the task comes back *"at a decision"*. The report is `fell_back`
|
|
766
|
+
(`failed` when no step ran) with `fallbackKind: "decision"`; the decision step has no verdict, so
|
|
767
|
+
nothing counts against it. A runner up to 0.3.18 reads the unknown kind as `known_bad_step`.
|
|
768
|
+
|
|
769
|
+
A plan whose chain has a decision keeps these protections when it stops *earlier* for another
|
|
770
|
+
reason — a parked or failed step, a judgement, the budget. Its note is the ordinary one above, but
|
|
771
|
+
with no intent line and no parameters, and its remaining planned steps end before the decision. Its
|
|
772
|
+
directive leaves the intent out too. An agent sent on to the recorded half's steps would start that
|
|
773
|
+
half's segment, and the plan would have decided after all.
|
|
774
|
+
|
|
751
775
|
With `BIR_REPLAY_RECORDED_OUTPUTS=1`, a step that could not be *sent* (a built-in, no proxy, off the
|
|
752
776
|
allowlist) is served its **recorded output** instead (§8.1), marked `(recorded)` in the bundle so the
|
|
753
777
|
model is never told a stale result is fresh. A timeout or an upstream error never is.
|
|
@@ -1010,10 +1034,16 @@ note replaces the call, in direct mode `run_scenario` answers with it). The deri
|
|
|
1010
1034
|
itself times out after max(`BIR_SERVICE_TIMEOUT_MS`, `BIR_DERIVE_BUDGET_MS`). None of these waits
|
|
1011
1035
|
happens while the service is cooling down after a failure: the turn simply runs normally.
|
|
1012
1036
|
|
|
1013
|
-
**Intent arming mid-turn is
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
arming. `
|
|
1037
|
+
**Intent arming mid-turn is on by default from 0.3.19.** An intent probe is sent from `PreToolUse`
|
|
1038
|
+
for hit counting. While arming is on — `BIR_SEGMENT_ARM` unset or `1`, the recorder's default from
|
|
1039
|
+
0.3.19, or `BIR_INTENT_ARM=1` — the probe may arm a plan in that call's place, and the hook waits at
|
|
1040
|
+
most `BIR_INTENT_ARM_BUDGET_MS` (**500**) for it and 1.5 s more for arming. With `BIR_SEGMENT_ARM=0`
|
|
1041
|
+
(and no `BIR_INTENT_ARM=1`), or `BIR_INTENT_ARM=0`, it is never waited on. Up to 0.3.18 arming was
|
|
1042
|
+
opt-in: a segment runs only mid-task, and a plan that stops at its decision hands the rest of the
|
|
1043
|
+
task to segments of its halves, so 0.3.19 turns it on. A probe asks the service to hand segments out
|
|
1044
|
+
(`acceptSegments`) only where one can arm: segment and intent arming on, in Claude Code. Any other
|
|
1045
|
+
probe — `BIR_INTENT_ARM=0`, or a session of another host — gets what would have armed instead, and
|
|
1046
|
+
it is logged (`replay.segment_would_arm`).
|
|
1017
1047
|
|
|
1018
1048
|
`UserPromptSubmit` must stay non-`async`: an async hook is fire-and-forget and cannot return
|
|
1019
1049
|
`additionalContext` at all.
|
|
@@ -1045,7 +1075,8 @@ stopped at a step that broke (a logic failure, a tool error, a call sent and nev
|
|
|
1045
1075
|
recorded stand-in (`BIR_REPLAY_RECORDED_OUTPUTS=1`) is never work done: it counts toward neither
|
|
1046
1076
|
`stepsPinned` nor the upgrade, and a direct run that used one is `diverged` at best. A call that ran
|
|
1047
1077
|
beside a pinned one hands over as `fell_back` or `failed` (§7), and so does a divergence whose
|
|
1048
|
-
bundle stops in front of the other half of an "if … otherwise" (§9.2)
|
|
1078
|
+
bundle stops in front of the other half of an "if … otherwise" (§9.2), or in front of the plan's
|
|
1079
|
+
decision (§8). A direct run cancelled by its caller
|
|
1049
1080
|
(Esc, or `bir-scenario`'s own timeout) books `fell_back` or `failed` with `measured: false`, never
|
|
1050
1081
|
`diverged`: that turn's cost says nothing about what the task costs.
|
|
1051
1082
|
|
|
@@ -1256,6 +1287,15 @@ tokens and every model in the list has a price; with no usage at all, the line s
|
|
|
1256
1287
|
`why="the host sent no usage"`. Such a host hands over one number for the whole run, so with more
|
|
1257
1288
|
than one plan in one run none of them is measured.
|
|
1258
1289
|
|
|
1290
|
+
**A plan stopped at its decision is billed up to the stop (0.3.19).** When a step of it ran, its
|
|
1291
|
+
hand-over takes a usage mark, and its report's window ends there: the agent's decision and the half
|
|
1292
|
+
it took are the agent's own, not the plan's (the service's segmentsbreakdown.md SB-26). This holds
|
|
1293
|
+
for a plan the prompt started, which gets a way to take a mark only when its stop is a decision, and
|
|
1294
|
+
for one an intent match started, whose window already ended at its own retire. Every other plan
|
|
1295
|
+
keeps its window — a plan of the prompt is billed for the whole turn up to the next plan, so the
|
|
1296
|
+
answer it was asked for stays on its bill — and every other hand-over still takes no mark. A
|
|
1297
|
+
decision reached before any step ran takes no mark either: its report is a sample of the whole turn.
|
|
1298
|
+
|
|
1259
1299
|
---
|
|
1260
1300
|
|
|
1261
1301
|
## 12. What the replayed turn records
|
|
@@ -327,7 +327,7 @@ Recording : yes
|
|
|
327
327
|
· the recorder runs in the background (pid 41820); its audit log is ~/.baseinstrunner/logs/<key>.log
|
|
328
328
|
· the BaseIn service last answered the recorder 12 s ago
|
|
329
329
|
· wrapped MCP calls are not approved by the runner: Claude Code asks as it would without it. For headless runs (`claude -p`) allow them in permissions.allow or --allowedTools, or set BIR_CORRELATION_DECISION=allow where the recorder starts
|
|
330
|
-
· recurring sub-tasks
|
|
330
|
+
· segments run mid-task — recurring sub-tasks, and the halves after a plan's decision — and so may whole plans (the default; BIR_SEGMENT_ARM=0 turns this off)
|
|
331
331
|
|
|
332
332
|
No problems found.
|
|
333
333
|
```
|
|
@@ -370,7 +370,8 @@ so there is exactly one switch and it cannot get out of step with itself.
|
|
|
370
370
|
| `BIR_REPLAY_RECORDED_OUTPUTS` | *(unset)* | `1` lets a step that could not be sent (a built-in off `BIR_BUILTIN_TOOLS`, no proxy, a server off the allowlist) stand on its recorded output instead of stopping the plan. Never for a timeout or an upstream error |
|
|
371
371
|
| `BIR_BUILTIN_TOOLS` | `Read,Glob,Grep,Edit,Write,Bash` | Built-ins the runner executes itself during a replay, in the session's own project directory and confined to it. A chain whose only non-MCP steps are on this list arms `direct`. A runner-side step never reaches the host's permission prompts, so leave off any tool you want the model to call itself; `Read,Glob,Grep` is the read-only set (the default up to 0.3.7). `none`: every built-in step is steered in the session. Claude Code sessions only: in a LangChain application a tool with one of these names is the application's own, and the runner runs none (§6.5) |
|
|
372
372
|
| `BIR_BASH_ALLOW` | *(unset: any command)* | Command prefixes a runner-side `Bash` step may run (`*` for all). Unset means any command; `none` means no command, and `Bash` steps are steered in the session. A plan's Bash steps share a working directory, exactly as the host's own Bash tool does — variables and functions do not persist, also as the host's. A call with `run_in_background`, or with its own `timeout` longer than the step's clock (`BIR_STEP_TIMEOUT_MS`), is never started on the runner: the plan hands over in front of it |
|
|
373
|
-
| `
|
|
373
|
+
| `BIR_SEGMENT_ARM` | *(unset: on, from 0.3.19)* | A segment the service hands out runs mid-task: a recurring sub-task, or a half of a task after a plan's decision. `0` is observe-only: what would have armed is logged (`replay.segment_would_arm`) and nothing is replaced. Up to 0.3.18 observe-only was the default. A recorder started before an update keeps its old default until `bir up --restart` |
|
|
374
|
+
| `BIR_INTENT_ARM` | *(unset)* | `1` lets a match found mid-turn arm a plan in the calling tool's place, which means waiting for it (`BIR_INTENT_ARM_BUDGET_MS`, 500); `0` never waits, and then nothing starts mid-task, segments included: what would have armed is logged; unset follows `BIR_SEGMENT_ARM`, so from 0.3.19 it is on unless `BIR_SEGMENT_ARM=0` |
|
|
374
375
|
| `BIR_VERBOSE` | *(unset)* | `1` adds per-step `replay.*` detail lines |
|
|
375
376
|
|
|
376
377
|
The `bir` server reads its own, from the environment Claude Code starts it with:
|
|
@@ -493,7 +494,11 @@ Everything replay decides is one line. `grep` for these:
|
|
|
493
494
|
| `replay.step` | One direct step executed |
|
|
494
495
|
| `replay.step_not_taken` | A step's test said this request does not need it, and nothing was sent for it ([calculatedReplay.md](calculatedReplay.md) §9.2). `plan.armed … tests=` lists the tested steps by their place in the plan: `4:hand_over,11:skip` |
|
|
495
496
|
| `replay.diverge` / `replay.compose` | Off-script, and the recovery |
|
|
496
|
-
| `replay.handover` | The plan stopped and handed the rest of the task to the model; `kind=` and `step=` say where |
|
|
497
|
+
| `replay.handover` | The plan stopped and handed the rest of the task to the model; `kind=` and `step=` say where. `kind=decision` (0.3.19) is a plan that ran the steps before its decision and stopped there, as planned (`via=divergence` when a divergence's bundle ran up to it) |
|
|
498
|
+
| `run.fragment … decision=true` / `run.fragment_dropped` | After a plan's decision: the agent's own work from its next call is recorded as a fragment, from which the service makes a calculated scenario of that way / a plan the intent check armed, or a next scenario, took that call instead, so nothing is recorded as the agent's own |
|
|
499
|
+
| `replay.next_ready` / `replay.next_hit` / `replay.next_miss` | 0.3.20: the calculated scenarios that can follow a plan's decision, armed while the agent decided / the agent's call was one's first step, value for value, so it runs in the call's place / none was used: `reason=other_values` (the same first tool with other values), `own_way` (a call that may change something and starts none of them), `not_ready` (the one that starts with that tool was not armed in time), `calls_limit` (5 calls started none), `turn_ended`. A call that only reads ends nothing |
|
|
500
|
+
| `run.fragment_closed` | 0.3.20: a scenario that follows the decision took over after the agent's own looks had opened the fragment; the fragment ends there, with what the agent did itself |
|
|
501
|
+
| `replay.decision … code=would_repeat` | 0.3.20: a plan an intent match armed after the agent's call would have made that call again (the same tool that may change something, or for a shell the same command), so it was not started (never twice) |
|
|
497
502
|
| `replay.vetoed` | A direct plan retired without running anything: `run_scenario` was not the first call, or never arrived |
|
|
498
503
|
| `replay.run_ambiguous` | Two sessions had `run_scenario` pending and the request named neither; nothing ran |
|
|
499
504
|
| `replay.cancelled` / `replay.note_kept` | A direct run's caller went away (Esc); no further step started, and the note is kept for the model's next call |
|
|
@@ -578,6 +583,11 @@ A recorded output that stood in for a step (`BIR_REPLAY_RECORDED_OUTPUTS=1`) is
|
|
|
578
583
|
work done. A direct run you cancel (Esc) is booked `fell_back` or `failed` with `measured: false`:
|
|
579
584
|
a turn cut short says nothing about what the task costs.
|
|
580
585
|
|
|
586
|
+
A plan that stops at its decision (runner 0.3.19 or newer) is booked `fell_back` with
|
|
587
|
+
`fallbackKind: decision`, or `failed` when no step ran. Nothing is counted against its steps, and
|
|
588
|
+
when a step ran its cost ends at the stop: the agent's decision and the half it took are not on the
|
|
589
|
+
plan's bill.
|
|
590
|
+
|
|
581
591
|
A turn's cost is **measured** only when its tokens were found and every model of the turn has a
|
|
582
592
|
price in the runner's table. A model with no price used to count as $0, and that zero was called
|
|
583
593
|
measured. Now such a turn is not measured, in Claude Code as in any host, and a `run.unmeasured`
|
|
@@ -656,6 +666,8 @@ Code's pool, as before.
|
|
|
656
666
|
| `run_scenario` answers "The prepared results are not available for this request. Do the task normally with your tools." | The plan could not be delivered — a recorder that does not answer, a timeout, an HTTP error — and the model does the task itself | The `scenario.answered` line on the `bir` server's stderr has the real reason. `BIR_SCENARIO_DEBUG=1` in that server's environment hands the raw reason to the model as an error, for debugging |
|
|
657
667
|
| A hand-over note says a step "was sent but did not answer … may or may not have completed" | The step timed out, or its upstream went away mid-call; nothing after it ran | Check whether its work happened before running it again — the model is told the same. Repeats: look at that upstream, or raise `BIR_STEP_TIMEOUT_MS` |
|
|
658
668
|
| A hand-over note says "a step never runs on another session's upstream" | Two Claude Code sessions are open in the project, and this one has not yet made a call of its own to that server, so bir cannot tell its proxy from the other session's. A direct plan is told apart by the process that started both the `bir` server and the session's proxies, so for one this means an older proxy or `bir` server (restart the session after an update), or a server started through a wrapper such as a shell or `npx` | Expected: the model calls the tool itself. Once the session's own call to the server has gone through, later replays use its proxy |
|
|
669
|
+
| A hand-over note says the plan "ran the steps before a decision, and stopped there" | The service split the plan at its "if … otherwise" (runner 0.3.19 or newer): the plan runs the steps both ways need, then the agent decides which way applies. The note quotes the request's own sentences with the condition | Expected. The agent's next step may start a segment of the way it chose (`replay.intent_armed … kind=segment`); with `BIR_SEGMENT_ARM=0` it does that part itself. `bir investigate` shows the run as `fell_back · decision` |
|
|
670
|
+
| A hand-over note says a step "did not run: it has failed before", and the step never failed | A runner up to 0.3.18 met a plan's decision, a stop it does not know. It stops there too, but its note names a failure and lists the remaining steps | Update the runner (`bir update`, then `bir up --restart`) |
|
|
659
671
|
| A hand-over note says a step "was not run by the host", or "ran with arguments a person chose, not the scenario's" | A LangChain application (§6.5). A person said no to the step, or answered in the tool's place, or a call limit blocked the call (`host_rejected`); or a person changed the call before it ran (`host_edited`) | Expected, and no fault of the plan: the step is booked `skipped`, the plan stops there and the model takes over. It is told not to run the step again — after a refusal, unless the user asks. §7 has what the report says |
|
|
660
672
|
| `recorder.send_failed … error="HTTP 409 … invalid_ticket"` | The execution report's ticket was already redeemed, or belongs to another scenario. Or the report came more than an hour after its plan was handed out | Harmless if the body says `duplicate: true`. Otherwise a rollover raced `SessionEnd`; the server books once. A report that came too late is not booked; the service as built (not deployed yet) takes a report for 24 hours |
|
|
661
673
|
| `recorder.send_failed … error="HTTP 429 …"` | The account is over one of the service's limits, and the answer is `429` with `Retry-After`. The limits are built on the service and not deployed yet; each is a setting there. By default: 120 runs a minute · 3,000 steps a minute (a tool call is two steps) · 10 title requests a minute · 20 requests in any one hour for Calculate, Prefer and Recalculate together (`bir scenario calc` then prints a line that begins `Could not start calculation (HTTP 429`) | What it costs: a create, a step or a report is sent once more, at most 2 s later, then dropped. A dropped step is a hole in its recording. A run whose create got the 429 runs without its plan. A 429 that the second try did not cure counts as the service being unwell: for 5 to 60 seconds no prompt of that recorder waits for a match. An intent probe over its own limit (1,500 a minute) is answered with no match, never with `429` |
|
|
@@ -1337,7 +1349,7 @@ Parameter database: target → setting (revision 4). Undo: bir scenario undo scn
|
|
|
1337
1349
|
| `revision_changed` | The plan changed since you read it: another edit, or a recalculation | `show` it again, check again, save again |
|
|
1338
1350
|
| `not_ready` or `scenario_calculating` | The scenario is not `ready`, or a recalculation is running | Wait for it to finish |
|
|
1339
1351
|
| `undo_refused` | The undo would stop a later step from giving its recorded call, or leave the final answer without its values | Fix that later step first. Or undo it on purpose: `--force --note "why"` |
|
|
1340
|
-
| `not_latest_edit` | The step has changed since the edit you named: a newer entry changed it. For a correction: the plan has changed since it | `bir` prints two commands: undo the newer one first, then yours |
|
|
1352
|
+
| `not_latest_edit` | The step has changed since the edit you named: a newer entry changed it. For a correction: the plan has changed since it. When the newer entry is an `ssplit_…` id, the plan was split at its decision after the correction, and a correction cannot be taken back while the split is in use | `bir` prints two commands: undo the newer one first, then yours. For a split, `bir` says to press Undo on the Split card of the plan's recording page first, and prints the command for yours |
|
|
1341
1353
|
| `already_undone` | That edit was undone already | To redo it, undo the undo. `bir` prints the command |
|
|
1342
1354
|
| `edit_replaced` | A recalculation replaced the plan after that edit | Nothing to undo; the edit is history now |
|
|
1343
1355
|
| `logic_sandbox_unavailable` | The service could not run step code just now (HTTP 503). Nothing was checked, saved or undone | Try again in a minute |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@basein/runner",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.20",
|
|
4
4
|
"description": "A recording MCP proxy: sits between any MCP client and its MCP servers, executes each call on the client's behalf, and records the run as a reusable BaseIn scenario.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|