@basein/runner 0.3.26 → 0.3.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/investigate.d.ts +13 -0
- package/dist/bin/investigate.js +67 -2
- package/dist/control/server.d.ts +12 -2
- package/dist/control/server.js +90 -15
- package/dist/replay/controller.d.ts +30 -1
- package/dist/replay/controller.js +28 -15
- package/dist/replay/plan.d.ts +41 -0
- package/dist/replay/plan.js +171 -3
- package/dist/replay/reads-only.d.ts +48 -0
- package/dist/replay/reads-only.js +161 -0
- package/package.json +1 -1
|
@@ -102,6 +102,19 @@ export interface Turn {
|
|
|
102
102
|
of?: string;
|
|
103
103
|
reason?: string;
|
|
104
104
|
};
|
|
105
|
+
/**
|
|
106
|
+
* The plan was retired before any step ran (`replay.vetoed`): the model
|
|
107
|
+
* changed something itself first, or its `run_scenario` never reached the
|
|
108
|
+
* runner. `code` from runners after 0.3.26; the reason text always.
|
|
109
|
+
*/
|
|
110
|
+
vetoed?: {
|
|
111
|
+
why?: string;
|
|
112
|
+
code?: string;
|
|
113
|
+
};
|
|
114
|
+
/** A direct plan waited through a call that only reads (`replay.direct_waits`, B2 is11.md). */
|
|
115
|
+
waited?: {
|
|
116
|
+
tool?: string;
|
|
117
|
+
};
|
|
105
118
|
/** `replay.*_failed`, `replay.arm_failed`, `replay.diverge`, `replay.handover` … */
|
|
106
119
|
incidents: JournalEntry[];
|
|
107
120
|
events: JournalEntry[];
|
package/dist/bin/investigate.js
CHANGED
|
@@ -164,6 +164,12 @@ export function turnsFromJournal(entries) {
|
|
|
164
164
|
case "run.repeat":
|
|
165
165
|
turn.repeat = { runId: str(e.run), of: str(e.of), reason: str(e.reason) };
|
|
166
166
|
break;
|
|
167
|
+
case "replay.vetoed":
|
|
168
|
+
turn.vetoed ??= { why: str(e.why), code: str(e.code) };
|
|
169
|
+
break;
|
|
170
|
+
case "replay.direct_waits":
|
|
171
|
+
turn.waited ??= { tool: str(e.tool) };
|
|
172
|
+
break;
|
|
167
173
|
case "run.finish":
|
|
168
174
|
turn.finishedAt = e.at;
|
|
169
175
|
turn.finish = {
|
|
@@ -340,6 +346,46 @@ function readingDecline(code, why) {
|
|
|
340
346
|
}
|
|
341
347
|
return undefined;
|
|
342
348
|
}
|
|
349
|
+
/**
|
|
350
|
+
* Why a plan that armed ran no step (`replay.vetoed`). The recording page
|
|
351
|
+
* once said "the runner did not say why" of such a turn, and guessed wrong
|
|
352
|
+
* (B2 is11.md): the runner had said, here, and only here.
|
|
353
|
+
*/
|
|
354
|
+
function vetoFinding(turn, v) {
|
|
355
|
+
const why = (v.why ?? "").replace(/ — the plan is retired without running any step$/, "");
|
|
356
|
+
const scenarioId = turn.plan?.scenarioId ?? turn.matched?.scenarioId ?? "<scnId>";
|
|
357
|
+
if (v.code === "run_refused" || (!v.code && /\brun_scenario (was called|failed|answered)\b/.test(why))) {
|
|
358
|
+
return {
|
|
359
|
+
code: "vetoed_run_refused",
|
|
360
|
+
severity: "info",
|
|
361
|
+
title: `The plan was not run: ${why}.`,
|
|
362
|
+
cause: "run_scenario was refused — by the person, or by a deny rule — or it failed before it reached the runner. Refused means refused: the plan ran no step, and the agent did the task.",
|
|
363
|
+
fix: "nothing, if it was refused on purpose. If it failed: `bir doctor` shows whether the bir server runs here, and `bir setup` lets Claude Code run it without asking.",
|
|
364
|
+
evidence: { code: v.code ?? null },
|
|
365
|
+
};
|
|
366
|
+
}
|
|
367
|
+
const tool = /\bfirst call (?:that may change something )?was (\S+), not run_scenario\b/.exec(why)?.[1];
|
|
368
|
+
if (!v.code) {
|
|
369
|
+
// Runners up to 0.3.26: any first call retired the plan, and no reason
|
|
370
|
+
// went with the report.
|
|
371
|
+
return {
|
|
372
|
+
code: "vetoed_first_call",
|
|
373
|
+
severity: "warn",
|
|
374
|
+
title: `The plan armed, and was not run: ${why}.`,
|
|
375
|
+
cause: "this runner retires a direct plan at the agent's first call that is not run_scenario, whatever it is — a file read included. A careful agent looks before it runs a plan that writes, the more so when the note names the recording's values where this request has others (another person, another date), and the replay is lost for it: the steps would have taken this request's values.",
|
|
376
|
+
fix: `update the runner when a version after 0.3.26 is out (\`bir update\`): it keeps the plan through calls that only read, and its note names this request's values in place of the recording's. \`bir scenario show ${scenarioId}\` shows which values the steps read from the request.`,
|
|
377
|
+
evidence: { tool: tool ?? null },
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
return {
|
|
381
|
+
code: "vetoed_changed_first",
|
|
382
|
+
severity: "warn",
|
|
383
|
+
title: `The plan armed, and was not run: ${why}.`,
|
|
384
|
+
cause: `the agent changed something itself (${tool ?? "a call"}) before it called run_scenario. Run after that, the plan could do the same work twice, so it ran no step and the agent did the task. Calls that only read before it would have kept the plan.`,
|
|
385
|
+
fix: `nothing is wrong with the plan. Why the agent went its own way is in its first lines of the turn; \`bir scenario show ${scenarioId}\` shows which values the steps read from the request.`,
|
|
386
|
+
evidence: { code: v.code, tool: tool ?? null },
|
|
387
|
+
};
|
|
388
|
+
}
|
|
343
389
|
export const LOW_SAVING_PCT = 0.3;
|
|
344
390
|
const usd = (v) => `$${v.toFixed(4)}`;
|
|
345
391
|
const pct = (v) => `${(v * 100).toFixed(1)}%`;
|
|
@@ -385,6 +431,17 @@ export function localFindings(turn) {
|
|
|
385
431
|
fix: "Check the incidents below and `bir doctor`; the service section says whether the scenario is ready.",
|
|
386
432
|
});
|
|
387
433
|
}
|
|
434
|
+
if (turn.vetoed)
|
|
435
|
+
out.push(vetoFinding(turn, turn.vetoed));
|
|
436
|
+
else if (turn.waited) {
|
|
437
|
+
out.push({
|
|
438
|
+
code: "direct_waited",
|
|
439
|
+
severity: "info",
|
|
440
|
+
title: `The agent's first call only read (${turn.waited.tool ?? "a read"}), and the plan waited for run_scenario.`,
|
|
441
|
+
cause: "A call that only reads does not retire a direct plan: it ran as the agent wrote it, and the agent was told once that the plan was still ready.",
|
|
442
|
+
fix: "",
|
|
443
|
+
});
|
|
444
|
+
}
|
|
388
445
|
if (turn.plan) {
|
|
389
446
|
const coverage = turn.plan.coverage ?? [];
|
|
390
447
|
const tools = turn.plan.tools ?? [];
|
|
@@ -419,10 +476,16 @@ export function localFindings(turn) {
|
|
|
419
476
|
});
|
|
420
477
|
}
|
|
421
478
|
if (turn.plan.mode === "direct") {
|
|
479
|
+
const steps = turn.plan.steps ?? coverage.length;
|
|
480
|
+
// A plan that armed ran nothing when it was retired first, or the turn
|
|
481
|
+
// reported it not steered: "28 steps ran" of such a turn was false.
|
|
482
|
+
const ranNone = turn.vetoed !== undefined || turn.reported?.outcome === "not_steered";
|
|
422
483
|
out.push({
|
|
423
484
|
code: "direct_mode",
|
|
424
485
|
severity: "info",
|
|
425
|
-
title:
|
|
486
|
+
title: ranNone
|
|
487
|
+
? `The plan armed in direct mode, ${steps} steps on the proxies and the runner, and none of them ran.`
|
|
488
|
+
: `The plan armed in direct mode: ${steps} steps ran on the proxies and the runner, no model in the loop.`,
|
|
426
489
|
cause: "Every step is a wrapped MCP tool this session has a proxy for, or a built-in the runner runs itself.",
|
|
427
490
|
fix: "",
|
|
428
491
|
});
|
|
@@ -814,7 +877,9 @@ export async function investigateCommand(args, deps) {
|
|
|
814
877
|
: t.decision?.verdict === "no-steer"
|
|
815
878
|
? `declined ${t.decision.code ?? ""}`.trim()
|
|
816
879
|
: t.plan
|
|
817
|
-
? `${t.plan.mode} ${t.done?.outcome ?? t.reported?.outcome ?? "armed"}
|
|
880
|
+
? `${t.plan.mode} ${t.done?.outcome ?? t.reported?.outcome ?? "armed"}${
|
|
881
|
+
// Retired before any step ran: the reason, where the runner gave one.
|
|
882
|
+
t.vetoed ? ` (${t.vetoed.code ?? "not run"})` : ""}`
|
|
818
883
|
: "matched";
|
|
819
884
|
const p = savedPct(t);
|
|
820
885
|
const saved = t.reported?.savedUsd != null ? ` saved ${usd(t.reported.savedUsd)}${p != null ? ` (${pct(p)})` : ""}` : "";
|
package/dist/control/server.d.ts
CHANGED
|
@@ -788,6 +788,13 @@ export declare class ControlServer {
|
|
|
788
788
|
* whether the service took it (null while the send is on its way).
|
|
789
789
|
*/
|
|
790
790
|
private readonly toolListsSent;
|
|
791
|
+
/**
|
|
792
|
+
* What each wrapped server says of its tools (MCP `annotations`), by the
|
|
793
|
+
* name the agent calls them: whether a call only looks, for a direct plan
|
|
794
|
+
* waiting through it (B2 is11.md). A server's word beats any guess from the
|
|
795
|
+
* name (replay/reads-only.ts). Kept from the lists proxies report.
|
|
796
|
+
*/
|
|
797
|
+
private readonly toolHints;
|
|
791
798
|
/**
|
|
792
799
|
* `POST /proxy/tools` — a proxy's server listed its tools (`tools/list`;
|
|
793
800
|
* the service's autocorrect.md AC12). Its names, descriptions, input
|
|
@@ -1139,8 +1146,11 @@ export declare class ControlServer {
|
|
|
1139
1146
|
private hasDirectCall;
|
|
1140
1147
|
/**
|
|
1141
1148
|
* Retire `state` without running anything: the model's `run_scenario` was
|
|
1142
|
-
* refused or failed
|
|
1143
|
-
* the task — never a
|
|
1149
|
+
* refused or failed, or the model changed something itself first. Books a
|
|
1150
|
+
* baseline sample (`not_steered`) — the agent did the task — never a
|
|
1151
|
+
* divergence. `code` goes with the report, so the recording page can say
|
|
1152
|
+
* why the plan did not run (B2 is11.md: it said "the runner did not say
|
|
1153
|
+
* why", and guessed wrong).
|
|
1144
1154
|
*/
|
|
1145
1155
|
private vetoPlan;
|
|
1146
1156
|
/**
|
package/dist/control/server.js
CHANGED
|
@@ -33,6 +33,7 @@ import { pidAlive } from "./discovery.js";
|
|
|
33
33
|
import { pinCompatible } from "./pin-guard.js";
|
|
34
34
|
import { agentDayOf, answerBeforePrompt, contextForToolUse, intentForToolUse, markTranscriptUsage, promptCount, recentToolResults, sessionTranscriptPath, settledLastAssistantText, settledTextAfterToolUse, settledTurnEnd, thinkingForToolUse, usageBetween, usageSince, } from "./transcript.js";
|
|
35
35
|
import { isHousekeepingCall } from "../record/housekeeping.js";
|
|
36
|
+
import { readsOnlyCall } from "../replay/reads-only.js";
|
|
36
37
|
import { asSent, userText } from "../record/prompt-text.js";
|
|
37
38
|
import { redact, redactText } from "../record/redact.js";
|
|
38
39
|
import { DEFAULT_MAX_STRING_BYTES, serializeCapped, truncateString, truncateStrings, } from "../record/truncate.js";
|
|
@@ -458,6 +459,24 @@ const HEALTH_SESSIONS_MAX = 50;
|
|
|
458
459
|
const HANDBACK_NOTE = "[BaseInstRunner calculated replay] The recorded sequence is no longer being supplied: carry on with the task yourself, choosing each tool call's arguments from what you have seen so far.";
|
|
459
460
|
/** What the model hears once a direct plan it never asked to run is retired. */
|
|
460
461
|
const DIRECT_SKIPPED_NOTE = "[BaseInstRunner calculated replay] The recorded scenario is not being run for this request: carry on with the task yourself.";
|
|
462
|
+
/**
|
|
463
|
+
* What the model hears, once, after its first call that only looks while a
|
|
464
|
+
* direct plan waits for `run_scenario` (B2 is11.md): the call did not cancel
|
|
465
|
+
* the plan, the steps take this request's values — named, where the reading
|
|
466
|
+
* named them — and a change made first would. In is11 the model had checked
|
|
467
|
+
* its notes because the note named the recording's person.
|
|
468
|
+
*/
|
|
469
|
+
function directWaitingNote(values) {
|
|
470
|
+
return [
|
|
471
|
+
"[BaseInstRunner calculated replay] The recorded sequence is still ready for this request:",
|
|
472
|
+
"a call that only reads does not cancel it.",
|
|
473
|
+
values
|
|
474
|
+
? `Its steps take this request's values, not the recording's: ${values}.`
|
|
475
|
+
: "Its steps take this request's values, not the recording's.",
|
|
476
|
+
"Call mcp__bir__run_scenario next, before any call that changes something:",
|
|
477
|
+
"after a change of your own it is not run.",
|
|
478
|
+
].join("\n");
|
|
479
|
+
}
|
|
461
480
|
/** What a second `run_scenario` for a plan that is already running is told. */
|
|
462
481
|
const ALREADY_RUNNING = "this scenario is already running for this turn — its results come back on the first run_scenario call; do not call it again or repeat its steps";
|
|
463
482
|
/** Request bodies past this are read and discarded, never buffered. */
|
|
@@ -2511,6 +2530,13 @@ export class ControlServer {
|
|
|
2511
2530
|
* whether the service took it (null while the send is on its way).
|
|
2512
2531
|
*/
|
|
2513
2532
|
toolListsSent = new Map();
|
|
2533
|
+
/**
|
|
2534
|
+
* What each wrapped server says of its tools (MCP `annotations`), by the
|
|
2535
|
+
* name the agent calls them: whether a call only looks, for a direct plan
|
|
2536
|
+
* waiting through it (B2 is11.md). A server's word beats any guess from the
|
|
2537
|
+
* name (replay/reads-only.ts). Kept from the lists proxies report.
|
|
2538
|
+
*/
|
|
2539
|
+
toolHints = new Map();
|
|
2514
2540
|
/**
|
|
2515
2541
|
* `POST /proxy/tools` — a proxy's server listed its tools (`tools/list`;
|
|
2516
2542
|
* the service's autocorrect.md AC12). Its names, descriptions, input
|
|
@@ -2530,12 +2556,26 @@ export class ControlServer {
|
|
|
2530
2556
|
if (this.isSessionOff(boundTo))
|
|
2531
2557
|
return { ok: true, off: true };
|
|
2532
2558
|
const serverName = typeof body.serverName === "string" ? body.serverName.trim().slice(0, 200) : "";
|
|
2533
|
-
if (!serverName || !Array.isArray(body.tools)
|
|
2559
|
+
if (!serverName || !Array.isArray(body.tools))
|
|
2534
2560
|
return { ok: true };
|
|
2535
2561
|
const tools = toolListForService(serverName, body.tools);
|
|
2536
2562
|
// A server still starting may list nothing: that replaces no list.
|
|
2537
2563
|
if (tools.length === 0)
|
|
2538
2564
|
return { ok: true };
|
|
2565
|
+
for (const t of tools) {
|
|
2566
|
+
if (t.annotations)
|
|
2567
|
+
this.toolHints.set(t.qualifiedName, t.annotations);
|
|
2568
|
+
else
|
|
2569
|
+
this.toolHints.delete(t.qualifiedName);
|
|
2570
|
+
}
|
|
2571
|
+
while (this.toolHints.size > 4_096) {
|
|
2572
|
+
const oldest = this.toolHints.keys().next().value;
|
|
2573
|
+
if (oldest === undefined)
|
|
2574
|
+
break;
|
|
2575
|
+
this.toolHints.delete(oldest);
|
|
2576
|
+
}
|
|
2577
|
+
if (!isToolListReporter(this.recorder))
|
|
2578
|
+
return { ok: true };
|
|
2539
2579
|
const project = this.projectKeyHere();
|
|
2540
2580
|
const key = `${accountOf(this.recorder) ?? ""}\u0000${project}\u0000${serverName}`;
|
|
2541
2581
|
const hash = createHash("sha256").update(JSON.stringify(tools)).digest("hex");
|
|
@@ -3837,6 +3877,8 @@ export class ControlServer {
|
|
|
3837
3877
|
const live = this.livePlan(run);
|
|
3838
3878
|
/** `run_scenario` for this session's own announced direct plan: allowed, see below. */
|
|
3839
3879
|
let allowDirect = false;
|
|
3880
|
+
/** A call that only looks, which a direct plan waits through (B2 is11.md): it runs as written. */
|
|
3881
|
+
let waitedRead = false;
|
|
3840
3882
|
if (this.replay.isDirectTool(toolName)) {
|
|
3841
3883
|
// `run_scenario` with no live plan does nothing but answer "no scenario
|
|
3842
3884
|
// is armed". Recorded, that answer is a failed tool call, and a recording
|
|
@@ -3865,19 +3907,43 @@ export class ControlServer {
|
|
|
3865
3907
|
// refused (the user said no, or a rule denied it) or it failed on the way.
|
|
3866
3908
|
// Refused means refused. The plan retires without running anything — it
|
|
3867
3909
|
// must not come back as a divergence that runs every step unattended.
|
|
3868
|
-
this.vetoPlan(run, live, "run_scenario was called but never reached the runner — refused or failed");
|
|
3910
|
+
this.vetoPlan(run, live, "run_scenario was called but never reached the runner — refused or failed", "run_refused");
|
|
3869
3911
|
}
|
|
3870
3912
|
else if (live?.mode === "direct" && !live.directRequested && !live.running && !this.runningPlans.has(live)) {
|
|
3871
|
-
// A direct plan whose
|
|
3872
|
-
// asked for it.
|
|
3873
|
-
// — the bir server failed to start, was
|
|
3874
|
-
// nothing —
|
|
3875
|
-
//
|
|
3913
|
+
// A direct plan whose next call is not `run_scenario`, and which never
|
|
3914
|
+
// asked for it. Nothing of the plan ever runs in this hook in its place:
|
|
3915
|
+
// when the model has no such tool — the bir server failed to start, was
|
|
3916
|
+
// disabled in /mcp, or lists nothing — a divergence would execute every
|
|
3917
|
+
// step, writes included, with no permission prompt, and deny the
|
|
3876
3918
|
// model's call: a broken delivery tool turned into an unattended run.
|
|
3877
|
-
//
|
|
3878
|
-
//
|
|
3879
|
-
|
|
3880
|
-
|
|
3919
|
+
//
|
|
3920
|
+
// A call that only looks does not retire the plan (B2 is11.md): it runs
|
|
3921
|
+
// as written, and the plan waits for `run_scenario`, told to the model
|
|
3922
|
+
// once on this call's PostToolUse. In is11 the model checked its own
|
|
3923
|
+
// notes first, because the note named the recording's person, and
|
|
3924
|
+
// losing the plan there cost a whole agent run for a value the steps
|
|
3925
|
+
// take from the request anyway. The first call that may change
|
|
3926
|
+
// something retires it, running nothing: run after the model's own
|
|
3927
|
+
// change, the plan could do the same work twice. That call runs as
|
|
3928
|
+
// written, and the model hears on its PostToolUse that the task is its
|
|
3929
|
+
// own.
|
|
3930
|
+
if (readsOnlyCall(toolName, modelArgs, this.toolHints.get(toolName))) {
|
|
3931
|
+
waitedRead = true;
|
|
3932
|
+
live.readsWaited = (live.readsWaited ?? 0) + 1;
|
|
3933
|
+
if (live.readsWaited === 1) {
|
|
3934
|
+
run.handback = directWaitingNote(live.plan?.requestValues());
|
|
3935
|
+
logLine("replay.direct_waits", {
|
|
3936
|
+
run: run.runId,
|
|
3937
|
+
scenario: live.scenarioId ?? undefined,
|
|
3938
|
+
tool: toolName,
|
|
3939
|
+
why: "the model's first call only reads — it runs as written, and the plan waits for run_scenario",
|
|
3940
|
+
});
|
|
3941
|
+
}
|
|
3942
|
+
}
|
|
3943
|
+
else {
|
|
3944
|
+
this.vetoPlan(run, live, `the model's first call that may change something was ${toolName}, not run_scenario${live.readsWaited ? ` (after ${live.readsWaited} that only read)` : ""}`, "changed_first");
|
|
3945
|
+
run.handback = DIRECT_SKIPPED_NOTE;
|
|
3946
|
+
}
|
|
3881
3947
|
}
|
|
3882
3948
|
const agentId = payload.agent_id ?? session.agentId;
|
|
3883
3949
|
// The real intent, read from every line sharing this message's id
|
|
@@ -3928,6 +3994,7 @@ export class ControlServer {
|
|
|
3928
3994
|
});
|
|
3929
3995
|
const action = await this.replay.preTool(steering, toolName, toolUseId, modelArgs, {
|
|
3930
3996
|
signal: answeredLate.signal,
|
|
3997
|
+
readsOnly: waitedRead && steering === live,
|
|
3931
3998
|
});
|
|
3932
3999
|
if (overran) {
|
|
3933
4000
|
// The host already has its answer. A divergence was answered with its
|
|
@@ -4469,20 +4536,28 @@ export class ControlServer {
|
|
|
4469
4536
|
}
|
|
4470
4537
|
/**
|
|
4471
4538
|
* Retire `state` without running anything: the model's `run_scenario` was
|
|
4472
|
-
* refused or failed
|
|
4473
|
-
* the task — never a
|
|
4539
|
+
* refused or failed, or the model changed something itself first. Books a
|
|
4540
|
+
* baseline sample (`not_steered`) — the agent did the task — never a
|
|
4541
|
+
* divergence. `code` goes with the report, so the recording page can say
|
|
4542
|
+
* why the plan did not run (B2 is11.md: it said "the runner did not say
|
|
4543
|
+
* why", and guessed wrong).
|
|
4474
4544
|
*/
|
|
4475
|
-
vetoPlan(run, state, why) {
|
|
4545
|
+
vetoPlan(run, state, why, code) {
|
|
4476
4546
|
if (state.retired)
|
|
4477
4547
|
return;
|
|
4478
4548
|
state.retired = true;
|
|
4479
4549
|
state.retiredAt = Date.now();
|
|
4480
4550
|
state.pinned.clear();
|
|
4481
4551
|
state.skipNote = undefined;
|
|
4552
|
+
// A continuation has a reason of its own for this, `not_taken`, which the
|
|
4553
|
+
// page words for a plan that follows a hand-over (restoreParkedFragment).
|
|
4554
|
+
if (code && !state.declined && !state.continuation)
|
|
4555
|
+
state.declined = code;
|
|
4482
4556
|
this.directCalls = this.directCalls.filter((c) => c.state !== state);
|
|
4483
4557
|
logLine("replay.vetoed", {
|
|
4484
4558
|
run: run.runId,
|
|
4485
4559
|
scenario: state.scenarioId ?? undefined,
|
|
4560
|
+
code,
|
|
4486
4561
|
why: `${why} — the plan is retired without running any step`,
|
|
4487
4562
|
});
|
|
4488
4563
|
}
|
|
@@ -5643,7 +5718,7 @@ export class ControlServer {
|
|
|
5643
5718
|
if (!this.runningPlans.has(waiting.state)) {
|
|
5644
5719
|
this.vetoPlan(waiting.run, waiting.state, failed
|
|
5645
5720
|
? "run_scenario failed before it reached the runner"
|
|
5646
|
-
: "run_scenario answered without ever reaching the runner");
|
|
5721
|
+
: "run_scenario answered without ever reaching the runner", "run_refused");
|
|
5647
5722
|
}
|
|
5648
5723
|
}
|
|
5649
5724
|
}
|
|
@@ -92,6 +92,13 @@ export interface PreToolOptions {
|
|
|
92
92
|
* route's deadline guard spoke first. No further step is started.
|
|
93
93
|
*/
|
|
94
94
|
signal?: AbortSignal;
|
|
95
|
+
/**
|
|
96
|
+
* The call only looks and changes nothing (replay/reads-only.ts), as the
|
|
97
|
+
* control server judged it with what the tool's server says of it. A direct
|
|
98
|
+
* plan the model has not asked for yet lets such a call through and waits
|
|
99
|
+
* for `run_scenario` (B2 is11.md); nothing runs in its place.
|
|
100
|
+
*/
|
|
101
|
+
readsOnly?: boolean;
|
|
95
102
|
}
|
|
96
103
|
/**
|
|
97
104
|
* What the caller knows about how a pinned call ended, beyond its error text
|
|
@@ -514,6 +521,12 @@ export interface ReplayState {
|
|
|
514
521
|
running?: Promise<RunArmedResult>;
|
|
515
522
|
/** The model called `run_scenario` for this plan (its `PreToolUse` was seen). */
|
|
516
523
|
directRequested?: boolean;
|
|
524
|
+
/**
|
|
525
|
+
* Calls that only look, made while this direct plan waited for
|
|
526
|
+
* `run_scenario` (B2 is11.md): each ran as the model wrote it, and the plan
|
|
527
|
+
* stayed announced. The first is told the plan still waits, once.
|
|
528
|
+
*/
|
|
529
|
+
readsWaited?: number;
|
|
517
530
|
/**
|
|
518
531
|
* The direct run's caller went away mid-run — Esc on `run_scenario`, or its
|
|
519
532
|
* client timing out. The turn it cut short is no measurement of the task, so
|
|
@@ -677,7 +690,23 @@ export type DeclineCode = "replay_disabled" | "not_ready" | "known_bad_first_ste
|
|
|
677
690
|
* failed on this request, so none of its steps could be computed. Set by
|
|
678
691
|
* the control server.
|
|
679
692
|
*/
|
|
680
|
-
| "params_failed"
|
|
693
|
+
| "params_failed"
|
|
694
|
+
/**
|
|
695
|
+
* A direct plan announced to the model, which made a call that may change
|
|
696
|
+
* something before it called `run_scenario`: the plan retired running
|
|
697
|
+
* nothing, because run after the model's own change it could do the same
|
|
698
|
+
* work twice. Calls that only look before it do not count (B2 is11.md).
|
|
699
|
+
* Set by the control server (runners after 0.3.26; up to it, such a plan
|
|
700
|
+
* was retired at any first call, and reported no reason).
|
|
701
|
+
*/
|
|
702
|
+
| "changed_first"
|
|
703
|
+
/**
|
|
704
|
+
* A direct plan whose `run_scenario` call never reached the runner: the
|
|
705
|
+
* person refused it, a rule denied it, or it failed on the way. Refused
|
|
706
|
+
* means refused: the plan retired running nothing. Set by the control
|
|
707
|
+
* server (runners after 0.3.26; up to it, reported with no reason).
|
|
708
|
+
*/
|
|
709
|
+
| "run_refused";
|
|
681
710
|
/**
|
|
682
711
|
* What a continuation continues (the service's fallbk.md Part 3): the plan
|
|
683
712
|
* that handed over, the step it stopped in front of, and the recording of the
|
|
@@ -700,21 +700,24 @@ export class ReplayController {
|
|
|
700
700
|
stopAt,
|
|
701
701
|
subTask: isSubTask(state),
|
|
702
702
|
});
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
703
|
+
// A steer directive shows the arguments the steps will run with, so the
|
|
704
|
+
// call the model writes is one the pin guard accepts (plan.ts,
|
|
705
|
+
// `steeringDirective`). A direct one says which of this request's values
|
|
706
|
+
// the steps take, and puts this request's values where the recording's
|
|
707
|
+
// stand in its intent (B2 is11.md): "set Olga up" under a request for
|
|
708
|
+
// Carlos made a careful model check first, and that check retired the
|
|
709
|
+
// plan. Both need the parameters settled — which costs nothing when the
|
|
710
|
+
// derivation has already answered: gate 7 waited for it, or there was
|
|
711
|
+
// nothing to derive. One turn of the loop lets an answer that is already
|
|
712
|
+
// in hand through; one still on its way is not waited for here, and the
|
|
713
|
+
// directive leaves the values out (a direct one names each recorded value
|
|
714
|
+
// by its role instead). A `paramsLogic` that throws is not judged here:
|
|
715
|
+
// the first call that needs the parameters meets the same failure, as
|
|
716
|
+
// before.
|
|
717
|
+
if (!deriveAnswered)
|
|
718
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
719
|
+
if (deriveAnswered)
|
|
720
|
+
await state.plan.ready().catch(() => undefined);
|
|
718
721
|
state.mode = mode;
|
|
719
722
|
state.stepsPlanned = steps.length;
|
|
720
723
|
// The chain this plan was served at: what its own calls are marked with
|
|
@@ -1055,6 +1058,16 @@ export class ReplayController {
|
|
|
1055
1058
|
});
|
|
1056
1059
|
return { kind: "passthrough" };
|
|
1057
1060
|
}
|
|
1061
|
+
// A CALL THAT ONLY LOOKS DOES NOT END A DIRECT PLAN (B2 is11.md). Asked
|
|
1062
|
+
// for `run_scenario` first, the model looked at something first instead —
|
|
1063
|
+
// in is11 its own notes, after the note had named another request's
|
|
1064
|
+
// person. Read as a divergence, this call would run every step here,
|
|
1065
|
+
// unattended; read as the model leaving the plan, it retired a plan that
|
|
1066
|
+
// would have done the task right. It is neither: it runs as written,
|
|
1067
|
+
// nothing runs in its place, and the plan waits for `run_scenario`. The
|
|
1068
|
+
// control server retires it at the first call that may change something.
|
|
1069
|
+
if (state.mode === "direct" && hook.readsOnly === true)
|
|
1070
|
+
return { kind: "passthrough" };
|
|
1058
1071
|
// Everything below — the wait for the parameters, and a divergence that
|
|
1059
1072
|
// executes the rest of the plan — happens inside this one hook, which the
|
|
1060
1073
|
// host waits on for a bounded time. One deadline covers all of it — and
|
package/dist/replay/plan.d.ts
CHANGED
|
@@ -514,6 +514,47 @@ export declare class ScenarioReplayPlan {
|
|
|
514
514
|
* body gets at most `bodyMs` ({@link PREVIEW_LOGIC_MS}) of it.
|
|
515
515
|
*/
|
|
516
516
|
private previewArguments;
|
|
517
|
+
/**
|
|
518
|
+
* This request's value of `key`, as a directive may show it, or undefined
|
|
519
|
+
* (B2 is11.md): once the reading has settled, a plain value the reading
|
|
520
|
+
* took from the request — never a mark, and never a value a step works out
|
|
521
|
+
* (`from: "step"`): before the run nobody has that one, and the recorded one
|
|
522
|
+
* is another request's (issue 1212).
|
|
523
|
+
*/
|
|
524
|
+
private shownValueOf;
|
|
525
|
+
/**
|
|
526
|
+
* A text written from the recording — the intent, a step's reasoning — put
|
|
527
|
+
* right for this request, for a directive to show (B2 is11.md).
|
|
528
|
+
*
|
|
529
|
+
* Both name the recording's values: "set Olga up", "moves each lead to
|
|
530
|
+
* Olga". Shown beside a request for Carlos, the name tells the model the
|
|
531
|
+
* plan is for someone else. A careful model then checks before it runs
|
|
532
|
+
* anything, and a direct plan's first call that was not `run_scenario`
|
|
533
|
+
* retired it — so the replay was lost on exactly the turn replay is for: the
|
|
534
|
+
* same task with a new value. The steps never read the text: they take
|
|
535
|
+
* Carlos from the request.
|
|
536
|
+
*
|
|
537
|
+
* Each recorded value of a parameter found in the text, as a whole word, is
|
|
538
|
+
* replaced by this request's own value where the plan has one it may show
|
|
539
|
+
* ({@link shownValueOf}), and else by the parameter's role (`<assignee
|
|
540
|
+
* name>`): a target the request did not name, a value a step works out, or
|
|
541
|
+
* any value while the reading has not settled. A value the same as the
|
|
542
|
+
* recorded one stays as it is. One pass, longest value first, so nothing put
|
|
543
|
+
* in is replaced again.
|
|
544
|
+
*/
|
|
545
|
+
forThisRequest(text: string): string;
|
|
546
|
+
/** {@link forThisRequest} as one function, built once for a directive's many texts. */
|
|
547
|
+
private rewriter;
|
|
548
|
+
/**
|
|
549
|
+
* This request's own values, for the direct directive's line that says the
|
|
550
|
+
* steps take them (B2 is11.md): what the reading took from the request.
|
|
551
|
+
* The values the request names in words come first — a name, a title: what
|
|
552
|
+
* a reader holds the note against — those unlike the recording's first;
|
|
553
|
+
* then the values worked out from the day ("tomorrow" as a date). Targets
|
|
554
|
+
* only: never a setting, never a value a step works out, never a mark.
|
|
555
|
+
* Undefined until the reading has settled, or when it named none.
|
|
556
|
+
*/
|
|
557
|
+
requestValues(): string | undefined;
|
|
517
558
|
/**
|
|
518
559
|
* The directive injected via `UserPromptSubmit`'s `additionalContext`.
|
|
519
560
|
*
|
package/dist/replay/plan.js
CHANGED
|
@@ -132,6 +132,47 @@ function clipReasoning(reasoning) {
|
|
|
132
132
|
const first = (reasoning ?? "").trim().split(/\r?\n/, 1)[0]?.trim() ?? "";
|
|
133
133
|
return first.length > MAX_REASONING_CHARS ? `${first.slice(0, MAX_REASONING_CHARS)}…` : first;
|
|
134
134
|
}
|
|
135
|
+
// ── the recording's values in a directive's text (B2 is11.md) ────────────────
|
|
136
|
+
/**
|
|
137
|
+
* A recorded value as a text may name it: a string or a number at least this
|
|
138
|
+
* long — a one-character sample ("5", "a") is found in too many words — and
|
|
139
|
+
* no longer than a name or a title is.
|
|
140
|
+
*/
|
|
141
|
+
const RECORDED_VALUE_MIN = 2;
|
|
142
|
+
const RECORDED_VALUE_MAX = 200;
|
|
143
|
+
/** One of this request's values, as a directive shows it: this long at most. */
|
|
144
|
+
const SHOWN_VALUE_MAX = 60;
|
|
145
|
+
/** This many of this request's values at most on a direct directive's values line… */
|
|
146
|
+
const SHOWN_VALUES = 4;
|
|
147
|
+
/** …and this long at most, all of them. */
|
|
148
|
+
const SHOWN_VALUES_CHARS = 200;
|
|
149
|
+
/** The recorded value `value` as a text may name it, or undefined. */
|
|
150
|
+
function recordedText(value) {
|
|
151
|
+
const text = typeof value === "number" && Number.isFinite(value)
|
|
152
|
+
? String(value)
|
|
153
|
+
: typeof value === "string"
|
|
154
|
+
? value.trim()
|
|
155
|
+
: undefined;
|
|
156
|
+
return text !== undefined && text.length >= RECORDED_VALUE_MIN && text.length <= RECORDED_VALUE_MAX ? text : undefined;
|
|
157
|
+
}
|
|
158
|
+
/** A value of this request's as plain text — never a mark — or undefined. */
|
|
159
|
+
function valueText(value) {
|
|
160
|
+
let text;
|
|
161
|
+
if (typeof value === "boolean")
|
|
162
|
+
text = String(value);
|
|
163
|
+
else if (typeof value === "number" && Number.isFinite(value))
|
|
164
|
+
text = String(value);
|
|
165
|
+
else if (typeof value === "string")
|
|
166
|
+
text = value.replace(/\s+/g, " ").trim();
|
|
167
|
+
if (!text || markedKey(text) !== undefined)
|
|
168
|
+
return undefined;
|
|
169
|
+
return text;
|
|
170
|
+
}
|
|
171
|
+
const clipValue = (text) => text.length > SHOWN_VALUE_MAX ? `${text.slice(0, SHOWN_VALUE_MAX)}…` : text;
|
|
172
|
+
const sameText = (a, b) => a.toLowerCase() === b.toLowerCase();
|
|
173
|
+
const escapeRegExp = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
174
|
+
/** A parameter's role, for a value a text must not name: `assignee_name` → `<assignee name>`. */
|
|
175
|
+
const roleOf = (key) => `<${key.replace(/[_-]+/g, " ").trim() || key}>`;
|
|
135
176
|
/** A step's computed arguments, as a steer directive shows them: this long at most. */
|
|
136
177
|
const MAX_DIRECTIVE_ARGS_CHARS = 240;
|
|
137
178
|
/**
|
|
@@ -854,6 +895,120 @@ export class ScenarioReplayPlan {
|
|
|
854
895
|
}
|
|
855
896
|
return out;
|
|
856
897
|
}
|
|
898
|
+
/**
|
|
899
|
+
* This request's value of `key`, as a directive may show it, or undefined
|
|
900
|
+
* (B2 is11.md): once the reading has settled, a plain value the reading
|
|
901
|
+
* took from the request — never a mark, and never a value a step works out
|
|
902
|
+
* (`from: "step"`): before the run nobody has that one, and the recorded one
|
|
903
|
+
* is another request's (issue 1212).
|
|
904
|
+
*/
|
|
905
|
+
shownValueOf(key, p) {
|
|
906
|
+
if (!this.paramsSettled || p?.from === "step")
|
|
907
|
+
return undefined;
|
|
908
|
+
return valueText(this.derivedParams[key]);
|
|
909
|
+
}
|
|
910
|
+
/**
|
|
911
|
+
* A text written from the recording — the intent, a step's reasoning — put
|
|
912
|
+
* right for this request, for a directive to show (B2 is11.md).
|
|
913
|
+
*
|
|
914
|
+
* Both name the recording's values: "set Olga up", "moves each lead to
|
|
915
|
+
* Olga". Shown beside a request for Carlos, the name tells the model the
|
|
916
|
+
* plan is for someone else. A careful model then checks before it runs
|
|
917
|
+
* anything, and a direct plan's first call that was not `run_scenario`
|
|
918
|
+
* retired it — so the replay was lost on exactly the turn replay is for: the
|
|
919
|
+
* same task with a new value. The steps never read the text: they take
|
|
920
|
+
* Carlos from the request.
|
|
921
|
+
*
|
|
922
|
+
* Each recorded value of a parameter found in the text, as a whole word, is
|
|
923
|
+
* replaced by this request's own value where the plan has one it may show
|
|
924
|
+
* ({@link shownValueOf}), and else by the parameter's role (`<assignee
|
|
925
|
+
* name>`): a target the request did not name, a value a step works out, or
|
|
926
|
+
* any value while the reading has not settled. A value the same as the
|
|
927
|
+
* recorded one stays as it is. One pass, longest value first, so nothing put
|
|
928
|
+
* in is replaced again.
|
|
929
|
+
*/
|
|
930
|
+
forThisRequest(text) {
|
|
931
|
+
return this.rewriter()(text);
|
|
932
|
+
}
|
|
933
|
+
/** {@link forThisRequest} as one function, built once for a directive's many texts. */
|
|
934
|
+
rewriter() {
|
|
935
|
+
const schema = this.scenario.paramsObject;
|
|
936
|
+
if (!schema || typeof schema !== "object")
|
|
937
|
+
return (text) => text;
|
|
938
|
+
/** Recorded value (lower case) → what goes in its place; null where it stays. */
|
|
939
|
+
const swaps = new Map();
|
|
940
|
+
for (const [key, p] of Object.entries(schema)) {
|
|
941
|
+
const recorded = recordedText(p?.sampleValue);
|
|
942
|
+
if (!recorded)
|
|
943
|
+
continue;
|
|
944
|
+
const now = this.shownValueOf(key, p);
|
|
945
|
+
const swap = now !== undefined && sameText(now, recorded) ? null : now !== undefined ? clipValue(now) : roleOf(key);
|
|
946
|
+
const k = recorded.toLowerCase();
|
|
947
|
+
// Two parameters recorded with one value: a change wins over "stays".
|
|
948
|
+
if (!swaps.has(k) || swaps.get(k) === null)
|
|
949
|
+
swaps.set(k, swap);
|
|
950
|
+
}
|
|
951
|
+
const found = [...swaps.entries()].filter(([, swap]) => swap !== null).map(([k]) => k);
|
|
952
|
+
if (found.length === 0)
|
|
953
|
+
return (text) => text;
|
|
954
|
+
found.sort((a, b) => b.length - a.length);
|
|
955
|
+
let pattern;
|
|
956
|
+
try {
|
|
957
|
+
pattern = new RegExp(`(?<![\\p{L}\\p{N}_])(?:${found.map(escapeRegExp).join("|")})(?![\\p{L}\\p{N}_])`, "giu");
|
|
958
|
+
}
|
|
959
|
+
catch {
|
|
960
|
+
return (text) => text;
|
|
961
|
+
}
|
|
962
|
+
return (text) => (text ? text.replace(pattern, (m) => swaps.get(m.toLowerCase()) ?? m) : text);
|
|
963
|
+
}
|
|
964
|
+
/**
|
|
965
|
+
* This request's own values, for the direct directive's line that says the
|
|
966
|
+
* steps take them (B2 is11.md): what the reading took from the request.
|
|
967
|
+
* The values the request names in words come first — a name, a title: what
|
|
968
|
+
* a reader holds the note against — those unlike the recording's first;
|
|
969
|
+
* then the values worked out from the day ("tomorrow" as a date). Targets
|
|
970
|
+
* only: never a setting, never a value a step works out, never a mark.
|
|
971
|
+
* Undefined until the reading has settled, or when it named none.
|
|
972
|
+
*/
|
|
973
|
+
requestValues() {
|
|
974
|
+
if (!this.paramsSettled)
|
|
975
|
+
return undefined;
|
|
976
|
+
const schema = this.scenario.paramsObject;
|
|
977
|
+
if (!schema || typeof schema !== "object")
|
|
978
|
+
return undefined;
|
|
979
|
+
const items = [];
|
|
980
|
+
let order = 0;
|
|
981
|
+
for (const [key, p] of Object.entries(schema)) {
|
|
982
|
+
order += 1;
|
|
983
|
+
if (p?.kind === "setting")
|
|
984
|
+
continue;
|
|
985
|
+
const now = this.shownValueOf(key, p);
|
|
986
|
+
if (now === undefined)
|
|
987
|
+
continue;
|
|
988
|
+
const recorded = recordedText(p?.sampleValue);
|
|
989
|
+
items.push({
|
|
990
|
+
key,
|
|
991
|
+
text: clipValue(now),
|
|
992
|
+
// Named in the request's own words; an older service sends no `from`.
|
|
993
|
+
said: p?.from === undefined || p.from === "prompt",
|
|
994
|
+
differs: recorded !== undefined && !sameText(now, recorded),
|
|
995
|
+
order,
|
|
996
|
+
});
|
|
997
|
+
}
|
|
998
|
+
items.sort((a, b) => Number(b.said) - Number(a.said) || Number(b.differs) - Number(a.differs) || a.order - b.order);
|
|
999
|
+
const parts = [];
|
|
1000
|
+
let chars = 0;
|
|
1001
|
+
for (const item of items) {
|
|
1002
|
+
if (parts.length >= SHOWN_VALUES)
|
|
1003
|
+
break;
|
|
1004
|
+
const part = `${item.key} ${JSON.stringify(item.text)}`;
|
|
1005
|
+
if (parts.length > 0 && chars + part.length > SHOWN_VALUES_CHARS)
|
|
1006
|
+
break;
|
|
1007
|
+
parts.push(part);
|
|
1008
|
+
chars += part.length + 2;
|
|
1009
|
+
}
|
|
1010
|
+
return parts.length > 0 ? parts.join(", ") : undefined;
|
|
1011
|
+
}
|
|
857
1012
|
/**
|
|
858
1013
|
* The directive injected via `UserPromptSubmit`'s `additionalContext`.
|
|
859
1014
|
*
|
|
@@ -880,9 +1035,14 @@ export class ScenarioReplayPlan {
|
|
|
880
1035
|
// A sub-task plan (armed mid-task) never tells the agent to stop calling
|
|
881
1036
|
// tools: the steps below are a part of the task it is already doing.
|
|
882
1037
|
const subTask = opts?.subTask ?? this.subTask;
|
|
1038
|
+
// The intent and each step's reasoning were written from the recording,
|
|
1039
|
+
// so they name its values ("set Olga up"); shown here, they say this
|
|
1040
|
+
// request's — or the value's role — never another request's (B2 is11.md).
|
|
1041
|
+
const fix = this.rewriter();
|
|
883
1042
|
// The intent is a paragraph the analyser wrote; it is context, not an
|
|
884
1043
|
// instruction, so it is clipped rather than dumped whole into the prompt.
|
|
885
|
-
const
|
|
1044
|
+
const written = fix(this.intent);
|
|
1045
|
+
const intent = written.length > 240 ? `${written.slice(0, 240)}…` : written;
|
|
886
1046
|
if (this.mode === "direct") {
|
|
887
1047
|
// The steps are not listed. The model does not call them, and what the
|
|
888
1048
|
// tool returns names every call and says whether the run was whole or
|
|
@@ -890,9 +1050,17 @@ export class ScenarioReplayPlan {
|
|
|
890
1050
|
// cost the agent up to 2,000 characters in the first call of every turn
|
|
891
1051
|
// a plan was armed for (docs/calculatedReplay.md, "How a bundle reads"),
|
|
892
1052
|
// and the tool's result shows an operator watching the session what ran.
|
|
1053
|
+
//
|
|
1054
|
+
// One line says where the steps' values come from, with this request's
|
|
1055
|
+
// own: a model that met another request's value in the note used to
|
|
1056
|
+
// stop to check, and its check retired the plan (B2 is11.md).
|
|
1057
|
+
const values = this.requestValues();
|
|
893
1058
|
return [
|
|
894
1059
|
"[BaseInstRunner calculated replay] A known-good tool sequence was recorded",
|
|
895
|
-
`for this
|
|
1060
|
+
`for a request like this one${intent ? ` (intent: ${intent})` : ""}.`,
|
|
1061
|
+
values
|
|
1062
|
+
? `Its steps take this request's values, not the recording's: ${values}.`
|
|
1063
|
+
: "Its steps take this request's values, not the recording's.",
|
|
896
1064
|
"",
|
|
897
1065
|
`Call ${directToolName} once, with no arguments, before any other tool. It runs the`,
|
|
898
1066
|
"sequence and returns what each call did.",
|
|
@@ -924,7 +1092,7 @@ export class ScenarioReplayPlan {
|
|
|
924
1092
|
return ` (only when ${t.text ?? "this request needs it"})`;
|
|
925
1093
|
};
|
|
926
1094
|
const listing = (withWhy, argsBefore) => planned.map((s, i) => {
|
|
927
|
-
const why = withWhy ? clipReasoning(s.reasoning) : "";
|
|
1095
|
+
const why = withWhy ? clipReasoning(fix(s.reasoning ?? "")) : "";
|
|
928
1096
|
const head = ` ${i + 1}. ${s.toolName}${onlyWhen(i)}${why ? ` — ${why}` : ""}`;
|
|
929
1097
|
const shown = i < argsBefore ? args[i] : undefined;
|
|
930
1098
|
return shown ? `${head}\n ${shown}` : head;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* reads-only — whether one call of the agent's only looks, and changes nothing.
|
|
3
|
+
*
|
|
4
|
+
* WHY THE RUNNER ASKS (B2 is11.md, 2026-11-11). A direct plan is announced
|
|
5
|
+
* with "call run_scenario before any other tool", and a first call that is
|
|
6
|
+
* not `run_scenario` used to retire it on the spot (control/server.ts). The
|
|
7
|
+
* rule is there for one danger: a model that has no `run_scenario` makes its
|
|
8
|
+
* own calls, and a divergence would then run every step, writes included,
|
|
9
|
+
* inside a hook nobody approves. But the call that retired the plan in is11
|
|
10
|
+
* was the agent reading its own notes file: it had seen another person's name
|
|
11
|
+
* in the note (Olga, the recording's; the request said Carlos) and checked
|
|
12
|
+
* first. The plan would have done the task right — its steps take the person
|
|
13
|
+
* from the request — and the turn cost a full agent run.
|
|
14
|
+
*
|
|
15
|
+
* So a call that only looks no longer retires a direct plan: it runs as the
|
|
16
|
+
* agent wrote it, and the plan waits, still announced, for `run_scenario`.
|
|
17
|
+
* The first call that may change something still retires it, running nothing
|
|
18
|
+
* (and nothing ever runs in a hook in its place): a plan run after the agent
|
|
19
|
+
* changed something itself could do the same work twice.
|
|
20
|
+
*
|
|
21
|
+
* WHEN IN DOUBT, IT IS A CHANGE. A call judged "only looks" that does change
|
|
22
|
+
* something leaves a plan armed that could then repeat that change; a change
|
|
23
|
+
* judged a change only ends a replay early, as every call did before. So:
|
|
24
|
+
*
|
|
25
|
+
* - Claude Code's built-ins by name: the reading ones only look, every other
|
|
26
|
+
* built-in (a sub-agent, an edit, an unknown new tool) is a change;
|
|
27
|
+
* - `Bash` by its command ({@link commandReadsOnly}), and `PowerShell` never;
|
|
28
|
+
* - an MCP tool by what its own server says of it (`annotations`), else by
|
|
29
|
+
* its name: a reading word and no writing word.
|
|
30
|
+
*
|
|
31
|
+
* The same rule as the service's `readsOnly` (correctionCheck.ts, AC13), kept
|
|
32
|
+
* in step by hand, stricter where a wrong "only looks" costs more here: a
|
|
33
|
+
* command substitution, a process substitution or an `awk` `system(` makes a
|
|
34
|
+
* command a change.
|
|
35
|
+
*/
|
|
36
|
+
import type { ToolAnnotations } from "../record/recorder.js";
|
|
37
|
+
/**
|
|
38
|
+
* Whether this call only looks. `toolInput` is the hook's `tool_input` (only a
|
|
39
|
+
* `Bash` call's `command` is read from it); `hints` is what the tool's own
|
|
40
|
+
* server listed for it, when it listed anything.
|
|
41
|
+
*/
|
|
42
|
+
export declare function readsOnlyCall(toolName: string, toolInput?: unknown, hints?: ToolAnnotations): boolean;
|
|
43
|
+
/**
|
|
44
|
+
* Whether a shell command only looks: every part of it is a looking command,
|
|
45
|
+
* nothing is written to a file, and nothing runs code it does not show.
|
|
46
|
+
*/
|
|
47
|
+
export declare function commandReadsOnly(command: string): boolean;
|
|
48
|
+
//# sourceMappingURL=reads-only.d.ts.map
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* reads-only — whether one call of the agent's only looks, and changes nothing.
|
|
3
|
+
*
|
|
4
|
+
* WHY THE RUNNER ASKS (B2 is11.md, 2026-11-11). A direct plan is announced
|
|
5
|
+
* with "call run_scenario before any other tool", and a first call that is
|
|
6
|
+
* not `run_scenario` used to retire it on the spot (control/server.ts). The
|
|
7
|
+
* rule is there for one danger: a model that has no `run_scenario` makes its
|
|
8
|
+
* own calls, and a divergence would then run every step, writes included,
|
|
9
|
+
* inside a hook nobody approves. But the call that retired the plan in is11
|
|
10
|
+
* was the agent reading its own notes file: it had seen another person's name
|
|
11
|
+
* in the note (Olga, the recording's; the request said Carlos) and checked
|
|
12
|
+
* first. The plan would have done the task right — its steps take the person
|
|
13
|
+
* from the request — and the turn cost a full agent run.
|
|
14
|
+
*
|
|
15
|
+
* So a call that only looks no longer retires a direct plan: it runs as the
|
|
16
|
+
* agent wrote it, and the plan waits, still announced, for `run_scenario`.
|
|
17
|
+
* The first call that may change something still retires it, running nothing
|
|
18
|
+
* (and nothing ever runs in a hook in its place): a plan run after the agent
|
|
19
|
+
* changed something itself could do the same work twice.
|
|
20
|
+
*
|
|
21
|
+
* WHEN IN DOUBT, IT IS A CHANGE. A call judged "only looks" that does change
|
|
22
|
+
* something leaves a plan armed that could then repeat that change; a change
|
|
23
|
+
* judged a change only ends a replay early, as every call did before. So:
|
|
24
|
+
*
|
|
25
|
+
* - Claude Code's built-ins by name: the reading ones only look, every other
|
|
26
|
+
* built-in (a sub-agent, an edit, an unknown new tool) is a change;
|
|
27
|
+
* - `Bash` by its command ({@link commandReadsOnly}), and `PowerShell` never;
|
|
28
|
+
* - an MCP tool by what its own server says of it (`annotations`), else by
|
|
29
|
+
* its name: a reading word and no writing word.
|
|
30
|
+
*
|
|
31
|
+
* The same rule as the service's `readsOnly` (correctionCheck.ts, AC13), kept
|
|
32
|
+
* in step by hand, stricter where a wrong "only looks" costs more here: a
|
|
33
|
+
* command substitution, a process substitution or an `awk` `system(` makes a
|
|
34
|
+
* command a change.
|
|
35
|
+
*/
|
|
36
|
+
/** Claude Code's built-ins that only read. `ToolSearch` and `TodoWrite` are housekeeping (record/housekeeping.ts). */
|
|
37
|
+
const READ_BUILTINS = new Set([
|
|
38
|
+
"Read",
|
|
39
|
+
"Grep",
|
|
40
|
+
"Glob",
|
|
41
|
+
"LS",
|
|
42
|
+
"WebFetch",
|
|
43
|
+
"WebSearch",
|
|
44
|
+
"NotebookRead",
|
|
45
|
+
"TaskOutput",
|
|
46
|
+
"BashOutput",
|
|
47
|
+
// The conversation's own: a question to the person, plan mode in or out.
|
|
48
|
+
// Nothing outside the conversation changes, and the plan may still be
|
|
49
|
+
// what the person says yes to.
|
|
50
|
+
"AskUserQuestion",
|
|
51
|
+
"EnterPlanMode",
|
|
52
|
+
"ExitPlanMode",
|
|
53
|
+
]);
|
|
54
|
+
// Not `query`, `select` or `check`: a database query, a selected option, a
|
|
55
|
+
// check-in can change something.
|
|
56
|
+
const READ_WORDS = new Set([
|
|
57
|
+
"list", "get", "search", "find", "read", "fetch", "lookup", "count", "view", "show",
|
|
58
|
+
"describe", "retrieve", "browse", "peek", "inspect", "snapshot",
|
|
59
|
+
]);
|
|
60
|
+
/**
|
|
61
|
+
* A word that may change something: a name holding one writes, whatever else
|
|
62
|
+
* it says (`get_or_create_contact`, `find_and_replace`). Some are nouns too
|
|
63
|
+
* (`get_schedule`): such a look is then taken for a change, which only ends a
|
|
64
|
+
* replay early, never lets a change through.
|
|
65
|
+
*/
|
|
66
|
+
const WRITE_WORDS = new Set([
|
|
67
|
+
"create", "add", "set", "update", "upsert", "insert", "put", "post", "send", "delete",
|
|
68
|
+
"remove", "mark", "replace", "archive", "move", "assign", "book", "cancel", "write",
|
|
69
|
+
"edit", "modify", "save", "submit", "approve", "reject", "close", "open", "merge",
|
|
70
|
+
"push", "publish", "upload", "invite", "schedule", "reply", "forward", "transfer",
|
|
71
|
+
"pay", "refund", "notify", "run", "execute", "exec", "apply", "reset", "start", "stop",
|
|
72
|
+
"kill", "enable", "disable", "grant", "revoke", "rename", "tag", "label", "comment",
|
|
73
|
+
]);
|
|
74
|
+
/** A tool name's words: `getJiraIssue` → get, jira, issue; `calendar_search_events` → calendar, search, events. */
|
|
75
|
+
function nameWords(name) {
|
|
76
|
+
return name
|
|
77
|
+
.replace(/([a-z\d])([A-Z])/g, "$1 $2")
|
|
78
|
+
.replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
|
|
79
|
+
.toLowerCase()
|
|
80
|
+
.split(/[^a-z\d]+/)
|
|
81
|
+
.filter(Boolean);
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Whether this call only looks. `toolInput` is the hook's `tool_input` (only a
|
|
85
|
+
* `Bash` call's `command` is read from it); `hints` is what the tool's own
|
|
86
|
+
* server listed for it, when it listed anything.
|
|
87
|
+
*/
|
|
88
|
+
export function readsOnlyCall(toolName, toolInput, hints) {
|
|
89
|
+
if (toolName === "Bash") {
|
|
90
|
+
const command = toolInput?.command;
|
|
91
|
+
return typeof command === "string" && commandReadsOnly(command);
|
|
92
|
+
}
|
|
93
|
+
if (READ_BUILTINS.has(toolName))
|
|
94
|
+
return true;
|
|
95
|
+
// Every other built-in — Write, Edit, PowerShell, a sub-agent, one added
|
|
96
|
+
// after this list was written — may change something.
|
|
97
|
+
if (!toolName.startsWith("mcp__"))
|
|
98
|
+
return false;
|
|
99
|
+
// A server's word beats any guess from the name.
|
|
100
|
+
if (hints?.readOnlyHint === false || hints?.destructiveHint === true)
|
|
101
|
+
return false;
|
|
102
|
+
if (hints?.readOnlyHint === true)
|
|
103
|
+
return true;
|
|
104
|
+
const words = nameWords(toolName.replace(/^mcp__.+?__/, ""));
|
|
105
|
+
if (words.some((w) => WRITE_WORDS.has(w)))
|
|
106
|
+
return false;
|
|
107
|
+
return words.some((w) => READ_WORDS.has(w));
|
|
108
|
+
}
|
|
109
|
+
/** Commands that only look (`ls`, `cat`, `grep …`). */
|
|
110
|
+
const READ_COMMANDS = new Set([
|
|
111
|
+
"ls", "dir", "cat", "type", "head", "tail", "less", "more", "grep", "rg", "egrep", "fgrep",
|
|
112
|
+
"find", "wc", "pwd", "echo", "printf", "date", "whoami", "which", "where", "stat", "file",
|
|
113
|
+
"du", "df", "tree", "jq", "sort", "uniq", "cut", "awk", "diff", "cmp", "basename", "dirname",
|
|
114
|
+
"realpath", "readlink", "true", "false", "test", "ps", "uname", "hostname", "id", "nproc",
|
|
115
|
+
"free", "uptime", "node --version", "npm --version",
|
|
116
|
+
]);
|
|
117
|
+
const READ_GIT = new Set([
|
|
118
|
+
"status", "log", "diff", "show", "branch", "rev-parse", "ls-files", "blame", "remote",
|
|
119
|
+
"describe", "config --get", "grep", "shortlog", "tag --list",
|
|
120
|
+
]);
|
|
121
|
+
/**
|
|
122
|
+
* Whether a shell command only looks: every part of it is a looking command,
|
|
123
|
+
* nothing is written to a file, and nothing runs code it does not show.
|
|
124
|
+
*/
|
|
125
|
+
export function commandReadsOnly(command) {
|
|
126
|
+
const text = command.trim();
|
|
127
|
+
if (!text)
|
|
128
|
+
return false;
|
|
129
|
+
// Code that runs out of sight: a substitution (`$(…)`, backticks), a
|
|
130
|
+
// process substitution, awk's `system(`.
|
|
131
|
+
if (/`|\$\(|<\(|>\(|\bsystem\s*\(/.test(text))
|
|
132
|
+
return false;
|
|
133
|
+
// A redirect into a file (not into /dev/null, not 2>&1), tee, or an in-place edit changes something.
|
|
134
|
+
if (/(^|[^0-9&])>{1,2}\s*(?!&|\/dev\/null)\S/.test(text) ||
|
|
135
|
+
/&>{1,2}\s*(?!\/dev\/null)\S/.test(text) ||
|
|
136
|
+
/\btee\b/.test(text) ||
|
|
137
|
+
/\bsed\b[^|;&]*\s(-i|--in-place)/.test(text)) {
|
|
138
|
+
return false;
|
|
139
|
+
}
|
|
140
|
+
if (/\bfind\b[^|;&]*\s-(delete|exec|execdir|ok|okdir|fprint|fprintf|fls)\b/.test(text))
|
|
141
|
+
return false;
|
|
142
|
+
// A job sent to the background (`cmd &`): not something this reads.
|
|
143
|
+
if (/(?<![>&])&(?![>&])/.test(text.replace(/&&/g, "")))
|
|
144
|
+
return false;
|
|
145
|
+
const parts = text
|
|
146
|
+
.split(/\|\||&&|[|;\n]/)
|
|
147
|
+
.map((p) => p.trim())
|
|
148
|
+
.filter(Boolean);
|
|
149
|
+
if (parts.length === 0)
|
|
150
|
+
return false;
|
|
151
|
+
return parts.every((part) => {
|
|
152
|
+
const words = part.replace(/^(\w+=\S+\s+)+/, "").split(/\s+/);
|
|
153
|
+
const cmd = (words[0] ?? "").replace(/^.*[\\/]/, "");
|
|
154
|
+
if (cmd === "git")
|
|
155
|
+
return READ_GIT.has(words[1] ?? "") || READ_GIT.has(`${words[1] ?? ""} ${words[2] ?? ""}`);
|
|
156
|
+
if (cmd === "sed")
|
|
157
|
+
return !/\s(-i|--in-place)/.test(part);
|
|
158
|
+
return READ_COMMANDS.has(cmd) || READ_COMMANDS.has(`${cmd} ${words[1] ?? ""}`);
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
//# sourceMappingURL=reads-only.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@basein/runner",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.27",
|
|
4
4
|
"description": "A recording MCP proxy: sits between any MCP client and its MCP servers, executes each call on the client's behalf, and records the run as a reusable BaseIn scenario.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|