@basein/runner 0.3.18 → 0.3.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/bir-hooks.d.ts +11 -8
- package/dist/bin/bir-hooks.js +11 -8
- package/dist/bin/bir.js +20 -7
- package/dist/bin/investigate.d.ts +9 -0
- package/dist/bin/investigate.js +31 -12
- package/dist/bin/scenario-edit.js +10 -0
- package/dist/control/recorder-main.d.ts +15 -0
- package/dist/control/recorder-main.js +33 -8
- package/dist/control/server.d.ts +91 -11
- package/dist/control/server.js +736 -49
- package/dist/record/recorder.d.ts +47 -4
- package/dist/record/recorder.js +3 -0
- package/dist/record/remote-recorder.d.ts +9 -1
- package/dist/record/remote-recorder.js +30 -0
- package/dist/record/switchable-recorder.js +2 -0
- package/dist/replay/condition-words.d.ts +28 -0
- package/dist/replay/condition-words.js +59 -0
- package/dist/replay/controller.d.ts +45 -6
- package/dist/replay/controller.js +184 -51
- package/dist/replay/derive.d.ts +10 -0
- package/dist/replay/derive.js +3 -0
- package/dist/replay/handover.d.ts +7 -0
- package/dist/replay/handover.js +89 -51
- package/dist/replay/plan.d.ts +11 -4
- package/dist/replay/plan.js +28 -8
- package/dist/replay/reads-only.d.ts +29 -0
- package/dist/replay/reads-only.js +261 -0
- package/dist/replay/types.d.ts +19 -3
- package/docs/calculatedReplay.md +45 -5
- package/docs/calculatedReplayGuide.md +16 -4
- package/package.json +1 -1
- package/production.md +34 -7
package/dist/control/server.js
CHANGED
|
@@ -37,9 +37,12 @@ import { asSent, userText } from "../record/prompt-text.js";
|
|
|
37
37
|
import { redact, redactText } from "../record/redact.js";
|
|
38
38
|
import { DEFAULT_MAX_STRING_BYTES, serializeCapped, truncateString, truncateStrings, } from "../record/truncate.js";
|
|
39
39
|
import { StepQueue } from "../record/queue.js";
|
|
40
|
-
import { NullRecorder, isAttentionReader, isIntentMatcher, isMatchAware, isRecorderHealthReporter, isRunCreationAware, isScenarioDisputer, isScenarioReporter, isServiceProber, isToolListReporter, } from "../record/recorder.js";
|
|
40
|
+
import { NullRecorder, isAttentionReader, isIntentMatcher, isMatchAware, isNextScenarioSource, isRecorderHealthReporter, isRunCreationAware, isScenarioDisputer, isScenarioReporter, isServiceProber, isToolListReporter, } from "../record/recorder.js";
|
|
41
41
|
import { ReplayController, POLL_HOLD_MS, REPORT_TOOL_NAME, } from "../replay/controller.js";
|
|
42
42
|
import { localDay } from "../replay/derive.js";
|
|
43
|
+
import { RecordedValueError, markedKey } from "../replay/plan.js";
|
|
44
|
+
import { evalToolInputLogic } from "../replay/logic.js";
|
|
45
|
+
import { callMayChange, readsOnly } from "../replay/reads-only.js";
|
|
43
46
|
import { calculateCostUsd, hasModelPricing } from "../replay/pricing.js";
|
|
44
47
|
import { logDetail, logLine, logOnce, errText } from "../util/log.js";
|
|
45
48
|
import { journal } from "../util/journal.js";
|
|
@@ -105,6 +108,118 @@ const PROMPT_WAIT_MS = 7_500;
|
|
|
105
108
|
const INTENT_ARM_PROBE_MS = 500;
|
|
106
109
|
/** What an intent-armed plan may spend arming — derivation — inside `PreToolUse`. */
|
|
107
110
|
const INTENT_ARM_DERIVE_MS = 1_500;
|
|
111
|
+
/**
|
|
112
|
+
* How long the agent's call after a decision may wait for a next scenario's
|
|
113
|
+
* values (the service's offer1.md O5): they are read while the agent decides,
|
|
114
|
+
* so this wait is only ever for a reading still on its way, or one read again
|
|
115
|
+
* with the call itself. The call is about to be replaced, so waiting costs the
|
|
116
|
+
* agent nothing; it stays well inside the route's own deadline.
|
|
117
|
+
*/
|
|
118
|
+
const NEXT_HIT_WAIT_MS = 6_000;
|
|
119
|
+
/** How long the question "what can follow this decision" may take. It is asked in the background. */
|
|
120
|
+
const NEXT_FETCH_MS = 5_000;
|
|
121
|
+
/** How long a next scenario's arm waits for its values to settle, in the background. */
|
|
122
|
+
const NEXT_READY_MS = 8_000;
|
|
123
|
+
/** At most this many scenarios follow one decision (the service sends no more either). */
|
|
124
|
+
const NEXT_MAX = 4;
|
|
125
|
+
/** The turn's request a decision fragment carries, in characters (offer1.md O2). */
|
|
126
|
+
const DECISION_REQUEST_MAX = 4_000;
|
|
127
|
+
/**
|
|
128
|
+
* How long the agent's call after a decision waits for the service's answer
|
|
129
|
+
* to "what can follow it" when it has not come yet. Only that answer: never
|
|
130
|
+
* the arms behind it.
|
|
131
|
+
*/
|
|
132
|
+
const NEXT_ROUTE_WAIT_MS = 1_000;
|
|
133
|
+
/** Tools whose call is its command: the same tool with another command is another call (offer1.md O8). */
|
|
134
|
+
const COMMAND_TOOLS = new Set(["Bash", "PowerShell"]);
|
|
135
|
+
/** A shell call's command, trimmed; empty when it has none. */
|
|
136
|
+
function commandOf(args) {
|
|
137
|
+
const c = args && typeof args === "object" ? args.command : undefined;
|
|
138
|
+
return typeof c === "string" ? c.trim() : "";
|
|
139
|
+
}
|
|
140
|
+
/** What `replay.next_miss` says for each way a decision's next scenarios end unused. */
|
|
141
|
+
const NEXT_MISS_WHY = {
|
|
142
|
+
other_values: "the agent's call has the tool a next scenario starts with, but not its values — it took that way its own way; that work is recorded, and becomes a scenario of its own",
|
|
143
|
+
own_way: "the agent's call may change something and starts none of them — it is doing a way of its own; that work is recorded, and becomes a scenario of its own, and none of these may start in the middle of it",
|
|
144
|
+
not_ready: "the scenario that starts with this call's tool was not armed in time, and the call may change something — the agent went on its own way",
|
|
145
|
+
calls_limit: "the agent's first calls after the decision started none of them — it is doing the work itself; that work is recorded, and becomes a scenario of its own",
|
|
146
|
+
turn_ended: "the turn ended before any of them was used",
|
|
147
|
+
no_delivery_tool: "the scenario would be delivered through mcp__bir__run_scenario, which no running bir server serves",
|
|
148
|
+
};
|
|
149
|
+
/** One argument as a call compares it: trimmed text for a scalar, canonical JSON for the rest. */
|
|
150
|
+
function argText(value) {
|
|
151
|
+
if (value === null || value === undefined)
|
|
152
|
+
return "";
|
|
153
|
+
if (typeof value === "string")
|
|
154
|
+
return value.trim();
|
|
155
|
+
if (typeof value === "number" || typeof value === "boolean")
|
|
156
|
+
return String(value);
|
|
157
|
+
return canonicalJson(value);
|
|
158
|
+
}
|
|
159
|
+
function canonicalJson(value) {
|
|
160
|
+
if (Array.isArray(value))
|
|
161
|
+
return `[${value.map(canonicalJson).join(",")}]`;
|
|
162
|
+
if (value && typeof value === "object") {
|
|
163
|
+
const o = value;
|
|
164
|
+
return `{${Object.keys(o)
|
|
165
|
+
.sort()
|
|
166
|
+
.map((k) => `${JSON.stringify(k)}:${canonicalJson(o[k])}`)
|
|
167
|
+
.join(",")}}`;
|
|
168
|
+
}
|
|
169
|
+
return JSON.stringify(value ?? null) ?? "null";
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* A planned first step's input against the agent's own arguments, value for
|
|
173
|
+
* value (offer1.md O5). `missing` when the only differences are values the
|
|
174
|
+
* plan does not have — empty, or held by a mark — that the agent gave. The
|
|
175
|
+
* runner's reserved arguments (`__bir…`) are never compared.
|
|
176
|
+
*/
|
|
177
|
+
function sameArguments(planned, agent) {
|
|
178
|
+
if (!agent || typeof agent !== "object" || Array.isArray(agent))
|
|
179
|
+
return "other";
|
|
180
|
+
const a = agent;
|
|
181
|
+
let missing = false;
|
|
182
|
+
for (const key of new Set([...Object.keys(planned), ...Object.keys(a)])) {
|
|
183
|
+
if (key.startsWith("__bir"))
|
|
184
|
+
continue;
|
|
185
|
+
const p = argText(planned[key]);
|
|
186
|
+
const v = argText(a[key]);
|
|
187
|
+
if (p === v)
|
|
188
|
+
continue;
|
|
189
|
+
if (v !== "" && (p === "" || markedKey(planned[key]) !== undefined)) {
|
|
190
|
+
missing = true;
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
return "other";
|
|
194
|
+
}
|
|
195
|
+
return missing ? "missing" : "hit";
|
|
196
|
+
}
|
|
197
|
+
/** A next scenario's plan that was never announced: it never runs, and is never reported. */
|
|
198
|
+
function dropNextState(state) {
|
|
199
|
+
state.plan = undefined;
|
|
200
|
+
state.mode = "none";
|
|
201
|
+
state.retired = true;
|
|
202
|
+
state.pinned.clear();
|
|
203
|
+
state.reported = true;
|
|
204
|
+
}
|
|
205
|
+
/** Drop an entry's plan, now or when its arm lands — all but `kept`, the one that hit. */
|
|
206
|
+
function dropNextEntry(entry, kept) {
|
|
207
|
+
if (entry.state) {
|
|
208
|
+
if (entry.state !== kept)
|
|
209
|
+
dropNextState(entry.state);
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
void entry.arming.then((late) => {
|
|
213
|
+
if (late && late !== kept)
|
|
214
|
+
dropNextState(late);
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* The agent's calls after a decision that are checked against its next
|
|
219
|
+
* scenarios: a scenario is a hit only on the way the agent starts, never in
|
|
220
|
+
* the middle of work it is already doing.
|
|
221
|
+
*/
|
|
222
|
+
const NEXT_CALLS_MAX = 5;
|
|
108
223
|
/** How long a fragment's `PreToolUse` waits to learn whether its run was created. */
|
|
109
224
|
const FRAGMENT_CREATE_MS = 300;
|
|
110
225
|
/** What `close()` spends draining the last runs to the service before it lets go. */
|
|
@@ -946,7 +1061,20 @@ export class ControlServer {
|
|
|
946
1061
|
lastStepHit: s.run.intent.lastStepHit ?? null,
|
|
947
1062
|
}
|
|
948
1063
|
: null,
|
|
949
|
-
fragment: s.run?.fragment
|
|
1064
|
+
fragment: s.run?.fragment
|
|
1065
|
+
? { state: s.run.fragment.state, ...(s.run.fragment.decision ? { decision: true } : {}) }
|
|
1066
|
+
: null,
|
|
1067
|
+
// The calculated scenarios that can follow a decision of this turn
|
|
1068
|
+
// (offer1.md O3): how many are ready for the agent's next call, and why
|
|
1069
|
+
// the set ended once it has.
|
|
1070
|
+
next: s.run?.next
|
|
1071
|
+
? {
|
|
1072
|
+
entries: s.run.next.entries.length,
|
|
1073
|
+
armed: s.run.next.entries.filter((e) => e.state?.plan && !e.state.retired).length,
|
|
1074
|
+
calls: s.run.next.calls,
|
|
1075
|
+
ended: s.run.next.ended ?? null,
|
|
1076
|
+
}
|
|
1077
|
+
: null,
|
|
950
1078
|
// The service's offer to record the turn as a recording's newest run,
|
|
951
1079
|
// and what became of it (newestRun.md).
|
|
952
1080
|
repeat: s.run?.repeat ? { state: s.run.repeat.state, of: s.run.repeat.runId } : null,
|
|
@@ -964,6 +1092,10 @@ export class ControlServer {
|
|
|
964
1092
|
// be invisible: an operator has to be able to see which of the two this
|
|
965
1093
|
// machine is doing without reading a log file.
|
|
966
1094
|
segmentArm: this.replay.segmentArm,
|
|
1095
|
+
// And whether a step's intent may start a plan in its own PreToolUse,
|
|
1096
|
+
// which means waiting for the probe there (it follows segmentArm unless
|
|
1097
|
+
// BIR_INTENT_ARM says otherwise). For `bir doctor`.
|
|
1098
|
+
intentArm: this.intentArming(),
|
|
967
1099
|
authUrl: resolveAuthUrl() || null,
|
|
968
1100
|
sessionId: this.sessionId,
|
|
969
1101
|
pid: process.pid,
|
|
@@ -2095,6 +2227,12 @@ export class ControlServer {
|
|
|
2095
2227
|
if (!otherHost(session))
|
|
2096
2228
|
state.reportTool = () => this.reportToolServed();
|
|
2097
2229
|
this.declineWithoutDeliveryTool(state);
|
|
2230
|
+
// A plan that stops at its decision is billed only up to the stop (the
|
|
2231
|
+
// service's segmentsbreakdown.md SB-26): its hand-over takes this mark. A
|
|
2232
|
+
// plan of the prompt without one keeps the window it always had.
|
|
2233
|
+
if (!otherHost(session) && this.replay.stopsAtDecision(state)) {
|
|
2234
|
+
state.markUsage = () => markTranscriptUsage(session.transcriptPath);
|
|
2235
|
+
}
|
|
2098
2236
|
// A steer plan with a shell or file-write step can only be followed if the
|
|
2099
2237
|
// directive shows its computed arguments (the pin guard holds those steps
|
|
2100
2238
|
// to their exact shape), so give its parameters a moment to settle —
|
|
@@ -2812,6 +2950,9 @@ export class ControlServer {
|
|
|
2812
2950
|
run.pendingDirective = undefined;
|
|
2813
2951
|
if (unannounced)
|
|
2814
2952
|
this.shelveUnannounced(unannounced);
|
|
2953
|
+
// The scenarios that could have followed a decision and were never used:
|
|
2954
|
+
// never announced, never reported (offer1.md O7).
|
|
2955
|
+
this.endNext(run, "turn_ended");
|
|
2815
2956
|
// A `run_scenario` still waiting for this run's plan will never get it.
|
|
2816
2957
|
this.directCalls = this.directCalls.filter((c) => c.run !== run);
|
|
2817
2958
|
const durationMs = Date.now() - run.startedAtMs;
|
|
@@ -3021,11 +3162,21 @@ export class ControlServer {
|
|
|
3021
3162
|
// that finished in three steps would be billed for everything the agent
|
|
3022
3163
|
// did afterwards, and its saving would read as a loss.
|
|
3023
3164
|
let end = nextMark;
|
|
3165
|
+
// A plan of the prompt stopped at its decision after a step ran: its
|
|
3166
|
+
// window ends at the hand-over (the service's segmentsbreakdown.md
|
|
3167
|
+
// SB-26), and the agent's decision and the half it took are not on its
|
|
3168
|
+
// bill. Every other plan of the prompt keeps its window: the answer is
|
|
3169
|
+
// still the plan's work.
|
|
3170
|
+
const atDecision = state.handover?.kind === "decision" && state.retiredMark !== undefined;
|
|
3024
3171
|
if (state.armedBy === "intent") {
|
|
3025
3172
|
windowCost = this.costSince(session, state.usageMark);
|
|
3026
3173
|
windowMs = (state.retiredAt ?? next?.armedAt ?? Date.now()) - state.armedAt;
|
|
3027
3174
|
end = state.retiredMark ?? nextMark;
|
|
3028
3175
|
}
|
|
3176
|
+
else if (atDecision) {
|
|
3177
|
+
windowMs = (state.retiredAt ?? Date.now()) - run.startedAtMs;
|
|
3178
|
+
end = state.retiredMark;
|
|
3179
|
+
}
|
|
3029
3180
|
else if (next) {
|
|
3030
3181
|
windowMs = next.armedAt - run.startedAtMs;
|
|
3031
3182
|
}
|
|
@@ -3047,11 +3198,15 @@ export class ControlServer {
|
|
|
3047
3198
|
});
|
|
3048
3199
|
if (!report)
|
|
3049
3200
|
return;
|
|
3050
|
-
if (state.armedBy === "intent") {
|
|
3201
|
+
if (state.armedBy === "intent" || atDecision) {
|
|
3051
3202
|
logDetail("replay.done", {
|
|
3052
3203
|
scenario: state.scenarioId ?? undefined,
|
|
3053
3204
|
windowMs: Math.max(0, windowMs),
|
|
3054
|
-
billedTo:
|
|
3205
|
+
billedTo: atDecision
|
|
3206
|
+
? "its decision"
|
|
3207
|
+
: state.retiredMark !== undefined
|
|
3208
|
+
? "its own retire"
|
|
3209
|
+
: "the next arm or the turn",
|
|
3055
3210
|
});
|
|
3056
3211
|
}
|
|
3057
3212
|
// The plan ran to its end, and no redo of the model's own was recorded:
|
|
@@ -3612,6 +3767,7 @@ export class ControlServer {
|
|
|
3612
3767
|
// A hand-over inside `preTool` (an input logic that threw) must still
|
|
3613
3768
|
// schedule the fragment, even though this call ends here.
|
|
3614
3769
|
this.noteHandover(run);
|
|
3770
|
+
this.noteDecision(session, run);
|
|
3615
3771
|
if (action.kind === "pin") {
|
|
3616
3772
|
// The same tool is not the same call. A pin swaps the model's
|
|
3617
3773
|
// arguments for the step's and approves them, so it is kept only for
|
|
@@ -3674,30 +3830,45 @@ export class ControlServer {
|
|
|
3674
3830
|
const note = planning ? undefined : this.replay.takeNote(run.replay);
|
|
3675
3831
|
if (note) {
|
|
3676
3832
|
this.noteHandover(run);
|
|
3833
|
+
this.noteDecision(session, run);
|
|
3677
3834
|
return this.preToolAnswer(this.replay.deliverInstead(note, toolName));
|
|
3678
3835
|
}
|
|
3679
|
-
//
|
|
3680
|
-
//
|
|
3681
|
-
// Opened before the probe below, so the turn is recorded from this call
|
|
3682
|
-
// on like any recording turn — a plan an intent match arms in it too.
|
|
3683
|
-
if (this.repeatDue(run))
|
|
3684
|
-
this.openRepeat(session, run);
|
|
3685
|
-
// 2. The model is driving: look for a scenario by this iteration's intent.
|
|
3686
|
-
// Never in plan mode, where nothing may run on the user's behalf.
|
|
3687
|
-
// If the deadline answers first, it answers `{}`: the call runs as
|
|
3688
|
-
// written, and a plan armed in its place was never announced.
|
|
3836
|
+
// If the deadline answers first, it answers `{}`: the call runs as
|
|
3837
|
+
// written, and a plan armed in its place below was never announced.
|
|
3689
3838
|
if (!planning && !lateSet) {
|
|
3690
3839
|
onLate?.(() => {
|
|
3691
3840
|
overran = true;
|
|
3692
3841
|
return {};
|
|
3693
3842
|
});
|
|
3694
3843
|
}
|
|
3695
|
-
|
|
3696
|
-
|
|
3697
|
-
|
|
3844
|
+
// 2. The agent's call after a plan's decision: the first step of a
|
|
3845
|
+
// calculated scenario that can follow it (the service's offer1.md
|
|
3846
|
+
// O4–O6)? Then that scenario runs in this call's place. Before the
|
|
3847
|
+
// probe below: this check is exact (the same tool, the same values)
|
|
3848
|
+
// and asks the service nothing at this call. Never in plan mode.
|
|
3849
|
+
if (!planning) {
|
|
3850
|
+
this.noteDecision(session, run);
|
|
3851
|
+
const hit = await this.tryNextScenarios(session, run, toolName, modelArgs, () => overran);
|
|
3852
|
+
if (hit)
|
|
3853
|
+
return hit;
|
|
3854
|
+
}
|
|
3855
|
+
// The model is doing the matched task itself, and the service asked for
|
|
3856
|
+
// that run to be recorded as the recording's newest (newestRun.md N1).
|
|
3857
|
+
// Opened before the probe below, so the turn is recorded from this call
|
|
3858
|
+
// on like any recording turn — a plan an intent match arms in it too.
|
|
3859
|
+
if (this.repeatDue(run))
|
|
3860
|
+
this.openRepeat(session, run);
|
|
3861
|
+
// 3. The model is driving: look for a scenario by this iteration's intent.
|
|
3862
|
+
// Never in plan mode, where nothing may run on the user's behalf, and
|
|
3863
|
+
// not once the deadline has answered: nothing new may start then.
|
|
3864
|
+
const armed = planning || overran ? undefined : await this.tryIntentMatch(session, run, toolName, intent, modelArgs);
|
|
3698
3865
|
if (armed) {
|
|
3699
|
-
if (!overran)
|
|
3866
|
+
if (!overran) {
|
|
3867
|
+
// The plan answers in this call's place: the agent's work from here
|
|
3868
|
+
// is that plan's, never the other half after a decision (SB-16).
|
|
3869
|
+
this.dropDecisionFragment(run, run.replay);
|
|
3700
3870
|
return armed;
|
|
3871
|
+
}
|
|
3701
3872
|
const late = run.replay;
|
|
3702
3873
|
if (late && late.armedBy === "intent" && late.plan && !late.retired) {
|
|
3703
3874
|
logLine("replay.decision", {
|
|
@@ -3711,12 +3882,17 @@ export class ControlServer {
|
|
|
3711
3882
|
}
|
|
3712
3883
|
// The call ran as the model wrote it: recorded below as its own.
|
|
3713
3884
|
}
|
|
3714
|
-
//
|
|
3885
|
+
// 4. No scenario took over: the model's own work after a hand-over is
|
|
3715
3886
|
// recorded as a fragment, starting with this call.
|
|
3716
3887
|
// Not for the call an intent match just armed a plan beside: the plan
|
|
3717
|
-
// runs next, and the fragment opens on the model's own work after it
|
|
3718
|
-
|
|
3719
|
-
|
|
3888
|
+
// runs next, and the fragment opens on the model's own work after it
|
|
3889
|
+
// — unless it was scheduled at a decision, whose fragment is the
|
|
3890
|
+
// other half only when the agent's own call opens it (SB-16).
|
|
3891
|
+
if (run.fragment?.state === "due") {
|
|
3892
|
+
if (!run.pendingDirective)
|
|
3893
|
+
await this.openFragment(session, run, reasoning, toolName, modelArgs);
|
|
3894
|
+
else
|
|
3895
|
+
this.dropDecisionFragment(run, run.pendingDirective.state);
|
|
3720
3896
|
}
|
|
3721
3897
|
}
|
|
3722
3898
|
// The arguments that will actually run — which is what must be correlated and
|
|
@@ -4314,14 +4490,19 @@ export class ControlServer {
|
|
|
4314
4490
|
return [...out];
|
|
4315
4491
|
}
|
|
4316
4492
|
/**
|
|
4317
|
-
* Per
|
|
4318
|
-
* (segmented.md R-OUT-6).
|
|
4493
|
+
* Per plan of this turn, its recording and how far the plan got in its own
|
|
4494
|
+
* chain (segmented.md R-OUT-6).
|
|
4319
4495
|
*
|
|
4320
4496
|
* The server uses it to refuse a segment that starts *before* where this turn
|
|
4321
4497
|
* already is: re-running steps the turn has just done is worse than not
|
|
4322
|
-
* replaying at all. The
|
|
4323
|
-
*
|
|
4324
|
-
*
|
|
4498
|
+
* replaying at all. The position is the plan's own: the step it handed over
|
|
4499
|
+
* at (its decision included), or its last step when it ran to its end. The
|
|
4500
|
+
* server maps it onto the recording — a segment's first position, a
|
|
4501
|
+
* decision's step before it — and takes the largest per recording (the
|
|
4502
|
+
* service's segmentsbreakdown.md D18). One entry per plan, so that two plans
|
|
4503
|
+
* of one recording, whose own positions are on different scales, never
|
|
4504
|
+
* hide each other. A plan declined for anything but a known-bad first step
|
|
4505
|
+
* reports nothing — it ran none of the recording, so it blocks none of it.
|
|
4325
4506
|
*
|
|
4326
4507
|
* Rebuilt on every probe rather than cached: a plan that hands over between
|
|
4327
4508
|
* two calls changes the answer.
|
|
@@ -4354,15 +4535,15 @@ export class ControlServer {
|
|
|
4354
4535
|
}
|
|
4355
4536
|
if (position === undefined)
|
|
4356
4537
|
continue;
|
|
4357
|
-
const seen = best.get(
|
|
4538
|
+
const seen = best.get(state.scenarioId);
|
|
4358
4539
|
if (!seen || position > seen.position) {
|
|
4359
|
-
best.set(
|
|
4540
|
+
best.set(state.scenarioId, { runId: recording, position });
|
|
4360
4541
|
}
|
|
4361
4542
|
}
|
|
4362
|
-
return [...best].map(([
|
|
4363
|
-
runId,
|
|
4543
|
+
return [...best].map(([scenarioId, v]) => ({
|
|
4544
|
+
runId: v.runId,
|
|
4364
4545
|
position: v.position,
|
|
4365
|
-
scenarioId
|
|
4546
|
+
scenarioId,
|
|
4366
4547
|
}));
|
|
4367
4548
|
}
|
|
4368
4549
|
/**
|
|
@@ -4446,7 +4627,10 @@ export class ControlServer {
|
|
|
4446
4627
|
stepPosition: run.ordering.next,
|
|
4447
4628
|
excludeRuns: this.excludeRunsFor(run),
|
|
4448
4629
|
ranThrough: this.ranThroughFor(run),
|
|
4449
|
-
|
|
4630
|
+
// Segments are asked for only where one can arm: a probe that is
|
|
4631
|
+
// waited on, in Claude Code (below). Any other probe gets the
|
|
4632
|
+
// service's `segmentWouldArm` instead, logged as observe-only.
|
|
4633
|
+
...(this.replay.segmentArm && this.intentArming() && !otherHost(session) ? { acceptSegments: true } : {}),
|
|
4450
4634
|
supportsCalls: true,
|
|
4451
4635
|
// Left out for Claude Code: a missing framework means Claude Code to
|
|
4452
4636
|
// the service, which matches a step only inside its own host.
|
|
@@ -4534,6 +4718,9 @@ export class ControlServer {
|
|
|
4534
4718
|
// the jail it may not leave (replay/builtins.ts). The session's own cwd
|
|
4535
4719
|
// from the hook payloads, never the daemon's.
|
|
4536
4720
|
state.sessionCwd = session.cwd ?? this.opts.cwd;
|
|
4721
|
+
// The request this plan serves, for a note at its decision (SB-18): the
|
|
4722
|
+
// turn's prompt. `arm` was given this step's intent instead.
|
|
4723
|
+
state.request = run.input || undefined;
|
|
4537
4724
|
this.declineWithoutDeliveryTool(state);
|
|
4538
4725
|
if (!state.plan) {
|
|
4539
4726
|
// A declined intent hit reports nothing: the turn's cost is not a
|
|
@@ -4541,6 +4728,41 @@ export class ControlServer {
|
|
|
4541
4728
|
state.reported = true;
|
|
4542
4729
|
return undefined;
|
|
4543
4730
|
}
|
|
4731
|
+
// HOW THE DIRECTIVE REACHES THE MODEL. Mid-task, the call that matched is
|
|
4732
|
+
// usually the model's own ordinary work — a `git status`, a file read —
|
|
4733
|
+
// and denying it, or swapping its output for a directive, shows the user a
|
|
4734
|
+
// blocked call and reads to the model like prompt injection. So the call
|
|
4735
|
+
// runs untouched and the directive rides on its PostToolUse, the channel a
|
|
4736
|
+
// hand-over note already uses; the plan starts at its first step after.
|
|
4737
|
+
//
|
|
4738
|
+
// The exception is a call that IS the plan's first step: letting it run
|
|
4739
|
+
// would have the plan run it a second time — a duplicated write. That one
|
|
4740
|
+
// is still answered in place. The same tool is not the same call, though:
|
|
4741
|
+
// it is held to the pin guard's rule (control/pin-guard.ts), so a `git
|
|
4742
|
+
// status` is never swapped for a deploy that happens to be a Bash step.
|
|
4743
|
+
const inPlace = this.isFirstStep(state, toolName, modelArgs);
|
|
4744
|
+
// NEVER TWICE (the service's offer1.md O8). A plan that starts after this
|
|
4745
|
+
// call — not in its place — and calls the same tool somewhere would make
|
|
4746
|
+
// that call again: the agent's e-mail, then the plan's own e-mail (issue
|
|
4747
|
+
// fx2). It is not started; the agent goes on. A read the runner knows is
|
|
4748
|
+
// only a read may repeat: reading twice changes nothing.
|
|
4749
|
+
if (!inPlace && this.planCallsTool(state, toolName, modelArgs)) {
|
|
4750
|
+
logLine("replay.decision", {
|
|
4751
|
+
verdict: "no-steer",
|
|
4752
|
+
code: "would_repeat",
|
|
4753
|
+
run: run.runId,
|
|
4754
|
+
scenario: state.scenarioId ?? undefined,
|
|
4755
|
+
kind: match.kind ?? "scenario",
|
|
4756
|
+
tool: toolName,
|
|
4757
|
+
why: "the plan would start after this call and make a call of the same tool again — never twice; the agent goes on",
|
|
4758
|
+
});
|
|
4759
|
+
state.plan = undefined;
|
|
4760
|
+
state.mode = "none";
|
|
4761
|
+
state.retired = true;
|
|
4762
|
+
state.pinned.clear();
|
|
4763
|
+
state.reported = true;
|
|
4764
|
+
return undefined;
|
|
4765
|
+
}
|
|
4544
4766
|
state.usageMark = markTranscriptUsage(session.transcriptPath);
|
|
4545
4767
|
// The closure, not the mark: only the control server holds the transcript
|
|
4546
4768
|
// path, and a completing retire happens in the controller (R-MONEY-3).
|
|
@@ -4563,19 +4785,6 @@ export class ControlServer {
|
|
|
4563
4785
|
run.replay = state;
|
|
4564
4786
|
run.intent.armed += 1;
|
|
4565
4787
|
const directive = this.replay.directiveFor(state) ?? "";
|
|
4566
|
-
// HOW THE DIRECTIVE REACHES THE MODEL. Mid-task, the call that matched is
|
|
4567
|
-
// usually the model's own ordinary work — a `git status`, a file read —
|
|
4568
|
-
// and denying it, or swapping its output for a directive, shows the user a
|
|
4569
|
-
// blocked call and reads to the model like prompt injection. So the call
|
|
4570
|
-
// runs untouched and the directive rides on its PostToolUse, the channel a
|
|
4571
|
-
// hand-over note already uses; the plan starts at its first step after.
|
|
4572
|
-
//
|
|
4573
|
-
// The exception is a call that IS the plan's first step: letting it run
|
|
4574
|
-
// would have the plan run it a second time — a duplicated write. That one
|
|
4575
|
-
// is still answered in place. The same tool is not the same call, though:
|
|
4576
|
-
// it is held to the pin guard's rule (control/pin-guard.ts), so a `git
|
|
4577
|
-
// status` is never swapped for a deploy that happens to be a Bash step.
|
|
4578
|
-
const inPlace = this.isFirstStep(state, toolName, modelArgs);
|
|
4579
4788
|
logLine("replay.intent_armed", {
|
|
4580
4789
|
run: run.runId,
|
|
4581
4790
|
scenario: state.scenarioId ?? undefined,
|
|
@@ -4616,6 +4825,433 @@ export class ControlServer {
|
|
|
4616
4825
|
return false;
|
|
4617
4826
|
}
|
|
4618
4827
|
}
|
|
4828
|
+
/**
|
|
4829
|
+
* The agent's call against `state`'s first step, value for value (the
|
|
4830
|
+
* service's offer1.md O5): `hit` when every argument is the step's own,
|
|
4831
|
+
* `missing` when the only differences are values the plan does not have
|
|
4832
|
+
* (a target the reading left empty, which the agent's call may hold), and
|
|
4833
|
+
* `other` otherwise.
|
|
4834
|
+
*/
|
|
4835
|
+
firstStepVerdict(state, toolName, modelArgs) {
|
|
4836
|
+
const plan = state.plan;
|
|
4837
|
+
if (!plan || plan.expectedTool() !== toolName)
|
|
4838
|
+
return "other";
|
|
4839
|
+
let first;
|
|
4840
|
+
try {
|
|
4841
|
+
const entry = plan.currentEntry();
|
|
4842
|
+
if (!entry || entry.depth !== 0)
|
|
4843
|
+
return "other";
|
|
4844
|
+
first = plan.toolInputForCurrentStep();
|
|
4845
|
+
}
|
|
4846
|
+
catch (err) {
|
|
4847
|
+
// A value the request did not give, or one only a step works out: the
|
|
4848
|
+
// agent's call may hold it.
|
|
4849
|
+
if (err instanceof RecordedValueError)
|
|
4850
|
+
return "missing";
|
|
4851
|
+
logDetail("replay.first_step_unknown", { tool: toolName, error: errText(err) });
|
|
4852
|
+
return "other";
|
|
4853
|
+
}
|
|
4854
|
+
return sameArguments(first, modelArgs);
|
|
4855
|
+
}
|
|
4856
|
+
/**
|
|
4857
|
+
* Whether `state`'s plan calls `toolName` in a step it would run: what makes
|
|
4858
|
+
* a plan started after a call of that tool make it a second time (the
|
|
4859
|
+
* service's offer1.md O8). A tool that only reads never counts — reading
|
|
4860
|
+
* twice changes nothing — by the service's own rule ({@link readsOnly}): a
|
|
4861
|
+
* `crm_search_customers` mid-task never keeps a plan that searches later from
|
|
4862
|
+
* starting.
|
|
4863
|
+
*/
|
|
4864
|
+
planCallsTool(state, toolName, modelArgs) {
|
|
4865
|
+
const plan = state.plan;
|
|
4866
|
+
if (!plan || readsOnly(toolName))
|
|
4867
|
+
return false;
|
|
4868
|
+
const same = plan
|
|
4869
|
+
.allEntries()
|
|
4870
|
+
.slice(0, plan.stepCount)
|
|
4871
|
+
.filter((e) => e.step.toolName === toolName);
|
|
4872
|
+
if (same.length === 0)
|
|
4873
|
+
return false;
|
|
4874
|
+
if (!COMMAND_TOOLS.has(toolName))
|
|
4875
|
+
return true;
|
|
4876
|
+
// A shell's call is its command: a `git status` is not repeated by a plan
|
|
4877
|
+
// that deploys. Only a step that would run this very command repeats it —
|
|
4878
|
+
// judged on the values the plan has now; a step whose command needs a
|
|
4879
|
+
// result first cannot be told, and does not count.
|
|
4880
|
+
const live = commandOf(modelArgs);
|
|
4881
|
+
if (!live)
|
|
4882
|
+
return false;
|
|
4883
|
+
const params = plan.parameters;
|
|
4884
|
+
return same.some((e) => {
|
|
4885
|
+
try {
|
|
4886
|
+
return commandOf(evalToolInputLogic(e.step.toolInputLogic ?? "return {};", params, e.frame.intent ?? "", {})) === live;
|
|
4887
|
+
}
|
|
4888
|
+
catch {
|
|
4889
|
+
return false;
|
|
4890
|
+
}
|
|
4891
|
+
});
|
|
4892
|
+
}
|
|
4893
|
+
// ── what can follow a decision (the service's offer1.md) ──────────────────
|
|
4894
|
+
/**
|
|
4895
|
+
* A plan of this turn stopped at its decision: ask the service, once per
|
|
4896
|
+
* plan, which calculated scenarios can follow it, and arm each in the
|
|
4897
|
+
* background while the agent reads the note and decides (offer1.md O3, O4).
|
|
4898
|
+
* Called wherever a hand-over is first seen; a second call for the same plan
|
|
4899
|
+
* does nothing.
|
|
4900
|
+
*/
|
|
4901
|
+
noteDecision(session, run) {
|
|
4902
|
+
if (run.finished || otherHost(session))
|
|
4903
|
+
return;
|
|
4904
|
+
if (!this.replay.enabled || !this.intentArming())
|
|
4905
|
+
return;
|
|
4906
|
+
if (!isNextScenarioSource(this.recorder))
|
|
4907
|
+
return;
|
|
4908
|
+
for (const state of run.replays) {
|
|
4909
|
+
const h = state.handover;
|
|
4910
|
+
if (h?.kind !== "decision" || !state.scenarioId)
|
|
4911
|
+
continue;
|
|
4912
|
+
const next = (run.next ??= { asked: new Set(), round: 0, answered: false, entries: [], calls: 0 });
|
|
4913
|
+
if (next.asked.has(state.scenarioId))
|
|
4914
|
+
continue;
|
|
4915
|
+
next.asked.add(state.scenarioId);
|
|
4916
|
+
// A new decision starts a new round: what could follow an earlier one is
|
|
4917
|
+
// no longer checked (its unused plans are dropped), and the agent's next
|
|
4918
|
+
// calls are checked against what can follow this one.
|
|
4919
|
+
for (const e of next.entries)
|
|
4920
|
+
dropNextEntry(e);
|
|
4921
|
+
next.entries = [];
|
|
4922
|
+
next.round += 1;
|
|
4923
|
+
next.answered = false;
|
|
4924
|
+
next.ended = undefined;
|
|
4925
|
+
next.calls = 0;
|
|
4926
|
+
next.asking = this.fetchNext(session, run, state.scenarioId, h.stepIndex, next.round);
|
|
4927
|
+
}
|
|
4928
|
+
}
|
|
4929
|
+
/**
|
|
4930
|
+
* Ask for a decision's next scenarios (offer1.md O3): each is checkable as
|
|
4931
|
+
* soon as the service answers, and armed in the background, each on its own
|
|
4932
|
+
* (O4). Resolves when the service has answered, not when the arms have.
|
|
4933
|
+
* Never rejects.
|
|
4934
|
+
*/
|
|
4935
|
+
async fetchNext(session, run, planId, stepIndex, round) {
|
|
4936
|
+
const startedAt = Date.now();
|
|
4937
|
+
let answer = null;
|
|
4938
|
+
try {
|
|
4939
|
+
if (isNextScenarioSource(this.recorder)) {
|
|
4940
|
+
answer = await this.recorder.nextScenarios(planId, { turnId: run.turnId, supportsCalls: true }, { timeoutMs: NEXT_FETCH_MS });
|
|
4941
|
+
}
|
|
4942
|
+
}
|
|
4943
|
+
catch {
|
|
4944
|
+
answer = null;
|
|
4945
|
+
}
|
|
4946
|
+
const next = run.next;
|
|
4947
|
+
// A later decision of the turn has its own round: this answer is no one's.
|
|
4948
|
+
if (!next || next.round !== round || next.ended || run.finished)
|
|
4949
|
+
return;
|
|
4950
|
+
next.answered = true;
|
|
4951
|
+
next.asking = undefined;
|
|
4952
|
+
const offered = (answer?.next ?? []).filter((m) => !!m && typeof m.scenarioId === "string");
|
|
4953
|
+
// A plan already armed this turn is never armed again from here (the
|
|
4954
|
+
// probe's own `exclude`): the plan that just stopped, above all.
|
|
4955
|
+
const taken = new Set(run.replays.map((s) => s.scenarioId).filter((id) => typeof id === "string"));
|
|
4956
|
+
const fresh = [];
|
|
4957
|
+
for (const match of offered) {
|
|
4958
|
+
if (next.entries.length + fresh.length >= NEXT_MAX)
|
|
4959
|
+
break;
|
|
4960
|
+
if (taken.has(match.scenarioId))
|
|
4961
|
+
continue;
|
|
4962
|
+
if ([...next.entries, ...fresh].some((e) => e.match.scenarioId === match.scenarioId))
|
|
4963
|
+
continue;
|
|
4964
|
+
const entry = {
|
|
4965
|
+
planId,
|
|
4966
|
+
stepIndex,
|
|
4967
|
+
// `arm` reads a numeric similarity, and the key says how it was found.
|
|
4968
|
+
match: { ...match, similarity: typeof match.similarity === "number" ? match.similarity : 1, key: "decision" },
|
|
4969
|
+
firstTool: typeof match.firstTool === "string" && match.firstTool ? match.firstTool : null,
|
|
4970
|
+
arming: Promise.resolve(undefined),
|
|
4971
|
+
};
|
|
4972
|
+
entry.arming = this.armNext(session, run, entry);
|
|
4973
|
+
fresh.push(entry);
|
|
4974
|
+
}
|
|
4975
|
+
// Checkable at once: a call waits only for the arm of a scenario that
|
|
4976
|
+
// starts with its own tool, never for the others.
|
|
4977
|
+
next.entries.push(...fresh);
|
|
4978
|
+
if (fresh.length === 0) {
|
|
4979
|
+
logDetail("replay.next_none", {
|
|
4980
|
+
run: run.runId,
|
|
4981
|
+
plan: planId,
|
|
4982
|
+
offered: offered.length,
|
|
4983
|
+
why: answer
|
|
4984
|
+
? "no calculated scenario follows this decision yet — what the agent does next is recorded, and becomes one"
|
|
4985
|
+
: "the service did not say what follows this decision — the agent goes on",
|
|
4986
|
+
});
|
|
4987
|
+
return;
|
|
4988
|
+
}
|
|
4989
|
+
// The audit line once every arm has answered; nobody waits for it.
|
|
4990
|
+
void Promise.all(fresh.map((e) => e.arming)).then(() => {
|
|
4991
|
+
if (run.finished || next.round !== round)
|
|
4992
|
+
return;
|
|
4993
|
+
logLine("replay.next_ready", {
|
|
4994
|
+
run: run.runId,
|
|
4995
|
+
plan: planId,
|
|
4996
|
+
n: fresh.filter((e) => e.state?.plan).length,
|
|
4997
|
+
of: fresh.length,
|
|
4998
|
+
ms: Date.now() - startedAt,
|
|
4999
|
+
scenarios: fresh
|
|
5000
|
+
.map((e) => `${e.match.scenarioId}:${e.firstTool ?? "?"}:${e.state?.plan ? "ready" : (e.state?.declined ?? "failed")}`)
|
|
5001
|
+
.join(","),
|
|
5002
|
+
why: "the calculated scenarios that can follow this decision are ready for the agent's next call",
|
|
5003
|
+
});
|
|
5004
|
+
});
|
|
5005
|
+
}
|
|
5006
|
+
/**
|
|
5007
|
+
* Arm one next scenario in the background (offer1.md O4): its values read
|
|
5008
|
+
* from the turn's request and its results so far — or, once more, with the
|
|
5009
|
+
* agent's own call (O5) — and settled, so a call is compared with the first
|
|
5010
|
+
* step as it would really run. Never rejects.
|
|
5011
|
+
*/
|
|
5012
|
+
async armNext(session, run, entry, liveCall) {
|
|
5013
|
+
try {
|
|
5014
|
+
const state = await this.replay.arm(entry.match, run.input, this.wrapped, "intent", {
|
|
5015
|
+
...(liveCall ? { liveCall } : {}),
|
|
5016
|
+
recentResults: this.recentResultsFor(session, "call"),
|
|
5017
|
+
agentDay: this.agentDayFor(session),
|
|
5018
|
+
atDecision: { scenarioId: entry.planId, stepIndex: entry.stepIndex },
|
|
5019
|
+
}, this.hostContext(session, run));
|
|
5020
|
+
if (state.plan)
|
|
5021
|
+
await this.withinBudget(state.plan.ready().catch(() => undefined), NEXT_READY_MS);
|
|
5022
|
+
entry.state = state;
|
|
5023
|
+
return state;
|
|
5024
|
+
}
|
|
5025
|
+
catch (err) {
|
|
5026
|
+
logDetail("replay.next_arm_failed", { scenario: entry.match.scenarioId ?? undefined, error: errText(err) });
|
|
5027
|
+
return undefined;
|
|
5028
|
+
}
|
|
5029
|
+
}
|
|
5030
|
+
/**
|
|
5031
|
+
* The agent's call after a decision, against the scenarios that can follow
|
|
5032
|
+
* it (offer1.md O5, O6). A hit — the call is a scenario's first step, value
|
|
5033
|
+
* for value — is answered in the call's place with that scenario's directive:
|
|
5034
|
+
* the agent calls `run_scenario`, and the scenario runs every step, this one
|
|
5035
|
+
* included, once. Undefined otherwise: the call runs as the agent wrote it.
|
|
5036
|
+
*/
|
|
5037
|
+
async tryNextScenarios(session, run, toolName, modelArgs, overran) {
|
|
5038
|
+
const next = run.next;
|
|
5039
|
+
if (!next || next.ended || run.finished || otherHost(session))
|
|
5040
|
+
return undefined;
|
|
5041
|
+
if (this.replay.isDirectTool(toolName))
|
|
5042
|
+
return undefined;
|
|
5043
|
+
const deadline = Date.now() + positiveEnv("BIR_NEXT_HIT_WAIT_MS", NEXT_HIT_WAIT_MS);
|
|
5044
|
+
const left = () => Math.max(0, deadline - Date.now());
|
|
5045
|
+
// The service has not answered yet — the agent decided quickly. Its answer
|
|
5046
|
+
// only is waited for, and briefly: not the arms that follow it.
|
|
5047
|
+
if (!next.answered && next.asking) {
|
|
5048
|
+
await this.withinBudget(next.asking, Math.min(left(), NEXT_ROUTE_WAIT_MS));
|
|
5049
|
+
if (overran() || next.ended || run.finished)
|
|
5050
|
+
return undefined;
|
|
5051
|
+
}
|
|
5052
|
+
next.calls += 1;
|
|
5053
|
+
/** A scenario that starts with this call's tool was compared, value for value, and missed. */
|
|
5054
|
+
let compared = false;
|
|
5055
|
+
/** One that starts with this call's tool was not armed in time: it misses this call only. */
|
|
5056
|
+
let notReady = false;
|
|
5057
|
+
/** Read the scenario's values once more, with the agent's call (O5); undefined when not ready in time. */
|
|
5058
|
+
const readAgain = async (entry, old) => {
|
|
5059
|
+
entry.rearmed = true;
|
|
5060
|
+
dropNextState(old);
|
|
5061
|
+
entry.state = undefined;
|
|
5062
|
+
const toolInput = serializeCapped(redact(modelArgs), { maxPayloadBytes: 2048, maxStringBytes: 256 });
|
|
5063
|
+
entry.arming = this.armNext(session, run, entry, { toolName, toolInput });
|
|
5064
|
+
return (await this.withinBudget(entry.arming, left())) ?? undefined;
|
|
5065
|
+
};
|
|
5066
|
+
for (const entry of next.entries) {
|
|
5067
|
+
if (entry.unhittable)
|
|
5068
|
+
continue;
|
|
5069
|
+
let state = entry.state;
|
|
5070
|
+
if (!state) {
|
|
5071
|
+
// Waited for only when the service named this call's tool as its first
|
|
5072
|
+
// step; any other is compared only once it is armed.
|
|
5073
|
+
if (entry.firstTool !== toolName)
|
|
5074
|
+
continue;
|
|
5075
|
+
state = (await this.withinBudget(entry.arming, left())) ?? undefined;
|
|
5076
|
+
if (overran() || next.ended)
|
|
5077
|
+
return undefined;
|
|
5078
|
+
if (!state) {
|
|
5079
|
+
notReady = true;
|
|
5080
|
+
continue;
|
|
5081
|
+
}
|
|
5082
|
+
}
|
|
5083
|
+
if (!state.plan || state.retired) {
|
|
5084
|
+
// Declined at the decision for a value the request never names (an
|
|
5085
|
+
// address a look-up returned): the agent's own call may hold it.
|
|
5086
|
+
if (entry.rearmed || state.declined !== "missing_target" || entry.firstTool !== toolName)
|
|
5087
|
+
continue;
|
|
5088
|
+
const again = await readAgain(entry, state);
|
|
5089
|
+
if (overran() || next.ended)
|
|
5090
|
+
return undefined;
|
|
5091
|
+
if (!again)
|
|
5092
|
+
notReady = true;
|
|
5093
|
+
else if (again.plan && !again.retired && this.firstStepVerdict(again, toolName, modelArgs) === "hit") {
|
|
5094
|
+
return this.nextHit(session, run, entry, again, toolName);
|
|
5095
|
+
}
|
|
5096
|
+
else if (again.plan?.expectedTool() === toolName)
|
|
5097
|
+
compared = true;
|
|
5098
|
+
continue;
|
|
5099
|
+
}
|
|
5100
|
+
if (state.plan.expectedTool() !== toolName)
|
|
5101
|
+
continue;
|
|
5102
|
+
// A first step inside a sub-task (a call row) is never the agent's call in place.
|
|
5103
|
+
const first = state.plan.currentEntry();
|
|
5104
|
+
if (!first || first.depth !== 0) {
|
|
5105
|
+
entry.unhittable = true;
|
|
5106
|
+
continue;
|
|
5107
|
+
}
|
|
5108
|
+
// Value for value (O5): the pin guard's "plausibly the step" is not
|
|
5109
|
+
// enough here — the scenario would run its own values, and an e-mail to
|
|
5110
|
+
// carlos must never become the recorded e-mail to olga.
|
|
5111
|
+
const verdict = this.firstStepVerdict(state, toolName, modelArgs);
|
|
5112
|
+
if (verdict === "hit")
|
|
5113
|
+
return this.nextHit(session, run, entry, state, toolName);
|
|
5114
|
+
// Only a value the plan lacks is read again, never one it read differently.
|
|
5115
|
+
if (verdict === "missing" && !entry.rearmed) {
|
|
5116
|
+
const again = await readAgain(entry, state);
|
|
5117
|
+
if (overran() || next.ended)
|
|
5118
|
+
return undefined;
|
|
5119
|
+
if (!again) {
|
|
5120
|
+
notReady = true;
|
|
5121
|
+
continue;
|
|
5122
|
+
}
|
|
5123
|
+
if (again.plan && !again.retired && this.firstStepVerdict(again, toolName, modelArgs) === "hit") {
|
|
5124
|
+
return this.nextHit(session, run, entry, again, toolName);
|
|
5125
|
+
}
|
|
5126
|
+
}
|
|
5127
|
+
compared = true;
|
|
5128
|
+
}
|
|
5129
|
+
// No hit.
|
|
5130
|
+
if (compared) {
|
|
5131
|
+
// The agent took a way a scenario starts with, with other values: its
|
|
5132
|
+
// own way, recorded, and a scenario of its own later (O7).
|
|
5133
|
+
this.endNext(run, "other_values", toolName);
|
|
5134
|
+
}
|
|
5135
|
+
else if (callMayChange(toolName, modelArgs)) {
|
|
5136
|
+
// Its first call that may change something and starts none of them: the
|
|
5137
|
+
// agent is doing a way of its own (O7), and from here a scenario must
|
|
5138
|
+
// never start in the middle of it.
|
|
5139
|
+
this.endNext(run, notReady ? "not_ready" : "own_way", toolName);
|
|
5140
|
+
}
|
|
5141
|
+
else {
|
|
5142
|
+
if (notReady) {
|
|
5143
|
+
logDetail("replay.next_not_ready", {
|
|
5144
|
+
run: run.runId,
|
|
5145
|
+
tool: toolName,
|
|
5146
|
+
why: "a scenario that starts with this tool was not armed in time — it misses this call only",
|
|
5147
|
+
});
|
|
5148
|
+
}
|
|
5149
|
+
// A look leaves them in place, for a while.
|
|
5150
|
+
if (next.calls >= NEXT_CALLS_MAX)
|
|
5151
|
+
this.endNext(run, "calls_limit", toolName);
|
|
5152
|
+
}
|
|
5153
|
+
return undefined;
|
|
5154
|
+
}
|
|
5155
|
+
/** A next scenario's hit: bookkeeping as for any plan armed mid-task, then its directive in the call's place. */
|
|
5156
|
+
nextHit(session, run, entry, state, toolName) {
|
|
5157
|
+
state.sessionId = session.sessionId;
|
|
5158
|
+
state.sessionCwd = session.cwd ?? this.opts.cwd;
|
|
5159
|
+
// The request this plan serves, for a note at a decision of its own (SB-18).
|
|
5160
|
+
state.request = run.input || undefined;
|
|
5161
|
+
if (this.declineWithoutDeliveryTool(state)) {
|
|
5162
|
+
state.reported = true;
|
|
5163
|
+
this.endNext(run, "no_delivery_tool", toolName);
|
|
5164
|
+
return undefined;
|
|
5165
|
+
}
|
|
5166
|
+
state.usageMark = markTranscriptUsage(session.transcriptPath);
|
|
5167
|
+
state.markUsage = () => markTranscriptUsage(session.transcriptPath);
|
|
5168
|
+
if (state.kind === "segment" && state.segment && state.scenarioId) {
|
|
5169
|
+
for (const earlier of run.replays) {
|
|
5170
|
+
if (earlier === state || !earlier.plan || earlier.kind !== "scenario")
|
|
5171
|
+
continue;
|
|
5172
|
+
if (earlier.matchedRunId !== state.segment.runId)
|
|
5173
|
+
continue;
|
|
5174
|
+
earlier.sharedWith.push(state.scenarioId);
|
|
5175
|
+
}
|
|
5176
|
+
}
|
|
5177
|
+
run.replays.push(state);
|
|
5178
|
+
run.replay = state;
|
|
5179
|
+
run.intent.armed += 1;
|
|
5180
|
+
// The agent's work from here is this scenario's, never recorded as the
|
|
5181
|
+
// agent's own after the decision: a fragment still due is dropped, and
|
|
5182
|
+
// one the agent's own looks opened ends with them.
|
|
5183
|
+
this.closeDecisionFragment(session, run, state);
|
|
5184
|
+
this.dropDecisionFragment(run, state);
|
|
5185
|
+
this.endNext(run, "hit", toolName, state);
|
|
5186
|
+
const directive = this.replay.directiveFor(state) ?? "";
|
|
5187
|
+
logLine("replay.next_hit", {
|
|
5188
|
+
run: run.runId,
|
|
5189
|
+
scenario: state.scenarioId ?? undefined,
|
|
5190
|
+
plan: entry.planId,
|
|
5191
|
+
kind: state.kind,
|
|
5192
|
+
mode: state.mode,
|
|
5193
|
+
tool: toolName,
|
|
5194
|
+
rearmed: entry.rearmed || undefined,
|
|
5195
|
+
why: "the agent's call is the first step of a calculated scenario that follows the decision — it runs in the call's place",
|
|
5196
|
+
});
|
|
5197
|
+
return this.preToolAnswer(this.replay.deliverInstead(directive, toolName));
|
|
5198
|
+
}
|
|
5199
|
+
/**
|
|
5200
|
+
* A next scenario hit after the agent's own calls — looks — opened the
|
|
5201
|
+
* fragment of its work after the decision: the fragment ends here, with what
|
|
5202
|
+
* the agent did itself, and nothing more of this turn is recorded into it.
|
|
5203
|
+
* The scenario's own calls, pinned in steer mode or run in direct mode, are
|
|
5204
|
+
* not the agent's work, and would make the fragment a copy of that scenario
|
|
5205
|
+
* (offer1.md O7). A fragment still due is dropped instead
|
|
5206
|
+
* ({@link dropDecisionFragment}).
|
|
5207
|
+
*/
|
|
5208
|
+
closeDecisionFragment(session, run, by) {
|
|
5209
|
+
const f = run.fragment;
|
|
5210
|
+
if (!f || f.state !== "open" || !f.decision || !run.recording)
|
|
5211
|
+
return;
|
|
5212
|
+
const runId = run.runId;
|
|
5213
|
+
const cost = this.costSince(session, f.usageMark);
|
|
5214
|
+
const durationMs = f.startedAtMs ? Date.now() - f.startedAtMs : 0;
|
|
5215
|
+
// Behind the fragment's own steps in the queue, as every finish is.
|
|
5216
|
+
this.queue.push(() => {
|
|
5217
|
+
this.recorder.finishRun(runId, undefined, {
|
|
5218
|
+
durationMs,
|
|
5219
|
+
costUsd: cost.measured ? cost.usd : undefined,
|
|
5220
|
+
});
|
|
5221
|
+
});
|
|
5222
|
+
run.recording = false;
|
|
5223
|
+
f.state = "closed";
|
|
5224
|
+
logLine("run.fragment_closed", {
|
|
5225
|
+
run: runId,
|
|
5226
|
+
scenario: f.scenarioId,
|
|
5227
|
+
step: f.stepIndex,
|
|
5228
|
+
by: by.scenarioId ?? undefined,
|
|
5229
|
+
why: "a scenario that follows the decision takes over after the agent's own first calls — the fragment ends with what the agent did itself",
|
|
5230
|
+
});
|
|
5231
|
+
}
|
|
5232
|
+
/**
|
|
5233
|
+
* End this turn's next scenarios (offer1.md O6): every one but the hit is
|
|
5234
|
+
* never announced, so never runs and is never reported.
|
|
5235
|
+
*/
|
|
5236
|
+
endNext(run, why, toolName, kept) {
|
|
5237
|
+
const next = run.next;
|
|
5238
|
+
if (!next || next.ended)
|
|
5239
|
+
return;
|
|
5240
|
+
next.ended = why;
|
|
5241
|
+
for (const e of next.entries)
|
|
5242
|
+
dropNextEntry(e, kept);
|
|
5243
|
+
if (why !== "hit" && next.entries.length > 0) {
|
|
5244
|
+
logLine("replay.next_miss", {
|
|
5245
|
+
run: run.runId,
|
|
5246
|
+
reason: why,
|
|
5247
|
+
tool: toolName,
|
|
5248
|
+
calls: next.calls,
|
|
5249
|
+
scenarios: next.entries.map((e) => e.match.scenarioId).join(","),
|
|
5250
|
+
why: NEXT_MISS_WHY[why] ?? "none of the scenarios that follow the decision was used",
|
|
5251
|
+
});
|
|
5252
|
+
}
|
|
5253
|
+
next.entries = [];
|
|
5254
|
+
}
|
|
4619
5255
|
/**
|
|
4620
5256
|
* Whether an intent probe may arm a plan in its own `PreToolUse` — and so
|
|
4621
5257
|
* whether that hook waits for it. See {@link ControlServerOptions.intentArm}.
|
|
@@ -4699,7 +5335,35 @@ export class ControlServer {
|
|
|
4699
5335
|
const h = state?.handover;
|
|
4700
5336
|
if (!h || !state?.scenarioId)
|
|
4701
5337
|
return;
|
|
4702
|
-
run.fragment = {
|
|
5338
|
+
run.fragment = {
|
|
5339
|
+
scenarioId: state.scenarioId,
|
|
5340
|
+
stepIndex: h.stepIndex,
|
|
5341
|
+
state: "due",
|
|
5342
|
+
...(h.kind === "decision" ? { decision: true } : {}),
|
|
5343
|
+
};
|
|
5344
|
+
}
|
|
5345
|
+
/**
|
|
5346
|
+
* Drop a fragment scheduled at a decision (the service's
|
|
5347
|
+
* segmentsbreakdown.md SB-16) when a plan the intent check armed takes the
|
|
5348
|
+
* agent's next call: what follows is that plan's half, not the other one,
|
|
5349
|
+
* and recorded as the other half it would become a wrong segment C. Cleared
|
|
5350
|
+
* rather than ended, so a later hand-over of that plan schedules its own
|
|
5351
|
+
* fragment, as any plan's does in a turn with none. Any other fragment is
|
|
5352
|
+
* left as it is.
|
|
5353
|
+
*/
|
|
5354
|
+
dropDecisionFragment(run, by) {
|
|
5355
|
+
const f = run.fragment;
|
|
5356
|
+
if (!f || f.state !== "due" || !f.decision)
|
|
5357
|
+
return;
|
|
5358
|
+
run.fragment = undefined;
|
|
5359
|
+
logLine("run.fragment_dropped", {
|
|
5360
|
+
run: run.runId,
|
|
5361
|
+
scenario: f.scenarioId,
|
|
5362
|
+
step: f.stepIndex,
|
|
5363
|
+
by: by?.scenarioId ?? undefined,
|
|
5364
|
+
kind: by?.kind,
|
|
5365
|
+
why: "a plan the intent check armed takes the agent's next step after the decision — that work is not the other half",
|
|
5366
|
+
});
|
|
4703
5367
|
}
|
|
4704
5368
|
/**
|
|
4705
5369
|
* Open the fragment run and record from this call on (fallbk.md D7).
|
|
@@ -4719,9 +5383,21 @@ export class ControlServer {
|
|
|
4719
5383
|
return;
|
|
4720
5384
|
const input = reasoning.trim() || `${toolName} ${serializeCapped(redact(args))}`.slice(0, 2_000);
|
|
4721
5385
|
const fallbackOf = { scenarioId: f.scenarioId, stepIndex: f.stepIndex };
|
|
5386
|
+
// The agent's own call after a plan's decision (SB-16): this runner's word
|
|
5387
|
+
// that the work that follows is the agent's choice, not a plan's half.
|
|
5388
|
+
// The service reads it beside its own mark of the fragment (the
|
|
5389
|
+
// service's segmentsbreakdown.md D14).
|
|
5390
|
+
// With the turn's own request (offer1.md O2): the service calculates what
|
|
5391
|
+
// the agent does here with it, so the values the request names are read
|
|
5392
|
+
// from the next request instead of being fixed. Redacted and cut, as the
|
|
5393
|
+
// prompt itself is wherever it leaves this machine.
|
|
5394
|
+
const request = run.input ? redactText(run.input).slice(0, DECISION_REQUEST_MAX) : "";
|
|
5395
|
+
const decisionOf = f.decision
|
|
5396
|
+
? { scenarioId: f.scenarioId, stepIndex: f.stepIndex, ...(request ? { request } : {}) }
|
|
5397
|
+
: undefined;
|
|
4722
5398
|
// The fragment is a run of the same host as the turn it continues.
|
|
4723
5399
|
const framework = frameworkOf(session);
|
|
4724
|
-
const id = this.recorder.startRun(input, { ...this.runMetadata(session), fallbackOf }, framework ? { fallbackOf, framework } : { fallbackOf });
|
|
5400
|
+
const id = this.recorder.startRun(input, { ...this.runMetadata(session), fallbackOf, ...(decisionOf ? { decisionOf } : {}) }, framework ? { fallbackOf, framework } : { fallbackOf });
|
|
4725
5401
|
for (const c of run.correlations.values()) {
|
|
4726
5402
|
c.settled = true;
|
|
4727
5403
|
if (c.fallback)
|
|
@@ -4738,6 +5414,7 @@ export class ControlServer {
|
|
|
4738
5414
|
run: id,
|
|
4739
5415
|
scenario: f.scenarioId,
|
|
4740
5416
|
step: f.stepIndex,
|
|
5417
|
+
decision: f.decision || undefined,
|
|
4741
5418
|
why: "recording the agent's own work after the scenario handed over",
|
|
4742
5419
|
});
|
|
4743
5420
|
if (!isRunCreationAware(this.recorder))
|
|
@@ -4977,12 +5654,17 @@ export class ControlServer {
|
|
|
4977
5654
|
const stashed = this.claimStashedNote(payload.session_id);
|
|
4978
5655
|
if (stashed)
|
|
4979
5656
|
context.push(stashed);
|
|
4980
|
-
const
|
|
5657
|
+
const postSession = this.sessions.get(payload.session_id ?? "unknown-session");
|
|
5658
|
+
const run = postSession?.run;
|
|
4981
5659
|
if (run && !run.finished) {
|
|
4982
5660
|
// The answer, if one comes, is written after this call (AC11).
|
|
4983
5661
|
if (typeof payload.tool_use_id === "string" && payload.tool_use_id)
|
|
4984
5662
|
run.lastToolUseId = payload.tool_use_id;
|
|
4985
5663
|
this.noteHandover(run);
|
|
5664
|
+
// A plan that stopped at its decision on this call: what can follow it is
|
|
5665
|
+
// asked for now, while the agent reads the note and decides (offer1.md O4).
|
|
5666
|
+
if (postSession)
|
|
5667
|
+
this.noteDecision(postSession, run);
|
|
4986
5668
|
const note = this.replay.takeNote(run.replay);
|
|
4987
5669
|
if (note)
|
|
4988
5670
|
context.push(note);
|
|
@@ -5547,6 +6229,11 @@ export class ControlServer {
|
|
|
5547
6229
|
const result = await this.replay.runArmed(state, signal);
|
|
5548
6230
|
if (signal?.aborted)
|
|
5549
6231
|
this.stashCancelledNote(call.sessionId, state);
|
|
6232
|
+
// A direct plan that stopped at its decision: what can follow it is asked
|
|
6233
|
+
// for at once, before the agent has even read the results (offer1.md O4).
|
|
6234
|
+
const runSession = this.sessions.get(call.sessionId);
|
|
6235
|
+
if (runSession && runSession.run === call.run && !signal?.aborted)
|
|
6236
|
+
this.noteDecision(runSession, call.run);
|
|
5550
6237
|
return result;
|
|
5551
6238
|
}
|
|
5552
6239
|
finally {
|