@basein/runner 0.3.22 → 0.3.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/bir-hooks.d.ts +5 -0
- package/dist/bin/bir-hooks.js +5 -0
- package/dist/control/pin-guard.d.ts +16 -0
- package/dist/control/pin-guard.js +36 -0
- package/dist/control/server.d.ts +63 -0
- package/dist/control/server.js +394 -8
- package/dist/record/recorder.d.ts +41 -1
- package/dist/record/remote-recorder.js +8 -0
- package/dist/replay/controller.d.ts +58 -3
- package/dist/replay/controller.js +56 -11
- package/docs/calculatedReplayGuide.md +1 -0
- package/package.json +1 -1
package/dist/bin/bir-hooks.d.ts
CHANGED
|
@@ -109,6 +109,11 @@
|
|
|
109
109
|
* call. Unset (or 0) is off: no probe is sent and
|
|
110
110
|
* nothing arms mid-turn
|
|
111
111
|
* BIR_INTENT_ARM_BUDGET_MS how long a call waits for that probe (default 500)
|
|
112
|
+
* BIR_CONTINUATIONS 0 stops asking, after a plan handed over, for the
|
|
113
|
+
* plan of the work recorded after an earlier
|
|
114
|
+
* hand-over at that step (default on: one request
|
|
115
|
+
* per hand-over; it runs only when the agent's next
|
|
116
|
+
* call is its first step)
|
|
112
117
|
* BIR_INTENT_REQUESTS_PER_TURN probes one turn may send (default 40)
|
|
113
118
|
* BIR_INTENT_MATCH_BUDGET_MS one intent probe's own ceiling (default 1500)
|
|
114
119
|
* BIR_REPLAY_BUDGET_MS whole-plan ceiling, direct mode (unset: the
|
package/dist/bin/bir-hooks.js
CHANGED
|
@@ -109,6 +109,11 @@
|
|
|
109
109
|
* call. Unset (or 0) is off: no probe is sent and
|
|
110
110
|
* nothing arms mid-turn
|
|
111
111
|
* BIR_INTENT_ARM_BUDGET_MS how long a call waits for that probe (default 500)
|
|
112
|
+
* BIR_CONTINUATIONS 0 stops asking, after a plan handed over, for the
|
|
113
|
+
* plan of the work recorded after an earlier
|
|
114
|
+
* hand-over at that step (default on: one request
|
|
115
|
+
* per hand-over; it runs only when the agent's next
|
|
116
|
+
* call is its first step)
|
|
112
117
|
* BIR_INTENT_REQUESTS_PER_TURN probes one turn may send (default 40)
|
|
113
118
|
* BIR_INTENT_MATCH_BUDGET_MS one intent probe's own ceiling (default 1500)
|
|
114
119
|
* BIR_REPLAY_BUDGET_MS whole-plan ceiling, direct mode (unset: the
|
|
@@ -85,4 +85,20 @@ export declare function commandShape(command: string, depth?: number, escape?: s
|
|
|
85
85
|
* only the general rule is applied there.
|
|
86
86
|
*/
|
|
87
87
|
export declare function pinCompatible(toolName: string, modelArgs: unknown, pinnedArgs: unknown, otherHost?: boolean): PinVerdict;
|
|
88
|
+
/**
|
|
89
|
+
* Whether the model's own call is a continuation's first step: the same call,
|
|
90
|
+
* not only the same kind of call (the service's fallbk.md Part 3).
|
|
91
|
+
*
|
|
92
|
+
* Stricter than {@link pinCompatible}, on purpose. A pin takes a call the
|
|
93
|
+
* model was told to make — the directive named the step — and one argument in
|
|
94
|
+
* common is enough to know it is that call. A continuation starts on a call
|
|
95
|
+
* nobody told the model to make: after a hand-over, the call is the model's
|
|
96
|
+
* own choice of what to do next, and the plan may run only when that choice
|
|
97
|
+
* is the one the recording made. Looking up another person, restarting
|
|
98
|
+
* another worker, reading another file is another choice. So the call must
|
|
99
|
+
* pass the pin guard, share at least one argument with the step, and hold
|
|
100
|
+
* the same value in every argument they share — text compared without
|
|
101
|
+
* surrounding space or letter case, anything else exactly.
|
|
102
|
+
*/
|
|
103
|
+
export declare function sameFirstCall(toolName: string, modelArgs: unknown, stepArgs: unknown): PinVerdict;
|
|
88
104
|
//# sourceMappingURL=pin-guard.d.ts.map
|
|
@@ -545,4 +545,40 @@ export function pinCompatible(toolName, modelArgs, pinnedArgs, otherHost = false
|
|
|
545
545
|
? { ok: true }
|
|
546
546
|
: { ok: false, why: "it shares no argument with the step" };
|
|
547
547
|
}
|
|
548
|
+
/**
|
|
549
|
+
* Whether the model's own call is a continuation's first step: the same call,
|
|
550
|
+
* not only the same kind of call (the service's fallbk.md Part 3).
|
|
551
|
+
*
|
|
552
|
+
* Stricter than {@link pinCompatible}, on purpose. A pin takes a call the
|
|
553
|
+
* model was told to make — the directive named the step — and one argument in
|
|
554
|
+
* common is enough to know it is that call. A continuation starts on a call
|
|
555
|
+
* nobody told the model to make: after a hand-over, the call is the model's
|
|
556
|
+
* own choice of what to do next, and the plan may run only when that choice
|
|
557
|
+
* is the one the recording made. Looking up another person, restarting
|
|
558
|
+
* another worker, reading another file is another choice. So the call must
|
|
559
|
+
* pass the pin guard, share at least one argument with the step, and hold
|
|
560
|
+
* the same value in every argument they share — text compared without
|
|
561
|
+
* surrounding space or letter case, anything else exactly.
|
|
562
|
+
*/
|
|
563
|
+
export function sameFirstCall(toolName, modelArgs, stepArgs) {
|
|
564
|
+
const verdict = pinCompatible(toolName, modelArgs, stepArgs);
|
|
565
|
+
if (!verdict.ok)
|
|
566
|
+
return verdict;
|
|
567
|
+
const model = record(modelArgs);
|
|
568
|
+
const step = record(stepArgs);
|
|
569
|
+
const shared = Object.keys(model).filter((key) => Object.prototype.hasOwnProperty.call(step, key));
|
|
570
|
+
if (shared.length === 0) {
|
|
571
|
+
return stableStringify(model) === stableStringify(step)
|
|
572
|
+
? { ok: true }
|
|
573
|
+
: { ok: false, why: "it shares no argument with the step" };
|
|
574
|
+
}
|
|
575
|
+
const differs = shared.find((key) => !sameValue(model[key], step[key]));
|
|
576
|
+
return differs === undefined ? { ok: true } : { ok: false, why: `its ${differs} is not the step's` };
|
|
577
|
+
}
|
|
578
|
+
function sameValue(a, b) {
|
|
579
|
+
if (typeof a === "string" && typeof b === "string") {
|
|
580
|
+
return a.trim().toLowerCase() === b.trim().toLowerCase();
|
|
581
|
+
}
|
|
582
|
+
return stableStringify(a) === stableStringify(b);
|
|
583
|
+
}
|
|
548
584
|
//# sourceMappingURL=pin-guard.js.map
|
package/dist/control/server.d.ts
CHANGED
|
@@ -282,6 +282,15 @@ export interface ControlServerOptions {
|
|
|
282
282
|
* failing. Unset or `BIR_INTENT_ARM=0` is off.
|
|
283
283
|
*/
|
|
284
284
|
intentArm?: boolean;
|
|
285
|
+
/**
|
|
286
|
+
* Whether the model's first call after a plan handed over asks the service
|
|
287
|
+
* for the plan of the work recorded after an earlier hand-over at that step,
|
|
288
|
+
* and runs it when that call is its first step (the service's fallbk.md
|
|
289
|
+
* Part 3). On by default: it is one request per hand-over, never per call.
|
|
290
|
+
* `BIR_CONTINUATIONS=0` turns it off; so does `BIR_INTENT_MATCH=0`, which
|
|
291
|
+
* turns off every mid-turn request.
|
|
292
|
+
*/
|
|
293
|
+
continuations?: boolean;
|
|
285
294
|
/**
|
|
286
295
|
* The verdict hot paths consult before they wait on the service. Defaults to
|
|
287
296
|
* the process-wide one every service call reports to; tests pass their own.
|
|
@@ -1156,6 +1165,60 @@ export declare class ControlServer {
|
|
|
1156
1165
|
* matched call has run by then, and the plan's first step would repeat it.
|
|
1157
1166
|
*/
|
|
1158
1167
|
private tryIntentMatch;
|
|
1168
|
+
/**
|
|
1169
|
+
* A continuation (the service's fallbk.md Part 3), on the model's first call
|
|
1170
|
+
* after a plan handed over.
|
|
1171
|
+
*
|
|
1172
|
+
* The model's work after an earlier hand-over of the same plan at the same
|
|
1173
|
+
* step was recorded once, as a fragment, and the service may have calculated
|
|
1174
|
+
* it into a plan. Until 0.3.23 nothing asked for that plan: the only way a
|
|
1175
|
+
* plan started mid-task was a match of the model's words against plan
|
|
1176
|
+
* intents ({@link tryIntentMatch}, off by default), and a one-line note
|
|
1177
|
+
* after a hand-over does not read like a plan's intent paragraph. Here the
|
|
1178
|
+
* hand-over itself is the question — the plan and the step — so the service
|
|
1179
|
+
* names one recording and matches no words.
|
|
1180
|
+
*
|
|
1181
|
+
* It arms only when the call the model is making is that plan's first step —
|
|
1182
|
+
* its tool, and arguments the pin guard would pin to the step's input — and
|
|
1183
|
+
* is then answered in that call's place, as an intent plan whose first step
|
|
1184
|
+
* the call is. The plan before it stopped because the next step needed a
|
|
1185
|
+
* judgement. This call is the model's judgement; a plan that starts with
|
|
1186
|
+
* another call is a way the model did not take, and must not run.
|
|
1187
|
+
*
|
|
1188
|
+
* Its values are read from the turn's own request — the rest of the task is
|
|
1189
|
+
* the one the user asked for — with the model's words, the call it was about
|
|
1190
|
+
* to make and this turn's tool results beside it, as for any plan started at
|
|
1191
|
+
* a tool call.
|
|
1192
|
+
*
|
|
1193
|
+
* Asked once per hand-over, waited on {@link CONTINUATION_PROBE_MS} and
|
|
1194
|
+
* {@link CONTINUATION_ARM_MS} at most, and not at all while the service is
|
|
1195
|
+
* failing. Every way it does not arm leaves the turn as it was: the model's
|
|
1196
|
+
* call runs, and its work is recorded as the fragment. A plan handed out and
|
|
1197
|
+
* not armed is reported for its ticket, with the reason.
|
|
1198
|
+
*/
|
|
1199
|
+
private tryContinuation;
|
|
1200
|
+
/**
|
|
1201
|
+
* A continuation handed out and not armed: kept, retired, for its ticket's
|
|
1202
|
+
* report at the seal (`not_steered`, with the reason, unmeasured). `code`
|
|
1203
|
+
* says why when no gate did.
|
|
1204
|
+
*/
|
|
1205
|
+
private keepDeclinedContinuation;
|
|
1206
|
+
/** A continuation request's answer that came after the call it was for went ahead. */
|
|
1207
|
+
private noteLateContinuation;
|
|
1208
|
+
/**
|
|
1209
|
+
* Every scenario this turn was handed and armed or declined: never handed
|
|
1210
|
+
* out again in it. A plan that handed over is among them, so its own
|
|
1211
|
+
* continuation request can never return it.
|
|
1212
|
+
*/
|
|
1213
|
+
private armedScenarioIds;
|
|
1214
|
+
/**
|
|
1215
|
+
* Whether the model's call is a continuation's first step: its tool, and
|
|
1216
|
+
* the step's own input in every argument the two share (pin-guard.ts
|
|
1217
|
+
* `sameFirstCall`). Anything that cannot be computed is "no".
|
|
1218
|
+
*/
|
|
1219
|
+
private isContinuationFirstCall;
|
|
1220
|
+
/** Whether continuations are asked for: on unless `BIR_CONTINUATIONS=0`. */
|
|
1221
|
+
private continuationsOn;
|
|
1159
1222
|
/**
|
|
1160
1223
|
* Whether the model's call is plausibly `state`'s first step: its tool, and
|
|
1161
1224
|
* arguments the pin guard would pin to that step's input. Anything that
|
package/dist/control/server.js
CHANGED
|
@@ -30,7 +30,7 @@ import { join, resolve as resolvePath } from "node:path";
|
|
|
30
30
|
import { BIR_CALL_ID, fingerprint, newCallId, parseQualifiedName, qualifyToolName, } from "./correlation.js";
|
|
31
31
|
import { StepIndexAllocator } from "./ordering.js";
|
|
32
32
|
import { pidAlive } from "./discovery.js";
|
|
33
|
-
import { pinCompatible } from "./pin-guard.js";
|
|
33
|
+
import { pinCompatible, sameFirstCall } from "./pin-guard.js";
|
|
34
34
|
import { agentDayOf, contextForToolUse, intentForToolUse, markTranscriptUsage, recentToolResults, settledLastAssistantText, settledTextAfterToolUse, thinkingForToolUse, usageSince, } from "./transcript.js";
|
|
35
35
|
import { isHousekeepingCall } from "../record/housekeeping.js";
|
|
36
36
|
import { asSent, userText } from "../record/prompt-text.js";
|
|
@@ -106,6 +106,27 @@ const INTENT_ARM_PROBE_MS = 500;
|
|
|
106
106
|
const INTENT_ARM_DERIVE_MS = 1_500;
|
|
107
107
|
/** How long a fragment's `PreToolUse` waits to learn whether its run was created. */
|
|
108
108
|
const FRAGMENT_CREATE_MS = 300;
|
|
109
|
+
/**
|
|
110
|
+
* How long the model's first call after a hand-over waits for the service to
|
|
111
|
+
* say whether the work recorded after an earlier hand-over at that step has a
|
|
112
|
+
* plan (the service's fallbk.md Part 3). One lookup, no words matched: it
|
|
113
|
+
* answers in a round trip. Asked once per hand-over, never per call.
|
|
114
|
+
*/
|
|
115
|
+
const CONTINUATION_PROBE_MS = 1_500;
|
|
116
|
+
/**
|
|
117
|
+
* How long the continuation request itself may take — longer than the call
|
|
118
|
+
* waits for it, so that an answer too late for the call still lands, and the
|
|
119
|
+
* ticket it may carry is reported as late instead of never.
|
|
120
|
+
*/
|
|
121
|
+
const CONTINUATION_REQUEST_MS = 6_000;
|
|
122
|
+
/**
|
|
123
|
+
* What a continuation may spend arming — reading its values — inside that
|
|
124
|
+
* `PreToolUse`. Longer than an intent arm may: it is asked once per hand-over,
|
|
125
|
+
* the plan it arms does the rest of the task, and a value the request names
|
|
126
|
+
* is read by the service, which gives a reading 6 s. With the probe it stays
|
|
127
|
+
* well inside the route's 12 s.
|
|
128
|
+
*/
|
|
129
|
+
const CONTINUATION_ARM_MS = 7_000;
|
|
109
130
|
/** What `close()` spends draining the last runs to the service before it lets go. */
|
|
110
131
|
const CLOSE_DRAIN_MS = 5_000;
|
|
111
132
|
/** How long a background drain follows one run before it stops waiting for it. */
|
|
@@ -471,6 +492,17 @@ function reportedArgs(args) {
|
|
|
471
492
|
function score(value) {
|
|
472
493
|
return typeof value === "number" && Number.isFinite(value) ? value.toFixed(3) : undefined;
|
|
473
494
|
}
|
|
495
|
+
/**
|
|
496
|
+
* The tool a served scenario starts with, or undefined when the payload does
|
|
497
|
+
* not say. Untyped JSON from the service, read without trusting its shape.
|
|
498
|
+
*/
|
|
499
|
+
function firstStepTool(scenario) {
|
|
500
|
+
const steps = scenario?.steps;
|
|
501
|
+
if (!Array.isArray(steps) || steps.length === 0)
|
|
502
|
+
return undefined;
|
|
503
|
+
const name = steps[0]?.toolName;
|
|
504
|
+
return typeof name === "string" ? name : undefined;
|
|
505
|
+
}
|
|
474
506
|
function positiveEnv(name, fallback) {
|
|
475
507
|
const n = Number(process.env[name]);
|
|
476
508
|
return Number.isFinite(n) && n > 0 ? n : fallback;
|
|
@@ -920,6 +952,14 @@ export class ControlServer {
|
|
|
920
952
|
unannounced: s.run.pendingDirective?.state === s.run.replay ? true : undefined,
|
|
921
953
|
running: this.runningPlans.has(s.run.replay) ? true : undefined,
|
|
922
954
|
armedBy: s.run.replay.armedBy,
|
|
955
|
+
// A continuation (the service's fallbk.md Part 3): the plan it
|
|
956
|
+
// continues, and the step that plan stopped in front of.
|
|
957
|
+
continues: s.run.replay.continuation
|
|
958
|
+
? {
|
|
959
|
+
scenarioId: s.run.replay.continuation.ofScenarioId,
|
|
960
|
+
stepIndex: s.run.replay.continuation.stepIndex,
|
|
961
|
+
}
|
|
962
|
+
: null,
|
|
923
963
|
handover: s.run.replay.handover
|
|
924
964
|
? { kind: s.run.replay.handover.kind, stepIndex: s.run.replay.handover.stepIndex }
|
|
925
965
|
: null,
|
|
@@ -934,7 +974,13 @@ export class ControlServer {
|
|
|
934
974
|
armed: s.run.intent.armed,
|
|
935
975
|
}
|
|
936
976
|
: null,
|
|
937
|
-
fragment: s.run?.fragment
|
|
977
|
+
fragment: s.run?.fragment
|
|
978
|
+
? {
|
|
979
|
+
state: s.run.fragment.state,
|
|
980
|
+
...(s.run.fragment.asked ? { asked: true } : {}),
|
|
981
|
+
...(s.run.fragment.counted ? { counted: true } : {}),
|
|
982
|
+
}
|
|
983
|
+
: null,
|
|
938
984
|
// The service's offer to record the turn as a recording's newest run,
|
|
939
985
|
// and what became of it (newestRun.md).
|
|
940
986
|
repeat: s.run?.repeat ? { state: s.run.repeat.state, of: s.run.repeat.runId } : null,
|
|
@@ -2989,12 +3035,36 @@ export class ControlServer {
|
|
|
2989
3035
|
? { runId: r.openedRunId, ownCalls: r.ownCalls ?? 0 }
|
|
2990
3036
|
: undefined;
|
|
2991
3037
|
const prompt = states.find((s) => s.armedBy === "prompt");
|
|
3038
|
+
// The plans that continued a plan in this turn (the service's fallbk.md
|
|
3039
|
+
// Part 3), directly or through one another: its hand-over's follow-on
|
|
3040
|
+
// work. Its baseline is the whole task, theirs only the part after the
|
|
3041
|
+
// hand-over, so its report bills the whole turn — its window runs through
|
|
3042
|
+
// theirs, and their readings are booked on it — and theirs book no saving.
|
|
3043
|
+
// Billed apart, the part after the hand-over would be saved twice.
|
|
3044
|
+
const continuedBy = (state, from) => {
|
|
3045
|
+
const chain = [];
|
|
3046
|
+
const ids = new Set(state.scenarioId ? [state.scenarioId] : []);
|
|
3047
|
+
for (const s of states.slice(from + 1)) {
|
|
3048
|
+
if (s.continuation && ids.has(s.continuation.ofScenarioId)) {
|
|
3049
|
+
chain.push(s);
|
|
3050
|
+
if (s.scenarioId)
|
|
3051
|
+
ids.add(s.scenarioId);
|
|
3052
|
+
}
|
|
3053
|
+
}
|
|
3054
|
+
// A continuation declined after its values were read spent that reading
|
|
3055
|
+
// on this turn too, and its own report books nothing.
|
|
3056
|
+
const declined = (run.declinedContinuations ?? []).filter((s) => s.continuation !== undefined && ids.has(s.continuation.ofScenarioId));
|
|
3057
|
+
const deriveUsd = [...chain, ...declined].reduce((usd, s) => usd + s.deriveCostUsd, 0);
|
|
3058
|
+
return { chain, deriveUsd };
|
|
3059
|
+
};
|
|
2992
3060
|
// One report per plan armed this turn (fallbk.md §Runner 4). Each is billed
|
|
2993
3061
|
// for its own window of the transcript — from when it armed to when the next
|
|
2994
3062
|
// one did — so the turn's tokens are split between them, never counted twice.
|
|
2995
3063
|
// A turn with a single prompt-armed plan is exactly the report it always was.
|
|
2996
3064
|
states.forEach((state, i) => {
|
|
2997
|
-
const
|
|
3065
|
+
const { chain, deriveUsd: chainDeriveUsd } = continuedBy(state, i);
|
|
3066
|
+
// The plan whose arm ends this one's window: never one that continued it.
|
|
3067
|
+
const next = states.slice(i + 1).find((s) => !chain.includes(s));
|
|
2998
3068
|
const nextMark = next?.armedBy === "intent" ? next.usageMark : undefined;
|
|
2999
3069
|
let windowCost = cost;
|
|
3000
3070
|
let windowMs = durationMs;
|
|
@@ -3027,6 +3097,7 @@ export class ControlServer {
|
|
|
3027
3097
|
runId: run.runId,
|
|
3028
3098
|
cutShort: cutShort || undefined,
|
|
3029
3099
|
redoneBy: state === prompt && state.ranToEnd === true ? redoneBy : undefined,
|
|
3100
|
+
chainDeriveUsd,
|
|
3030
3101
|
});
|
|
3031
3102
|
if (!report)
|
|
3032
3103
|
return;
|
|
@@ -3087,6 +3158,23 @@ export class ControlServer {
|
|
|
3087
3158
|
if (report)
|
|
3088
3159
|
this.queue.push(() => recorder.reportExecution(report));
|
|
3089
3160
|
}
|
|
3161
|
+
// Continuations handed out and not armed (the service's fallbk.md Part 3):
|
|
3162
|
+
// a report for each ticket, with the reason, so the console says why and
|
|
3163
|
+
// not "no report". Nothing of theirs ran: unmeasured and costless.
|
|
3164
|
+
for (const declined of run.declinedContinuations ?? []) {
|
|
3165
|
+
if (states.includes(declined))
|
|
3166
|
+
continue;
|
|
3167
|
+
const report = this.replay.buildReport(declined, {
|
|
3168
|
+
sessionCostUsd: 0,
|
|
3169
|
+
measured: false,
|
|
3170
|
+
durationMs: 0,
|
|
3171
|
+
prompt: run.input || undefined,
|
|
3172
|
+
siblings: [...states, declined],
|
|
3173
|
+
runId: run.runId,
|
|
3174
|
+
});
|
|
3175
|
+
if (report)
|
|
3176
|
+
this.queue.push(() => recorder.reportExecution(report));
|
|
3177
|
+
}
|
|
3090
3178
|
}
|
|
3091
3179
|
/**
|
|
3092
3180
|
* The plan's own calls of the turn, whole, as an answered turn's report
|
|
@@ -3675,7 +3763,39 @@ export class ControlServer {
|
|
|
3675
3763
|
return {};
|
|
3676
3764
|
});
|
|
3677
3765
|
}
|
|
3678
|
-
|
|
3766
|
+
// 2a. The model's first call after a plan handed over: the work recorded
|
|
3767
|
+
// after an earlier hand-over at the same step may have a plan of its
|
|
3768
|
+
// own (the service's fallbk.md Part 3). Asked with the hand-over, not
|
|
3769
|
+
// with words, and armed only when this call is its first step.
|
|
3770
|
+
const dueFragment = run.fragment;
|
|
3771
|
+
const before = run.replay;
|
|
3772
|
+
const continued = planning
|
|
3773
|
+
? undefined
|
|
3774
|
+
: await this.tryContinuation(session, run, toolName, intent, modelArgs);
|
|
3775
|
+
if (continued) {
|
|
3776
|
+
if (!overran)
|
|
3777
|
+
return continued;
|
|
3778
|
+
// The deadline answered first: the call ran as the model wrote it, and
|
|
3779
|
+
// the plan was never announced. The model's own work from this call is
|
|
3780
|
+
// the fragment, as it would have been with no plan.
|
|
3781
|
+
const late = run.replay;
|
|
3782
|
+
if (late && late !== before && late.continuation) {
|
|
3783
|
+
logLine("replay.decision", {
|
|
3784
|
+
verdict: "no-steer",
|
|
3785
|
+
code: "arm_unannounced",
|
|
3786
|
+
run: run.runId,
|
|
3787
|
+
scenario: late.scenarioId ?? undefined,
|
|
3788
|
+
why: "the call had to go ahead before this continuation's directive could replace it — retired unannounced, so never run",
|
|
3789
|
+
});
|
|
3790
|
+
const at = run.replays.indexOf(late);
|
|
3791
|
+
if (at >= 0)
|
|
3792
|
+
run.replays.splice(at, 1);
|
|
3793
|
+
run.replay = before;
|
|
3794
|
+
this.keepDeclinedContinuation(session, run, late, "arm_unannounced");
|
|
3795
|
+
run.fragment = dueFragment;
|
|
3796
|
+
}
|
|
3797
|
+
}
|
|
3798
|
+
const armed = planning || continued
|
|
3679
3799
|
? undefined
|
|
3680
3800
|
: await this.tryIntentMatch(session, run, toolName, intent, modelArgs);
|
|
3681
3801
|
if (armed) {
|
|
@@ -4327,9 +4447,7 @@ export class ControlServer {
|
|
|
4327
4447
|
run.intent.requests += 1;
|
|
4328
4448
|
// Every scenario armed this turn, so a continuation cannot re-arm the plan
|
|
4329
4449
|
// that just handed over.
|
|
4330
|
-
const exclude = run
|
|
4331
|
-
.map((s) => s.scenarioId)
|
|
4332
|
-
.filter((id) => typeof id === "string");
|
|
4450
|
+
const exclude = this.armedScenarioIds(run);
|
|
4333
4451
|
const startedAt = Date.now();
|
|
4334
4452
|
const framework = frameworkOf(session);
|
|
4335
4453
|
const probe = this.recorder
|
|
@@ -4441,6 +4559,271 @@ export class ControlServer {
|
|
|
4441
4559
|
run.pendingDirective = { state, text: directive };
|
|
4442
4560
|
return undefined;
|
|
4443
4561
|
}
|
|
4562
|
+
/**
|
|
4563
|
+
* A continuation (the service's fallbk.md Part 3), on the model's first call
|
|
4564
|
+
* after a plan handed over.
|
|
4565
|
+
*
|
|
4566
|
+
* The model's work after an earlier hand-over of the same plan at the same
|
|
4567
|
+
* step was recorded once, as a fragment, and the service may have calculated
|
|
4568
|
+
* it into a plan. Until 0.3.23 nothing asked for that plan: the only way a
|
|
4569
|
+
* plan started mid-task was a match of the model's words against plan
|
|
4570
|
+
* intents ({@link tryIntentMatch}, off by default), and a one-line note
|
|
4571
|
+
* after a hand-over does not read like a plan's intent paragraph. Here the
|
|
4572
|
+
* hand-over itself is the question — the plan and the step — so the service
|
|
4573
|
+
* names one recording and matches no words.
|
|
4574
|
+
*
|
|
4575
|
+
* It arms only when the call the model is making is that plan's first step —
|
|
4576
|
+
* its tool, and arguments the pin guard would pin to the step's input — and
|
|
4577
|
+
* is then answered in that call's place, as an intent plan whose first step
|
|
4578
|
+
* the call is. The plan before it stopped because the next step needed a
|
|
4579
|
+
* judgement. This call is the model's judgement; a plan that starts with
|
|
4580
|
+
* another call is a way the model did not take, and must not run.
|
|
4581
|
+
*
|
|
4582
|
+
* Its values are read from the turn's own request — the rest of the task is
|
|
4583
|
+
* the one the user asked for — with the model's words, the call it was about
|
|
4584
|
+
* to make and this turn's tool results beside it, as for any plan started at
|
|
4585
|
+
* a tool call.
|
|
4586
|
+
*
|
|
4587
|
+
* Asked once per hand-over, waited on {@link CONTINUATION_PROBE_MS} and
|
|
4588
|
+
* {@link CONTINUATION_ARM_MS} at most, and not at all while the service is
|
|
4589
|
+
* failing. Every way it does not arm leaves the turn as it was: the model's
|
|
4590
|
+
* call runs, and its work is recorded as the fragment. A plan handed out and
|
|
4591
|
+
* not armed is reported for its ticket, with the reason.
|
|
4592
|
+
*/
|
|
4593
|
+
async tryContinuation(session, run, toolName, intent, modelArgs) {
|
|
4594
|
+
const f = run.fragment;
|
|
4595
|
+
if (!f || f.state !== "due" || f.asked)
|
|
4596
|
+
return undefined;
|
|
4597
|
+
if (!this.replay.enabled || !this.replay.intentMatch.enabled || !this.continuationsOn())
|
|
4598
|
+
return undefined;
|
|
4599
|
+
// Never in a session of another host: a plan is started there by the
|
|
4600
|
+
// prompt, and stepped through by an adapter that was told of it then.
|
|
4601
|
+
if (otherHost(session) || this.recordsOnly(session, run))
|
|
4602
|
+
return undefined;
|
|
4603
|
+
if (!isIntentMatcher(this.recorder))
|
|
4604
|
+
return undefined;
|
|
4605
|
+
// The model calling the delivery tool itself is no step of a plan.
|
|
4606
|
+
if (this.replay.isDirectTool(toolName))
|
|
4607
|
+
return undefined;
|
|
4608
|
+
if (!this.service.canWait()) {
|
|
4609
|
+
logDetail("replay.continuation_skipped", { run: run.runId, tool: toolName, why: "service unhealthy" });
|
|
4610
|
+
return undefined;
|
|
4611
|
+
}
|
|
4612
|
+
f.asked = true;
|
|
4613
|
+
const text = (intent?.text ?? "").trim().slice(0, 4_000);
|
|
4614
|
+
const framework = frameworkOf(session);
|
|
4615
|
+
const handover = { scenarioId: f.scenarioId, stepIndex: f.stepIndex };
|
|
4616
|
+
let landed = false;
|
|
4617
|
+
const probe = this.recorder
|
|
4618
|
+
.matchIntent({
|
|
4619
|
+
text,
|
|
4620
|
+
currentToolName: toolName,
|
|
4621
|
+
exclude: this.armedScenarioIds(run),
|
|
4622
|
+
handover,
|
|
4623
|
+
...(framework ? { framework } : {}),
|
|
4624
|
+
}, { timeoutMs: CONTINUATION_REQUEST_MS })
|
|
4625
|
+
.catch(() => null)
|
|
4626
|
+
.then((answer) => {
|
|
4627
|
+
landed = true;
|
|
4628
|
+
return answer;
|
|
4629
|
+
});
|
|
4630
|
+
const answer = await this.withinBudget(probe, CONTINUATION_PROBE_MS);
|
|
4631
|
+
if (!answer) {
|
|
4632
|
+
if (!landed) {
|
|
4633
|
+
// Too slow, not failed: the service most likely counted the hand-over
|
|
4634
|
+
// already, so the fragment says so. A plan its late answer hands out
|
|
4635
|
+
// is reported as late, for its ticket.
|
|
4636
|
+
f.counted = true;
|
|
4637
|
+
void probe.then((late) => this.noteLateContinuation(run, handover, late));
|
|
4638
|
+
}
|
|
4639
|
+
logDetail("replay.continuation_miss", {
|
|
4640
|
+
run: run.runId,
|
|
4641
|
+
tool: toolName,
|
|
4642
|
+
why: landed ? "the request failed" : "no answer in time",
|
|
4643
|
+
});
|
|
4644
|
+
return undefined;
|
|
4645
|
+
}
|
|
4646
|
+
// Only a service that knows continuations answers one. An older service
|
|
4647
|
+
// matched the text by words instead, and that is no continuation.
|
|
4648
|
+
if (answer.continuation === undefined)
|
|
4649
|
+
return undefined;
|
|
4650
|
+
f.counted = answer.continuation?.counted === true;
|
|
4651
|
+
const match = answer.matched;
|
|
4652
|
+
if (!answer.continuation || !match) {
|
|
4653
|
+
logDetail("replay.continuation_none", {
|
|
4654
|
+
run: run.runId,
|
|
4655
|
+
continues: `${f.scenarioId}@${f.stepIndex}`,
|
|
4656
|
+
recording: answer.continuation?.runId,
|
|
4657
|
+
why: answer.continuation
|
|
4658
|
+
? "the work recorded after an earlier hand-over here has no plan ready"
|
|
4659
|
+
: "no hand-over at this step was recorded before",
|
|
4660
|
+
});
|
|
4661
|
+
return undefined;
|
|
4662
|
+
}
|
|
4663
|
+
const continuation = {
|
|
4664
|
+
ofScenarioId: f.scenarioId,
|
|
4665
|
+
stepIndex: f.stepIndex,
|
|
4666
|
+
runId: answer.continuation.runId,
|
|
4667
|
+
};
|
|
4668
|
+
// Before anything is read: the plan starts with another tool than the
|
|
4669
|
+
// model's call, so the model chose another way.
|
|
4670
|
+
const first = firstStepTool(match.scenario);
|
|
4671
|
+
if (match.scenario && first !== undefined && first !== toolName) {
|
|
4672
|
+
this.keepDeclinedContinuation(session, run, this.replay.declinedContinuation(match, "not_first_step", continuation));
|
|
4673
|
+
return undefined;
|
|
4674
|
+
}
|
|
4675
|
+
const startedAt = Date.now();
|
|
4676
|
+
// Valid JSON with the serializer's own truncation marker, never a raw
|
|
4677
|
+
// slice: the live call handed to the reading.
|
|
4678
|
+
const toolInput = serializeCapped(redact(modelArgs), {
|
|
4679
|
+
maxPayloadBytes: 2048,
|
|
4680
|
+
maxStringBytes: 256,
|
|
4681
|
+
});
|
|
4682
|
+
// The rest of the task the user asked for: its values are in the turn's
|
|
4683
|
+
// request, and the model's words say what it is doing next.
|
|
4684
|
+
const prompt = [run.input, text].filter((s) => s.trim().length > 0).join("\n\n");
|
|
4685
|
+
const arming = this.armOrLog(run, this.replay.arm(match, prompt, this.wrapped, "intent", {
|
|
4686
|
+
liveCall: { toolName, toolInput },
|
|
4687
|
+
recentResults: this.recentResultsFor(session, "call"),
|
|
4688
|
+
agentDay: this.agentDayFor(session),
|
|
4689
|
+
}, undefined, { continuation }));
|
|
4690
|
+
const state = await this.withinBudget(arming, CONTINUATION_ARM_MS);
|
|
4691
|
+
if (state === undefined)
|
|
4692
|
+
return undefined; // it threw, and said so
|
|
4693
|
+
if (state === null) {
|
|
4694
|
+
logLine("replay.decision", {
|
|
4695
|
+
verdict: "no-steer",
|
|
4696
|
+
code: "arm_timeout",
|
|
4697
|
+
run: run.runId,
|
|
4698
|
+
scenario: match.scenarioId ?? undefined,
|
|
4699
|
+
continues: `${f.scenarioId}@${f.stepIndex}`,
|
|
4700
|
+
ms: CONTINUATION_ARM_MS,
|
|
4701
|
+
why: "the continuation's values were not read before this call had to go ahead — never delivered, so never run",
|
|
4702
|
+
});
|
|
4703
|
+
void arming.then((late) => {
|
|
4704
|
+
if (late)
|
|
4705
|
+
this.keepDeclinedContinuation(session, run, late, "arm_timeout");
|
|
4706
|
+
});
|
|
4707
|
+
return undefined;
|
|
4708
|
+
}
|
|
4709
|
+
state.sessionId = session.sessionId;
|
|
4710
|
+
state.sessionCwd = session.cwd ?? this.opts.cwd;
|
|
4711
|
+
this.declineWithoutDeliveryTool(state);
|
|
4712
|
+
if (!state.plan) {
|
|
4713
|
+
// A gate declined it, and said why (`state.declined`).
|
|
4714
|
+
this.keepDeclinedContinuation(session, run, state);
|
|
4715
|
+
return undefined;
|
|
4716
|
+
}
|
|
4717
|
+
// Its values are in: the model's call must be its first step, arguments
|
|
4718
|
+
// and all — the same call, not merely one the pin guard would take over
|
|
4719
|
+
// (pin-guard.ts `sameFirstCall`).
|
|
4720
|
+
await this.withinBudget(state.plan.ready().catch(() => undefined), Math.max(0, CONTINUATION_ARM_MS - (Date.now() - startedAt)));
|
|
4721
|
+
if (!this.isContinuationFirstCall(state, toolName, modelArgs)) {
|
|
4722
|
+
this.keepDeclinedContinuation(session, run, state, "not_first_step");
|
|
4723
|
+
return undefined;
|
|
4724
|
+
}
|
|
4725
|
+
state.usageMark = markTranscriptUsage(session.transcriptPath);
|
|
4726
|
+
state.markUsage = () => markTranscriptUsage(session.transcriptPath);
|
|
4727
|
+
run.replays.push(state);
|
|
4728
|
+
run.replay = state;
|
|
4729
|
+
// From here the continuation is this turn's plan. A hand-over of its own,
|
|
4730
|
+
// if it makes one, schedules its own fragment: the one the plan before it
|
|
4731
|
+
// left due is not opened.
|
|
4732
|
+
run.fragment = undefined;
|
|
4733
|
+
const directive = this.replay.directiveFor(state) ?? "";
|
|
4734
|
+
logLine("replay.continuation_armed", {
|
|
4735
|
+
run: run.runId,
|
|
4736
|
+
scenario: state.scenarioId ?? undefined,
|
|
4737
|
+
continues: `${continuation.ofScenarioId}@${continuation.stepIndex}`,
|
|
4738
|
+
recording: continuation.runId,
|
|
4739
|
+
mode: state.mode,
|
|
4740
|
+
instead: toolName,
|
|
4741
|
+
ms: Date.now() - startedAt,
|
|
4742
|
+
why: "the model's call after the hand-over is the first step of the plan calculated from the work recorded after an earlier hand-over at this step",
|
|
4743
|
+
});
|
|
4744
|
+
return this.preToolAnswer(this.replay.deliverInstead(directive, toolName));
|
|
4745
|
+
}
|
|
4746
|
+
/**
|
|
4747
|
+
* A continuation handed out and not armed: kept, retired, for its ticket's
|
|
4748
|
+
* report at the seal (`not_steered`, with the reason, unmeasured). `code`
|
|
4749
|
+
* says why when no gate did.
|
|
4750
|
+
*/
|
|
4751
|
+
keepDeclinedContinuation(session, run, state, code) {
|
|
4752
|
+
if (code) {
|
|
4753
|
+
state.declined = code;
|
|
4754
|
+
logLine("replay.decision", {
|
|
4755
|
+
verdict: "no-steer",
|
|
4756
|
+
code,
|
|
4757
|
+
run: run.runId,
|
|
4758
|
+
scenario: state.scenarioId ?? undefined,
|
|
4759
|
+
continues: state.continuation
|
|
4760
|
+
? `${state.continuation.ofScenarioId}@${state.continuation.stepIndex}`
|
|
4761
|
+
: undefined,
|
|
4762
|
+
why: code === "not_first_step"
|
|
4763
|
+
? "the model's call after the hand-over is not this continuation's first step — the model chose another way"
|
|
4764
|
+
: "the continuation was not armed in time",
|
|
4765
|
+
});
|
|
4766
|
+
}
|
|
4767
|
+
state.sessionId = session.sessionId;
|
|
4768
|
+
state.plan = undefined;
|
|
4769
|
+
state.mode = "none";
|
|
4770
|
+
state.stepsPlanned = 0;
|
|
4771
|
+
state.retired = true;
|
|
4772
|
+
state.pinned.clear();
|
|
4773
|
+
// A seal has reported already: nothing more can be said for this turn.
|
|
4774
|
+
if (run.finished)
|
|
4775
|
+
return;
|
|
4776
|
+
(run.declinedContinuations ??= []).push(state);
|
|
4777
|
+
}
|
|
4778
|
+
/** A continuation request's answer that came after the call it was for went ahead. */
|
|
4779
|
+
noteLateContinuation(run, handover, late) {
|
|
4780
|
+
const match = late?.matched;
|
|
4781
|
+
if (!late?.continuation || !match?.executionTicket || run.finished)
|
|
4782
|
+
return;
|
|
4783
|
+
const state = this.replay.declinedContinuation(match, "match_late", {
|
|
4784
|
+
ofScenarioId: handover.scenarioId,
|
|
4785
|
+
stepIndex: handover.stepIndex,
|
|
4786
|
+
runId: late.continuation.runId,
|
|
4787
|
+
});
|
|
4788
|
+
(run.declinedContinuations ??= []).push(state);
|
|
4789
|
+
}
|
|
4790
|
+
/**
|
|
4791
|
+
* Every scenario this turn was handed and armed or declined: never handed
|
|
4792
|
+
* out again in it. A plan that handed over is among them, so its own
|
|
4793
|
+
* continuation request can never return it.
|
|
4794
|
+
*/
|
|
4795
|
+
armedScenarioIds(run) {
|
|
4796
|
+
return [...run.replays, ...(run.declinedContinuations ?? [])]
|
|
4797
|
+
.map((s) => s.scenarioId)
|
|
4798
|
+
.filter((id) => typeof id === "string");
|
|
4799
|
+
}
|
|
4800
|
+
/**
|
|
4801
|
+
* Whether the model's call is a continuation's first step: its tool, and
|
|
4802
|
+
* the step's own input in every argument the two share (pin-guard.ts
|
|
4803
|
+
* `sameFirstCall`). Anything that cannot be computed is "no".
|
|
4804
|
+
*/
|
|
4805
|
+
isContinuationFirstCall(state, toolName, modelArgs) {
|
|
4806
|
+
const plan = state.plan;
|
|
4807
|
+
if (!plan || plan.expectedTool() !== toolName)
|
|
4808
|
+
return false;
|
|
4809
|
+
try {
|
|
4810
|
+
const verdict = sameFirstCall(toolName, modelArgs, plan.toolInputForCurrentStep());
|
|
4811
|
+
if (!verdict.ok) {
|
|
4812
|
+
logDetail("replay.continuation_other_call", { tool: toolName, why: verdict.why });
|
|
4813
|
+
}
|
|
4814
|
+
return verdict.ok;
|
|
4815
|
+
}
|
|
4816
|
+
catch (err) {
|
|
4817
|
+
logDetail("replay.first_step_unknown", { tool: toolName, error: errText(err) });
|
|
4818
|
+
return false;
|
|
4819
|
+
}
|
|
4820
|
+
}
|
|
4821
|
+
/** Whether continuations are asked for: on unless `BIR_CONTINUATIONS=0`. */
|
|
4822
|
+
continuationsOn() {
|
|
4823
|
+
if (this.opts.continuations !== undefined)
|
|
4824
|
+
return this.opts.continuations;
|
|
4825
|
+
return process.env.BIR_CONTINUATIONS !== "0";
|
|
4826
|
+
}
|
|
4444
4827
|
/**
|
|
4445
4828
|
* Whether the model's call is plausibly `state`'s first step: its tool, and
|
|
4446
4829
|
* arguments the pin guard would pin to that step's input. Anything that
|
|
@@ -4521,9 +4904,12 @@ export class ControlServer {
|
|
|
4521
4904
|
return;
|
|
4522
4905
|
const input = reasoning.trim() || `${toolName} ${serializeCapped(redact(args))}`.slice(0, 2_000);
|
|
4523
4906
|
const fallbackOf = { scenarioId: f.scenarioId, stepIndex: f.stepIndex };
|
|
4907
|
+
// The service counted this hand-over already, when it was asked for its
|
|
4908
|
+
// continuation (Part 3): the create says so, and is not counted again.
|
|
4909
|
+
const sent = f.counted ? { ...fallbackOf, counted: true } : fallbackOf;
|
|
4524
4910
|
// The fragment is a run of the same host as the turn it continues.
|
|
4525
4911
|
const framework = frameworkOf(session);
|
|
4526
|
-
const id = this.recorder.startRun(input, { ...this.runMetadata(session), fallbackOf }, framework ? { fallbackOf, framework } : { fallbackOf });
|
|
4912
|
+
const id = this.recorder.startRun(input, { ...this.runMetadata(session), fallbackOf }, framework ? { fallbackOf: sent, framework } : { fallbackOf: sent });
|
|
4527
4913
|
for (const c of run.correlations.values()) {
|
|
4528
4914
|
c.settled = true;
|
|
4529
4915
|
if (c.fallback)
|
|
@@ -7,9 +7,15 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import type { ExecutionStage, ExecutionStepResult, FallbackKind } from "../replay/types.js";
|
|
9
9
|
export interface StartRunOptions {
|
|
10
|
+
/**
|
|
11
|
+
* Marks a *fragment* (fallbk.md D7). `counted`: the runner asked for this
|
|
12
|
+
* hand-over's continuation first, and the service counted the hand-over as
|
|
13
|
+
* a hit then, so this request must not count it again (Part 3).
|
|
14
|
+
*/
|
|
10
15
|
fallbackOf?: {
|
|
11
16
|
scenarioId: string;
|
|
12
17
|
stepIndex: number;
|
|
18
|
+
counted?: boolean;
|
|
13
19
|
};
|
|
14
20
|
/**
|
|
15
21
|
* Marks a *repeat* (the service's newestRun.md): a matched turn whose plan
|
|
@@ -287,7 +293,12 @@ export interface ServiceProber {
|
|
|
287
293
|
export declare function isServiceProber(r: Recorder): r is Recorder & ServiceProber;
|
|
288
294
|
/** What a ReAct iteration sends to find a scenario by its intent (fallbk.md §Runner 4). */
|
|
289
295
|
export interface IntentMatchRequest {
|
|
290
|
-
/**
|
|
296
|
+
/**
|
|
297
|
+
* What the agent wrote, or was thinking, as it chose the call. Never empty
|
|
298
|
+
* on an intent probe. A continuation request ({@link handover}) is answered
|
|
299
|
+
* by the hand-over alone: it sends the text, possibly empty, and the service
|
|
300
|
+
* does not read it.
|
|
301
|
+
*/
|
|
291
302
|
text: string;
|
|
292
303
|
currentToolName?: string;
|
|
293
304
|
/** Scenario ids already armed this turn. */
|
|
@@ -298,12 +309,32 @@ export interface IntentMatchRequest {
|
|
|
298
309
|
* what a missing value means to the service.
|
|
299
310
|
*/
|
|
300
311
|
framework?: string;
|
|
312
|
+
/**
|
|
313
|
+
* A continuation request (the service's fallbk.md Part 3): the plan that
|
|
314
|
+
* just handed the task to the model, and the step it stopped in front of.
|
|
315
|
+
* The service answers with the plan of the work recorded after an earlier
|
|
316
|
+
* hand-over at that step, if there is one, and matches no words.
|
|
317
|
+
*/
|
|
318
|
+
handover?: {
|
|
319
|
+
scenarioId: string;
|
|
320
|
+
stepIndex: number;
|
|
321
|
+
};
|
|
301
322
|
}
|
|
302
323
|
/** The answer to a probe. */
|
|
303
324
|
export interface IntentMatchAnswer {
|
|
304
325
|
matched: RunMatch | null;
|
|
305
326
|
/** The best score the service saw, matched or not — for the log, to tune by. */
|
|
306
327
|
bestSimilarity?: number | null;
|
|
328
|
+
/**
|
|
329
|
+
* The answer to a continuation request: the recording of the earlier
|
|
330
|
+
* hand-over at that step, and whether this hand-over was counted as a hit
|
|
331
|
+
* on it. Null when there is none; absent from a service that does not know
|
|
332
|
+
* continuations, which matched the text by words instead.
|
|
333
|
+
*/
|
|
334
|
+
continuation?: {
|
|
335
|
+
runId: string;
|
|
336
|
+
counted?: boolean;
|
|
337
|
+
} | null;
|
|
307
338
|
}
|
|
308
339
|
/** How long a probe's caller will wait; the request is abandoned past it. */
|
|
309
340
|
export interface IntentMatchOptions {
|
|
@@ -374,6 +405,15 @@ export interface ExecutionReport {
|
|
|
374
405
|
* which is what every report said before this field existed.
|
|
375
406
|
*/
|
|
376
407
|
baselineEligible?: boolean;
|
|
408
|
+
/**
|
|
409
|
+
* Set on a continuation's report (the service's fallbk.md Part 3): the plan
|
|
410
|
+
* it continued in this turn, which handed over in front of it. That plan's
|
|
411
|
+
* report bills the whole turn against its own baseline, which covers this
|
|
412
|
+
* part of the task too, so the service books this one no saving.
|
|
413
|
+
*/
|
|
414
|
+
continues?: {
|
|
415
|
+
scenarioId: string;
|
|
416
|
+
};
|
|
377
417
|
/**
|
|
378
418
|
* Which gate of the ladder declined, on a `not_steered` report.
|
|
379
419
|
*
|
|
@@ -683,6 +683,9 @@ export class RemoteRecorder {
|
|
|
683
683
|
return {
|
|
684
684
|
matched: b.matched ?? null,
|
|
685
685
|
...(typeof b.bestSimilarity === "number" ? { bestSimilarity: b.bestSimilarity } : {}),
|
|
686
|
+
// Null and absent mean different things: no continuation, or a
|
|
687
|
+
// service that does not know continuations (fallbk.md Part 3).
|
|
688
|
+
...(b.continuation !== undefined ? { continuation: b.continuation } : {}),
|
|
686
689
|
};
|
|
687
690
|
}
|
|
688
691
|
catch (err) {
|
|
@@ -1040,6 +1043,9 @@ export class RemoteRecorder {
|
|
|
1040
1043
|
fallbackStepIndex: report.fallbackStepIndex,
|
|
1041
1044
|
fallbackKind: report.fallbackKind,
|
|
1042
1045
|
baselineEligible: report.baselineEligible,
|
|
1046
|
+
// A continuation's report: the plan it continued, whose report
|
|
1047
|
+
// books the turn (the service's fallbk.md Part 3). Additive.
|
|
1048
|
+
continues: report.continues,
|
|
1043
1049
|
// Why nothing ran, when nothing ran. Additive: an older service drops
|
|
1044
1050
|
// the field and books the cost exactly as it always did.
|
|
1045
1051
|
declined: report.declined,
|
|
@@ -1070,6 +1076,8 @@ export class RemoteRecorder {
|
|
|
1070
1076
|
stepsFailed: failed || undefined,
|
|
1071
1077
|
baselineEligible: report.baselineEligible === false ? false : undefined,
|
|
1072
1078
|
declined: report.declined,
|
|
1079
|
+
// A continuation's report: the plan whose report books the turn.
|
|
1080
|
+
continues: report.continues?.scenarioId,
|
|
1073
1081
|
// `bir investigate` prints it beside the outcome: "redone by run_…".
|
|
1074
1082
|
redoneBy: report.redoneBy?.runId,
|
|
1075
1083
|
ownCalls: report.redoneBy?.ownCalls,
|
|
@@ -388,6 +388,16 @@ export interface ReplayState {
|
|
|
388
388
|
retired: boolean;
|
|
389
389
|
/** What armed it: the prompt, or a ReAct iteration's intent (fallbk.md). */
|
|
390
390
|
armedBy: "prompt" | "intent";
|
|
391
|
+
/**
|
|
392
|
+
* Set on a continuation (the service's fallbk.md Part 3): a plan armed
|
|
393
|
+
* mid-turn because the plan before it handed over at a step, and the work
|
|
394
|
+
* recorded after an earlier hand-over at that step was calculated into this
|
|
395
|
+
* plan. `armedBy` is `intent` — it arms mid-turn, and is billed for its own
|
|
396
|
+
* window like any plan armed there — but it finishes the user's request
|
|
397
|
+
* rather than a part of the agent's own task, and its report books no
|
|
398
|
+
* saving: the plan it continued books the whole turn.
|
|
399
|
+
*/
|
|
400
|
+
continuation?: ContinuationOf;
|
|
391
401
|
/**
|
|
392
402
|
* The Claude Code session this plan serves, set by the control server. Its
|
|
393
403
|
* steps run on that session's own proxies (executor.ts). Absent for `bir
|
|
@@ -596,7 +606,33 @@ export type DeclineCode = "replay_disabled" | "not_ready" | "known_bad_first_ste
|
|
|
596
606
|
* the prompt had been answered, or while prompts were not waiting on it
|
|
597
607
|
* (the match gate). Set by the control server, like the two above.
|
|
598
608
|
*/
|
|
599
|
-
| "match_late"
|
|
609
|
+
| "match_late"
|
|
610
|
+
/**
|
|
611
|
+
* A continuation (the service's fallbk.md Part 3) whose first step is not
|
|
612
|
+
* the call the model is making after the hand-over: another tool, or the
|
|
613
|
+
* same tool on something else. The model chose what to do next — the
|
|
614
|
+
* judgement the plan before it stopped for — and chose otherwise. Set by
|
|
615
|
+
* the control server.
|
|
616
|
+
*/
|
|
617
|
+
| "not_first_step";
|
|
618
|
+
/**
|
|
619
|
+
* What a continuation continues (the service's fallbk.md Part 3): the plan
|
|
620
|
+
* that handed over, the step it stopped in front of, and the recording of the
|
|
621
|
+
* work done after an earlier hand-over there, which this plan was calculated
|
|
622
|
+
* from.
|
|
623
|
+
*/
|
|
624
|
+
export interface ContinuationOf {
|
|
625
|
+
ofScenarioId: string;
|
|
626
|
+
stepIndex: number;
|
|
627
|
+
runId: string;
|
|
628
|
+
}
|
|
629
|
+
/**
|
|
630
|
+
* Whether a plan is a part of the task the agent is already doing — armed
|
|
631
|
+
* mid-turn by its intent — so its words say "continue" and never "answer the
|
|
632
|
+
* user". A continuation is armed mid-turn too, and it is the rest of the
|
|
633
|
+
* user's request: it ends with the answer.
|
|
634
|
+
*/
|
|
635
|
+
export declare function isSubTask(state: Pick<ReplayState, "armedBy" | "continuation">): boolean;
|
|
600
636
|
/** What `PreToolUse` should do about this call. */
|
|
601
637
|
export type PreToolAction =
|
|
602
638
|
/** Pin the arguments and let the real tool run. */
|
|
@@ -674,6 +710,14 @@ export declare class ReplayController {
|
|
|
674
710
|
* itself as one that never reported.
|
|
675
711
|
*/
|
|
676
712
|
lateMatch(match: RunMatchLike, host?: HostContext): ReplayState;
|
|
713
|
+
/**
|
|
714
|
+
* The state of a continuation handed out and not armed (the service's
|
|
715
|
+
* fallbk.md Part 3): declined before any gate ran — its first step is not
|
|
716
|
+
* the model's call (`not_first_step`), or the service's answer came too
|
|
717
|
+
* late (`match_late`). Like {@link lateMatch}, it exists only for its
|
|
718
|
+
* ticket's report, with the reason.
|
|
719
|
+
*/
|
|
720
|
+
declinedContinuation(match: RunMatchLike, code: DeclineCode, continuation: ContinuationOf): ReplayState;
|
|
677
721
|
/**
|
|
678
722
|
* Run the gate ladder and, if every gate passes, arm a plan.
|
|
679
723
|
*
|
|
@@ -682,9 +726,13 @@ export declare class ReplayController {
|
|
|
682
726
|
* up with a denominator nobody measured.
|
|
683
727
|
*
|
|
684
728
|
* `host` is given for a session whose host is not Claude Code, and is kept
|
|
685
|
-
* on the state (see {@link HostContext}).
|
|
729
|
+
* on the state (see {@link HostContext}). `extra.continuation` arms a
|
|
730
|
+
* continuation (the service's fallbk.md Part 3): see
|
|
731
|
+
* {@link ReplayState.continuation}.
|
|
686
732
|
*/
|
|
687
|
-
arm(match: RunMatchLike, prompt: string, wrapped: ReadonlySet<string>, armedBy?: "prompt" | "intent", ctx?: DeriveContext, host?: HostContext
|
|
733
|
+
arm(match: RunMatchLike, prompt: string, wrapped: ReadonlySet<string>, armedBy?: "prompt" | "intent", ctx?: DeriveContext, host?: HostContext, extra?: {
|
|
734
|
+
continuation?: ContinuationOf;
|
|
735
|
+
}): Promise<ReplayState>;
|
|
688
736
|
/**
|
|
689
737
|
* The armed plan as data: the steps it will run, in order, and where it
|
|
690
738
|
* stops when that is before its end. For a host that steps through the plan
|
|
@@ -960,6 +1008,13 @@ export declare class ReplayController {
|
|
|
960
1008
|
runId: string;
|
|
961
1009
|
ownCalls: number;
|
|
962
1010
|
};
|
|
1011
|
+
/**
|
|
1012
|
+
* What the continuations of this plan spent reading their values this
|
|
1013
|
+
* turn (the service's fallbk.md Part 3). Booked here, on the plan whose
|
|
1014
|
+
* report bills the whole turn: theirs book no saving, so a cost left on
|
|
1015
|
+
* them would be left out of the turn's saving.
|
|
1016
|
+
*/
|
|
1017
|
+
chainDeriveUsd?: number;
|
|
963
1018
|
}): ExecutionReport | undefined;
|
|
964
1019
|
/**
|
|
965
1020
|
* The step a failed replay is fairly blamed on: the first whose own logic
|
|
@@ -196,6 +196,15 @@ const MAX_STEP_ERROR_CHARS = 500;
|
|
|
196
196
|
* cannot hold more than a few megabytes in the recorder.
|
|
197
197
|
*/
|
|
198
198
|
const TURN_CALLS_MAX_CHARS = 8 * 1024 * 1024;
|
|
199
|
+
/**
|
|
200
|
+
* Whether a plan is a part of the task the agent is already doing — armed
|
|
201
|
+
* mid-turn by its intent — so its words say "continue" and never "answer the
|
|
202
|
+
* user". A continuation is armed mid-turn too, and it is the rest of the
|
|
203
|
+
* user's request: it ends with the answer.
|
|
204
|
+
*/
|
|
205
|
+
export function isSubTask(state) {
|
|
206
|
+
return state.armedBy === "intent" && state.continuation === undefined;
|
|
207
|
+
}
|
|
199
208
|
export class ReplayController {
|
|
200
209
|
enabled;
|
|
201
210
|
work = new ProxyWorkQueue();
|
|
@@ -365,6 +374,30 @@ export class ReplayController {
|
|
|
365
374
|
state.retired = true;
|
|
366
375
|
return state;
|
|
367
376
|
}
|
|
377
|
+
/**
|
|
378
|
+
* The state of a continuation handed out and not armed (the service's
|
|
379
|
+
* fallbk.md Part 3): declined before any gate ran — its first step is not
|
|
380
|
+
* the model's call (`not_first_step`), or the service's answer came too
|
|
381
|
+
* late (`match_late`). Like {@link lateMatch}, it exists only for its
|
|
382
|
+
* ticket's report, with the reason.
|
|
383
|
+
*/
|
|
384
|
+
declinedContinuation(match, code, continuation) {
|
|
385
|
+
const state = this.blankState(match, new Set(), "intent");
|
|
386
|
+
state.continuation = continuation;
|
|
387
|
+
state.declined = code;
|
|
388
|
+
state.retired = true;
|
|
389
|
+
logLine("replay.decision", {
|
|
390
|
+
verdict: "no-steer",
|
|
391
|
+
code,
|
|
392
|
+
run: match.runId,
|
|
393
|
+
scenario: match.scenarioId ?? undefined,
|
|
394
|
+
continues: `${continuation.ofScenarioId}@${continuation.stepIndex}`,
|
|
395
|
+
why: code === "not_first_step"
|
|
396
|
+
? "a continuation starts only on its first step, and the model's call after the hand-over is another one"
|
|
397
|
+
: "the continuation came after the call it could have replaced had gone ahead",
|
|
398
|
+
});
|
|
399
|
+
return state;
|
|
400
|
+
}
|
|
368
401
|
/**
|
|
369
402
|
* Run the gate ladder and, if every gate passes, arm a plan.
|
|
370
403
|
*
|
|
@@ -373,10 +406,14 @@ export class ReplayController {
|
|
|
373
406
|
* up with a denominator nobody measured.
|
|
374
407
|
*
|
|
375
408
|
* `host` is given for a session whose host is not Claude Code, and is kept
|
|
376
|
-
* on the state (see {@link HostContext}).
|
|
409
|
+
* on the state (see {@link HostContext}). `extra.continuation` arms a
|
|
410
|
+
* continuation (the service's fallbk.md Part 3): see
|
|
411
|
+
* {@link ReplayState.continuation}.
|
|
377
412
|
*/
|
|
378
|
-
async arm(match, prompt, wrapped, armedBy = "prompt", ctx = {}, host) {
|
|
413
|
+
async arm(match, prompt, wrapped, armedBy = "prompt", ctx = {}, host, extra = {}) {
|
|
379
414
|
const state = this.blankState(match, wrapped, armedBy, host);
|
|
415
|
+
if (extra.continuation)
|
|
416
|
+
state.continuation = extra.continuation;
|
|
380
417
|
const decline = (why, code) => {
|
|
381
418
|
state.declined = code;
|
|
382
419
|
logLine("replay.decision", {
|
|
@@ -597,12 +634,13 @@ export class ReplayController {
|
|
|
597
634
|
}
|
|
598
635
|
// A plan armed by intent runs inside a task the agent is already doing, so
|
|
599
636
|
// its directive and its bundles say "continue", never "answer the user".
|
|
637
|
+
// A continuation is the rest of the user's request: it ends with the answer.
|
|
600
638
|
state.plan = new ScenarioReplayPlan({
|
|
601
639
|
scenario,
|
|
602
640
|
params,
|
|
603
641
|
mode,
|
|
604
642
|
stopAt,
|
|
605
|
-
subTask:
|
|
643
|
+
subTask: isSubTask(state),
|
|
606
644
|
});
|
|
607
645
|
if (mode === "steer") {
|
|
608
646
|
// A steer directive shows the arguments the steps will run with, so the
|
|
@@ -643,7 +681,11 @@ export class ReplayController {
|
|
|
643
681
|
host: host?.app,
|
|
644
682
|
steps: steps.length,
|
|
645
683
|
stopsAt: stopAt,
|
|
646
|
-
by: armedBy === "intent" ? "intent" : undefined,
|
|
684
|
+
by: armedBy === "intent" ? (state.continuation ? "continuation" : "intent") : undefined,
|
|
685
|
+
// The plan it continues and the step that plan stopped in front of.
|
|
686
|
+
continues: state.continuation
|
|
687
|
+
? `${state.continuation.ofScenarioId}@${state.continuation.stepIndex}`
|
|
688
|
+
: undefined,
|
|
647
689
|
similarity: match.similarity.toFixed(3),
|
|
648
690
|
// Which steps the service told this plan to stop in front of, and why.
|
|
649
691
|
stops: stopLines(steps),
|
|
@@ -752,7 +794,7 @@ export class ReplayController {
|
|
|
752
794
|
directiveFor(state, opts = {}) {
|
|
753
795
|
const asked = opts.previewMs;
|
|
754
796
|
return state.plan?.steeringDirective(DIRECT_TOOL_NAME, {
|
|
755
|
-
subTask: state
|
|
797
|
+
subTask: isSubTask(state),
|
|
756
798
|
// NaN is a caller's arithmetic gone wrong: no time known, no preview.
|
|
757
799
|
previewBudgetMs: typeof asked === "number"
|
|
758
800
|
? Math.max(0, Math.min(Number.isNaN(asked) ? 0 : asked, PREVIEW_BUDGET_MS))
|
|
@@ -1499,10 +1541,9 @@ export class ReplayController {
|
|
|
1499
1541
|
const what = left.length === 1 ? `Not needed for this request: ${named[0]}${why}.` : `Not needed for this request: ${named.join(", ")}${why}.`;
|
|
1500
1542
|
let next;
|
|
1501
1543
|
if (plan.isDone() && !plan.atStop()) {
|
|
1502
|
-
next =
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
: "The sequence is complete: answer the user's request from the tool results.";
|
|
1544
|
+
next = isSubTask(state)
|
|
1545
|
+
? "The sequence is complete: continue your task, calling tools as needed."
|
|
1546
|
+
: "The sequence is complete: answer the user's request from the tool results.";
|
|
1506
1547
|
}
|
|
1507
1548
|
else {
|
|
1508
1549
|
const step = plan.currentStep();
|
|
@@ -1890,7 +1931,8 @@ export class ReplayController {
|
|
|
1890
1931
|
if (state.reported || !state.scenarioId)
|
|
1891
1932
|
return undefined;
|
|
1892
1933
|
state.reported = true;
|
|
1893
|
-
const
|
|
1934
|
+
const deriveCostUsd = state.deriveCostUsd + Math.max(0, d.chainDeriveUsd ?? 0);
|
|
1935
|
+
const total = deriveCostUsd + d.sessionCostUsd + state.fallbackCostUsd;
|
|
1894
1936
|
const steps = this.stepResultsOf(state);
|
|
1895
1937
|
const isBaseline = state.outcome === "not_steered" || state.outcome === "failed";
|
|
1896
1938
|
// A costless decline is noise on both sides — *unless* it says something the
|
|
@@ -1912,7 +1954,7 @@ export class ReplayController {
|
|
|
1912
1954
|
runId: d.runId,
|
|
1913
1955
|
ticket: state.ticket,
|
|
1914
1956
|
outcome: state.outcome,
|
|
1915
|
-
deriveCostUsd
|
|
1957
|
+
deriveCostUsd,
|
|
1916
1958
|
sessionCostUsd: d.sessionCostUsd,
|
|
1917
1959
|
fallbackCostUsd: state.fallbackCostUsd,
|
|
1918
1960
|
durationMs: d.durationMs,
|
|
@@ -1955,6 +1997,9 @@ export class ReplayController {
|
|
|
1955
1997
|
: undefined,
|
|
1956
1998
|
// The agent did not take the plan's result and redid the task (AC1).
|
|
1957
1999
|
...(d.redoneBy ? { redoneBy: d.redoneBy } : {}),
|
|
2000
|
+
// A continuation (the service's fallbk.md Part 3): the plan it continued
|
|
2001
|
+
// books the whole turn, so this report books no saving of its own.
|
|
2002
|
+
...(state.continuation ? { continues: { scenarioId: state.continuation.ofScenarioId } } : {}),
|
|
1958
2003
|
// A turn in which another plan ran is not a measurement of what this task
|
|
1959
2004
|
// costs unaided, so it is no baseline sample (segmented.md R-MONEY-5).
|
|
1960
2005
|
// This covers two plans that both armed, and — the case that was silently
|
|
@@ -373,6 +373,7 @@ so there is exactly one switch and it cannot get out of step with itself.
|
|
|
373
373
|
| `BIR_BASH_ALLOW` | *(unset: any command)* | Command prefixes a runner-side `Bash` step may run (`*` for all). Unset means any command; `none` means no command, and `Bash` steps are steered in the session. A plan's Bash steps share a working directory, exactly as the host's own Bash tool does — variables and functions do not persist, also as the host's. A call with `run_in_background`, or with its own `timeout` longer than the step's clock (`BIR_STEP_TIMEOUT_MS`), is never started on the runner: the plan hands over in front of it |
|
|
374
374
|
| `BIR_PATH_ALLOW` | *(unset: any folder, from 0.3.21)* | Folders a runner-side `Read`, `Glob`, `Grep`, `Edit` or `Write` may touch (`*` for all). Unset means any path, as the agent's own tools reach it — a plan that reads the agent's memory file under `~/.claude` runs. `project` is the session's project folder alone (the rule up to 0.3.20); add folders after a comma, `~` for the home folder: `project,~/.claude`. Under a list, a path outside it, symlinks included, is refused and the plan hands over in front of it. `none` means no folder, and the file steps are steered in the session. `Bash` is `BIR_BASH_ALLOW`'s |
|
|
375
375
|
| `BIR_INTENT_ARM` | *(unset: off)* | `1` lets a whole plan whose intent matches the agent's reasoning mid-turn arm in the calling tool's place, which means waiting for it (`BIR_INTENT_ARM_BUDGET_MS`, 500). Unset or `0`: off — no probe is sent and nothing arms mid-turn, as up to 0.3.18 |
|
|
376
|
+
| `BIR_CONTINUATIONS` | *(unset: on, from 0.3.23)* | After a plan hands the task to the agent at a step, the agent's first call asks the service for the plan of the work recorded after an earlier hand-over at that step — once per hand-over, waiting 1.5 s at most for the answer and up to 7 s for its values — and runs it in that call's place only when the call is its first step (the same tool, on the same thing). Its saving is counted on the plan that handed over. `0`: off, and the agent's work is recorded as before. `BIR_INTENT_MATCH=0` turns it off too |
|
|
376
377
|
| `BIR_VERBOSE` | *(unset)* | `1` adds per-step `replay.*` detail lines |
|
|
377
378
|
|
|
378
379
|
The `bir` server reads its own, from the environment Claude Code starts it with:
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@basein/runner",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.23",
|
|
4
4
|
"description": "A recording MCP proxy: sits between any MCP client and its MCP servers, executes each call on the client's behalf, and records the run as a reusable BaseIn scenario.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|