@basein/runner 0.3.27 → 0.3.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/bin/bir-scenario.js +7 -5
- package/dist/bin/bir.js +36 -5
- package/dist/bin/scenario-edit.d.ts +102 -1
- package/dist/bin/scenario-edit.js +318 -23
- package/dist/control/server.d.ts +19 -0
- package/dist/control/server.js +80 -2
- package/dist/control/transcript.d.ts +9 -0
- package/dist/control/transcript.js +56 -9
- package/dist/record/recorder.d.ts +50 -0
- package/dist/record/remote-recorder.d.ts +9 -1
- package/dist/record/remote-recorder.js +38 -0
- package/dist/replay/controller.d.ts +41 -1
- package/dist/replay/controller.js +109 -4
- package/docs/calculatedReplayGuide.md +74 -1
- package/package.json +1 -1
|
@@ -126,7 +126,8 @@ const indexes = new Map();
|
|
|
126
126
|
* ({@link subagentUsage}): transcript path → the main index they were read
|
|
127
127
|
* beside, one usage-only index per file, and `gone`: what the files whose
|
|
128
128
|
* index was let go of had used (one past {@link MAX_SUBAGENT_FILES}, one read
|
|
129
|
-
* again from its start after a rewrite)
|
|
129
|
+
* again from its start after a rewrite), and `goneCalls`: their model calls.
|
|
130
|
+
* Dropped with the main index.
|
|
130
131
|
*/
|
|
131
132
|
const subagentIndexes = new Map();
|
|
132
133
|
/** Sub-agent files read per transcript, at most. */
|
|
@@ -150,6 +151,7 @@ function freshIndex(st, usageOnly) {
|
|
|
150
151
|
// A sub-agent's file shares its transcript's set: a message that a host
|
|
151
152
|
// writes into both files is counted once, whichever is read first.
|
|
152
153
|
usageSeen: usageOnly?.seen ?? new Set(),
|
|
154
|
+
calls: 0,
|
|
153
155
|
lastAssistantText: "",
|
|
154
156
|
firstUserPrompt: "",
|
|
155
157
|
sawPrompt: false,
|
|
@@ -388,6 +390,8 @@ function subagentFiles(transcriptPath) {
|
|
|
388
390
|
* read beside the transcript's set of counted messages, so nothing counts
|
|
389
391
|
* twice. A total that fell made the turn across the fall cost $0 for that
|
|
390
392
|
* model, the main transcript's own tokens with it, and the file $0 for good.
|
|
393
|
+
*
|
|
394
|
+
* `calls` is the same files' model calls, on the same terms.
|
|
391
395
|
*/
|
|
392
396
|
function subagentUsage(transcriptPath, main) {
|
|
393
397
|
const files = subagentFiles(transcriptPath);
|
|
@@ -397,16 +401,26 @@ function subagentUsage(transcriptPath, main) {
|
|
|
397
401
|
known = undefined;
|
|
398
402
|
}
|
|
399
403
|
if (files.length === 0 && !known)
|
|
400
|
-
return [];
|
|
404
|
+
return { usage: [], calls: 0 };
|
|
401
405
|
if (!known) {
|
|
402
|
-
known = { beside: main, files: new Map(), gone: new Map() };
|
|
406
|
+
known = { beside: main, files: new Map(), gone: new Map(), goneCalls: 0 };
|
|
403
407
|
subagentIndexes.set(transcriptPath, known);
|
|
404
408
|
}
|
|
405
|
-
const
|
|
406
|
-
const usageOnly = {
|
|
409
|
+
const store = known;
|
|
410
|
+
const usageOnly = {
|
|
411
|
+
seen: main.usageSeen,
|
|
412
|
+
retire: (old) => {
|
|
413
|
+
addUsage(store.gone, old.usage);
|
|
414
|
+
store.goneCalls += old.calls;
|
|
415
|
+
},
|
|
416
|
+
};
|
|
407
417
|
for (const file of files)
|
|
408
|
-
indexIn(
|
|
409
|
-
|
|
418
|
+
indexIn(store.files, file, MAX_SUBAGENT_FILES, usageOnly);
|
|
419
|
+
const read = [...store.files.values()];
|
|
420
|
+
return {
|
|
421
|
+
usage: [store.gone, ...read.map((idx) => idx.usage)],
|
|
422
|
+
calls: read.reduce((n, idx) => n + idx.calls, store.goneCalls),
|
|
423
|
+
};
|
|
410
424
|
}
|
|
411
425
|
/** `from`'s tokens added into `into`, key by key. */
|
|
412
426
|
function addUsage(into, from) {
|
|
@@ -478,6 +492,9 @@ function foldUsage(idx, message) {
|
|
|
478
492
|
return;
|
|
479
493
|
if (id)
|
|
480
494
|
idx.usageSeen.add(id);
|
|
495
|
+
// One message is one request: a model call (LP11), on the very terms its
|
|
496
|
+
// tokens are counted.
|
|
497
|
+
idx.calls += 1;
|
|
481
498
|
const model = message?.model ?? "unknown";
|
|
482
499
|
const input = count(usage.input_tokens);
|
|
483
500
|
const read = count(usage.cache_read_input_tokens);
|
|
@@ -938,19 +955,49 @@ const count = (value) => typeof value === "number" && Number.isFinite(value) &&
|
|
|
938
955
|
*
|
|
939
956
|
* The running totals live in the index ({@link foldLine}); this returns a copy,
|
|
940
957
|
* because a mark is kept and compared long after the index has moved on.
|
|
958
|
+
*
|
|
959
|
+
* The model calls those totals were counted from ride beside the copy, in
|
|
960
|
+
* {@link MARK_CALLS}: a key of the map would be read as a model, and priced.
|
|
941
961
|
*/
|
|
942
962
|
function totalUsage(transcriptPath) {
|
|
943
963
|
const totals = new Map();
|
|
944
964
|
const idx = indexFor(transcriptPath);
|
|
945
|
-
if (!idx)
|
|
965
|
+
if (!idx) {
|
|
966
|
+
MARK_CALLS.set(totals, 0);
|
|
946
967
|
return totals;
|
|
968
|
+
}
|
|
947
969
|
for (const [model, u] of idx.usage)
|
|
948
970
|
totals.set(model, { ...u });
|
|
949
971
|
// And what the session's sub-agents used, each in a file of its own.
|
|
950
|
-
|
|
972
|
+
const subagents = subagentUsage(transcriptPath, idx);
|
|
973
|
+
for (const usage of subagents.usage)
|
|
951
974
|
addUsage(totals, usage);
|
|
975
|
+
MARK_CALLS.set(totals, idx.calls + subagents.calls);
|
|
952
976
|
return totals;
|
|
953
977
|
}
|
|
978
|
+
/**
|
|
979
|
+
* Model calls counted into a mark, by the mark: every mark this module makes
|
|
980
|
+
* has its entry, and a map made anywhere else has none ({@link callsBetween}).
|
|
981
|
+
* Weak, so a mark nobody holds any more takes its count with it.
|
|
982
|
+
*/
|
|
983
|
+
const MARK_CALLS = new WeakMap();
|
|
984
|
+
/**
|
|
985
|
+
* The model calls between two marks of one transcript, `from` the earlier:
|
|
986
|
+
* assistant messages, each counted once by its `message.id` as its tokens are
|
|
987
|
+
* ({@link totalUsage}), the sub-agents' files included — the calls behind the
|
|
988
|
+
* tokens {@link usageBetween} gives for the same two marks. No `mark` counts
|
|
989
|
+
* from the start, as there. Clamped at 0, as the tokens are, for a transcript
|
|
990
|
+
* rewritten between the two. Undefined when either mark was not made here.
|
|
991
|
+
*/
|
|
992
|
+
export function callsBetween(mark, now) {
|
|
993
|
+
const to = MARK_CALLS.get(now);
|
|
994
|
+
if (to === undefined)
|
|
995
|
+
return undefined;
|
|
996
|
+
if (mark === undefined)
|
|
997
|
+
return to;
|
|
998
|
+
const from = MARK_CALLS.get(mark);
|
|
999
|
+
return from === undefined ? undefined : Math.max(0, to - from);
|
|
1000
|
+
}
|
|
954
1001
|
/**
|
|
955
1002
|
* Snapshot cumulative usage at the moment a run starts.
|
|
956
1003
|
*
|
|
@@ -471,7 +471,57 @@ export interface ExecutionReport {
|
|
|
471
471
|
* (record/remote-recorder.ts, `answerCallsBody`).
|
|
472
472
|
*/
|
|
473
473
|
answerCalls?: AnswerCall[];
|
|
474
|
+
/**
|
|
475
|
+
* What the runner can say of the turn beyond its cost (the service's
|
|
476
|
+
* learningPhase.md LP11): how the plan ran, the model calls and tokens of
|
|
477
|
+
* the window its cost was read for, how much the plan handed the agent to
|
|
478
|
+
* read, and why it handed over. The learning phase's rules about cost and
|
|
479
|
+
* time read it (P7, P8). Additive: an older service drops it, and a report
|
|
480
|
+
* without it is the body it always was.
|
|
481
|
+
*/
|
|
482
|
+
turn?: ExecutionTurn;
|
|
483
|
+
}
|
|
484
|
+
/**
|
|
485
|
+
* {@link ExecutionReport.turn}. Every key is always there, `null` when the
|
|
486
|
+
* runner does not know it: no transcript mark, another host's run, nothing
|
|
487
|
+
* handed to the agent, no answer read.
|
|
488
|
+
*/
|
|
489
|
+
export interface ExecutionTurn {
|
|
490
|
+
/** How the plan ran, as coverage.ts decided it when it armed; null for a plan that never armed. */
|
|
491
|
+
mode: "direct" | "steer" | null;
|
|
492
|
+
/** The window's model calls: assistant messages, counted once per `message.id`, sub-agents' files included. */
|
|
493
|
+
modelCalls: number | null;
|
|
494
|
+
inputTokens: number | null;
|
|
495
|
+
outputTokens: number | null;
|
|
496
|
+
cacheReadTokens: number | null;
|
|
497
|
+
/** Every cache write, the 1-hour ones among them. */
|
|
498
|
+
cacheWriteTokens: number | null;
|
|
499
|
+
/**
|
|
500
|
+
* Characters of what the plan handed the agent to read: `run_scenario`'s
|
|
501
|
+
* answer (the bundle, or the hand-over note), a divergence's bundle, and
|
|
502
|
+
* the notes told in place of a call or beside its result. The announcement
|
|
503
|
+
* the prompt carries is not counted. Null for a plan that never armed.
|
|
504
|
+
*/
|
|
505
|
+
bundleChars: number | null;
|
|
506
|
+
/** The brief bundle (0.3.25) was handed; false for a full one; null when no bundle was. */
|
|
507
|
+
brief: boolean | null;
|
|
508
|
+
/** Characters of the agent's final answer, when the runner read it (AC11); else null. */
|
|
509
|
+
answerChars: number | null;
|
|
510
|
+
/**
|
|
511
|
+
* On a hand-over: where the stopped step stood (`stop`, as a fragment's
|
|
512
|
+
* create call sends it — `not_run`, `failed`, `ran`, `uncertain`, …; null
|
|
513
|
+
* for a stop in front of a parked step or the other half) and why it
|
|
514
|
+
* stopped (`reason`: `budget`, `cancelled`, `test_failed`, `concurrent`,
|
|
515
|
+
* `failed`, … for a step that stopped, else the hand-over's kind). Null
|
|
516
|
+
* when the plan did not hand over.
|
|
517
|
+
*/
|
|
518
|
+
handover: {
|
|
519
|
+
stop: string | null;
|
|
520
|
+
reason: string | null;
|
|
521
|
+
} | null;
|
|
474
522
|
}
|
|
523
|
+
/** The part of {@link ExecutionTurn} that only the transcript can say: the control server reads it. */
|
|
524
|
+
export type TurnUsage = Pick<ExecutionTurn, "modelCalls" | "inputTokens" | "outputTokens" | "cacheReadTokens" | "cacheWriteTokens">;
|
|
475
525
|
/** One call of a plan, whole, as {@link ExecutionReport.answerCalls} carries it. */
|
|
476
526
|
export interface AnswerCall {
|
|
477
527
|
/** The step's own `stepIndex`. */
|
|
@@ -61,7 +61,7 @@
|
|
|
61
61
|
*/
|
|
62
62
|
import { type AuthSession } from "../auth/client.js";
|
|
63
63
|
import { type ServiceHealth } from "../util/service-health.js";
|
|
64
|
-
import type { AnswerCall, AttentionPlan, DisputeAnswer, ExecutionReport, FlushOptions, IntentMatchAnswer, IntentMatchOptions, IntentMatchRequest, Recorder, RecorderHealth, RecorderHealthReporter, RunCreation, RunMatch, RunMetrics, StartRunOptions, StepMetadata, ToolListEntry } from "./recorder.js";
|
|
64
|
+
import type { AnswerCall, AttentionPlan, DisputeAnswer, ExecutionReport, ExecutionTurn, FlushOptions, IntentMatchAnswer, IntentMatchOptions, IntentMatchRequest, Recorder, RecorderHealth, RecorderHealthReporter, RunCreation, RunMatch, RunMetrics, StartRunOptions, StepMetadata, ToolListEntry } from "./recorder.js";
|
|
65
65
|
/**
|
|
66
66
|
* Caps for the free text this recorder sends, in UTF-8 bytes. Tool payloads
|
|
67
67
|
* arrive capped at 64 KiB by their producers; text never passed through one,
|
|
@@ -129,6 +129,14 @@ export interface RemoteRecorderOptions {
|
|
|
129
129
|
/** The longest one queued send waits for a failing service ({@link COOLDOWN_WAIT_MAX_MS}). Tests shorten it. */
|
|
130
130
|
cooldownWaitMaxMs?: number;
|
|
131
131
|
}
|
|
132
|
+
/**
|
|
133
|
+
* A report's `turn` as it is sent (the service's learningPhase.md LP11): its
|
|
134
|
+
* ten keys and no other, each a whole count of at least 0, a short code or
|
|
135
|
+
* null — so that no value a service's schema could refuse costs the report
|
|
136
|
+
* that carries it, and with it the turn's cost. Undefined, and so left out of
|
|
137
|
+
* the body, when the report has none.
|
|
138
|
+
*/
|
|
139
|
+
export declare function turnBody(turn: ExecutionTurn | undefined): ExecutionTurn | undefined;
|
|
132
140
|
/** What one answered turn's call may carry of its output, in UTF-8 bytes (review item 12). */
|
|
133
141
|
export declare const ANSWER_CALL_OUTPUT_MAX_BYTES: number;
|
|
134
142
|
/** All of one report's answer calls together, serialized, in UTF-8 bytes. */
|
|
@@ -274,6 +274,39 @@ function reportBody(fields, steps, extra = {}) {
|
|
|
274
274
|
add(i, "toolOutput");
|
|
275
275
|
return JSON.stringify({ ...fields, steps: sent, ...extra });
|
|
276
276
|
}
|
|
277
|
+
/** Longest `stop` or `reason` a report's `turn` carries: they are codes. */
|
|
278
|
+
const TURN_CODE_MAX = 64;
|
|
279
|
+
/**
|
|
280
|
+
* A report's `turn` as it is sent (the service's learningPhase.md LP11): its
|
|
281
|
+
* ten keys and no other, each a whole count of at least 0, a short code or
|
|
282
|
+
* null — so that no value a service's schema could refuse costs the report
|
|
283
|
+
* that carries it, and with it the turn's cost. Undefined, and so left out of
|
|
284
|
+
* the body, when the report has none.
|
|
285
|
+
*/
|
|
286
|
+
export function turnBody(turn) {
|
|
287
|
+
if (!turn || typeof turn !== "object")
|
|
288
|
+
return undefined;
|
|
289
|
+
const count = (n) => typeof n === "number" && Number.isFinite(n) && n >= 0 ? Math.round(n) : null;
|
|
290
|
+
const code = (s) => (typeof s === "string" && s !== "" ? s.slice(0, TURN_CODE_MAX) : null);
|
|
291
|
+
try {
|
|
292
|
+
const h = turn.handover;
|
|
293
|
+
return {
|
|
294
|
+
mode: turn.mode === "direct" || turn.mode === "steer" ? turn.mode : null,
|
|
295
|
+
modelCalls: count(turn.modelCalls),
|
|
296
|
+
inputTokens: count(turn.inputTokens),
|
|
297
|
+
outputTokens: count(turn.outputTokens),
|
|
298
|
+
cacheReadTokens: count(turn.cacheReadTokens),
|
|
299
|
+
cacheWriteTokens: count(turn.cacheWriteTokens),
|
|
300
|
+
bundleChars: count(turn.bundleChars),
|
|
301
|
+
brief: typeof turn.brief === "boolean" ? turn.brief : null,
|
|
302
|
+
answerChars: count(turn.answerChars),
|
|
303
|
+
handover: h && typeof h === "object" ? { stop: code(h.stop), reason: code(h.reason) } : null,
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
catch {
|
|
307
|
+
return undefined;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
277
310
|
/** What one answered turn's call may carry of its output, in UTF-8 bytes (review item 12). */
|
|
278
311
|
export const ANSWER_CALL_OUTPUT_MAX_BYTES = 64 * 1024;
|
|
279
312
|
/** All of one report's answer calls together, serialized, in UTF-8 bytes. */
|
|
@@ -1139,6 +1172,11 @@ export class RemoteRecorder {
|
|
|
1139
1172
|
// ...or only answered: its answer, capped and redacted like any
|
|
1140
1173
|
// text (AC11). Additive, like `redoneBy`.
|
|
1141
1174
|
answer: out(answer),
|
|
1175
|
+
// How the plan ran, its model calls and tokens, what the agent
|
|
1176
|
+
// read and why it handed over (the service's learningPhase.md
|
|
1177
|
+
// LP11). A few hundred bytes, inside the budget the payloads give
|
|
1178
|
+
// way to. Additive: an older service drops it.
|
|
1179
|
+
turn: turnBody(report.turn),
|
|
1142
1180
|
}, steps,
|
|
1143
1181
|
// Beside the answer, the plan's calls whole (review item 12),
|
|
1144
1182
|
// outside the steps' budget. Additive: an older service drops them.
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* degrades to "run the turn normally" or returns something the session can
|
|
12
12
|
* ignore. Nothing throws at a hook.
|
|
13
13
|
*/
|
|
14
|
-
import type { ExecutionReport } from "../record/recorder.js";
|
|
14
|
+
import type { ExecutionReport, ExecutionTurn, TurnUsage } from "../record/recorder.js";
|
|
15
15
|
import { type ReplayMode, type StepReach } from "./coverage.js";
|
|
16
16
|
import { deriveParameters, type AgentDay, type DeriveContext, type DeriveResult } from "./derive.js";
|
|
17
17
|
import { type HandoverStopReason } from "./handover.js";
|
|
@@ -360,6 +360,15 @@ export interface Handover {
|
|
|
360
360
|
error?: string;
|
|
361
361
|
/** For a `step_failed`: where the stopped step stood ({@link stopOf}). */
|
|
362
362
|
stop?: string;
|
|
363
|
+
/**
|
|
364
|
+
* Why it stopped, as a code: for a `step_failed` the stop's own reason
|
|
365
|
+
* (`budget`, `cancelled`, `test_failed`, `concurrent`, `host_rejected`, …),
|
|
366
|
+
* `failed` for a plain failure; for another kind the kind itself. Only the
|
|
367
|
+
* report reads it (the service's learningPhase.md LP11): {@link stop}
|
|
368
|
+
* folds the stops in front of a step into `not_run`, and that is all the
|
|
369
|
+
* service needs to continue one, not to tell why a plan stopped.
|
|
370
|
+
*/
|
|
371
|
+
reason?: string;
|
|
363
372
|
/** What the model is told. Always built, even when delivery fails. */
|
|
364
373
|
note: string;
|
|
365
374
|
/** Set once the note reached the model, so it is never delivered twice. */
|
|
@@ -468,6 +477,16 @@ export interface ReplayState {
|
|
|
468
477
|
bashCwd?: string;
|
|
469
478
|
/** Set when the plan handed the task to the model part-way (fallbk.md). */
|
|
470
479
|
handover?: Handover;
|
|
480
|
+
/**
|
|
481
|
+
* Characters this plan handed the agent to read so far — `run_scenario`'s
|
|
482
|
+
* answer, a divergence's bundle, the notes told in place of a call or
|
|
483
|
+
* beside its result — for the report's `turn` (the service's
|
|
484
|
+
* learningPhase.md LP11). Counted where the text leaves for the agent,
|
|
485
|
+
* never where it is built: a note made and never delivered was not read.
|
|
486
|
+
*/
|
|
487
|
+
handedChars?: number;
|
|
488
|
+
/** The bundle this plan handed the agent, when it handed one: brief (0.3.25), or the full one. */
|
|
489
|
+
bundleKind?: "brief" | "full";
|
|
471
490
|
/**
|
|
472
491
|
* Every planned step ran and the model was handed the results to answer
|
|
473
492
|
* from: a steered plan whose last step threaded, a direct run that ran to
|
|
@@ -725,6 +744,14 @@ export interface ContinuationOf {
|
|
|
725
744
|
* user's request: it ends with the answer.
|
|
726
745
|
*/
|
|
727
746
|
export declare function isSubTask(state: Pick<ReplayState, "armedBy" | "continuation">): boolean;
|
|
747
|
+
/**
|
|
748
|
+
* The report's `turn` (the service's learningPhase.md LP11): what this plan
|
|
749
|
+
* did in the turn, from its state, and the window's tokens and model calls,
|
|
750
|
+
* which only the control server can read (`usage`; undefined when it could
|
|
751
|
+
* not). Every value is null when it is not known, and nothing here throws: a
|
|
752
|
+
* state it cannot read is a `turn` of nulls, never a report lost.
|
|
753
|
+
*/
|
|
754
|
+
export declare function turnOf(state: ReplayState, usage?: TurnUsage): ExecutionTurn;
|
|
728
755
|
/** What `PreToolUse` should do about this call. */
|
|
729
756
|
export type PreToolAction =
|
|
730
757
|
/** Pin the arguments and let the real tool run. */
|
|
@@ -903,6 +930,13 @@ export declare class ReplayController {
|
|
|
903
930
|
* Returns it once and marks it delivered; undefined when there is none.
|
|
904
931
|
*/
|
|
905
932
|
takeNote(state: ReplayState | undefined): string | undefined;
|
|
933
|
+
/**
|
|
934
|
+
* The hand-over note `run_scenario` answers with: claimed when it is still
|
|
935
|
+
* to tell ({@link takeNote}), and told again when it was told already — the
|
|
936
|
+
* same text the run always answered (`takeNote(state) ?? the note`), now
|
|
937
|
+
* counted as read either way.
|
|
938
|
+
*/
|
|
939
|
+
private runNote;
|
|
906
940
|
/** Whether a note about this plan is still to be told to the model: a hand-over's, or the unseen results'. */
|
|
907
941
|
hasUndeliveredNote(state: ReplayState | undefined): boolean;
|
|
908
942
|
/**
|
|
@@ -1120,6 +1154,12 @@ export declare class ReplayController {
|
|
|
1120
1154
|
* own baseline, so the turn's saving is booked somewhere.
|
|
1121
1155
|
*/
|
|
1122
1156
|
booksOwnSaving?: boolean;
|
|
1157
|
+
/**
|
|
1158
|
+
* The model calls and tokens of the window `sessionCostUsd` was read
|
|
1159
|
+
* for, for the report's `turn` (the service's learningPhase.md LP11).
|
|
1160
|
+
* Only the control server holds the transcript; absent, they are null.
|
|
1161
|
+
*/
|
|
1162
|
+
usage?: TurnUsage;
|
|
1123
1163
|
}): ExecutionReport | undefined;
|
|
1124
1164
|
/**
|
|
1125
1165
|
* The step a failed replay is fairly blamed on: the first whose own logic
|
|
@@ -221,6 +221,10 @@ export function sentACall(state) {
|
|
|
221
221
|
}
|
|
222
222
|
return false;
|
|
223
223
|
}
|
|
224
|
+
/** {@link Handover.reason}: a stopped step's own reason, `failed` for a plain failure, else the kind. */
|
|
225
|
+
function handoverReason(kind, stopReason) {
|
|
226
|
+
return kind === "step_failed" ? (stopReason ?? "failed") : kind;
|
|
227
|
+
}
|
|
224
228
|
/**
|
|
225
229
|
* How long a call waits for the previous pin to thread before it is judged to
|
|
226
230
|
* be running beside it: the control server's hook-view fallback
|
|
@@ -255,6 +259,59 @@ const TURN_CALLS_MAX_CHARS = 8 * 1024 * 1024;
|
|
|
255
259
|
export function isSubTask(state) {
|
|
256
260
|
return state.armedBy === "intent" && state.continuation === undefined;
|
|
257
261
|
}
|
|
262
|
+
/**
|
|
263
|
+
* Count text the plan hands the agent to read ({@link ReplayState.handedChars}),
|
|
264
|
+
* where it leaves for the agent. Adds a length and nothing else.
|
|
265
|
+
*/
|
|
266
|
+
function handed(state, text) {
|
|
267
|
+
if (!state || typeof text !== "string" || text.length === 0)
|
|
268
|
+
return;
|
|
269
|
+
state.handedChars = (state.handedChars ?? 0) + text.length;
|
|
270
|
+
}
|
|
271
|
+
/** Longest `stop` or `reason` a report's `turn` carries: they are codes, and a code is short. */
|
|
272
|
+
const TURN_CODE_MAX = 64;
|
|
273
|
+
const turnCode = (value) => typeof value === "string" && value !== "" ? value.slice(0, TURN_CODE_MAX) : null;
|
|
274
|
+
/**
|
|
275
|
+
* The report's `turn` (the service's learningPhase.md LP11): what this plan
|
|
276
|
+
* did in the turn, from its state, and the window's tokens and model calls,
|
|
277
|
+
* which only the control server can read (`usage`; undefined when it could
|
|
278
|
+
* not). Every value is null when it is not known, and nothing here throws: a
|
|
279
|
+
* state it cannot read is a `turn` of nulls, never a report lost.
|
|
280
|
+
*/
|
|
281
|
+
export function turnOf(state, usage) {
|
|
282
|
+
const count = (n) => typeof n === "number" && Number.isFinite(n) && n >= 0 ? Math.round(n) : null;
|
|
283
|
+
try {
|
|
284
|
+
const armed = state.mode === "direct" || state.mode === "steer";
|
|
285
|
+
const h = state.handover;
|
|
286
|
+
return {
|
|
287
|
+
mode: armed ? state.mode : null,
|
|
288
|
+
modelCalls: count(usage?.modelCalls),
|
|
289
|
+
inputTokens: count(usage?.inputTokens),
|
|
290
|
+
outputTokens: count(usage?.outputTokens),
|
|
291
|
+
cacheReadTokens: count(usage?.cacheReadTokens),
|
|
292
|
+
cacheWriteTokens: count(usage?.cacheWriteTokens),
|
|
293
|
+
bundleChars: armed ? count(state.handedChars ?? 0) : null,
|
|
294
|
+
brief: state.bundleKind === "brief" ? true : state.bundleKind === "full" ? false : null,
|
|
295
|
+
// Read by the control server after the turn, when it reads the answer (AC11).
|
|
296
|
+
answerChars: null,
|
|
297
|
+
handover: h ? { stop: turnCode(h.stop), reason: turnCode(h.reason) } : null,
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
catch {
|
|
301
|
+
return {
|
|
302
|
+
mode: null,
|
|
303
|
+
modelCalls: null,
|
|
304
|
+
inputTokens: null,
|
|
305
|
+
outputTokens: null,
|
|
306
|
+
cacheReadTokens: null,
|
|
307
|
+
cacheWriteTokens: null,
|
|
308
|
+
bundleChars: null,
|
|
309
|
+
brief: null,
|
|
310
|
+
answerChars: null,
|
|
311
|
+
handover: null,
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
}
|
|
258
315
|
export class ReplayController {
|
|
259
316
|
enabled;
|
|
260
317
|
work = new ProxyWorkQueue();
|
|
@@ -970,6 +1027,7 @@ export class ReplayController {
|
|
|
970
1027
|
scenario: state.scenarioId ?? undefined,
|
|
971
1028
|
bytes: h.note.length,
|
|
972
1029
|
});
|
|
1030
|
+
handed(state, h.note);
|
|
973
1031
|
return h.note;
|
|
974
1032
|
}
|
|
975
1033
|
// A plan that ran to its end with nobody there to read it has no
|
|
@@ -978,10 +1036,25 @@ export class ReplayController {
|
|
|
978
1036
|
if (u && !u.delivered) {
|
|
979
1037
|
u.delivered = true;
|
|
980
1038
|
logDetail("replay.unseen_delivered", { scenario: state.scenarioId ?? undefined, bytes: u.note.length });
|
|
1039
|
+
handed(state, u.note);
|
|
981
1040
|
return u.note;
|
|
982
1041
|
}
|
|
983
1042
|
return undefined;
|
|
984
1043
|
}
|
|
1044
|
+
/**
|
|
1045
|
+
* The hand-over note `run_scenario` answers with: claimed when it is still
|
|
1046
|
+
* to tell ({@link takeNote}), and told again when it was told already — the
|
|
1047
|
+
* same text the run always answered (`takeNote(state) ?? the note`), now
|
|
1048
|
+
* counted as read either way.
|
|
1049
|
+
*/
|
|
1050
|
+
runNote(state) {
|
|
1051
|
+
const note = this.takeNote(state);
|
|
1052
|
+
if (note !== undefined)
|
|
1053
|
+
return note;
|
|
1054
|
+
const again = state.handover?.note;
|
|
1055
|
+
handed(state, again);
|
|
1056
|
+
return again;
|
|
1057
|
+
}
|
|
985
1058
|
/** Whether a note about this plan is still to be told to the model: a hand-over's, or the unseen results'. */
|
|
986
1059
|
hasUndeliveredNote(state) {
|
|
987
1060
|
return Boolean((state?.handover && !state.handover.delivered) || (state?.unseen && !state.unseen.delivered));
|
|
@@ -1155,6 +1228,7 @@ export class ReplayController {
|
|
|
1155
1228
|
if (skip && toolName !== plan.expectedTool() && skip.tools.has(toolName) && (skip.refused ?? 0) < 2) {
|
|
1156
1229
|
skip.delivered = true;
|
|
1157
1230
|
skip.refused = (skip.refused ?? 0) + 1;
|
|
1231
|
+
handed(state, skip.text);
|
|
1158
1232
|
return this.deliverInstead(skip.text, toolName, otherHost);
|
|
1159
1233
|
}
|
|
1160
1234
|
// A call to the next step's tool: a note the model has read is done
|
|
@@ -1686,6 +1760,7 @@ export class ReplayController {
|
|
|
1686
1760
|
if (!skip || skip.delivered)
|
|
1687
1761
|
return undefined;
|
|
1688
1762
|
skip.delivered = true;
|
|
1763
|
+
handed(state, skip.text);
|
|
1689
1764
|
return skip.text;
|
|
1690
1765
|
}
|
|
1691
1766
|
/**
|
|
@@ -1699,6 +1774,7 @@ export class ReplayController {
|
|
|
1699
1774
|
if (!skip || skip.delivered || !skip.tools.has(toolName))
|
|
1700
1775
|
return undefined;
|
|
1701
1776
|
skip.delivered = true;
|
|
1777
|
+
handed(state, skip.text);
|
|
1702
1778
|
return skip.text;
|
|
1703
1779
|
}
|
|
1704
1780
|
/**
|
|
@@ -1887,7 +1963,7 @@ export class ReplayController {
|
|
|
1887
1963
|
delivery: "none",
|
|
1888
1964
|
stopReason: "not_run",
|
|
1889
1965
|
});
|
|
1890
|
-
return handedOver(this.
|
|
1966
|
+
return handedOver(this.runNote(state)) ?? { ok: false, why: ready.error };
|
|
1891
1967
|
}
|
|
1892
1968
|
const deadline = Date.now() + this.budgets.planMs;
|
|
1893
1969
|
let result;
|
|
@@ -1981,12 +2057,15 @@ export class ReplayController {
|
|
|
1981
2057
|
? { ok: true, text: `${note}\n\n${result.text}`, responseModel: {}, steps: result.executed + result.recorded, partial: true }
|
|
1982
2058
|
: (handedOver(note) ?? { ok: false, why: note });
|
|
1983
2059
|
}
|
|
1984
|
-
const note = this.
|
|
2060
|
+
const note = this.runNote(state) ?? "";
|
|
1985
2061
|
if (!live) {
|
|
1986
2062
|
// Nothing ran: the tool answers with an error that says so, which is
|
|
1987
2063
|
// what a model reacts to best.
|
|
1988
2064
|
return handedOver(note) ?? { ok: false, why: note };
|
|
1989
2065
|
}
|
|
2066
|
+
// The note in front was counted as it was claimed; the rest of the answer here.
|
|
2067
|
+
handed(state, `\n\n${result.text}`);
|
|
2068
|
+
state.bundleKind = result.brief ? "brief" : "full";
|
|
1990
2069
|
return {
|
|
1991
2070
|
ok: true,
|
|
1992
2071
|
text: `${note}\n\n${result.text}`,
|
|
@@ -2006,7 +2085,7 @@ export class ReplayController {
|
|
|
2006
2085
|
ran: 0,
|
|
2007
2086
|
noneNeeded: true,
|
|
2008
2087
|
});
|
|
2009
|
-
return handedOver(this.
|
|
2088
|
+
return handedOver(this.runNote(state)) ?? { ok: false, why: "" };
|
|
2010
2089
|
}
|
|
2011
2090
|
if (result.executed === 0) {
|
|
2012
2091
|
// Every step stood on its recorded output (BIR_REPLAY_RECORDED_OUTPUTS):
|
|
@@ -2021,7 +2100,7 @@ export class ReplayController {
|
|
|
2021
2100
|
stopReason: "not_run",
|
|
2022
2101
|
ran: 0,
|
|
2023
2102
|
});
|
|
2024
|
-
return handedOver(this.
|
|
2103
|
+
return handedOver(this.runNote(state)) ?? { ok: false, why: "" };
|
|
2025
2104
|
}
|
|
2026
2105
|
// Live steps only, as above: a recorded stand-in did no work here.
|
|
2027
2106
|
state.stepsPinned += result.executed;
|
|
@@ -2087,6 +2166,19 @@ export class ReplayController {
|
|
|
2087
2166
|
why: "the caller went away during the last step — every step ran and nobody read the results",
|
|
2088
2167
|
});
|
|
2089
2168
|
}
|
|
2169
|
+
else {
|
|
2170
|
+
// What `run_scenario` hands the agent: the bundle, and the response
|
|
2171
|
+
// model `bir-scenario` puts under it when there is one.
|
|
2172
|
+
handed(state, result.text);
|
|
2173
|
+
try {
|
|
2174
|
+
if (Object.keys(responseModel).length > 0)
|
|
2175
|
+
handed(state, JSON.stringify(responseModel));
|
|
2176
|
+
}
|
|
2177
|
+
catch {
|
|
2178
|
+
// A model no JSON can carry reaches the agent as nothing: nothing to count.
|
|
2179
|
+
}
|
|
2180
|
+
state.bundleKind = result.brief ? "brief" : "full";
|
|
2181
|
+
}
|
|
2090
2182
|
return {
|
|
2091
2183
|
ok: true,
|
|
2092
2184
|
text: result.text,
|
|
@@ -2202,6 +2294,10 @@ export class ReplayController {
|
|
|
2202
2294
|
d.cutShort === true
|
|
2203
2295
|
? false
|
|
2204
2296
|
: undefined,
|
|
2297
|
+
// How the plan ran, what the turn's model calls and tokens were, what
|
|
2298
|
+
// the agent was handed to read, and why it handed over (LP11). Only
|
|
2299
|
+
// facts the runner already had; an older service drops the field.
|
|
2300
|
+
turn: turnOf(state, d.usage),
|
|
2205
2301
|
};
|
|
2206
2302
|
}
|
|
2207
2303
|
/**
|
|
@@ -2582,12 +2678,14 @@ export class ReplayController {
|
|
|
2582
2678
|
// done its work, and the model must know before it does the task itself.
|
|
2583
2679
|
if (stop?.reason === "uncertain" && stop.position !== undefined) {
|
|
2584
2680
|
const note = this.stopNote(state, plan, stop, "none", composed.left);
|
|
2681
|
+
handed(state, note);
|
|
2585
2682
|
return this.deliverInstead(note, toolName, state.host !== undefined);
|
|
2586
2683
|
}
|
|
2587
2684
|
// The rest is the other half (AC5): the model is told so, in place of
|
|
2588
2685
|
// its call, before it does any of the plan's remaining steps.
|
|
2589
2686
|
if (stop?.kind === "other_branch" && stop.position !== undefined) {
|
|
2590
2687
|
const note = this.stopNote(state, plan, stop, "none", composed.left);
|
|
2688
|
+
handed(state, note);
|
|
2591
2689
|
return this.deliverInstead(note, toolName, state.host !== undefined);
|
|
2592
2690
|
}
|
|
2593
2691
|
// Every remaining step was left out by its test: the steps the session
|
|
@@ -2615,6 +2713,7 @@ export class ReplayController {
|
|
|
2615
2713
|
kind: stopKind,
|
|
2616
2714
|
stepIndex: parked.stepIndex,
|
|
2617
2715
|
toolName: parked.toolName ?? undefined,
|
|
2716
|
+
reason: stopKind,
|
|
2618
2717
|
note: this.noteFor(state, {
|
|
2619
2718
|
kind: stopKind,
|
|
2620
2719
|
step: parked,
|
|
@@ -2638,6 +2737,10 @@ export class ReplayController {
|
|
|
2638
2737
|
// reason is read as adversarial interception.
|
|
2639
2738
|
const action = this.deliverInstead(text, toolName, state.host !== undefined);
|
|
2640
2739
|
logDetail("replay.inject", { channel: action.kind, bytes: text.length });
|
|
2740
|
+
// What the agent reads of it: the whole text as a refusal's reason, as
|
|
2741
|
+
// much as the channel carries as a command's output.
|
|
2742
|
+
handed(state, action.kind === "bash" ? text.slice(0, BASH_CHANNEL_MAX) : text);
|
|
2743
|
+
state.bundleKind = "full";
|
|
2641
2744
|
return action;
|
|
2642
2745
|
}
|
|
2643
2746
|
/**
|
|
@@ -2675,6 +2778,7 @@ export class ReplayController {
|
|
|
2675
2778
|
...(stop.kind === "step_failed"
|
|
2676
2779
|
? { stop: stopOf({ ranThisStep: stop.stage === "tool_output_logic", stopReason: stop.reason }) }
|
|
2677
2780
|
: {}),
|
|
2781
|
+
reason: handoverReason(stop.kind, stop.reason),
|
|
2678
2782
|
note,
|
|
2679
2783
|
delivered: true,
|
|
2680
2784
|
};
|
|
@@ -2699,6 +2803,7 @@ export class ReplayController {
|
|
|
2699
2803
|
toolName: h.step?.toolName ?? undefined,
|
|
2700
2804
|
error: h.error,
|
|
2701
2805
|
...(h.kind === "step_failed" ? { stop: stopOf(h) } : {}),
|
|
2806
|
+
reason: handoverReason(h.kind, h.stopReason),
|
|
2702
2807
|
note: this.noteFor(state, h),
|
|
2703
2808
|
delivered: false,
|
|
2704
2809
|
};
|