@basein/runner 0.3.15 → 0.3.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +7 -0
  2. package/dist/bin/bir-scenario.js +8 -5
  3. package/dist/bin/bir.js +2 -1
  4. package/dist/bin/investigate.d.ts +71 -0
  5. package/dist/bin/investigate.js +145 -13
  6. package/dist/bin/scenario-edit.d.ts +45 -1
  7. package/dist/bin/scenario-edit.js +134 -14
  8. package/dist/control/server.d.ts +16 -0
  9. package/dist/control/server.js +156 -7
  10. package/dist/control/transcript.d.ts +18 -0
  11. package/dist/control/transcript.js +55 -0
  12. package/dist/record/recorder.d.ts +47 -4
  13. package/dist/record/remote-recorder.d.ts +5 -4
  14. package/dist/record/remote-recorder.js +23 -1
  15. package/dist/record/switchable-recorder.d.ts +1 -5
  16. package/dist/replay/bundle.d.ts +10 -0
  17. package/dist/replay/bundle.js +23 -2
  18. package/dist/replay/controller.d.ts +133 -0
  19. package/dist/replay/controller.js +496 -59
  20. package/dist/replay/flatten.d.ts +39 -3
  21. package/dist/replay/flatten.js +42 -2
  22. package/dist/replay/handover.d.ts +39 -1
  23. package/dist/replay/handover.js +81 -2
  24. package/dist/replay/logic.d.ts +8 -0
  25. package/dist/replay/logic.js +19 -0
  26. package/dist/replay/plan.d.ts +165 -4
  27. package/dist/replay/plan.js +488 -25
  28. package/dist/replay/source-run.d.ts +14 -1
  29. package/dist/replay/source-run.js +25 -2
  30. package/dist/replay/targets.js +8 -1
  31. package/dist/replay/types.d.ts +52 -3
  32. package/dist/replay/types.js +14 -0
  33. package/dist/util/service-headers.d.ts +7 -1
  34. package/dist/util/service-headers.js +7 -1
  35. package/docs/BaseInstRunner.md +3 -1
  36. package/docs/calculatedReplay.md +117 -6
  37. package/docs/calculatedReplayGuide.md +41 -3
  38. package/package.json +1 -1
@@ -66,7 +66,7 @@ const USAGE = {
66
66
  ' [--freeze | --unfreeze] [--note "why"] [--force --note "why"] [--revision <n>] [--json]\n' +
67
67
  ' bir scenario edit <runId|scnId> --param <name> --kind target|setting [--note "why"] [--revision <n>] [--json]',
68
68
  edits: "usage: bir scenario edits <runId|scnId> [--json]",
69
- undo: 'usage: bir scenario undo <runId|scnId> [--edit <sedit_id|pedit_id> | --step <n> | --param <name>] [--force --note "why"] [--json]',
69
+ undo: 'usage: bir scenario undo <runId|scnId> [--edit <sedit_id|pedit_id|scor_id> | --step <n> | --param <name>] [--force --note "why"] [--json]',
70
70
  calc: "usage: bir scenario calc <runId|scnId> [--force [--discard-edits]]",
71
71
  editing: "usage: bir scenario editing on|off|status",
72
72
  };
@@ -594,13 +594,19 @@ function refusal(out, report, opening, hint) {
594
594
  }
595
595
  out(hint);
596
596
  }
597
- // ── edits / undo ──────────────────────────────────────────────────────────────
598
- /** The service's history: step edits, and — from a service that has them — kind changes. */
597
+ /**
598
+ * The service's history: step edits, and — from a service that has them —
599
+ * kind changes and corrections (autocorrect.md AC7). An older service sends
600
+ * neither list, and there is nothing of them to print.
601
+ */
599
602
  function historyOf(body) {
600
603
  const b = body;
601
604
  return {
602
605
  edits: Array.isArray(b?.edits) ? b.edits : [],
603
606
  paramEdits: Array.isArray(b?.paramEdits) ? b.paramEdits : [],
607
+ corrections: Array.isArray(b?.corrections)
608
+ ? b.corrections.filter((c) => typeof c?.id === "string" && typeof c?.createdAt === "string")
609
+ : [],
604
610
  };
605
611
  }
606
612
  async function fetchEdits(scenarioId, deps) {
@@ -628,15 +634,16 @@ async function listEdits(args, deps) {
628
634
  out(jsonText(reply.body));
629
635
  return 0;
630
636
  }
631
- const { edits, paramEdits } = historyOf(reply.body);
632
- if (edits.length === 0 && paramEdits.length === 0) {
637
+ const { edits, paramEdits, corrections } = historyOf(reply.body);
638
+ if (edits.length === 0 && paramEdits.length === 0 && corrections.length === 0) {
633
639
  out(`No hand edits of ${target}.`);
634
640
  return 0;
635
641
  }
636
642
  // Ids are printed whole: they are what `undo --edit` takes. The detail line
637
643
  // is indented rather than aligned under the date, which a 42-character id
638
644
  // would push past most terminals' width. One list, newest first: a step's
639
- // edits and a parameter's kind changes are walked back by the same `undo`.
645
+ // edits, a parameter's kind changes and the service's corrections of the
646
+ // plan are walked back by the same `undo`.
640
647
  const noteOf = (note) => (note ? ` "${note.replace(/\s+/g, " ")}"` : "");
641
648
  const rows = [
642
649
  ...edits.map((e) => ({
@@ -649,6 +656,11 @@ async function listEdits(args, deps) {
649
656
  line: `${p.id} ${when(p.createdAt)} param ${p.param} ${String(p.kind).padEnd(7)} revision ${p.chainRevisionBefore} → ${p.chainRevisionAfter}${noteOf(p.note)}`,
650
657
  detail: paramSummary(p),
651
658
  })),
659
+ ...corrections.map((c) => ({
660
+ at: c.createdAt,
661
+ line: `${c.id} ${when(c.createdAt)} correction ${String(c.state).padEnd(8)} revision ${c.chainRevisionBefore ?? "?"} → ${c.chainRevisionAfter ?? "—"}`,
662
+ detail: correctionSummary(c),
663
+ })),
652
664
  ].sort((x, y) => (x.at < y.at ? 1 : x.at > y.at ? -1 : 0));
653
665
  for (const r of rows) {
654
666
  out(r.line);
@@ -656,6 +668,62 @@ async function listEdits(args, deps) {
656
668
  }
657
669
  return 0;
658
670
  }
671
+ /** `[11,12,13,15]` → "11–13, 15": the step lists of a correction. */
672
+ export function stepRanges(steps) {
673
+ const sorted = [...new Set((steps ?? []).filter((n) => Number.isInteger(n)))].sort((a, b) => a - b);
674
+ const parts = [];
675
+ for (let i = 0; i < sorted.length;) {
676
+ let j = i;
677
+ while (j + 1 < sorted.length && sorted[j + 1] === sorted[j] + 1)
678
+ j += 1;
679
+ parts.push(j === i ? String(sorted[i]) : `${sorted[i]}–${sorted[j]}`);
680
+ i = j + 1;
681
+ }
682
+ return parts.join(", ");
683
+ }
684
+ /** The second line of a correction in `edits`: what it changed, from which run, and what became of it. */
685
+ export function correctionSummary(c) {
686
+ const parts = [];
687
+ if (c.summary)
688
+ parts.push(c.summary.replace(/\s+/g, " ").trim());
689
+ const changes = [];
690
+ const count = (steps) => steps?.length ?? 0;
691
+ if (count(c.added))
692
+ changes.push(`${count(c.added)} step${count(c.added) === 1 ? "" : "s"} added (${stepRanges(c.added)})`);
693
+ if (count(c.tested))
694
+ changes.push(`tests on step${count(c.tested) === 1 ? "" : "s"} ${stepRanges(c.tested)}`);
695
+ if (count(c.changed))
696
+ changes.push(`step${count(c.changed) === 1 ? "" : "s"} ${stepRanges(c.changed)} changed`);
697
+ if (c.answerChanged)
698
+ changes.push("the answer changed");
699
+ if (changes.length > 0)
700
+ parts.push(changes.join(", "));
701
+ if (c.redoneRunId)
702
+ parts.push(`from the redo ${c.redoneRunId}${c.verdictKind ? ` (${c.verdictKind})` : ""}`);
703
+ switch (c.state) {
704
+ case "proposed":
705
+ parts.push('shown, not used yet — "Use the correction" on the recording page uses it');
706
+ break;
707
+ case "used":
708
+ parts.push(`used${c.usedAt ? ` ${when(c.usedAt)}` : ""}`);
709
+ break;
710
+ case "failed": {
711
+ const first = c.problems?.[0];
712
+ parts.push(`not used: its check failed${first ? ` — ${first}` : ""}`);
713
+ break;
714
+ }
715
+ case "undone":
716
+ parts.push(`undone${c.undoneAt ? ` ${when(c.undoneAt)}` : ""}`);
717
+ break;
718
+ case "replaced":
719
+ parts.push("replaced by a recalculation");
720
+ break;
721
+ default:
722
+ if (c.state)
723
+ parts.push(String(c.state));
724
+ }
725
+ return parts.join("; ");
726
+ }
659
727
  /** The second line of a kind change in `edits`: the kinds, and what became of it. */
660
728
  function paramSummary(p) {
661
729
  const parts = [];
@@ -722,14 +790,29 @@ export function newestUndoableParam(paramEdits, param) {
722
790
  .sort((x, y) => (x.createdAt < y.createdAt ? 1 : x.createdAt > y.createdAt ? -1 : 0))
723
791
  .find((e) => e.kind !== "revert" && !e.revertedAt && !e.replacedAt && (param === undefined || e.param === param));
724
792
  }
793
+ /**
794
+ * The correction `undo` without `--edit` takes back, when it is the newest
795
+ * change of all: one in use, not undone. A plan-wide change, so never one
796
+ * when `--step` or `--param` narrows the undo to a step or a parameter.
797
+ */
798
+ export function newestUndoableCorrection(corrections) {
799
+ return [...corrections]
800
+ .filter((c) => c.state === "used" && !c.undoneAt)
801
+ .sort((x, y) => {
802
+ const a = x.usedAt ?? x.createdAt;
803
+ const b = y.usedAt ?? y.createdAt;
804
+ return a < b ? 1 : a > b ? -1 : 0;
805
+ })[0];
806
+ }
725
807
  const isParamEditId = (id) => id.startsWith("pedit_");
808
+ const isCorrectionId = (id) => id.startsWith("scor_");
726
809
  async function undo(args, deps) {
727
810
  const { out } = deps;
728
811
  const target = args.positionals[1];
729
812
  if (!target)
730
813
  return usageError(deps, "undo");
731
- if (args.edit !== undefined && !/^(sedit|pedit)_[\w-]+$/.test(args.edit)) {
732
- return usageError(deps, "undo", `--edit takes a sedit_… or pedit_… id, as \`bir scenario edits\` prints it; got ${args.edit}.`);
814
+ if (args.edit !== undefined && !/^(sedit|pedit|scor)_[\w-]+$/.test(args.edit)) {
815
+ return usageError(deps, "undo", `--edit takes a sedit_…, pedit_… or scor_… id, as \`bir scenario edits\` prints it; got ${args.edit}.`);
733
816
  }
734
817
  const picks = [args.edit, args.step, args.param].filter((v) => v !== undefined).length;
735
818
  if (picks > 1) {
@@ -754,11 +837,18 @@ async function undo(args, deps) {
754
837
  const listed = await fetchEdits(resolved.id, deps);
755
838
  if ("reply" in listed)
756
839
  return explain(listed.reply, args, deps, { verb: "Could not read the history", target });
757
- // Undo walks back across both lists (D9): the newest change of either
758
- // kind goes first, unless --step or --param narrows it to one.
840
+ // Undo walks back across every list (D9): the newest change of any kind
841
+ // goes first — a correction in use too, which is a change of the whole
842
+ // plan (autocorrect.md AC7) — unless --step or --param narrows it to one.
759
843
  const stepPick = args.param === undefined ? newestUndoable(listed.edits, args.step) : undefined;
760
844
  const paramPick = args.step === undefined ? newestUndoableParam(listed.paramEdits, args.param) : undefined;
761
- const pick = !stepPick ? paramPick : !paramPick ? stepPick : paramPick.createdAt > stepPick.createdAt ? paramPick : stepPick;
845
+ const correctionPick = args.step === undefined && args.param === undefined ? newestUndoableCorrection(listed.corrections) : undefined;
846
+ const candidates = [
847
+ ...(stepPick ? [{ id: stepPick.id, at: stepPick.createdAt }] : []),
848
+ ...(paramPick ? [{ id: paramPick.id, at: paramPick.createdAt }] : []),
849
+ ...(correctionPick ? [{ id: correctionPick.id, at: correctionPick.usedAt ?? correctionPick.createdAt }] : []),
850
+ ];
851
+ const pick = candidates.sort((x, y) => (x.at < y.at ? 1 : x.at > y.at ? -1 : 0))[0];
762
852
  if (!pick) {
763
853
  return localFailure(args, deps, "nothing_to_undo", args.step !== undefined
764
854
  ? `Nothing to undo: step ${args.step} of ${target} has no hand edit that can be undone.`
@@ -793,6 +883,8 @@ async function undo(args, deps) {
793
883
  }
794
884
  if (isParamEditId(editId))
795
885
  return paramUndone(reply.body, editId, target, args, deps);
886
+ if (isCorrectionId(editId))
887
+ return correctionUndone(reply.body, editId, target, deps);
796
888
  if (answer?.report)
797
889
  renderReport(out, answer.report, { brief: true, undo: true });
798
890
  if (refused && answer?.report) {
@@ -820,6 +912,20 @@ async function undo(args, deps) {
820
912
  }
821
913
  return 0;
822
914
  }
915
+ /**
916
+ * A correction taken back: the plan is as it was before it, exactly — and
917
+ * held, because that plan was the one the agent rejected. Said so, with the
918
+ * way to let it go.
919
+ */
920
+ function correctionUndone(answer, editId, target, deps) {
921
+ const revision = answer?.chainRevision ?? answer?.correction?.chainRevisionAfter;
922
+ deps.out(`The plan is back as it was before correction ${answer?.correction?.id ?? editId} (revision ${revision ?? "?"}).`);
923
+ if (answer?.held) {
924
+ deps.out(` It is held${answer.held.reason ? ` (${answer.held.reason})` : ""}: the plan from before is the one the agent rejected, so it is not used — the agent does the task — until it is corrected again or someone presses "Use it again" on the recording page.`);
925
+ }
926
+ deps.out(` One further back: bir scenario undo ${target}`);
927
+ return 0;
928
+ }
823
929
  /**
824
930
  * A kind change undone. Nothing ran, so there is no report: the parameter's
825
931
  * kind as it is now, what that means for the next prompt, and the way on —
@@ -884,8 +990,14 @@ async function calc(args, deps) {
884
990
  if (status === 409 && code === "scenario_has_edits") {
885
991
  const steps = body.editedSteps;
886
992
  const list = Array.isArray(steps) ? steps.map(String) : [];
993
+ // A correction the service made from a redo is protected the same way
994
+ // (autocorrect.md AC6): a recalculation from the recording would bring
995
+ // back the one-half plan it fixed.
996
+ const correction = body.correctionId;
887
997
  const which = list.length === 0 ? "some steps were" : list.length === 1 ? `step ${list[0]} was` : `steps ${list.join(", ")} were`;
888
- out(`Not recalculated: ${which} edited by hand, and a recalculation rebuilds every step.`);
998
+ out(typeof correction === "string"
999
+ ? `Not recalculated: the plan carries correction ${correction}, and a recalculation from the recording would throw it away.`
1000
+ : `Not recalculated: ${which} edited by hand, and a recalculation rebuilds every step.`);
889
1001
  out(` To recalculate anyway and discard the edits: bir scenario calc ${target} --force --discard-edits`);
890
1002
  out(" (the edits stay in `bir scenario edits`, marked replaced)");
891
1003
  return 1;
@@ -1057,6 +1169,8 @@ function explain(reply, args, deps, e) {
1057
1169
  const stepFlag = e.step !== undefined ? ` --step ${e.step}` : "";
1058
1170
  // A kind change (findoffer.md) is about a parameter, and nothing in it is checked.
1059
1171
  const aboutParam = e.param !== undefined || (e.editId?.startsWith("pedit_") ?? false);
1172
+ // A correction (autocorrect.md AC7) is about the whole plan.
1173
+ const aboutCorrection = e.editId?.startsWith("scor_") ?? false;
1060
1174
  if (reply.status === 404) {
1061
1175
  out(e.notFound ??
1062
1176
  `${e.verb} (not_found): no such scenario, step or edit — or it is not yours. Only a scenario's owner can see or change it.`);
@@ -1110,13 +1224,19 @@ function explain(reply, args, deps, e) {
1110
1224
  // newer entry puts the step back as this one left it, and then this one
1111
1225
  // can be undone. It used to loop, when undo only toggled (C23).
1112
1226
  const latest = typeof body.latestEditId === "string" ? body.latestEditId : "<the newer edit>";
1113
- out(`${e.verb} (not_latest_edit): the ${aboutParam ? "parameter" : "step"} has changed since that edit — ${latest} changed it after. Undo the newer one first, then this one:`);
1227
+ out(aboutCorrection
1228
+ ? `${e.verb} (not_latest_edit): the plan has changed since that correction — ${latest} changed it after. Undo the newer one first, then this one:`
1229
+ : `${e.verb} (not_latest_edit): the ${aboutParam ? "parameter" : "step"} has changed since that edit — ${latest} changed it after. Undo the newer one first, then this one:`);
1114
1230
  out(` bir scenario undo ${target} --edit ${latest}`);
1115
1231
  out(` bir scenario undo ${target} --edit ${e.editId ?? "<this edit>"}`);
1116
1232
  return 1;
1117
1233
  }
1118
1234
  case "already_undone": {
1119
1235
  const by = typeof body.revertedById === "string" ? body.revertedById : undefined;
1236
+ if (aboutCorrection) {
1237
+ out(`${e.verb} (already_undone): ${e.editId ?? "that correction"} was taken back already; \`bir scenario edits ${target}\` lists what the plan has now.`);
1238
+ return 1;
1239
+ }
1120
1240
  out(`${e.verb} (already_undone): ${e.editId ?? "that edit"} was undone already${by ? `, by ${by}` : ""}. To redo it, undo that undo:`);
1121
1241
  out(` bir scenario undo ${target} --edit ${by ?? "<the undo's sedit_ id, from bir scenario edits>"}`);
1122
1242
  return 1;
@@ -1124,7 +1244,7 @@ function explain(reply, args, deps, e) {
1124
1244
  case "edit_replaced":
1125
1245
  out(aboutParam
1126
1246
  ? `${e.verb} (edit_replaced): a recalculation dropped that parameter after the change. It stays in \`bir scenario edits\`, but there is nothing left to undo.`
1127
- : `${e.verb} (edit_replaced): a recalculation replaced the plan after that edit. It stays in \`bir scenario edits\`, but there is nothing left to undo.`);
1247
+ : `${e.verb} (edit_replaced): a recalculation replaced the plan after that ${aboutCorrection ? "correction" : "edit"}. It stays in \`bir scenario edits\`, but there is nothing left to undo.`);
1128
1248
  return 1;
1129
1249
  case "rate_limited": {
1130
1250
  const ms = typeof body.retryAfterMs === "number" ? body.retryAfterMs : undefined;
@@ -1107,6 +1107,22 @@ export declare class ControlServer {
1107
1107
  * answered `id: null` — the service records no repeat of it — drops them.
1108
1108
  */
1109
1109
  private openRepeat;
1110
+ /**
1111
+ * Record the prompt plan's own calls of this turn into the `redone` repeat
1112
+ * `id`, in the order they went out, before the agent's first call of its
1113
+ * own (the service's autocorrect.md AC2). Each is a pair, both rows marked
1114
+ * `byPlan` with the plan, its step and its chain revision, so the service
1115
+ * stores them without a vector and counts none of them as the agent's.
1116
+ * Payloads are whole — redacted and capped as any recorded call's — where
1117
+ * the report has only 3,800 bytes of each. Returns how many it recorded.
1118
+ */
1119
+ private recordPlanCalls;
1120
+ /**
1121
+ * One tool call of the model's own was recorded into `run`: counted for
1122
+ * the plan's report when it is the `redone` repeat (AC1). A step a plan
1123
+ * pinned is the plan's doing, not the model's, and is not counted.
1124
+ */
1125
+ private noteOwnCall;
1110
1126
  /**
1111
1127
  * Why the matched turn's plan did not run, as the service shows it: what
1112
1128
  * the service withheld it for, else the code of the gate that declined it
@@ -31,7 +31,7 @@ import { BIR_CALL_ID, fingerprint, newCallId, parseQualifiedName, qualifyToolNam
31
31
  import { StepIndexAllocator } from "./ordering.js";
32
32
  import { pidAlive } from "./discovery.js";
33
33
  import { pinCompatible } from "./pin-guard.js";
34
- import { contextForToolUse, intentForToolUse, markTranscriptUsage, recentToolResults, settledLastAssistantText, thinkingForToolUse, usageSince, } from "./transcript.js";
34
+ import { contextForToolUse, intentForToolUse, markTranscriptUsage, recentToolResults, settledLastAssistantText, settledTextAfterToolUse, thinkingForToolUse, usageSince, } from "./transcript.js";
35
35
  import { isHousekeepingCall } from "../record/housekeeping.js";
36
36
  import { asSent, userText } from "../record/prompt-text.js";
37
37
  import { redact, redactText } from "../record/redact.js";
@@ -1811,6 +1811,11 @@ export class ControlServer {
1811
1811
  // the jail it may not leave (replay/builtins.ts). The session's own cwd
1812
1812
  // from the hook payloads, never the daemon's.
1813
1813
  state.sessionCwd = session.cwd ?? this.opts.cwd;
1814
+ // The service offered to record this turn as the recording's newest run:
1815
+ // should the agent redo the task after the plan, the plan's own calls go
1816
+ // into that run first, so they are kept whole as they run (AC2).
1817
+ if (match.repeat?.runId)
1818
+ state.keepCalls = true;
1814
1819
  this.declineWithoutDeliveryTool(state);
1815
1820
  // A steer plan with a shell or file-write step can only be followed if the
1816
1821
  // directive shows its computed arguments (the pin guard holds those steps
@@ -2289,7 +2294,14 @@ export class ControlServer {
2289
2294
  // A matched turn records nothing, but it still owes the ledger a number —
2290
2295
  // including when it declined to steer, which is a *baseline* sample and is
2291
2296
  // what every saving is measured against (docs/calculatedReplay.md §11).
2292
- this.reportExecution(session, run, durationMs, cost, cutShort);
2297
+ this.reportExecution(session, run, durationMs, cost, cutShort, reason);
2298
+ // The plan's calls kept for a redo went into the repeat when it opened
2299
+ // (AC2); nothing reads them after the turn, and they can run to megabytes
2300
+ // on a sealed run the session keeps as its previous one.
2301
+ for (const s of run.replays.length > 0 ? run.replays : run.replay ? [run.replay] : []) {
2302
+ s.turnCalls = undefined;
2303
+ s.turnCallChars = 0;
2304
+ }
2293
2305
  // A fragment is recorded from the hand-over on (fallbk.md D7), so its cost
2294
2306
  // and duration are the model's own work, not the steps the scenario did.
2295
2307
  // So is a repeat opened after the plan ran to its end (newestRun.md N7). Another host
@@ -2323,8 +2335,10 @@ export class ControlServer {
2323
2335
  // Resolve the answer *outside* the queue: the wait below is up to two
2324
2336
  // seconds, and the queue is serial — blocking it here would delay the
2325
2337
  // first step of the user's next turn by that much. Kept on the run, so a
2326
- // drain of this run waits for this seal and for nobody else's.
2327
- run.sealing = answerText
2338
+ // drain of this run waits for this seal and for nobody else's — beside a
2339
+ // plan's report that waits for the answer too (AC11), never in its place.
2340
+ const before = run.sealing;
2341
+ const finishing = answerText
2328
2342
  .then((answer) => {
2329
2343
  run.finishQueued = true;
2330
2344
  this.queue.push(() => {
@@ -2347,6 +2361,7 @@ export class ControlServer {
2347
2361
  });
2348
2362
  });
2349
2363
  });
2364
+ run.sealing = before ? Promise.all([before, finishing]).then(() => undefined) : finishing;
2350
2365
  }
2351
2366
  logLine("run.finish", {
2352
2367
  run: run.runId,
@@ -2439,7 +2454,9 @@ export class ControlServer {
2439
2454
  }
2440
2455
  reportExecution(session, run, durationMs, cost,
2441
2456
  /** The run was closed before its task ended for the caller (another host only). */
2442
- cutShort = false) {
2457
+ cutShort = false,
2458
+ /** Why the turn was sealed: only an ordinary `stop` ends with an answer (AC11). */
2459
+ sealedBy = "stop") {
2443
2460
  if (!isScenarioReporter(this.recorder))
2444
2461
  return;
2445
2462
  const states = run.replays.length > 0 ? run.replays : run.replay ? [run.replay] : [];
@@ -2447,6 +2464,14 @@ export class ControlServer {
2447
2464
  // Another host hands over one number for the whole run. It cannot be
2448
2465
  // split between two plans, so with more than one neither is measured.
2449
2466
  const unsplit = otherHost(session) && states.length > 1;
2467
+ // The plan ran to its end and the agent redid the task in a repeat the
2468
+ // service took (newestRun.md N7): the prompt plan's report says so, with
2469
+ // the repeat's id and the agent's own calls in it (autocorrect.md AC1).
2470
+ const r = run.repeat;
2471
+ const redoneBy = r?.state === "open" && r.reason === "redone" && r.openedRunId
2472
+ ? { runId: r.openedRunId, ownCalls: r.ownCalls ?? 0 }
2473
+ : undefined;
2474
+ const prompt = states.find((s) => s.armedBy === "prompt");
2450
2475
  // One report per plan armed this turn (fallbk.md §Runner 4). Each is billed
2451
2476
  // for its own window of the transcript — from when it armed to when the next
2452
2477
  // one did — so the turn's tokens are split between them, never counted twice.
@@ -2484,6 +2509,7 @@ export class ControlServer {
2484
2509
  siblings: states,
2485
2510
  runId: run.runId,
2486
2511
  cutShort: cutShort || undefined,
2512
+ redoneBy: state === prompt && state.ranToEnd === true ? redoneBy : undefined,
2487
2513
  });
2488
2514
  if (!report)
2489
2515
  return;
@@ -2494,6 +2520,30 @@ export class ControlServer {
2494
2520
  billedTo: state.retiredMark !== undefined ? "its own retire" : "the next arm or the turn",
2495
2521
  });
2496
2522
  }
2523
+ // The plan ran to its end, and no redo of the model's own was recorded:
2524
+ // the model only answered. Its answer goes with the report, so that the
2525
+ // service can tell whether it took the plan's result (the service's
2526
+ // autocorrect.md AC11). Read off the queue, as a recording's answer is —
2527
+ // the wait is up to two seconds, and the queue is serial — and kept on
2528
+ // the run, so a drain of this run waits for it. Only after an ordinary
2529
+ // stop, and only the text written after the turn's last call: a turn cut
2530
+ // short has no answer, and an older text is none. Claude Code only: the
2531
+ // service corrects no other host's plan.
2532
+ if (state === prompt &&
2533
+ state.ranToEnd === true &&
2534
+ !redoneBy &&
2535
+ report.outcome === "steered_full" &&
2536
+ sealedBy === "stop" &&
2537
+ run.lastToolUseId !== undefined &&
2538
+ !otherHost(session)) {
2539
+ const sent = settledTextAfterToolUse(session.transcriptPath, run.lastToolUseId)
2540
+ .catch(() => "")
2541
+ .then((answer) => {
2542
+ this.queue.push(() => recorder.reportExecution(answer ? { ...report, answer } : report));
2543
+ });
2544
+ run.sealing = run.sealing ? Promise.all([run.sealing, sent]).then(() => undefined) : sent;
2545
+ return;
2546
+ }
2497
2547
  this.queue.push(() => recorder.reportExecution(report));
2498
2548
  });
2499
2549
  // The prompt match that landed too late (see `noteLateMatch`). Not when a
@@ -2894,6 +2944,7 @@ export class ControlServer {
2894
2944
  if (!steering.retired) {
2895
2945
  steering.pinned.clear();
2896
2946
  steering.retired = true;
2947
+ steering.skipNote = undefined;
2897
2948
  logLine("replay.pin_overran", {
2898
2949
  scenario: steering.scenarioId ?? undefined,
2899
2950
  tool: toolName,
@@ -2915,9 +2966,25 @@ export class ControlServer {
2915
2966
  if (verdict.ok) {
2916
2967
  pinned = action.input;
2917
2968
  this.declinedPins.delete(steering);
2969
+ this.replay.pinTaken(steering);
2918
2970
  }
2919
2971
  else {
2920
2972
  steering.pinned.delete(toolUseId);
2973
+ // Not the next step — and perhaps the step a test just left out,
2974
+ // which shares its tool, made because the note saying so had not
2975
+ // reached the model yet (AC5). The note goes in its place: a step
2976
+ // left out never runs because the note was late. The plan stays
2977
+ // armed for the step that is next.
2978
+ const skipped = this.replay.skipNoteFor(steering, toolName);
2979
+ if (skipped) {
2980
+ logLine("replay.pin_declined", {
2981
+ scenario: steering.scenarioId ?? undefined,
2982
+ tool: toolName,
2983
+ step: action.stepIndex,
2984
+ why: "a call to the step a test left out — the note saying so is given in its place",
2985
+ });
2986
+ return this.preToolAnswer(this.replay.deliverInstead(skipped, toolName));
2987
+ }
2921
2988
  this.declinedPins.add(steering);
2922
2989
  logLine("replay.pin_declined", {
2923
2990
  scenario: steering.scenarioId ?? undefined,
@@ -2938,6 +3005,13 @@ export class ControlServer {
2938
3005
  // a plan nobody would ever hear of.
2939
3006
  if (!overran && !pinned && (!run.replay?.plan || run.replay.retired)) {
2940
3007
  // ── falling back to the model (fallbk.md) ──────────────────────────────
3008
+ // 0. A call to a step a test left out (AC5), when the note saying so has
3009
+ // not reached the model yet — the plan may have finished on it. The
3010
+ // note goes in its place: a step left out must never run because the
3011
+ // note was late.
3012
+ const skipped = planning ? undefined : this.replay.skipNoteFor(run.replay, toolName);
3013
+ if (skipped)
3014
+ return this.preToolAnswer(this.replay.deliverInstead(skipped, toolName));
2941
3015
  // 1. A hand-over noticed where no hook answer could carry it — a proxy
2942
3016
  // report threaded after its PostToolUse had already answered — reaches
2943
3017
  // the model in place of this call. Never silence (D4).
@@ -3071,6 +3145,7 @@ export class ControlServer {
3071
3145
  ? (run.replay?.scenarioId ?? undefined)
3072
3146
  : undefined;
3073
3147
  if (run.recording) {
3148
+ this.noteOwnCall(run, pinnedBy);
3074
3149
  this.queue.push(() => {
3075
3150
  this.recorder.recordToolSelected(run.runId, pair.selected, {
3076
3151
  toolName,
@@ -3302,6 +3377,7 @@ export class ControlServer {
3302
3377
  toolName,
3303
3378
  });
3304
3379
  if (run.recording) {
3380
+ this.noteOwnCall(run);
3305
3381
  const toolInput = serializeCapped(redact(modelArgs));
3306
3382
  this.queue.push(() => {
3307
3383
  this.recorder.recordToolSelected(run.runId, pair.selected, {
@@ -3360,6 +3436,7 @@ export class ControlServer {
3360
3436
  state.retired = true;
3361
3437
  state.retiredAt = Date.now();
3362
3438
  state.pinned.clear();
3439
+ state.skipNote = undefined;
3363
3440
  this.directCalls = this.directCalls.filter((c) => c.state !== state);
3364
3441
  logLine("replay.vetoed", {
3365
3442
  run: run.runId,
@@ -3390,6 +3467,8 @@ export class ControlServer {
3390
3467
  state.retired = true;
3391
3468
  state.retiredAt = Date.now();
3392
3469
  state.pinned.clear();
3470
+ // A note naming a step to call next is void with the plan (AC5 review).
3471
+ state.skipNote = undefined;
3393
3472
  run.handback = HANDBACK_NOTE;
3394
3473
  logLine("replay.pin_abandoned", {
3395
3474
  run: run.runId,
@@ -4099,6 +4178,13 @@ export class ControlServer {
4099
4178
  run.ordering = new StepIndexAllocator();
4100
4179
  run.recording = true;
4101
4180
  r.state = "open";
4181
+ r.openedRunId = id;
4182
+ r.reason = reason;
4183
+ r.ownCalls = 0;
4184
+ // The plan's own calls of this turn go in first, whole and marked, so the
4185
+ // newest run holds the whole turn: the agent's work is built on what the
4186
+ // plan's calls returned (the service's autocorrect.md AC2).
4187
+ const planCalls = redone ? this.recordPlanCalls(run, id) : 0;
4102
4188
  if (redone) {
4103
4189
  // The plan's steps are not the model's work: this run is billed and
4104
4190
  // timed from its own first call on, as a fragment is.
@@ -4109,6 +4195,7 @@ export class ControlServer {
4109
4195
  run: id,
4110
4196
  of: r.runId,
4111
4197
  reason,
4198
+ planCalls: planCalls || undefined,
4112
4199
  why: redone
4113
4200
  ? "the plan ran to its end, and the agent went on with calls of its own instead of answering from its results — recorded as that recording's newest run"
4114
4201
  : "the agent is doing a recorded task itself — recorded as that recording's newest run",
@@ -4119,10 +4206,15 @@ export class ControlServer {
4119
4206
  .runCreated(id)
4120
4207
  .catch(() => ({ created: true }))
4121
4208
  .then((created) => {
4122
- if (created.created || run.runId !== id)
4209
+ if (created.created)
4123
4210
  return;
4124
- run.recording = false;
4211
+ // Refused: never named as the turn's redo (the service's
4212
+ // autocorrect.md AC1) — also when the turn has moved on to another
4213
+ // run since, which used to leave the repeat `open` and named.
4125
4214
  r.state = "refused";
4215
+ if (run.runId !== id)
4216
+ return;
4217
+ run.recording = false;
4126
4218
  logLine("run.repeat_refused", {
4127
4219
  run: id,
4128
4220
  of: r.runId,
@@ -4130,6 +4222,54 @@ export class ControlServer {
4130
4222
  });
4131
4223
  });
4132
4224
  }
4225
+ /**
4226
+ * Record the prompt plan's own calls of this turn into the `redone` repeat
4227
+ * `id`, in the order they went out, before the agent's first call of its
4228
+ * own (the service's autocorrect.md AC2). Each is a pair, both rows marked
4229
+ * `byPlan` with the plan, its step and its chain revision, so the service
4230
+ * stores them without a vector and counts none of them as the agent's.
4231
+ * Payloads are whole — redacted and capped as any recorded call's — where
4232
+ * the report has only 3,800 bytes of each. Returns how many it recorded.
4233
+ */
4234
+ recordPlanCalls(run, id) {
4235
+ const prompt = run.replays.find((s) => s.armedBy === "prompt");
4236
+ const calls = this.replay.turnCallsOf(prompt);
4237
+ for (const call of calls) {
4238
+ const pair = run.ordering.allocatePair();
4239
+ const byPlan = {
4240
+ scenarioId: call.scenarioId,
4241
+ stepIndex: call.stepIndex,
4242
+ // Left out, never null, when the plan was served without one: the
4243
+ // service's row schema takes a number or nothing.
4244
+ ...(call.chainRevision !== null ? { chainRevision: call.chainRevision } : {}),
4245
+ };
4246
+ this.queue.push(() => {
4247
+ this.recorder.recordToolSelected(id, pair.selected, {
4248
+ toolName: call.toolName,
4249
+ toolInput: call.input,
4250
+ metadata: { byPlan },
4251
+ });
4252
+ this.recorder.recordToolResponse(id, pair.response, {
4253
+ toolName: call.toolName,
4254
+ toolOutput: call.error === undefined ? call.output : undefined,
4255
+ toolError: call.error,
4256
+ metadata: { byPlan },
4257
+ });
4258
+ });
4259
+ }
4260
+ return calls.length;
4261
+ }
4262
+ /**
4263
+ * One tool call of the model's own was recorded into `run`: counted for
4264
+ * the plan's report when it is the `redone` repeat (AC1). A step a plan
4265
+ * pinned is the plan's doing, not the model's, and is not counted.
4266
+ */
4267
+ noteOwnCall(run, pinnedBy) {
4268
+ const r = run.repeat;
4269
+ if (pinnedBy || !r || r.state !== "open" || run.runId !== r.openedRunId)
4270
+ return;
4271
+ r.ownCalls = (r.ownCalls ?? 0) + 1;
4272
+ }
4133
4273
  /**
4134
4274
  * Why the matched turn's plan did not run, as the service shows it: what
4135
4275
  * the service withheld it for, else the code of the gate that declined it
@@ -4171,10 +4311,18 @@ export class ControlServer {
4171
4311
  context.push(stashed);
4172
4312
  const run = this.sessions.get(payload.session_id ?? "unknown-session")?.run;
4173
4313
  if (run && !run.finished) {
4314
+ // The answer, if one comes, is written after this call (AC11).
4315
+ if (typeof payload.tool_use_id === "string" && payload.tool_use_id)
4316
+ run.lastToolUseId = payload.tool_use_id;
4174
4317
  this.noteHandover(run);
4175
4318
  const note = this.replay.takeNote(run.replay);
4176
4319
  if (note)
4177
4320
  context.push(note);
4321
+ // The steps a test just left out (AC5): said before the model makes its
4322
+ // next call, which must not be one of them.
4323
+ const skipped = this.replay.takeSkipNote(run.replay);
4324
+ if (skipped)
4325
+ context.push(skipped);
4178
4326
  if (run.handback) {
4179
4327
  context.push(run.handback);
4180
4328
  run.handback = undefined;
@@ -5150,6 +5298,7 @@ export class ControlServer {
5150
5298
  record(run, pair, d) {
5151
5299
  if (!run.recording)
5152
5300
  return;
5301
+ this.noteOwnCall(run, d.pinnedBy);
5153
5302
  const toolInput = serializeCapped(redact(d.args));
5154
5303
  const toolOutput = d.isError ? undefined : serializeCapped(redact(d.result));
5155
5304
  const toolError = d.isError
@@ -34,6 +34,24 @@
34
34
  * birth time, mtime, and the bytes at its head and at the offset) and read
35
35
  * again from the start, which is exactly what the old code did every time.
36
36
  */
37
+ /**
38
+ * The model's text after the message that made the tool call `toolUseId`: its
39
+ * answer, once that call's result came back (the service's autocorrect.md
40
+ * AC11). Empty while none has been written, and when the call is not in the
41
+ * transcript — never an older text, which `lastAssistantText` can be: the
42
+ * sentence before a call, or the turn before's answer.
43
+ */
44
+ export declare function textAfterToolUse(transcriptPath: string | undefined, toolUseId: string | undefined): string;
45
+ /**
46
+ * {@link textAfterToolUse}, waited for: up to `timeoutMs` for a text to appear,
47
+ * then until it stops changing for `confirmMs`. Empty when none came in time —
48
+ * a plan's report then goes without an answer, rather than with a wrong one.
49
+ */
50
+ export declare function settledTextAfterToolUse(transcriptPath: string | undefined, toolUseId: string | undefined, opts?: {
51
+ timeoutMs?: number;
52
+ confirmMs?: number;
53
+ pollMs?: number;
54
+ }): Promise<string>;
37
55
  /** Forget every cached index (tests; a caller that knows files were replaced). */
38
56
  export declare function forgetTranscripts(): void;
39
57
  /**