@basein/runner 0.3.14 → 0.3.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1811,6 +1811,11 @@ export class ControlServer {
1811
1811
  // the jail it may not leave (replay/builtins.ts). The session's own cwd
1812
1812
  // from the hook payloads, never the daemon's.
1813
1813
  state.sessionCwd = session.cwd ?? this.opts.cwd;
1814
+ // The service offered to record this turn as the recording's newest run:
1815
+ // should the agent redo the task after the plan, the plan's own calls go
1816
+ // into that run first, so they are kept whole as they run (AC2).
1817
+ if (match.repeat?.runId)
1818
+ state.keepCalls = true;
1814
1819
  this.declineWithoutDeliveryTool(state);
1815
1820
  // A steer plan with a shell or file-write step can only be followed if the
1816
1821
  // directive shows its computed arguments (the pin guard holds those steps
@@ -2290,14 +2295,24 @@ export class ControlServer {
2290
2295
  // including when it declined to steer, which is a *baseline* sample and is
2291
2296
  // what every saving is measured against (docs/calculatedReplay.md §11).
2292
2297
  this.reportExecution(session, run, durationMs, cost, cutShort);
2298
+ // The plan's calls kept for a redo went into the repeat when it opened
2299
+ // (AC2); nothing reads them after the turn, and they can run to megabytes
2300
+ // on a sealed run the session keeps as its previous one.
2301
+ for (const s of run.replays.length > 0 ? run.replays : run.replay ? [run.replay] : []) {
2302
+ s.turnCalls = undefined;
2303
+ s.turnCallChars = 0;
2304
+ }
2293
2305
  // A fragment is recorded from the hand-over on (fallbk.md D7), so its cost
2294
2306
  // and duration are the model's own work, not the steps the scenario did.
2295
- // Another host hands over one number for the whole run, and it is the
2296
- // fragment's: the plan's own steps made no model call, so all of it was
2297
- // spent after the hand-over.
2307
+ // So is a repeat opened after the plan ran to its end (newestRun.md N7). Another host
2308
+ // hands over one number for the whole run, and it is the fragment's: the
2309
+ // plan's own steps made no model call, so all of it was spent after the
2310
+ // hand-over.
2298
2311
  const fragment = run.fragment?.state === "open" ? run.fragment : undefined;
2299
- const recordedCost = fragment && !other ? this.costSince(session, fragment.usageMark) : cost;
2300
- const recordedMs = fragment?.startedAtMs ? Date.now() - fragment.startedAtMs : durationMs;
2312
+ const repeatAfterPlan = !fragment && run.repeat?.state === "open" && run.repeat.startedAtMs !== undefined ? run.repeat : undefined;
2313
+ const ownWork = fragment ?? repeatAfterPlan;
2314
+ const recordedCost = ownWork && !other ? this.costSince(session, ownWork.usageMark) : cost;
2315
+ const recordedMs = ownWork?.startedAtMs ? Date.now() - ownWork.startedAtMs : durationMs;
2301
2316
  if (run.recording) {
2302
2317
  const transcriptPath = session.transcriptPath;
2303
2318
  // The answer is the host's when it handed one over, and the
@@ -2444,6 +2459,14 @@ export class ControlServer {
2444
2459
  // Another host hands over one number for the whole run. It cannot be
2445
2460
  // split between two plans, so with more than one neither is measured.
2446
2461
  const unsplit = otherHost(session) && states.length > 1;
2462
+ // The plan ran to its end and the agent redid the task in a repeat the
2463
+ // service took (newestRun.md N7): the prompt plan's report says so, with
2464
+ // the repeat's id and the agent's own calls in it (autocorrect.md AC1).
2465
+ const r = run.repeat;
2466
+ const redoneBy = r?.state === "open" && r.reason === "redone" && r.openedRunId
2467
+ ? { runId: r.openedRunId, ownCalls: r.ownCalls ?? 0 }
2468
+ : undefined;
2469
+ const prompt = states.find((s) => s.armedBy === "prompt");
2447
2470
  // One report per plan armed this turn (fallbk.md §Runner 4). Each is billed
2448
2471
  // for its own window of the transcript — from when it armed to when the next
2449
2472
  // one did — so the turn's tokens are split between them, never counted twice.
@@ -2481,6 +2504,7 @@ export class ControlServer {
2481
2504
  siblings: states,
2482
2505
  runId: run.runId,
2483
2506
  cutShort: cutShort || undefined,
2507
+ redoneBy: state === prompt && state.ranToEnd === true ? redoneBy : undefined,
2484
2508
  });
2485
2509
  if (!report)
2486
2510
  return;
@@ -2891,6 +2915,7 @@ export class ControlServer {
2891
2915
  if (!steering.retired) {
2892
2916
  steering.pinned.clear();
2893
2917
  steering.retired = true;
2918
+ steering.skipNote = undefined;
2894
2919
  logLine("replay.pin_overran", {
2895
2920
  scenario: steering.scenarioId ?? undefined,
2896
2921
  tool: toolName,
@@ -2912,9 +2937,25 @@ export class ControlServer {
2912
2937
  if (verdict.ok) {
2913
2938
  pinned = action.input;
2914
2939
  this.declinedPins.delete(steering);
2940
+ this.replay.pinTaken(steering);
2915
2941
  }
2916
2942
  else {
2917
2943
  steering.pinned.delete(toolUseId);
2944
+ // Not the next step — and perhaps the step a test just left out,
2945
+ // which shares its tool, made because the note saying so had not
2946
+ // reached the model yet (AC5). The note goes in its place: a step
2947
+ // left out never runs because the note was late. The plan stays
2948
+ // armed for the step that is next.
2949
+ const skipped = this.replay.skipNoteFor(steering, toolName);
2950
+ if (skipped) {
2951
+ logLine("replay.pin_declined", {
2952
+ scenario: steering.scenarioId ?? undefined,
2953
+ tool: toolName,
2954
+ step: action.stepIndex,
2955
+ why: "a call to the step a test left out — the note saying so is given in its place",
2956
+ });
2957
+ return this.preToolAnswer(this.replay.deliverInstead(skipped, toolName));
2958
+ }
2918
2959
  this.declinedPins.add(steering);
2919
2960
  logLine("replay.pin_declined", {
2920
2961
  scenario: steering.scenarioId ?? undefined,
@@ -2935,6 +2976,13 @@ export class ControlServer {
2935
2976
  // a plan nobody would ever hear of.
2936
2977
  if (!overran && !pinned && (!run.replay?.plan || run.replay.retired)) {
2937
2978
  // ── falling back to the model (fallbk.md) ──────────────────────────────
2979
+ // 0. A call to a step a test left out (AC5), when the note saying so has
2980
+ // not reached the model yet — the plan may have finished on it. The
2981
+ // note goes in its place: a step left out must never run because the
2982
+ // note was late.
2983
+ const skipped = planning ? undefined : this.replay.skipNoteFor(run.replay, toolName);
2984
+ if (skipped)
2985
+ return this.preToolAnswer(this.replay.deliverInstead(skipped, toolName));
2938
2986
  // 1. A hand-over noticed where no hook answer could carry it — a proxy
2939
2987
  // report threaded after its PostToolUse had already answered — reaches
2940
2988
  // the model in place of this call. Never silence (D4).
@@ -3068,6 +3116,7 @@ export class ControlServer {
3068
3116
  ? (run.replay?.scenarioId ?? undefined)
3069
3117
  : undefined;
3070
3118
  if (run.recording) {
3119
+ this.noteOwnCall(run, pinnedBy);
3071
3120
  this.queue.push(() => {
3072
3121
  this.recorder.recordToolSelected(run.runId, pair.selected, {
3073
3122
  toolName,
@@ -3299,6 +3348,7 @@ export class ControlServer {
3299
3348
  toolName,
3300
3349
  });
3301
3350
  if (run.recording) {
3351
+ this.noteOwnCall(run);
3302
3352
  const toolInput = serializeCapped(redact(modelArgs));
3303
3353
  this.queue.push(() => {
3304
3354
  this.recorder.recordToolSelected(run.runId, pair.selected, {
@@ -3357,6 +3407,7 @@ export class ControlServer {
3357
3407
  state.retired = true;
3358
3408
  state.retiredAt = Date.now();
3359
3409
  state.pinned.clear();
3410
+ state.skipNote = undefined;
3360
3411
  this.directCalls = this.directCalls.filter((c) => c.state !== state);
3361
3412
  logLine("replay.vetoed", {
3362
3413
  run: run.runId,
@@ -3387,6 +3438,8 @@ export class ControlServer {
3387
3438
  state.retired = true;
3388
3439
  state.retiredAt = Date.now();
3389
3440
  state.pinned.clear();
3441
+ // A note naming a step to call next is void with the plan (AC5 review).
3442
+ state.skipNote = undefined;
3390
3443
  run.handback = HANDBACK_NOTE;
3391
3444
  logLine("replay.pin_abandoned", {
3392
3445
  run: run.runId,
@@ -4032,10 +4085,20 @@ export class ControlServer {
4032
4085
  }
4033
4086
  /**
4034
4087
  * Whether this matched turn's task is now the model's own, so its work is
4035
- * the repeat the service offered to record (the service's newestRun.md N1):
4036
- * offered, nothing recording, no fragment, and no plan of the prompt match
4037
- * that ran a step or still may. A plan that handed over is the fragment's to
4038
- * record (fallbk.md D7).
4088
+ * the repeat the service offered to record: offered, nothing recording, no
4089
+ * fragment, and the prompt match's plan done with — retired, not running,
4090
+ * not diverging — and either:
4091
+ *
4092
+ * - it ran no step: the model does the whole task (newestRun.md N1); or
4093
+ * - it ran to its end, and the model, told to answer from its results,
4094
+ * called tools of its own instead (N7). Before 0.3.15 that work was
4095
+ * lost: a model that did not take the plan's result as the answer and
4096
+ * did the task again left nothing to fix the plan from.
4097
+ *
4098
+ * A plan that handed over is the fragment's to record (fallbk.md D7). One
4099
+ * that ran part of the task and stopped without a hand-over records
4100
+ * nothing, as before: the call where it stopped ran while it was still
4101
+ * live, so a run of what came after would lack the step that matters.
4039
4102
  */
4040
4103
  repeatDue(run) {
4041
4104
  const r = run.repeat;
@@ -4044,23 +4107,34 @@ export class ControlServer {
4044
4107
  const prompt = run.replays.find((s) => s.armedBy === "prompt");
4045
4108
  if (!prompt?.plan)
4046
4109
  return true;
4047
- return prompt.retired && prompt.stepsPinned === 0 && !prompt.handover;
4110
+ if (!prompt.retired || prompt.diverging || this.runningPlans.has(prompt) || prompt.handover)
4111
+ return false;
4112
+ return prompt.stepsPinned === 0 || prompt.ranToEnd === true;
4048
4113
  }
4049
4114
  /**
4050
- * Open the repeat run and record from this call on (newestRun.md N1).
4115
+ * Whether the prompt match's plan ran to its end before the model went on
4116
+ * by itself (newestRun.md N7). False when it ran no step, which is N1.
4117
+ */
4118
+ redoneAfterPlan(run) {
4119
+ const prompt = run.replays.find((s) => s.armedBy === "prompt");
4120
+ return prompt?.plan !== undefined && prompt.ranToEnd === true;
4121
+ }
4122
+ /**
4123
+ * Open the repeat run and record from this call on (newestRun.md N1, N7).
4051
4124
  *
4052
4125
  * The same swap as {@link openFragment}: nothing allocated before it belongs
4053
- * to it. Unlike a fragment it carries the turn's prompt, and its cost and
4054
- * duration are the whole turn's, because no plan did any of the task. Nothing
4055
- * waits for the service's answer: the steps queue behind the create, and a
4056
- * create answered `id: null` — the service records no repeat of it — drops
4057
- * them.
4126
+ * to it. Unlike a fragment it carries the turn's prompt. When no plan did
4127
+ * any of the task, its cost and duration are the whole turn's; after a plan
4128
+ * that ran to its end, they start here, as a fragment's do. Nothing waits
4129
+ * for the service's answer: the steps queue behind the create, and a create
4130
+ * answered `id: null` — the service records no repeat of it — drops them.
4058
4131
  */
4059
4132
  openRepeat(session, run) {
4060
4133
  const r = run.repeat;
4061
4134
  if (!r || r.state !== "offered")
4062
4135
  return;
4063
- const reason = this.repeatReason(run);
4136
+ const redone = this.redoneAfterPlan(run);
4137
+ const reason = redone ? "redone" : this.repeatReason(run);
4064
4138
  const repeatOf = reason ? { runId: r.runId, reason } : { runId: r.runId };
4065
4139
  // The repeat is a run of the same host as the turn it records.
4066
4140
  const framework = frameworkOf(session);
@@ -4075,11 +4149,27 @@ export class ControlServer {
4075
4149
  run.ordering = new StepIndexAllocator();
4076
4150
  run.recording = true;
4077
4151
  r.state = "open";
4152
+ r.openedRunId = id;
4153
+ r.reason = reason;
4154
+ r.ownCalls = 0;
4155
+ // The plan's own calls of this turn go in first, whole and marked, so the
4156
+ // newest run holds the whole turn: the agent's work is built on what the
4157
+ // plan's calls returned (the service's autocorrect.md AC2).
4158
+ const planCalls = redone ? this.recordPlanCalls(run, id) : 0;
4159
+ if (redone) {
4160
+ // The plan's steps are not the model's work: this run is billed and
4161
+ // timed from its own first call on, as a fragment is.
4162
+ r.usageMark = markTranscriptUsage(session.transcriptPath);
4163
+ r.startedAtMs = Date.now();
4164
+ }
4078
4165
  logLine("run.repeat", {
4079
4166
  run: id,
4080
4167
  of: r.runId,
4081
4168
  reason,
4082
- why: "the agent is doing a recorded task itself — recorded as that recording's newest run",
4169
+ planCalls: planCalls || undefined,
4170
+ why: redone
4171
+ ? "the plan ran to its end, and the agent went on with calls of its own instead of answering from its results — recorded as that recording's newest run"
4172
+ : "the agent is doing a recorded task itself — recorded as that recording's newest run",
4083
4173
  });
4084
4174
  if (!isRunCreationAware(this.recorder))
4085
4175
  return;
@@ -4087,10 +4177,15 @@ export class ControlServer {
4087
4177
  .runCreated(id)
4088
4178
  .catch(() => ({ created: true }))
4089
4179
  .then((created) => {
4090
- if (created.created || run.runId !== id)
4180
+ if (created.created)
4091
4181
  return;
4092
- run.recording = false;
4182
+ // Refused: never named as the turn's redo (the service's
4183
+ // autocorrect.md AC1) — also when the turn has moved on to another
4184
+ // run since, which used to leave the repeat `open` and named.
4093
4185
  r.state = "refused";
4186
+ if (run.runId !== id)
4187
+ return;
4188
+ run.recording = false;
4094
4189
  logLine("run.repeat_refused", {
4095
4190
  run: id,
4096
4191
  of: r.runId,
@@ -4098,6 +4193,54 @@ export class ControlServer {
4098
4193
  });
4099
4194
  });
4100
4195
  }
4196
+ /**
4197
+ * Record the prompt plan's own calls of this turn into the `redone` repeat
4198
+ * `id`, in the order they went out, before the agent's first call of its
4199
+ * own (the service's autocorrect.md AC2). Each is a pair, both rows marked
4200
+ * `byPlan` with the plan, its step and its chain revision, so the service
4201
+ * stores them without a vector and counts none of them as the agent's.
4202
+ * Payloads are whole — redacted and capped as any recorded call's — where
4203
+ * the report has only 3,800 bytes of each. Returns how many it recorded.
4204
+ */
4205
+ recordPlanCalls(run, id) {
4206
+ const prompt = run.replays.find((s) => s.armedBy === "prompt");
4207
+ const calls = this.replay.turnCallsOf(prompt);
4208
+ for (const call of calls) {
4209
+ const pair = run.ordering.allocatePair();
4210
+ const byPlan = {
4211
+ scenarioId: call.scenarioId,
4212
+ stepIndex: call.stepIndex,
4213
+ // Left out, never null, when the plan was served without one: the
4214
+ // service's row schema takes a number or nothing.
4215
+ ...(call.chainRevision !== null ? { chainRevision: call.chainRevision } : {}),
4216
+ };
4217
+ this.queue.push(() => {
4218
+ this.recorder.recordToolSelected(id, pair.selected, {
4219
+ toolName: call.toolName,
4220
+ toolInput: call.input,
4221
+ metadata: { byPlan },
4222
+ });
4223
+ this.recorder.recordToolResponse(id, pair.response, {
4224
+ toolName: call.toolName,
4225
+ toolOutput: call.error === undefined ? call.output : undefined,
4226
+ toolError: call.error,
4227
+ metadata: { byPlan },
4228
+ });
4229
+ });
4230
+ }
4231
+ return calls.length;
4232
+ }
4233
+ /**
4234
+ * One tool call of the model's own was recorded into `run`: counted for
4235
+ * the plan's report when it is the `redone` repeat (AC1). A step a plan
4236
+ * pinned is the plan's doing, not the model's, and is not counted.
4237
+ */
4238
+ noteOwnCall(run, pinnedBy) {
4239
+ const r = run.repeat;
4240
+ if (pinnedBy || !r || r.state !== "open" || run.runId !== r.openedRunId)
4241
+ return;
4242
+ r.ownCalls = (r.ownCalls ?? 0) + 1;
4243
+ }
4101
4244
  /**
4102
4245
  * Why the matched turn's plan did not run, as the service shows it: what
4103
4246
  * the service withheld it for, else the code of the gate that declined it
@@ -4143,6 +4286,11 @@ export class ControlServer {
4143
4286
  const note = this.replay.takeNote(run.replay);
4144
4287
  if (note)
4145
4288
  context.push(note);
4289
+ // The steps a test just left out (AC5): said before the model makes its
4290
+ // next call, which must not be one of them.
4291
+ const skipped = this.replay.takeSkipNote(run.replay);
4292
+ if (skipped)
4293
+ context.push(skipped);
4146
4294
  if (run.handback) {
4147
4295
  context.push(run.handback);
4148
4296
  run.handback = undefined;
@@ -5118,6 +5266,7 @@ export class ControlServer {
5118
5266
  record(run, pair, d) {
5119
5267
  if (!run.recording)
5120
5268
  return;
5269
+ this.noteOwnCall(run, d.pinnedBy);
5121
5270
  const toolInput = serializeCapped(redact(d.args));
5122
5271
  const toolOutput = d.isError ? undefined : serializeCapped(redact(d.result));
5123
5272
  const toolError = d.isError
@@ -13,9 +13,12 @@ export interface StartRunOptions {
13
13
  };
14
14
  /**
15
15
  * Marks a *repeat* (the service's newestRun.md): a matched turn whose plan
16
- * did not run, so the model is doing the recording's task itself. Sent only
17
- * when the match offered it ({@link RunMatch.repeat}). `reason` is why the
18
- * plan did not run. The service never prompt-matches a repeat.
16
+ * did not run, so the model is doing the recording's task itself — or whose
17
+ * plan ran to its end, after which the model went on with calls of its own
18
+ * instead of answering from its results (N7). Sent only when the match
19
+ * offered it ({@link RunMatch.repeat}). `reason` is why the plan did not
20
+ * run, or `redone` after a plan that ran to its end. The service never
21
+ * prompt-matches a repeat.
19
22
  */
20
23
  repeatOf?: {
21
24
  runId: string;
@@ -58,6 +61,27 @@ export interface FlushOptions {
58
61
  */
59
62
  timeoutMs?: number;
60
63
  }
64
+ /**
65
+ * A plan's own call recorded into the turn's newest run (the service's
66
+ * autocorrect.md AC2): which plan made it, which of its steps, and the chain
67
+ * revision it was served at — left out when the service served none, since
68
+ * its schema takes a number or nothing, never null.
69
+ */
70
+ export interface PlanCallMark {
71
+ scenarioId: string;
72
+ stepIndex: number;
73
+ chainRevision?: number;
74
+ }
75
+ /**
76
+ * A recorded step's metadata. `pinnedBy`: a plan executed this step live in
77
+ * the turn (R-INTENT-7). `byPlan`: the plan made this call before the agent
78
+ * redid the task, and it is recorded first into the agent's repeat so the
79
+ * repeat holds the whole turn (AC2). The service stores neither with a vector.
80
+ */
81
+ export type StepMetadata = {
82
+ pinnedBy?: string;
83
+ byPlan?: PlanCallMark;
84
+ };
61
85
  export interface Recorder {
62
86
  /**
63
87
  * Create the run; returns the client-generated runId synchronously.
@@ -83,17 +107,18 @@ export interface Recorder {
83
107
  };
84
108
  /**
85
109
  * Set when a plan executed this step, to the plan's scenario id
86
- * (R-INTENT-7). The service stores such a step without a vector, so the
110
+ * (R-INTENT-7), or made it before the agent redid the task (`byPlan`,
111
+ * AC2). The service stores such a step without a vector, so the
87
112
  * system's own replays never count as recurrences of the work.
88
113
  */
89
- metadata?: {
90
- pinnedBy: string;
91
- };
114
+ metadata?: StepMetadata;
92
115
  }): string;
93
116
  recordToolResponse(runId: string, stepIndex: number, d: {
94
117
  toolName: string;
95
118
  toolOutput?: string;
96
119
  toolError?: string;
120
+ /** `byPlan` on the answer of a plan's own call (AC2); omitted otherwise. */
121
+ metadata?: StepMetadata;
97
122
  }): string;
98
123
  recordFinalAnswer(runId: string, stepIndex: number, d: {
99
124
  answer: string;
@@ -414,6 +439,19 @@ export interface ExecutionReport {
414
439
  * person reading the audit log, this is for the console.
415
440
  */
416
441
  declined?: string;
442
+ /**
443
+ * The plan ran to its end and the agent did not take its result: it went on
444
+ * with calls of its own, recorded as the recording's newest run with the
445
+ * reason `redone` (the service's newestRun.md N7, autocorrect.md AC1).
446
+ * `runId` is that repeat's own id; `ownCalls` the agent's own tool calls
447
+ * recorded into it — not the plan's calls recorded first (AC2), and not a
448
+ * step a plan pinned. Only on the report of the prompt's plan, and only when
449
+ * the service took the repeat; an older service drops the field.
450
+ */
451
+ redoneBy?: {
452
+ runId: string;
453
+ ownCalls: number;
454
+ };
417
455
  }
418
456
  /** Optional capability: reporting needs a service, and a NullRecorder has none. */
419
457
  export interface ScenarioReporter {
@@ -50,7 +50,7 @@
50
50
  */
51
51
  import { type AuthSession } from "../auth/client.js";
52
52
  import { type ServiceHealth } from "../util/service-health.js";
53
- import type { ExecutionReport, FlushOptions, IntentMatchAnswer, IntentMatchOptions, IntentMatchRequest, Recorder, RecorderHealth, RecorderHealthReporter, RunCreation, RunMatch, RunMetrics, StartRunOptions } from "./recorder.js";
53
+ import type { ExecutionReport, FlushOptions, IntentMatchAnswer, IntentMatchOptions, IntentMatchRequest, Recorder, RecorderHealth, RecorderHealthReporter, RunCreation, RunMatch, RunMetrics, StartRunOptions, StepMetadata } from "./recorder.js";
54
54
  /**
55
55
  * Caps for the free text this recorder sends, in UTF-8 bytes. Tool payloads
56
56
  * arrive capped at 64 KiB by their producers; text never passed through one,
@@ -225,14 +225,13 @@ export declare class RemoteRecorder implements Recorder, RecorderHealthReporter
225
225
  text: string;
226
226
  source: "thinking";
227
227
  };
228
- metadata?: {
229
- pinnedBy: string;
230
- };
228
+ metadata?: StepMetadata;
231
229
  }): string;
232
230
  recordToolResponse(runId: string, stepIndex: number, data: {
233
231
  toolName: string;
234
232
  toolOutput?: string;
235
233
  toolError?: string;
234
+ metadata?: StepMetadata;
236
235
  }): string;
237
236
  recordFinalAnswer(runId: string, stepIndex: number, data: {
238
237
  answer: string;
@@ -674,6 +674,10 @@ export class RemoteRecorder {
674
674
  toolName: data.toolName,
675
675
  toolOutput: out(toolOutput),
676
676
  toolError: out(toolError),
677
+ // Omitted, not nulled, when absent: an ordinary answer is the body it
678
+ // has always been. Only a plan's own call recorded into a redo carries
679
+ // one (`byPlan`, the service's autocorrect.md AC2).
680
+ metadata: data.metadata,
677
681
  }));
678
682
  return id;
679
683
  }
@@ -744,6 +748,12 @@ export class RemoteRecorder {
744
748
  kind: "report",
745
749
  bytes: sizeOf(prompt, error, ...(steps ?? []).flatMap((s) => [s.toolInput, s.toolOutput])) + 1024,
746
750
  send: async () => {
751
+ // A redo whose run the service refused to create is no run of its:
752
+ // never named (the service's autocorrect.md AC1), even when the
753
+ // refusal came after the report was built.
754
+ const redoneBy = report.redoneBy && this.runs.get(report.redoneBy.runId)?.settled?.created.created === false
755
+ ? undefined
756
+ : report.redoneBy;
747
757
  const body = await this.request("POST", `/scenarios/${report.scenarioId}/executions`,
748
758
  // Per-step verdicts and the headline failure (errorshandling.md) ride
749
759
  // beside the costs. All optional on the service side, so an older
@@ -776,6 +786,9 @@ export class RemoteRecorder {
776
786
  // Why nothing ran, when nothing ran. Additive: an older service drops
777
787
  // the field and books the cost exactly as it always did.
778
788
  declined: report.declined,
789
+ // The plan ran to its end and the agent redid the task (the
790
+ // service's autocorrect.md AC1). Additive, like `declined`.
791
+ redoneBy,
779
792
  }, steps), { retryTransient: Boolean(report.ticket) });
780
793
  const r = (body ?? {});
781
794
  const failed = report.steps?.filter((s) => s.status === "failed").length ?? 0;
@@ -795,6 +808,9 @@ export class RemoteRecorder {
795
808
  baselineEligible: report.baselineEligible === false ? false : undefined,
796
809
  sharedWith: report.sharedWith?.length,
797
810
  declined: report.declined,
811
+ // `bir investigate` prints it beside the outcome: "redone by run_…".
812
+ redoneBy: report.redoneBy?.runId,
813
+ ownCalls: report.redoneBy?.ownCalls,
798
814
  });
799
815
  },
800
816
  });
@@ -90,11 +90,7 @@ export declare class SwitchableRecorder implements Recorder {
90
90
  currentAccessToken(): string | undefined;
91
91
  startRun(input: string, metadata?: Record<string, unknown>, options?: StartRunOptions): string;
92
92
  recordToolSelected(runId: string, stepIndex: number, d: Parameters<Recorder["recordToolSelected"]>[2]): string;
93
- recordToolResponse(runId: string, stepIndex: number, d: {
94
- toolName: string;
95
- toolOutput?: string;
96
- toolError?: string;
97
- }): string;
93
+ recordToolResponse(runId: string, stepIndex: number, d: Parameters<Recorder["recordToolResponse"]>[2]): string;
98
94
  recordFinalAnswer(runId: string, stepIndex: number, d: {
99
95
  answer: string;
100
96
  }): string;
@@ -34,7 +34,17 @@ export interface BundleEntry {
34
34
  response: string;
35
35
  /** True when `response` is a recorded output, not a fresh run. */
36
36
  recorded?: boolean;
37
+ /**
38
+ * Set on a step its test left out (the service's autocorrect.md AC5): no
39
+ * call was made, and the bundle says so in its place, so the model can tell
40
+ * a whole answer from a part. `text` is the test in words, when it has some.
41
+ */
42
+ notNeeded?: {
43
+ text: string | null;
44
+ };
37
45
  }
46
+ /** The test's words, one line and short, for a bundle or a note. */
47
+ export declare function testWords(text: string | null | undefined): string | null;
38
48
  /** Cap a computed input for inclusion in a bundle. */
39
49
  export declare function bundleInput(value: unknown): string;
40
50
  export interface AssembleOptions {
@@ -30,6 +30,13 @@ export const BASH_CHANNEL_MAX = 25_000;
30
30
  const MAX_INPUT_CHARS = 2_000;
31
31
  /** Fixed structural overhead reserved per step (labels, separators). */
32
32
  const PER_STEP_OVERHEAD = 80;
33
+ /** The test's words, one line and short, for a bundle or a note. */
34
+ export function testWords(text) {
35
+ const line = (text ?? "").trim().split(/\r?\n/, 1)[0]?.trim() ?? "";
36
+ if (!line)
37
+ return null;
38
+ return line.length > 200 ? `${line.slice(0, 200)}…` : line;
39
+ }
33
40
  /** Cap a computed input for inclusion in a bundle. */
34
41
  export function bundleInput(value) {
35
42
  let text;
@@ -90,7 +97,9 @@ const SUBTASK_FOOTER = "\n\nThe calls above are done. Continue your task, callin
90
97
  * Compose the bundle. `maxChars` is a hard ceiling on the returned string.
91
98
  */
92
99
  export function assembleBundle(entries, maxChars, opts = {}) {
93
- const n = entries.length;
100
+ // A step left out has no response to share the budget with: only the steps
101
+ // that answered split it.
102
+ const n = entries.filter((e) => !e.notNeeded).length;
94
103
  const anyRecorded = entries.some((e) => e.recorded);
95
104
  // A hand-over outranks a sub-task: it says "continue" too, and it also has to
96
105
  // say that the plan stopped short. A sub-task outranks the live and mixed
@@ -111,10 +120,22 @@ export function assembleBundle(entries, maxChars, opts = {}) {
111
120
  : LIVE_FOOTER;
112
121
  const fixedCost = header.length +
113
122
  footer.length +
114
- entries.reduce((sum, e) => sum + PER_STEP_OVERHEAD + e.toolName.length + e.input.length, 0);
123
+ entries.reduce((sum, e) => sum +
124
+ PER_STEP_OVERHEAD +
125
+ e.toolName.length +
126
+ e.input.length +
127
+ // A left-out step's line, which has no response to take from.
128
+ (e.notNeeded ? PER_STEP_OVERHEAD + (testWords(e.notNeeded.text)?.length ?? 0) : 0), 0);
115
129
  const responseBudget = Math.max(0, maxChars - fixedCost);
116
130
  const perResponse = n > 0 ? Math.floor(responseBudget / n) : responseBudget;
117
131
  const sections = entries.map((e, i) => {
132
+ if (e.notNeeded) {
133
+ // Never "skipped" or "failed": the step was left out on purpose, because
134
+ // this request does not need it, and nothing is missing for its absence.
135
+ const why = testWords(e.notNeeded.text);
136
+ return (`\n\n--- Step ${i + 1}: ${e.toolName} (not needed) ---\n` +
137
+ `Not run: this request does not need it${why ? ` — it is only for when ${why}` : ""}.`);
138
+ }
118
139
  const response = truncate(e.response, perResponse);
119
140
  return (`\n\n--- Step ${i + 1}: ${e.toolName} ---\n` +
120
141
  `Call: ${e.input}\n` +