@basein/runner 0.3.14 → 0.3.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/bin/bir-scenario.js +8 -5
- package/dist/bin/bir.js +2 -1
- package/dist/bin/investigate.d.ts +71 -0
- package/dist/bin/investigate.js +145 -13
- package/dist/bin/scenario-edit.d.ts +45 -1
- package/dist/bin/scenario-edit.js +134 -14
- package/dist/control/server.d.ts +41 -10
- package/dist/control/server.js +169 -20
- package/dist/record/recorder.d.ts +45 -7
- package/dist/record/remote-recorder.d.ts +3 -4
- package/dist/record/remote-recorder.js +16 -0
- package/dist/record/switchable-recorder.d.ts +1 -5
- package/dist/replay/bundle.d.ts +10 -0
- package/dist/replay/bundle.js +23 -2
- package/dist/replay/controller.d.ts +151 -0
- package/dist/replay/controller.js +510 -58
- package/dist/replay/flatten.d.ts +39 -3
- package/dist/replay/flatten.js +42 -2
- package/dist/replay/handover.d.ts +39 -1
- package/dist/replay/handover.js +81 -2
- package/dist/replay/logic.d.ts +8 -0
- package/dist/replay/logic.js +19 -0
- package/dist/replay/plan.d.ts +165 -4
- package/dist/replay/plan.js +488 -25
- package/dist/replay/source-run.d.ts +14 -1
- package/dist/replay/source-run.js +25 -2
- package/dist/replay/targets.js +8 -1
- package/dist/replay/types.d.ts +52 -3
- package/dist/replay/types.js +14 -0
- package/dist/util/service-headers.d.ts +7 -1
- package/dist/util/service-headers.js +7 -1
- package/docs/BaseInstRunner.md +3 -1
- package/docs/calculatedReplay.md +117 -6
- package/docs/calculatedReplayGuide.md +41 -3
- package/package.json +1 -1
package/dist/control/server.js
CHANGED
|
@@ -1811,6 +1811,11 @@ export class ControlServer {
|
|
|
1811
1811
|
// the jail it may not leave (replay/builtins.ts). The session's own cwd
|
|
1812
1812
|
// from the hook payloads, never the daemon's.
|
|
1813
1813
|
state.sessionCwd = session.cwd ?? this.opts.cwd;
|
|
1814
|
+
// The service offered to record this turn as the recording's newest run:
|
|
1815
|
+
// should the agent redo the task after the plan, the plan's own calls go
|
|
1816
|
+
// into that run first, so they are kept whole as they run (AC2).
|
|
1817
|
+
if (match.repeat?.runId)
|
|
1818
|
+
state.keepCalls = true;
|
|
1814
1819
|
this.declineWithoutDeliveryTool(state);
|
|
1815
1820
|
// A steer plan with a shell or file-write step can only be followed if the
|
|
1816
1821
|
// directive shows its computed arguments (the pin guard holds those steps
|
|
@@ -2290,14 +2295,24 @@ export class ControlServer {
|
|
|
2290
2295
|
// including when it declined to steer, which is a *baseline* sample and is
|
|
2291
2296
|
// what every saving is measured against (docs/calculatedReplay.md §11).
|
|
2292
2297
|
this.reportExecution(session, run, durationMs, cost, cutShort);
|
|
2298
|
+
// The plan's calls kept for a redo went into the repeat when it opened
|
|
2299
|
+
// (AC2); nothing reads them after the turn, and they can run to megabytes
|
|
2300
|
+
// on a sealed run the session keeps as its previous one.
|
|
2301
|
+
for (const s of run.replays.length > 0 ? run.replays : run.replay ? [run.replay] : []) {
|
|
2302
|
+
s.turnCalls = undefined;
|
|
2303
|
+
s.turnCallChars = 0;
|
|
2304
|
+
}
|
|
2293
2305
|
// A fragment is recorded from the hand-over on (fallbk.md D7), so its cost
|
|
2294
2306
|
// and duration are the model's own work, not the steps the scenario did.
|
|
2295
|
-
//
|
|
2296
|
-
//
|
|
2297
|
-
// spent after the
|
|
2307
|
+
// So is a repeat opened after the plan ran to its end (newestRun.md N7). Another host
|
|
2308
|
+
// hands over one number for the whole run, and it is the fragment's: the
|
|
2309
|
+
// plan's own steps made no model call, so all of it was spent after the
|
|
2310
|
+
// hand-over.
|
|
2298
2311
|
const fragment = run.fragment?.state === "open" ? run.fragment : undefined;
|
|
2299
|
-
const
|
|
2300
|
-
const
|
|
2312
|
+
const repeatAfterPlan = !fragment && run.repeat?.state === "open" && run.repeat.startedAtMs !== undefined ? run.repeat : undefined;
|
|
2313
|
+
const ownWork = fragment ?? repeatAfterPlan;
|
|
2314
|
+
const recordedCost = ownWork && !other ? this.costSince(session, ownWork.usageMark) : cost;
|
|
2315
|
+
const recordedMs = ownWork?.startedAtMs ? Date.now() - ownWork.startedAtMs : durationMs;
|
|
2301
2316
|
if (run.recording) {
|
|
2302
2317
|
const transcriptPath = session.transcriptPath;
|
|
2303
2318
|
// The answer is the host's when it handed one over, and the
|
|
@@ -2444,6 +2459,14 @@ export class ControlServer {
|
|
|
2444
2459
|
// Another host hands over one number for the whole run. It cannot be
|
|
2445
2460
|
// split between two plans, so with more than one neither is measured.
|
|
2446
2461
|
const unsplit = otherHost(session) && states.length > 1;
|
|
2462
|
+
// The plan ran to its end and the agent redid the task in a repeat the
|
|
2463
|
+
// service took (newestRun.md N7): the prompt plan's report says so, with
|
|
2464
|
+
// the repeat's id and the agent's own calls in it (autocorrect.md AC1).
|
|
2465
|
+
const r = run.repeat;
|
|
2466
|
+
const redoneBy = r?.state === "open" && r.reason === "redone" && r.openedRunId
|
|
2467
|
+
? { runId: r.openedRunId, ownCalls: r.ownCalls ?? 0 }
|
|
2468
|
+
: undefined;
|
|
2469
|
+
const prompt = states.find((s) => s.armedBy === "prompt");
|
|
2447
2470
|
// One report per plan armed this turn (fallbk.md §Runner 4). Each is billed
|
|
2448
2471
|
// for its own window of the transcript — from when it armed to when the next
|
|
2449
2472
|
// one did — so the turn's tokens are split between them, never counted twice.
|
|
@@ -2481,6 +2504,7 @@ export class ControlServer {
|
|
|
2481
2504
|
siblings: states,
|
|
2482
2505
|
runId: run.runId,
|
|
2483
2506
|
cutShort: cutShort || undefined,
|
|
2507
|
+
redoneBy: state === prompt && state.ranToEnd === true ? redoneBy : undefined,
|
|
2484
2508
|
});
|
|
2485
2509
|
if (!report)
|
|
2486
2510
|
return;
|
|
@@ -2891,6 +2915,7 @@ export class ControlServer {
|
|
|
2891
2915
|
if (!steering.retired) {
|
|
2892
2916
|
steering.pinned.clear();
|
|
2893
2917
|
steering.retired = true;
|
|
2918
|
+
steering.skipNote = undefined;
|
|
2894
2919
|
logLine("replay.pin_overran", {
|
|
2895
2920
|
scenario: steering.scenarioId ?? undefined,
|
|
2896
2921
|
tool: toolName,
|
|
@@ -2912,9 +2937,25 @@ export class ControlServer {
|
|
|
2912
2937
|
if (verdict.ok) {
|
|
2913
2938
|
pinned = action.input;
|
|
2914
2939
|
this.declinedPins.delete(steering);
|
|
2940
|
+
this.replay.pinTaken(steering);
|
|
2915
2941
|
}
|
|
2916
2942
|
else {
|
|
2917
2943
|
steering.pinned.delete(toolUseId);
|
|
2944
|
+
// Not the next step — and perhaps the step a test just left out,
|
|
2945
|
+
// which shares its tool, made because the note saying so had not
|
|
2946
|
+
// reached the model yet (AC5). The note goes in its place: a step
|
|
2947
|
+
// left out never runs because the note was late. The plan stays
|
|
2948
|
+
// armed for the step that is next.
|
|
2949
|
+
const skipped = this.replay.skipNoteFor(steering, toolName);
|
|
2950
|
+
if (skipped) {
|
|
2951
|
+
logLine("replay.pin_declined", {
|
|
2952
|
+
scenario: steering.scenarioId ?? undefined,
|
|
2953
|
+
tool: toolName,
|
|
2954
|
+
step: action.stepIndex,
|
|
2955
|
+
why: "a call to the step a test left out — the note saying so is given in its place",
|
|
2956
|
+
});
|
|
2957
|
+
return this.preToolAnswer(this.replay.deliverInstead(skipped, toolName));
|
|
2958
|
+
}
|
|
2918
2959
|
this.declinedPins.add(steering);
|
|
2919
2960
|
logLine("replay.pin_declined", {
|
|
2920
2961
|
scenario: steering.scenarioId ?? undefined,
|
|
@@ -2935,6 +2976,13 @@ export class ControlServer {
|
|
|
2935
2976
|
// a plan nobody would ever hear of.
|
|
2936
2977
|
if (!overran && !pinned && (!run.replay?.plan || run.replay.retired)) {
|
|
2937
2978
|
// ── falling back to the model (fallbk.md) ──────────────────────────────
|
|
2979
|
+
// 0. A call to a step a test left out (AC5), when the note saying so has
|
|
2980
|
+
// not reached the model yet — the plan may have finished on it. The
|
|
2981
|
+
// note goes in its place: a step left out must never run because the
|
|
2982
|
+
// note was late.
|
|
2983
|
+
const skipped = planning ? undefined : this.replay.skipNoteFor(run.replay, toolName);
|
|
2984
|
+
if (skipped)
|
|
2985
|
+
return this.preToolAnswer(this.replay.deliverInstead(skipped, toolName));
|
|
2938
2986
|
// 1. A hand-over noticed where no hook answer could carry it — a proxy
|
|
2939
2987
|
// report threaded after its PostToolUse had already answered — reaches
|
|
2940
2988
|
// the model in place of this call. Never silence (D4).
|
|
@@ -3068,6 +3116,7 @@ export class ControlServer {
|
|
|
3068
3116
|
? (run.replay?.scenarioId ?? undefined)
|
|
3069
3117
|
: undefined;
|
|
3070
3118
|
if (run.recording) {
|
|
3119
|
+
this.noteOwnCall(run, pinnedBy);
|
|
3071
3120
|
this.queue.push(() => {
|
|
3072
3121
|
this.recorder.recordToolSelected(run.runId, pair.selected, {
|
|
3073
3122
|
toolName,
|
|
@@ -3299,6 +3348,7 @@ export class ControlServer {
|
|
|
3299
3348
|
toolName,
|
|
3300
3349
|
});
|
|
3301
3350
|
if (run.recording) {
|
|
3351
|
+
this.noteOwnCall(run);
|
|
3302
3352
|
const toolInput = serializeCapped(redact(modelArgs));
|
|
3303
3353
|
this.queue.push(() => {
|
|
3304
3354
|
this.recorder.recordToolSelected(run.runId, pair.selected, {
|
|
@@ -3357,6 +3407,7 @@ export class ControlServer {
|
|
|
3357
3407
|
state.retired = true;
|
|
3358
3408
|
state.retiredAt = Date.now();
|
|
3359
3409
|
state.pinned.clear();
|
|
3410
|
+
state.skipNote = undefined;
|
|
3360
3411
|
this.directCalls = this.directCalls.filter((c) => c.state !== state);
|
|
3361
3412
|
logLine("replay.vetoed", {
|
|
3362
3413
|
run: run.runId,
|
|
@@ -3387,6 +3438,8 @@ export class ControlServer {
|
|
|
3387
3438
|
state.retired = true;
|
|
3388
3439
|
state.retiredAt = Date.now();
|
|
3389
3440
|
state.pinned.clear();
|
|
3441
|
+
// A note naming a step to call next is void with the plan (AC5 review).
|
|
3442
|
+
state.skipNote = undefined;
|
|
3390
3443
|
run.handback = HANDBACK_NOTE;
|
|
3391
3444
|
logLine("replay.pin_abandoned", {
|
|
3392
3445
|
run: run.runId,
|
|
@@ -4032,10 +4085,20 @@ export class ControlServer {
|
|
|
4032
4085
|
}
|
|
4033
4086
|
/**
|
|
4034
4087
|
* Whether this matched turn's task is now the model's own, so its work is
|
|
4035
|
-
* the repeat the service offered to record
|
|
4036
|
-
*
|
|
4037
|
-
*
|
|
4038
|
-
*
|
|
4088
|
+
* the repeat the service offered to record: offered, nothing recording, no
|
|
4089
|
+
* fragment, and the prompt match's plan done with — retired, not running,
|
|
4090
|
+
* not diverging — and either:
|
|
4091
|
+
*
|
|
4092
|
+
* - it ran no step: the model does the whole task (newestRun.md N1); or
|
|
4093
|
+
* - it ran to its end, and the model, told to answer from its results,
|
|
4094
|
+
* called tools of its own instead (N7). Before 0.3.15 that work was
|
|
4095
|
+
* lost: a model that did not take the plan's result as the answer and
|
|
4096
|
+
* did the task again left nothing to fix the plan from.
|
|
4097
|
+
*
|
|
4098
|
+
* A plan that handed over is the fragment's to record (fallbk.md D7). One
|
|
4099
|
+
* that ran part of the task and stopped without a hand-over records
|
|
4100
|
+
* nothing, as before: the call where it stopped ran while it was still
|
|
4101
|
+
* live, so a run of what came after would lack the step that matters.
|
|
4039
4102
|
*/
|
|
4040
4103
|
repeatDue(run) {
|
|
4041
4104
|
const r = run.repeat;
|
|
@@ -4044,23 +4107,34 @@ export class ControlServer {
|
|
|
4044
4107
|
const prompt = run.replays.find((s) => s.armedBy === "prompt");
|
|
4045
4108
|
if (!prompt?.plan)
|
|
4046
4109
|
return true;
|
|
4047
|
-
|
|
4110
|
+
if (!prompt.retired || prompt.diverging || this.runningPlans.has(prompt) || prompt.handover)
|
|
4111
|
+
return false;
|
|
4112
|
+
return prompt.stepsPinned === 0 || prompt.ranToEnd === true;
|
|
4048
4113
|
}
|
|
4049
4114
|
/**
|
|
4050
|
-
*
|
|
4115
|
+
* Whether the prompt match's plan ran to its end before the model went on
|
|
4116
|
+
* by itself (newestRun.md N7). False when it ran no step, which is N1.
|
|
4117
|
+
*/
|
|
4118
|
+
redoneAfterPlan(run) {
|
|
4119
|
+
const prompt = run.replays.find((s) => s.armedBy === "prompt");
|
|
4120
|
+
return prompt?.plan !== undefined && prompt.ranToEnd === true;
|
|
4121
|
+
}
|
|
4122
|
+
/**
|
|
4123
|
+
* Open the repeat run and record from this call on (newestRun.md N1, N7).
|
|
4051
4124
|
*
|
|
4052
4125
|
* The same swap as {@link openFragment}: nothing allocated before it belongs
|
|
4053
|
-
* to it. Unlike a fragment it carries the turn's prompt
|
|
4054
|
-
* duration are the whole turn's
|
|
4055
|
-
*
|
|
4056
|
-
*
|
|
4057
|
-
* them.
|
|
4126
|
+
* to it. Unlike a fragment it carries the turn's prompt. When no plan did
|
|
4127
|
+
* any of the task, its cost and duration are the whole turn's; after a plan
|
|
4128
|
+
* that ran to its end, they start here, as a fragment's do. Nothing waits
|
|
4129
|
+
* for the service's answer: the steps queue behind the create, and a create
|
|
4130
|
+
* answered `id: null` — the service records no repeat of it — drops them.
|
|
4058
4131
|
*/
|
|
4059
4132
|
openRepeat(session, run) {
|
|
4060
4133
|
const r = run.repeat;
|
|
4061
4134
|
if (!r || r.state !== "offered")
|
|
4062
4135
|
return;
|
|
4063
|
-
const
|
|
4136
|
+
const redone = this.redoneAfterPlan(run);
|
|
4137
|
+
const reason = redone ? "redone" : this.repeatReason(run);
|
|
4064
4138
|
const repeatOf = reason ? { runId: r.runId, reason } : { runId: r.runId };
|
|
4065
4139
|
// The repeat is a run of the same host as the turn it records.
|
|
4066
4140
|
const framework = frameworkOf(session);
|
|
@@ -4075,11 +4149,27 @@ export class ControlServer {
|
|
|
4075
4149
|
run.ordering = new StepIndexAllocator();
|
|
4076
4150
|
run.recording = true;
|
|
4077
4151
|
r.state = "open";
|
|
4152
|
+
r.openedRunId = id;
|
|
4153
|
+
r.reason = reason;
|
|
4154
|
+
r.ownCalls = 0;
|
|
4155
|
+
// The plan's own calls of this turn go in first, whole and marked, so the
|
|
4156
|
+
// newest run holds the whole turn: the agent's work is built on what the
|
|
4157
|
+
// plan's calls returned (the service's autocorrect.md AC2).
|
|
4158
|
+
const planCalls = redone ? this.recordPlanCalls(run, id) : 0;
|
|
4159
|
+
if (redone) {
|
|
4160
|
+
// The plan's steps are not the model's work: this run is billed and
|
|
4161
|
+
// timed from its own first call on, as a fragment is.
|
|
4162
|
+
r.usageMark = markTranscriptUsage(session.transcriptPath);
|
|
4163
|
+
r.startedAtMs = Date.now();
|
|
4164
|
+
}
|
|
4078
4165
|
logLine("run.repeat", {
|
|
4079
4166
|
run: id,
|
|
4080
4167
|
of: r.runId,
|
|
4081
4168
|
reason,
|
|
4082
|
-
|
|
4169
|
+
planCalls: planCalls || undefined,
|
|
4170
|
+
why: redone
|
|
4171
|
+
? "the plan ran to its end, and the agent went on with calls of its own instead of answering from its results — recorded as that recording's newest run"
|
|
4172
|
+
: "the agent is doing a recorded task itself — recorded as that recording's newest run",
|
|
4083
4173
|
});
|
|
4084
4174
|
if (!isRunCreationAware(this.recorder))
|
|
4085
4175
|
return;
|
|
@@ -4087,10 +4177,15 @@ export class ControlServer {
|
|
|
4087
4177
|
.runCreated(id)
|
|
4088
4178
|
.catch(() => ({ created: true }))
|
|
4089
4179
|
.then((created) => {
|
|
4090
|
-
if (created.created
|
|
4180
|
+
if (created.created)
|
|
4091
4181
|
return;
|
|
4092
|
-
|
|
4182
|
+
// Refused: never named as the turn's redo (the service's
|
|
4183
|
+
// autocorrect.md AC1) — also when the turn has moved on to another
|
|
4184
|
+
// run since, which used to leave the repeat `open` and named.
|
|
4093
4185
|
r.state = "refused";
|
|
4186
|
+
if (run.runId !== id)
|
|
4187
|
+
return;
|
|
4188
|
+
run.recording = false;
|
|
4094
4189
|
logLine("run.repeat_refused", {
|
|
4095
4190
|
run: id,
|
|
4096
4191
|
of: r.runId,
|
|
@@ -4098,6 +4193,54 @@ export class ControlServer {
|
|
|
4098
4193
|
});
|
|
4099
4194
|
});
|
|
4100
4195
|
}
|
|
4196
|
+
/**
|
|
4197
|
+
* Record the prompt plan's own calls of this turn into the `redone` repeat
|
|
4198
|
+
* `id`, in the order they went out, before the agent's first call of its
|
|
4199
|
+
* own (the service's autocorrect.md AC2). Each is a pair, both rows marked
|
|
4200
|
+
* `byPlan` with the plan, its step and its chain revision, so the service
|
|
4201
|
+
* stores them without a vector and counts none of them as the agent's.
|
|
4202
|
+
* Payloads are whole — redacted and capped as any recorded call's — where
|
|
4203
|
+
* the report has only 3,800 bytes of each. Returns how many it recorded.
|
|
4204
|
+
*/
|
|
4205
|
+
recordPlanCalls(run, id) {
|
|
4206
|
+
const prompt = run.replays.find((s) => s.armedBy === "prompt");
|
|
4207
|
+
const calls = this.replay.turnCallsOf(prompt);
|
|
4208
|
+
for (const call of calls) {
|
|
4209
|
+
const pair = run.ordering.allocatePair();
|
|
4210
|
+
const byPlan = {
|
|
4211
|
+
scenarioId: call.scenarioId,
|
|
4212
|
+
stepIndex: call.stepIndex,
|
|
4213
|
+
// Left out, never null, when the plan was served without one: the
|
|
4214
|
+
// service's row schema takes a number or nothing.
|
|
4215
|
+
...(call.chainRevision !== null ? { chainRevision: call.chainRevision } : {}),
|
|
4216
|
+
};
|
|
4217
|
+
this.queue.push(() => {
|
|
4218
|
+
this.recorder.recordToolSelected(id, pair.selected, {
|
|
4219
|
+
toolName: call.toolName,
|
|
4220
|
+
toolInput: call.input,
|
|
4221
|
+
metadata: { byPlan },
|
|
4222
|
+
});
|
|
4223
|
+
this.recorder.recordToolResponse(id, pair.response, {
|
|
4224
|
+
toolName: call.toolName,
|
|
4225
|
+
toolOutput: call.error === undefined ? call.output : undefined,
|
|
4226
|
+
toolError: call.error,
|
|
4227
|
+
metadata: { byPlan },
|
|
4228
|
+
});
|
|
4229
|
+
});
|
|
4230
|
+
}
|
|
4231
|
+
return calls.length;
|
|
4232
|
+
}
|
|
4233
|
+
/**
|
|
4234
|
+
* One tool call of the model's own was recorded into `run`: counted for
|
|
4235
|
+
* the plan's report when it is the `redone` repeat (AC1). A step a plan
|
|
4236
|
+
* pinned is the plan's doing, not the model's, and is not counted.
|
|
4237
|
+
*/
|
|
4238
|
+
noteOwnCall(run, pinnedBy) {
|
|
4239
|
+
const r = run.repeat;
|
|
4240
|
+
if (pinnedBy || !r || r.state !== "open" || run.runId !== r.openedRunId)
|
|
4241
|
+
return;
|
|
4242
|
+
r.ownCalls = (r.ownCalls ?? 0) + 1;
|
|
4243
|
+
}
|
|
4101
4244
|
/**
|
|
4102
4245
|
* Why the matched turn's plan did not run, as the service shows it: what
|
|
4103
4246
|
* the service withheld it for, else the code of the gate that declined it
|
|
@@ -4143,6 +4286,11 @@ export class ControlServer {
|
|
|
4143
4286
|
const note = this.replay.takeNote(run.replay);
|
|
4144
4287
|
if (note)
|
|
4145
4288
|
context.push(note);
|
|
4289
|
+
// The steps a test just left out (AC5): said before the model makes its
|
|
4290
|
+
// next call, which must not be one of them.
|
|
4291
|
+
const skipped = this.replay.takeSkipNote(run.replay);
|
|
4292
|
+
if (skipped)
|
|
4293
|
+
context.push(skipped);
|
|
4146
4294
|
if (run.handback) {
|
|
4147
4295
|
context.push(run.handback);
|
|
4148
4296
|
run.handback = undefined;
|
|
@@ -5118,6 +5266,7 @@ export class ControlServer {
|
|
|
5118
5266
|
record(run, pair, d) {
|
|
5119
5267
|
if (!run.recording)
|
|
5120
5268
|
return;
|
|
5269
|
+
this.noteOwnCall(run, d.pinnedBy);
|
|
5121
5270
|
const toolInput = serializeCapped(redact(d.args));
|
|
5122
5271
|
const toolOutput = d.isError ? undefined : serializeCapped(redact(d.result));
|
|
5123
5272
|
const toolError = d.isError
|
|
@@ -13,9 +13,12 @@ export interface StartRunOptions {
|
|
|
13
13
|
};
|
|
14
14
|
/**
|
|
15
15
|
* Marks a *repeat* (the service's newestRun.md): a matched turn whose plan
|
|
16
|
-
* did not run, so the model is doing the recording's task itself
|
|
17
|
-
*
|
|
18
|
-
*
|
|
16
|
+
* did not run, so the model is doing the recording's task itself — or whose
|
|
17
|
+
* plan ran to its end, after which the model went on with calls of its own
|
|
18
|
+
* instead of answering from its results (N7). Sent only when the match
|
|
19
|
+
* offered it ({@link RunMatch.repeat}). `reason` is why the plan did not
|
|
20
|
+
* run, or `redone` after a plan that ran to its end. The service never
|
|
21
|
+
* prompt-matches a repeat.
|
|
19
22
|
*/
|
|
20
23
|
repeatOf?: {
|
|
21
24
|
runId: string;
|
|
@@ -58,6 +61,27 @@ export interface FlushOptions {
|
|
|
58
61
|
*/
|
|
59
62
|
timeoutMs?: number;
|
|
60
63
|
}
|
|
64
|
+
/**
|
|
65
|
+
* A plan's own call recorded into the turn's newest run (the service's
|
|
66
|
+
* autocorrect.md AC2): which plan made it, which of its steps, and the chain
|
|
67
|
+
* revision it was served at — left out when the service served none, since
|
|
68
|
+
* its schema takes a number or nothing, never null.
|
|
69
|
+
*/
|
|
70
|
+
export interface PlanCallMark {
|
|
71
|
+
scenarioId: string;
|
|
72
|
+
stepIndex: number;
|
|
73
|
+
chainRevision?: number;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* A recorded step's metadata. `pinnedBy`: a plan executed this step live in
|
|
77
|
+
* the turn (R-INTENT-7). `byPlan`: the plan made this call before the agent
|
|
78
|
+
* redid the task, and it is recorded first into the agent's repeat so the
|
|
79
|
+
* repeat holds the whole turn (AC2). The service stores neither with a vector.
|
|
80
|
+
*/
|
|
81
|
+
export type StepMetadata = {
|
|
82
|
+
pinnedBy?: string;
|
|
83
|
+
byPlan?: PlanCallMark;
|
|
84
|
+
};
|
|
61
85
|
export interface Recorder {
|
|
62
86
|
/**
|
|
63
87
|
* Create the run; returns the client-generated runId synchronously.
|
|
@@ -83,17 +107,18 @@ export interface Recorder {
|
|
|
83
107
|
};
|
|
84
108
|
/**
|
|
85
109
|
* Set when a plan executed this step, to the plan's scenario id
|
|
86
|
-
* (R-INTENT-7)
|
|
110
|
+
* (R-INTENT-7), or made it before the agent redid the task (`byPlan`,
|
|
111
|
+
* AC2). The service stores such a step without a vector, so the
|
|
87
112
|
* system's own replays never count as recurrences of the work.
|
|
88
113
|
*/
|
|
89
|
-
metadata?:
|
|
90
|
-
pinnedBy: string;
|
|
91
|
-
};
|
|
114
|
+
metadata?: StepMetadata;
|
|
92
115
|
}): string;
|
|
93
116
|
recordToolResponse(runId: string, stepIndex: number, d: {
|
|
94
117
|
toolName: string;
|
|
95
118
|
toolOutput?: string;
|
|
96
119
|
toolError?: string;
|
|
120
|
+
/** `byPlan` on the answer of a plan's own call (AC2); omitted otherwise. */
|
|
121
|
+
metadata?: StepMetadata;
|
|
97
122
|
}): string;
|
|
98
123
|
recordFinalAnswer(runId: string, stepIndex: number, d: {
|
|
99
124
|
answer: string;
|
|
@@ -414,6 +439,19 @@ export interface ExecutionReport {
|
|
|
414
439
|
* person reading the audit log, this is for the console.
|
|
415
440
|
*/
|
|
416
441
|
declined?: string;
|
|
442
|
+
/**
|
|
443
|
+
* The plan ran to its end and the agent did not take its result: it went on
|
|
444
|
+
* with calls of its own, recorded as the recording's newest run with the
|
|
445
|
+
* reason `redone` (the service's newestRun.md N7, autocorrect.md AC1).
|
|
446
|
+
* `runId` is that repeat's own id; `ownCalls` the agent's own tool calls
|
|
447
|
+
* recorded into it — not the plan's calls recorded first (AC2), and not a
|
|
448
|
+
* step a plan pinned. Only on the report of the prompt's plan, and only when
|
|
449
|
+
* the service took the repeat; an older service drops the field.
|
|
450
|
+
*/
|
|
451
|
+
redoneBy?: {
|
|
452
|
+
runId: string;
|
|
453
|
+
ownCalls: number;
|
|
454
|
+
};
|
|
417
455
|
}
|
|
418
456
|
/** Optional capability: reporting needs a service, and a NullRecorder has none. */
|
|
419
457
|
export interface ScenarioReporter {
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
*/
|
|
51
51
|
import { type AuthSession } from "../auth/client.js";
|
|
52
52
|
import { type ServiceHealth } from "../util/service-health.js";
|
|
53
|
-
import type { ExecutionReport, FlushOptions, IntentMatchAnswer, IntentMatchOptions, IntentMatchRequest, Recorder, RecorderHealth, RecorderHealthReporter, RunCreation, RunMatch, RunMetrics, StartRunOptions } from "./recorder.js";
|
|
53
|
+
import type { ExecutionReport, FlushOptions, IntentMatchAnswer, IntentMatchOptions, IntentMatchRequest, Recorder, RecorderHealth, RecorderHealthReporter, RunCreation, RunMatch, RunMetrics, StartRunOptions, StepMetadata } from "./recorder.js";
|
|
54
54
|
/**
|
|
55
55
|
* Caps for the free text this recorder sends, in UTF-8 bytes. Tool payloads
|
|
56
56
|
* arrive capped at 64 KiB by their producers; text never passed through one,
|
|
@@ -225,14 +225,13 @@ export declare class RemoteRecorder implements Recorder, RecorderHealthReporter
|
|
|
225
225
|
text: string;
|
|
226
226
|
source: "thinking";
|
|
227
227
|
};
|
|
228
|
-
metadata?:
|
|
229
|
-
pinnedBy: string;
|
|
230
|
-
};
|
|
228
|
+
metadata?: StepMetadata;
|
|
231
229
|
}): string;
|
|
232
230
|
recordToolResponse(runId: string, stepIndex: number, data: {
|
|
233
231
|
toolName: string;
|
|
234
232
|
toolOutput?: string;
|
|
235
233
|
toolError?: string;
|
|
234
|
+
metadata?: StepMetadata;
|
|
236
235
|
}): string;
|
|
237
236
|
recordFinalAnswer(runId: string, stepIndex: number, data: {
|
|
238
237
|
answer: string;
|
|
@@ -674,6 +674,10 @@ export class RemoteRecorder {
|
|
|
674
674
|
toolName: data.toolName,
|
|
675
675
|
toolOutput: out(toolOutput),
|
|
676
676
|
toolError: out(toolError),
|
|
677
|
+
// Omitted, not nulled, when absent: an ordinary answer is the body it
|
|
678
|
+
// has always been. Only a plan's own call recorded into a redo carries
|
|
679
|
+
// one (`byPlan`, the service's autocorrect.md AC2).
|
|
680
|
+
metadata: data.metadata,
|
|
677
681
|
}));
|
|
678
682
|
return id;
|
|
679
683
|
}
|
|
@@ -744,6 +748,12 @@ export class RemoteRecorder {
|
|
|
744
748
|
kind: "report",
|
|
745
749
|
bytes: sizeOf(prompt, error, ...(steps ?? []).flatMap((s) => [s.toolInput, s.toolOutput])) + 1024,
|
|
746
750
|
send: async () => {
|
|
751
|
+
// A redo whose run the service refused to create is no run of its:
|
|
752
|
+
// never named (the service's autocorrect.md AC1), even when the
|
|
753
|
+
// refusal came after the report was built.
|
|
754
|
+
const redoneBy = report.redoneBy && this.runs.get(report.redoneBy.runId)?.settled?.created.created === false
|
|
755
|
+
? undefined
|
|
756
|
+
: report.redoneBy;
|
|
747
757
|
const body = await this.request("POST", `/scenarios/${report.scenarioId}/executions`,
|
|
748
758
|
// Per-step verdicts and the headline failure (errorshandling.md) ride
|
|
749
759
|
// beside the costs. All optional on the service side, so an older
|
|
@@ -776,6 +786,9 @@ export class RemoteRecorder {
|
|
|
776
786
|
// Why nothing ran, when nothing ran. Additive: an older service drops
|
|
777
787
|
// the field and books the cost exactly as it always did.
|
|
778
788
|
declined: report.declined,
|
|
789
|
+
// The plan ran to its end and the agent redid the task (the
|
|
790
|
+
// service's autocorrect.md AC1). Additive, like `declined`.
|
|
791
|
+
redoneBy,
|
|
779
792
|
}, steps), { retryTransient: Boolean(report.ticket) });
|
|
780
793
|
const r = (body ?? {});
|
|
781
794
|
const failed = report.steps?.filter((s) => s.status === "failed").length ?? 0;
|
|
@@ -795,6 +808,9 @@ export class RemoteRecorder {
|
|
|
795
808
|
baselineEligible: report.baselineEligible === false ? false : undefined,
|
|
796
809
|
sharedWith: report.sharedWith?.length,
|
|
797
810
|
declined: report.declined,
|
|
811
|
+
// `bir investigate` prints it beside the outcome: "redone by run_…".
|
|
812
|
+
redoneBy: report.redoneBy?.runId,
|
|
813
|
+
ownCalls: report.redoneBy?.ownCalls,
|
|
798
814
|
});
|
|
799
815
|
},
|
|
800
816
|
});
|
|
@@ -90,11 +90,7 @@ export declare class SwitchableRecorder implements Recorder {
|
|
|
90
90
|
currentAccessToken(): string | undefined;
|
|
91
91
|
startRun(input: string, metadata?: Record<string, unknown>, options?: StartRunOptions): string;
|
|
92
92
|
recordToolSelected(runId: string, stepIndex: number, d: Parameters<Recorder["recordToolSelected"]>[2]): string;
|
|
93
|
-
recordToolResponse(runId: string, stepIndex: number, d:
|
|
94
|
-
toolName: string;
|
|
95
|
-
toolOutput?: string;
|
|
96
|
-
toolError?: string;
|
|
97
|
-
}): string;
|
|
93
|
+
recordToolResponse(runId: string, stepIndex: number, d: Parameters<Recorder["recordToolResponse"]>[2]): string;
|
|
98
94
|
recordFinalAnswer(runId: string, stepIndex: number, d: {
|
|
99
95
|
answer: string;
|
|
100
96
|
}): string;
|
package/dist/replay/bundle.d.ts
CHANGED
|
@@ -34,7 +34,17 @@ export interface BundleEntry {
|
|
|
34
34
|
response: string;
|
|
35
35
|
/** True when `response` is a recorded output, not a fresh run. */
|
|
36
36
|
recorded?: boolean;
|
|
37
|
+
/**
|
|
38
|
+
* Set on a step its test left out (the service's autocorrect.md AC5): no
|
|
39
|
+
* call was made, and the bundle says so in its place, so the model can tell
|
|
40
|
+
* a whole answer from a part. `text` is the test in words, when it has some.
|
|
41
|
+
*/
|
|
42
|
+
notNeeded?: {
|
|
43
|
+
text: string | null;
|
|
44
|
+
};
|
|
37
45
|
}
|
|
46
|
+
/** The test's words, one line and short, for a bundle or a note. */
|
|
47
|
+
export declare function testWords(text: string | null | undefined): string | null;
|
|
38
48
|
/** Cap a computed input for inclusion in a bundle. */
|
|
39
49
|
export declare function bundleInput(value: unknown): string;
|
|
40
50
|
export interface AssembleOptions {
|
package/dist/replay/bundle.js
CHANGED
|
@@ -30,6 +30,13 @@ export const BASH_CHANNEL_MAX = 25_000;
|
|
|
30
30
|
const MAX_INPUT_CHARS = 2_000;
|
|
31
31
|
/** Fixed structural overhead reserved per step (labels, separators). */
|
|
32
32
|
const PER_STEP_OVERHEAD = 80;
|
|
33
|
+
/** The test's words, one line and short, for a bundle or a note. */
|
|
34
|
+
export function testWords(text) {
|
|
35
|
+
const line = (text ?? "").trim().split(/\r?\n/, 1)[0]?.trim() ?? "";
|
|
36
|
+
if (!line)
|
|
37
|
+
return null;
|
|
38
|
+
return line.length > 200 ? `${line.slice(0, 200)}…` : line;
|
|
39
|
+
}
|
|
33
40
|
/** Cap a computed input for inclusion in a bundle. */
|
|
34
41
|
export function bundleInput(value) {
|
|
35
42
|
let text;
|
|
@@ -90,7 +97,9 @@ const SUBTASK_FOOTER = "\n\nThe calls above are done. Continue your task, callin
|
|
|
90
97
|
* Compose the bundle. `maxChars` is a hard ceiling on the returned string.
|
|
91
98
|
*/
|
|
92
99
|
export function assembleBundle(entries, maxChars, opts = {}) {
|
|
93
|
-
|
|
100
|
+
// A step left out has no response to share the budget with: only the steps
|
|
101
|
+
// that answered split it.
|
|
102
|
+
const n = entries.filter((e) => !e.notNeeded).length;
|
|
94
103
|
const anyRecorded = entries.some((e) => e.recorded);
|
|
95
104
|
// A hand-over outranks a sub-task: it says "continue" too, and it also has to
|
|
96
105
|
// say that the plan stopped short. A sub-task outranks the live and mixed
|
|
@@ -111,10 +120,22 @@ export function assembleBundle(entries, maxChars, opts = {}) {
|
|
|
111
120
|
: LIVE_FOOTER;
|
|
112
121
|
const fixedCost = header.length +
|
|
113
122
|
footer.length +
|
|
114
|
-
entries.reduce((sum, e) => sum +
|
|
123
|
+
entries.reduce((sum, e) => sum +
|
|
124
|
+
PER_STEP_OVERHEAD +
|
|
125
|
+
e.toolName.length +
|
|
126
|
+
e.input.length +
|
|
127
|
+
// A left-out step's line, which has no response to take from.
|
|
128
|
+
(e.notNeeded ? PER_STEP_OVERHEAD + (testWords(e.notNeeded.text)?.length ?? 0) : 0), 0);
|
|
115
129
|
const responseBudget = Math.max(0, maxChars - fixedCost);
|
|
116
130
|
const perResponse = n > 0 ? Math.floor(responseBudget / n) : responseBudget;
|
|
117
131
|
const sections = entries.map((e, i) => {
|
|
132
|
+
if (e.notNeeded) {
|
|
133
|
+
// Never "skipped" or "failed": the step was left out on purpose, because
|
|
134
|
+
// this request does not need it, and nothing is missing for its absence.
|
|
135
|
+
const why = testWords(e.notNeeded.text);
|
|
136
|
+
return (`\n\n--- Step ${i + 1}: ${e.toolName} (not needed) ---\n` +
|
|
137
|
+
`Not run: this request does not need it${why ? ` — it is only for when ${why}` : ""}.`);
|
|
138
|
+
}
|
|
118
139
|
const response = truncate(e.response, perResponse);
|
|
119
140
|
return (`\n\n--- Step ${i + 1}: ${e.toolName} ---\n` +
|
|
120
141
|
`Call: ${e.input}\n` +
|