@basein/runner 0.3.8 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/record/remote-recorder.d.ts +8 -0
- package/dist/record/remote-recorder.js +76 -9
- package/dist/record/truncate.d.ts +11 -0
- package/dist/record/truncate.js +11 -0
- package/dist/replay/controller.d.ts +14 -1
- package/dist/replay/controller.js +92 -7
- package/dist/replay/plan.d.ts +6 -0
- package/dist/replay/plan.js +13 -3
- package/dist/replay/types.d.ts +16 -0
- package/docs/calculatedReplay.md +37 -2
- package/docs/calculatedReplayGuide.md +1 -0
- package/package.json +1 -1
- package/production.md +1 -0
|
@@ -69,6 +69,14 @@ export declare const TEXT_CAPS: {
|
|
|
69
69
|
/** An execution report's error lines, rendered in a web page. */
|
|
70
70
|
readonly reportError: number;
|
|
71
71
|
};
|
|
72
|
+
/**
|
|
73
|
+
* The most an execution report's request may weigh. The service reads this
|
|
74
|
+
* route with its 100 kB JSON parser — only `/recordings` has the larger one —
|
|
75
|
+
* and a report it refuses for its size loses the turn's cost with it. So the
|
|
76
|
+
* steps' tool payloads get what is left of this after everything else in the
|
|
77
|
+
* report: the costs, the prompt, the verdicts.
|
|
78
|
+
*/
|
|
79
|
+
export declare const REPORT_MAX_BYTES: number;
|
|
72
80
|
/** Queued sends past which the oldest unsent step is dropped (`BIR_RECORDER_MAX_PENDING`). */
|
|
73
81
|
export declare const DEFAULT_MAX_PENDING_SENDS = 500;
|
|
74
82
|
/** Queued bytes past which the same happens (`BIR_RECORDER_MAX_PENDING_BYTES`). */
|
|
@@ -52,7 +52,7 @@ import { credentialsPath, describeBody, isAuthResponse, issuedFor, loadCredentia
|
|
|
52
52
|
import { errText, logDetail, logEvery, logLine } from "../util/log.js";
|
|
53
53
|
import { isUnhealthyStatus, serviceHealth, serviceTimeoutMs, timeoutSignal, } from "../util/service-health.js";
|
|
54
54
|
import { redactJsonText, redactText } from "./redact.js";
|
|
55
|
-
import { capJsonText, capText, DEFAULT_MAX_PAYLOAD_BYTES, preCut } from "./truncate.js";
|
|
55
|
+
import { capJsonText, capText, DEFAULT_MAX_PAYLOAD_BYTES, preCut, REPORT_PAYLOAD_MAX_BYTES, } from "./truncate.js";
|
|
56
56
|
/**
|
|
57
57
|
* Caps for the free text this recorder sends, in UTF-8 bytes. Tool payloads
|
|
58
58
|
* arrive capped at 64 KiB by their producers; text never passed through one,
|
|
@@ -73,6 +73,14 @@ export const TEXT_CAPS = {
|
|
|
73
73
|
/** An execution report's error lines, rendered in a web page. */
|
|
74
74
|
reportError: 2 * 1024,
|
|
75
75
|
};
|
|
76
|
+
/**
|
|
77
|
+
* The most an execution report's request may weigh. The service reads this
|
|
78
|
+
* route with its 100 kB JSON parser — only `/recordings` has the larger one —
|
|
79
|
+
* and a report it refuses for its size loses the turn's cost with it. So the
|
|
80
|
+
* steps' tool payloads get what is left of this after everything else in the
|
|
81
|
+
* report: the costs, the prompt, the verdicts.
|
|
82
|
+
*/
|
|
83
|
+
export const REPORT_MAX_BYTES = 96 * 1024;
|
|
76
84
|
/** Queued sends past which the oldest unsent step is dropped (`BIR_RECORDER_MAX_PENDING`). */
|
|
77
85
|
export const DEFAULT_MAX_PENDING_SENDS = 500;
|
|
78
86
|
/** Queued bytes past which the same happens (`BIR_RECORDER_MAX_PENDING_BYTES`). */
|
|
@@ -177,6 +185,61 @@ function sizeOf(...parts) {
|
|
|
177
185
|
n += p?.text.length ?? 0;
|
|
178
186
|
return n;
|
|
179
187
|
}
|
|
188
|
+
function pendStep(step) {
|
|
189
|
+
return {
|
|
190
|
+
step,
|
|
191
|
+
toolInput: pend(step.toolInput, REPORT_PAYLOAD_MAX_BYTES, true),
|
|
192
|
+
toolOutput: pend(step.toolOutput, REPORT_PAYLOAD_MAX_BYTES, true),
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* An execution report's request body: `fields`, and its steps as sent — each
|
|
197
|
+
* error and each tool payload capped and redacted, like every other field here.
|
|
198
|
+
*
|
|
199
|
+
* The payloads are what makes a report heavy, so they are added last and only
|
|
200
|
+
* while the whole body stays within {@link REPORT_MAX_BYTES}: the failed
|
|
201
|
+
* steps' first — theirs are the ones somebody will read — then every step's
|
|
202
|
+
* input, then the outputs, each in the order the steps ran. A payload that
|
|
203
|
+
* does not fit is left out; its step keeps its verdict, and the costs the
|
|
204
|
+
* report exists for are never what gives way.
|
|
205
|
+
*/
|
|
206
|
+
function reportBody(fields, steps) {
|
|
207
|
+
if (!steps)
|
|
208
|
+
return JSON.stringify(fields);
|
|
209
|
+
const sent = steps.map(({ step }) => ({
|
|
210
|
+
...step,
|
|
211
|
+
error: typeof step.error === "string"
|
|
212
|
+
? capText(step.error, TEXT_CAPS.reportError, redactText)
|
|
213
|
+
: step.error,
|
|
214
|
+
toolInput: undefined,
|
|
215
|
+
toolOutput: undefined,
|
|
216
|
+
}));
|
|
217
|
+
const bare = Buffer.byteLength(JSON.stringify({ ...fields, steps: sent }), "utf8");
|
|
218
|
+
let left = REPORT_MAX_BYTES - bare;
|
|
219
|
+
const add = (i, key) => {
|
|
220
|
+
if (sent[i][key] !== undefined)
|
|
221
|
+
return;
|
|
222
|
+
const text = out(steps[i][key]);
|
|
223
|
+
if (text === undefined)
|
|
224
|
+
return;
|
|
225
|
+
// The value as a JSON string, and `,"<key>":` in front of it.
|
|
226
|
+
const size = Buffer.byteLength(JSON.stringify(text), "utf8") + key.length + 4;
|
|
227
|
+
if (size > left)
|
|
228
|
+
return;
|
|
229
|
+
sent[i][key] = text;
|
|
230
|
+
left -= size;
|
|
231
|
+
};
|
|
232
|
+
const all = sent.map((_, i) => i);
|
|
233
|
+
for (const i of all.filter((n) => sent[n].status === "failed")) {
|
|
234
|
+
add(i, "toolInput");
|
|
235
|
+
add(i, "toolOutput");
|
|
236
|
+
}
|
|
237
|
+
for (const i of all)
|
|
238
|
+
add(i, "toolInput");
|
|
239
|
+
for (const i of all)
|
|
240
|
+
add(i, "toolOutput");
|
|
241
|
+
return JSON.stringify({ ...fields, steps: sent });
|
|
242
|
+
}
|
|
180
243
|
/** A tool error is JSON when a producer serialized it, and prose when it did not. */
|
|
181
244
|
function pendError(text) {
|
|
182
245
|
if (typeof text !== "string")
|
|
@@ -609,12 +672,21 @@ export class RemoteRecorder {
|
|
|
609
672
|
const costUsd = report.deriveCostUsd + report.sessionCostUsd + report.fallbackCostUsd;
|
|
610
673
|
const prompt = pend(report.prompt, TEXT_CAPS.prompt);
|
|
611
674
|
const error = pend(report.error, TEXT_CAPS.reportError);
|
|
675
|
+
const steps = report.steps?.map(pendStep);
|
|
612
676
|
this.enqueue({
|
|
613
677
|
lane: EXECUTIONS,
|
|
614
678
|
kind: "report",
|
|
615
|
-
bytes: sizeOf(prompt, error) + 1024,
|
|
679
|
+
bytes: sizeOf(prompt, error, ...(steps ?? []).flatMap((s) => [s.toolInput, s.toolOutput])) + 1024,
|
|
616
680
|
send: async () => {
|
|
617
|
-
const body = await this.request("POST", `/scenarios/${report.scenarioId}/executions`,
|
|
681
|
+
const body = await this.request("POST", `/scenarios/${report.scenarioId}/executions`,
|
|
682
|
+
// Per-step verdicts and the headline failure (errorshandling.md) ride
|
|
683
|
+
// beside the costs. All optional on the service side, so an older
|
|
684
|
+
// service ignores them rather than rejecting the report — and the
|
|
685
|
+
// costs still book. Errors are shown on a web page, so they are
|
|
686
|
+
// capped and redacted like any text; so is what each step sent and
|
|
687
|
+
// received, which the page shows under the verdict in place of the
|
|
688
|
+
// recording's sample (`reportBody`).
|
|
689
|
+
reportBody({
|
|
618
690
|
ticket: report.ticket,
|
|
619
691
|
outcome: report.outcome,
|
|
620
692
|
costUsd,
|
|
@@ -627,11 +699,6 @@ export class RemoteRecorder {
|
|
|
627
699
|
measured: report.measured,
|
|
628
700
|
pricingVersion: report.pricingVersion,
|
|
629
701
|
prompt: out(prompt),
|
|
630
|
-
// Per-step verdicts and the headline failure (errorshandling.md). All
|
|
631
|
-
// optional on the service side, so an older service ignores them rather
|
|
632
|
-
// than rejecting the report — and the costs still book. Errors are
|
|
633
|
-
// shown on a web page, so they are capped and redacted like any text.
|
|
634
|
-
steps: report.steps?.map((s) => typeof s.error === "string" ? { ...s, error: capText(s.error, TEXT_CAPS.reportError, redactText) } : s),
|
|
635
702
|
error: out(error),
|
|
636
703
|
errorStage: report.errorStage,
|
|
637
704
|
errorStepIndex: report.errorStepIndex,
|
|
@@ -643,7 +710,7 @@ export class RemoteRecorder {
|
|
|
643
710
|
// Why nothing ran, when nothing ran. Additive: an older service drops
|
|
644
711
|
// the field and books the cost exactly as it always did.
|
|
645
712
|
declined: report.declined,
|
|
646
|
-
}), { retryTransient: Boolean(report.ticket) });
|
|
713
|
+
}, steps), { retryTransient: Boolean(report.ticket) });
|
|
647
714
|
const r = (body ?? {});
|
|
648
715
|
const failed = report.steps?.filter((s) => s.status === "failed").length ?? 0;
|
|
649
716
|
logLine("execution.reported", {
|
|
@@ -20,6 +20,17 @@
|
|
|
20
20
|
export declare const DEFAULT_MAX_PAYLOAD_BYTES: number;
|
|
21
21
|
/** 8 KiB per individual string leaf. */
|
|
22
22
|
export declare const DEFAULT_MAX_STRING_BYTES: number;
|
|
23
|
+
/**
|
|
24
|
+
* What one tool payload may take inside an execution report. A report carries
|
|
25
|
+
* every step of a replay in one small request, so a step's input or output
|
|
26
|
+
* gets far less there than the 64 KiB a recorded step has to itself.
|
|
27
|
+
*
|
|
28
|
+
* 3,800 bytes, not a round 4 KiB: the service keeps 4,000 characters of each,
|
|
29
|
+
* and a payload cut here still fits that with its marker. So the scenario's
|
|
30
|
+
* page shows this cut, which says how large the payload was, and never a
|
|
31
|
+
* second one made on top of it.
|
|
32
|
+
*/
|
|
33
|
+
export declare const REPORT_PAYLOAD_MAX_BYTES = 3800;
|
|
23
34
|
export interface TruncateOptions {
|
|
24
35
|
maxPayloadBytes?: number;
|
|
25
36
|
maxStringBytes?: number;
|
package/dist/record/truncate.js
CHANGED
|
@@ -20,6 +20,17 @@
|
|
|
20
20
|
export const DEFAULT_MAX_PAYLOAD_BYTES = 64 * 1024;
|
|
21
21
|
/** 8 KiB per individual string leaf. */
|
|
22
22
|
export const DEFAULT_MAX_STRING_BYTES = 8 * 1024;
|
|
23
|
+
/**
|
|
24
|
+
* What one tool payload may take inside an execution report. A report carries
|
|
25
|
+
* every step of a replay in one small request, so a step's input or output
|
|
26
|
+
* gets far less there than the 64 KiB a recorded step has to itself.
|
|
27
|
+
*
|
|
28
|
+
* 3,800 bytes, not a round 4 KiB: the service keeps 4,000 characters of each,
|
|
29
|
+
* and a payload cut here still fits that with its marker. So the scenario's
|
|
30
|
+
* page shows this cut, which says how large the payload was, and never a
|
|
31
|
+
* second one made on top of it.
|
|
32
|
+
*/
|
|
33
|
+
export const REPORT_PAYLOAD_MAX_BYTES = 3_800;
|
|
23
34
|
const utf8 = (s) => Buffer.byteLength(s, "utf8");
|
|
24
35
|
/** The marker {@link truncateString} appends, so a second cut can report the first size. */
|
|
25
36
|
const MARKER = /… \[truncated \d+ of (\d+) bytes\]$/;
|
|
@@ -273,6 +273,12 @@ interface PinnedCall {
|
|
|
273
273
|
toolName: string;
|
|
274
274
|
/** When it was pinned — a steered step's duration is measured across hooks. */
|
|
275
275
|
pinnedAt: number;
|
|
276
|
+
/**
|
|
277
|
+
* The arguments the pin gave the call, for the step's verdict: the plan
|
|
278
|
+
* never sees a steered call, so this is the only place they are known when
|
|
279
|
+
* its result lands.
|
|
280
|
+
*/
|
|
281
|
+
input?: Record<string, unknown>;
|
|
276
282
|
/**
|
|
277
283
|
* When the host said the call finished (`PostToolUse`), if the control
|
|
278
284
|
* server told us ({@link ReplayController.hostCompleted}). A wrapped step
|
|
@@ -854,7 +860,14 @@ export declare class ReplayController {
|
|
|
854
860
|
* never be told something the log does not also say.
|
|
855
861
|
*/
|
|
856
862
|
private observeStep;
|
|
857
|
-
/**
|
|
863
|
+
/**
|
|
864
|
+
* Upsert one step's verdict. Later news about a step replaces earlier news.
|
|
865
|
+
*
|
|
866
|
+
* The verdict keeps what the step sent and what came back, when the caller
|
|
867
|
+
* has them. They are this run's, which the service can learn nowhere else:
|
|
868
|
+
* a matched turn is not recorded, so without them the scenario's page shows
|
|
869
|
+
* the recording's input and output beside this run's verdict.
|
|
870
|
+
*/
|
|
858
871
|
private recordStep;
|
|
859
872
|
/** The step verdicts, in chain order, for the execution report. */
|
|
860
873
|
private stepResultsOf;
|
|
@@ -13,8 +13,8 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import { parseQualifiedName } from "../control/correlation.js";
|
|
15
15
|
import { isHousekeepingCall } from "../record/housekeeping.js";
|
|
16
|
-
import { redact } from "../record/redact.js";
|
|
17
|
-
import { serializeCapped } from "../record/truncate.js";
|
|
16
|
+
import { REDACT_SCAN_MAX_CHARS, redact, redactJsonText, redactText } from "../record/redact.js";
|
|
17
|
+
import { capText, REPORT_PAYLOAD_MAX_BYTES, serializeCapped } from "../record/truncate.js";
|
|
18
18
|
import { logDetail, logLine, errText } from "../util/log.js";
|
|
19
19
|
import { serviceHealth, serviceTimeoutMs, timeoutSignal, } from "../util/service-health.js";
|
|
20
20
|
import { bashCommandFor } from "./bash-channel.js";
|
|
@@ -779,7 +779,13 @@ export class ReplayController {
|
|
|
779
779
|
// The model does the task itself — told so in place of this call.
|
|
780
780
|
const step = plan.currentStep();
|
|
781
781
|
if (step && ready.blame) {
|
|
782
|
-
this.recordStep(state, {
|
|
782
|
+
this.recordStep(state, {
|
|
783
|
+
step,
|
|
784
|
+
entry: plan.currentEntry(),
|
|
785
|
+
outcome: "failed",
|
|
786
|
+
stage: ready.stage,
|
|
787
|
+
error: ready.error,
|
|
788
|
+
});
|
|
783
789
|
}
|
|
784
790
|
this.handOver(state, {
|
|
785
791
|
kind: "step_failed",
|
|
@@ -829,6 +835,7 @@ export class ReplayController {
|
|
|
829
835
|
reach: this.reachFor(state, step),
|
|
830
836
|
toolName,
|
|
831
837
|
pinnedAt: Date.now(),
|
|
838
|
+
input,
|
|
832
839
|
});
|
|
833
840
|
logDetail("replay.pin", { n: plan.currentStepIndex, tool: toolName });
|
|
834
841
|
return { kind: "pin", input, stepIndex: plan.currentStepIndex };
|
|
@@ -850,6 +857,7 @@ export class ReplayController {
|
|
|
850
857
|
if (step) {
|
|
851
858
|
this.recordStep(state, {
|
|
852
859
|
step,
|
|
860
|
+
entry: plan.currentEntry(),
|
|
853
861
|
outcome: "failed",
|
|
854
862
|
stage: "tool_input_logic",
|
|
855
863
|
error: errText(err),
|
|
@@ -1043,6 +1051,10 @@ export class ReplayController {
|
|
|
1043
1051
|
// reading it after the call would judge the *next* step, and reading it
|
|
1044
1052
|
// before assumes no other pin ever moved it.
|
|
1045
1053
|
const step = plan.allSteps()[pin.stepIndex];
|
|
1054
|
+
// The entry it ran as, so its verdict is kept under the scenario that owns
|
|
1055
|
+
// the step: a called segment numbers its steps from zero too, and without
|
|
1056
|
+
// this its step 0 replaced the caller's (segmented.md R-CALL-13).
|
|
1057
|
+
const entry = plan.allEntries()[pin.stepIndex];
|
|
1046
1058
|
// The output threads into the step at the cursor. One pin at a time
|
|
1047
1059
|
// (`preTool`) keeps that the pin's own step; if it ever is not, threading
|
|
1048
1060
|
// would feed one step's output to another's logic, so the plan stops.
|
|
@@ -1057,6 +1069,9 @@ export class ReplayController {
|
|
|
1057
1069
|
`the plan's cursor is at step ${plan.currentStepIndex}, not the pinned step ${pin.stepIndex}`;
|
|
1058
1070
|
const info = {
|
|
1059
1071
|
step,
|
|
1072
|
+
entry,
|
|
1073
|
+
input: pin.input,
|
|
1074
|
+
output: answerOf(output),
|
|
1060
1075
|
outcome: "failed",
|
|
1061
1076
|
stage: "tool_call",
|
|
1062
1077
|
error,
|
|
@@ -1068,7 +1083,7 @@ export class ReplayController {
|
|
|
1068
1083
|
this.handOver(state, {
|
|
1069
1084
|
kind: "step_failed",
|
|
1070
1085
|
step,
|
|
1071
|
-
entry
|
|
1086
|
+
entry,
|
|
1072
1087
|
position: pin.stepIndex,
|
|
1073
1088
|
error,
|
|
1074
1089
|
delivery: "above",
|
|
@@ -1090,6 +1105,10 @@ export class ReplayController {
|
|
|
1090
1105
|
if (step) {
|
|
1091
1106
|
this.recordStep(state, {
|
|
1092
1107
|
step,
|
|
1108
|
+
entry,
|
|
1109
|
+
input: pin.input,
|
|
1110
|
+
// What the logic threw on — the output is usually the explanation.
|
|
1111
|
+
output,
|
|
1093
1112
|
outcome: "failed",
|
|
1094
1113
|
stage: "tool_output_logic",
|
|
1095
1114
|
error: errText(err),
|
|
@@ -1125,6 +1144,9 @@ export class ReplayController {
|
|
|
1125
1144
|
if (step && toolError) {
|
|
1126
1145
|
const info = {
|
|
1127
1146
|
step,
|
|
1147
|
+
entry,
|
|
1148
|
+
input: pin.input,
|
|
1149
|
+
output,
|
|
1128
1150
|
outcome: "failed",
|
|
1129
1151
|
stage: "tool_call",
|
|
1130
1152
|
error: toolError,
|
|
@@ -1146,6 +1168,9 @@ export class ReplayController {
|
|
|
1146
1168
|
if (step) {
|
|
1147
1169
|
this.recordStep(state, {
|
|
1148
1170
|
step,
|
|
1171
|
+
entry,
|
|
1172
|
+
input: pin.input,
|
|
1173
|
+
output,
|
|
1149
1174
|
outcome: "executed",
|
|
1150
1175
|
ms: Date.now() - pin.pinnedAt,
|
|
1151
1176
|
});
|
|
@@ -1210,13 +1235,23 @@ export class ReplayController {
|
|
|
1210
1235
|
? `${failure.error} — it was sent, and may or may not have completed`
|
|
1211
1236
|
: failure.error
|
|
1212
1237
|
: toolResultError(output);
|
|
1238
|
+
const entry = plan.allEntries()[pin.stepIndex];
|
|
1213
1239
|
if (error !== undefined) {
|
|
1214
|
-
const info = {
|
|
1240
|
+
const info = {
|
|
1241
|
+
step,
|
|
1242
|
+
entry,
|
|
1243
|
+
input: pin.input,
|
|
1244
|
+
output: answerOf(output),
|
|
1245
|
+
outcome: "failed",
|
|
1246
|
+
stage: "tool_call",
|
|
1247
|
+
error,
|
|
1248
|
+
ms,
|
|
1249
|
+
};
|
|
1215
1250
|
this.logStep(info);
|
|
1216
1251
|
this.recordStep(state, info);
|
|
1217
1252
|
return;
|
|
1218
1253
|
}
|
|
1219
|
-
this.recordStep(state, { step, outcome: "executed", ms });
|
|
1254
|
+
this.recordStep(state, { step, entry, input: pin.input, output, outcome: "executed", ms });
|
|
1220
1255
|
state.stepsPinned += 1;
|
|
1221
1256
|
this.upgrade(state, "fell_back");
|
|
1222
1257
|
logDetail("replay.thread_late", {
|
|
@@ -2308,7 +2343,14 @@ export class ReplayController {
|
|
|
2308
2343
|
this.recordStep(state, info);
|
|
2309
2344
|
};
|
|
2310
2345
|
}
|
|
2311
|
-
/**
|
|
2346
|
+
/**
|
|
2347
|
+
* Upsert one step's verdict. Later news about a step replaces earlier news.
|
|
2348
|
+
*
|
|
2349
|
+
* The verdict keeps what the step sent and what came back, when the caller
|
|
2350
|
+
* has them. They are this run's, which the service can learn nowhere else:
|
|
2351
|
+
* a matched turn is not recorded, so without them the scenario's page shows
|
|
2352
|
+
* the recording's input and output beside this run's verdict.
|
|
2353
|
+
*/
|
|
2312
2354
|
recordStep(state, info) {
|
|
2313
2355
|
const status = info.outcome === "executed" ? "ok" : info.outcome;
|
|
2314
2356
|
// Keyed by the *flat* position, not by the step's own index: two scenarios
|
|
@@ -2325,6 +2367,16 @@ export class ReplayController {
|
|
|
2325
2367
|
? `${info.error.slice(0, MAX_STEP_ERROR_CHARS)}…`
|
|
2326
2368
|
: info.error
|
|
2327
2369
|
: undefined;
|
|
2370
|
+
// Payloads only for a step of the reported scenario's own chain. The
|
|
2371
|
+
// console still files an execution's verdicts by step number alone
|
|
2372
|
+
// (plan-services.md W1.9), and a called segment numbers its steps from
|
|
2373
|
+
// zero as well: a called step's call would be shown under the caller's
|
|
2374
|
+
// step of the same number — the very mix-up the payloads are there to end.
|
|
2375
|
+
const own = !info.entry || info.entry.depth === 0;
|
|
2376
|
+
// A step whose input logic threw never had an input: the `{}` the plan
|
|
2377
|
+
// reports there is a placeholder, not a call with no arguments.
|
|
2378
|
+
const sent = own && info.stage !== "tool_input_logic" ? info.input : undefined;
|
|
2379
|
+
const received = own ? info.output : undefined;
|
|
2328
2380
|
state.stepResults.set(key, {
|
|
2329
2381
|
stepIndex: info.step.stepIndex,
|
|
2330
2382
|
toolName: info.step.toolName ?? "",
|
|
@@ -2334,6 +2386,8 @@ export class ReplayController {
|
|
|
2334
2386
|
stage: info.stage ?? (status === "skipped" ? "tool_call" : undefined),
|
|
2335
2387
|
error,
|
|
2336
2388
|
durationMs: info.ms,
|
|
2389
|
+
toolInput: sent === undefined ? undefined : reportPayload(serializeCapped(redact(sent)), true),
|
|
2390
|
+
toolOutput: received === undefined ? undefined : reportPayload(received),
|
|
2337
2391
|
// The tool is gone from its server: the service repairs the plan without
|
|
2338
2392
|
// waiting out the grace (plan-services.md D8, kind 2).
|
|
2339
2393
|
toolMissing: info.toolMissing || undefined,
|
|
@@ -2507,6 +2561,37 @@ function pinFailure(pin, output, failed, how) {
|
|
|
2507
2561
|
: "the tool reported an error");
|
|
2508
2562
|
return { error, reason };
|
|
2509
2563
|
}
|
|
2564
|
+
/**
|
|
2565
|
+
* What a call that did not thread answered with, if it answered anything.
|
|
2566
|
+
* `null` is a proxy that lost the call (tool-error.ts, `isNoResult`), and `{}`
|
|
2567
|
+
* is how a hook that carried no `tool_response` reaches here: neither is an
|
|
2568
|
+
* output anybody received.
|
|
2569
|
+
*/
|
|
2570
|
+
function answerOf(output) {
|
|
2571
|
+
return isNoResult(output) || output.trim() === "{}" ? undefined : output;
|
|
2572
|
+
}
|
|
2573
|
+
/**
|
|
2574
|
+
* A step's input or output as its verdict carries it: redacted as a recorded
|
|
2575
|
+
* step's is, and cut — visibly — to what a report has room for.
|
|
2576
|
+
*
|
|
2577
|
+
* Cut as text rather than kept as JSON: these are read on the scenario's page,
|
|
2578
|
+
* never parsed, and the first kilobytes of the payload itself read better than
|
|
2579
|
+
* a preview of it inside a second layer of escapes.
|
|
2580
|
+
*
|
|
2581
|
+
* `clean` says it was redacted a moment ago — the input, serialized by the
|
|
2582
|
+
* caller. An output may not have been: the hook's view of a steered built-in
|
|
2583
|
+
* is threaded as the host gave it. So an output is redacted here, and as JSON,
|
|
2584
|
+
* where a secret is judged by its key as well as by its shape.
|
|
2585
|
+
*/
|
|
2586
|
+
function reportPayload(json, clean = false) {
|
|
2587
|
+
if (clean)
|
|
2588
|
+
return capText(json, REPORT_PAYLOAD_MAX_BYTES);
|
|
2589
|
+
// Larger than any producer here makes: only what a report keeps is scanned.
|
|
2590
|
+
if (json.length > REDACT_SCAN_MAX_CHARS) {
|
|
2591
|
+
return capText(json, REPORT_PAYLOAD_MAX_BYTES, redactText);
|
|
2592
|
+
}
|
|
2593
|
+
return capText(redactJsonText(json), REPORT_PAYLOAD_MAX_BYTES);
|
|
2594
|
+
}
|
|
2510
2595
|
/**
|
|
2511
2596
|
* The index a hand-over reports, in the *caller's* chain (R-CALL-32).
|
|
2512
2597
|
*
|
package/dist/replay/plan.d.ts
CHANGED
|
@@ -105,6 +105,12 @@ export interface StepInfo {
|
|
|
105
105
|
*/
|
|
106
106
|
entry?: FlatEntry;
|
|
107
107
|
input: Record<string, unknown>;
|
|
108
|
+
/**
|
|
109
|
+
* What the step's output logic was given: the tool's answer, or for
|
|
110
|
+
* `recorded` the source run's output that stood in for it. Absent when the
|
|
111
|
+
* call produced none — `skipped`, or a failure before the tool ran.
|
|
112
|
+
*/
|
|
113
|
+
output?: string;
|
|
108
114
|
outcome: "executed" | "recorded" | "skipped" | "failed";
|
|
109
115
|
/** Which part of the step broke. Only set for `failed`. */
|
|
110
116
|
stage?: ExecutionStage;
|
package/dist/replay/plan.js
CHANGED
|
@@ -215,8 +215,11 @@ function settingsOf(paramsObject, mapped) {
|
|
|
215
215
|
* still abandons the whole bundle rather than shipping a half-computed one. All
|
|
216
216
|
* this adds is that the failure is now *attributable* — "step 2's tool input
|
|
217
217
|
* logic", rather than a stack trace the console cannot line up against a chain.
|
|
218
|
+
*
|
|
219
|
+
* `output` is what the body was given, for a stage that runs on one: a throw
|
|
220
|
+
* in the output logic is only readable next to the output it threw on.
|
|
218
221
|
*/
|
|
219
|
-
function reportingStage(onStep, entry, stage, input, startedAt, fn) {
|
|
222
|
+
function reportingStage(onStep, entry, stage, input, startedAt, fn, output) {
|
|
220
223
|
try {
|
|
221
224
|
return fn();
|
|
222
225
|
}
|
|
@@ -225,6 +228,7 @@ function reportingStage(onStep, entry, stage, input, startedAt, fn) {
|
|
|
225
228
|
step: entry.step,
|
|
226
229
|
entry,
|
|
227
230
|
input,
|
|
231
|
+
output,
|
|
228
232
|
outcome: "failed",
|
|
229
233
|
stage,
|
|
230
234
|
error: err instanceof Error ? err.message : String(err),
|
|
@@ -817,6 +821,7 @@ export class ScenarioReplayPlan {
|
|
|
817
821
|
step,
|
|
818
822
|
entry,
|
|
819
823
|
input: computed,
|
|
824
|
+
output: response,
|
|
820
825
|
outcome: "failed",
|
|
821
826
|
stage: "tool_call",
|
|
822
827
|
error: toolError,
|
|
@@ -845,7 +850,7 @@ export class ScenarioReplayPlan {
|
|
|
845
850
|
Object.assign(resp.get(entry.frame.id) ?? {}, derived);
|
|
846
851
|
}
|
|
847
852
|
this.leaveFrame(entry, resp);
|
|
848
|
-
});
|
|
853
|
+
}, response);
|
|
849
854
|
}
|
|
850
855
|
catch (err) {
|
|
851
856
|
if (!stopOnFailure)
|
|
@@ -872,6 +877,7 @@ export class ScenarioReplayPlan {
|
|
|
872
877
|
step,
|
|
873
878
|
entry,
|
|
874
879
|
input: computed,
|
|
880
|
+
output: response,
|
|
875
881
|
outcome: error ? "failed" : "executed",
|
|
876
882
|
stage: error ? "tool_call" : undefined,
|
|
877
883
|
error,
|
|
@@ -977,6 +983,7 @@ export class ScenarioReplayPlan {
|
|
|
977
983
|
step,
|
|
978
984
|
entry,
|
|
979
985
|
input: computed,
|
|
986
|
+
output: response,
|
|
980
987
|
outcome: "failed",
|
|
981
988
|
stage: "tool_call",
|
|
982
989
|
error: toolError,
|
|
@@ -997,7 +1004,7 @@ export class ScenarioReplayPlan {
|
|
|
997
1004
|
}
|
|
998
1005
|
let derivedKeys;
|
|
999
1006
|
try {
|
|
1000
|
-
derivedKeys = reportingStage(onStep, entry, "tool_output_logic", computed, startedAt, () => this.applyOutput(response));
|
|
1007
|
+
derivedKeys = reportingStage(onStep, entry, "tool_output_logic", computed, startedAt, () => this.applyOutput(response), response);
|
|
1001
1008
|
}
|
|
1002
1009
|
catch (err) {
|
|
1003
1010
|
if (!stopOnFailure)
|
|
@@ -1024,6 +1031,7 @@ export class ScenarioReplayPlan {
|
|
|
1024
1031
|
step,
|
|
1025
1032
|
entry,
|
|
1026
1033
|
input: computed,
|
|
1034
|
+
output: response,
|
|
1027
1035
|
outcome: error ? "failed" : "executed",
|
|
1028
1036
|
stage: error ? "tool_call" : undefined,
|
|
1029
1037
|
error,
|
|
@@ -1095,6 +1103,8 @@ export class ScenarioReplayPlan {
|
|
|
1095
1103
|
step,
|
|
1096
1104
|
entry,
|
|
1097
1105
|
input: computed,
|
|
1106
|
+
// The stand-in is what the step's output logic runs on from here.
|
|
1107
|
+
output: standIn,
|
|
1098
1108
|
outcome: standIn !== undefined ? "recorded" : "skipped",
|
|
1099
1109
|
error,
|
|
1100
1110
|
toolMissing: isMissingToolError(error) || undefined,
|
package/dist/replay/types.d.ts
CHANGED
|
@@ -249,6 +249,22 @@ export interface ExecutionStepResult {
|
|
|
249
249
|
/** Message only — never a tool payload; this is rendered in a web page. */
|
|
250
250
|
error?: string;
|
|
251
251
|
durationMs?: number;
|
|
252
|
+
/**
|
|
253
|
+
* What the step sent its tool on this run: the arguments its input logic
|
|
254
|
+
* computed, serialized and redacted as a recorded step's are, and cut to
|
|
255
|
+
* what a report carries. Absent when the step broke before it had any.
|
|
256
|
+
*
|
|
257
|
+
* Without the two payloads the scenario's page can only put the recording's
|
|
258
|
+
* sample under this run's verdict and time — a call that was not made, and
|
|
259
|
+
* an answer that was not given, on a card that says "passed".
|
|
260
|
+
*/
|
|
261
|
+
toolInput?: string;
|
|
262
|
+
/**
|
|
263
|
+
* What the step's output logic was given, on the same terms: the tool's
|
|
264
|
+
* answer, or for `recorded` the source run's output that stood in for it.
|
|
265
|
+
* Absent on a step that got none.
|
|
266
|
+
*/
|
|
267
|
+
toolOutput?: string;
|
|
252
268
|
/**
|
|
253
269
|
* The step's tool no longer exists on its server, read from the error text
|
|
254
270
|
* (plan-services.md D8, kind 2). The service then repairs the plan without
|
package/docs/calculatedReplay.md
CHANGED
|
@@ -903,9 +903,38 @@ steps: Array<{
|
|
|
903
903
|
stage?: ExecutionStage; // where in the step, for `failed` / `skipped`
|
|
904
904
|
error?: string; // message only — this is rendered in a web page
|
|
905
905
|
durationMs?: number;
|
|
906
|
+
toolInput?: string; // what the step sent its tool, as JSON text
|
|
907
|
+
toolOutput?: string; // what its output logic was given
|
|
906
908
|
}>
|
|
907
909
|
```
|
|
908
910
|
|
|
911
|
+
**What the step sent and received.** `toolInput` is the arguments the step's input logic computed
|
|
912
|
+
on this turn, and `toolOutput` what its output logic was given: the tool's answer, or for `recorded`
|
|
913
|
+
the source run's output that stood in. They are this run's, and the service can learn them nowhere
|
|
914
|
+
else — a matched turn is not recorded (§12). Without them the recording page can only put the
|
|
915
|
+
recording's own sample under this run's verdict and time: a transfer of 3500 beside "passed", after
|
|
916
|
+
a replay that sent 3600.
|
|
917
|
+
|
|
918
|
+
- **Redacted and capped.** Each is serialized and redacted as a recorded step's payload is
|
|
919
|
+
(`serializeCapped(redact(…))`; an output is redacted again here, as JSON, because the hook's view
|
|
920
|
+
of a steered built-in is threaded as the host gave it). Then it is cut to 3,800
|
|
921
|
+
bytes (`REPORT_PAYLOAD_MAX_BYTES`), as text and with a marker that says how much was dropped: these
|
|
922
|
+
are read on a page, never parsed, and with its marker a cut payload still fits the 4,000
|
|
923
|
+
characters the service keeps of each — so the page shows this cut and its sizes, not a second one.
|
|
924
|
+
The recorder redacts once more as it sends.
|
|
925
|
+
- **Inside one small request.** The service reads `POST /scenarios/:id/executions` with its 100 kB
|
|
926
|
+
JSON parser (only `/recordings` has the 5 MB one) and answers a larger body 413, which would lose
|
|
927
|
+
the turn's cost with it. So the recorder holds the whole report to 96 KiB (`REPORT_MAX_BYTES`):
|
|
928
|
+
it writes the costs and the verdicts first, then adds payloads while they fit — the failed steps'
|
|
929
|
+
first, then every step's input, then the outputs, in the order the steps ran. A payload that does
|
|
930
|
+
not fit is left out, and its step keeps its verdict.
|
|
931
|
+
- **Absent where there was none.** A step whose input logic threw has no input. A `skipped` step
|
|
932
|
+
has no output, and neither has a call the host says failed without a result.
|
|
933
|
+
- **Only for the reported scenario's own steps.** A step inside a called segment is reported with
|
|
934
|
+
its verdict and its `scenarioId`, without payloads: the console still files verdicts by step
|
|
935
|
+
number alone (the service's plan-services.md, W1.9), and a called segment numbers its steps from
|
|
936
|
+
zero too.
|
|
937
|
+
|
|
909
938
|
`ok` and `recorded` are passes (`recorded` with an asterisk: the values came from the source run,
|
|
910
939
|
which happens only with `BIR_REPLAY_RECORDED_OUTPUTS=1` and in `bir replay`). `skipped` produced no
|
|
911
940
|
result here: its call could not be sent (a built-in outside the session, no proxy for its server,
|
|
@@ -926,8 +955,10 @@ The verdicts come from `plan.ts`'s `onStep`, which has always known them — wit
|
|
|
926
955
|
logic failure is announced *before* the throw propagates (`reportingStage`), because after it
|
|
927
956
|
nothing knows which step the stack trace belonged to. In steer mode there is no loop to observe, so
|
|
928
957
|
the controller records the verdict at both hooks: `PreToolUse` for an input-logic throw, `PostToolUse`
|
|
929
|
-
for the pass or the output-logic throw
|
|
930
|
-
|
|
958
|
+
for the pass or the output-logic throw — and keeps the pinned arguments on the pin, so the verdict
|
|
959
|
+
`PostToolUse` writes can say what the call ran with. `ReplayState.stepResults` is keyed by the
|
|
960
|
+
scenario that owns the step and its index, so the second word on a step replaces the first and a
|
|
961
|
+
called segment's step 0 never replaces the caller's.
|
|
931
962
|
|
|
932
963
|
One consequence for the rule that a costless decline is not worth sending (`buildReport`, and the
|
|
933
964
|
service's matching `202 { recorded: false }`): it now *is* worth sending when a step broke. No cost
|
|
@@ -991,6 +1022,10 @@ trusting it.
|
|
|
991
1022
|
would 404. `run.recording = false` already handles it — `record()` early-returns and `sealRun` skips
|
|
992
1023
|
both the final answer and `finishRun`.
|
|
993
1024
|
|
|
1025
|
+
What the turn's steps sent and received is not lost with it: the execution report carries them, on
|
|
1026
|
+
each step's verdict (§11.2a). That is the only copy the service gets, and it is what the recording
|
|
1027
|
+
page shows as the scenario's last run.
|
|
1028
|
+
|
|
994
1029
|
Three consequences to keep straight:
|
|
995
1030
|
|
|
996
1031
|
1. **Step indices are still allocated.** `ordering` runs regardless, correlations are still minted
|
|
@@ -572,6 +572,7 @@ matches. A prompt that missed both is matched once more on its task line (`PROMP
|
|
|
572
572
|
| A parked step ran again after an hour, or a plan was recalculated by itself | The service's grace period (D8): a parked step is served with `stop: null` once an hour for a day and a success clears its count; if it keeps failing, or its tool is gone, the next repeat of the prompt is recorded in full and the plan is calculated again from it | Nothing: that is the repair. `bir investigate` says which it is — a step still being retried, a parked step whose repair is due, or a plan being calculated again. To repair now by hand: `bir scenario calc <runId> --force` for a whole-run plan; from the next version, `bir scenario calc <scnId> --force` works for any plan, a sub-task one included. A plan with steps edited by hand is not repaired by itself: the parked step stays parked, and `bir investigate` says the repair was skipped. Then fix the step (§9.2); or take the edits back with `bir scenario undo`, once per edit, and it is repaired as usual; or recalculate now and drop the edits: `bir scenario calc <scnId> --force --discard-edits` |
|
|
573
573
|
| `sessionCostUsd` grows every prompt in a session | The per-run usage watermark is missing or not taken | Design §11.4. Reports without a mark must carry `measured: false` |
|
|
574
574
|
| Everything works, nothing is saved | A replayed turn is deliberately not recorded (design §12) | Correct. The matched run stays canonical and its `iterations` is bumped |
|
|
575
|
+
| The console shows a replayed step's input and output with the recording's values, beside this run's verdict and time | The run was reported by a runner up to 0.3.8, which sent each step's verdict and duration and nothing of what it sent or received. The page then falls back to the recording's sample (design §11.2a) | Update the runner: a newer one sends each replayed step's input and output with its verdict, redacted and cut to under 4 KB, and the card shows them as *last run*. To see what an older replay ran with: the result of `mcp__bir__run_scenario` in the session's transcript, or `lastParams` in `bir scenario show scn_…` |
|
|
575
576
|
| `bir doctor`: the SessionStart hook points at a Node that is gone | The hook names Node by absolute path. That Node was uninstalled, moved, or was a version manager's per-shell copy; the hook then fails on every session, no recorder is started, and runs record Tier 2 | `bir setup` here again — it rewrites the hook to the Node that runs it |
|
|
576
577
|
| `control.port_busy` in the log; the recorder is not running | Another program holds this project's port, or the system excludes it (on Windows, a range Hyper-V or WSL reserved). A background recorder with a recorded port does not fall back to another, because every hook but SessionStart names this one; only a `bir-hooks` you run by hand in a terminal still does | `bir setup --port <another>` in this directory — the fix the log line itself names. It rewrites the hook URLs and the scenario server's address together and restarts the recorder |
|
|
577
578
|
| The recorder was started by the hook, and `bir doctor` says `servers=(all wrapped)` although you set `BIR_REPLAY_ALLOW_SERVERS` | The hook starts the recorder with Claude Code's environment, not your terminal's; a variable set in one terminal is not seen | `bir replay allow <a,b>` — stored for the project, read by every recorder — then `bir up --restart` |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@basein/runner",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.9",
|
|
4
4
|
"description": "A recording MCP proxy: sits between any MCP client and its MCP servers, executes each call on the client's behalf, and records the run as a reusable BaseIn scenario.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/production.md
CHANGED
|
@@ -518,6 +518,7 @@ Over 2-minute runs with one turn every 8 s (92 hooks):
|
|
|
518
518
|
- Non-JSON tool errors: 8 KiB.
|
|
519
519
|
- Report errors: 2 KiB.
|
|
520
520
|
- Tool payloads: 64 KiB.
|
|
521
|
+
- A replayed step's input and output, in the execution report: 3,800 bytes each, redacted like a recorded step's. The whole report stays under 96 KiB, because the service reads that route with its 100 kB parser: a payload that does not fit is left out, and its step keeps its verdict.
|
|
521
522
|
|
|
522
523
|
Truncation is always marked with the original size.
|
|
523
524
|
|