@basein/runner 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/auth/client.d.ts +12 -0
- package/dist/auth/client.js +41 -0
- package/dist/bin/bir-hooks.d.ts +18 -3
- package/dist/bin/bir-hooks.js +70 -4
- package/dist/bin/bir.js +56 -3
- package/dist/control/server.d.ts +84 -1
- package/dist/control/server.js +555 -51
- package/dist/control/transcript.d.ts +40 -0
- package/dist/control/transcript.js +105 -0
- package/dist/record/recorder.d.ts +178 -4
- package/dist/record/recorder.js +6 -0
- package/dist/record/remote-recorder.d.ts +20 -2
- package/dist/record/remote-recorder.js +66 -6
- package/dist/replay/bundle.d.ts +10 -1
- package/dist/replay/bundle.js +41 -3
- package/dist/replay/controller.d.ts +179 -6
- package/dist/replay/controller.js +584 -55
- package/dist/replay/coverage.js +2 -2
- package/dist/replay/derive.d.ts +54 -7
- package/dist/replay/derive.js +174 -17
- package/dist/replay/flatten.d.ts +125 -0
- package/dist/replay/flatten.js +182 -0
- package/dist/replay/handover.d.ts +60 -0
- package/dist/replay/handover.js +82 -0
- package/dist/replay/logic.d.ts +11 -0
- package/dist/replay/logic.js +17 -0
- package/dist/replay/plan.d.ts +105 -8
- package/dist/replay/plan.js +309 -47
- package/dist/replay/source-run.d.ts +24 -10
- package/dist/replay/source-run.js +65 -30
- package/dist/replay/types.d.ts +108 -5
- package/dist/replay/types.js +33 -3
- package/docs/calculatedReplay.md +16 -8
- package/docs/calculatedReplayGuide.md +4 -4
- package/package.json +1 -1
package/dist/replay/plan.js
CHANGED
|
@@ -20,9 +20,29 @@
|
|
|
20
20
|
* step to know its reach, so it can choose between the proxy, a recorded
|
|
21
21
|
* output, and skipping.
|
|
22
22
|
*/
|
|
23
|
-
import { evalParamsLogic, evalResponseParamsLogic, evalToolInputLogic, evalToolOutputLogic, } from "./logic.js";
|
|
23
|
+
import { evalParamMapLogic, evalParamsLogic, evalResponseParamsLogic, evalResultMapLogic, evalToolInputLogic, evalToolOutputLogic, } from "./logic.js";
|
|
24
|
+
import { flatEntriesOf } from "./flatten.js";
|
|
24
25
|
import { assembleBundle, bundleInput, MAX_REPLAY_REASON } from "./bundle.js";
|
|
25
26
|
import { toolResultError } from "./tool-error.js";
|
|
27
|
+
/**
|
|
28
|
+
* The recorded values for the *settings* a mapping did not name (R-CALL-21).
|
|
29
|
+
*
|
|
30
|
+
* A target must always be supplied by the caller; a setting is what the segment
|
|
31
|
+
* was recorded doing, and taking its sample here is what makes "may be omitted"
|
|
32
|
+
* true rather than "runs with undefined".
|
|
33
|
+
*/
|
|
34
|
+
function settingsOf(paramsObject, mapped) {
|
|
35
|
+
const out = {};
|
|
36
|
+
if (!paramsObject || typeof paramsObject !== "object")
|
|
37
|
+
return out;
|
|
38
|
+
for (const [name, p] of Object.entries(paramsObject)) {
|
|
39
|
+
if (name in mapped)
|
|
40
|
+
continue;
|
|
41
|
+
if (p?.kind === "setting" && p && "sampleValue" in p)
|
|
42
|
+
out[name] = p.sampleValue;
|
|
43
|
+
}
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
26
46
|
/**
|
|
27
47
|
* Evaluate one of the scenario's logic bodies, naming the step and the stage to
|
|
28
48
|
* the observer if it throws — then rethrow, unchanged and uncaught.
|
|
@@ -32,13 +52,14 @@ import { toolResultError } from "./tool-error.js";
|
|
|
32
52
|
* this adds is that the failure is now *attributable* — "step 2's tool input
|
|
33
53
|
* logic", rather than a stack trace the console cannot line up against a chain.
|
|
34
54
|
*/
|
|
35
|
-
function reportingStage(onStep,
|
|
55
|
+
function reportingStage(onStep, entry, stage, input, startedAt, fn) {
|
|
36
56
|
try {
|
|
37
57
|
return fn();
|
|
38
58
|
}
|
|
39
59
|
catch (err) {
|
|
40
60
|
onStep?.({
|
|
41
|
-
step,
|
|
61
|
+
step: entry.step,
|
|
62
|
+
entry,
|
|
42
63
|
input,
|
|
43
64
|
outcome: "failed",
|
|
44
65
|
stage,
|
|
@@ -54,21 +75,107 @@ export class ScenarioReplayPlan {
|
|
|
54
75
|
mode;
|
|
55
76
|
intent;
|
|
56
77
|
scenario;
|
|
57
|
-
|
|
78
|
+
/** The flat list this plan walks. One entry per step that will really run. */
|
|
79
|
+
entries;
|
|
58
80
|
paramsPromise;
|
|
59
81
|
params = {};
|
|
60
82
|
respParams;
|
|
61
83
|
stepIndex = 0;
|
|
62
84
|
readyPromise;
|
|
85
|
+
/** Exclusive end of the planned steps. Equal to the chain length unless a step is parked. */
|
|
86
|
+
stopAt;
|
|
87
|
+
/** This plan is a sub-task inside the agent's own task (segmented.md R-OUT-13). */
|
|
88
|
+
subTask;
|
|
63
89
|
constructor(opts) {
|
|
64
90
|
this.scenario = opts.scenario;
|
|
65
91
|
this.scenarioId = opts.scenario.id;
|
|
66
92
|
this.runId = opts.scenario.runId;
|
|
67
93
|
this.intent = opts.scenario.intent ?? "";
|
|
68
|
-
this.
|
|
94
|
+
this.entries = opts.entries ?? flatEntriesOf(opts.scenario);
|
|
69
95
|
this.mode = opts.mode;
|
|
70
96
|
this.paramsPromise = opts.params;
|
|
71
97
|
this.respParams = { ...(opts.respParamsInit ?? {}) };
|
|
98
|
+
// The caller's own frame accumulates into the plan's `respParams`, so
|
|
99
|
+
// `accumulated`, `responseModel` and every depth-0 step see one object.
|
|
100
|
+
if (this.entries[0])
|
|
101
|
+
this.rootFrame().respParams = this.respParams;
|
|
102
|
+
this.subTask = opts.subTask === true;
|
|
103
|
+
const stop = opts.stopAt;
|
|
104
|
+
this.stopAt =
|
|
105
|
+
typeof stop === "number" && stop >= 0 && stop < this.entries.length
|
|
106
|
+
? stop
|
|
107
|
+
: this.entries.length;
|
|
108
|
+
}
|
|
109
|
+
/** The caller's own frame — the one at depth 0. */
|
|
110
|
+
rootFrame() {
|
|
111
|
+
let frame = this.entries[0].frame;
|
|
112
|
+
while (frame.parent)
|
|
113
|
+
frame = frame.parent;
|
|
114
|
+
return frame;
|
|
115
|
+
}
|
|
116
|
+
/** The parameters one entry's logic runs with (segmented.md R-CALL-29). */
|
|
117
|
+
paramsFor(entry) {
|
|
118
|
+
return entry.depth === 0 ? this.params : (entry.frame.parameters ?? {});
|
|
119
|
+
}
|
|
120
|
+
/** The accumulated values one entry's logic runs with. */
|
|
121
|
+
respFor(entry) {
|
|
122
|
+
return entry.depth === 0 ? this.respParams : entry.frame.respParams;
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Entering a called frame: build the segment's parameters from the caller's
|
|
126
|
+
* state (R-CALL-29).
|
|
127
|
+
*
|
|
128
|
+
* The recipe is the dry run's, exactly (9.8 step 10): the mapping's output,
|
|
129
|
+
* plus the segment's own recorded sample for every *setting* the mapping left
|
|
130
|
+
* out. A mapping that omits a setting therefore cannot pass the service's
|
|
131
|
+
* check and then run `undefined` here.
|
|
132
|
+
*/
|
|
133
|
+
enterFrame(entry, resp) {
|
|
134
|
+
if (!entry.first || entry.depth === 0)
|
|
135
|
+
return;
|
|
136
|
+
const frame = entry.frame;
|
|
137
|
+
const parent = frame.parent;
|
|
138
|
+
const parentParams = parent
|
|
139
|
+
? parent.depth === 0
|
|
140
|
+
? this.params
|
|
141
|
+
: (parent.parameters ?? {})
|
|
142
|
+
: this.params;
|
|
143
|
+
const parentResp = parent
|
|
144
|
+
? resp
|
|
145
|
+
? (resp.get(parent.id) ?? parent.respParams)
|
|
146
|
+
: parent.respParams
|
|
147
|
+
: this.respParams;
|
|
148
|
+
const mapped = frame.paramMapLogic
|
|
149
|
+
? evalParamMapLogic(frame.paramMapLogic, parentParams, parent?.intent ?? this.intent, {
|
|
150
|
+
...parentResp,
|
|
151
|
+
})
|
|
152
|
+
: {};
|
|
153
|
+
frame.parameters = { ...settingsOf(frame.paramsObject, mapped), ...mapped };
|
|
154
|
+
if (resp)
|
|
155
|
+
resp.set(frame.id, {});
|
|
156
|
+
else
|
|
157
|
+
frame.respParams = {};
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Leaving a called frame: hand what the segment emitted back to its caller
|
|
161
|
+
* (R-CALL-29). `out` is the frame's own accumulated `respParams`.
|
|
162
|
+
*/
|
|
163
|
+
leaveFrame(entry, resp) {
|
|
164
|
+
if (!entry.last || entry.depth === 0)
|
|
165
|
+
return;
|
|
166
|
+
const frame = entry.frame;
|
|
167
|
+
const parent = frame.parent;
|
|
168
|
+
if (!frame.resultMapLogic || !parent)
|
|
169
|
+
return;
|
|
170
|
+
const parentParams = parent.depth === 0 ? this.params : (parent.parameters ?? {});
|
|
171
|
+
const target = resp
|
|
172
|
+
? (resp.get(parent.id) ?? parent.respParams)
|
|
173
|
+
: parent.depth === 0
|
|
174
|
+
? this.respParams
|
|
175
|
+
: parent.respParams;
|
|
176
|
+
const own = resp ? (resp.get(frame.id) ?? frame.respParams) : frame.respParams;
|
|
177
|
+
const out = evalResultMapLogic(frame.resultMapLogic, { ...own }, parentParams, parent.intent, { ...target });
|
|
178
|
+
Object.assign(target, out);
|
|
72
179
|
}
|
|
73
180
|
/**
|
|
74
181
|
* Await the derivation, then apply `paramsLogic` — once, however many callers
|
|
@@ -96,8 +203,37 @@ export class ScenarioReplayPlan {
|
|
|
96
203
|
get currentStepIndex() {
|
|
97
204
|
return this.stepIndex;
|
|
98
205
|
}
|
|
206
|
+
/** Steps this plan will run — the chain up to the first known-bad step. */
|
|
99
207
|
get stepCount() {
|
|
100
|
-
return this.
|
|
208
|
+
return this.stopAt;
|
|
209
|
+
}
|
|
210
|
+
/** Every step this plan could run, planned or not. */
|
|
211
|
+
get totalSteps() {
|
|
212
|
+
return this.entries.length;
|
|
213
|
+
}
|
|
214
|
+
/** True when the plan ends in front of a known-bad step (fallbk.md D3). */
|
|
215
|
+
stopsEarly() {
|
|
216
|
+
return this.stopAt < this.entries.length;
|
|
217
|
+
}
|
|
218
|
+
/** The known-bad step the plan stops in front of, if any. */
|
|
219
|
+
stopStep() {
|
|
220
|
+
return this.stopsEarly() ? this.entries[this.stopAt]?.step : undefined;
|
|
221
|
+
}
|
|
222
|
+
/** The entry the plan stops in front of, with the frame that owns it. */
|
|
223
|
+
stopEntry() {
|
|
224
|
+
return this.stopsEarly() ? this.entries[this.stopAt] : undefined;
|
|
225
|
+
}
|
|
226
|
+
/** Tool names from position `from` to the end of the flat chain. */
|
|
227
|
+
toolsFrom(from) {
|
|
228
|
+
return this.entries.slice(Math.max(0, from)).map((e) => e.step.toolName ?? "");
|
|
229
|
+
}
|
|
230
|
+
/** Every entry in order — for the report, and for the owning scenario ids. */
|
|
231
|
+
allEntries() {
|
|
232
|
+
return this.entries;
|
|
233
|
+
}
|
|
234
|
+
/** The entry awaiting execution, or undefined when the plan is done. */
|
|
235
|
+
currentEntry() {
|
|
236
|
+
return this.stepIndex < this.stopAt ? this.entries[this.stepIndex] : undefined;
|
|
101
237
|
}
|
|
102
238
|
/** The parameters in force. Empty until {@link ready} resolves. */
|
|
103
239
|
get parameters() {
|
|
@@ -109,19 +245,19 @@ export class ScenarioReplayPlan {
|
|
|
109
245
|
}
|
|
110
246
|
/** The step awaiting execution, or undefined when the plan is done. */
|
|
111
247
|
currentStep() {
|
|
112
|
-
return this.
|
|
248
|
+
return this.currentEntry()?.step;
|
|
113
249
|
}
|
|
114
250
|
/** The tool the current step expects, or undefined when done. */
|
|
115
251
|
expectedTool() {
|
|
116
|
-
return this.
|
|
252
|
+
return this.currentStep()?.toolName ?? undefined;
|
|
117
253
|
}
|
|
118
|
-
/** True once every step has been applied — the plan should be retired. */
|
|
254
|
+
/** True once every planned step has been applied — the plan should be retired. */
|
|
119
255
|
isDone() {
|
|
120
|
-
return this.stepIndex >= this.
|
|
256
|
+
return this.stepIndex >= this.stopAt;
|
|
121
257
|
}
|
|
122
258
|
/** Every step in order — for the directive, and for coverage reporting. */
|
|
123
259
|
allSteps() {
|
|
124
|
-
return this.
|
|
260
|
+
return this.entries.map((e) => e.step);
|
|
125
261
|
}
|
|
126
262
|
/**
|
|
127
263
|
* True when `toolName` is one of this scenario's own tools.
|
|
@@ -133,7 +269,7 @@ export class ScenarioReplayPlan {
|
|
|
133
269
|
* abandoned the sequence.
|
|
134
270
|
*/
|
|
135
271
|
usesTool(toolName) {
|
|
136
|
-
return this.
|
|
272
|
+
return this.entries.some((e) => e.step.toolName === toolName);
|
|
137
273
|
}
|
|
138
274
|
/**
|
|
139
275
|
* Compute the pinned input for the current step. Throws when there is no
|
|
@@ -141,10 +277,13 @@ export class ScenarioReplayPlan {
|
|
|
141
277
|
* a throw as divergence.
|
|
142
278
|
*/
|
|
143
279
|
toolInputForCurrentStep() {
|
|
144
|
-
const
|
|
145
|
-
if (!
|
|
280
|
+
const entry = this.entries[this.stepIndex];
|
|
281
|
+
if (!entry)
|
|
146
282
|
throw new Error("no current step to compute input for");
|
|
147
|
-
|
|
283
|
+
// Entering a called frame happens here, on its first step, because this is
|
|
284
|
+
// the first moment the caller's own accumulation is final (R-CALL-29).
|
|
285
|
+
this.enterFrame(entry);
|
|
286
|
+
return evalToolInputLogic(entry.step.toolInputLogic ?? "return {};", this.paramsFor(entry), entry.frame.intent, this.respFor(entry));
|
|
148
287
|
}
|
|
149
288
|
/**
|
|
150
289
|
* Thread a real tool output into `respParams` and advance.
|
|
@@ -155,15 +294,20 @@ export class ScenarioReplayPlan {
|
|
|
155
294
|
* Throws when `toolOutputLogic` fails; the caller aborts to a normal turn.
|
|
156
295
|
*/
|
|
157
296
|
applyOutput(realOutput) {
|
|
158
|
-
const
|
|
159
|
-
if (!
|
|
297
|
+
const entry = this.entries[this.stepIndex];
|
|
298
|
+
if (!entry)
|
|
160
299
|
return [];
|
|
161
300
|
let derivedKeys = [];
|
|
162
|
-
if (step.toolOutputLogic) {
|
|
163
|
-
const derived = evalToolOutputLogic(step.toolOutputLogic, realOutput, this.
|
|
301
|
+
if (entry.step.toolOutputLogic) {
|
|
302
|
+
const derived = evalToolOutputLogic(entry.step.toolOutputLogic, realOutput, this.paramsFor(entry), entry.frame.intent, this.respFor(entry));
|
|
164
303
|
derivedKeys = Object.keys(derived);
|
|
165
|
-
|
|
304
|
+
// Mutated, not replaced: the frame's store is the object the plan and
|
|
305
|
+
// every later step of that frame already hold.
|
|
306
|
+
Object.assign(this.respFor(entry), derived);
|
|
166
307
|
}
|
|
308
|
+
// Leaving a called frame hands its results back to the caller before the
|
|
309
|
+
// caller's next step reads them (R-CALL-29).
|
|
310
|
+
this.leaveFrame(entry);
|
|
167
311
|
this.stepIndex += 1;
|
|
168
312
|
return derivedKeys;
|
|
169
313
|
}
|
|
@@ -181,14 +325,22 @@ export class ScenarioReplayPlan {
|
|
|
181
325
|
* model initiates the expected calls — the arguments are supplied by the
|
|
182
326
|
* system, so the model need not compute them.
|
|
183
327
|
*/
|
|
184
|
-
steeringDirective(directToolName = "mcp__bir__run_scenario") {
|
|
328
|
+
steeringDirective(directToolName = "mcp__bir__run_scenario", opts) {
|
|
329
|
+
// A sub-task plan (segmented.md R-OUT-13) never tells the agent to stop
|
|
330
|
+
// calling tools: the steps below are a part of the task it is already doing.
|
|
331
|
+
const subTask = opts?.subTask ?? this.subTask;
|
|
185
332
|
// The intent is a paragraph the analyser wrote; it is context, not an
|
|
186
333
|
// instruction, so it is clipped rather than dumped whole into the prompt.
|
|
187
334
|
const intent = this.intent.length > 240 ? `${this.intent.slice(0, 240)}…` : this.intent;
|
|
188
|
-
const lines = this.
|
|
189
|
-
const why =
|
|
190
|
-
return ` ${i + 1}. ${
|
|
335
|
+
const lines = this.entries.slice(0, this.stopAt).map((e, i) => {
|
|
336
|
+
const why = e.step.reasoning ? ` — ${e.step.reasoning}` : "";
|
|
337
|
+
return ` ${i + 1}. ${e.step.toolName ?? ""}${why}`;
|
|
191
338
|
});
|
|
339
|
+
// A parked step (fallbk.md D3): the model is told up front that the plan is
|
|
340
|
+
// partial, so the hand-over that follows is expected rather than a surprise.
|
|
341
|
+
const tail = this.stopsEarly()
|
|
342
|
+
? ["", "The system will then hand the rest of the task back to you."]
|
|
343
|
+
: [];
|
|
192
344
|
if (this.mode === "direct") {
|
|
193
345
|
// The sequence is listed even though the model does not call it itself:
|
|
194
346
|
// it is about to receive these results, and knowing what was run is what
|
|
@@ -203,7 +355,12 @@ export class ScenarioReplayPlan {
|
|
|
203
355
|
...lines,
|
|
204
356
|
"",
|
|
205
357
|
`Call ${directToolName} once, with no arguments, before any other tool.`,
|
|
206
|
-
|
|
358
|
+
subTask
|
|
359
|
+
? "Then continue your task from the results it returns, calling tools as needed."
|
|
360
|
+
: this.stopsEarly()
|
|
361
|
+
? "Then continue the user's request from the results it returns."
|
|
362
|
+
: "Then answer the user's request from the results it returns.",
|
|
363
|
+
...tail,
|
|
207
364
|
].join("\n");
|
|
208
365
|
}
|
|
209
366
|
return [
|
|
@@ -214,8 +371,17 @@ export class ScenarioReplayPlan {
|
|
|
214
371
|
...(intent ? ["", `Context: ${intent}`] : []),
|
|
215
372
|
"",
|
|
216
373
|
"The system supplies the exact arguments for each call — you do not need to",
|
|
217
|
-
|
|
218
|
-
|
|
374
|
+
...(subTask
|
|
375
|
+
? [
|
|
376
|
+
"compute them. Do not call any other tools until this sequence is complete.",
|
|
377
|
+
"These steps are part of your task: when they are done, continue your task,",
|
|
378
|
+
"calling tools as needed.",
|
|
379
|
+
]
|
|
380
|
+
: [
|
|
381
|
+
"compute them. Do not call any other tools until this sequence is complete,",
|
|
382
|
+
"then answer the user's request from the tool results.",
|
|
383
|
+
]),
|
|
384
|
+
...tail,
|
|
219
385
|
].join("\n");
|
|
220
386
|
}
|
|
221
387
|
/**
|
|
@@ -233,18 +399,27 @@ export class ScenarioReplayPlan {
|
|
|
233
399
|
* Skipping instead would drop the step *and* stop threading `respParams`, so
|
|
234
400
|
* every later step reading from it fails too.
|
|
235
401
|
*/
|
|
236
|
-
async composeBundle(maxChars, execute, recordedOutputFor, onStep) {
|
|
402
|
+
async composeBundle(maxChars, execute, recordedOutputFor, onStep, opts = {}) {
|
|
237
403
|
// A local copy: composing must not corrupt the live plan's accumulation if
|
|
238
404
|
// the caller decides to keep steering afterwards.
|
|
239
|
-
|
|
405
|
+
// One copy per frame, for the same reason the plan copied its own: composing
|
|
406
|
+
// must not corrupt the live accumulation if the caller keeps steering.
|
|
407
|
+
const resp = new Map();
|
|
408
|
+
if (this.entries[0])
|
|
409
|
+
resp.set(this.rootFrame().id, { ...this.respParams });
|
|
240
410
|
const entries = [];
|
|
241
411
|
let executed = 0;
|
|
242
412
|
let recordedCount = 0;
|
|
243
413
|
let skipped = 0;
|
|
244
414
|
let errored = 0;
|
|
245
|
-
for (const
|
|
415
|
+
for (const entry of this.entries.slice(this.stepIndex, this.stopAt)) {
|
|
416
|
+
const step = entry.step;
|
|
246
417
|
const startedAt = Date.now();
|
|
247
|
-
const computed = reportingStage(onStep,
|
|
418
|
+
const computed = reportingStage(onStep, entry, "tool_input_logic", {}, startedAt, () => {
|
|
419
|
+
this.enterFrame(entry, resp);
|
|
420
|
+
const respParams = resp.get(entry.frame.id) ?? {};
|
|
421
|
+
return evalToolInputLogic(step.toolInputLogic ?? "return {};", this.paramsFor(entry), entry.frame.intent, respParams);
|
|
422
|
+
});
|
|
248
423
|
let response;
|
|
249
424
|
let recorded = false;
|
|
250
425
|
try {
|
|
@@ -255,26 +430,27 @@ export class ScenarioReplayPlan {
|
|
|
255
430
|
const error = err instanceof Error ? err.message : String(err);
|
|
256
431
|
const fallback = (typeof step.recordedOutput === "string" && step.recordedOutput.length > 0
|
|
257
432
|
? step.recordedOutput
|
|
258
|
-
: undefined) ?? (await recordedOutputFor?.(
|
|
433
|
+
: undefined) ?? (await recordedOutputFor?.(entry).catch(() => undefined));
|
|
259
434
|
if (!fallback) {
|
|
260
435
|
skipped += 1;
|
|
261
|
-
onStep?.({ step, input: computed, outcome: "skipped", error, ms: Date.now() - startedAt });
|
|
436
|
+
onStep?.({ step, entry, input: computed, outcome: "skipped", error, ms: Date.now() - startedAt });
|
|
262
437
|
continue;
|
|
263
438
|
}
|
|
264
439
|
response = fallback;
|
|
265
440
|
recorded = true;
|
|
266
441
|
recordedCount += 1;
|
|
267
|
-
onStep?.({ step, input: computed, outcome: "recorded", error, ms: Date.now() - startedAt });
|
|
442
|
+
onStep?.({ step, entry, input: computed, outcome: "recorded", error, ms: Date.now() - startedAt });
|
|
268
443
|
}
|
|
269
444
|
// Thread the output for later steps' inputs. A recorded output threads
|
|
270
445
|
// too: `toolOutputLogic` was authored against exactly this shape, and a
|
|
271
446
|
// stale value beats a missing one downstream.
|
|
272
447
|
let derivedKeys = [];
|
|
273
448
|
if (step.toolOutputLogic) {
|
|
274
|
-
const derived = reportingStage(onStep,
|
|
449
|
+
const derived = reportingStage(onStep, entry, "tool_output_logic", computed, startedAt, () => evalToolOutputLogic(step.toolOutputLogic, response, this.paramsFor(entry), entry.frame.intent, resp.get(entry.frame.id) ?? {}));
|
|
275
450
|
derivedKeys = Object.keys(derived);
|
|
276
|
-
Object.assign(
|
|
451
|
+
Object.assign(resp.get(entry.frame.id) ?? {}, derived);
|
|
277
452
|
}
|
|
453
|
+
this.leaveFrame(entry, resp);
|
|
278
454
|
if (!recorded) {
|
|
279
455
|
// A step's verdict is its output, not its resolution: a tool that ran
|
|
280
456
|
// and answered "Error: …" resolves like any other, and a chain of eight
|
|
@@ -284,6 +460,7 @@ export class ScenarioReplayPlan {
|
|
|
284
460
|
errored += 1;
|
|
285
461
|
onStep?.({
|
|
286
462
|
step,
|
|
463
|
+
entry,
|
|
287
464
|
input: computed,
|
|
288
465
|
outcome: error ? "failed" : "executed",
|
|
289
466
|
stage: error ? "tool_call" : undefined,
|
|
@@ -293,14 +470,17 @@ export class ScenarioReplayPlan {
|
|
|
293
470
|
});
|
|
294
471
|
}
|
|
295
472
|
entries.push({
|
|
296
|
-
toolName: step.toolName,
|
|
473
|
+
toolName: step.toolName ?? "",
|
|
297
474
|
input: bundleInput(computed),
|
|
298
475
|
response,
|
|
299
476
|
recorded,
|
|
300
477
|
});
|
|
301
478
|
}
|
|
302
479
|
return {
|
|
303
|
-
text: assembleBundle(entries, maxChars
|
|
480
|
+
text: assembleBundle(entries, maxChars, {
|
|
481
|
+
handover: opts.handover,
|
|
482
|
+
subTask: this.subTask,
|
|
483
|
+
}),
|
|
304
484
|
executed,
|
|
305
485
|
recorded: recordedCount,
|
|
306
486
|
skipped,
|
|
@@ -321,21 +501,39 @@ export class ScenarioReplayPlan {
|
|
|
321
501
|
* an unbounded payload into the model's context for a scenario whose tools
|
|
322
502
|
* answer in megabytes (docs/mcpmark.md §12).
|
|
323
503
|
*/
|
|
324
|
-
async runToCompletion(execute, recordedOutputFor, onStep, deadline, maxChars = MAX_REPLAY_REASON) {
|
|
504
|
+
async runToCompletion(execute, recordedOutputFor, onStep, deadline, maxChars = MAX_REPLAY_REASON, opts = {}) {
|
|
325
505
|
const entries = [];
|
|
326
506
|
let executed = 0;
|
|
327
507
|
let recordedCount = 0;
|
|
328
508
|
let skipped = 0;
|
|
329
509
|
let errored = 0;
|
|
330
510
|
let partial = false;
|
|
511
|
+
let stopped;
|
|
512
|
+
const stopOnFailure = opts.stopOnFailure === true;
|
|
513
|
+
const errText = (err) => (err instanceof Error ? err.message : String(err));
|
|
331
514
|
while (!this.isDone()) {
|
|
332
515
|
if (deadline !== undefined && Date.now() >= deadline) {
|
|
333
516
|
partial = true;
|
|
334
517
|
break;
|
|
335
518
|
}
|
|
336
|
-
const
|
|
519
|
+
const entry = this.entries[this.stepIndex];
|
|
520
|
+
const step = entry.step;
|
|
337
521
|
const startedAt = Date.now();
|
|
338
|
-
|
|
522
|
+
let computed;
|
|
523
|
+
try {
|
|
524
|
+
computed = reportingStage(onStep, entry, "tool_input_logic", {}, startedAt, () => this.toolInputForCurrentStep());
|
|
525
|
+
}
|
|
526
|
+
catch (err) {
|
|
527
|
+
if (!stopOnFailure)
|
|
528
|
+
throw err;
|
|
529
|
+
stopped = {
|
|
530
|
+
kind: "step_failed",
|
|
531
|
+
stepIndex: step.stepIndex,
|
|
532
|
+
stage: "tool_input_logic",
|
|
533
|
+
error: errText(err),
|
|
534
|
+
};
|
|
535
|
+
break;
|
|
536
|
+
}
|
|
339
537
|
let response;
|
|
340
538
|
let recorded = false;
|
|
341
539
|
try {
|
|
@@ -346,20 +544,75 @@ export class ScenarioReplayPlan {
|
|
|
346
544
|
const error = err instanceof Error ? err.message : String(err);
|
|
347
545
|
const fallback = (typeof step.recordedOutput === "string" && step.recordedOutput.length > 0
|
|
348
546
|
? step.recordedOutput
|
|
349
|
-
: undefined) ?? (await recordedOutputFor?.(
|
|
547
|
+
: undefined) ?? (await recordedOutputFor?.(entry).catch(() => undefined));
|
|
350
548
|
if (!fallback) {
|
|
351
549
|
skipped += 1;
|
|
352
|
-
onStep?.({ step, input: computed, outcome: "skipped", error, ms: Date.now() - startedAt });
|
|
550
|
+
onStep?.({ step, entry, input: computed, outcome: "skipped", error, ms: Date.now() - startedAt });
|
|
353
551
|
// Advance past a step that cannot run, without threading anything.
|
|
552
|
+
// The frame is still left behind it: a sub-task whose last step could
|
|
553
|
+
// not run here still hands back whatever its earlier steps emitted.
|
|
554
|
+
this.leaveFrame(entry);
|
|
354
555
|
this.stepIndex += 1;
|
|
355
556
|
continue;
|
|
356
557
|
}
|
|
357
558
|
response = fallback;
|
|
358
559
|
recorded = true;
|
|
359
560
|
recordedCount += 1;
|
|
360
|
-
onStep?.({ step, input: computed, outcome: "recorded", error, ms: Date.now() - startedAt });
|
|
561
|
+
onStep?.({ step, entry, input: computed, outcome: "recorded", error, ms: Date.now() - startedAt });
|
|
562
|
+
}
|
|
563
|
+
if (stopOnFailure && !recorded) {
|
|
564
|
+
// Judged before the output logic, which was written against a success
|
|
565
|
+
// and would only throw a second, less useful error on this one.
|
|
566
|
+
const toolError = toolResultError(response);
|
|
567
|
+
if (toolError) {
|
|
568
|
+
errored += 1;
|
|
569
|
+
onStep?.({
|
|
570
|
+
step,
|
|
571
|
+
entry,
|
|
572
|
+
input: computed,
|
|
573
|
+
outcome: "failed",
|
|
574
|
+
stage: "tool_call",
|
|
575
|
+
error: toolError,
|
|
576
|
+
ms: Date.now() - startedAt,
|
|
577
|
+
});
|
|
578
|
+
entries.push({
|
|
579
|
+
toolName: step.toolName ?? "",
|
|
580
|
+
input: bundleInput(computed),
|
|
581
|
+
response,
|
|
582
|
+
recorded,
|
|
583
|
+
});
|
|
584
|
+
this.stepIndex += 1;
|
|
585
|
+
stopped = {
|
|
586
|
+
kind: "step_failed",
|
|
587
|
+
stepIndex: step.stepIndex,
|
|
588
|
+
stage: "tool_call",
|
|
589
|
+
error: toolError,
|
|
590
|
+
};
|
|
591
|
+
break;
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
let derivedKeys;
|
|
595
|
+
try {
|
|
596
|
+
derivedKeys = reportingStage(onStep, entry, "tool_output_logic", computed, startedAt, () => this.applyOutput(response));
|
|
597
|
+
}
|
|
598
|
+
catch (err) {
|
|
599
|
+
if (!stopOnFailure)
|
|
600
|
+
throw err;
|
|
601
|
+
// The tool ran; its output is real and belongs in front of the model.
|
|
602
|
+
entries.push({
|
|
603
|
+
toolName: step.toolName ?? "",
|
|
604
|
+
input: bundleInput(computed),
|
|
605
|
+
response,
|
|
606
|
+
recorded,
|
|
607
|
+
});
|
|
608
|
+
stopped = {
|
|
609
|
+
kind: "step_failed",
|
|
610
|
+
stepIndex: step.stepIndex,
|
|
611
|
+
stage: "tool_output_logic",
|
|
612
|
+
error: errText(err),
|
|
613
|
+
};
|
|
614
|
+
break;
|
|
361
615
|
}
|
|
362
|
-
const derivedKeys = reportingStage(onStep, step, "tool_output_logic", computed, startedAt, () => this.applyOutput(response));
|
|
363
616
|
if (!recorded) {
|
|
364
617
|
// A step's verdict is its output, not its resolution: a tool that ran
|
|
365
618
|
// and answered "Error: …" resolves like any other, and a chain of eight
|
|
@@ -369,6 +622,7 @@ export class ScenarioReplayPlan {
|
|
|
369
622
|
errored += 1;
|
|
370
623
|
onStep?.({
|
|
371
624
|
step,
|
|
625
|
+
entry,
|
|
372
626
|
input: computed,
|
|
373
627
|
outcome: error ? "failed" : "executed",
|
|
374
628
|
stage: error ? "tool_call" : undefined,
|
|
@@ -378,19 +632,27 @@ export class ScenarioReplayPlan {
|
|
|
378
632
|
});
|
|
379
633
|
}
|
|
380
634
|
entries.push({
|
|
381
|
-
toolName: step.toolName,
|
|
635
|
+
toolName: step.toolName ?? "",
|
|
382
636
|
input: bundleInput(computed),
|
|
383
637
|
response,
|
|
384
638
|
recorded,
|
|
385
639
|
});
|
|
386
640
|
}
|
|
641
|
+
// Every planned step ran and the plan ends in front of a parked one.
|
|
642
|
+
if (!stopped && !partial && this.stopsEarly() && this.isDone()) {
|
|
643
|
+
stopped = { kind: "known_bad_step", stepIndex: this.entries[this.stopAt].stepIndex };
|
|
644
|
+
}
|
|
387
645
|
return {
|
|
388
|
-
text: assembleBundle(entries, maxChars
|
|
646
|
+
text: assembleBundle(entries, maxChars, {
|
|
647
|
+
handover: stopped !== undefined,
|
|
648
|
+
subTask: this.subTask,
|
|
649
|
+
}),
|
|
389
650
|
executed,
|
|
390
651
|
recorded: recordedCount,
|
|
391
652
|
skipped,
|
|
392
653
|
errored,
|
|
393
654
|
partial,
|
|
655
|
+
stopped,
|
|
394
656
|
};
|
|
395
657
|
}
|
|
396
658
|
}
|
|
@@ -21,30 +21,44 @@
|
|
|
21
21
|
*
|
|
22
22
|
* Fetched **lazily** — only when a step actually needs it — and cached for the
|
|
23
23
|
* turn, so a fully-executable replay never makes this call at all.
|
|
24
|
+
*
|
|
25
|
+
* A chain with calls draws on **several** recordings: a called segment's steps
|
|
26
|
+
* take their outputs from that segment's own recording, within its own range
|
|
27
|
+
* (segmented.md R-CALL-31). So the rows are cached per run id, and the cursors
|
|
28
|
+
* are kept per frame — a segment called twice in one chain replays from the
|
|
29
|
+
* start of its range both times, as two separate replays of it would.
|
|
24
30
|
*/
|
|
25
|
-
import type {
|
|
31
|
+
import type { FlatEntry } from "./flatten.js";
|
|
26
32
|
export interface SourceRunOptions {
|
|
27
33
|
baseUrl: string;
|
|
34
|
+
/** The run the plan's own scenario came from; others are fetched on demand. */
|
|
28
35
|
runId: string;
|
|
29
36
|
/** Bearer token; refreshed by the caller, read fresh on each fetch. */
|
|
30
37
|
token: () => string;
|
|
31
38
|
fetchImpl?: typeof fetch;
|
|
32
39
|
timeoutMs?: number;
|
|
33
40
|
}
|
|
34
|
-
export declare class
|
|
41
|
+
export declare class SourceRunCache {
|
|
35
42
|
private readonly opts;
|
|
36
|
-
/**
|
|
37
|
-
private
|
|
38
|
-
private
|
|
39
|
-
|
|
43
|
+
/** run id → every recorded output of that run, in order. */
|
|
44
|
+
private readonly runs;
|
|
45
|
+
private readonly loading;
|
|
46
|
+
/** `frameId#toolName` → how many of that tool the frame has drawn. */
|
|
47
|
+
private readonly cursor;
|
|
40
48
|
constructor(opts: SourceRunOptions);
|
|
41
49
|
/**
|
|
42
|
-
* The next recorded output for
|
|
43
|
-
*
|
|
44
|
-
*
|
|
50
|
+
* The next recorded output for this entry's tool, from this entry's own
|
|
51
|
+
* recording and range, or undefined when there is none left (or the run could
|
|
52
|
+
* not be fetched). Never throws — a missing recorded output means the step is
|
|
53
|
+
* skipped, which the caller already handles.
|
|
45
54
|
*/
|
|
46
|
-
outputFor(
|
|
55
|
+
outputFor(entry: FlatEntry): Promise<string | undefined>;
|
|
47
56
|
private load;
|
|
48
57
|
private fetchRun;
|
|
49
58
|
}
|
|
59
|
+
/**
|
|
60
|
+
* The name this class had before one chain could draw on more than one
|
|
61
|
+
* recording. Kept so nothing outside has to change its import.
|
|
62
|
+
*/
|
|
63
|
+
export { SourceRunCache as SourceRunOutputs };
|
|
50
64
|
//# sourceMappingURL=source-run.d.ts.map
|