@basein/runner 0.3.23 → 0.3.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/control/pin-guard.d.ts +0 -16
- package/dist/control/pin-guard.js +0 -36
- package/dist/control/server.d.ts +29 -12
- package/dist/control/server.js +201 -82
- package/dist/record/recorder.d.ts +14 -4
- package/dist/replay/bundle.d.ts +76 -0
- package/dist/replay/bundle.js +311 -14
- package/dist/replay/controller.d.ts +24 -10
- package/dist/replay/controller.js +29 -15
- package/dist/replay/plan.d.ts +15 -0
- package/dist/replay/plan.js +48 -30
- package/docs/calculatedReplay.md +32 -0
- package/docs/calculatedReplayGuide.md +1 -1
- package/package.json +1 -1
package/dist/replay/bundle.d.ts
CHANGED
|
@@ -13,6 +13,20 @@
|
|
|
13
13
|
* fresh execution (§8.1). It is surfaced in the header *and* per entry, because
|
|
14
14
|
* a bundle that quietly passed off a stale result as a live one would be the
|
|
15
15
|
* worst outcome available here.
|
|
16
|
+
*
|
|
17
|
+
* The agent pays to take the bundle in, so it carries nothing the model already
|
|
18
|
+
* has (docs/calculatedReplay.md, "How a bundle reads"): a result is its text,
|
|
19
|
+
* not MCP's JSON envelope, and consecutive calls to one tool share a section
|
|
20
|
+
* that says once what they share.
|
|
21
|
+
*
|
|
22
|
+
* And it carries detail only where detail is needed. A run that went entirely
|
|
23
|
+
* well is laid out briefly ({@link AssembleOptions.brief}): the agent's job is
|
|
24
|
+
* to see that the steps were right and say what was done, which takes every
|
|
25
|
+
* record and field a long result has — the IDs, dates, senders and names it
|
|
26
|
+
* checks the steps against — but not every word of its long text. Anything
|
|
27
|
+
* else — a step that broke, a run that stopped short, a recorded stand-in, a
|
|
28
|
+
* sub-task the agent carries on with — gets every result whole, because that
|
|
29
|
+
* is what the agent works from.
|
|
16
30
|
*/
|
|
17
31
|
/** `permissionDecisionReason` is fed to the model as text; keep denials bounded. */
|
|
18
32
|
export declare const MAX_REPLAY_REASON = 60000;
|
|
@@ -63,11 +77,73 @@ export interface AssembleOptions {
|
|
|
63
77
|
* leave the agent its tools.
|
|
64
78
|
*/
|
|
65
79
|
reportTool?: string;
|
|
80
|
+
/**
|
|
81
|
+
* Lay the run out briefly if it went entirely well, with `summary` — the
|
|
82
|
+
* plan's response model — in a few words above the footer. Asked for by a
|
|
83
|
+
* caller whose agent answers from the bundle and nothing else (`bir-
|
|
84
|
+
* scenario`'s run of a whole plan); `summary` may be absent or `{}`.
|
|
85
|
+
*
|
|
86
|
+
* Entirely well means every entry ran live and none reported an error, and
|
|
87
|
+
* the agent is to answer now: with `handover` or `subTask` set, a recorded
|
|
88
|
+
* entry, or a result marked as an error, the bundle is laid out in full as
|
|
89
|
+
* if this were not set. The summary is then the caller's to deliver.
|
|
90
|
+
*/
|
|
91
|
+
brief?: {
|
|
92
|
+
summary?: Record<string, unknown>;
|
|
93
|
+
};
|
|
66
94
|
}
|
|
67
95
|
/**
|
|
68
96
|
* Compose the bundle. `maxChars` is a hard ceiling on the returned string.
|
|
69
97
|
*/
|
|
70
98
|
export declare function assembleBundle(entries: readonly BundleEntry[], maxChars: number, opts?: AssembleOptions): string;
|
|
99
|
+
/**
|
|
100
|
+
* Whether a run went entirely well and the agent answers now — the one case a
|
|
101
|
+
* bundle may be brief ({@link AssembleOptions.brief}). A run that went wrong
|
|
102
|
+
* anywhere, stood on a recording, or is one the agent carries on from is what
|
|
103
|
+
* the agent works from next: it gets every result.
|
|
104
|
+
*/
|
|
105
|
+
export declare function wentWell(entries: readonly BundleEntry[], opts: Pick<AssembleOptions, "handover" | "subTask">): boolean;
|
|
106
|
+
/**
|
|
107
|
+
* JSON text with every string longer than 40 characters cut to its first 40
|
|
108
|
+
* and "…". Keys, numbers and short strings stay as they are, and an escape is
|
|
109
|
+
* never split. Works on JSON a parser would refuse — a tool's `NaN`, a result
|
|
110
|
+
* cut short — because it walks the text's string literals from the start, so
|
|
111
|
+
* it always knows which quote opens one: the text between two strings, a
|
|
112
|
+
* list of numbers say, is never taken for one. An unterminated string is left
|
|
113
|
+
* as it is.
|
|
114
|
+
*/
|
|
115
|
+
export declare function cutLongText(json: string): string;
|
|
116
|
+
/**
|
|
117
|
+
* `key = value` for each field, each value in a few words ({@link briefValue}),
|
|
118
|
+
* ended at a field's end with "…" rather than run past `max` characters.
|
|
119
|
+
*/
|
|
120
|
+
export declare function briefFields(fields: Record<string, unknown>, max: number): string;
|
|
121
|
+
/**
|
|
122
|
+
* A value of the plan's summary in a few words. Numbers, booleans and short
|
|
123
|
+
* strings whole; a long string cut; a list or object whose JSON is short —
|
|
124
|
+
* a few IDs, names or addresses — whole; a longer list counted; a longer
|
|
125
|
+
* object by its first fields, one level down, below which it is counted too.
|
|
126
|
+
* A summary's long lists are mostly the records the steps above already show.
|
|
127
|
+
*/
|
|
128
|
+
export declare function briefValue(value: unknown, depth?: number): string;
|
|
129
|
+
/**
|
|
130
|
+
* A tool result as the model should read it.
|
|
131
|
+
*
|
|
132
|
+
* Executors hand the bundle the JSON of MCP's `CallToolResult`. Laid out as it
|
|
133
|
+
* came, every result pays for the envelope —
|
|
134
|
+
* `{"content":[{"type":"text","text":"…"}],"isError":false}`, some fifty
|
|
135
|
+
* characters — and for a second layer of escaping: every quote in the text
|
|
136
|
+
* written `\"`, every backslash `\\`. Neither tells the model anything, and on
|
|
137
|
+
* a long plan they are a large share of what it reads (and of what the agent
|
|
138
|
+
* pays to put in its context). A result whose content is text and nothing else
|
|
139
|
+
* is shown as that text, marked when `isError` is true. Anything else — an
|
|
140
|
+
* image, a resource, structured content, an answer that is not a
|
|
141
|
+
* `CallToolResult` at all — is shown exactly as it came.
|
|
142
|
+
*/
|
|
143
|
+
export declare function resultText(response: string): {
|
|
144
|
+
text: string;
|
|
145
|
+
error: boolean;
|
|
146
|
+
};
|
|
71
147
|
/** Truncate to `max` chars, appending a marker naming how much was dropped. */
|
|
72
148
|
export declare function truncate(s: string, max: number): string;
|
|
73
149
|
//# sourceMappingURL=bundle.d.ts.map
|
package/dist/replay/bundle.js
CHANGED
|
@@ -13,6 +13,20 @@
|
|
|
13
13
|
* fresh execution (§8.1). It is surfaced in the header *and* per entry, because
|
|
14
14
|
* a bundle that quietly passed off a stale result as a live one would be the
|
|
15
15
|
* worst outcome available here.
|
|
16
|
+
*
|
|
17
|
+
* The agent pays to take the bundle in, so it carries nothing the model already
|
|
18
|
+
* has (docs/calculatedReplay.md, "How a bundle reads"): a result is its text,
|
|
19
|
+
* not MCP's JSON envelope, and consecutive calls to one tool share a section
|
|
20
|
+
* that says once what they share.
|
|
21
|
+
*
|
|
22
|
+
* And it carries detail only where detail is needed. A run that went entirely
|
|
23
|
+
* well is laid out briefly ({@link AssembleOptions.brief}): the agent's job is
|
|
24
|
+
* to see that the steps were right and say what was done, which takes every
|
|
25
|
+
* record and field a long result has — the IDs, dates, senders and names it
|
|
26
|
+
* checks the steps against — but not every word of its long text. Anything
|
|
27
|
+
* else — a step that broke, a run that stopped short, a recorded stand-in, a
|
|
28
|
+
* sub-task the agent carries on with — gets every result whole, because that
|
|
29
|
+
* is what the agent works from.
|
|
16
30
|
*/
|
|
17
31
|
/** `permissionDecisionReason` is fed to the model as text; keep denials bounded. */
|
|
18
32
|
export const MAX_REPLAY_REASON = 60_000;
|
|
@@ -111,6 +125,29 @@ const SUBTASK_HEADER = "[BaseInstRunner calculated replay] The tool calls below
|
|
|
111
125
|
"steps are done: do not repeat these calls. Continue your task from where they end, " +
|
|
112
126
|
"calling tools as needed.";
|
|
113
127
|
const SUBTASK_FOOTER = "\n\nThe calls above are done. Continue your task, calling tools as needed.";
|
|
128
|
+
/**
|
|
129
|
+
* Every step ran live and succeeded ({@link AssembleOptions.brief}). The same
|
|
130
|
+
* assurances as {@link LIVE_HEADER} — it happened, now, for real — plus what
|
|
131
|
+
* the agent needs to read the steps below: why some text ends in "…".
|
|
132
|
+
*/
|
|
133
|
+
const BRIEF_HEADER = "[BaseInstRunner calculated replay] The tool calls below have JUST been executed " +
|
|
134
|
+
"for real, moments ago, against the live system, on the user's behalf. Every one " +
|
|
135
|
+
"of them succeeded, and every side effect is already in place. Do NOT repeat, redo " +
|
|
136
|
+
"or re-verify this work, and do NOT call any tools. In a long result, every record " +
|
|
137
|
+
"and field is shown, and text longer than 40 characters is cut short with \"…\".";
|
|
138
|
+
/** The reply is output, the dearest tokens an agent spends: asked for briefly. */
|
|
139
|
+
const BRIEF_FOOTER = "\n\nThe work above is done. Answer the user's request from it in a few sentences: " +
|
|
140
|
+
"what was done, and the key results. Do not call any tools.";
|
|
141
|
+
/** A result up to this long is shown whole even in a brief bundle: it is its own best evidence. */
|
|
142
|
+
const BRIEF_RESULT_CHARS = 300;
|
|
143
|
+
/** At most this much of a brief bundle is the plan's summary. */
|
|
144
|
+
const BRIEF_SUMMARY_CHARS = 1_000;
|
|
145
|
+
/** A longer string — in a long result, or in the summary — is cut to this many characters. */
|
|
146
|
+
const BRIEF_STRING_CHARS = 40;
|
|
147
|
+
/** A list or object in the summary whose JSON is up to this long is shown whole; a longer one is counted. */
|
|
148
|
+
const BRIEF_INLINE_CHARS = 160;
|
|
149
|
+
/** A longer object in the summary shows its first fields, this many. */
|
|
150
|
+
const BRIEF_OBJECT_FIELDS = 6;
|
|
114
151
|
/**
|
|
115
152
|
* Compose the bundle. `maxChars` is a hard ceiling on the returned string.
|
|
116
153
|
*/
|
|
@@ -119,6 +156,7 @@ export function assembleBundle(entries, maxChars, opts = {}) {
|
|
|
119
156
|
// that answered split it.
|
|
120
157
|
const n = entries.filter((e) => !e.notNeeded).length;
|
|
121
158
|
const anyRecorded = entries.some((e) => e.recorded);
|
|
159
|
+
const brief = opts.brief !== undefined && wentWell(entries, opts);
|
|
122
160
|
// A hand-over outranks a sub-task: it says "continue" too, and it also has to
|
|
123
161
|
// say that the plan stopped short. A sub-task outranks the live and mixed
|
|
124
162
|
// headers, whose wording forbids further tool calls.
|
|
@@ -126,14 +164,16 @@ export function assembleBundle(entries, maxChars, opts = {}) {
|
|
|
126
164
|
? HANDOVER_HEADER
|
|
127
165
|
: opts.subTask
|
|
128
166
|
? SUBTASK_HEADER
|
|
129
|
-
: withReportException(anyRecorded ? MIXED_HEADER : LIVE_HEADER, opts.reportTool);
|
|
167
|
+
: withReportException(brief ? BRIEF_HEADER : anyRecorded ? MIXED_HEADER : LIVE_HEADER, opts.reportTool);
|
|
130
168
|
const footer = opts.handover
|
|
131
169
|
? HANDOVER_FOOTER
|
|
132
170
|
: opts.subTask
|
|
133
171
|
? SUBTASK_FOOTER
|
|
134
|
-
: withReportException(anyRecorded ? MIXED_FOOTER : LIVE_FOOTER, opts.reportTool);
|
|
172
|
+
: withReportException(brief ? BRIEF_FOOTER : anyRecorded ? MIXED_FOOTER : LIVE_FOOTER, opts.reportTool);
|
|
173
|
+
const summary = brief ? briefSummary(opts.brief?.summary) : "";
|
|
135
174
|
const fixedCost = header.length +
|
|
136
175
|
footer.length +
|
|
176
|
+
summary.length +
|
|
137
177
|
entries.reduce((sum, e) => sum +
|
|
138
178
|
PER_STEP_OVERHEAD +
|
|
139
179
|
e.toolName.length +
|
|
@@ -142,20 +182,277 @@ export function assembleBundle(entries, maxChars, opts = {}) {
|
|
|
142
182
|
(e.notNeeded ? PER_STEP_OVERHEAD + (testWords(e.notNeeded.text)?.length ?? 0) : 0), 0);
|
|
143
183
|
const responseBudget = Math.max(0, maxChars - fixedCost);
|
|
144
184
|
const perResponse = n > 0 ? Math.floor(responseBudget / n) : responseBudget;
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
185
|
+
const show = (e) => {
|
|
186
|
+
const result = resultText(e.response);
|
|
187
|
+
return {
|
|
188
|
+
text: brief ? briefResult(result.text, perResponse) : truncate(result.text, perResponse),
|
|
189
|
+
marks: resultMarks(e.recorded, result.error),
|
|
190
|
+
};
|
|
191
|
+
};
|
|
192
|
+
const sections = [];
|
|
193
|
+
for (const run of consecutiveRuns(entries)) {
|
|
194
|
+
if (run.length === 1) {
|
|
195
|
+
const { entry: e, index: i } = run[0];
|
|
196
|
+
if (e.notNeeded) {
|
|
197
|
+
// Never "skipped" or "failed": the step was left out on purpose, because
|
|
198
|
+
// this request does not need it, and nothing is missing for its absence.
|
|
199
|
+
const why = testWords(e.notNeeded.text);
|
|
200
|
+
sections.push(`\n\n--- Step ${i + 1}: ${e.toolName} (not needed) ---\n` +
|
|
201
|
+
`Not run: this request does not need it${why ? ` — it is only for when ${why}` : ""}.`);
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
const shown = show(e);
|
|
205
|
+
sections.push(`\n\n--- Step ${i + 1}: ${e.toolName} ---\n` +
|
|
206
|
+
`Call: ${e.input}\n` +
|
|
207
|
+
`Result${shown.marks}: ${shown.text}`);
|
|
208
|
+
continue;
|
|
209
|
+
}
|
|
210
|
+
sections.push(runSection(run, show));
|
|
211
|
+
}
|
|
212
|
+
return truncate(header + sections.join("") + summary + footer, maxChars);
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Whether a run went entirely well and the agent answers now — the one case a
|
|
216
|
+
* bundle may be brief ({@link AssembleOptions.brief}). A run that went wrong
|
|
217
|
+
* anywhere, stood on a recording, or is one the agent carries on from is what
|
|
218
|
+
* the agent works from next: it gets every result.
|
|
219
|
+
*/
|
|
220
|
+
export function wentWell(entries, opts) {
|
|
221
|
+
return (!opts.handover &&
|
|
222
|
+
!opts.subTask &&
|
|
223
|
+
entries.every((e) => e.notNeeded || (!e.recorded && !resultText(e.response).error)));
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* A result in a brief bundle. Whole when it is short. A long one that is JSON
|
|
227
|
+
* — a search's records, nearly always — keeps every record and field, and
|
|
228
|
+
* only its long text is cut ({@link cutLongText}): the agent checks the steps
|
|
229
|
+
* against the IDs, dates, senders and names, and those are short. A long
|
|
230
|
+
* result in words is shown whole: it may be the very answer.
|
|
231
|
+
*
|
|
232
|
+
* Measured on B2's length tests: showing only the values the plan took from a
|
|
233
|
+
* result instead left the agent unable to check the plan's choices — "I
|
|
234
|
+
* couldn't confirm these were the newest", "or who sent them" — in one reply
|
|
235
|
+
* in six, and once it reported a correct run as wrong.
|
|
236
|
+
*/
|
|
237
|
+
function briefResult(text, budget) {
|
|
238
|
+
if (text.length > BRIEF_RESULT_CHARS && /^\s*[[{]/.test(text))
|
|
239
|
+
return truncate(cutLongText(text), budget);
|
|
240
|
+
return truncate(text, budget);
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* JSON text with every string longer than 40 characters cut to its first 40
|
|
244
|
+
* and "…". Keys, numbers and short strings stay as they are, and an escape is
|
|
245
|
+
* never split. Works on JSON a parser would refuse — a tool's `NaN`, a result
|
|
246
|
+
* cut short — because it walks the text's string literals from the start, so
|
|
247
|
+
* it always knows which quote opens one: the text between two strings, a
|
|
248
|
+
* list of numbers say, is never taken for one. An unterminated string is left
|
|
249
|
+
* as it is.
|
|
250
|
+
*/
|
|
251
|
+
export function cutLongText(json) {
|
|
252
|
+
let out = "";
|
|
253
|
+
let at = 0;
|
|
254
|
+
while (at < json.length) {
|
|
255
|
+
const open = json.indexOf('"', at);
|
|
256
|
+
if (open < 0)
|
|
257
|
+
return out + json.slice(at);
|
|
258
|
+
out += json.slice(at, open + 1);
|
|
259
|
+
// Walk the string a unit at a time — a character, or an escape with what
|
|
260
|
+
// it escapes — to its closing quote, noting where the 40th unit ends.
|
|
261
|
+
let end = open + 1;
|
|
262
|
+
let units = 0;
|
|
263
|
+
let cutAt = -1;
|
|
264
|
+
while (end < json.length && json[end] !== '"') {
|
|
265
|
+
end += json[end] === "\\" ? 2 : 1;
|
|
266
|
+
units += 1;
|
|
267
|
+
if (units === BRIEF_STRING_CHARS)
|
|
268
|
+
cutAt = end;
|
|
269
|
+
}
|
|
270
|
+
if (end >= json.length)
|
|
271
|
+
return out + json.slice(open + 1);
|
|
272
|
+
out += units > BRIEF_STRING_CHARS ? `${json.slice(open + 1, cutAt)}…"` : json.slice(open + 1, end + 1);
|
|
273
|
+
at = end + 1;
|
|
274
|
+
}
|
|
275
|
+
return out;
|
|
276
|
+
}
|
|
277
|
+
/**
|
|
278
|
+
* The plan's summary (its response model) in a few words, or "" when it has
|
|
279
|
+
* nothing to add: none at all, or only the note that it was left out — which
|
|
280
|
+
* sends the model to the steps, and they are right above it.
|
|
281
|
+
*/
|
|
282
|
+
function briefSummary(model) {
|
|
283
|
+
if (!model)
|
|
284
|
+
return "";
|
|
285
|
+
const keys = Object.keys(model);
|
|
286
|
+
if (keys.length === 0 || keys.every((k) => k === "summaryLeftOut"))
|
|
287
|
+
return "";
|
|
288
|
+
return `\n\nThe plan's summary: ${briefFields(model, BRIEF_SUMMARY_CHARS)}`;
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* `key = value` for each field, each value in a few words ({@link briefValue}),
|
|
292
|
+
* ended at a field's end with "…" rather than run past `max` characters.
|
|
293
|
+
*/
|
|
294
|
+
export function briefFields(fields, max) {
|
|
295
|
+
let out = "";
|
|
296
|
+
for (const [key, value] of Object.entries(fields)) {
|
|
297
|
+
const part = `${key} = ${briefValue(value)}`;
|
|
298
|
+
const next = out ? `${out}; ${part}` : part;
|
|
299
|
+
if (next.length > max)
|
|
300
|
+
return out ? `${out}; …` : truncate(part, max);
|
|
301
|
+
out = next;
|
|
302
|
+
}
|
|
303
|
+
return out;
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* A value of the plan's summary in a few words. Numbers, booleans and short
|
|
307
|
+
* strings whole; a long string cut; a list or object whose JSON is short —
|
|
308
|
+
* a few IDs, names or addresses — whole; a longer list counted; a longer
|
|
309
|
+
* object by its first fields, one level down, below which it is counted too.
|
|
310
|
+
* A summary's long lists are mostly the records the steps above already show.
|
|
311
|
+
*/
|
|
312
|
+
export function briefValue(value, depth = 0) {
|
|
313
|
+
if (typeof value === "string") {
|
|
314
|
+
return JSON.stringify(value.length > BRIEF_STRING_CHARS ? `${value.slice(0, BRIEF_STRING_CHARS)}…` : value);
|
|
315
|
+
}
|
|
316
|
+
if (value === null || typeof value !== "object")
|
|
317
|
+
return String(value);
|
|
318
|
+
let whole;
|
|
319
|
+
try {
|
|
320
|
+
whole = JSON.stringify(value) ?? "";
|
|
321
|
+
}
|
|
322
|
+
catch {
|
|
323
|
+
return "[unserializable]";
|
|
324
|
+
}
|
|
325
|
+
if (whole.length <= BRIEF_INLINE_CHARS)
|
|
326
|
+
return whole;
|
|
327
|
+
if (Array.isArray(value))
|
|
328
|
+
return `${value.length} ${value.length === 1 ? "item" : "items"}`;
|
|
329
|
+
const fields = Object.entries(value);
|
|
330
|
+
if (depth >= 1)
|
|
331
|
+
return `{${fields.length} ${fields.length === 1 ? "field" : "fields"}}`;
|
|
332
|
+
const shown = fields.slice(0, BRIEF_OBJECT_FIELDS).map(([k, v]) => `${k}: ${briefValue(v, depth + 1)}`);
|
|
333
|
+
return `{${shown.join(", ")}${fields.length > BRIEF_OBJECT_FIELDS ? ", …" : ""}}`;
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* A tool result as the model should read it.
|
|
337
|
+
*
|
|
338
|
+
* Executors hand the bundle the JSON of MCP's `CallToolResult`. Laid out as it
|
|
339
|
+
* came, every result pays for the envelope —
|
|
340
|
+
* `{"content":[{"type":"text","text":"…"}],"isError":false}`, some fifty
|
|
341
|
+
* characters — and for a second layer of escaping: every quote in the text
|
|
342
|
+
* written `\"`, every backslash `\\`. Neither tells the model anything, and on
|
|
343
|
+
* a long plan they are a large share of what it reads (and of what the agent
|
|
344
|
+
* pays to put in its context). A result whose content is text and nothing else
|
|
345
|
+
* is shown as that text, marked when `isError` is true. Anything else — an
|
|
346
|
+
* image, a resource, structured content, an answer that is not a
|
|
347
|
+
* `CallToolResult` at all — is shown exactly as it came.
|
|
348
|
+
*/
|
|
349
|
+
export function resultText(response) {
|
|
350
|
+
const asIs = { text: response, error: false };
|
|
351
|
+
let parsed;
|
|
352
|
+
try {
|
|
353
|
+
parsed = JSON.parse(response);
|
|
354
|
+
}
|
|
355
|
+
catch {
|
|
356
|
+
return asIs;
|
|
357
|
+
}
|
|
358
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
|
|
359
|
+
return asIs;
|
|
360
|
+
const result = parsed;
|
|
361
|
+
if (!Object.keys(result).every((k) => k === "content" || k === "isError"))
|
|
362
|
+
return asIs;
|
|
363
|
+
if (result.isError !== undefined && typeof result.isError !== "boolean")
|
|
364
|
+
return asIs;
|
|
365
|
+
if (!Array.isArray(result.content) || result.content.length === 0)
|
|
366
|
+
return asIs;
|
|
367
|
+
const texts = [];
|
|
368
|
+
for (const item of result.content) {
|
|
369
|
+
if (!item || typeof item !== "object" || Array.isArray(item))
|
|
370
|
+
return asIs;
|
|
371
|
+
const part = item;
|
|
372
|
+
if (part.type !== "text" || typeof part.text !== "string")
|
|
373
|
+
return asIs;
|
|
374
|
+
if (!Object.keys(part).every((k) => k === "type" || k === "text"))
|
|
375
|
+
return asIs;
|
|
376
|
+
texts.push(part.text);
|
|
377
|
+
}
|
|
378
|
+
return { text: texts.join("\n"), error: result.isError === true };
|
|
379
|
+
}
|
|
380
|
+
/** `(recorded)`, `(error)` or `(recorded, error)` after a result's label; "" for a plain live result. */
|
|
381
|
+
function resultMarks(recorded, error) {
|
|
382
|
+
const marks = [recorded ? "recorded" : "", error ? "error" : ""].filter(Boolean);
|
|
383
|
+
return marks.length > 0 ? ` (${marks.join(", ")})` : "";
|
|
384
|
+
}
|
|
385
|
+
/**
|
|
386
|
+
* The entries cut into runs of consecutive calls to one tool. A long plan is
|
|
387
|
+
* mostly such runs — eleven look-ups, then thirty-two forwards — and laying
|
|
388
|
+
* each step out on its own repeats the tool's name and the section frame every
|
|
389
|
+
* time. A step left out stands alone, so its "not needed" line reads as before.
|
|
390
|
+
*/
|
|
391
|
+
function consecutiveRuns(entries) {
|
|
392
|
+
const runs = [];
|
|
393
|
+
entries.forEach((entry, index) => {
|
|
394
|
+
const last = runs.at(-1);
|
|
395
|
+
const prev = last?.at(-1)?.entry;
|
|
396
|
+
if (last && prev && !prev.notNeeded && !entry.notNeeded && prev.toolName === entry.toolName) {
|
|
397
|
+
last.push({ entry, index });
|
|
398
|
+
}
|
|
399
|
+
else {
|
|
400
|
+
runs.push([{ entry, index }]);
|
|
152
401
|
}
|
|
153
|
-
const response = truncate(e.response, perResponse);
|
|
154
|
-
return (`\n\n--- Step ${i + 1}: ${e.toolName} ---\n` +
|
|
155
|
-
`Call: ${e.input}\n` +
|
|
156
|
-
`Result${e.recorded ? " (recorded)" : ""}: ${response}`);
|
|
157
402
|
});
|
|
158
|
-
return
|
|
403
|
+
return runs;
|
|
404
|
+
}
|
|
405
|
+
/**
|
|
406
|
+
* Two or more consecutive calls to one tool, as one section: one line per call,
|
|
407
|
+
* numbered as its step, with its result after the arrow. What every call shares
|
|
408
|
+
* is said once above the calls — input fields with the same value in each, and
|
|
409
|
+
* the answer, when every call got the same one. Each result is shown as `show`
|
|
410
|
+
* shows a step on its own, budget included.
|
|
411
|
+
*/
|
|
412
|
+
function runSection(run, show) {
|
|
413
|
+
const first = run[0];
|
|
414
|
+
const head = `\n\n--- Steps ${first.index + 1}-${run[run.length - 1].index + 1}: ${first.entry.toolName} (${run.length} calls) ---`;
|
|
415
|
+
const inputs = sharedInput(run.map(({ entry }) => entry.input));
|
|
416
|
+
const shown = run.map(({ entry, index }, i) => ({ step: index + 1, input: inputs.own[i], ...show(entry) }));
|
|
417
|
+
const lines = inputs.shared === null ? "" : `\nIn every call: ${inputs.shared}`;
|
|
418
|
+
const same = shown.every((s) => s.text === shown[0].text && s.marks === shown[0].marks);
|
|
419
|
+
if (same) {
|
|
420
|
+
return (`${head}${lines}\nEvery call's result${shown[0].marks}: ${shown[0].text}` +
|
|
421
|
+
shown.map((s) => `\n${s.step}: ${s.input}`).join(""));
|
|
422
|
+
}
|
|
423
|
+
return head + lines + shown.map((s) => `\n${s.step}${s.marks}: ${s.input} -> ${s.text}`).join("");
|
|
424
|
+
}
|
|
425
|
+
/**
|
|
426
|
+
* The input fields a run of calls all share, said once, and what is left of
|
|
427
|
+
* each call's input. A forward sends thirty-two emails to one recipient; the
|
|
428
|
+
* recipient need not be read thirty-two times. Only plain JSON objects are
|
|
429
|
+
* split, and only on fields present in every call with the same value; an
|
|
430
|
+
* input that was capped, or is not an object, leaves the run as it came.
|
|
431
|
+
*/
|
|
432
|
+
function sharedInput(inputs) {
|
|
433
|
+
const asCame = { shared: null, own: [...inputs] };
|
|
434
|
+
const objects = [];
|
|
435
|
+
for (const input of inputs) {
|
|
436
|
+
let parsed;
|
|
437
|
+
try {
|
|
438
|
+
parsed = JSON.parse(input);
|
|
439
|
+
}
|
|
440
|
+
catch {
|
|
441
|
+
return asCame;
|
|
442
|
+
}
|
|
443
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
|
|
444
|
+
return asCame;
|
|
445
|
+
objects.push(parsed);
|
|
446
|
+
}
|
|
447
|
+
const value = (o, key) => JSON.stringify(o[key]);
|
|
448
|
+
const common = Object.keys(objects[0]).filter((key) => objects.every((o) => Object.hasOwn(o, key) && value(o, key) === value(objects[0], key)));
|
|
449
|
+
if (common.length === 0)
|
|
450
|
+
return asCame;
|
|
451
|
+
const pick = (o, keep) => JSON.stringify(Object.fromEntries(Object.entries(o).filter(([key]) => keep(key))));
|
|
452
|
+
return {
|
|
453
|
+
shared: pick(objects[0], (key) => common.includes(key)),
|
|
454
|
+
own: objects.map((o) => pick(o, (key) => !common.includes(key))),
|
|
455
|
+
};
|
|
159
456
|
}
|
|
160
457
|
/** Truncate to `max` chars, appending a marker naming how much was dropped. */
|
|
161
458
|
export function truncate(s, max) {
|
|
@@ -199,7 +199,8 @@ export declare function clampDirectRun(budgets: ReplayBudgets, logicMs?: number)
|
|
|
199
199
|
*
|
|
200
200
|
* - **ok** — the plan ran to its end: `{ ok: true, text, responseModel, steps,
|
|
201
201
|
* partial: false }`. `text` is the bundle; `responseModel` what the
|
|
202
|
-
* scenario's own logic built from it (`{}` when it has none
|
|
202
|
+
* scenario's own logic built from it (`{}` when it has none, and when the
|
|
203
|
+
* bundle is brief and carries it already — plan.ts, `DriveOptions.brief`).
|
|
203
204
|
* - **partial** — the plan stopped part-way after running steps here: `{ ok:
|
|
204
205
|
* true, text, responseModel: {}, steps, partial }`. `text` opens with the
|
|
205
206
|
* hand-over note (text beginning `[BaseInstRunner calculated replay]`) and
|
|
@@ -608,13 +609,21 @@ export type DeclineCode = "replay_disabled" | "not_ready" | "known_bad_first_ste
|
|
|
608
609
|
*/
|
|
609
610
|
| "match_late"
|
|
610
611
|
/**
|
|
611
|
-
* A continuation (the service's fallbk.md Part 3)
|
|
612
|
-
*
|
|
613
|
-
*
|
|
614
|
-
*
|
|
612
|
+
* A continuation (the service's fallbk.md Part 3) armed after the hand-over
|
|
613
|
+
* and told to the model, which never ran it: `run_scenario` was never asked
|
|
614
|
+
* for — another call of the same message came first, the model did the
|
|
615
|
+
* rest its own way, or the call was refused. Set by the control server.
|
|
616
|
+
* (Runner 0.3.23 armed a continuation only when the model's call was its
|
|
617
|
+
* first step and declined it otherwise as `not_first_step`. From 0.3.24 it
|
|
618
|
+
* arms on the model's first call after the hand-over, whatever that is.)
|
|
619
|
+
*/
|
|
620
|
+
| "not_taken"
|
|
621
|
+
/**
|
|
622
|
+
* A continuation whose own code for working out its values (`paramsLogic`)
|
|
623
|
+
* failed on this request, so none of its steps could be computed. Set by
|
|
615
624
|
* the control server.
|
|
616
625
|
*/
|
|
617
|
-
| "
|
|
626
|
+
| "params_failed";
|
|
618
627
|
/**
|
|
619
628
|
* What a continuation continues (the service's fallbk.md Part 3): the plan
|
|
620
629
|
* that handed over, the step it stopped in front of, and the recording of the
|
|
@@ -712,10 +721,9 @@ export declare class ReplayController {
|
|
|
712
721
|
lateMatch(match: RunMatchLike, host?: HostContext): ReplayState;
|
|
713
722
|
/**
|
|
714
723
|
* The state of a continuation handed out and not armed (the service's
|
|
715
|
-
* fallbk.md Part 3): declined before any gate ran —
|
|
716
|
-
*
|
|
717
|
-
*
|
|
718
|
-
* ticket's report, with the reason.
|
|
724
|
+
* fallbk.md Part 3): declined before any gate ran — the service's answer
|
|
725
|
+
* came too late (`match_late`). Like {@link lateMatch}, it exists only for
|
|
726
|
+
* its ticket's report, with the reason.
|
|
719
727
|
*/
|
|
720
728
|
declinedContinuation(match: RunMatchLike, code: DeclineCode, continuation: ContinuationOf): ReplayState;
|
|
721
729
|
/**
|
|
@@ -1015,6 +1023,12 @@ export declare class ReplayController {
|
|
|
1015
1023
|
* them would be left out of the turn's saving.
|
|
1016
1024
|
*/
|
|
1017
1025
|
chainDeriveUsd?: number;
|
|
1026
|
+
/**
|
|
1027
|
+
* A continuation whose plan before it ran no step (its report is
|
|
1028
|
+
* `failed`, which books no saving): this one books its own, against its
|
|
1029
|
+
* own baseline, so the turn's saving is booked somewhere.
|
|
1030
|
+
*/
|
|
1031
|
+
booksOwnSaving?: boolean;
|
|
1018
1032
|
}): ExecutionReport | undefined;
|
|
1019
1033
|
/**
|
|
1020
1034
|
* The step a failed replay is fairly blamed on: the first whose own logic
|
|
@@ -376,10 +376,9 @@ export class ReplayController {
|
|
|
376
376
|
}
|
|
377
377
|
/**
|
|
378
378
|
* The state of a continuation handed out and not armed (the service's
|
|
379
|
-
* fallbk.md Part 3): declined before any gate ran —
|
|
380
|
-
*
|
|
381
|
-
*
|
|
382
|
-
* ticket's report, with the reason.
|
|
379
|
+
* fallbk.md Part 3): declined before any gate ran — the service's answer
|
|
380
|
+
* came too late (`match_late`). Like {@link lateMatch}, it exists only for
|
|
381
|
+
* its ticket's report, with the reason.
|
|
383
382
|
*/
|
|
384
383
|
declinedContinuation(match, code, continuation) {
|
|
385
384
|
const state = this.blankState(match, new Set(), "intent");
|
|
@@ -392,9 +391,7 @@ export class ReplayController {
|
|
|
392
391
|
run: match.runId,
|
|
393
392
|
scenario: match.scenarioId ?? undefined,
|
|
394
393
|
continues: `${continuation.ofScenarioId}@${continuation.stepIndex}`,
|
|
395
|
-
why:
|
|
396
|
-
? "a continuation starts only on its first step, and the model's call after the hand-over is another one"
|
|
397
|
-
: "the continuation came after the call it could have replaced had gone ahead",
|
|
394
|
+
why: "the continuation came after the call it could have replaced had gone ahead",
|
|
398
395
|
});
|
|
399
396
|
return state;
|
|
400
397
|
}
|
|
@@ -1760,7 +1757,15 @@ export class ReplayController {
|
|
|
1760
1757
|
const deadline = Date.now() + this.budgets.planMs;
|
|
1761
1758
|
let result;
|
|
1762
1759
|
try {
|
|
1763
|
-
result = await plan.runToCompletion(this.executeStep(state, { deadline }), this.recordedSource(state), this.observeStep(state), deadline, MAX_REPLAY_REASON, {
|
|
1760
|
+
result = await plan.runToCompletion(this.executeStep(state, { deadline }), this.recordedSource(state), this.observeStep(state), deadline, MAX_REPLAY_REASON, {
|
|
1761
|
+
stopOnFailure: true,
|
|
1762
|
+
recordedFallback: this.recordedFallback,
|
|
1763
|
+
signal,
|
|
1764
|
+
reportTool: this.reportToolFor(state),
|
|
1765
|
+
// The agent answers from the bundle alone: a run that went entirely
|
|
1766
|
+
// well is laid out briefly, with the plan's summary in it.
|
|
1767
|
+
brief: true,
|
|
1768
|
+
});
|
|
1764
1769
|
}
|
|
1765
1770
|
catch (err) {
|
|
1766
1771
|
// The scenario's own logic failed. Retire and let the model do the work.
|
|
@@ -1878,12 +1883,18 @@ export class ReplayController {
|
|
|
1878
1883
|
}
|
|
1879
1884
|
// Live steps only, as above: a recorded stand-in did no work here.
|
|
1880
1885
|
state.stepsPinned += result.executed;
|
|
1886
|
+
// A brief bundle carries the plan's summary itself, in a few words, so it
|
|
1887
|
+
// is not delivered a second time beside it.
|
|
1881
1888
|
let responseModel = {};
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1889
|
+
if (result.summaryError)
|
|
1890
|
+
logDetail("replay.response_model_failed", { error: result.summaryError });
|
|
1891
|
+
if (!result.brief) {
|
|
1892
|
+
try {
|
|
1893
|
+
responseModel = plan.responseModel();
|
|
1894
|
+
}
|
|
1895
|
+
catch (err) {
|
|
1896
|
+
logDetail("replay.response_model_failed", { error: errText(err) });
|
|
1897
|
+
}
|
|
1887
1898
|
}
|
|
1888
1899
|
// A plan whose tools errored ran to the end and did nothing. `steered_full`
|
|
1889
1900
|
// would book the largest saving available for it; `failed` books a baseline
|
|
@@ -1998,8 +2009,11 @@ export class ReplayController {
|
|
|
1998
2009
|
// The agent did not take the plan's result and redid the task (AC1).
|
|
1999
2010
|
...(d.redoneBy ? { redoneBy: d.redoneBy } : {}),
|
|
2000
2011
|
// A continuation (the service's fallbk.md Part 3): the plan it continued
|
|
2001
|
-
// books the whole turn, so this report books no saving of its own
|
|
2002
|
-
|
|
2012
|
+
// books the whole turn, so this report books no saving of its own —
|
|
2013
|
+
// unless that plan ran nothing and books none.
|
|
2014
|
+
...(state.continuation && !d.booksOwnSaving
|
|
2015
|
+
? { continues: { scenarioId: state.continuation.ofScenarioId } }
|
|
2016
|
+
: {}),
|
|
2003
2017
|
// A turn in which another plan ran is not a measurement of what this task
|
|
2004
2018
|
// costs unaided, so it is no baseline sample (segmented.md R-MONEY-5).
|
|
2005
2019
|
// This covers two plans that both armed, and — the case that was silently
|
package/dist/replay/plan.d.ts
CHANGED
|
@@ -125,6 +125,14 @@ export interface DriveOptions {
|
|
|
125
125
|
* they always were (bundle.ts, `AssembleOptions.reportTool`).
|
|
126
126
|
*/
|
|
127
127
|
reportTool?: string;
|
|
128
|
+
/**
|
|
129
|
+
* Lay a run that went entirely well out briefly, with the plan's summary in
|
|
130
|
+
* it (bundle.ts, `AssembleOptions.brief`), and say so in the result's
|
|
131
|
+
* `brief`. For a caller whose agent answers from the bundle alone —
|
|
132
|
+
* `bir-scenario`'s run of a whole plan — not for an operator's `bir replay`,
|
|
133
|
+
* which reads every result. A run that went wrong is laid out in full.
|
|
134
|
+
*/
|
|
135
|
+
brief?: boolean;
|
|
128
136
|
}
|
|
129
137
|
/**
|
|
130
138
|
* Supplies a step's recorded output when its tool cannot run here (§8.1).
|
|
@@ -213,6 +221,13 @@ export interface ComposeResult {
|
|
|
213
221
|
* the end of the planned steps.
|
|
214
222
|
*/
|
|
215
223
|
endedAt: number;
|
|
224
|
+
/**
|
|
225
|
+
* The bundle is brief ({@link DriveOptions.brief}), and the plan's summary,
|
|
226
|
+
* when it has one, is in `text`: the caller must not deliver it again.
|
|
227
|
+
*/
|
|
228
|
+
brief?: boolean;
|
|
229
|
+
/** Building the summary for a brief bundle threw: why. The bundle went without it. */
|
|
230
|
+
summaryError?: string;
|
|
216
231
|
}
|
|
217
232
|
/** Where, and why, a run stopped short of the chain's end (fallbk.md). */
|
|
218
233
|
export interface PlanStop {
|