@basein/runner 0.3.24 → 0.3.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/replay/bundle.d.ts +76 -0
- package/dist/replay/bundle.js +311 -14
- package/dist/replay/controller.d.ts +2 -1
- package/dist/replay/controller.js +20 -6
- package/dist/replay/plan.d.ts +15 -0
- package/dist/replay/plan.js +48 -30
- package/docs/calculatedReplay.md +32 -0
- package/package.json +1 -1
package/dist/replay/bundle.d.ts
CHANGED
|
@@ -13,6 +13,20 @@
|
|
|
13
13
|
* fresh execution (§8.1). It is surfaced in the header *and* per entry, because
|
|
14
14
|
* a bundle that quietly passed off a stale result as a live one would be the
|
|
15
15
|
* worst outcome available here.
|
|
16
|
+
*
|
|
17
|
+
* The agent pays to take the bundle in, so it carries nothing the model already
|
|
18
|
+
* has (docs/calculatedReplay.md, "How a bundle reads"): a result is its text,
|
|
19
|
+
* not MCP's JSON envelope, and consecutive calls to one tool share a section
|
|
20
|
+
* that says once what they share.
|
|
21
|
+
*
|
|
22
|
+
* And it carries detail only where detail is needed. A run that went entirely
|
|
23
|
+
* well is laid out briefly ({@link AssembleOptions.brief}): the agent's job is
|
|
24
|
+
* to see that the steps were right and say what was done, which takes every
|
|
25
|
+
* record and field a long result has — the IDs, dates, senders and names it
|
|
26
|
+
* checks the steps against — but not every word of its long text. Anything
|
|
27
|
+
* else — a step that broke, a run that stopped short, a recorded stand-in, a
|
|
28
|
+
* sub-task the agent carries on with — gets every result whole, because that
|
|
29
|
+
* is what the agent works from.
|
|
16
30
|
*/
|
|
17
31
|
/** `permissionDecisionReason` is fed to the model as text; keep denials bounded. */
|
|
18
32
|
export declare const MAX_REPLAY_REASON = 60000;
|
|
@@ -63,11 +77,73 @@ export interface AssembleOptions {
|
|
|
63
77
|
* leave the agent its tools.
|
|
64
78
|
*/
|
|
65
79
|
reportTool?: string;
|
|
80
|
+
/**
|
|
81
|
+
* Lay the run out briefly if it went entirely well, with `summary` — the
|
|
82
|
+
* plan's response model — in a few words above the footer. Asked for by a
|
|
83
|
+
* caller whose agent answers from the bundle and nothing else (`bir-
|
|
84
|
+
* scenario`'s run of a whole plan); `summary` may be absent or `{}`.
|
|
85
|
+
*
|
|
86
|
+
* Entirely well means every entry ran live and none reported an error, and
|
|
87
|
+
* the agent is to answer now: with `handover` or `subTask` set, a recorded
|
|
88
|
+
* entry, or a result marked as an error, the bundle is laid out in full as
|
|
89
|
+
* if this were not set. The summary is then the caller's to deliver.
|
|
90
|
+
*/
|
|
91
|
+
brief?: {
|
|
92
|
+
summary?: Record<string, unknown>;
|
|
93
|
+
};
|
|
66
94
|
}
|
|
67
95
|
/**
|
|
68
96
|
* Compose the bundle. `maxChars` is a hard ceiling on the returned string.
|
|
69
97
|
*/
|
|
70
98
|
export declare function assembleBundle(entries: readonly BundleEntry[], maxChars: number, opts?: AssembleOptions): string;
|
|
99
|
+
/**
|
|
100
|
+
* Whether a run went entirely well and the agent answers now — the one case a
|
|
101
|
+
* bundle may be brief ({@link AssembleOptions.brief}). A run that went wrong
|
|
102
|
+
* anywhere, stood on a recording, or is one the agent carries on from is what
|
|
103
|
+
* the agent works from next: it gets every result.
|
|
104
|
+
*/
|
|
105
|
+
export declare function wentWell(entries: readonly BundleEntry[], opts: Pick<AssembleOptions, "handover" | "subTask">): boolean;
|
|
106
|
+
/**
|
|
107
|
+
* JSON text with every string longer than 40 characters cut to its first 40
|
|
108
|
+
* and "…". Keys, numbers and short strings stay as they are, and an escape is
|
|
109
|
+
* never split. Works on JSON a parser would refuse — a tool's `NaN`, a result
|
|
110
|
+
* cut short — because it walks the text's string literals from the start, so
|
|
111
|
+
* it always knows which quote opens one: the text between two strings, a
|
|
112
|
+
* list of numbers say, is never taken for one. An unterminated string is left
|
|
113
|
+
* as it is.
|
|
114
|
+
*/
|
|
115
|
+
export declare function cutLongText(json: string): string;
|
|
116
|
+
/**
|
|
117
|
+
* `key = value` for each field, each value in a few words ({@link briefValue}),
|
|
118
|
+
* ended at a field's end with "…" rather than run past `max` characters.
|
|
119
|
+
*/
|
|
120
|
+
export declare function briefFields(fields: Record<string, unknown>, max: number): string;
|
|
121
|
+
/**
|
|
122
|
+
* A value of the plan's summary in a few words. Numbers, booleans and short
|
|
123
|
+
* strings whole; a long string cut; a list or object whose JSON is short —
|
|
124
|
+
* a few IDs, names or addresses — whole; a longer list counted; a longer
|
|
125
|
+
* object by its first fields, one level down, below which it is counted too.
|
|
126
|
+
* A summary's long lists are mostly the records the steps above already show.
|
|
127
|
+
*/
|
|
128
|
+
export declare function briefValue(value: unknown, depth?: number): string;
|
|
129
|
+
/**
|
|
130
|
+
* A tool result as the model should read it.
|
|
131
|
+
*
|
|
132
|
+
* Executors hand the bundle the JSON of MCP's `CallToolResult`. Laid out as it
|
|
133
|
+
* came, every result pays for the envelope —
|
|
134
|
+
* `{"content":[{"type":"text","text":"…"}],"isError":false}`, some fifty
|
|
135
|
+
* characters — and for a second layer of escaping: every quote in the text
|
|
136
|
+
* written `\"`, every backslash `\\`. Neither tells the model anything, and on
|
|
137
|
+
* a long plan they are a large share of what it reads (and of what the agent
|
|
138
|
+
* pays to put in its context). A result whose content is text and nothing else
|
|
139
|
+
* is shown as that text, marked when `isError` is true. Anything else — an
|
|
140
|
+
* image, a resource, structured content, an answer that is not a
|
|
141
|
+
* `CallToolResult` at all — is shown exactly as it came.
|
|
142
|
+
*/
|
|
143
|
+
export declare function resultText(response: string): {
|
|
144
|
+
text: string;
|
|
145
|
+
error: boolean;
|
|
146
|
+
};
|
|
71
147
|
/** Truncate to `max` chars, appending a marker naming how much was dropped. */
|
|
72
148
|
export declare function truncate(s: string, max: number): string;
|
|
73
149
|
//# sourceMappingURL=bundle.d.ts.map
|
package/dist/replay/bundle.js
CHANGED
|
@@ -13,6 +13,20 @@
|
|
|
13
13
|
* fresh execution (§8.1). It is surfaced in the header *and* per entry, because
|
|
14
14
|
* a bundle that quietly passed off a stale result as a live one would be the
|
|
15
15
|
* worst outcome available here.
|
|
16
|
+
*
|
|
17
|
+
* The agent pays to take the bundle in, so it carries nothing the model already
|
|
18
|
+
* has (docs/calculatedReplay.md, "How a bundle reads"): a result is its text,
|
|
19
|
+
* not MCP's JSON envelope, and consecutive calls to one tool share a section
|
|
20
|
+
* that says once what they share.
|
|
21
|
+
*
|
|
22
|
+
* And it carries detail only where detail is needed. A run that went entirely
|
|
23
|
+
* well is laid out briefly ({@link AssembleOptions.brief}): the agent's job is
|
|
24
|
+
* to see that the steps were right and say what was done, which takes every
|
|
25
|
+
* record and field a long result has — the IDs, dates, senders and names it
|
|
26
|
+
* checks the steps against — but not every word of its long text. Anything
|
|
27
|
+
* else — a step that broke, a run that stopped short, a recorded stand-in, a
|
|
28
|
+
* sub-task the agent carries on with — gets every result whole, because that
|
|
29
|
+
* is what the agent works from.
|
|
16
30
|
*/
|
|
17
31
|
/** `permissionDecisionReason` is fed to the model as text; keep denials bounded. */
|
|
18
32
|
export const MAX_REPLAY_REASON = 60_000;
|
|
@@ -111,6 +125,29 @@ const SUBTASK_HEADER = "[BaseInstRunner calculated replay] The tool calls below
|
|
|
111
125
|
"steps are done: do not repeat these calls. Continue your task from where they end, " +
|
|
112
126
|
"calling tools as needed.";
|
|
113
127
|
const SUBTASK_FOOTER = "\n\nThe calls above are done. Continue your task, calling tools as needed.";
|
|
128
|
+
/**
|
|
129
|
+
* Every step ran live and succeeded ({@link AssembleOptions.brief}). The same
|
|
130
|
+
* assurances as {@link LIVE_HEADER} — it happened, now, for real — plus what
|
|
131
|
+
* the agent needs to read the steps below: why some text ends in "…".
|
|
132
|
+
*/
|
|
133
|
+
const BRIEF_HEADER = "[BaseInstRunner calculated replay] The tool calls below have JUST been executed " +
|
|
134
|
+
"for real, moments ago, against the live system, on the user's behalf. Every one " +
|
|
135
|
+
"of them succeeded, and every side effect is already in place. Do NOT repeat, redo " +
|
|
136
|
+
"or re-verify this work, and do NOT call any tools. In a long result, every record " +
|
|
137
|
+
"and field is shown, and text longer than 40 characters is cut short with \"…\".";
|
|
138
|
+
/** The reply is output, the dearest tokens an agent spends: asked for briefly. */
|
|
139
|
+
const BRIEF_FOOTER = "\n\nThe work above is done. Answer the user's request from it in a few sentences: " +
|
|
140
|
+
"what was done, and the key results. Do not call any tools.";
|
|
141
|
+
/** A result up to this long is shown whole even in a brief bundle: it is its own best evidence. */
|
|
142
|
+
const BRIEF_RESULT_CHARS = 300;
|
|
143
|
+
/** At most this much of a brief bundle is the plan's summary. */
|
|
144
|
+
const BRIEF_SUMMARY_CHARS = 1_000;
|
|
145
|
+
/** A longer string — in a long result, or in the summary — is cut to this many characters. */
|
|
146
|
+
const BRIEF_STRING_CHARS = 40;
|
|
147
|
+
/** A list or object in the summary whose JSON is up to this long is shown whole; a longer one is counted. */
|
|
148
|
+
const BRIEF_INLINE_CHARS = 160;
|
|
149
|
+
/** A longer object in the summary shows its first fields, this many. */
|
|
150
|
+
const BRIEF_OBJECT_FIELDS = 6;
|
|
114
151
|
/**
|
|
115
152
|
* Compose the bundle. `maxChars` is a hard ceiling on the returned string.
|
|
116
153
|
*/
|
|
@@ -119,6 +156,7 @@ export function assembleBundle(entries, maxChars, opts = {}) {
|
|
|
119
156
|
// that answered split it.
|
|
120
157
|
const n = entries.filter((e) => !e.notNeeded).length;
|
|
121
158
|
const anyRecorded = entries.some((e) => e.recorded);
|
|
159
|
+
const brief = opts.brief !== undefined && wentWell(entries, opts);
|
|
122
160
|
// A hand-over outranks a sub-task: it says "continue" too, and it also has to
|
|
123
161
|
// say that the plan stopped short. A sub-task outranks the live and mixed
|
|
124
162
|
// headers, whose wording forbids further tool calls.
|
|
@@ -126,14 +164,16 @@ export function assembleBundle(entries, maxChars, opts = {}) {
|
|
|
126
164
|
? HANDOVER_HEADER
|
|
127
165
|
: opts.subTask
|
|
128
166
|
? SUBTASK_HEADER
|
|
129
|
-
: withReportException(anyRecorded ? MIXED_HEADER : LIVE_HEADER, opts.reportTool);
|
|
167
|
+
: withReportException(brief ? BRIEF_HEADER : anyRecorded ? MIXED_HEADER : LIVE_HEADER, opts.reportTool);
|
|
130
168
|
const footer = opts.handover
|
|
131
169
|
? HANDOVER_FOOTER
|
|
132
170
|
: opts.subTask
|
|
133
171
|
? SUBTASK_FOOTER
|
|
134
|
-
: withReportException(anyRecorded ? MIXED_FOOTER : LIVE_FOOTER, opts.reportTool);
|
|
172
|
+
: withReportException(brief ? BRIEF_FOOTER : anyRecorded ? MIXED_FOOTER : LIVE_FOOTER, opts.reportTool);
|
|
173
|
+
const summary = brief ? briefSummary(opts.brief?.summary) : "";
|
|
135
174
|
const fixedCost = header.length +
|
|
136
175
|
footer.length +
|
|
176
|
+
summary.length +
|
|
137
177
|
entries.reduce((sum, e) => sum +
|
|
138
178
|
PER_STEP_OVERHEAD +
|
|
139
179
|
e.toolName.length +
|
|
@@ -142,20 +182,277 @@ export function assembleBundle(entries, maxChars, opts = {}) {
|
|
|
142
182
|
(e.notNeeded ? PER_STEP_OVERHEAD + (testWords(e.notNeeded.text)?.length ?? 0) : 0), 0);
|
|
143
183
|
const responseBudget = Math.max(0, maxChars - fixedCost);
|
|
144
184
|
const perResponse = n > 0 ? Math.floor(responseBudget / n) : responseBudget;
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
185
|
+
const show = (e) => {
|
|
186
|
+
const result = resultText(e.response);
|
|
187
|
+
return {
|
|
188
|
+
text: brief ? briefResult(result.text, perResponse) : truncate(result.text, perResponse),
|
|
189
|
+
marks: resultMarks(e.recorded, result.error),
|
|
190
|
+
};
|
|
191
|
+
};
|
|
192
|
+
const sections = [];
|
|
193
|
+
for (const run of consecutiveRuns(entries)) {
|
|
194
|
+
if (run.length === 1) {
|
|
195
|
+
const { entry: e, index: i } = run[0];
|
|
196
|
+
if (e.notNeeded) {
|
|
197
|
+
// Never "skipped" or "failed": the step was left out on purpose, because
|
|
198
|
+
// this request does not need it, and nothing is missing for its absence.
|
|
199
|
+
const why = testWords(e.notNeeded.text);
|
|
200
|
+
sections.push(`\n\n--- Step ${i + 1}: ${e.toolName} (not needed) ---\n` +
|
|
201
|
+
`Not run: this request does not need it${why ? ` — it is only for when ${why}` : ""}.`);
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
const shown = show(e);
|
|
205
|
+
sections.push(`\n\n--- Step ${i + 1}: ${e.toolName} ---\n` +
|
|
206
|
+
`Call: ${e.input}\n` +
|
|
207
|
+
`Result${shown.marks}: ${shown.text}`);
|
|
208
|
+
continue;
|
|
209
|
+
}
|
|
210
|
+
sections.push(runSection(run, show));
|
|
211
|
+
}
|
|
212
|
+
return truncate(header + sections.join("") + summary + footer, maxChars);
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Whether a run went entirely well and the agent answers now — the one case a
|
|
216
|
+
* bundle may be brief ({@link AssembleOptions.brief}). A run that went wrong
|
|
217
|
+
* anywhere, stood on a recording, or is one the agent carries on from is what
|
|
218
|
+
* the agent works from next: it gets every result.
|
|
219
|
+
*/
|
|
220
|
+
export function wentWell(entries, opts) {
|
|
221
|
+
return (!opts.handover &&
|
|
222
|
+
!opts.subTask &&
|
|
223
|
+
entries.every((e) => e.notNeeded || (!e.recorded && !resultText(e.response).error)));
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* A result in a brief bundle. Whole when it is short. A long one that is JSON
|
|
227
|
+
* — a search's records, nearly always — keeps every record and field, and
|
|
228
|
+
* only its long text is cut ({@link cutLongText}): the agent checks the steps
|
|
229
|
+
* against the IDs, dates, senders and names, and those are short. A long
|
|
230
|
+
* result in words is shown whole: it may be the very answer.
|
|
231
|
+
*
|
|
232
|
+
* Measured on B2's length tests: showing only the values the plan took from a
|
|
233
|
+
* result instead left the agent unable to check the plan's choices — "I
|
|
234
|
+
* couldn't confirm these were the newest", "or who sent them" — in one reply
|
|
235
|
+
* in six, and once it reported a correct run as wrong.
|
|
236
|
+
*/
|
|
237
|
+
function briefResult(text, budget) {
|
|
238
|
+
if (text.length > BRIEF_RESULT_CHARS && /^\s*[[{]/.test(text))
|
|
239
|
+
return truncate(cutLongText(text), budget);
|
|
240
|
+
return truncate(text, budget);
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* JSON text with every string longer than 40 characters cut to its first 40
|
|
244
|
+
* and "…". Keys, numbers and short strings stay as they are, and an escape is
|
|
245
|
+
* never split. Works on JSON a parser would refuse — a tool's `NaN`, a result
|
|
246
|
+
* cut short — because it walks the text's string literals from the start, so
|
|
247
|
+
* it always knows which quote opens one: the text between two strings, a
|
|
248
|
+
* list of numbers say, is never taken for one. An unterminated string is left
|
|
249
|
+
* as it is.
|
|
250
|
+
*/
|
|
251
|
+
export function cutLongText(json) {
|
|
252
|
+
let out = "";
|
|
253
|
+
let at = 0;
|
|
254
|
+
while (at < json.length) {
|
|
255
|
+
const open = json.indexOf('"', at);
|
|
256
|
+
if (open < 0)
|
|
257
|
+
return out + json.slice(at);
|
|
258
|
+
out += json.slice(at, open + 1);
|
|
259
|
+
// Walk the string a unit at a time — a character, or an escape with what
|
|
260
|
+
// it escapes — to its closing quote, noting where the 40th unit ends.
|
|
261
|
+
let end = open + 1;
|
|
262
|
+
let units = 0;
|
|
263
|
+
let cutAt = -1;
|
|
264
|
+
while (end < json.length && json[end] !== '"') {
|
|
265
|
+
end += json[end] === "\\" ? 2 : 1;
|
|
266
|
+
units += 1;
|
|
267
|
+
if (units === BRIEF_STRING_CHARS)
|
|
268
|
+
cutAt = end;
|
|
269
|
+
}
|
|
270
|
+
if (end >= json.length)
|
|
271
|
+
return out + json.slice(open + 1);
|
|
272
|
+
out += units > BRIEF_STRING_CHARS ? `${json.slice(open + 1, cutAt)}…"` : json.slice(open + 1, end + 1);
|
|
273
|
+
at = end + 1;
|
|
274
|
+
}
|
|
275
|
+
return out;
|
|
276
|
+
}
|
|
277
|
+
/**
|
|
278
|
+
* The plan's summary (its response model) in a few words, or "" when it has
|
|
279
|
+
* nothing to add: none at all, or only the note that it was left out — which
|
|
280
|
+
* sends the model to the steps, and they are right above it.
|
|
281
|
+
*/
|
|
282
|
+
function briefSummary(model) {
|
|
283
|
+
if (!model)
|
|
284
|
+
return "";
|
|
285
|
+
const keys = Object.keys(model);
|
|
286
|
+
if (keys.length === 0 || keys.every((k) => k === "summaryLeftOut"))
|
|
287
|
+
return "";
|
|
288
|
+
return `\n\nThe plan's summary: ${briefFields(model, BRIEF_SUMMARY_CHARS)}`;
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* `key = value` for each field, each value in a few words ({@link briefValue}),
|
|
292
|
+
* ended at a field's end with "…" rather than run past `max` characters.
|
|
293
|
+
*/
|
|
294
|
+
export function briefFields(fields, max) {
|
|
295
|
+
let out = "";
|
|
296
|
+
for (const [key, value] of Object.entries(fields)) {
|
|
297
|
+
const part = `${key} = ${briefValue(value)}`;
|
|
298
|
+
const next = out ? `${out}; ${part}` : part;
|
|
299
|
+
if (next.length > max)
|
|
300
|
+
return out ? `${out}; …` : truncate(part, max);
|
|
301
|
+
out = next;
|
|
302
|
+
}
|
|
303
|
+
return out;
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* A value of the plan's summary in a few words. Numbers, booleans and short
|
|
307
|
+
* strings whole; a long string cut; a list or object whose JSON is short —
|
|
308
|
+
* a few IDs, names or addresses — whole; a longer list counted; a longer
|
|
309
|
+
* object by its first fields, one level down, below which it is counted too.
|
|
310
|
+
* A summary's long lists are mostly the records the steps above already show.
|
|
311
|
+
*/
|
|
312
|
+
export function briefValue(value, depth = 0) {
|
|
313
|
+
if (typeof value === "string") {
|
|
314
|
+
return JSON.stringify(value.length > BRIEF_STRING_CHARS ? `${value.slice(0, BRIEF_STRING_CHARS)}…` : value);
|
|
315
|
+
}
|
|
316
|
+
if (value === null || typeof value !== "object")
|
|
317
|
+
return String(value);
|
|
318
|
+
let whole;
|
|
319
|
+
try {
|
|
320
|
+
whole = JSON.stringify(value) ?? "";
|
|
321
|
+
}
|
|
322
|
+
catch {
|
|
323
|
+
return "[unserializable]";
|
|
324
|
+
}
|
|
325
|
+
if (whole.length <= BRIEF_INLINE_CHARS)
|
|
326
|
+
return whole;
|
|
327
|
+
if (Array.isArray(value))
|
|
328
|
+
return `${value.length} ${value.length === 1 ? "item" : "items"}`;
|
|
329
|
+
const fields = Object.entries(value);
|
|
330
|
+
if (depth >= 1)
|
|
331
|
+
return `{${fields.length} ${fields.length === 1 ? "field" : "fields"}}`;
|
|
332
|
+
const shown = fields.slice(0, BRIEF_OBJECT_FIELDS).map(([k, v]) => `${k}: ${briefValue(v, depth + 1)}`);
|
|
333
|
+
return `{${shown.join(", ")}${fields.length > BRIEF_OBJECT_FIELDS ? ", …" : ""}}`;
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* A tool result as the model should read it.
|
|
337
|
+
*
|
|
338
|
+
* Executors hand the bundle the JSON of MCP's `CallToolResult`. Laid out as it
|
|
339
|
+
* came, every result pays for the envelope —
|
|
340
|
+
* `{"content":[{"type":"text","text":"…"}],"isError":false}`, some fifty
|
|
341
|
+
* characters — and for a second layer of escaping: every quote in the text
|
|
342
|
+
* written `\"`, every backslash `\\`. Neither tells the model anything, and on
|
|
343
|
+
* a long plan they are a large share of what it reads (and of what the agent
|
|
344
|
+
* pays to put in its context). A result whose content is text and nothing else
|
|
345
|
+
* is shown as that text, marked when `isError` is true. Anything else — an
|
|
346
|
+
* image, a resource, structured content, an answer that is not a
|
|
347
|
+
* `CallToolResult` at all — is shown exactly as it came.
|
|
348
|
+
*/
|
|
349
|
+
export function resultText(response) {
|
|
350
|
+
const asIs = { text: response, error: false };
|
|
351
|
+
let parsed;
|
|
352
|
+
try {
|
|
353
|
+
parsed = JSON.parse(response);
|
|
354
|
+
}
|
|
355
|
+
catch {
|
|
356
|
+
return asIs;
|
|
357
|
+
}
|
|
358
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
|
|
359
|
+
return asIs;
|
|
360
|
+
const result = parsed;
|
|
361
|
+
if (!Object.keys(result).every((k) => k === "content" || k === "isError"))
|
|
362
|
+
return asIs;
|
|
363
|
+
if (result.isError !== undefined && typeof result.isError !== "boolean")
|
|
364
|
+
return asIs;
|
|
365
|
+
if (!Array.isArray(result.content) || result.content.length === 0)
|
|
366
|
+
return asIs;
|
|
367
|
+
const texts = [];
|
|
368
|
+
for (const item of result.content) {
|
|
369
|
+
if (!item || typeof item !== "object" || Array.isArray(item))
|
|
370
|
+
return asIs;
|
|
371
|
+
const part = item;
|
|
372
|
+
if (part.type !== "text" || typeof part.text !== "string")
|
|
373
|
+
return asIs;
|
|
374
|
+
if (!Object.keys(part).every((k) => k === "type" || k === "text"))
|
|
375
|
+
return asIs;
|
|
376
|
+
texts.push(part.text);
|
|
377
|
+
}
|
|
378
|
+
return { text: texts.join("\n"), error: result.isError === true };
|
|
379
|
+
}
|
|
380
|
+
/** `(recorded)`, `(error)` or `(recorded, error)` after a result's label; "" for a plain live result. */
|
|
381
|
+
function resultMarks(recorded, error) {
|
|
382
|
+
const marks = [recorded ? "recorded" : "", error ? "error" : ""].filter(Boolean);
|
|
383
|
+
return marks.length > 0 ? ` (${marks.join(", ")})` : "";
|
|
384
|
+
}
|
|
385
|
+
/**
|
|
386
|
+
* The entries cut into runs of consecutive calls to one tool. A long plan is
|
|
387
|
+
* mostly such runs — eleven look-ups, then thirty-two forwards — and laying
|
|
388
|
+
* each step out on its own repeats the tool's name and the section frame every
|
|
389
|
+
* time. A step left out stands alone, so its "not needed" line reads as before.
|
|
390
|
+
*/
|
|
391
|
+
function consecutiveRuns(entries) {
|
|
392
|
+
const runs = [];
|
|
393
|
+
entries.forEach((entry, index) => {
|
|
394
|
+
const last = runs.at(-1);
|
|
395
|
+
const prev = last?.at(-1)?.entry;
|
|
396
|
+
if (last && prev && !prev.notNeeded && !entry.notNeeded && prev.toolName === entry.toolName) {
|
|
397
|
+
last.push({ entry, index });
|
|
398
|
+
}
|
|
399
|
+
else {
|
|
400
|
+
runs.push([{ entry, index }]);
|
|
152
401
|
}
|
|
153
|
-
const response = truncate(e.response, perResponse);
|
|
154
|
-
return (`\n\n--- Step ${i + 1}: ${e.toolName} ---\n` +
|
|
155
|
-
`Call: ${e.input}\n` +
|
|
156
|
-
`Result${e.recorded ? " (recorded)" : ""}: ${response}`);
|
|
157
402
|
});
|
|
158
|
-
return
|
|
403
|
+
return runs;
|
|
404
|
+
}
|
|
405
|
+
/**
|
|
406
|
+
* Two or more consecutive calls to one tool, as one section: one line per call,
|
|
407
|
+
* numbered as its step, with its result after the arrow. What every call shares
|
|
408
|
+
* is said once above the calls — input fields with the same value in each, and
|
|
409
|
+
* the answer, when every call got the same one. Each result is shown as `show`
|
|
410
|
+
* shows a step on its own, budget included.
|
|
411
|
+
*/
|
|
412
|
+
function runSection(run, show) {
|
|
413
|
+
const first = run[0];
|
|
414
|
+
const head = `\n\n--- Steps ${first.index + 1}-${run[run.length - 1].index + 1}: ${first.entry.toolName} (${run.length} calls) ---`;
|
|
415
|
+
const inputs = sharedInput(run.map(({ entry }) => entry.input));
|
|
416
|
+
const shown = run.map(({ entry, index }, i) => ({ step: index + 1, input: inputs.own[i], ...show(entry) }));
|
|
417
|
+
const lines = inputs.shared === null ? "" : `\nIn every call: ${inputs.shared}`;
|
|
418
|
+
const same = shown.every((s) => s.text === shown[0].text && s.marks === shown[0].marks);
|
|
419
|
+
if (same) {
|
|
420
|
+
return (`${head}${lines}\nEvery call's result${shown[0].marks}: ${shown[0].text}` +
|
|
421
|
+
shown.map((s) => `\n${s.step}: ${s.input}`).join(""));
|
|
422
|
+
}
|
|
423
|
+
return head + lines + shown.map((s) => `\n${s.step}${s.marks}: ${s.input} -> ${s.text}`).join("");
|
|
424
|
+
}
|
|
425
|
+
/**
|
|
426
|
+
* The input fields a run of calls all share, said once, and what is left of
|
|
427
|
+
* each call's input. A forward sends thirty-two emails to one recipient; the
|
|
428
|
+
* recipient need not be read thirty-two times. Only plain JSON objects are
|
|
429
|
+
* split, and only on fields present in every call with the same value; an
|
|
430
|
+
* input that was capped, or is not an object, leaves the run as it came.
|
|
431
|
+
*/
|
|
432
|
+
function sharedInput(inputs) {
|
|
433
|
+
const asCame = { shared: null, own: [...inputs] };
|
|
434
|
+
const objects = [];
|
|
435
|
+
for (const input of inputs) {
|
|
436
|
+
let parsed;
|
|
437
|
+
try {
|
|
438
|
+
parsed = JSON.parse(input);
|
|
439
|
+
}
|
|
440
|
+
catch {
|
|
441
|
+
return asCame;
|
|
442
|
+
}
|
|
443
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
|
|
444
|
+
return asCame;
|
|
445
|
+
objects.push(parsed);
|
|
446
|
+
}
|
|
447
|
+
const value = (o, key) => JSON.stringify(o[key]);
|
|
448
|
+
const common = Object.keys(objects[0]).filter((key) => objects.every((o) => Object.hasOwn(o, key) && value(o, key) === value(objects[0], key)));
|
|
449
|
+
if (common.length === 0)
|
|
450
|
+
return asCame;
|
|
451
|
+
const pick = (o, keep) => JSON.stringify(Object.fromEntries(Object.entries(o).filter(([key]) => keep(key))));
|
|
452
|
+
return {
|
|
453
|
+
shared: pick(objects[0], (key) => common.includes(key)),
|
|
454
|
+
own: objects.map((o) => pick(o, (key) => !common.includes(key))),
|
|
455
|
+
};
|
|
159
456
|
}
|
|
160
457
|
/** Truncate to `max` chars, appending a marker naming how much was dropped. */
|
|
161
458
|
export function truncate(s, max) {
|
|
@@ -199,7 +199,8 @@ export declare function clampDirectRun(budgets: ReplayBudgets, logicMs?: number)
|
|
|
199
199
|
*
|
|
200
200
|
* - **ok** — the plan ran to its end: `{ ok: true, text, responseModel, steps,
|
|
201
201
|
* partial: false }`. `text` is the bundle; `responseModel` what the
|
|
202
|
-
* scenario's own logic built from it (`{}` when it has none
|
|
202
|
+
* scenario's own logic built from it (`{}` when it has none, and when the
|
|
203
|
+
* bundle is brief and carries it already — plan.ts, `DriveOptions.brief`).
|
|
203
204
|
* - **partial** — the plan stopped part-way after running steps here: `{ ok:
|
|
204
205
|
* true, text, responseModel: {}, steps, partial }`. `text` opens with the
|
|
205
206
|
* hand-over note (text beginning `[BaseInstRunner calculated replay]`) and
|
|
@@ -1757,7 +1757,15 @@ export class ReplayController {
|
|
|
1757
1757
|
const deadline = Date.now() + this.budgets.planMs;
|
|
1758
1758
|
let result;
|
|
1759
1759
|
try {
|
|
1760
|
-
result = await plan.runToCompletion(this.executeStep(state, { deadline }), this.recordedSource(state), this.observeStep(state), deadline, MAX_REPLAY_REASON, {
|
|
1760
|
+
result = await plan.runToCompletion(this.executeStep(state, { deadline }), this.recordedSource(state), this.observeStep(state), deadline, MAX_REPLAY_REASON, {
|
|
1761
|
+
stopOnFailure: true,
|
|
1762
|
+
recordedFallback: this.recordedFallback,
|
|
1763
|
+
signal,
|
|
1764
|
+
reportTool: this.reportToolFor(state),
|
|
1765
|
+
// The agent answers from the bundle alone: a run that went entirely
|
|
1766
|
+
// well is laid out briefly, with the plan's summary in it.
|
|
1767
|
+
brief: true,
|
|
1768
|
+
});
|
|
1761
1769
|
}
|
|
1762
1770
|
catch (err) {
|
|
1763
1771
|
// The scenario's own logic failed. Retire and let the model do the work.
|
|
@@ -1875,12 +1883,18 @@ export class ReplayController {
|
|
|
1875
1883
|
}
|
|
1876
1884
|
// Live steps only, as above: a recorded stand-in did no work here.
|
|
1877
1885
|
state.stepsPinned += result.executed;
|
|
1886
|
+
// A brief bundle carries the plan's summary itself, in a few words, so it
|
|
1887
|
+
// is not delivered a second time beside it.
|
|
1878
1888
|
let responseModel = {};
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1889
|
+
if (result.summaryError)
|
|
1890
|
+
logDetail("replay.response_model_failed", { error: result.summaryError });
|
|
1891
|
+
if (!result.brief) {
|
|
1892
|
+
try {
|
|
1893
|
+
responseModel = plan.responseModel();
|
|
1894
|
+
}
|
|
1895
|
+
catch (err) {
|
|
1896
|
+
logDetail("replay.response_model_failed", { error: errText(err) });
|
|
1897
|
+
}
|
|
1884
1898
|
}
|
|
1885
1899
|
// A plan whose tools errored ran to the end and did nothing. `steered_full`
|
|
1886
1900
|
// would book the largest saving available for it; `failed` books a baseline
|
package/dist/replay/plan.d.ts
CHANGED
|
@@ -125,6 +125,14 @@ export interface DriveOptions {
|
|
|
125
125
|
* they always were (bundle.ts, `AssembleOptions.reportTool`).
|
|
126
126
|
*/
|
|
127
127
|
reportTool?: string;
|
|
128
|
+
/**
|
|
129
|
+
* Lay a run that went entirely well out briefly, with the plan's summary in
|
|
130
|
+
* it (bundle.ts, `AssembleOptions.brief`), and say so in the result's
|
|
131
|
+
* `brief`. For a caller whose agent answers from the bundle alone —
|
|
132
|
+
* `bir-scenario`'s run of a whole plan — not for an operator's `bir replay`,
|
|
133
|
+
* which reads every result. A run that went wrong is laid out in full.
|
|
134
|
+
*/
|
|
135
|
+
brief?: boolean;
|
|
128
136
|
}
|
|
129
137
|
/**
|
|
130
138
|
* Supplies a step's recorded output when its tool cannot run here (§8.1).
|
|
@@ -213,6 +221,13 @@ export interface ComposeResult {
|
|
|
213
221
|
* the end of the planned steps.
|
|
214
222
|
*/
|
|
215
223
|
endedAt: number;
|
|
224
|
+
/**
|
|
225
|
+
* The bundle is brief ({@link DriveOptions.brief}), and the plan's summary,
|
|
226
|
+
* when it has one, is in `text`: the caller must not deliver it again.
|
|
227
|
+
*/
|
|
228
|
+
brief?: boolean;
|
|
229
|
+
/** Building the summary for a brief bundle threw: why. The bundle went without it. */
|
|
230
|
+
summaryError?: string;
|
|
216
231
|
}
|
|
217
232
|
/** Where, and why, a run stopped short of the chain's end (fallbk.md). */
|
|
218
233
|
export interface PlanStop {
|
package/dist/replay/plan.js
CHANGED
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
* output, and skipping.
|
|
22
22
|
*/
|
|
23
23
|
import { evalParamsLogic, evalResponseParamsLogic, evalToolInputLogic, evalToolOutputLogic, evalWhenLogic, LogicTimeoutError, } from "./logic.js";
|
|
24
|
-
import { assembleBundle, bundleInput, MAX_REPLAY_REASON, testWords } from "./bundle.js";
|
|
24
|
+
import { assembleBundle, bundleInput, MAX_REPLAY_REASON, testWords, wentWell, } from "./bundle.js";
|
|
25
25
|
import { redact, redactText } from "../record/redact.js";
|
|
26
26
|
import { sinkOfPlan, unsafeParameter } from "./safety.js";
|
|
27
27
|
import { testedOnlyTargets } from "./targets.js";
|
|
@@ -883,6 +883,29 @@ export class ScenarioReplayPlan {
|
|
|
883
883
|
// The intent is a paragraph the analyser wrote; it is context, not an
|
|
884
884
|
// instruction, so it is clipped rather than dumped whole into the prompt.
|
|
885
885
|
const intent = this.intent.length > 240 ? `${this.intent.slice(0, 240)}…` : this.intent;
|
|
886
|
+
if (this.mode === "direct") {
|
|
887
|
+
// The steps are not listed. The model does not call them, and what the
|
|
888
|
+
// tool returns names every call and says whether the run was whole or
|
|
889
|
+
// stopped short — what a list here once told it ahead of time. The list
|
|
890
|
+
// cost the agent up to 2,000 characters in the first call of every turn
|
|
891
|
+
// a plan was armed for (docs/calculatedReplay.md, "How a bundle reads"),
|
|
892
|
+
// and the tool's result shows an operator watching the session what ran.
|
|
893
|
+
return [
|
|
894
|
+
"[BaseInstRunner calculated replay] A known-good tool sequence was recorded",
|
|
895
|
+
`for this request${intent ? ` (intent: ${intent})` : ""}.`,
|
|
896
|
+
"",
|
|
897
|
+
`Call ${directToolName} once, with no arguments, before any other tool. It runs the`,
|
|
898
|
+
"sequence and returns what each call did.",
|
|
899
|
+
subTask
|
|
900
|
+
? "Then continue your task from the results it returns, calling tools as needed."
|
|
901
|
+
: this.stopsEarly()
|
|
902
|
+
? "Then continue the user's request from the results it returns."
|
|
903
|
+
: "Then answer the user's request from the results it returns.",
|
|
904
|
+
// A parked step (fallbk.md D3): the model is told up front that the
|
|
905
|
+
// plan is partial, so the hand-over that follows is expected.
|
|
906
|
+
...(this.stopsEarly() ? ["", "The system will then hand the rest of the task back to you."] : []),
|
|
907
|
+
].join("\n");
|
|
908
|
+
}
|
|
886
909
|
// Each step's reasoning is another session's narration ("I'll now drop the
|
|
887
910
|
// temp table…"), injected into this one. It is a label here, not a
|
|
888
911
|
// paragraph: its first line, clipped. A chain long enough that even the
|
|
@@ -931,30 +954,6 @@ export class ScenarioReplayPlan {
|
|
|
931
954
|
const tail = this.stopsEarly()
|
|
932
955
|
? ["", "The system will then hand the rest of the task back to you."]
|
|
933
956
|
: [];
|
|
934
|
-
if (this.mode === "direct") {
|
|
935
|
-
// The sequence is listed even though the model does not call it itself:
|
|
936
|
-
// it is about to receive these results, and knowing what was run is what
|
|
937
|
-
// lets it tell a complete answer from a partial one. It is also the only
|
|
938
|
-
// place an operator watching the session can see what a replay is doing.
|
|
939
|
-
return [
|
|
940
|
-
"[BaseInstRunner calculated replay] A known-good tool sequence was recorded",
|
|
941
|
-
`for this request${intent ? ` (intent: ${intent})` : ""}.`,
|
|
942
|
-
"",
|
|
943
|
-
anyTest
|
|
944
|
-
? 'It runs these tools, in order, and returns their results. A step marked "only when" runs only when this request needs it, and the results say which ones did not:'
|
|
945
|
-
: "It runs these tools, in order, and returns their results:",
|
|
946
|
-
"",
|
|
947
|
-
...lines,
|
|
948
|
-
"",
|
|
949
|
-
`Call ${directToolName} once, with no arguments, before any other tool.`,
|
|
950
|
-
subTask
|
|
951
|
-
? "Then continue your task from the results it returns, calling tools as needed."
|
|
952
|
-
: this.stopsEarly()
|
|
953
|
-
? "Then continue the user's request from the results it returns."
|
|
954
|
-
: "Then answer the user's request from the results it returns.",
|
|
955
|
-
...tail,
|
|
956
|
-
].join("\n");
|
|
957
|
-
}
|
|
958
957
|
return [
|
|
959
958
|
"[BaseInstRunner calculated replay] A known-good tool sequence was previously",
|
|
960
959
|
"recorded for this request. Call exactly these tools, in order, one at a time:",
|
|
@@ -1343,12 +1342,29 @@ export class ScenarioReplayPlan {
|
|
|
1343
1342
|
position: this.stopAt,
|
|
1344
1343
|
};
|
|
1345
1344
|
}
|
|
1345
|
+
const layout = {
|
|
1346
|
+
handover: stopped !== undefined,
|
|
1347
|
+
subTask: this.subTask,
|
|
1348
|
+
reportTool: opts.reportTool,
|
|
1349
|
+
};
|
|
1350
|
+
// Brief only for a run that went entirely well (bundle.ts, `wentWell`). A
|
|
1351
|
+
// tool error that is not an MCP `isError` — a JSON-RPC error, which the
|
|
1352
|
+
// bundle cannot tell from a result — is counted in `errored`, and is just
|
|
1353
|
+
// as much a reason for the agent to see everything.
|
|
1354
|
+
const brief = opts.brief === true && errored === 0 && wentWell(entries, layout);
|
|
1355
|
+
let summaryError;
|
|
1356
|
+
if (brief) {
|
|
1357
|
+
let summary;
|
|
1358
|
+
try {
|
|
1359
|
+
summary = this.responseModel();
|
|
1360
|
+
}
|
|
1361
|
+
catch (err) {
|
|
1362
|
+
summaryError = errText(err);
|
|
1363
|
+
}
|
|
1364
|
+
layout.brief = { summary };
|
|
1365
|
+
}
|
|
1346
1366
|
return {
|
|
1347
|
-
text: assembleBundle(entries, maxChars,
|
|
1348
|
-
handover: stopped !== undefined,
|
|
1349
|
-
subTask: this.subTask,
|
|
1350
|
-
reportTool: opts.reportTool,
|
|
1351
|
-
}),
|
|
1367
|
+
text: assembleBundle(entries, maxChars, layout),
|
|
1352
1368
|
executed,
|
|
1353
1369
|
recorded: recordedCount,
|
|
1354
1370
|
skipped,
|
|
@@ -1359,6 +1375,8 @@ export class ScenarioReplayPlan {
|
|
|
1359
1375
|
stopped,
|
|
1360
1376
|
aborted: aborted || undefined,
|
|
1361
1377
|
endedAt: this.stepIndex,
|
|
1378
|
+
brief: brief || undefined,
|
|
1379
|
+
summaryError,
|
|
1362
1380
|
};
|
|
1363
1381
|
}
|
|
1364
1382
|
/**
|
package/docs/calculatedReplay.md
CHANGED
|
@@ -771,6 +771,38 @@ Delivery, best channel first. (A direct plan's own results go through `run_scena
|
|
|
771
771
|
(`assembleBundle`/`truncate`, portable verbatim from `payload-executor.ts`). Works for any tool,
|
|
772
772
|
and is read as adversarial — hence last.
|
|
773
773
|
|
|
774
|
+
**How a bundle reads** ([`bundle.ts`](../src/replay/bundle.ts); the same in a direct run's answer,
|
|
775
|
+
§6). The agent pays once to take the bundle in, as new input, and on a long plan that was most of a
|
|
776
|
+
replayed task's bill — so nothing in it repeats what the model already has. Each result is shown as
|
|
777
|
+
its text, not as MCP's `CallToolResult` JSON (the envelope and its second layer of escaping told the
|
|
778
|
+
model nothing), with `(error)` when `isError` is set; a result with anything but text — an image, a
|
|
779
|
+
resource, structured content — is shown as it came. Consecutive calls to one tool share a section,
|
|
780
|
+
`--- Steps a-b: tool (n calls) ---`, with one numbered line per call, `n: input -> result`. Input
|
|
781
|
+
fields with the same value in every call are said once above them (`In every call: {…}`), and so is
|
|
782
|
+
an answer every call got (`Every call's result: …`). A step on its own, and a step left out, keep
|
|
783
|
+
`--- Step n: tool ---`. Nothing is dropped, and each result keeps its fair share of the cap. On
|
|
784
|
+
B2's length tests this made bundles 27% shorter, 33% at 50–60 steps.
|
|
785
|
+
|
|
786
|
+
**A run that went well reads briefly** (`AssembleOptions.brief`, asked for by a direct run, §6). When
|
|
787
|
+
every step ran live and succeeded and the agent is to answer now, its job is to see that the steps
|
|
788
|
+
were right and say what was done — not to read every word of every result. So the header says every
|
|
789
|
+
call succeeded; a result up to 300 characters is shown whole; a longer one that is JSON keeps every
|
|
790
|
+
record and field, and only its text longer than 40 characters is cut, with "…" (`cutLongText`: the
|
|
791
|
+
IDs, dates, senders and names the agent checks the steps against are short; an e-mail's body is
|
|
792
|
+
not). A long result in words is shown whole — it may be the very answer. Showing only the values the
|
|
793
|
+
plan took from a long result was tried first: it left the agent unable to check the plan's choices
|
|
794
|
+
("I couldn't confirm these were the newest") in one reply in six on B2's length tests, and once it
|
|
795
|
+
reported a correct run as wrong. The plan's summary (its response model) is said in a few words
|
|
796
|
+
(short lists whole, long ones counted), in the bundle, above the footer — so `bir-scenario` gets
|
|
797
|
+
`responseModel: {}` and does not append it again — and is left out when it only says it was left
|
|
798
|
+
out. The footer asks for
|
|
799
|
+
the answer in a few sentences: the reply is output, the dearest tokens an agent spends. Anything
|
|
800
|
+
that went wrong — a step that reported an error, a run that stopped short, a recorded stand-in, a
|
|
801
|
+
sub-task the agent carries on with — gets the full bundle above, because that is what the agent
|
|
802
|
+
works from next. For the same reason a direct run's directive (`steeringDirective`) no longer lists
|
|
803
|
+
the planned steps: the model calls none of them, and the answer names every call and says whether
|
|
804
|
+
the run was whole. An operator's `bir replay` and a divergence bundle are always full.
|
|
805
|
+
|
|
774
806
|
**Tier 2 — abort.** Composition itself threw (`replay.compose_failed`). Retire the plan, return
|
|
775
807
|
`{}`, continue as an ordinary turn. Outcome `failed`.
|
|
776
808
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@basein/runner",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.25",
|
|
4
4
|
"description": "A recording MCP proxy: sits between any MCP client and its MCP servers, executes each call on the client's behalf, and records the run as a reusable BaseIn scenario.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|