@basein/runner 0.3.17 → 0.3.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/bir-hooks.d.ts +11 -8
- package/dist/bin/bir-hooks.js +11 -8
- package/dist/bin/bir-scenario.d.ts +5 -1
- package/dist/bin/bir-scenario.js +155 -5
- package/dist/bin/bir.js +22 -9
- package/dist/bin/investigate.d.ts +9 -0
- package/dist/bin/investigate.js +26 -12
- package/dist/bin/scenario-edit.d.ts +7 -0
- package/dist/bin/scenario-edit.js +19 -2
- package/dist/bin/scenario-liveness.d.ts +12 -1
- package/dist/bin/scenario-liveness.js +8 -4
- package/dist/bin/setup.js +3 -3
- package/dist/control/client.d.ts +5 -0
- package/dist/control/client.js +7 -0
- package/dist/control/ensure-hook.d.ts +14 -3
- package/dist/control/ensure-hook.js +18 -5
- package/dist/control/recorder-main.d.ts +15 -0
- package/dist/control/recorder-main.js +33 -8
- package/dist/control/server.d.ts +192 -13
- package/dist/control/server.js +819 -41
- package/dist/control/transcript.d.ts +19 -0
- package/dist/control/transcript.js +57 -0
- package/dist/proxy/intercept.js +13 -0
- package/dist/proxy/session.d.ts +23 -0
- package/dist/proxy/session.js +71 -3
- package/dist/record/recorder.d.ts +156 -2
- package/dist/record/recorder.js +9 -0
- package/dist/record/remote-recorder.d.ts +59 -1
- package/dist/record/remote-recorder.js +297 -20
- package/dist/record/switchable-recorder.js +6 -0
- package/dist/replay/bundle.d.ts +8 -0
- package/dist/replay/bundle.js +20 -6
- package/dist/replay/condition-words.d.ts +28 -0
- package/dist/replay/condition-words.js +59 -0
- package/dist/replay/controller.d.ts +78 -7
- package/dist/replay/controller.js +318 -62
- package/dist/replay/derive.d.ts +15 -0
- package/dist/replay/derive.js +5 -0
- package/dist/replay/handover.d.ts +7 -0
- package/dist/replay/handover.js +89 -51
- package/dist/replay/plan.d.ts +101 -5
- package/dist/replay/plan.js +260 -18
- package/dist/replay/source-run.d.ts +2 -2
- package/dist/replay/source-run.js +3 -3
- package/dist/replay/targets.d.ts +40 -1
- package/dist/replay/targets.js +105 -3
- package/dist/replay/types.d.ts +25 -8
- package/docs/calculatedReplay.md +45 -5
- package/docs/calculatedReplayGuide.md +13 -4
- package/package.json +1 -1
- package/production.md +21 -7
|
@@ -34,6 +34,25 @@
|
|
|
34
34
|
* birth time, mtime, and the bytes at its head and at the offset) and read
|
|
35
35
|
* again from the start, which is exactly what the old code did every time.
|
|
36
36
|
*/
|
|
37
|
+
/** The day the agent was told it is, and the words that told it (issue 1212). */
|
|
38
|
+
export interface AgentDay {
|
|
39
|
+
/** "Today's date is Thursday, 2023-11-30 and the current time is 00:00:00." */
|
|
40
|
+
line: string;
|
|
41
|
+
/** The `YYYY-MM-DD` in the line, when it has one. */
|
|
42
|
+
day?: string;
|
|
43
|
+
/** `system`: a system prompt the host sent. `host`: the host's own date reminder. */
|
|
44
|
+
source: "system" | "host";
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* The day the agent of this session was told it is (issue 1212), as its host
|
|
48
|
+
* wrote it into the transcript: a system prompt's own "Today's date is …"
|
|
49
|
+
* first — a benchmark's "Thursday, 2023-11-30", which the agent goes by —
|
|
50
|
+
* and else the host's date reminder, this machine's day. Undefined when the
|
|
51
|
+
* transcript has neither yet: its host writes both just after the prompt
|
|
52
|
+
* hook, so the first prompt of a session finds them only when its plan
|
|
53
|
+
* starts.
|
|
54
|
+
*/
|
|
55
|
+
export declare function agentDayOf(transcriptPath: string | undefined): AgentDay | undefined;
|
|
37
56
|
/**
|
|
38
57
|
* The model's text after the message that made the tool call `toolUseId`: its
|
|
39
58
|
* answer, once that call's result came back (the service's autocorrect.md
|
|
@@ -135,6 +135,8 @@ function freshIndex(st) {
|
|
|
135
135
|
results: [],
|
|
136
136
|
messages: new Map(),
|
|
137
137
|
toolUses: new Map(),
|
|
138
|
+
systemDayLine: "",
|
|
139
|
+
hostDay: "",
|
|
138
140
|
};
|
|
139
141
|
}
|
|
140
142
|
/** Whether `idx` can be continued for a file that now looks like `st`. */
|
|
@@ -303,9 +305,35 @@ function ingest(idx, bytes) {
|
|
|
303
305
|
if (line)
|
|
304
306
|
foldLine(idx, line);
|
|
305
307
|
}
|
|
308
|
+
/** The line that tells the agent the day, out of a system prompt's text: "Today's date is …", "Today is …". */
|
|
309
|
+
const DAY_LINE = /Today(?:['’]s date)? is [^\n]{4,200}/i;
|
|
310
|
+
function dayLineIn(text) {
|
|
311
|
+
const whole = Array.isArray(text)
|
|
312
|
+
? text.map((t) => (typeof t === "string" ? t : t && typeof t === "object" ? String(t.text ?? "") : "")).join("\n")
|
|
313
|
+
: typeof text === "string"
|
|
314
|
+
? text
|
|
315
|
+
: "";
|
|
316
|
+
// The whole line, never cut at a full stop: "Nov. 30, 2023" has one inside.
|
|
317
|
+
return redactText(DAY_LINE.exec(whole)?.[0] ?? "").slice(0, 200);
|
|
318
|
+
}
|
|
306
319
|
/** What one line contributes to every answer this module gives. */
|
|
307
320
|
function foldLine(idx, line) {
|
|
308
321
|
idx.lineNo += 1;
|
|
322
|
+
// The day the agent was told it is (issue 1212): a system prompt's
|
|
323
|
+
// "Today's date is …", and the host's own date reminder beside it.
|
|
324
|
+
if (line.type === "attachment" && line.attachment?.type === "prompt_snapshot") {
|
|
325
|
+
const said = dayLineIn(line.attachment.systemPrompt);
|
|
326
|
+
if (said)
|
|
327
|
+
idx.systemDayLine = said;
|
|
328
|
+
}
|
|
329
|
+
else if (line.type === "api-request-shape") {
|
|
330
|
+
const said = dayLineIn(line.shape?.system);
|
|
331
|
+
if (said)
|
|
332
|
+
idx.systemDayLine = said;
|
|
333
|
+
}
|
|
334
|
+
else if (line.type === "attachment" && line.attachment?.type === "date" && typeof line.attachment.date === "string") {
|
|
335
|
+
idx.hostDay = line.attachment.date.slice(0, 10);
|
|
336
|
+
}
|
|
309
337
|
const message = line.message;
|
|
310
338
|
const role = message?.role ?? line.type;
|
|
311
339
|
const blocks = blocksOf(line);
|
|
@@ -397,6 +425,35 @@ function foldLine(idx, line) {
|
|
|
397
425
|
}
|
|
398
426
|
}
|
|
399
427
|
}
|
|
428
|
+
function isoDayIn(text) {
|
|
429
|
+
const m = /(?<!\d)(\d{4})-(\d{2})-(\d{2})(?!\d)/.exec(text);
|
|
430
|
+
if (!m)
|
|
431
|
+
return undefined;
|
|
432
|
+
const [y, mo, d] = [Number(m[1]), Number(m[2]), Number(m[3])];
|
|
433
|
+
const at = new Date(Date.UTC(y, mo - 1, d));
|
|
434
|
+
return at.getUTCFullYear() === y && at.getUTCMonth() === mo - 1 && at.getUTCDate() === d ? m[0] : undefined;
|
|
435
|
+
}
|
|
436
|
+
/**
|
|
437
|
+
* The day the agent of this session was told it is (issue 1212), as its host
|
|
438
|
+
* wrote it into the transcript: a system prompt's own "Today's date is …"
|
|
439
|
+
* first — a benchmark's "Thursday, 2023-11-30", which the agent goes by —
|
|
440
|
+
* and else the host's date reminder, this machine's day. Undefined when the
|
|
441
|
+
* transcript has neither yet: its host writes both just after the prompt
|
|
442
|
+
* hook, so the first prompt of a session finds them only when its plan
|
|
443
|
+
* starts.
|
|
444
|
+
*/
|
|
445
|
+
export function agentDayOf(transcriptPath) {
|
|
446
|
+
if (!transcriptPath)
|
|
447
|
+
return undefined;
|
|
448
|
+
const idx = indexFor(transcriptPath);
|
|
449
|
+
if (!idx)
|
|
450
|
+
return undefined;
|
|
451
|
+
if (idx.systemDayLine)
|
|
452
|
+
return { line: idx.systemDayLine, day: isoDayIn(idx.systemDayLine), source: "system" };
|
|
453
|
+
if (idx.hostDay)
|
|
454
|
+
return { line: `Today's date is ${idx.hostDay}.`, day: isoDayIn(idx.hostDay), source: "host" };
|
|
455
|
+
return undefined;
|
|
456
|
+
}
|
|
400
457
|
/**
|
|
401
458
|
* The model's text after the message that made the tool call `toolUseId`: its
|
|
402
459
|
* answer, once that call's result came back (the service's autocorrect.md
|
package/dist/proxy/intercept.js
CHANGED
|
@@ -140,6 +140,19 @@ export class RecordingInterceptor {
|
|
|
140
140
|
}
|
|
141
141
|
onUpstreamResponse(pending, response) {
|
|
142
142
|
if (pending.method === "tools/list") {
|
|
143
|
+
// The server's tools, as it listed them (the service's autocorrect.md
|
|
144
|
+
// AC12): what a correction from an agent's answer may write a step with.
|
|
145
|
+
// Its own answer, never the rewrite below, and on a later tick.
|
|
146
|
+
if (!response.error) {
|
|
147
|
+
const result = response.result;
|
|
148
|
+
const listed = result?.tools;
|
|
149
|
+
if (Array.isArray(listed)) {
|
|
150
|
+
const cursor = pending.params?.cursor;
|
|
151
|
+
const next = result?.nextCursor;
|
|
152
|
+
const page = { first: cursor === undefined || cursor === null || cursor === "", last: !(typeof next === "string" && next.length > 0) };
|
|
153
|
+
setImmediate(() => this.session.reportTools(listed, page));
|
|
154
|
+
}
|
|
155
|
+
}
|
|
143
156
|
// The only rewrite in the whole proxy, and only when a control server may
|
|
144
157
|
// inject call ids into this server's calls (Phase 6). Decided by the
|
|
145
158
|
// session so it does not depend on whether registration happened to beat
|
package/dist/proxy/session.d.ts
CHANGED
|
@@ -148,6 +148,16 @@ export declare class ProxySession {
|
|
|
148
148
|
private queue;
|
|
149
149
|
/** Steps that arrived before the tier was known. */
|
|
150
150
|
private buffer;
|
|
151
|
+
/**
|
|
152
|
+
* The newest tool list the server gave (`tools/list`, every page of it;
|
|
153
|
+
* the service's autocorrect.md AC12), kept to send once bound, and the
|
|
154
|
+
* control server it went to.
|
|
155
|
+
*/
|
|
156
|
+
private toolList?;
|
|
157
|
+
private toolPages;
|
|
158
|
+
private toolListSentTo?;
|
|
159
|
+
/** One later try of a tool list whose post to the control server failed. */
|
|
160
|
+
private toolRetry?;
|
|
151
161
|
private lossyValue;
|
|
152
162
|
private negotiation?;
|
|
153
163
|
private downgrading?;
|
|
@@ -306,6 +316,19 @@ export declare class ProxySession {
|
|
|
306
316
|
*/
|
|
307
317
|
private becomePassthrough;
|
|
308
318
|
private buildRecorder;
|
|
319
|
+
/**
|
|
320
|
+
* One page of the server's `tools/list` answer (the service's
|
|
321
|
+
* autocorrect.md AC12): the whole list goes to the control server once its
|
|
322
|
+
* last page is in, which sends the names, descriptions and input schemas to
|
|
323
|
+
* the service when they changed. Only where this proxy records: a server
|
|
324
|
+
* nothing is recorded of sends nothing of its tools either.
|
|
325
|
+
*/
|
|
326
|
+
reportTools(tools: readonly unknown[], page: {
|
|
327
|
+
first: boolean;
|
|
328
|
+
last: boolean;
|
|
329
|
+
}): void;
|
|
330
|
+
/** Send the kept tool list to the control server this proxy is bound to, once. */
|
|
331
|
+
private sendTools;
|
|
309
332
|
/**
|
|
310
333
|
* Record one completed `tools/call`. Fire-and-forget by contract: it is called
|
|
311
334
|
* *after* the result is already on its way to the host, and it never awaits.
|
package/dist/proxy/session.js
CHANGED
|
@@ -47,7 +47,7 @@ import { readFile } from "node:fs/promises";
|
|
|
47
47
|
import { resolve as resolvePath } from "node:path";
|
|
48
48
|
import { ControlClient } from "../control/client.js";
|
|
49
49
|
import { readDiscovery } from "../control/discovery.js";
|
|
50
|
-
import { qualifyToolName } from "../control/correlation.js";
|
|
50
|
+
import { newCallId, qualifyToolName } from "../control/correlation.js";
|
|
51
51
|
import { projectRecord } from "../config/generate.js";
|
|
52
52
|
import { installedPath, normalizePath } from "../control/paths.js";
|
|
53
53
|
import { authenticate, loadCredentials, normalizeAuthUrl, resolveAuthUrl } from "../auth/client.js";
|
|
@@ -241,6 +241,16 @@ export class ProxySession {
|
|
|
241
241
|
});
|
|
242
242
|
/** Steps that arrived before the tier was known. */
|
|
243
243
|
buffer = [];
|
|
244
|
+
/**
|
|
245
|
+
* The newest tool list the server gave (`tools/list`, every page of it;
|
|
246
|
+
* the service's autocorrect.md AC12), kept to send once bound, and the
|
|
247
|
+
* control server it went to.
|
|
248
|
+
*/
|
|
249
|
+
toolList;
|
|
250
|
+
toolPages = [];
|
|
251
|
+
toolListSentTo;
|
|
252
|
+
/** One later try of a tool list whose post to the control server failed. */
|
|
253
|
+
toolRetry;
|
|
244
254
|
lossyValue = false;
|
|
245
255
|
negotiation;
|
|
246
256
|
downgrading;
|
|
@@ -484,6 +494,11 @@ export class ProxySession {
|
|
|
484
494
|
else
|
|
485
495
|
logLine("tier.rebound", { ...fields, was });
|
|
486
496
|
this.flushBuffer();
|
|
497
|
+
// Every (re)registration hands the list over again, also to a recorder
|
|
498
|
+
// that restarted on the same address: the control server says whether
|
|
499
|
+
// the service still needs it.
|
|
500
|
+
this.toolListSentTo = undefined;
|
|
501
|
+
this.sendTools();
|
|
487
502
|
this.replayOn = registered.replay === true;
|
|
488
503
|
this.pollHoldMs = registered.pollHoldMs ?? 25_000;
|
|
489
504
|
// A bound proxy always has a work loop — paced to one poll every
|
|
@@ -785,6 +800,50 @@ export class ProxySession {
|
|
|
785
800
|
}
|
|
786
801
|
return new RemoteRecorder({ baseUrl, session });
|
|
787
802
|
}
|
|
803
|
+
/**
|
|
804
|
+
* One page of the server's `tools/list` answer (the service's
|
|
805
|
+
* autocorrect.md AC12): the whole list goes to the control server once its
|
|
806
|
+
* last page is in, which sends the names, descriptions and input schemas to
|
|
807
|
+
* the service when they changed. Only where this proxy records: a server
|
|
808
|
+
* nothing is recorded of sends nothing of its tools either.
|
|
809
|
+
*/
|
|
810
|
+
reportTools(tools, page) {
|
|
811
|
+
if (page.first)
|
|
812
|
+
this.toolPages = [];
|
|
813
|
+
this.toolPages.push(...tools.slice(0, 1_000 - this.toolPages.length));
|
|
814
|
+
if (!page.last)
|
|
815
|
+
return;
|
|
816
|
+
const list = this.toolPages;
|
|
817
|
+
this.toolPages = [];
|
|
818
|
+
// A server still starting may list nothing: the list before it stands.
|
|
819
|
+
if (list.length === 0)
|
|
820
|
+
return;
|
|
821
|
+
this.toolList = list;
|
|
822
|
+
this.toolListSentTo = undefined;
|
|
823
|
+
this.sendTools();
|
|
824
|
+
}
|
|
825
|
+
/** Send the kept tool list to the control server this proxy is bound to, once. */
|
|
826
|
+
sendTools() {
|
|
827
|
+
if (this.closing || !this.toolList || !this.recording || this.tierValue !== "bound" || !this.control)
|
|
828
|
+
return;
|
|
829
|
+
if (this.toolListSentTo === this.control)
|
|
830
|
+
return;
|
|
831
|
+
const control = this.control;
|
|
832
|
+
this.toolListSentTo = control;
|
|
833
|
+
void control.tools(this.serverName, this.toolList).then((ok) => {
|
|
834
|
+
if (ok || this.toolListSentTo !== control)
|
|
835
|
+
return;
|
|
836
|
+
this.toolListSentTo = undefined;
|
|
837
|
+
// Tried once more a little later; after that, at the next bind or list.
|
|
838
|
+
if (!this.toolRetry && !this.closing) {
|
|
839
|
+
this.toolRetry = setTimeout(() => {
|
|
840
|
+
this.toolRetry = undefined;
|
|
841
|
+
this.sendTools();
|
|
842
|
+
}, 30_000);
|
|
843
|
+
this.toolRetry.unref?.();
|
|
844
|
+
}
|
|
845
|
+
});
|
|
846
|
+
}
|
|
788
847
|
/**
|
|
789
848
|
* Record one completed `tools/call`. Fire-and-forget by contract: it is called
|
|
790
849
|
* *after* the result is already on its way to the host, and it never awaits.
|
|
@@ -867,9 +926,18 @@ export class ProxySession {
|
|
|
867
926
|
const toolError = step.isError
|
|
868
927
|
? (step.errorMessage ?? serializeCapped(redact(step.result)))
|
|
869
928
|
: undefined;
|
|
929
|
+
// One id on both rows, and the failure marked (the service's review of
|
|
930
|
+
// calcscenarioautocorrect.md, item 7): the service pairs a call with its
|
|
931
|
+
// answer by it, not by the order they came in.
|
|
932
|
+
const callId = step.callId ?? newCallId();
|
|
870
933
|
this.queue.push(async () => {
|
|
871
|
-
recorder.recordToolSelected(runId, pair.selected, { toolName, toolInput });
|
|
872
|
-
recorder.recordToolResponse(runId, pair.response, {
|
|
934
|
+
recorder.recordToolSelected(runId, pair.selected, { toolName, toolInput, metadata: { callId } });
|
|
935
|
+
recorder.recordToolResponse(runId, pair.response, {
|
|
936
|
+
toolName,
|
|
937
|
+
toolOutput,
|
|
938
|
+
toolError,
|
|
939
|
+
metadata: step.isError ? { callId, isError: true } : { callId },
|
|
940
|
+
});
|
|
873
941
|
// Wait for the service to take them before the next step goes. The
|
|
874
942
|
// recorder's own per-run chain is unbounded; waiting here is what makes
|
|
875
943
|
// the step queue's bound apply to what is actually waiting on the
|
|
@@ -49,6 +49,29 @@ export interface RunMetrics {
|
|
|
49
49
|
incomplete?: {
|
|
50
50
|
lossy: boolean;
|
|
51
51
|
};
|
|
52
|
+
/**
|
|
53
|
+
* The day the agent of this run was told it is (the service's review of
|
|
54
|
+
* calcscenarioautocorrect.md, item 1). The service checks a corrected plan
|
|
55
|
+
* against this very run, and reads its prompt with this day — not the day
|
|
56
|
+
* the run reached the service, which is years off for a benchmark and a day
|
|
57
|
+
* off around midnight. See {@link RunAgentDay}.
|
|
58
|
+
*/
|
|
59
|
+
agentDay?: RunAgentDay;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* What a run's finish says of the agent's day. Each field is left out when it
|
|
63
|
+
* is not known, and the whole object when neither is.
|
|
64
|
+
*/
|
|
65
|
+
export interface RunAgentDay {
|
|
66
|
+
/**
|
|
67
|
+
* The line of the agent's own system prompt that gives the date ("Today's
|
|
68
|
+
* date is Thursday, 2023-11-30 …"), as its host wrote it into the session.
|
|
69
|
+
* Never Claude Code's date reminder: that is only this machine's day. At
|
|
70
|
+
* most 400 characters.
|
|
71
|
+
*/
|
|
72
|
+
said?: string;
|
|
73
|
+
/** This machine's calendar day, `YYYY-MM-DD` in its own time zone, when the run's prompt came. */
|
|
74
|
+
local?: string;
|
|
52
75
|
}
|
|
53
76
|
/** How long a caller is prepared to wait for {@link Recorder.flush}. */
|
|
54
77
|
export interface FlushOptions {
|
|
@@ -77,10 +100,21 @@ export interface PlanCallMark {
|
|
|
77
100
|
* the turn (R-INTENT-7). `byPlan`: the plan made this call before the agent
|
|
78
101
|
* redid the task, and it is recorded first into the agent's repeat so the
|
|
79
102
|
* repeat holds the whole turn (AC2). The service stores neither with a vector.
|
|
103
|
+
*
|
|
104
|
+
* `callId` is on both rows of one tool call, its `tool_selected` and its
|
|
105
|
+
* `tool_response`, and ties the two together (the service's review of
|
|
106
|
+
* calcscenarioautocorrect.md, item 7): Claude Code's `tool_use_id`, or the id
|
|
107
|
+
* the runner made for a call no hook named. Paired by tool name in the order
|
|
108
|
+
* they came, two calls of one tool made at once and answered out of order
|
|
109
|
+
* would take each other's answers. `isError` is on a `tool_response` row whose
|
|
110
|
+
* call failed — its MCP result said `isError`, or the host reported the call
|
|
111
|
+
* as failed — so a failed call is never read as one that worked.
|
|
80
112
|
*/
|
|
81
113
|
export type StepMetadata = {
|
|
82
114
|
pinnedBy?: string;
|
|
83
115
|
byPlan?: PlanCallMark;
|
|
116
|
+
callId?: string;
|
|
117
|
+
isError?: boolean;
|
|
84
118
|
};
|
|
85
119
|
export interface Recorder {
|
|
86
120
|
/**
|
|
@@ -301,13 +335,21 @@ export interface IntentMatchRequest {
|
|
|
301
335
|
stepPosition?: number;
|
|
302
336
|
/** Recordings this turn must not count hits against (R-HIT-7). */
|
|
303
337
|
excludeRuns?: string[];
|
|
304
|
-
/**
|
|
338
|
+
/**
|
|
339
|
+
* Per plan of this turn: its recording, and how far it got in its own chain
|
|
340
|
+
* — the step it handed over at (a decision included), or its last step when
|
|
341
|
+
* it ran to its end (R-OUT-6). One entry per plan; the service maps each onto
|
|
342
|
+
* the recording and keeps the largest (its segmentsbreakdown.md D18).
|
|
343
|
+
*/
|
|
305
344
|
ranThrough?: Array<{
|
|
306
345
|
runId: string;
|
|
307
346
|
position: number;
|
|
308
347
|
scenarioId?: string;
|
|
309
348
|
}>;
|
|
310
|
-
/**
|
|
349
|
+
/**
|
|
350
|
+
* Present only while segment arming is on (R-OUT-9, R-OUT-10): the
|
|
351
|
+
* recorder's default from 0.3.19, absent with `BIR_SEGMENT_ARM=0`.
|
|
352
|
+
*/
|
|
311
353
|
acceptSegments?: boolean;
|
|
312
354
|
/** Always true from this runner (R-CALL-28). */
|
|
313
355
|
supportsCalls?: boolean;
|
|
@@ -460,12 +502,124 @@ export interface ExecutionReport {
|
|
|
460
502
|
* ends; an older service drops the field.
|
|
461
503
|
*/
|
|
462
504
|
answer?: string;
|
|
505
|
+
/**
|
|
506
|
+
* The plan's own calls of the turn, whole, sent only beside {@link answer}
|
|
507
|
+
* (the service's review of calcscenarioautocorrect.md, item 12). A step's
|
|
508
|
+
* payloads in {@link steps} are cut to 3,800 bytes, and a correction made
|
|
509
|
+
* from the answer cannot test a condition on what a long output says — a
|
|
510
|
+
* CRM list, a long series. Each output here is cut at 64 KiB, and all of
|
|
511
|
+
* them together are held to 600,000 bytes, the largest cut first
|
|
512
|
+
* (record/remote-recorder.ts, `answerCallsBody`).
|
|
513
|
+
*/
|
|
514
|
+
answerCalls?: AnswerCall[];
|
|
515
|
+
}
|
|
516
|
+
/** One call of a plan, whole, as {@link ExecutionReport.answerCalls} carries it. */
|
|
517
|
+
export interface AnswerCall {
|
|
518
|
+
/** The step's own `stepIndex`. */
|
|
519
|
+
stepIndex: number;
|
|
520
|
+
toolName: string;
|
|
521
|
+
/** What the step sent: redacted, at most 64 KiB serialized. */
|
|
522
|
+
input: Record<string, unknown>;
|
|
523
|
+
/** What came back — the tool's whole answer, as recorded — or its error. */
|
|
524
|
+
output: string;
|
|
525
|
+
/** The call failed: `output` is what it reported instead. */
|
|
526
|
+
isError?: boolean;
|
|
527
|
+
}
|
|
528
|
+
/**
|
|
529
|
+
* What a wrapped server says of one of its tools (MCP `annotations`), as the
|
|
530
|
+
* service takes it (the service's review of calcscenarioautocorrect.md, items
|
|
531
|
+
* 4 and 16): whether it only reads, whether it can destroy something. The
|
|
532
|
+
* service judges "is this call a change?" by these first, and by the tool's
|
|
533
|
+
* name only when its server says nothing. Only these five keys; the hints are
|
|
534
|
+
* hints, as MCP says, and a server's word only.
|
|
535
|
+
*/
|
|
536
|
+
export interface ToolAnnotations {
|
|
537
|
+
title?: string;
|
|
538
|
+
readOnlyHint?: boolean;
|
|
539
|
+
destructiveHint?: boolean;
|
|
540
|
+
idempotentHint?: boolean;
|
|
541
|
+
openWorldHint?: boolean;
|
|
542
|
+
}
|
|
543
|
+
/** One tool of a wrapped MCP server, as the service keeps it (its autocorrect.md AC12). */
|
|
544
|
+
export interface ToolListEntry {
|
|
545
|
+
/** The server's own name for it. */
|
|
546
|
+
name: string;
|
|
547
|
+
/** As the agent calls it: `mcp__<server>__<tool>`. */
|
|
548
|
+
qualifiedName: string;
|
|
549
|
+
description?: string;
|
|
550
|
+
inputSchema?: Record<string, unknown>;
|
|
551
|
+
/** What its server says of it, when it says anything. */
|
|
552
|
+
annotations?: ToolAnnotations;
|
|
463
553
|
}
|
|
554
|
+
/** Optional capability: a server's tool list goes to the service (AC12). A NullRecorder has none. */
|
|
555
|
+
export interface ToolListReporter {
|
|
556
|
+
/**
|
|
557
|
+
* Resolves true when the service took the list. Never throws. `project` is
|
|
558
|
+
* the list's project: the first 16 hex characters of the sha256 of its
|
|
559
|
+
* folder (control/server.ts, `projectKey`), the same value every run of
|
|
560
|
+
* that project carries as `metadata.project`. Left out of the body when
|
|
561
|
+
* absent, as an older caller sends it.
|
|
562
|
+
*/
|
|
563
|
+
reportTools(serverName: string, tools: readonly ToolListEntry[], project?: string): Promise<boolean>;
|
|
564
|
+
}
|
|
565
|
+
export declare function isToolListReporter(r: Recorder): r is Recorder & ToolListReporter;
|
|
464
566
|
/** Optional capability: reporting needs a service, and a NullRecorder has none. */
|
|
465
567
|
export interface ScenarioReporter {
|
|
466
568
|
reportExecution(report: ExecutionReport): void;
|
|
467
569
|
}
|
|
468
570
|
export declare function isScenarioReporter(r: Recorder): r is Recorder & ScenarioReporter;
|
|
571
|
+
/**
|
|
572
|
+
* One plan that waits for its owner (`GET /scenarios/attention`; the service's
|
|
573
|
+
* review of calcscenarioautocorrect.md, item 8): held, and not handed out, so
|
|
574
|
+
* every prompt of its task is a full agent run until somebody looks.
|
|
575
|
+
*/
|
|
576
|
+
export interface AttentionPlan {
|
|
577
|
+
scenarioId: string;
|
|
578
|
+
title: string;
|
|
579
|
+
/** When the hold began, ISO 8601. Null for a plan that is not held: only a correction waits. */
|
|
580
|
+
heldSince: string | null;
|
|
581
|
+
/** Why it is held; null when it is not. */
|
|
582
|
+
reason: "checking" | "rejected" | "rejected_again" | "verifying" | "verify_failed" | "verify_stale" | null;
|
|
583
|
+
/** `proposed`: a correction waits for the owner's word. */
|
|
584
|
+
correction: "proposed" | null;
|
|
585
|
+
/** The plan's recording page. */
|
|
586
|
+
url: string;
|
|
587
|
+
}
|
|
588
|
+
/**
|
|
589
|
+
* Optional capability: the plans of this account that need their owner, for
|
|
590
|
+
* the line a session start shows. Null whenever nothing can be said — no
|
|
591
|
+
* service, an older one (404), a slow or failing one. Never throws.
|
|
592
|
+
*/
|
|
593
|
+
export interface AttentionReader {
|
|
594
|
+
scenarioAttention(opts?: {
|
|
595
|
+
timeoutMs?: number;
|
|
596
|
+
}): Promise<AttentionPlan[] | null>;
|
|
597
|
+
}
|
|
598
|
+
export declare function isAttentionReader(r: Recorder): r is Recorder & AttentionReader;
|
|
599
|
+
/** What the service answered a report that a replay was wrong. */
|
|
600
|
+
export interface DisputeAnswer {
|
|
601
|
+
/** The service took the report. */
|
|
602
|
+
ok: boolean;
|
|
603
|
+
/** The plan is held now: not handed out until it is checked. */
|
|
604
|
+
held?: boolean;
|
|
605
|
+
/** Why not, in the service's own short words, or the runner's own code. */
|
|
606
|
+
why?: string;
|
|
607
|
+
}
|
|
608
|
+
/**
|
|
609
|
+
* Optional capability: the agent said a replay's results do not answer the
|
|
610
|
+
* request (`mcp__bir__report_wrong_result`; the service's review of
|
|
611
|
+
* calcscenarioautocorrect.md, item 19). `POST /scenarios/:id/dispute`.
|
|
612
|
+
* Never throws: every failure is an answer with `ok: false`.
|
|
613
|
+
*/
|
|
614
|
+
export interface ScenarioDisputer {
|
|
615
|
+
disputeScenario(scenarioId: string, report: {
|
|
616
|
+
reason: string;
|
|
617
|
+
executionId?: string;
|
|
618
|
+
}, opts?: {
|
|
619
|
+
timeoutMs?: number;
|
|
620
|
+
}): Promise<DisputeAnswer>;
|
|
621
|
+
}
|
|
622
|
+
export declare function isScenarioDisputer(r: Recorder): r is Recorder & ScenarioDisputer;
|
|
469
623
|
/**
|
|
470
624
|
* A recorder that drops everything. Used when auth is unavailable: recording is
|
|
471
625
|
* best-effort, and a session must never fail because BaseIn is unreachable (§10).
|
package/dist/record/recorder.js
CHANGED
|
@@ -20,9 +20,18 @@ export function isServiceProber(r) {
|
|
|
20
20
|
export function isIntentMatcher(r) {
|
|
21
21
|
return typeof r.matchIntent === "function";
|
|
22
22
|
}
|
|
23
|
+
export function isToolListReporter(r) {
|
|
24
|
+
return typeof r.reportTools === "function";
|
|
25
|
+
}
|
|
23
26
|
export function isScenarioReporter(r) {
|
|
24
27
|
return typeof r.reportExecution === "function";
|
|
25
28
|
}
|
|
29
|
+
export function isAttentionReader(r) {
|
|
30
|
+
return typeof r.scenarioAttention === "function";
|
|
31
|
+
}
|
|
32
|
+
export function isScenarioDisputer(r) {
|
|
33
|
+
return typeof r.disputeScenario === "function";
|
|
34
|
+
}
|
|
26
35
|
/**
|
|
27
36
|
* A recorder that drops everything. Used when auth is unavailable: recording is
|
|
28
37
|
* best-effort, and a session must never fail because BaseIn is unreachable (§10).
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
*/
|
|
51
51
|
import { type AuthSession } from "../auth/client.js";
|
|
52
52
|
import { type ServiceHealth } from "../util/service-health.js";
|
|
53
|
-
import type { ExecutionReport, FlushOptions, IntentMatchAnswer, IntentMatchOptions, IntentMatchRequest, Recorder, RecorderHealth, RecorderHealthReporter, RunCreation, RunMatch, RunMetrics, StartRunOptions, StepMetadata } from "./recorder.js";
|
|
53
|
+
import type { AnswerCall, AttentionPlan, DisputeAnswer, ExecutionReport, FlushOptions, IntentMatchAnswer, IntentMatchOptions, IntentMatchRequest, Recorder, RecorderHealth, RecorderHealthReporter, RunCreation, RunMatch, RunMetrics, StartRunOptions, StepMetadata, ToolListEntry } from "./recorder.js";
|
|
54
54
|
/**
|
|
55
55
|
* Caps for the free text this recorder sends, in UTF-8 bytes. Tool payloads
|
|
56
56
|
* arrive capped at 64 KiB by their producers; text never passed through one,
|
|
@@ -118,6 +118,23 @@ export interface RemoteRecorderOptions {
|
|
|
118
118
|
/** The longest one queued send waits for a failing service ({@link COOLDOWN_WAIT_MAX_MS}). Tests shorten it. */
|
|
119
119
|
cooldownWaitMaxMs?: number;
|
|
120
120
|
}
|
|
121
|
+
/** What one answered turn's call may carry of its output, in UTF-8 bytes (review item 12). */
|
|
122
|
+
export declare const ANSWER_CALL_OUTPUT_MAX_BYTES: number;
|
|
123
|
+
/** All of one report's answer calls together, serialized, in UTF-8 bytes. */
|
|
124
|
+
export declare const ANSWER_CALLS_MAX_BYTES = 600000;
|
|
125
|
+
/**
|
|
126
|
+
* An answered turn's calls as they are sent (the service's review of
|
|
127
|
+
* calcscenarioautocorrect.md, item 12): each input redacted again, each output
|
|
128
|
+
* redacted and cut at {@link ANSWER_CALL_OUTPUT_MAX_BYTES}, and all of them
|
|
129
|
+
* held to `maxBytes` serialized by cutting the largest outputs first — every
|
|
130
|
+
* output above one common length is cut to it, the longest length that fits.
|
|
131
|
+
* The cut is the recordings' own, "… [truncated N of M bytes]", so a reader
|
|
132
|
+
* can tell a short answer from a cut one. Undefined when there is nothing to
|
|
133
|
+
* send, or when even outputs of {@link ANSWER_CALL_OUTPUT_MIN_BYTES} would not
|
|
134
|
+
* fit (inputs that large): the service then reads the report's cut copies,
|
|
135
|
+
* as it did before, rather than half a list.
|
|
136
|
+
*/
|
|
137
|
+
export declare function answerCallsBody(calls: readonly AnswerCall[] | undefined, maxBytes?: number): AnswerCall[] | undefined;
|
|
121
138
|
export declare class RemoteRecorder implements Recorder, RecorderHealthReporter {
|
|
122
139
|
private readonly baseUrl;
|
|
123
140
|
private readonly framework;
|
|
@@ -247,6 +264,47 @@ export declare class RemoteRecorder implements Recorder, RecorderHealthReporter
|
|
|
247
264
|
* or refused in between is as lost as one before.
|
|
248
265
|
*/
|
|
249
266
|
private incompleteOf;
|
|
267
|
+
/**
|
|
268
|
+
* One wrapped server's tool list (the service's autocorrect.md AC12): the
|
|
269
|
+
* names, descriptions and input schemas a correction from an agent's answer
|
|
270
|
+
* writes a step with. On its own lane, and never counted as a recording's
|
|
271
|
+
* lost send. Quiet towards the service's health: a list the service refuses
|
|
272
|
+
* (an older one's 404, the route's own rate limit, a fault of that route)
|
|
273
|
+
* opens no breaker the hooks wait on. Resolves true when the service took it.
|
|
274
|
+
*
|
|
275
|
+
* `project` is the list's project (review item 16): two projects with a
|
|
276
|
+
* server of one name keep a list each. An older service drops the field.
|
|
277
|
+
*/
|
|
278
|
+
reportTools(serverName: string, tools: readonly ToolListEntry[], project?: string): Promise<boolean>;
|
|
279
|
+
/** The account this recorder sends for, as signed in now. */
|
|
280
|
+
accountId(): string | undefined;
|
|
281
|
+
/**
|
|
282
|
+
* The plans of this account that wait for their owner (`GET
|
|
283
|
+
* /scenarios/attention`; the service's review of calcscenarioautocorrect.md,
|
|
284
|
+
* item 8): what the line a session start shows is made of. Asked on a
|
|
285
|
+
* session start's path, so it is bounded (1.5 s by default), asked only
|
|
286
|
+
* while the service is known to answer, and quiet towards its health: a
|
|
287
|
+
* slow or refusing answer here opens no breaker the hooks wait on. Null on
|
|
288
|
+
* any failure — an older service's 404 included — and never a throw.
|
|
289
|
+
*/
|
|
290
|
+
scenarioAttention(opts?: {
|
|
291
|
+
timeoutMs?: number;
|
|
292
|
+
}): Promise<AttentionPlan[] | null>;
|
|
293
|
+
/**
|
|
294
|
+
* The agent said a replay's results do not answer its request (`POST
|
|
295
|
+
* /scenarios/:scenarioId/dispute`; the service's review of
|
|
296
|
+
* calcscenarioautocorrect.md, item 19). `executionId` is the turn's
|
|
297
|
+
* execution ticket — the id its report books under at the end of the turn,
|
|
298
|
+
* so while the turn runs the service has the ticket and not yet the row.
|
|
299
|
+
* Awaited — the agent waits for one sentence back — bounded (10 s by
|
|
300
|
+
* default), and quiet towards the service's health. Never throws.
|
|
301
|
+
*/
|
|
302
|
+
disputeScenario(scenarioId: string, report: {
|
|
303
|
+
reason: string;
|
|
304
|
+
executionId?: string;
|
|
305
|
+
}, opts?: {
|
|
306
|
+
timeoutMs?: number;
|
|
307
|
+
}): Promise<DisputeAnswer>;
|
|
250
308
|
/**
|
|
251
309
|
* Report what a matched turn cost (docs/calculatedReplay.md §11.2).
|
|
252
310
|
*
|