@basein/runner 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/auth/client.d.ts +12 -0
- package/dist/auth/client.js +41 -0
- package/dist/bin/bir-hooks.d.ts +18 -3
- package/dist/bin/bir-hooks.js +70 -4
- package/dist/bin/bir.js +56 -3
- package/dist/control/server.d.ts +84 -1
- package/dist/control/server.js +555 -51
- package/dist/control/transcript.d.ts +40 -0
- package/dist/control/transcript.js +105 -0
- package/dist/record/recorder.d.ts +178 -4
- package/dist/record/recorder.js +6 -0
- package/dist/record/remote-recorder.d.ts +20 -2
- package/dist/record/remote-recorder.js +66 -6
- package/dist/replay/bundle.d.ts +10 -1
- package/dist/replay/bundle.js +41 -3
- package/dist/replay/controller.d.ts +179 -6
- package/dist/replay/controller.js +584 -55
- package/dist/replay/coverage.js +2 -2
- package/dist/replay/derive.d.ts +54 -7
- package/dist/replay/derive.js +174 -17
- package/dist/replay/flatten.d.ts +125 -0
- package/dist/replay/flatten.js +182 -0
- package/dist/replay/handover.d.ts +60 -0
- package/dist/replay/handover.js +82 -0
- package/dist/replay/logic.d.ts +11 -0
- package/dist/replay/logic.js +17 -0
- package/dist/replay/plan.d.ts +105 -8
- package/dist/replay/plan.js +309 -47
- package/dist/replay/source-run.d.ts +24 -10
- package/dist/replay/source-run.js +65 -30
- package/dist/replay/types.d.ts +108 -5
- package/dist/replay/types.js +33 -3
- package/docs/calculatedReplay.md +16 -8
- package/docs/calculatedReplayGuide.md +4 -4
- package/package.json +1 -1
|
@@ -18,6 +18,46 @@
|
|
|
18
18
|
* ever required for a recording to be valid — a missing context is a missing
|
|
19
19
|
* nicety, not a failed step.
|
|
20
20
|
*/
|
|
21
|
+
/**
|
|
22
|
+
* The last `n` tool results of the *current* prompt, redacted and capped.
|
|
23
|
+
*
|
|
24
|
+
* Derivation reads them because a target is often named by an earlier step's
|
|
25
|
+
* output and never by the prompt — "the worst service" is a name the agent
|
|
26
|
+
* learned from a tool, not from the user (segmented.md R-PARAM-2).
|
|
27
|
+
*
|
|
28
|
+
* "This prompt" is everything after the last user-role line that carries text.
|
|
29
|
+
* A transcript compacted mid-turn yields whatever follows that line, which may
|
|
30
|
+
* be nothing; a transcript with no user prompt at all (a run opened by a proxy
|
|
31
|
+
* step) yields nothing. Results with `is_error: true` are kept: a failure is
|
|
32
|
+
* still the agent's evidence about what it is working on.
|
|
33
|
+
*/
|
|
34
|
+
export declare function recentToolResults(transcriptPath: string | undefined, n?: number, maxBytes?: number): string[];
|
|
35
|
+
/** Which rung of R-INTENT-1 a step's intent came from. */
|
|
36
|
+
export type IntentSource = "text" | "thinking" | "tool";
|
|
37
|
+
/** What the agent meant by a tool call, and where that was read from. */
|
|
38
|
+
export interface ToolUseIntent {
|
|
39
|
+
text: string;
|
|
40
|
+
source: "text" | "thinking";
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* The intent of one tool call, read the way the transcript really stores it
|
|
44
|
+
* (segmented.md R-INTENT-3).
|
|
45
|
+
*
|
|
46
|
+
* Claude Code writes **one line per content block**, and every line of a
|
|
47
|
+
* multi-block assistant message repeats the same `message.id`. So the `text`
|
|
48
|
+
* block, the `thinking` block and the `tool_use` block of one message are three
|
|
49
|
+
* separate lines. {@link contextForToolUse} reads only the line holding the
|
|
50
|
+
* `tool_use` block, which is why it has returned "" in practice since it was
|
|
51
|
+
* written: the text is always on a sibling line.
|
|
52
|
+
*
|
|
53
|
+
* Order is R-INTENT-1: the written text, else the thinking text, else nothing —
|
|
54
|
+
* and the caller then lets the *service* build a line from the tool name and
|
|
55
|
+
* arguments, so the runner never invents one.
|
|
56
|
+
*
|
|
57
|
+
* `sources` is `BIR_INTENT_SOURCES`: a source not in the set is skipped and the
|
|
58
|
+
* next rung tried. It governs only what may be *sent*; it never stops a probe.
|
|
59
|
+
*/
|
|
60
|
+
export declare function intentForToolUse(transcriptPath: string | undefined, toolUseId: string, sources?: ReadonlySet<IntentSource>): ToolUseIntent | undefined;
|
|
21
61
|
/** The assistant text that accompanied `toolUseId`, or "" when unavailable. */
|
|
22
62
|
export declare function contextForToolUse(transcriptPath: string | undefined, toolUseId: string): string;
|
|
23
63
|
/** The last assistant text in the transcript — the turn's final answer. */
|
|
@@ -19,6 +19,8 @@
|
|
|
19
19
|
* nicety, not a failed step.
|
|
20
20
|
*/
|
|
21
21
|
import { readFileSync } from "node:fs";
|
|
22
|
+
import { redactText } from "../record/redact.js";
|
|
23
|
+
import { truncateString } from "../record/truncate.js";
|
|
22
24
|
function readLines(transcriptPath) {
|
|
23
25
|
let raw;
|
|
24
26
|
try {
|
|
@@ -56,6 +58,109 @@ function textOf(blocks) {
|
|
|
56
58
|
.join("\n")
|
|
57
59
|
.trim();
|
|
58
60
|
}
|
|
61
|
+
/** One `tool_result` block's payload, flattened to text. */
|
|
62
|
+
function resultTextOf(block) {
|
|
63
|
+
const content = block.content;
|
|
64
|
+
if (typeof content === "string")
|
|
65
|
+
return content;
|
|
66
|
+
if (Array.isArray(content))
|
|
67
|
+
return textOf(content);
|
|
68
|
+
if (typeof block.text === "string")
|
|
69
|
+
return block.text;
|
|
70
|
+
return "";
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* The last `n` tool results of the *current* prompt, redacted and capped.
|
|
74
|
+
*
|
|
75
|
+
* Derivation reads them because a target is often named by an earlier step's
|
|
76
|
+
* output and never by the prompt — "the worst service" is a name the agent
|
|
77
|
+
* learned from a tool, not from the user (segmented.md R-PARAM-2).
|
|
78
|
+
*
|
|
79
|
+
* "This prompt" is everything after the last user-role line that carries text.
|
|
80
|
+
* A transcript compacted mid-turn yields whatever follows that line, which may
|
|
81
|
+
* be nothing; a transcript with no user prompt at all (a run opened by a proxy
|
|
82
|
+
* step) yields nothing. Results with `is_error: true` are kept: a failure is
|
|
83
|
+
* still the agent's evidence about what it is working on.
|
|
84
|
+
*/
|
|
85
|
+
export function recentToolResults(transcriptPath, n = 5, maxBytes = 2048) {
|
|
86
|
+
if (!transcriptPath || n <= 0)
|
|
87
|
+
return [];
|
|
88
|
+
const lines = readLines(transcriptPath);
|
|
89
|
+
let start = -1;
|
|
90
|
+
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
|
91
|
+
const role = lines[i].message?.role ?? lines[i].type;
|
|
92
|
+
if (role !== "user")
|
|
93
|
+
continue;
|
|
94
|
+
if (textOf(blocksOf(lines[i]))) {
|
|
95
|
+
start = i;
|
|
96
|
+
break;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
if (start === -1)
|
|
100
|
+
return [];
|
|
101
|
+
const out = [];
|
|
102
|
+
for (let i = start; i < lines.length; i += 1) {
|
|
103
|
+
const role = lines[i].message?.role ?? lines[i].type;
|
|
104
|
+
if (role !== "user")
|
|
105
|
+
continue;
|
|
106
|
+
for (const block of blocksOf(lines[i])) {
|
|
107
|
+
if (block.type !== "tool_result")
|
|
108
|
+
continue;
|
|
109
|
+
const text = resultTextOf(block);
|
|
110
|
+
if (text)
|
|
111
|
+
out.push(text);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
return out.slice(-n).map((t) => truncateString(redactText(t), maxBytes));
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* The intent of one tool call, read the way the transcript really stores it
|
|
118
|
+
* (segmented.md R-INTENT-3).
|
|
119
|
+
*
|
|
120
|
+
* Claude Code writes **one line per content block**, and every line of a
|
|
121
|
+
* multi-block assistant message repeats the same `message.id`. So the `text`
|
|
122
|
+
* block, the `thinking` block and the `tool_use` block of one message are three
|
|
123
|
+
* separate lines. {@link contextForToolUse} reads only the line holding the
|
|
124
|
+
* `tool_use` block, which is why it has returned "" in practice since it was
|
|
125
|
+
* written: the text is always on a sibling line.
|
|
126
|
+
*
|
|
127
|
+
* Order is R-INTENT-1: the written text, else the thinking text, else nothing —
|
|
128
|
+
* and the caller then lets the *service* build a line from the tool name and
|
|
129
|
+
* arguments, so the runner never invents one.
|
|
130
|
+
*
|
|
131
|
+
* `sources` is `BIR_INTENT_SOURCES`: a source not in the set is skipped and the
|
|
132
|
+
* next rung tried. It governs only what may be *sent*; it never stops a probe.
|
|
133
|
+
*/
|
|
134
|
+
export function intentForToolUse(transcriptPath, toolUseId, sources = new Set(["text", "thinking", "tool"])) {
|
|
135
|
+
if (!transcriptPath || !toolUseId)
|
|
136
|
+
return undefined;
|
|
137
|
+
const lines = readLines(transcriptPath);
|
|
138
|
+
const holder = lines.find((line) => blocksOf(line).some((b) => b.type === "tool_use" && b.id === toolUseId));
|
|
139
|
+
if (!holder)
|
|
140
|
+
return undefined;
|
|
141
|
+
// No `message.id` means an older transcript, or a hand-written fixture: the
|
|
142
|
+
// line is the whole message and there are no siblings to gather.
|
|
143
|
+
const messageId = holder.message?.id;
|
|
144
|
+
const siblings = messageId
|
|
145
|
+
? lines.filter((line) => line.message?.id === messageId)
|
|
146
|
+
: [holder];
|
|
147
|
+
if (sources.has("text")) {
|
|
148
|
+
const text = textOf(siblings.flatMap((line) => blocksOf(line)));
|
|
149
|
+
if (text)
|
|
150
|
+
return { text, source: "text" };
|
|
151
|
+
}
|
|
152
|
+
if (sources.has("thinking")) {
|
|
153
|
+
const thinking = siblings
|
|
154
|
+
.flatMap((line) => blocksOf(line))
|
|
155
|
+
.filter((b) => b.type === "thinking" && typeof b.thinking === "string")
|
|
156
|
+
.map((b) => b.thinking)
|
|
157
|
+
.join("\n")
|
|
158
|
+
.trim();
|
|
159
|
+
if (thinking)
|
|
160
|
+
return { text: thinking, source: "thinking" };
|
|
161
|
+
}
|
|
162
|
+
return undefined;
|
|
163
|
+
}
|
|
59
164
|
/** The assistant text that accompanied `toolUseId`, or "" when unavailable. */
|
|
60
165
|
export function contextForToolUse(transcriptPath, toolUseId) {
|
|
61
166
|
if (!transcriptPath || !toolUseId)
|
|
@@ -5,18 +5,48 @@
|
|
|
5
5
|
* service needs no new endpoints for v1: a BaseInstRunner run is the same row
|
|
6
6
|
* shape as an RRepeat one, discriminated only by `metadata.recorder`.
|
|
7
7
|
*/
|
|
8
|
-
import type { ExecutionStage, ExecutionStepResult } from "../replay/types.js";
|
|
8
|
+
import type { ExecutionStage, ExecutionStepResult, FallbackKind } from "../replay/types.js";
|
|
9
|
+
export interface StartRunOptions {
|
|
10
|
+
fallbackOf?: {
|
|
11
|
+
scenarioId: string;
|
|
12
|
+
stepIndex: number;
|
|
13
|
+
};
|
|
14
|
+
}
|
|
9
15
|
export interface RunMetrics {
|
|
10
16
|
costUsd?: number;
|
|
11
17
|
durationMs?: number;
|
|
12
18
|
}
|
|
13
19
|
export interface Recorder {
|
|
14
|
-
/**
|
|
15
|
-
|
|
20
|
+
/**
|
|
21
|
+
* Create the run; returns the client-generated runId synchronously.
|
|
22
|
+
*
|
|
23
|
+
* `options.fallbackOf` marks a *fragment* (fallbk.md D7): the model's own work
|
|
24
|
+
* after a calculated scenario handed it the task at `stepIndex`. The service
|
|
25
|
+
* never prompt-matches a fragment.
|
|
26
|
+
*/
|
|
27
|
+
startRun(input: string, metadata?: Record<string, unknown>, options?: StartRunOptions): string;
|
|
16
28
|
recordToolSelected(runId: string, stepIndex: number, d: {
|
|
17
29
|
toolName: string;
|
|
18
30
|
toolInput: string;
|
|
19
31
|
context?: string;
|
|
32
|
+
/**
|
|
33
|
+
* The thinking text this call was chosen in, when there was no written
|
|
34
|
+
* text (segmented.md R-INTENT-1). It travels to the service only to be
|
|
35
|
+
* embedded and is never stored as text (R-INTENT-2), which is why it is
|
|
36
|
+
* carried apart from `context` rather than folded into it.
|
|
37
|
+
*/
|
|
38
|
+
intent?: {
|
|
39
|
+
text: string;
|
|
40
|
+
source: "thinking";
|
|
41
|
+
};
|
|
42
|
+
/**
|
|
43
|
+
* Set when a plan executed this step, to the plan's scenario id
|
|
44
|
+
* (R-INTENT-7). The service stores such a step without a vector, so the
|
|
45
|
+
* system's own replays never count as recurrences of the work.
|
|
46
|
+
*/
|
|
47
|
+
metadata?: {
|
|
48
|
+
pinnedBy: string;
|
|
49
|
+
};
|
|
20
50
|
}): string;
|
|
21
51
|
recordToolResponse(runId: string, stepIndex: number, d: {
|
|
22
52
|
toolName: string;
|
|
@@ -36,18 +66,145 @@ export interface Recorder {
|
|
|
36
66
|
* callers must stop recording it. What the caller does *instead* of recording is
|
|
37
67
|
* v2's subject (docs/calculatedReplay.md); stopping is mandatory either way.
|
|
38
68
|
*/
|
|
69
|
+
/** Which of a segment's two keys scored against the live step (R-OUT-3). */
|
|
70
|
+
export type MatchKey = "intent" | "firstStep";
|
|
71
|
+
/** The cheap model's answer about one live step and one segment (R-VERIFY-1). */
|
|
72
|
+
export interface MatchVerdict {
|
|
73
|
+
model: string;
|
|
74
|
+
verdict: "same" | "different" | "skipped";
|
|
75
|
+
reason?: string;
|
|
76
|
+
}
|
|
77
|
+
/** Where a handed-out segment sits in its recording (R-OUT-8). */
|
|
78
|
+
export interface MatchSegment {
|
|
79
|
+
runId: string;
|
|
80
|
+
stepFrom: number;
|
|
81
|
+
stepTo: number;
|
|
82
|
+
/** `tool_selected` rows of the recording before `stepFrom` (R-OUT-6). */
|
|
83
|
+
firstPosition: number;
|
|
84
|
+
hitCount?: number;
|
|
85
|
+
}
|
|
39
86
|
export interface RunMatch {
|
|
40
87
|
runId: string;
|
|
41
88
|
scenarioId: string | null;
|
|
42
89
|
similarity: number;
|
|
43
90
|
scenario: Record<string, unknown> | null;
|
|
44
91
|
executionTicket?: string;
|
|
92
|
+
/**
|
|
93
|
+
* Server policy for this scenario (fallbk.md D8). A step whose `failureCount`
|
|
94
|
+
* is greater than `maxStepFailures` is handed to the model, not attempted.
|
|
95
|
+
* Absent from an older service, which means never.
|
|
96
|
+
*/
|
|
97
|
+
fallback?: {
|
|
98
|
+
maxStepFailures: number;
|
|
99
|
+
};
|
|
100
|
+
/**
|
|
101
|
+
* What kind of row matched (segmented.md R-OUT-7). Absent from an older
|
|
102
|
+
* service, and then read as `scenario`.
|
|
103
|
+
*/
|
|
104
|
+
kind?: "scenario" | "segment";
|
|
105
|
+
segment?: MatchSegment;
|
|
106
|
+
key?: MatchKey;
|
|
107
|
+
verified?: MatchVerdict;
|
|
108
|
+
/** The frozen name, beside the runtime `intent` the plan runs on (R-INTENT-12). */
|
|
109
|
+
intentName?: string;
|
|
45
110
|
}
|
|
46
111
|
/** Optional capability: `null` means "fresh run, keep recording". */
|
|
47
112
|
export interface MatchAware {
|
|
48
113
|
getMatch(runId: string): Promise<RunMatch | null>;
|
|
49
114
|
}
|
|
50
115
|
export declare function isMatchAware(r: Recorder): r is Recorder & MatchAware;
|
|
116
|
+
/**
|
|
117
|
+
* Optional capability: whether the service actually created a run.
|
|
118
|
+
*
|
|
119
|
+
* A fragment's create call can answer `id: null` — a hand-over at the same step
|
|
120
|
+
* was recorded before, and this one is a hit on it — and then every step post
|
|
121
|
+
* for the id would 404. Resolves true when a run exists (or nobody can say).
|
|
122
|
+
*/
|
|
123
|
+
/** What the create call said about a run: fresh, or a hit on a fragment. */
|
|
124
|
+
export interface RunCreation {
|
|
125
|
+
created: boolean;
|
|
126
|
+
/** Set when this was a hit on an existing fragment (R-FALL-7, R-HIT-7). */
|
|
127
|
+
fragmentOf?: {
|
|
128
|
+
runId: string;
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
export interface RunCreationAware {
|
|
132
|
+
runCreated(runId: string): Promise<RunCreation>;
|
|
133
|
+
}
|
|
134
|
+
export declare function isRunCreationAware(r: Recorder): r is Recorder & RunCreationAware;
|
|
135
|
+
/** What a ReAct iteration sends to find a scenario by its intent (fallbk.md §Runner 4). */
|
|
136
|
+
export interface IntentMatchRequest {
|
|
137
|
+
/**
|
|
138
|
+
* The step's intent (segmented.md R-INTENT-1). May be empty: a call made with
|
|
139
|
+
* no reasoning still probes, and the service builds a line from the tool name
|
|
140
|
+
* and arguments instead (R-HIT-4).
|
|
141
|
+
*/
|
|
142
|
+
text: string;
|
|
143
|
+
currentToolName?: string;
|
|
144
|
+
/** Scenario ids already armed this turn. */
|
|
145
|
+
exclude?: string[];
|
|
146
|
+
/** Which rung the text came from (R-INTENT-3). */
|
|
147
|
+
source?: IntentSourceName;
|
|
148
|
+
/** The same value as `currentToolName`; what the new server reads (R-HIT-6). */
|
|
149
|
+
toolName?: string;
|
|
150
|
+
/** Redacted and capped arguments of the call about to be made (R-HIT-4). */
|
|
151
|
+
toolInput?: string;
|
|
152
|
+
/** The turn's identity, fixed for its whole life (R-HIT-6). */
|
|
153
|
+
turnId?: string;
|
|
154
|
+
/** `run.ordering.next` — where this call sits in the turn. */
|
|
155
|
+
stepPosition?: number;
|
|
156
|
+
/** Recordings this turn must not count hits against (R-HIT-7). */
|
|
157
|
+
excludeRuns?: string[];
|
|
158
|
+
/** Per recording a plan of this turn came from, the largest position (R-OUT-6). */
|
|
159
|
+
ranThrough?: Array<{
|
|
160
|
+
runId: string;
|
|
161
|
+
position: number;
|
|
162
|
+
scenarioId?: string;
|
|
163
|
+
}>;
|
|
164
|
+
/** Present only with `BIR_SEGMENT_ARM` (R-OUT-9, R-OUT-10). */
|
|
165
|
+
acceptSegments?: boolean;
|
|
166
|
+
/** Always true from this runner (R-CALL-28). */
|
|
167
|
+
supportsCalls?: boolean;
|
|
168
|
+
}
|
|
169
|
+
/** Which text a step's intent came from. */
|
|
170
|
+
export type IntentSourceName = "text" | "thinking" | "tool";
|
|
171
|
+
/** One recorded step a live step matched (R-HIT-6). */
|
|
172
|
+
export interface StepHit {
|
|
173
|
+
runId: string;
|
|
174
|
+
stepIndex: number;
|
|
175
|
+
similarity: number;
|
|
176
|
+
hitCount?: number;
|
|
177
|
+
}
|
|
178
|
+
/** What a segment *would* have armed, when no ticket was issued (R-OUT-9). */
|
|
179
|
+
export interface SegmentWouldArm {
|
|
180
|
+
scenarioId: string;
|
|
181
|
+
runId: string;
|
|
182
|
+
key?: MatchKey;
|
|
183
|
+
similarity: number;
|
|
184
|
+
band?: string;
|
|
185
|
+
verified?: MatchVerdict;
|
|
186
|
+
tool?: string;
|
|
187
|
+
firstTool?: string;
|
|
188
|
+
stepFrom?: number;
|
|
189
|
+
stepTo?: number;
|
|
190
|
+
hitCount?: number;
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* The whole answer to a probe, not just its match. The counting fields are what
|
|
194
|
+
* make the observe-only period readable (R-OUT-10), so they must survive the
|
|
195
|
+
* trip rather than being discarded with the rest of the body.
|
|
196
|
+
*/
|
|
197
|
+
export interface IntentMatchAnswer {
|
|
198
|
+
matched: RunMatch | null;
|
|
199
|
+
stepHit?: StepHit | null;
|
|
200
|
+
bestStepSimilarity?: number;
|
|
201
|
+
segmentWouldArm?: SegmentWouldArm;
|
|
202
|
+
}
|
|
203
|
+
/** Optional capability: intent matching needs a service. */
|
|
204
|
+
export interface IntentMatcher {
|
|
205
|
+
matchIntent(req: IntentMatchRequest): Promise<IntentMatchAnswer | null>;
|
|
206
|
+
}
|
|
207
|
+
export declare function isIntentMatcher(r: Recorder): r is Recorder & IntentMatcher;
|
|
51
208
|
/**
|
|
52
209
|
* What a matched turn actually cost, reported once per match
|
|
53
210
|
* (docs/calculatedReplay.md §11).
|
|
@@ -61,7 +218,7 @@ export interface ExecutionReport {
|
|
|
61
218
|
scenarioId: string;
|
|
62
219
|
/** The match's claim token. It *is* the execution row's id, so a doubled report books once. */
|
|
63
220
|
ticket?: string;
|
|
64
|
-
outcome: "steered_full" | "diverged" | "not_steered" | "failed";
|
|
221
|
+
outcome: "steered_full" | "diverged" | "not_steered" | "failed" | "fell_back";
|
|
65
222
|
/** The derivation call — the only tokens replay itself spends. */
|
|
66
223
|
deriveCostUsd: number;
|
|
67
224
|
/** The live turn, from the transcript usage delta. */
|
|
@@ -89,6 +246,23 @@ export interface ExecutionReport {
|
|
|
89
246
|
errorStage?: ExecutionStage;
|
|
90
247
|
errorStepIndex?: number;
|
|
91
248
|
errorToolName?: string;
|
|
249
|
+
/** Where a `fell_back` execution handed over, and why (fallbk.md). */
|
|
250
|
+
fallbackStepIndex?: number;
|
|
251
|
+
fallbackKind?: FallbackKind;
|
|
252
|
+
/**
|
|
253
|
+
* False when another plan armed in this same turn, so this report's cost is
|
|
254
|
+
* not a measurement of what the task costs unaided (segmented.md R-MONEY-5).
|
|
255
|
+
* The service then adds no baseline sample from it. Absent means eligible,
|
|
256
|
+
* which is what every report said before this field existed.
|
|
257
|
+
*/
|
|
258
|
+
baselineEligible?: boolean;
|
|
259
|
+
/**
|
|
260
|
+
* Segments of this plan's own recording that ran later in the same turn
|
|
261
|
+
* (segmented.md R-MONEY-4). The service freezes this row's baseline at what
|
|
262
|
+
* is left after their share, so one recording's steps are never counted in
|
|
263
|
+
* two baselines of one turn. The segments' own rows are untouched.
|
|
264
|
+
*/
|
|
265
|
+
sharedWith?: string[];
|
|
92
266
|
}
|
|
93
267
|
/** Optional capability: reporting needs a service, and a NullRecorder has none. */
|
|
94
268
|
export interface ScenarioReporter {
|
package/dist/record/recorder.js
CHANGED
|
@@ -8,6 +8,12 @@
|
|
|
8
8
|
export function isMatchAware(r) {
|
|
9
9
|
return typeof r.getMatch === "function";
|
|
10
10
|
}
|
|
11
|
+
export function isRunCreationAware(r) {
|
|
12
|
+
return typeof r.runCreated === "function";
|
|
13
|
+
}
|
|
14
|
+
export function isIntentMatcher(r) {
|
|
15
|
+
return typeof r.matchIntent === "function";
|
|
16
|
+
}
|
|
11
17
|
export function isScenarioReporter(r) {
|
|
12
18
|
return typeof r.reportExecution === "function";
|
|
13
19
|
}
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
* {@link RemoteRecorder.getMatch} is how they find out. v1 only stops; replay is v2.
|
|
24
24
|
*/
|
|
25
25
|
import { type AuthSession } from "../auth/client.js";
|
|
26
|
-
import type { ExecutionReport, Recorder, RunMatch, RunMetrics } from "./recorder.js";
|
|
26
|
+
import type { ExecutionReport, IntentMatchAnswer, IntentMatchRequest, Recorder, RunCreation, RunMatch, RunMetrics, StartRunOptions } from "./recorder.js";
|
|
27
27
|
export interface RemoteRecorderOptions {
|
|
28
28
|
baseUrl: string;
|
|
29
29
|
session: AuthSession;
|
|
@@ -38,19 +38,37 @@ export declare class RemoteRecorder implements Recorder {
|
|
|
38
38
|
private readonly chains;
|
|
39
39
|
/** runId → the similar-prompt outcome of its create call. */
|
|
40
40
|
private readonly matches;
|
|
41
|
+
/** runId → what the create call said (not created: a hit, nothing to record). */
|
|
42
|
+
private readonly created;
|
|
41
43
|
/**
|
|
42
44
|
* Execution reports ride their own chain: a matched turn has no run on the
|
|
43
45
|
* service (that is what a match means), so there is no run chain to append to.
|
|
44
46
|
*/
|
|
45
47
|
private readonly executionChainKey;
|
|
46
48
|
constructor(opts: RemoteRecorderOptions);
|
|
47
|
-
startRun(input: string, metadata?: Record<string, unknown
|
|
49
|
+
startRun(input: string, metadata?: Record<string, unknown>, options?: StartRunOptions): string;
|
|
50
|
+
/** What the create call said. "Created" for an id this recorder never opened. */
|
|
51
|
+
runCreated(runId: string): Promise<RunCreation>;
|
|
52
|
+
/**
|
|
53
|
+
* Match a ReAct iteration's reasoning against the caller's scenario intents
|
|
54
|
+
* (fallbk.md §Runner 4). Awaited, not queued: the caller is a `PreToolUse`
|
|
55
|
+
* holding a budget, and the answer decides what that hook says. A service that
|
|
56
|
+
* predates the route (404) or any failure is a miss.
|
|
57
|
+
*/
|
|
58
|
+
matchIntent(req: IntentMatchRequest): Promise<IntentMatchAnswer | null>;
|
|
48
59
|
/** Await the similar-prompt match for a run's create call (null = fresh run). */
|
|
49
60
|
getMatch(runId: string): Promise<RunMatch | null>;
|
|
50
61
|
recordToolSelected(runId: string, stepIndex: number, data: {
|
|
51
62
|
toolName: string;
|
|
52
63
|
toolInput: string;
|
|
53
64
|
context?: string;
|
|
65
|
+
intent?: {
|
|
66
|
+
text: string;
|
|
67
|
+
source: "thinking";
|
|
68
|
+
};
|
|
69
|
+
metadata?: {
|
|
70
|
+
pinnedBy: string;
|
|
71
|
+
};
|
|
54
72
|
}): string;
|
|
55
73
|
recordToolResponse(runId: string, stepIndex: number, data: {
|
|
56
74
|
toolName: string;
|
|
@@ -33,6 +33,8 @@ export class RemoteRecorder {
|
|
|
33
33
|
chains = new Map();
|
|
34
34
|
/** runId → the similar-prompt outcome of its create call. */
|
|
35
35
|
matches = new Map();
|
|
36
|
+
/** runId → what the create call said (not created: a hit, nothing to record). */
|
|
37
|
+
created = new Map();
|
|
36
38
|
/**
|
|
37
39
|
* Execution reports ride their own chain: a matched turn has no run on the
|
|
38
40
|
* service (that is what a match means), so there is no run chain to append to.
|
|
@@ -43,12 +45,16 @@ export class RemoteRecorder {
|
|
|
43
45
|
this.session = opts.session;
|
|
44
46
|
this.framework = opts.framework ?? "baseinstrunner";
|
|
45
47
|
}
|
|
46
|
-
startRun(input, metadata) {
|
|
48
|
+
startRun(input, metadata, options) {
|
|
47
49
|
const id = "run_" + randomUUID();
|
|
48
50
|
let resolveMatch;
|
|
51
|
+
let resolveCreated;
|
|
49
52
|
this.matches.set(id, new Promise((r) => {
|
|
50
53
|
resolveMatch = r;
|
|
51
54
|
}));
|
|
55
|
+
this.created.set(id, new Promise((r) => {
|
|
56
|
+
resolveCreated = r;
|
|
57
|
+
}));
|
|
52
58
|
this.enqueue(id, async () => {
|
|
53
59
|
try {
|
|
54
60
|
const body = await this.post("/recordings/runs", {
|
|
@@ -56,19 +62,60 @@ export class RemoteRecorder {
|
|
|
56
62
|
framework: this.framework,
|
|
57
63
|
input,
|
|
58
64
|
metadata,
|
|
65
|
+
// A fragment (fallbk.md D7). Omitted, not null, so an older service
|
|
66
|
+
// that does not know the field sees the body it always saw.
|
|
67
|
+
...(options?.fallbackOf ? { fallbackOf: options.fallbackOf } : {}),
|
|
68
|
+
// A chain with calls is inlined for this runner rather than stopped in
|
|
69
|
+
// front of its first call (segmented.md R-CALL-28).
|
|
70
|
+
supportsCalls: true,
|
|
59
71
|
});
|
|
60
|
-
const
|
|
61
|
-
?
|
|
62
|
-
:
|
|
63
|
-
resolveMatch(matched);
|
|
72
|
+
const b = body && typeof body === "object"
|
|
73
|
+
? body
|
|
74
|
+
: {};
|
|
75
|
+
resolveMatch(b.matched ?? null);
|
|
76
|
+
// `id: null` is the service saying it kept a run of its own. Which run
|
|
77
|
+
// that was matters: a turn must not count step hits against the
|
|
78
|
+
// recording it is itself a repeat of (R-HIT-7).
|
|
79
|
+
resolveCreated({ created: b.id !== null, fragmentOf: b.fragmentOf });
|
|
64
80
|
}
|
|
65
81
|
catch (err) {
|
|
66
82
|
resolveMatch(null); // fail-safe: on a create error, record normally
|
|
83
|
+
resolveCreated({ created: true });
|
|
67
84
|
throw err; // preserve enqueue's dropped-send logging
|
|
68
85
|
}
|
|
69
86
|
});
|
|
70
87
|
return id;
|
|
71
88
|
}
|
|
89
|
+
/** What the create call said. "Created" for an id this recorder never opened. */
|
|
90
|
+
async runCreated(runId) {
|
|
91
|
+
const pending = this.created.get(runId);
|
|
92
|
+
return pending ? await pending : { created: true };
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Match a ReAct iteration's reasoning against the caller's scenario intents
|
|
96
|
+
* (fallbk.md §Runner 4). Awaited, not queued: the caller is a `PreToolUse`
|
|
97
|
+
* holding a budget, and the answer decides what that hook says. A service that
|
|
98
|
+
* predates the route (404) or any failure is a miss.
|
|
99
|
+
*/
|
|
100
|
+
async matchIntent(req) {
|
|
101
|
+
try {
|
|
102
|
+
const body = await this.post("/scenarios/match-intent", req);
|
|
103
|
+
// The whole answer, not just `matched`: the counting fields are what make
|
|
104
|
+
// the observe-only period readable (segmented.md R-OUT-10), and an older
|
|
105
|
+
// server simply sends none of them (R-COMPAT-2).
|
|
106
|
+
const b = body && typeof body === "object" ? body : {};
|
|
107
|
+
return {
|
|
108
|
+
matched: b.matched ?? null,
|
|
109
|
+
stepHit: b.stepHit ?? null,
|
|
110
|
+
bestStepSimilarity: b.bestStepSimilarity,
|
|
111
|
+
segmentWouldArm: b.segmentWouldArm,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
catch (err) {
|
|
115
|
+
logLine("replay.intent_match_failed", { error: errText(err), why: "treated as a miss" });
|
|
116
|
+
return null;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
72
119
|
/** Await the similar-prompt match for a run's create call (null = fresh run). */
|
|
73
120
|
async getMatch(runId) {
|
|
74
121
|
const pending = this.matches.get(runId);
|
|
@@ -76,13 +123,19 @@ export class RemoteRecorder {
|
|
|
76
123
|
}
|
|
77
124
|
recordToolSelected(runId, stepIndex, data) {
|
|
78
125
|
const id = "step_" + randomUUID();
|
|
79
|
-
this.enqueue(runId, () =>
|
|
126
|
+
this.enqueue(runId, () =>
|
|
127
|
+
// `intent` and `metadata` are omitted, not nulled, when absent, so an
|
|
128
|
+
// older service sees the body it has always seen (segmented.md
|
|
129
|
+
// R-COMPAT-1).
|
|
130
|
+
this.post(`/recordings/runs/${runId}/steps`, {
|
|
80
131
|
id,
|
|
81
132
|
stepIndex,
|
|
82
133
|
type: "tool_selected",
|
|
83
134
|
toolName: data.toolName,
|
|
84
135
|
toolInput: data.toolInput,
|
|
85
136
|
context: data.context,
|
|
137
|
+
intent: data.intent,
|
|
138
|
+
metadata: data.metadata,
|
|
86
139
|
}));
|
|
87
140
|
return id;
|
|
88
141
|
}
|
|
@@ -155,6 +208,10 @@ export class RemoteRecorder {
|
|
|
155
208
|
errorStage: report.errorStage,
|
|
156
209
|
errorStepIndex: report.errorStepIndex,
|
|
157
210
|
errorToolName: report.errorToolName,
|
|
211
|
+
fallbackStepIndex: report.fallbackStepIndex,
|
|
212
|
+
fallbackKind: report.fallbackKind,
|
|
213
|
+
baselineEligible: report.baselineEligible,
|
|
214
|
+
sharedWith: report.sharedWith,
|
|
158
215
|
});
|
|
159
216
|
const r = (body ?? {});
|
|
160
217
|
const failed = report.steps?.filter((s) => s.status === "failed").length ?? 0;
|
|
@@ -169,6 +226,8 @@ export class RemoteRecorder {
|
|
|
169
226
|
measured: report.measured,
|
|
170
227
|
steps: report.steps?.length,
|
|
171
228
|
stepsFailed: failed || undefined,
|
|
229
|
+
baselineEligible: report.baselineEligible === false ? false : undefined,
|
|
230
|
+
sharedWith: report.sharedWith?.length,
|
|
172
231
|
});
|
|
173
232
|
});
|
|
174
233
|
}
|
|
@@ -177,6 +236,7 @@ export class RemoteRecorder {
|
|
|
177
236
|
await (this.chains.get(runId) ?? Promise.resolve());
|
|
178
237
|
this.chains.delete(runId);
|
|
179
238
|
this.matches.delete(runId);
|
|
239
|
+
this.created.delete(runId);
|
|
180
240
|
// Reports live on their own chain (a matched run has no run chain of its own
|
|
181
241
|
// — the service created no run), so draining that chain is a separate step.
|
|
182
242
|
await (this.chains.get(this.executionChainKey) ?? Promise.resolve());
|
package/dist/replay/bundle.d.ts
CHANGED
|
@@ -27,10 +27,19 @@ export interface BundleEntry {
|
|
|
27
27
|
}
|
|
28
28
|
/** Cap a computed input for inclusion in a bundle. */
|
|
29
29
|
export declare function bundleInput(value: unknown): string;
|
|
30
|
+
export interface AssembleOptions {
|
|
31
|
+
/** The plan stopped part-way and the model is to carry on (fallbk.md). */
|
|
32
|
+
handover?: boolean;
|
|
33
|
+
/**
|
|
34
|
+
* The plan was armed mid-task by intent (segmented.md R-OUT-13). Its results
|
|
35
|
+
* feed the agent's own task, which continues. `handover` wins over this.
|
|
36
|
+
*/
|
|
37
|
+
subTask?: boolean;
|
|
38
|
+
}
|
|
30
39
|
/**
|
|
31
40
|
* Compose the bundle. `maxChars` is a hard ceiling on the returned string.
|
|
32
41
|
*/
|
|
33
|
-
export declare function assembleBundle(entries: readonly BundleEntry[], maxChars: number): string;
|
|
42
|
+
export declare function assembleBundle(entries: readonly BundleEntry[], maxChars: number, opts?: AssembleOptions): string;
|
|
34
43
|
/** Truncate to `max` chars, appending a marker naming how much was dropped. */
|
|
35
44
|
export declare function truncate(s: string, max: number): string;
|
|
36
45
|
//# sourceMappingURL=bundle.d.ts.map
|
package/dist/replay/bundle.js
CHANGED
|
@@ -53,14 +53,52 @@ const MIXED_HEADER = "[BaseInstRunner calculated replay] A known-good tool seque
|
|
|
53
53
|
"because their tool cannot be executed here — treat them as possibly out of date. " +
|
|
54
54
|
"Use them to answer the user's request now — do NOT call any tools.";
|
|
55
55
|
const MIXED_FOOTER = "\n\nAnswer the user's request using the results above. Do not call any tools.";
|
|
56
|
+
/**
|
|
57
|
+
* The scenario stopped before the task was done (fallbk.md), so the model must
|
|
58
|
+
* NOT be told to answer without tools — the rest of the work is its own. The
|
|
59
|
+
* hand-over note in front of the bundle says what is left.
|
|
60
|
+
*/
|
|
61
|
+
const HANDOVER_HEADER = "[BaseInstRunner calculated replay] The tool calls below ran for real, moments ago, on " +
|
|
62
|
+
"the user's behalf, and their side effects are already in place. Results marked " +
|
|
63
|
+
"(recorded) come from an earlier recording because their tool cannot run here. The " +
|
|
64
|
+
"scenario stopped before the task was finished: do not repeat these calls, and " +
|
|
65
|
+
"continue the task yourself from where they end.";
|
|
66
|
+
const HANDOVER_FOOTER = "\n\nThe calls above are done. Continue the rest of the task yourself, calling tools as needed.";
|
|
67
|
+
/**
|
|
68
|
+
* The plan ran inside the agent's own task (segmented.md R-OUT-13): it finished,
|
|
69
|
+
* but the task did not, so the model must not be told to answer without tools.
|
|
70
|
+
* This is {@link HANDOVER_HEADER} without "The scenario stopped before the task
|
|
71
|
+
* was finished" — nothing stopped, the sub-task is simply done.
|
|
72
|
+
*/
|
|
73
|
+
const SUBTASK_HEADER = "[BaseInstRunner calculated replay] The tool calls below ran for real, moments ago, on " +
|
|
74
|
+
"the user's behalf, and their side effects are already in place. Results marked " +
|
|
75
|
+
"(recorded) come from an earlier recording because their tool cannot run here. These " +
|
|
76
|
+
"steps are done: do not repeat these calls. Continue your task from where they end, " +
|
|
77
|
+
"calling tools as needed.";
|
|
78
|
+
const SUBTASK_FOOTER = "\n\nThe calls above are done. Continue your task, calling tools as needed.";
|
|
56
79
|
/**
|
|
57
80
|
* Compose the bundle. `maxChars` is a hard ceiling on the returned string.
|
|
58
81
|
*/
|
|
59
|
-
export function assembleBundle(entries, maxChars) {
|
|
82
|
+
export function assembleBundle(entries, maxChars, opts = {}) {
|
|
60
83
|
const n = entries.length;
|
|
61
84
|
const anyRecorded = entries.some((e) => e.recorded);
|
|
62
|
-
|
|
63
|
-
|
|
85
|
+
// A hand-over outranks a sub-task: it says "continue" too, and it also has to
|
|
86
|
+
// say that the plan stopped short. A sub-task outranks the live and mixed
|
|
87
|
+
// headers, whose wording forbids further tool calls (segmented.md 10.8 §6).
|
|
88
|
+
const header = opts.handover
|
|
89
|
+
? HANDOVER_HEADER
|
|
90
|
+
: opts.subTask
|
|
91
|
+
? SUBTASK_HEADER
|
|
92
|
+
: anyRecorded
|
|
93
|
+
? MIXED_HEADER
|
|
94
|
+
: LIVE_HEADER;
|
|
95
|
+
const footer = opts.handover
|
|
96
|
+
? HANDOVER_FOOTER
|
|
97
|
+
: opts.subTask
|
|
98
|
+
? SUBTASK_FOOTER
|
|
99
|
+
: anyRecorded
|
|
100
|
+
? MIXED_FOOTER
|
|
101
|
+
: LIVE_FOOTER;
|
|
64
102
|
const fixedCost = header.length +
|
|
65
103
|
footer.length +
|
|
66
104
|
entries.reduce((sum, e) => sum + PER_STEP_OVERHEAD + e.toolName.length + e.input.length, 0);
|