@alisio/core 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/changelog/data.js +16 -0
  2. package/dist/eval/canonical.d.ts +14 -0
  3. package/dist/eval/canonical.js +40 -0
  4. package/dist/eval/cassette.d.ts +180 -0
  5. package/dist/eval/cassette.js +424 -0
  6. package/dist/eval/compare.d.ts +147 -0
  7. package/dist/eval/compare.js +436 -0
  8. package/dist/eval/engine.d.ts +144 -0
  9. package/dist/eval/engine.js +728 -0
  10. package/dist/eval/export.d.ts +40 -0
  11. package/dist/eval/export.js +90 -0
  12. package/dist/eval/fingerprint.d.ts +88 -0
  13. package/dist/eval/fingerprint.js +186 -0
  14. package/dist/eval/normalize.d.ts +68 -0
  15. package/dist/eval/normalize.js +138 -0
  16. package/dist/eval/paths.d.ts +11 -0
  17. package/dist/eval/paths.js +50 -0
  18. package/dist/eval/plan.d.ts +26 -0
  19. package/dist/eval/plan.js +44 -0
  20. package/dist/eval/report.d.ts +53 -0
  21. package/dist/eval/report.js +258 -0
  22. package/dist/eval/run-suite.d.ts +68 -0
  23. package/dist/eval/run-suite.js +94 -0
  24. package/dist/eval/secrets.d.ts +12 -0
  25. package/dist/eval/secrets.js +45 -0
  26. package/dist/eval/stats.d.ts +17 -0
  27. package/dist/eval/stats.js +50 -0
  28. package/dist/eval/suite.d.ts +20 -0
  29. package/dist/eval/suite.js +165 -0
  30. package/dist/eval/variants.d.ts +35 -0
  31. package/dist/eval/variants.js +137 -0
  32. package/dist/eval/verifiers.d.ts +49 -0
  33. package/dist/eval/verifiers.js +264 -0
  34. package/dist/forge/javascript/hosts/worker-host.d.ts +9 -0
  35. package/dist/forge/javascript/hosts/worker-host.js +16 -1
  36. package/dist/forge/kernel.js +3 -2
  37. package/dist/index.d.ts +16 -1
  38. package/dist/index.js +16 -1
  39. package/dist/plan/exit-plan.js +2 -1
  40. package/dist/runtime/ids.d.ts +21 -0
  41. package/dist/runtime/ids.js +41 -1
  42. package/dist/runtime/store.js +3 -2
  43. package/package.json +2 -2
@@ -2,6 +2,22 @@
2
2
  export const CHANGELOG_DATA = {
3
3
  schema: 1,
4
4
  entries: [
5
+ {
6
+ version: "0.7.0",
7
+ date: "2026-10-14",
8
+ sections: [
9
+ {
10
+ title: "Added",
11
+ items: [
12
+ "`alisio eval list|run|record|compare|variants|export`: run evaluation suites against any configured provider, compare variants and export the results.",
13
+ "Reliability Lab suites with deterministic verifiers (files, commands, tool sequences), so a pass or fail never depends on a model judging itself.",
14
+ "`pass@k` and `pass^k` metrics with Wilson confidence intervals, so a comparison tells you whether a difference is real or noise.",
15
+ "Cassettes: `alisio eval record` captures provider responses and later runs replay them offline, without network or cost.",
16
+ "Bundled suites `smoke-coding`, `tool-sequences` and `long-context`, available from a source checkout.",
17
+ ],
18
+ },
19
+ ],
20
+ },
5
21
  {
6
22
  version: "0.6.0",
7
23
  date: "2026-10-12",
@@ -0,0 +1,14 @@
1
+ /**
2
+ * Canonical JSON of the reliability lab (specs/010 D10): equal values give equal text, so the same
3
+ * text can be hashed for the configuration hash, argument keys and cassette fingerprints.
4
+ *
5
+ * Rules: object keys sorted by UTF-16 code unit (the default string order), `undefined` members
6
+ * dropped (in arrays they become `null`, as `JSON.stringify` does), `null` kept, `-0` written as
7
+ * `0`, non-finite numbers rejected (JSON cannot carry them), string VALUES normalized to NFC
8
+ * (keys are left as written).
9
+ */
10
+ /** JSON text with the canonical rules above. */
11
+ export declare function canonicalJson(value: unknown): string;
12
+ export declare const sha256Hex: (data: string | Buffer) => string;
13
+ /** `sha256:<64 hex>` of the canonical JSON of `value`. */
14
+ export declare function hashCanonical(value: unknown): string;
@@ -0,0 +1,40 @@
1
+ /**
2
+ * Canonical JSON of the reliability lab (specs/010 D10): equal values give equal text, so the same
3
+ * text can be hashed for the configuration hash, argument keys and cassette fingerprints.
4
+ *
5
+ * Rules: object keys sorted by UTF-16 code unit (the default string order), `undefined` members
6
+ * dropped (in arrays they become `null`, as `JSON.stringify` does), `null` kept, `-0` written as
7
+ * `0`, non-finite numbers rejected (JSON cannot carry them), string VALUES normalized to NFC
8
+ * (keys are left as written).
9
+ */
10
+ import { createHash } from "node:crypto";
11
+ function normalize(value) {
12
+ if (typeof value === "string")
13
+ return value.normalize("NFC");
14
+ if (typeof value === "number") {
15
+ if (!Number.isFinite(value))
16
+ throw new TypeError(`Cannot canonicalize a non-finite number (${value}).`);
17
+ return value === 0 ? 0 : value;
18
+ }
19
+ if (Array.isArray(value))
20
+ return value.map((item) => (item === undefined ? null : normalize(item)));
21
+ if (value && typeof value === "object") {
22
+ const out = {};
23
+ for (const key of Object.keys(value).sort()) {
24
+ const member = value[key];
25
+ if (member !== undefined)
26
+ out[key] = normalize(member);
27
+ }
28
+ return out;
29
+ }
30
+ return value;
31
+ }
32
+ /** JSON text with the canonical rules above. */
33
+ export function canonicalJson(value) {
34
+ return JSON.stringify(normalize(value));
35
+ }
36
+ export const sha256Hex = (data) => createHash("sha256").update(data).digest("hex");
37
+ /** `sha256:<64 hex>` of the canonical JSON of `value`. */
38
+ export function hashCanonical(value) {
39
+ return `sha256:${sha256Hex(canonicalJson(value))}`;
40
+ }
@@ -0,0 +1,180 @@
1
+ /**
2
+ * Cassettes of the reliability lab (specs/010 5.3, D10, D11): `cassette/v1` JSONL, the provider
3
+ * that records a live run and the provider that serves it back.
4
+ *
5
+ * line 1 header: {v:1, cassetteVersion, suiteId, taskId, harnessVersion, recordedAt, configHash,
6
+ * provider, model, models?, texts, toolSets}
7
+ * line n one call to `ModelProvider.stream`: {call, lane, fingerprint, parts, messages?, events,
8
+ * error?}
9
+ *
10
+ * `completed.message` is stored VERBATIM (tool-call ids, `providerData`, `truncated`) together with
11
+ * its usage; `reasoning_delta` is dropped (the SDK says it is never replayed). A provider error
12
+ * (for example the truncation signal) is stored as `error: {message, code}` so recovery replays.
13
+ *
14
+ * Replay serves the recorded events in strict order PER LANE. A lane is a session (the registered
15
+ * main session by its `<SESSION#n>` placeholder); a session the run did not register (a child /
16
+ * subagent) is bound to the lane whose first request has the same fingerprint, so concurrent
17
+ * children may interleave differently. v1 limitation: two children with an identical first request
18
+ * are interchangeable, and a child whose first request changes is reported as a divergence.
19
+ *
20
+ * This module (and everything it imports) must not touch the network: a test checks that
21
+ * statically, and replay needs no credentials. Replay promises no timing.
22
+ */
23
+ import type { ModelInfo, ModelProvider, ProviderEvent, Usage } from "@alisio/sdk";
24
+ import { type CassetteTexts, type FingerprintDiff, type FingerprintParts, type MessageView, type StreamRequest } from "./fingerprint.ts";
25
+ import type { Normalizer } from "./normalize.ts";
26
+ export declare const MAX_CASSETTE_BYTES: number;
27
+ export declare const MAX_CASSETTE_DIR_BYTES: number;
28
+ export declare const CASSETTE_SUFFIX = ".cassette.jsonl";
29
+ export type RecordedEvent = {
30
+ type: "text_delta";
31
+ delta: string;
32
+ } | {
33
+ type: "completed";
34
+ message: Extract<ProviderEvent, {
35
+ type: "completed";
36
+ }>["message"];
37
+ usage?: Usage;
38
+ };
39
+ export interface CassetteHeader {
40
+ v: 1;
41
+ cassetteVersion: number;
42
+ suiteId: string;
43
+ taskId: string;
44
+ harnessVersion: string;
45
+ recordedAt: number;
46
+ configHash: string;
47
+ provider: string;
48
+ model: string;
49
+ models?: ModelInfo[];
50
+ texts: CassetteTexts["texts"];
51
+ toolSets: CassetteTexts["toolSets"];
52
+ }
53
+ export interface CassetteCall {
54
+ call: number;
55
+ lane: string;
56
+ fingerprint: string;
57
+ parts: FingerprintParts;
58
+ /** Views of the messages that first appear in this call (the earlier ones are in earlier lines). */
59
+ messages?: Record<string, MessageView>;
60
+ events: RecordedEvent[];
61
+ error?: {
62
+ message: string;
63
+ code?: string;
64
+ };
65
+ }
66
+ export interface Cassette {
67
+ header: CassetteHeader;
68
+ calls: CassetteCall[];
69
+ }
70
+ export declare class CassetteFormatError extends Error {
71
+ constructor(message: string);
72
+ }
73
+ export declare class CassetteWriteError extends Error {
74
+ readonly reason: "secret" | "size" | "total";
75
+ constructor(message: string, reason: "secret" | "size" | "total");
76
+ }
77
+ export declare function serializeCassette(cassette: Cassette): string;
78
+ export declare function parseCassette(text: string): Cassette;
79
+ export declare function readCassette(path: string): Cassette;
80
+ export interface WriteOptions {
81
+ /** Exact credential values of the run (provider profile, matching env vars). */
82
+ secrets?: readonly string[];
83
+ maxBytes?: number;
84
+ maxDirBytes?: number;
85
+ }
86
+ /**
87
+ * Writes `cassette` to `path` unless it holds a secret (REFUSES: it never strips, which would make
88
+ * the cassette lie about what the model saw) or exceeds the size limits (256 KB each, 2 MB for the
89
+ * directory). Nothing is written on refusal.
90
+ */
91
+ export declare function writeCassette(path: string, cassette: Cassette, options?: WriteOptions): void;
92
+ export interface CassetteMeta {
93
+ suiteId: string;
94
+ taskId: string;
95
+ harnessVersion: string;
96
+ configHash: string;
97
+ recordedAt: number;
98
+ provider: string;
99
+ model: string;
100
+ cassetteVersion?: number;
101
+ }
102
+ /** Collects the calls a `RecordingProvider` sees; `toCassette` turns them into the file content. */
103
+ export declare class CassetteRecorder {
104
+ readonly calls: CassetteCall[];
105
+ /** Why the recording cannot be trusted (set, never cleared); `toCassette` refuses then. */
106
+ unstable: string | undefined;
107
+ models: ModelInfo[] | undefined;
108
+ private normalizer;
109
+ private readonly store;
110
+ private readonly childLanes;
111
+ private readonly laneCounts;
112
+ bind(normalizer: Normalizer): void;
113
+ /** Opens the record of the next call. */
114
+ begin(request: StreamRequest): CassetteCall;
115
+ private laneFor;
116
+ markUnstable(reason: string): void;
117
+ toCassette(meta: CassetteMeta): Cassette;
118
+ }
119
+ /**
120
+ * Wraps a real provider and tees what it emits into `sink`. The caller still receives every event
121
+ * (reasoning included). A call aborted before any event (a first-token retry, a timeout) or one the
122
+ * consumer abandons makes the recording unstable: replaying it would not reproduce the run.
123
+ */
124
+ export declare class RecordingProvider implements ModelProvider {
125
+ private readonly inner;
126
+ private readonly sink;
127
+ readonly id: string;
128
+ readonly model: string;
129
+ readonly listModels?: ModelProvider["listModels"];
130
+ readonly dispose?: ModelProvider["dispose"];
131
+ constructor(inner: ModelProvider, sink: CassetteRecorder);
132
+ stream(request: StreamRequest): AsyncGenerator<ProviderEvent>;
133
+ }
134
+ /** The request of a call differs from the recorded one. Carries the structured diff. */
135
+ export declare class ReplayDivergenceError extends Error {
136
+ readonly call: number;
137
+ readonly lane: string;
138
+ readonly diff: FingerprintDiff;
139
+ constructor(call: number, lane: string, diff: FingerprintDiff);
140
+ }
141
+ /** The run asked for a call the cassette has no (more) recording for. */
142
+ export declare class ReplayExhausted extends Error {
143
+ readonly call: number;
144
+ readonly lane: string;
145
+ constructor(call: number, lane: string, detail: string);
146
+ }
147
+ /** Recorded calls the run never asked for (`ReplayProvider.assertConsumed`). */
148
+ export declare class ReplayUnconsumedError extends Error {
149
+ readonly leftover: Array<{
150
+ call: number;
151
+ lane: string;
152
+ }>;
153
+ constructor(leftover: Array<{
154
+ call: number;
155
+ lane: string;
156
+ }>);
157
+ }
158
+ export declare class ReplayProvider implements ModelProvider {
159
+ readonly id: string;
160
+ readonly model: string;
161
+ readonly listModels?: ModelProvider["listModels"];
162
+ private readonly store;
163
+ private readonly lanes;
164
+ private readonly position;
165
+ private readonly bindings;
166
+ private readonly claimed;
167
+ private requests;
168
+ private readonly normalizer;
169
+ constructor(cassette: Cassette, options: {
170
+ normalizer: Normalizer;
171
+ });
172
+ /** The first divergence or exhaustion, kept even if the runner wraps the error it received. */
173
+ failure: ReplayDivergenceError | ReplayExhausted | undefined;
174
+ stream(request: StreamRequest): AsyncGenerator<ProviderEvent>;
175
+ /** The recorded call this request must be, consumed; throws when it is not. */
176
+ private match;
177
+ private laneFor;
178
+ /** Throws `ReplayUnconsumedError` when recorded calls were never requested (extra calls). */
179
+ assertConsumed(): void;
180
+ }
@@ -0,0 +1,424 @@
1
+ /**
2
+ * Cassettes of the reliability lab (specs/010 5.3, D10, D11): `cassette/v1` JSONL, the provider
3
+ * that records a live run and the provider that serves it back.
4
+ *
5
+ * line 1 header: {v:1, cassetteVersion, suiteId, taskId, harnessVersion, recordedAt, configHash,
6
+ * provider, model, models?, texts, toolSets}
7
+ * line n one call to `ModelProvider.stream`: {call, lane, fingerprint, parts, messages?, events,
8
+ * error?}
9
+ *
10
+ * `completed.message` is stored VERBATIM (tool-call ids, `providerData`, `truncated`) together with
11
+ * its usage; `reasoning_delta` is dropped (the SDK says it is never replayed). A provider error
12
+ * (for example the truncation signal) is stored as `error: {message, code}` so recovery replays.
13
+ *
14
+ * Replay serves the recorded events in strict order PER LANE. A lane is a session (the registered
15
+ * main session by its `<SESSION#n>` placeholder); a session the run did not register (a child /
16
+ * subagent) is bound to the lane whose first request has the same fingerprint, so concurrent
17
+ * children may interleave differently. v1 limitation: two children with an identical first request
18
+ * are interchangeable, and a child whose first request changes is reported as a divergence.
19
+ *
20
+ * This module (and everything it imports) must not touch the network: a test checks that
21
+ * statically, and replay needs no credentials. Replay promises no timing.
22
+ */
23
+ import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
24
+ import { dirname, join } from "node:path";
25
+ import { absorb, computeFingerprint, diffFingerprints, emptyStore, } from "./fingerprint.js";
26
+ import { findSecrets } from "./secrets.js";
27
+ export const MAX_CASSETTE_BYTES = 256 * 1024;
28
+ export const MAX_CASSETTE_DIR_BYTES = 2 * 1024 * 1024;
29
+ export const CASSETTE_SUFFIX = ".cassette.jsonl";
30
+ const NO_SESSION = "<NO_SESSION>";
31
+ export class CassetteFormatError extends Error {
32
+ constructor(message) {
33
+ super(message);
34
+ this.name = "CassetteFormatError";
35
+ }
36
+ }
37
+ export class CassetteWriteError extends Error {
38
+ reason;
39
+ constructor(message, reason) {
40
+ super(message);
41
+ this.reason = reason;
42
+ this.name = "CassetteWriteError";
43
+ }
44
+ }
45
+ // ---------------------------------------------------------------------------------------------
46
+ // File format
47
+ // ---------------------------------------------------------------------------------------------
48
+ export function serializeCassette(cassette) {
49
+ return `${[cassette.header, ...cassette.calls].map((line) => JSON.stringify(line)).join("\n")}\n`;
50
+ }
51
+ const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
52
+ export function parseCassette(text) {
53
+ const lines = text.split("\n").filter((line) => line.trim() !== "");
54
+ if (lines.length === 0)
55
+ throw new CassetteFormatError("The cassette is empty.");
56
+ const parsed = lines.map((line, index) => {
57
+ try {
58
+ return JSON.parse(line);
59
+ }
60
+ catch {
61
+ throw new CassetteFormatError(`Cassette line ${index + 1} is not valid JSON.`);
62
+ }
63
+ });
64
+ const header = parsed[0];
65
+ if (!isRecord(header) || header.v !== 1)
66
+ throw new CassetteFormatError('Unsupported cassette: line 1 must be a cassette/v1 header ({"v":1,...}). Record it again with this version of alisio.');
67
+ for (const key of ["suiteId", "taskId", "harnessVersion", "configHash", "provider", "model"])
68
+ if (typeof header[key] !== "string")
69
+ throw new CassetteFormatError(`Cassette header line 1 lacks the string "${key}".`);
70
+ if (!isRecord(header.texts) || !isRecord(header.toolSets))
71
+ throw new CassetteFormatError("Cassette header line 1 lacks its texts or toolSets.");
72
+ const calls = parsed.slice(1).map((line, index) => {
73
+ const at = index + 2;
74
+ if (!isRecord(line) ||
75
+ typeof line.call !== "number" ||
76
+ typeof line.lane !== "string" ||
77
+ typeof line.fingerprint !== "string" ||
78
+ !isRecord(line.parts) ||
79
+ !Array.isArray(line.events))
80
+ throw new CassetteFormatError(`Cassette line ${at} is not a call record.`);
81
+ if (line.call !== index)
82
+ throw new CassetteFormatError(`Cassette line ${at}: call ${line.call}, expected ${index} (calls must be numbered in order).`);
83
+ for (const event of line.events)
84
+ if (!isRecord(event) || (event.type !== "text_delta" && event.type !== "completed"))
85
+ throw new CassetteFormatError(`Cassette line ${at} holds an unknown event.`);
86
+ return line;
87
+ });
88
+ return { header: header, calls };
89
+ }
90
+ export function readCassette(path) {
91
+ let text;
92
+ try {
93
+ text = readFileSync(path, "utf8");
94
+ }
95
+ catch {
96
+ throw new CassetteFormatError(`Cannot read the cassette ${path}. Record it first with \`alisio eval record\`.`);
97
+ }
98
+ try {
99
+ return parseCassette(text);
100
+ }
101
+ catch (error) {
102
+ throw new CassetteFormatError(`${path}: ${error instanceof Error ? error.message : String(error)}`);
103
+ }
104
+ }
105
+ /** Every string leaf with its path (`calls[0].events[1].message.text`). */
106
+ function* leaves(value, path) {
107
+ if (typeof value === "string")
108
+ yield [path, value];
109
+ else if (Array.isArray(value))
110
+ for (let i = 0; i < value.length; i++)
111
+ yield* leaves(value[i], `${path}[${i}]`);
112
+ else if (value && typeof value === "object")
113
+ for (const [key, member] of Object.entries(value))
114
+ yield* leaves(member, `${path}.${key}`);
115
+ }
116
+ /**
117
+ * Writes `cassette` to `path` unless it holds a secret (REFUSES: it never strips, which would make
118
+ * the cassette lie about what the model saw) or exceeds the size limits (256 KB each, 2 MB for the
119
+ * directory). Nothing is written on refusal.
120
+ */
121
+ export function writeCassette(path, cassette, options = {}) {
122
+ for (const [where, value] of [
123
+ ...leaves(cassette.header, "header"),
124
+ ...leaves(cassette.calls, "calls"),
125
+ ]) {
126
+ const hits = findSecrets(value, options.secrets ?? []);
127
+ if (hits.length > 0)
128
+ throw new CassetteWriteError(`Refusing to write the cassette: ${where} looks like a secret (${hits.join("; ")}). Remove it from the fixture, prompt or tool output and record again.`, "secret");
129
+ }
130
+ const text = serializeCassette(cassette);
131
+ const bytes = Buffer.byteLength(text);
132
+ const maxBytes = options.maxBytes ?? MAX_CASSETTE_BYTES;
133
+ if (bytes > maxBytes)
134
+ throw new CassetteWriteError(`Refusing to write the cassette: ${bytes} bytes exceeds the ${Math.round(maxBytes / 1024)} KB limit per cassette. Record a shorter task.`, "size");
135
+ const dir = dirname(path);
136
+ let others = 0;
137
+ if (existsSync(dir))
138
+ for (const name of readdirSync(dir))
139
+ if (name.endsWith(CASSETTE_SUFFIX) && join(dir, name) !== path)
140
+ others += statSync(join(dir, name)).size;
141
+ const maxDir = options.maxDirBytes ?? MAX_CASSETTE_DIR_BYTES;
142
+ if (others + bytes > maxDir)
143
+ throw new CassetteWriteError(`Refusing to write the cassette: the cassettes in ${dir} would total ${others + bytes} bytes, over the ${Math.round(maxDir / (1024 * 1024))} MB limit.`, "total");
144
+ mkdirSync(dir, { recursive: true });
145
+ writeFileSync(path, text);
146
+ }
147
+ // ---------------------------------------------------------------------------------------------
148
+ // Lanes
149
+ // ---------------------------------------------------------------------------------------------
150
+ /** Lane of a session the run did not register: named after its first request. */
151
+ const childLaneBase = (fingerprint) => `L:${fingerprint.slice(7, 19)}`;
152
+ const childLaneName = (base, n) => (n === 1 ? base : `${base}#${n}`);
153
+ /** Collects the calls a `RecordingProvider` sees; `toCassette` turns them into the file content. */
154
+ export class CassetteRecorder {
155
+ calls = [];
156
+ /** Why the recording cannot be trusted (set, never cleared); `toCassette` refuses then. */
157
+ unstable;
158
+ models;
159
+ normalizer;
160
+ store = emptyStore();
161
+ childLanes = new Map();
162
+ laneCounts = new Map();
163
+ bind(normalizer) {
164
+ this.normalizer = normalizer;
165
+ }
166
+ /** Opens the record of the next call. */
167
+ begin(request) {
168
+ if (!this.normalizer)
169
+ throw new Error("CassetteRecorder.bind(normalizer) was not called.");
170
+ const material = computeFingerprint(request, this.normalizer);
171
+ const messages = absorb(this.store, material);
172
+ const entry = {
173
+ call: this.calls.length,
174
+ lane: this.laneFor(request, material),
175
+ fingerprint: material.fingerprint,
176
+ parts: material.parts,
177
+ ...(Object.keys(messages).length > 0 ? { messages } : {}),
178
+ events: [],
179
+ };
180
+ this.calls.push(entry);
181
+ return entry;
182
+ }
183
+ laneFor(request, material) {
184
+ const label = this.normalizer?.sessionLabel(request.sessionId);
185
+ if (label)
186
+ return label;
187
+ if (request.sessionId === undefined)
188
+ return NO_SESSION;
189
+ let lane = this.childLanes.get(request.sessionId);
190
+ if (!lane) {
191
+ const base = childLaneBase(material.fingerprint);
192
+ const n = (this.laneCounts.get(base) ?? 0) + 1;
193
+ this.laneCounts.set(base, n);
194
+ lane = childLaneName(base, n);
195
+ this.childLanes.set(request.sessionId, lane);
196
+ }
197
+ return lane;
198
+ }
199
+ markUnstable(reason) {
200
+ this.unstable ??= `${reason} The recording is unstable: re-record.`;
201
+ }
202
+ toCassette(meta) {
203
+ if (this.unstable)
204
+ throw new Error(`Cannot build the cassette. ${this.unstable}`);
205
+ return {
206
+ header: {
207
+ v: 1,
208
+ cassetteVersion: meta.cassetteVersion ?? 1,
209
+ suiteId: meta.suiteId,
210
+ taskId: meta.taskId,
211
+ harnessVersion: meta.harnessVersion,
212
+ recordedAt: meta.recordedAt,
213
+ configHash: meta.configHash,
214
+ provider: meta.provider,
215
+ model: meta.model,
216
+ ...(this.models ? { models: this.models } : {}),
217
+ texts: this.store.texts,
218
+ toolSets: this.store.toolSets,
219
+ },
220
+ calls: this.calls,
221
+ };
222
+ }
223
+ }
224
+ /**
225
+ * Wraps a real provider and tees what it emits into `sink`. The caller still receives every event
226
+ * (reasoning included). A call aborted before any event (a first-token retry, a timeout) or one the
227
+ * consumer abandons makes the recording unstable: replaying it would not reproduce the run.
228
+ */
229
+ export class RecordingProvider {
230
+ inner;
231
+ sink;
232
+ id;
233
+ model;
234
+ listModels;
235
+ dispose;
236
+ constructor(inner, sink) {
237
+ this.inner = inner;
238
+ this.sink = sink;
239
+ this.id = inner.id;
240
+ this.model = inner.model;
241
+ const list = inner.listModels?.bind(inner);
242
+ if (list)
243
+ this.listModels = async (signal) => {
244
+ const models = await list(signal);
245
+ sink.models = models;
246
+ return models;
247
+ };
248
+ const dispose = inner.dispose?.bind(inner);
249
+ if (dispose)
250
+ this.dispose = () => dispose();
251
+ }
252
+ async *stream(request) {
253
+ const entry = this.sink.begin(request);
254
+ let finished = false;
255
+ try {
256
+ for await (const event of this.inner.stream(request)) {
257
+ if (event.type === "text_delta")
258
+ entry.events.push({ type: "text_delta", delta: event.delta });
259
+ else if (event.type === "completed")
260
+ entry.events.push({
261
+ type: "completed",
262
+ message: event.message,
263
+ ...(event.usage ? { usage: event.usage } : {}),
264
+ });
265
+ yield event;
266
+ }
267
+ finished = true;
268
+ }
269
+ catch (error) {
270
+ if (request.signal.aborted) {
271
+ this.sink.markUnstable(`Call ${entry.call} was aborted ${entry.events.length === 0 ? "before any event" : "mid-stream"} (a first-token retry, a timeout or a budget).`);
272
+ }
273
+ else {
274
+ const { message } = error instanceof Error ? error : new Error(String(error));
275
+ const code = error?.code;
276
+ entry.error = { message, ...(typeof code === "string" ? { code } : {}) };
277
+ }
278
+ finished = true;
279
+ throw error;
280
+ }
281
+ finally {
282
+ if (!finished)
283
+ this.sink.markUnstable(`Call ${entry.call} was abandoned by the consumer${request.signal.aborted ? " after an abort" : ""}.`);
284
+ }
285
+ }
286
+ }
287
+ // ---------------------------------------------------------------------------------------------
288
+ // Replay
289
+ // ---------------------------------------------------------------------------------------------
290
+ const describeDiff = (diff) => diff.length === 0 ? "the request changed" : diff.map((entry) => entry.field).join(", ");
291
+ /** The request of a call differs from the recorded one. Carries the structured diff. */
292
+ export class ReplayDivergenceError extends Error {
293
+ call;
294
+ lane;
295
+ diff;
296
+ constructor(call, lane, diff) {
297
+ super(`Replay diverged at call ${call} (lane ${lane}): ${describeDiff(diff)}`);
298
+ this.call = call;
299
+ this.lane = lane;
300
+ this.diff = diff;
301
+ this.name = "ReplayDivergenceError";
302
+ }
303
+ }
304
+ /** The run asked for a call the cassette has no (more) recording for. */
305
+ export class ReplayExhausted extends Error {
306
+ call;
307
+ lane;
308
+ constructor(call, lane, detail) {
309
+ super(`Replay exhausted at request ${call} (lane ${lane}): ${detail}`);
310
+ this.call = call;
311
+ this.lane = lane;
312
+ this.name = "ReplayExhausted";
313
+ }
314
+ }
315
+ /** Recorded calls the run never asked for (`ReplayProvider.assertConsumed`). */
316
+ export class ReplayUnconsumedError extends Error {
317
+ leftover;
318
+ constructor(leftover) {
319
+ super(`${leftover.length} recorded call(s) were never requested (first: call ${leftover[0]?.call} in lane ${leftover[0]?.lane}).`);
320
+ this.leftover = leftover;
321
+ this.name = "ReplayUnconsumedError";
322
+ }
323
+ }
324
+ export class ReplayProvider {
325
+ id;
326
+ model;
327
+ listModels;
328
+ store;
329
+ lanes = new Map();
330
+ position = new Map();
331
+ bindings = new Map();
332
+ claimed = new Set();
333
+ requests = 0;
334
+ normalizer;
335
+ constructor(cassette, options) {
336
+ this.normalizer = options.normalizer;
337
+ this.id = cassette.header.provider;
338
+ this.model = cassette.header.model;
339
+ const models = cassette.header.models;
340
+ if (models)
341
+ this.listModels = async () => models;
342
+ this.store = {
343
+ texts: cassette.header.texts,
344
+ toolSets: cassette.header.toolSets,
345
+ messages: Object.assign({}, ...cassette.calls.map((c) => c.messages ?? {})),
346
+ };
347
+ for (const call of cassette.calls) {
348
+ const queue = this.lanes.get(call.lane) ?? [];
349
+ queue.push(call);
350
+ this.lanes.set(call.lane, queue);
351
+ }
352
+ }
353
+ /** The first divergence or exhaustion, kept even if the runner wraps the error it received. */
354
+ failure;
355
+ async *stream(request) {
356
+ request.signal.throwIfAborted();
357
+ let expected;
358
+ try {
359
+ expected = this.match(request);
360
+ }
361
+ catch (error) {
362
+ if (error instanceof ReplayDivergenceError || error instanceof ReplayExhausted)
363
+ this.failure ??= error;
364
+ throw error;
365
+ }
366
+ for (const event of expected.events) {
367
+ request.signal.throwIfAborted();
368
+ yield event;
369
+ }
370
+ if (expected.error)
371
+ throw Object.assign(new Error(expected.error.message), expected.error.code ? { code: expected.error.code } : {});
372
+ }
373
+ /** The recorded call this request must be, consumed; throws when it is not. */
374
+ match(request) {
375
+ const number = this.requests++;
376
+ const material = computeFingerprint(request, this.normalizer);
377
+ const lane = this.laneFor(request, material, number);
378
+ const expected = this.lanes.get(lane)?.[this.position.get(lane) ?? 0];
379
+ if (!expected)
380
+ throw new ReplayExhausted(number, lane, `the cassette has no further recorded call for this lane (${this.lanes.get(lane)?.length ?? 0} recorded).`);
381
+ if (expected.fingerprint !== material.fingerprint)
382
+ throw new ReplayDivergenceError(expected.call, lane, diffFingerprints(expected.call, expected.parts, this.store, material));
383
+ this.position.set(lane, (this.position.get(lane) ?? 0) + 1);
384
+ return expected;
385
+ }
386
+ laneFor(request, material, number) {
387
+ const label = this.normalizer.sessionLabel(request.sessionId);
388
+ if (label)
389
+ return label;
390
+ if (request.sessionId === undefined)
391
+ return NO_SESSION;
392
+ const bound = this.bindings.get(request.sessionId);
393
+ if (bound)
394
+ return bound;
395
+ const base = childLaneBase(material.fingerprint);
396
+ for (let n = 1; this.lanes.has(childLaneName(base, n)); n++) {
397
+ const candidate = childLaneName(base, n);
398
+ if (!this.claimed.has(candidate)) {
399
+ this.claimed.add(candidate);
400
+ this.bindings.set(request.sessionId, candidate);
401
+ return candidate;
402
+ }
403
+ }
404
+ // No recorded child starts with this request: explain it against the next unclaimed child.
405
+ const next = [...this.lanes.entries()]
406
+ .filter(([lane]) => lane.startsWith("L:") && !this.claimed.has(lane))
407
+ .sort(([, a], [, b]) => (a[0]?.call ?? 0) - (b[0]?.call ?? 0))[0];
408
+ const first = next?.[1][0];
409
+ if (next && first) {
410
+ throw new ReplayDivergenceError(first.call, next[0], diffFingerprints(first.call, first.parts, this.store, material));
411
+ }
412
+ throw new ReplayExhausted(number, base, "no recorded child session starts with this request.");
413
+ }
414
+ /** Throws `ReplayUnconsumedError` when recorded calls were never requested (extra calls). */
415
+ assertConsumed() {
416
+ const leftover = [];
417
+ for (const [lane, queue] of this.lanes)
418
+ for (const call of queue.slice(this.position.get(lane) ?? 0))
419
+ leftover.push({ call: call.call, lane });
420
+ leftover.sort((a, b) => a.call - b.call);
421
+ if (leftover.length > 0)
422
+ throw new ReplayUnconsumedError(leftover);
423
+ }
424
+ }