@statelyai/agent 2.0.0-alpha.36 → 2.0.0-alpha.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/dist/{effects-CuOUNYtZ.mjs → effects-BT_cSY0j.mjs} +95 -11
  3. package/dist/{effects-B23bFZEs.cjs → effects-DDseYtJk.cjs} +95 -11
  4. package/dist/index.cjs +6 -6
  5. package/dist/index.d.cts +3 -3
  6. package/dist/index.d.mts +3 -3
  7. package/dist/index.mjs +6 -6
  8. package/dist/otel.d.cts +1 -1
  9. package/dist/otel.d.mts +1 -1
  10. package/dist/{run-agent-BcQ-9NRT.d.cts → run-agent-30mH2moE.d.cts} +2 -2
  11. package/dist/{run-agent-C86drZtN.d.mts → run-agent-DlPzBWj-.d.mts} +2 -2
  12. package/dist/{setup-agent-jlo4vtaZ.d.cts → setup-agent-BaNyz6Q3.d.cts} +1 -1
  13. package/dist/{setup-agent-BIKcbVXV.d.mts → setup-agent-C1CG_nfX.d.mts} +1 -1
  14. package/dist/testing.cjs +2411 -1799
  15. package/dist/testing.d.cts +167 -4
  16. package/dist/testing.d.mts +167 -4
  17. package/dist/testing.mjs +2412 -1801
  18. package/dist/{usage-HtnrUsl9.d.cts → usage-Bm5nvpow.d.cts} +5 -0
  19. package/dist/{usage-DoP0Oi84.d.mts → usage-BnikdOca.d.mts} +5 -0
  20. package/dist/validate.cjs +1 -1
  21. package/dist/validate.d.cts +2 -2
  22. package/dist/validate.d.mts +2 -2
  23. package/dist/validate.mjs +1 -1
  24. package/dist/{verify-Dpz1u5Pe.mjs → verify-ChyX6jq0.mjs} +75 -20
  25. package/dist/{verify-Bink-Drw.cjs → verify-CwlombbC.cjs} +75 -20
  26. package/dist/{verify-CciG8UP-.d.cts → verify-DJHfEfm6.d.cts} +9 -4
  27. package/dist/{verify-Buh8XSUr.d.mts → verify-i9Hiwc5g.d.mts} +9 -4
  28. package/docs/decisions.md +1 -1
  29. package/docs/event-log.md +3 -2
  30. package/docs/hosts.md +2 -2
  31. package/docs/machines-as-data.md +5 -5
  32. package/docs/persistence.md +76 -5
  33. package/docs/quickstart.md +1 -1
  34. package/docs/steps.md +2 -0
  35. package/docs/usage-and-budgets.md +2 -0
  36. package/docs/verify.md +7 -2
  37. package/package.json +1 -1
  38. package/skills/build-machine/SKILL.md +4 -1
package/dist/testing.cjs CHANGED
@@ -1,1959 +1,2570 @@
1
1
  Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
2
2
  const require_event_log = require("./event-log-Dkc-yIYu.cjs");
3
- const require_effects = require("./effects-B23bFZEs.cjs");
4
- const require_verify = require("./verify-Bink-Drw.cjs");
3
+ const require_decision = require("./decision-CMeuNZ1g.cjs");
4
+ const require_act = require("./act-CrX0JwWY.cjs");
5
+ const require_effects = require("./effects-DDseYtJk.cjs");
6
+ const require_event_log_store = require("./event-log-store-Cl-YWDnq.cjs");
7
+ const require_verify = require("./verify-CwlombbC.cjs");
5
8
  let xstate = require("xstate");
6
- //#region src/trajectory.ts
9
+ //#region src/equivalence.ts
7
10
  /**
8
- * Classifies one item. Event-shaped things (log entries, events) become events;
9
- * everything else is a state value. A snapshot is unwrapped to its `value`.
10
- * @internal
11
+ * Verified equivalence — prove a port behaves like the code it replaces.
12
+ *
13
+ * A port is equivalent when, given the same model answers and the same tool
14
+ * results, it produces the same OBSERVABLE EFFECTS: the same number of model
15
+ * calls, the same tool calls with the same arguments, and the same output.
16
+ * Prompts, internal steps, and states are free to differ.
17
+ *
18
+ * 1. Record the original: wrap its model (`recorder.model`, an AI SDK
19
+ * `LanguageModelV3`, or `recorder.openai`, an `openai` client) and its tools
20
+ * (`recorder.tool` / `recorder.tools`), run a scenario, call
21
+ * `recorder.output(result)`, and save `recorder.recording`.
22
+ * 2. Replay it on the port: `createRecorder({ replay: recording })` answers
23
+ * every model call and tool call from the recording, with no API key and no
24
+ * side effects, and records what the port did.
25
+ * 3. `compareRecordings(original, port)` reports the first effect that differs.
26
+ *
27
+ * A port that changes how it calls the model on purpose (a model-picked action
28
+ * that became `agent.decide`, a stream that became `doGenerate`) still replays:
29
+ * `translate` turns the recorded answer into one that fits, `fallback` answers
30
+ * the calls the original never made, and `compareRecordings(…, { kinds })`
31
+ * compares only tool calls and output.
32
+ *
33
+ * Any other client (`@anthropic-ai/sdk`, `@google/genai`, a framework's own
34
+ * model client) records through `recorder.client` or `recorder.fn`: each call
35
+ * is a `call` effect, answered in replay by name and order.
36
+ *
37
+ * Provider-free: every seam is wrapped structurally, so this module imports
38
+ * neither `ai` nor `openai`.
39
+ * @module
11
40
  */
12
- function normalize$1(item) {
13
- if (require_event_log.isRecord(item)) {
14
- const nested = item["event"];
15
- if (require_event_log.isRecord(nested) && typeof nested["type"] === "string") return {
16
- kind: "event",
17
- event: nested
18
- };
19
- if (typeof item["type"] === "string") return {
20
- kind: "event",
21
- event: item
22
- };
23
- if ("value" in item) return {
24
- kind: "state",
25
- value: item["value"]
26
- };
41
+ /**
42
+ * Thrown in replay when the port asks for something the recording cannot
43
+ * answer: a model call past the last recorded one, or a tool call the original
44
+ * never made. The effect is still recorded, so {@link compareRecordings}
45
+ * reports it.
46
+ */
47
+ var RecordingDivergenceError = class extends Error {
48
+ name = "RecordingDivergenceError";
49
+ };
50
+ function toRecordedError(error) {
51
+ if (!(error instanceof Error)) return {
52
+ name: "Error",
53
+ message: String(error)
54
+ };
55
+ const recorded = {
56
+ name: error.name,
57
+ message: error.message
58
+ };
59
+ if (Object.prototype.propertyIsEnumerable.call(error, "name")) recorded.ownName = true;
60
+ const code = error.code;
61
+ if (code !== void 0) recorded.code = toJson(code);
62
+ const data = Object.fromEntries(Object.entries(error).filter(([key, value]) => key !== "code" && (typeof value === "number" || typeof value === "boolean")));
63
+ if (Object.keys(data).length) recorded.data = data;
64
+ const markers = /* @__PURE__ */ new Set();
65
+ for (let holder = error; holder; holder = Object.getPrototypeOf(holder)) for (const symbol of Object.getOwnPropertySymbols(holder)) {
66
+ const key = Symbol.keyFor(symbol);
67
+ if (key !== void 0 && error[symbol] === true) markers.add(key);
27
68
  }
28
- return {
29
- kind: "state",
30
- value: item
69
+ if (markers.size) recorded.markers = [...markers];
70
+ return recorded;
71
+ }
72
+ function fromRecordedError({ data, markers, name, ownName, ...fields }) {
73
+ const error = Object.assign(new Error(fields.message), fields, data);
74
+ if (ownName || name !== "Error") Object.defineProperty(error, "name", {
75
+ value: name,
76
+ writable: true,
77
+ configurable: true,
78
+ enumerable: ownName ?? false
79
+ });
80
+ for (const marker of markers ?? []) error[Symbol.for(marker)] = true;
81
+ return error;
82
+ }
83
+ /** What two runs compare of an error: its data and markers may differ. */
84
+ function errorIdentity$1(error) {
85
+ return error && {
86
+ name: error.name,
87
+ message: error.message,
88
+ code: error.code
31
89
  };
32
90
  }
33
- /** Every leaf of a state value as a dot path (`{ review: 'editing' }` → `review.editing`). @internal */
34
- function statePaths$1(value) {
35
- if (typeof value === "string") return [value];
36
- if (!require_event_log.isRecord(value)) return [];
37
- return Object.entries(value).flatMap(([key, child]) => statePaths$1(child).map((path) => `${key}.${path}`));
91
+ /** A JSON-safe copy: Dates become ISO strings, Errors become {@link RecordedError}. */
92
+ function toJson(value) {
93
+ if (value === void 0) return void 0;
94
+ return JSON.parse(JSON.stringify(value, function(key, item) {
95
+ const raw = this[key];
96
+ if (raw instanceof Date) return raw.toISOString();
97
+ if (item instanceof Error) return toRecordedError(item);
98
+ if (typeof item === "bigint") return item.toString();
99
+ if (item instanceof Uint8Array) return Array.from(item);
100
+ return item;
101
+ }));
38
102
  }
39
- /**
40
- * A string expectation matches a state value when it names one of its leaves or
41
- * an ancestor of one, so `'review'` matches `{ review: 'editing' }`. @internal
42
- */
43
- function stateMatchesPath(path, value) {
44
- return statePaths$1(value).some((leaf) => leaf === path || leaf.startsWith(`${path}.`));
103
+ /** Restores the `Date` timestamps the AI SDK puts on responses. */
104
+ function reviveTimestamps(value) {
105
+ if (value && typeof value === "object") {
106
+ for (const holder of [value, value.response]) if (holder && typeof holder["timestamp"] === "string") holder["timestamp"] = new Date(holder["timestamp"]);
107
+ }
108
+ return value;
45
109
  }
46
- function deepEqual(a, b) {
47
- if (Object.is(a, b)) return true;
48
- if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => deepEqual(item, b[index]));
49
- if (!require_event_log.isRecord(a) || !require_event_log.isRecord(b)) return false;
50
- const keys = Object.keys(a);
51
- return keys.length === Object.keys(b).length && keys.every((key) => deepEqual(a[key], b[key]));
110
+ function omit(value, keys) {
111
+ if (!value || typeof value !== "object") return value;
112
+ return Object.fromEntries(Object.entries(value).filter(([key]) => !keys.includes(key)));
52
113
  }
53
114
  /**
54
- * Normalizes ONE expected item into a predicate over actual items, so the
55
- * expectation is classified once instead of once per candidate:
56
- *
57
- * - A string expectation matches an event's `type`, or a state value it names.
58
- * - An event expectation matches an event with the same `type` whose other
59
- * declared keys deep-equal, so payload details are opt-in.
60
- * - A state-value expectation matches structurally.
115
+ * Key-order-independent JSON, for equality.
61
116
  * @internal
62
117
  */
63
- function matcherFor(expected) {
64
- if (typeof expected === "string") return (actual) => {
65
- const actualItem = normalize$1(actual);
66
- return actualItem.kind === "event" ? actualItem.event.type === expected : stateMatchesPath(expected, actualItem.value);
67
- };
68
- const expectedItem = normalize$1(expected);
69
- switch (expectedItem.kind) {
70
- case "event": return (actual) => {
71
- const actualItem = normalize$1(actual);
72
- return actualItem.kind === "event" && Object.keys(expectedItem.event).every((key) => deepEqual(expectedItem.event[key], actualItem.event[key]));
73
- };
74
- case "state": return (actual) => {
75
- const actualItem = normalize$1(actual);
76
- return actualItem.kind === "state" && deepEqual(expectedItem.value, actualItem.value);
77
- };
78
- }
118
+ function stableStringify(value) {
119
+ if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
120
+ if (value && typeof value === "object") return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`;
121
+ return JSON.stringify(value) ?? "undefined";
122
+ }
123
+ function jsonEqual(a, b) {
124
+ return stableStringify(toJson(a)) === stableStringify(toJson(b));
125
+ }
126
+ function streamFrom(parts) {
127
+ return new ReadableStream({ start(controller) {
128
+ for (const part of parts) controller.enqueue(reviveTimestamps(structuredClone(part)));
129
+ controller.close();
130
+ } });
131
+ }
132
+ async function* iterate(chunks) {
133
+ for (const chunk of chunks) yield structuredClone(chunk);
79
134
  }
80
135
  /**
81
- * Matches a run's trajectory against an expected one.
82
- *
83
- * Both trajectories may be state values or events collected from
84
- * `onTransition`;
85
- * items are compared by shape, so `['prompting', 'drafting']` scores a state
86
- * path and `['PROMPT_SUBMITTED', 'SEND']` scores an event log with the same
87
- * call.
88
- *
89
- * Semantics are an ordered subsequence: every expected item must appear in
90
- * `actual`, in order, with gaps allowed. Pass `{ exact: true }` to require
91
- * equality instead.
92
- *
93
- * @example Trajectory assertions in a test
94
- * ```ts
95
- * const statePath: unknown[] = [];
96
- * const result = await runToQuiescence(
97
- * createAgentRuntime(machine, {
98
- * executors,
99
- * onTransition: (snapshot) => statePath.push(snapshot.value),
100
- * }),
101
- * { input },
102
- * );
103
- *
104
- * const path = matchesTrajectory(statePath, ['prompting', 'drafting', 'sent']);
105
- * expect(path.matched, JSON.stringify(path.firstMiss)).toBe(true);
106
- * expect(matchesTrajectory(events, ['PROMPT_SUBMITTED', 'SEND']).matched).toBe(true);
107
- * ```
108
- *
109
- * @example Partial credit in an eval scorer
110
- * ```ts
111
- * const match = matchesTrajectory(output.statePath, expected.statePath);
112
- * return { name: 'state_path', score: match.score, metadata: { ...match.firstMiss } };
113
- * ```
136
+ * Creates a {@link Recorder}. With no options it records: every wrapped call
137
+ * runs for real. With `replay` it answers from a recording instead.
114
138
  */
115
- function matchesTrajectory(actual, expected, options = {}) {
116
- const expectedCount = expected.length;
117
- if (expectedCount === 0) throw new require_event_log.AgentError("empty-trajectory", "matchesTrajectory: expected trajectory must contain at least one item.");
118
- const exact = options.exact === true;
119
- let cursor = 0;
120
- let matchedCount = 0;
121
- let firstMiss;
122
- for (let index = 0; index < expectedCount; index++) {
123
- const want = expected[index];
124
- const matches = matcherFor(want);
125
- const searchedFrom = exact ? index : cursor;
126
- let found = -1;
127
- for (let at = searchedFrom; at < actual.length; at++) {
128
- if (matches(actual[at])) {
129
- found = at;
130
- break;
139
+ function createRecorder(options = {}) {
140
+ const effects = [];
141
+ const replay = options.replay?.effects;
142
+ const replayedModels = replay?.filter((effect) => effect.kind === "model");
143
+ const usedModels = /* @__PURE__ */ new Set();
144
+ const usedTools = /* @__PURE__ */ new Set();
145
+ const usedCalls = /* @__PURE__ */ new Set();
146
+ const callCounts = /* @__PURE__ */ new Map();
147
+ const normalizeEffect = (effect, list) => options.normalize?.(effect, list.indexOf(effect), list) ?? effect;
148
+ let modelCalls = 0;
149
+ let toolsInFlight = 0;
150
+ /**
151
+ * The earliest unused recorded call that fits this one: one `translate`
152
+ * adapts, or one with the same method and shape. Not by position, so a call
153
+ * the port added or dropped does not shift every answer after it.
154
+ */
155
+ function nextReplayedModel(effect) {
156
+ for (const recorded of replayedModels) {
157
+ if (usedModels.has(recorded)) continue;
158
+ const translated = options.translate?.(structuredClone(recorded), effect);
159
+ if (translated) {
160
+ effect.translated = true;
161
+ return {
162
+ recorded,
163
+ answer: translated
164
+ };
131
165
  }
132
- if (exact) break;
166
+ if (recorded.method === effect.method && modelCallsMatch(options.modelCalls, recorded, effect)) return {
167
+ recorded,
168
+ answer: recorded
169
+ };
133
170
  }
134
- if (found === -1) {
135
- firstMiss = {
136
- index,
137
- expected: want,
138
- searchedFrom
139
- };
140
- break;
141
- }
142
- matchedCount++;
143
- cursor = found + 1;
144
171
  }
145
- const matched = !firstMiss && (!exact || actual.length === expectedCount);
146
- const denominator = exact ? Math.max(expectedCount, actual.length) : expectedCount;
147
- return {
148
- matched,
149
- matchedCount,
150
- expectedCount,
151
- score: matched ? 1 : denominator === 0 ? 1 : matchedCount / denominator,
152
- ...firstMiss ? { firstMiss } : {}
153
- };
154
- }
155
- //#endregion
156
- //#region src/seam.ts
157
- /** The only own keys an executor result carries. @internal */
158
- const TEXT_RESULT_KEYS = /* @__PURE__ */ new Set([
159
- "result",
160
- "messages",
161
- "usage",
162
- "raw"
163
- ]);
164
- /**
165
- * True when a scripted entry is the executor result rather than the output
166
- * value: it owns a `result` key and owns NOTHING outside the result's own
167
- * vocabulary. Bare `'result' in value` would swallow an output object's
168
- * siblings (`{ result: 'draft', confidence: 0.9 }` would lose `confidence`)
169
- * and would also match an inherited `result`. @internal
170
- */
171
- function isTextResult(value) {
172
- return Object.hasOwn(value, "result") && Object.keys(value).every((key) => TEXT_RESULT_KEYS.has(key));
173
- }
174
- /**
175
- * Resolves ONE scripted entry to an executor result: a function entry is
176
- * called with the request, and the value is taken as the executor result only
177
- * when it is one (see {@link isTextResult}). @internal
178
- */
179
- async function resolveSeamScriptEntry(entry, request, info) {
180
- const value = typeof entry === "function" ? await entry(request, info) : entry;
181
- return require_event_log.isRecord(value) && isTextResult(value) ? value : { result: value };
182
- }
183
- /** Names a pending text request in an error message. @internal */
184
- function describeText(request) {
185
- return request.name ? `'${request.name}' (model '${request.model}')` : `(model '${request.model}')`;
186
- }
187
- /**
188
- * Stream semantics with no model: the whole text lands as one chunk. @internal
189
- */
190
- function emitScriptedChunk(result, info) {
191
- const text = require_event_log.isRecord(result) ? result["result"] : void 0;
192
- if (typeof text === "string") info?.onChunk?.(text);
193
- }
194
- /**
195
- * The seam's own answer, for scoring: the executor result's `result`.
196
- * @internal
197
- */
198
- async function seamOutputOf(result, request) {
199
- if (!require_event_log.isRecord(result)) return;
200
- return await require_event_log.normalizeGeneratorResult(result, `seam '${request.name ?? request.model}'`);
201
- }
202
- /**
203
- * Runs a machine end to end with one model call under test, and slices the run
204
- * at that call.
205
- *
206
- * The `before`/`after` slices are the point: `after.statePath` is the branch
207
- * the seam chose and `after.events` is the same question against the durable
208
- * log, both ready for `matchesTrajectory`. The state slice splits where the
209
- * state path stood when the seam answered; the event slice splits at the
210
- * seam's own effect completion (the first `xstate.done.*`/`xstate.error.*`
211
- * entry appended after the call was made).
212
- *
213
- * @example No API key: the seam is scripted too, so the whole thing runs offline.
214
- * ```ts
215
- * const run = await runSeam(emailDrafter, {
216
- * scripts: { writeFollowUps: [followUps], emailDrafter: [draft] },
217
- * seam: { request: 'writeFollowUps' },
218
- * respond: ({ state }) => (state === 'prompting' ? { type: 'PROMPT_SUBMITTED', prompt } : null),
219
- * });
220
- *
221
- * matchesTrajectory(run.after.statePath, ['needsMoreInfo', 'drafting']);
222
- * ```
223
- *
224
- * @example A candidate prompt at the seam, scored against the same rows.
225
- * ```ts
226
- * const { generateText } = createAiSdkExecutors({ models });
227
- * const run = await runSeam(emailDrafter, { ...row, candidate: generateText });
228
- * ```
229
- */
230
- async function runSeam(machine, options) {
231
- const { seam, candidate } = options;
232
- const queues = new Map(Object.entries(options.scripts ?? {}).map(([key, answers]) => [key, [...answers]]));
233
- const statePath = [];
234
- let calls = 0;
235
- let seamMatches = 0;
236
- let seamOutput;
237
- let seamUsage;
238
- const callLog = [];
239
- let seamReached = false;
240
- let callsBeforeSeam = -1;
241
- let seamStateAt = 0;
242
- let seamEventAt = 0;
243
- let liveEvents = 0;
244
- const queueKeyOf = (request) => request.name !== void 0 && queues.has(request.name) ? request.name : request.model;
172
+ function noAnswerMessage(effect) {
173
+ const unused = replayedModels.filter((recorded) => !usedModels.has(recorded));
174
+ return unused.length ? `Model call #${effect.call} (${effect.method}, ${JSON.stringify(modelCallShape(effect))}) fits no unused recorded call: ${unused.map((recorded) => `#${recorded.call} (${recorded.method}, ${JSON.stringify(modelCallShape(recorded))})`).join(", ")}. A recorded answer only fits a call of the same method and shape; pass \`translate\` to adapt one.` : `Model call #${effect.call} was never made by the recorded run (it made ${replayedModels.length}).`;
175
+ }
245
176
  /**
246
- * Consumes this request's slot in the call plan, or resolves `undefined` when
247
- * its queue is dry. With `repeatLast`, the last entry is replayed instead of
248
- * running dry.
177
+ * Records a model call around `run`, or answers it from the recording.
178
+ * `fallback` runs it on the stand-in when the recording has no answer.
249
179
  */
250
- const takeScriptedSlot = async (request, info) => {
251
- const queue = queues.get(queueKeyOf(request));
252
- if (!queue?.length) return;
253
- return resolveSeamScriptEntry(options.repeatLast && queue.length === 1 ? queue[0] : queue.shift(), request, info);
254
- };
255
- const scriptedAnswer = async (request, info) => {
256
- const scripted = await takeScriptedSlot(request, info);
257
- if (!scripted) throw new require_event_log.AgentError("seam-script-exhausted", `runSeam: no scripted answer left for request ${describeText(request)}. Add an entry to \`scripts.${queueKeyOf(request)}\`, or pass \`repeatLast: true\` to replay its last entry down a longer branch.`);
258
- return scripted;
259
- };
260
- const route = async (request, info) => {
261
- const callIndex = calls++;
262
- const isSeam = request.name === seam.request && seamMatches++ === (seam.occurrence ?? 0);
263
- const ledger = (source) => {
264
- callLog.push({
265
- ...request.name !== void 0 ? { name: request.name } : {},
266
- model: request.model,
267
- key: queueKeyOf(request),
268
- source,
269
- seam: isSeam
270
- });
180
+ async function modelCall(seam, method, request, streamed, run, fallback) {
181
+ const effect = {
182
+ kind: "model",
183
+ seam,
184
+ method,
185
+ call: modelCalls++,
186
+ request: toJson(omit(request, [
187
+ "abortSignal",
188
+ "headers",
189
+ "signal"
190
+ ]))
271
191
  };
272
- if (isSeam && candidate) {
273
- await takeScriptedSlot(request, info);
274
- seamReached = true;
275
- callsBeforeSeam = callIndex;
276
- seamStateAt = statePath.length;
277
- seamEventAt = liveEvents;
278
- const result = await candidate(request, info);
279
- seamOutput = await seamOutputOf(result, request);
280
- seamUsage = require_event_log.getCallUsage(result);
281
- ledger("candidate");
282
- return result;
283
- }
284
- const scripted = await scriptedAnswer(request, info);
285
- ledger("script");
286
- if (!isSeam) return scripted;
287
- seamReached = true;
288
- callsBeforeSeam = callIndex;
289
- seamStateAt = statePath.length;
290
- seamEventAt = liveEvents;
291
- seamOutput = scripted.result;
292
- seamUsage = scripted.usage;
293
- return scripted;
294
- };
295
- const executors = {
296
- ...options.executors,
297
- generateText: route,
298
- streamText: async (request, info) => {
299
- const result = await route(request, info);
300
- emitScriptedChunk(result, info);
301
- return result;
302
- }
303
- };
304
- let snapshot;
305
- let event;
306
- const events = [];
307
- let result;
308
- const maxTurns = options.maxTurns ?? 12;
309
- for (let turn = 0; turn <= maxTurns; turn++) {
310
- result = await require_effects.runToQuiescence(require_effects.createAgentRuntime(machine, {
311
- ...options.actors ? { actors: options.actors } : {},
312
- executors,
313
- onTransition: (next, causedBy) => {
314
- if (turn > 0 && causedBy.type === "@xstate.init") return;
315
- events.push(causedBy);
316
- liveEvents = events.length;
317
- statePath.push(next.value);
192
+ effects.push(effect);
193
+ if (replay) {
194
+ const replayed = nextReplayedModel(effect);
195
+ if (replayed) {
196
+ const { recorded, answer } = replayed;
197
+ usedModels.add(recorded);
198
+ effect.replayed = recorded.call;
199
+ if (answer.error) {
200
+ effect.error = answer.error;
201
+ throw fromRecordedError(answer.error);
202
+ }
203
+ effect.response = answer.response;
204
+ effect.stream = answer.stream;
205
+ if (seam === "openai") return streamed ? iterate(answer.stream ?? []) : structuredClone(answer.response);
206
+ return streamed ? {
207
+ ...reviveTimestamps(structuredClone(answer.response) ?? {}),
208
+ stream: streamFrom(answer.stream ?? [])
209
+ } : reviveTimestamps(structuredClone(answer.response));
318
210
  }
319
- }), {
320
- ...snapshot ? { snapshot } : { input: options.input },
321
- ...event ? { event } : {}
322
- });
323
- if (result.status !== "idle" || turn === maxTurns) break;
324
- const next = await options.respond?.({
325
- snapshot: result.snapshot,
326
- state: result.snapshot.value,
327
- meta: require_event_log.getStateMeta(result.snapshot),
328
- turn,
329
- result
330
- });
331
- if (!next) break;
332
- event = next;
333
- snapshot = result.persist();
334
- }
335
- let splitAt = events.length;
336
- if (seamReached) {
337
- splitAt = seamEventAt;
338
- for (let index = seamEventAt; index < events.length; index++) {
339
- const type = events[index].type;
340
- if (type.startsWith("xstate.done") || type.startsWith("xstate.error")) {
341
- splitAt = index;
342
- break;
211
+ if (!fallback) {
212
+ const error = new RecordingDivergenceError(noAnswerMessage(effect));
213
+ effect.error = toRecordedError(error);
214
+ throw error;
343
215
  }
216
+ effect.unrecorded = true;
217
+ run = fallback;
344
218
  }
345
- }
346
- const splitStateAt = seamReached ? seamStateAt : statePath.length;
347
- return {
348
- result,
349
- seamOutput,
350
- ...seamUsage !== void 0 ? { seamUsage } : {},
351
- callsBeforeSeam,
352
- calls: callLog,
353
- before: {
354
- statePath: statePath.slice(0, splitStateAt),
355
- events: events.slice(0, splitAt)
356
- },
357
- after: {
358
- statePath: statePath.slice(splitStateAt),
359
- events: events.slice(splitAt)
219
+ let result;
220
+ try {
221
+ result = await run();
222
+ } catch (error) {
223
+ effect.error = toRecordedError(error);
224
+ throw error;
360
225
  }
361
- };
362
- }
363
- //#endregion
364
- //#region src/internal/trace-states.ts
365
- /** The `value` of a trace event's `from` or `snapshot`. @internal */
366
- function valueOf(holder) {
367
- return require_event_log.isRecord(holder) ? holder.value : void 0;
368
- }
369
- /** A decision request on its first attempt (no failed attempts yet). @internal */
370
- function isFirstDecision(request) {
371
- return require_event_log.isRecord(request) && request.kind === "decision" && (!Array.isArray(request.attempts) || request.attempts.length === 0);
372
- }
373
- /**
374
- * Every active leaf of a state value as a dot path:
375
- * `'a'` → `['a']`; `{ a: 'b' }` → `['a.b']`; `{ p: { x: 'one', y: {} } }` → `['p.x.one', 'p.y']`.
376
- * @internal
377
- */
378
- function statePaths(value, prefix = "") {
379
- if (typeof value === "string") return [prefix + value];
380
- if (!require_event_log.isRecord(value)) return [];
381
- const entries = Object.entries(value);
382
- if (entries.length === 0) return prefix ? [prefix.slice(0, -1)] : [];
383
- return entries.flatMap(([key, child]) => statePaths(child, `${prefix}${key}.`));
384
- }
385
- /**
386
- * A trace's `machine.transition` events as steps, skipping `@xstate.init`.
387
- * A first-attempt decision's `request.start` names the decision on the step
388
- * that follows it, skipping `@agent.usage` steps. Reads live and serialized traces alike.
389
- * @internal
390
- */
391
- function traceSteps(trace) {
392
- const steps = [];
393
- let decision;
394
- for (const event of trace) {
395
- if (event.type === "request.start" && isFirstDecision(event.request)) {
396
- const name = event.request.name;
397
- decision = typeof name === "string" ? name : void 0;
398
- continue;
226
+ if (!streamed) {
227
+ effect.response = toJson(result);
228
+ return result;
399
229
  }
400
- if (event.type !== "machine.transition") continue;
401
- const type = require_event_log.isRecord(event.event) ? event.event.type : void 0;
402
- if (type === "@xstate.init") continue;
403
- const fromValue = valueOf(event.from);
404
- const toValue = valueOf(event.snapshot);
405
- steps.push({
406
- from: statePaths(fromValue),
407
- fromValue,
408
- event: typeof type === "string" ? type : "",
409
- payload: event.event,
410
- to: statePaths(toValue),
411
- toValue,
412
- ignored: event.ignored === true,
413
- ...decision === void 0 || type === "@agent.usage" ? {} : { decision }
414
- });
415
- if (type !== "@agent.usage") decision = void 0;
416
- }
417
- return steps;
418
- }
419
- //#endregion
420
- //#region src/fold-traces.ts
421
- /**
422
- * Folds many runs' traces into per-state stats: how many runs visited each
423
- * state, how many of those failed, how many options the guards withheld
424
- * at decisions made there, how many events it ignored, and how many failures
425
- * `blame` traced back to it. Pure; reads only:
426
- *
427
- * - `machine.transition`: the run visited `from.value` and `snapshot.value`;
428
- * with `ignored`, each of those states ignored one event.
429
- * - `request.start` of a decision, first attempt only: its `request.withheld`
430
- * counts toward the state the run was in, the `from.value` of the
431
- * transition its choice causes.
432
- *
433
- * A trace is one run's events, live (`AgentTraceEvent`) or serialized
434
- * (`serializeTraceEvent`). A host-driven `decideNext` traces nothing; push
435
- * `{ type: 'request.start', request: result.request }` onto the trace before
436
- * sending its event. Parallel states count once per active leaf.
437
- *
438
- * `failures` counts a failed run against every state it visited, so states
439
- * after a bad decision rank as high as the decision. Sort by `blamed` to see
440
- * where failures start.
441
- *
442
- * @example
443
- * ```ts no-check
444
- * const stats = foldTraces(traces, {
445
- * failed: (trace) => trace.at(-1)?.type === 'run.end' && trace.at(-1).status === 'error',
446
- * });
447
- * ```
448
- */
449
- function foldTraces(traces, options) {
450
- const stats = /* @__PURE__ */ new Map();
451
- const statsFor = (state) => {
452
- let entry = stats.get(state);
453
- if (!entry) {
454
- entry = {
455
- state,
456
- visits: 0,
457
- failures: 0,
458
- withheld: 0,
459
- ignored: 0,
460
- blamed: 0
461
- };
462
- stats.set(state, entry);
230
+ const parts = [];
231
+ effect.stream = parts;
232
+ if (seam === "openai") {
233
+ const source = result;
234
+ return (async function* () {
235
+ for await (const chunk of source) {
236
+ parts.push(toJson(chunk));
237
+ yield chunk;
238
+ }
239
+ })();
463
240
  }
464
- return entry;
465
- };
466
- for (const trace of traces) {
467
- const visited = /* @__PURE__ */ new Set();
468
- let current = [];
469
- const visit = (state) => {
470
- statsFor(state);
471
- visited.add(state);
241
+ const { stream, ...rest } = result;
242
+ effect.response = toJson(rest);
243
+ return {
244
+ ...rest,
245
+ stream: stream.pipeThrough(new TransformStream({ transform(part, controller) {
246
+ parts.push(toJson(part));
247
+ controller.enqueue(part);
248
+ } }))
472
249
  };
473
- for (const event of trace) if (event.type === "machine.transition") {
474
- for (const state of statePaths(valueOf(event.from))) visit(state);
475
- current = statePaths(valueOf(event.snapshot));
476
- for (const state of current) visit(state);
477
- if (event.ignored === true) for (const state of current) statsFor(state).ignored++;
478
- } else if (event.type === "request.start" && isFirstDecision(event.request)) {
479
- const withheld = event.request.withheld;
480
- if (!Array.isArray(withheld)) continue;
481
- for (const state of current) statsFor(state).withheld += withheld.length;
250
+ }
251
+ /** `inputOf` picks the recorded input from the call's arguments. */
252
+ function tool(name, fn, inputOf = (args) => args.length === 1 ? args[0] : args) {
253
+ return ((...args) => {
254
+ const input = toJson(inputOf(args));
255
+ const effect = {
256
+ kind: "tool",
257
+ name,
258
+ step: modelCalls,
259
+ input,
260
+ ...toolsInFlight > 0 ? { concurrent: true } : {}
261
+ };
262
+ effects.push(effect);
263
+ return track(call(effect, args));
264
+ });
265
+ function call(effect, args) {
266
+ const { input } = effect;
267
+ if (replay && options.tools !== "live") {
268
+ const wanted = normalizeEffect(effect, effects).input;
269
+ const matches = (candidate) => candidate.kind === "tool" && !usedTools.has(candidate) && candidate.name === name && jsonEqual(normalizeEffect(candidate, replay).input, wanted);
270
+ const recorded = replay.find(matches);
271
+ if (!recorded && options.fallback?.tools) {
272
+ effect.unrecorded = true;
273
+ return runLive(effect, args);
274
+ }
275
+ const settle = (value) => {
276
+ if (recorded?.sync) return value();
277
+ try {
278
+ return Promise.resolve(value());
279
+ } catch (error) {
280
+ return Promise.reject(error);
281
+ }
282
+ };
283
+ return settle(() => {
284
+ if (!recorded) {
285
+ const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)}; the recorded run made no such call.`);
286
+ effect.error = toRecordedError(error);
287
+ throw error;
288
+ }
289
+ usedTools.add(recorded);
290
+ if (recorded.sync) effect.sync = true;
291
+ if (recorded.error) {
292
+ effect.error = recorded.error;
293
+ throw fromRecordedError(recorded.error);
294
+ }
295
+ effect.output = recorded.output;
296
+ return structuredClone(recorded.output);
297
+ });
298
+ }
299
+ return runLive(effect, args);
482
300
  }
483
- const failed = options.failed(trace);
484
- for (const state of visited) {
485
- const entry = statsFor(state);
486
- entry.visits++;
487
- if (failed) entry.failures++;
301
+ function runLive(effect, args) {
302
+ let output;
303
+ try {
304
+ output = fn(...args);
305
+ } catch (error) {
306
+ effect.sync = true;
307
+ effect.error = toRecordedError(error);
308
+ throw error;
309
+ }
310
+ if (!(output instanceof Promise)) {
311
+ effect.sync = true;
312
+ effect.output = toJson(output);
313
+ return output;
314
+ }
315
+ return output.then((value) => {
316
+ effect.output = toJson(value);
317
+ return value;
318
+ }, (error) => {
319
+ effect.error = toRecordedError(error);
320
+ throw error;
321
+ });
488
322
  }
489
- const blamed = failed ? options.blame?.(trace) : void 0;
490
- if (blamed !== void 0) statsFor(blamed).blamed++;
491
323
  }
492
- return [...stats.values()];
493
- }
494
- //#endregion
495
- //#region src/trace-coverage.ts
496
- /** Mirrors xstate@6.0.0-alpha.69 `matchesEventDescriptor` (src/utils.ts), which the package does not export. */
497
- function matchesDescriptor(eventType, descriptor) {
498
- if (descriptor === eventType || descriptor === "*") return true;
499
- if (!descriptor.endsWith(".*")) return false;
500
- const tokens = descriptor.split(".");
501
- const eventTokens = eventType.split(".");
502
- for (let index = 0; index < tokens.length; index++) {
503
- if (tokens[index] === "*") return index === tokens.length - 1;
504
- if (tokens[index] !== eventTokens[index]) return false;
505
- }
506
- return true;
507
- }
508
- /** Whether a definition's `matches` pattern (e.g. an invoke's `actorId`) accepts the traced event. */
509
- function matchesPattern(definition, event) {
510
- const pattern = definition.matches;
511
- if (!require_event_log.isRecord(pattern) || !require_event_log.isRecord(event)) return true;
512
- return Object.entries(pattern).every(([key, value]) => Object.is(event[key], value));
513
- }
514
- /**
515
- * The descriptor XState would try first at `node` for `event`, guards aside:
516
- * the exact key, then partial and wildcard keys longest first, as in
517
- * xstate@6.0.0-alpha.69 `getCandidates` (src/stateUtils.ts).
518
- */
519
- function candidateAt(node, type, event) {
520
- const keys = [...node.transitions.keys()].filter((key) => key !== type && matchesDescriptor(type, key)).sort((a, b) => b.length - a.length);
521
- if (node.transitions.has(type)) keys.unshift(type);
522
- return keys.find((key) => node.transitions.get(key).some((d) => matchesPattern(d, event)));
523
- }
524
- const pathOf = (node) => node.path.join(".");
525
- const sameLeaves = (a, b) => statePaths(a).sort().join("|") === statePaths(b).sort().join("|");
526
- /**
527
- * The leaves a trace passed through inside its macrosteps: states an `always`
528
- * transition left before the step ended, which no traced snapshot holds.
529
- * Re-derived with XState's pure `getMicrosteps` from the snapshot before each
530
- * step (the previous step's, or the fresh start's microsteps from its input).
531
- * A step that cannot be recomputed, or whose recomputation ends elsewhere
532
- * than the traced snapshot, adds none. Only `value` and `context` are rebuilt,
533
- * so a guard that reads live children can take another route to the same
534
- * leaves, and that route is credited.
535
- */
536
- function passedThrough(machine, trace) {
537
- const leaves = /* @__PURE__ */ new Set();
538
- let resumed = false;
539
- let previous;
540
- for (const event of trace) {
541
- if (event.type === "run.start") resumed = event.snapshot !== void 0;
542
- if (event.type !== "machine.transition" || !require_event_log.isRecord(event.event)) continue;
543
- const snapshot = require_event_log.isRecord(event.snapshot) ? event.snapshot : void 0;
544
- try {
545
- const microsteps = event.event.type === "@xstate.init" ? resumed ? [] : (0, xstate.getInitialMicrosteps)(machine, event.event.input) : previous && sameLeaves(previous.value, valueOf(event.from)) ? (0, xstate.getMicrosteps)(machine, machine.resolveState({
546
- value: previous.value,
547
- context: previous.context
548
- }), event.event) : [];
549
- const last = microsteps.at(-1)?.[0];
550
- if (last && snapshot && sameLeaves(last.value, snapshot.value)) for (const [step] of microsteps) for (const leaf of statePaths(step.value)) leaves.add(leaf);
551
- } catch {}
552
- previous = snapshot;
553
- }
554
- return leaves;
555
- }
556
- const ratioOf = (visited, total) => total === 0 ? 1 : visited / total;
557
- /**
558
- * Measures how much of a machine a set of runs exercised: which atomic and
559
- * final states they visited, and which handlers they took. A handler is a
560
- * `(declaring state, event descriptor)` pair from `on`, invoke
561
- * `onDone`/`onError`, and `after`, with the targets it declares and the
562
- * leaves the runs landed in after it. Pure; reads only `machine.transition`
563
- * events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
564
- *
565
- * A step credits one handler per active leaf: the nearest node, from the leaf
566
- * up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
567
- * event, preferred in XState's order. Guards are not replayed, so a step
568
- * whose nearest handler was rejected and fell through to an ancestor is
569
- * credited to the nearest one. XState keys invoke and `after` handlers by
570
- * bare type (`xstate.done.actor`), one per state however many invokes; a
571
- * definition's `matches` (an invoke's `actorId`) must accept the traced event
572
- * for its state to be credited. Ignored steps credit no handler. `always`
573
- * handlers are listed as `unobservable`, never as unvisited; a state one
574
- * passes through is visited, re-derived from each step's microsteps.
575
- *
576
- * @example
577
- * ```ts no-check
578
- * const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
579
- * expect(report.unvisitedHandlers).toEqual([]);
580
- * ```
581
- */
582
- function traceCoverage(machine, traces, options = {}) {
583
- const nodes = /* @__PURE__ */ new Map();
584
- const states = [];
585
- const handlers = [];
586
- const handlerAt = /* @__PURE__ */ new Map();
587
- const unobservable = [];
588
- const walk = (node) => {
589
- const state = pathOf(node);
590
- nodes.set(state, node);
591
- if (node.type === "atomic" || node.type === "final") states.push({
592
- state,
593
- runs: 0
594
- });
595
- const byEvent = /* @__PURE__ */ new Map();
596
- for (const [event, definitions] of node.transitions) {
597
- const dynamic = definitions.some((d) => typeof d.to === "function");
598
- const declared = [...new Set(definitions.flatMap((d) => (d.target ?? []).map(pathOf)))];
599
- const handler = {
600
- state,
601
- event,
602
- runs: 0,
603
- targets: {
604
- declared: dynamic ? "dynamic" : declared,
605
- seen: [],
606
- unreached: []
607
- }
324
+ function fn(name, run, callOptions = {}) {
325
+ const { redact, serialize = toJson, revive } = callOptions;
326
+ return (async (...args) => {
327
+ const passed = [...args];
328
+ while (passed.length && passed.at(-1) === void 0) passed.pop();
329
+ const call = callCounts.get(name) ?? 0;
330
+ callCounts.set(name, call + 1);
331
+ const effect = {
332
+ kind: "call",
333
+ name,
334
+ call,
335
+ args: toJson(redact ? redact(passed) : passed)
608
336
  };
609
- handlers.push(handler);
610
- byEvent.set(event, handler);
611
- }
612
- handlerAt.set(node, byEvent);
613
- if (node.always?.length) unobservable.push({
614
- state,
615
- event: "always"
616
- });
617
- for (const child of Object.values(node.states)) walk(child);
618
- };
619
- walk(machine.root);
620
- const stateRuns = new Map(states.map((row) => [row.state, row]));
621
- const seen = /* @__PURE__ */ new Map();
622
- for (const trace of traces) {
623
- const steps = traceSteps(trace);
624
- const visited = new Set(steps[0]?.from);
625
- for (const event of trace) {
626
- if (event.type !== "machine.transition" || !require_event_log.isRecord(event.event)) continue;
627
- if (event.event.type !== "@xstate.init" || !require_event_log.isRecord(event.snapshot)) continue;
628
- for (const leaf of statePaths(event.snapshot.value)) visited.add(leaf);
629
- }
630
- if (unobservable.length > 0) for (const leaf of passedThrough(machine, trace)) visited.add(leaf);
631
- const credited = /* @__PURE__ */ new Set();
632
- for (const step of steps) {
633
- for (const leaf of step.to) visited.add(leaf);
634
- if (step.ignored) continue;
635
- const hits = /* @__PURE__ */ new Set();
636
- for (const leaf of step.from) for (let node = nodes.get(leaf); node; node = node.parent) {
637
- if (node.type === "final") continue;
638
- const event = candidateAt(node, step.event, step.payload);
639
- if (event === void 0) continue;
640
- hits.add(handlerAt.get(node).get(event));
641
- break;
337
+ effects.push(effect);
338
+ if (replay) {
339
+ const recorded = replay.find((candidate) => candidate.kind === "call" && candidate.name === name && !usedCalls.has(candidate));
340
+ if (!recorded) {
341
+ const error = new RecordingDivergenceError(`Call #${call} to '${name}' was never made by the recorded run.`);
342
+ effect.error = toRecordedError(error);
343
+ throw error;
344
+ }
345
+ usedCalls.add(recorded);
346
+ if (recorded.error) {
347
+ effect.error = recorded.error;
348
+ throw fromRecordedError(recorded.error);
349
+ }
350
+ if (recorded.stream) {
351
+ effect.stream = recorded.stream;
352
+ const chunks = structuredClone(recorded.stream);
353
+ return revive ? revive(chunks) : iterate(chunks);
354
+ }
355
+ effect.result = recorded.result;
356
+ const result = structuredClone(recorded.result);
357
+ return revive ? revive(result) : result;
642
358
  }
643
- for (const handler of hits) {
644
- credited.add(handler);
645
- let leaves = seen.get(handler);
646
- if (!leaves) seen.set(handler, leaves = /* @__PURE__ */ new Set());
647
- for (const leaf of step.to) leaves.add(leaf);
359
+ let result;
360
+ try {
361
+ result = await run(...args);
362
+ } catch (error) {
363
+ effect.error = toRecordedError(error);
364
+ throw error;
648
365
  }
649
- }
650
- for (const state of visited) {
651
- const row = stateRuns.get(state);
652
- if (row) row.runs++;
653
- }
654
- for (const handler of credited) handler.runs++;
655
- }
656
- for (const handler of handlers) {
657
- const leaves = [...seen.get(handler) ?? []];
658
- handler.targets.seen = leaves;
659
- const { declared } = handler.targets;
660
- if (declared === "dynamic") continue;
661
- handler.targets.unreached = declared.filter((target) => !leaves.some((leaf) => leaf === target || leaf.startsWith(`${target}.`)));
662
- }
663
- const reachableComplete = options.reachable ? require_verify.isExplorationComplete(options.reachable) : void 0;
664
- let unreachable;
665
- if (options.reachable && reachableComplete) {
666
- const reached = new Set(options.reachable.reachedStates.flatMap((value) => statePaths(value)));
667
- unreachable = states.filter((row) => !reached.has(row.state) && row.runs === 0).map((row) => row.state);
366
+ if (!require_effects.isAsyncIterable(result)) {
367
+ effect.result = toJson(serialize(result));
368
+ return result;
369
+ }
370
+ const chunks = [];
371
+ effect.stream = chunks;
372
+ const source = result;
373
+ return (async function* () {
374
+ try {
375
+ for await (const chunk of source) {
376
+ chunks.push(toJson(serialize(chunk)));
377
+ yield chunk;
378
+ }
379
+ } catch (error) {
380
+ effect.error = toRecordedError(error);
381
+ throw error;
382
+ }
383
+ })();
384
+ });
668
385
  }
669
- const counted = unreachable ? states.filter((row) => !unreachable.includes(row.state)) : states;
670
- const unvisitedStates = counted.filter((row) => row.runs === 0).map((row) => row.state);
671
- const unvisitedHandlers = handlers.filter((handler) => handler.runs === 0).map(({ state, event }) => ({
672
- state,
673
- event
674
- }));
675
- return {
676
- states,
677
- handlers,
678
- unvisitedStates,
679
- unvisitedHandlers,
680
- unobservable,
681
- ...unreachable ? { unreachable } : {},
682
- ...reachableComplete === void 0 ? {} : { reachableComplete },
683
- ratio: {
684
- states: ratioOf(counted.length - unvisitedStates.length, counted.length),
685
- handlers: ratioOf(handlers.length - unvisitedHandlers.length, handlers.length)
686
- }
687
- };
688
- }
689
- //#endregion
690
- //#region src/first-divergence.ts
691
- const defaultKey = (step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`;
692
- const isErrorLike = (value) => require_event_log.isError(value) || require_event_log.isRecord(value) && typeof value.name === "string" && typeof value.message === "string";
693
- const normalizeErrors = (value, seen = /* @__PURE__ */ new WeakSet()) => {
694
- if (value === null || typeof value !== "object") return value;
695
- if (seen.has(value)) return "[circular]";
696
- seen.add(value);
697
- try {
698
- if (isErrorLike(value) && (require_event_log.isError(value) || "stack" in value || "code" in value)) {
699
- const { name, message, code, cause } = value;
700
- return {
701
- name,
702
- message,
703
- ...typeof code === "string" ? { code } : {},
704
- ...cause !== void 0 ? { cause: normalizeErrors(cause, seen) } : {}
386
+ /** Counts an async tool call as running until it settles. */
387
+ function track(result) {
388
+ if (result instanceof Promise) {
389
+ toolsInFlight++;
390
+ const settled = () => {
391
+ toolsInFlight--;
705
392
  };
393
+ result.then(settled, settled);
706
394
  }
707
- if (Array.isArray(value)) return value.map((item) => normalizeErrors(item, seen));
708
- if (!require_event_log.isPlainObject(value)) return value;
709
- const copy = Object.create(null);
710
- for (const key of Object.keys(value)) copy[key] = normalizeErrors(value[key], seen);
711
- return copy;
712
- } finally {
713
- seen.delete(value);
395
+ return result;
714
396
  }
715
- };
716
- const payloadKey = (step) => {
717
- if (!require_event_log.isRecord(step.payload)) return `${defaultKey(step)}|${require_event_log.canonicalEventJson(step.payload)}`;
718
- const { sessionId: _sessionId, ...payload } = step.payload;
719
- return `${defaultKey(step)}|${require_event_log.canonicalEventJson(normalizeErrors(payload))}`;
720
- };
721
- const isTrace = (items) => items.length === 0 || items.some((item) => require_event_log.isRecord(item) && (item.type === "machine.transition" || item.type === "run.start" || item.type === "run.end"));
722
- /**
723
- * Finds the first step where a run departs from a reference, so a failure is
724
- * blamed on the state it started in rather than on every state the run went
725
- * through afterwards.
726
- *
727
- * - Against a reference **trace**, such as a passing run of the same
728
- * scenario, steps are compared in order by `key`. The first step that
729
- * differs, or the point where either trace ends early, is the divergence.
730
- * - Against an expected **trajectory** (`TrajectoryItem[]`, as for
731
- * {@link matchesTrajectory}), the run is laid out as its initial state, then
732
- * each step's event and new state. The first expected item it never reaches
733
- * is the divergence, blamed on the step after the last item it did reach. A
734
- * run that goes on past the trajectory has not diverged.
735
- *
736
- * An empty reference is a trace with no steps, so any step diverges from it.
737
- * A trace is one run's events, live or serialized. Steps the machine ignored
738
- * changed nothing and are left out on both sides. Returns `undefined` when the
739
- * run does not diverge.
740
- *
741
- * @example
742
- * ```ts no-check
743
- * const stats = foldTraces(traces, {
744
- * failed,
745
- * blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
746
- * });
747
- * ```
748
- */
749
- function firstDivergence(actual, reference, options = {}) {
750
- const steps = traceSteps(actual).filter((step) => !step.ignored);
751
- return isTrace(reference) ? againstTrace(steps, traceSteps(reference).filter((step) => !step.ignored), options.key === "payload" ? payloadKey : options.key ?? defaultKey) : againstTrajectory(actual, steps, reference);
752
- }
753
- function diverged(steps, index, expected, initial = steps[0]?.fromValue) {
754
- const step = steps[index];
755
397
  return {
756
- index,
757
- state: (step ? step.from[0] : steps.at(-1)?.to[0] ?? statePaths(initial)[0]) ?? "",
758
- ...step?.decision === void 0 ? {} : { decision: step.decision },
759
- ...step ? { actual: step } : {},
760
- ...expected === void 0 ? {} : { expected }
398
+ model(model) {
399
+ const base = model;
400
+ const stand = options.fallback?.model;
401
+ const wrapped = Object.create(base);
402
+ for (const method of [
403
+ "doGenerate",
404
+ "doStream",
405
+ "doDecide",
406
+ "doEvaluate"
407
+ ]) {
408
+ if (typeof base[method] !== "function") continue;
409
+ const invoke = (target, callOptions) => target[method].call(target, callOptions);
410
+ wrapped[method] = (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0);
411
+ }
412
+ return wrapped;
413
+ },
414
+ openai(client) {
415
+ const wrapCreate = (resource, method) => new Proxy(resource, { get(target, key, receiver) {
416
+ if (key !== "create") return Reflect.get(target, key, receiver);
417
+ return (params, requestOptions) => {
418
+ let api;
419
+ const create = (resource) => () => {
420
+ api = Reflect.get(resource, "create").call(resource, params, requestOptions);
421
+ return Promise.resolve(api);
422
+ };
423
+ const stand = fallbackResource(method);
424
+ const promise = modelCall("openai", method, params, params?.stream === true, create(target), stand && create(stand));
425
+ return Object.assign(promise, { withResponse: async () => {
426
+ const data = await promise;
427
+ const meta = await api?.withResponse?.();
428
+ return {
429
+ data,
430
+ response: meta?.response ?? new Response(null),
431
+ request_id: meta?.request_id ?? null
432
+ };
433
+ } });
434
+ };
435
+ } });
436
+ const fallbackResource = (method) => {
437
+ let value = options.fallback?.openai;
438
+ for (const key of method.split(".").slice(0, -1)) value = value?.[key];
439
+ return value && typeof value === "object" ? value : void 0;
440
+ };
441
+ const wrapped = /* @__PURE__ */ new Map();
442
+ const wrap = (key, make) => {
443
+ if (!wrapped.has(key)) wrapped.set(key, make());
444
+ return wrapped.get(key);
445
+ };
446
+ return new Proxy(client, { get(target, key, receiver) {
447
+ const value = Reflect.get(target, key, receiver);
448
+ if (key === "responses" && value) return wrap(key, () => wrapCreate(value, "responses.create"));
449
+ if (key === "chat" && value) return wrap(key, () => new Proxy(value, { get(chat, chatKey, chatReceiver) {
450
+ const inner = Reflect.get(chat, chatKey, chatReceiver);
451
+ return chatKey === "completions" && inner ? wrap("chat.completions", () => wrapCreate(inner, "chat.completions.create")) : inner;
452
+ } }));
453
+ return value;
454
+ } });
455
+ },
456
+ tool: (name, fn) => tool(name, fn),
457
+ fn,
458
+ client(name, client, { methods, ...callOptions }) {
459
+ const paths = methods.map((method) => method.split("."));
460
+ const startsWith = (path, prefix) => prefix.every((key, index) => path[index] === key);
461
+ const wrapAt = (target, prefix) => {
462
+ const wrapped = /* @__PURE__ */ new Map();
463
+ return new Proxy(target, { get(object, key, receiver) {
464
+ if (typeof key !== "string") return Reflect.get(object, key, receiver);
465
+ const path = [...prefix, key];
466
+ const method = paths.some((p) => p.length === path.length && startsWith(p, path));
467
+ if (!method && !paths.some((p) => startsWith(p, path))) return Reflect.get(object, key, receiver);
468
+ if (!wrapped.has(key)) {
469
+ const value = Reflect.get(object, key, receiver);
470
+ wrapped.set(key, method ? fn(`${name}.${path.join(".")}`, (...args) => value.apply(object, args), callOptions) : wrapAt(value && typeof value === "object" ? value : {}, path));
471
+ }
472
+ return wrapped.get(key);
473
+ } });
474
+ };
475
+ return wrapAt(client, []);
476
+ },
477
+ tools(tools) {
478
+ return Object.fromEntries(Object.entries(tools).map(([name, definition]) => [name, definition.execute ? {
479
+ ...definition,
480
+ execute: tool(name, definition.execute, (args) => args[0])
481
+ } : definition]));
482
+ },
483
+ output(value) {
484
+ effects.push({
485
+ kind: "output",
486
+ value: toJson(value)
487
+ });
488
+ },
489
+ get recording() {
490
+ return {
491
+ version: 1,
492
+ effects: toJson(effects)
493
+ };
494
+ }
761
495
  };
762
496
  }
763
- function againstTrace(steps, reference, key) {
764
- const length = Math.max(steps.length, reference.length);
765
- for (let index = 0; index < length; index++) {
766
- const step = steps[index];
767
- const want = reference[index];
768
- if (step && want && key(step) === key(want)) continue;
769
- return diverged(steps, index, want, steps[0]?.fromValue ?? reference[0]?.fromValue);
770
- }
497
+ /** True when two model calls offer the same tool names and response format. */
498
+ function sameModelCallShape(expected, actual) {
499
+ return jsonEqual(modelCallShape(expected), modelCallShape(actual));
771
500
  }
772
- function againstTrajectory(trace, steps, expected) {
773
- const initial = initialValue(trace, steps);
774
- const timeline = initial === void 0 ? [] : [{ value: initial }];
775
- const stepAt = initial === void 0 ? [] : [0];
776
- steps.forEach((step, index) => {
777
- timeline.push(step.payload);
778
- stepAt.push(index);
779
- if (JSON.stringify(step.toValue) !== JSON.stringify(step.fromValue)) {
780
- timeline.push({ value: step.toValue });
781
- stepAt.push(index);
782
- }
783
- });
784
- const match = matchesTrajectory(timeline, expected);
785
- if (!match.firstMiss) return void 0;
786
- const at = match.firstMiss.searchedFrom;
787
- return diverged(steps, at < timeline.length ? stepAt[at] ?? steps.length : steps.length, match.firstMiss.expected, initial);
501
+ function modelCallsMatch(match, expected, actual) {
502
+ if (match === "none") return true;
503
+ if (typeof match === "function") return match(expected, actual);
504
+ return sameModelCallShape(expected, actual);
788
505
  }
789
- function initialValue(trace, steps) {
790
- for (const event of trace) if (event.type === "machine.transition" && require_event_log.isRecord(event.snapshot)) return event.snapshot.value;
791
- return steps[0]?.fromValue;
506
+ function shapeMessage(expected, actual) {
507
+ return `Model call #${expected.call} changed shape: ${JSON.stringify(modelCallShape(expected))} → ${JSON.stringify(modelCallShape(actual))}. A recorded answer only fits a call of the same shape.`;
792
508
  }
793
- //#endregion
794
- //#region src/propose-from-traces.ts
795
509
  /**
796
- * Groups the approval verdicts in many runs' traces by `key` and proposes an
797
- * `allow` or `deny` rule for each. Pure and deterministic; a proposal is for
798
- * a person to review, and nothing here changes a machine.
799
- *
800
- * Reads `machine.transition` events: a verdict is a transition, not ignored,
801
- * out of `approvalState` on an approve or reject event. Rows come in the
802
- * order keys were first seen.
803
- *
804
- * @example
805
- * ```ts no-check
806
- * const rows = proposeApprovalRules(traces, {
807
- * approvalState: "awaitingApproval",
808
- * approveEvents: ["APPROVE"],
809
- * rejectEvents: ["REJECT"],
810
- * key: ({ snapshot }) => `${snapshot.context.pending.tool}:${snapshot.context.pending.path}`,
811
- * });
812
- * ```
510
+ * Offered tool names and the response format, or a decision's questions and
511
+ * their options: what a recorded answer depends on.
813
512
  */
814
- function proposeApprovalRules(traces, options) {
815
- const rows = /* @__PURE__ */ new Map();
816
- const rowFor = (key) => {
817
- let row = rows.get(key);
818
- if (!row) {
819
- row = {
820
- key,
821
- approved: 0,
822
- rejected: 0,
823
- violations: 0,
824
- runs: [],
825
- proposal: null,
826
- support: 0
827
- };
828
- rows.set(key, row);
829
- }
830
- return row;
513
+ function modelCallShape(effect) {
514
+ const request = effect.request ?? {};
515
+ const questions = request["questions"];
516
+ if (questions && typeof questions === "object") return { questions: Object.entries(questions).map(([name, question]) => {
517
+ const criteria = question["criteria"];
518
+ if (question["type"] === "choice" && criteria && typeof criteria === "object") return [
519
+ name,
520
+ question["type"],
521
+ Object.keys(criteria).sort()
522
+ ];
523
+ if (question["type"] === "score" && Array.isArray(criteria)) return [
524
+ name,
525
+ question["type"],
526
+ criteria.length
527
+ ];
528
+ return [name, question["type"]];
529
+ }).sort((x, y) => String(x[0]).localeCompare(String(y[0]))) };
530
+ return {
531
+ tools: (Array.isArray(request["tools"]) ? request["tools"] : []).map((tool) => tool["name"] ?? tool["function"]?.["name"]).sort(),
532
+ format: request["responseFormat"]?.["type"] ?? request["response_format"]?.["type"] ?? (request["text"]?.["format"])?.["type"] ?? "text"
831
533
  };
832
- const inState = (value) => statePaths(value).some((path) => path === options.approvalState || path.startsWith(`${options.approvalState}.`));
833
- let index = 0;
834
- for (const trace of traces) {
835
- const run = index++;
836
- const seen = /* @__PURE__ */ new Set();
837
- let snapshot;
838
- for (const event of trace) {
839
- if (event.type !== "machine.transition") continue;
840
- const before = snapshot;
841
- snapshot = event.snapshot;
842
- const type = require_event_log.isRecord(event.event) ? event.event.type : void 0;
843
- if (event.ignored === true || typeof type !== "string" || !inState(valueOf(event.from))) continue;
844
- const approve = options.approveEvents.includes(type);
845
- if (!approve && !options.rejectEvents.includes(type)) continue;
846
- const key = options.key({
847
- event: event.event,
848
- snapshot: before,
849
- trace
850
- });
851
- if (key === void 0) continue;
852
- const row = rowFor(key);
853
- if (approve) row.approved++;
854
- else row.rejected++;
855
- if (!seen.has(key)) {
856
- seen.add(key);
857
- row.runs.push(run);
858
- }
859
- }
860
- for (const key of options.violations?.(trace) ?? []) {
861
- const row = rows.get(key);
862
- if (row) row.violations++;
534
+ }
535
+ /**
536
+ * Puts each group of concurrent tool calls in a stable order, so parallel tool
537
+ * calls that finished in a different order still compare equal. A tool call
538
+ * that started after the previous one settled begins a new group, so
539
+ * sequential calls keep their order.
540
+ */
541
+ function normalize$1(effects) {
542
+ const result = [];
543
+ let batch = [];
544
+ const flush = () => {
545
+ result.push(...batch.sort((a, b) => stableStringify([a.name, a.input]).localeCompare(stableStringify([b.name, b.input]))));
546
+ batch = [];
547
+ };
548
+ for (const effect of effects) {
549
+ if (effect.kind !== "tool" || !effect.concurrent) flush();
550
+ if (effect.kind === "tool") batch.push(effect);
551
+ else result.push(effect);
552
+ }
553
+ flush();
554
+ return result;
555
+ }
556
+ function describe(effect) {
557
+ switch (effect.kind) {
558
+ case "model": return `model call #${effect.call} (${effect.method})`;
559
+ case "tool": return `tool '${effect.name}' with ${JSON.stringify(effect.input)}`;
560
+ case "call": return `call #${effect.call} to '${effect.name}' with ${JSON.stringify(effect.args)}`;
561
+ case "output": return `output ${JSON.stringify(effect.value)}`;
562
+ }
563
+ }
564
+ /** Why two effects differ, or `undefined` when they are the same. */
565
+ function differ(left, right, options) {
566
+ if (left.kind !== right.kind) return {
567
+ reason: "kind",
568
+ message: `Expected ${describe(left)}; the port made ${describe(right)}.`
569
+ };
570
+ if (left.kind === "model" && right.kind === "model") {
571
+ if (right.translated && right.replayed === left.call) return void 0;
572
+ if (left.method !== right.method) return {
573
+ reason: "model-method",
574
+ message: `Model call #${left.call} used ${left.method}; the port used ${right.method}.`
575
+ };
576
+ if (!modelCallsMatch(options.modelCalls, left, right)) return {
577
+ reason: "model-call-shape",
578
+ message: shapeMessage(left, right)
579
+ };
580
+ if (!jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) {
581
+ const outcome = (error) => error ? `failed with ${error.name}: ${error.message}` : "succeeded";
582
+ return {
583
+ reason: "model-error",
584
+ message: `Model call #${left.call} ${outcome(left.error)}; in the port it ${outcome(right.error)}.`
585
+ };
863
586
  }
587
+ return;
864
588
  }
865
- for (const row of rows.values()) {
866
- row.support = row.runs.length;
867
- row.proposal = row.approved > 0 && row.rejected === 0 && row.violations === 0 ? "allow" : row.rejected > 0 && row.approved === 0 ? "deny" : null;
589
+ if (left.kind === "tool" && right.kind === "tool") {
590
+ if (left.name !== right.name) return {
591
+ reason: "tool-name",
592
+ message: `Expected ${describe(left)}; the port made ${describe(right)}.`
593
+ };
594
+ if (!jsonEqual(left.input, right.input)) return {
595
+ reason: "tool-input",
596
+ message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.input)}.`
597
+ };
598
+ if (!jsonEqual(left.output, right.output) || !jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) return {
599
+ reason: "tool-result",
600
+ message: `Tool '${left.name}' returned a different result.`
601
+ };
602
+ return;
868
603
  }
869
- return [...rows.values()];
604
+ if (left.kind === "call" && right.kind === "call") {
605
+ if (left.name !== right.name) return {
606
+ reason: "call-name",
607
+ message: `Expected ${describe(left)}; the port made ${describe(right)}.`
608
+ };
609
+ if (!jsonEqual(left.args, right.args)) return {
610
+ reason: "call-args",
611
+ message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.args)}.`
612
+ };
613
+ if (!jsonEqual(left.result, right.result) || !jsonEqual(left.stream, right.stream) || !jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) return {
614
+ reason: "call-result",
615
+ message: `Call #${left.call} to '${left.name}' returned a different result.`
616
+ };
617
+ return;
618
+ }
619
+ if (left.kind === "output" && right.kind === "output" && !jsonEqual(left.value, right.value)) return {
620
+ reason: "output",
621
+ message: `Expected ${describe(left)}; the port returned ${JSON.stringify(right.value)}.`
622
+ };
870
623
  }
871
624
  /**
872
- * Proposes a value for each cap (retry count, steps, batch size) from many
873
- * runs' traces: the 95th percentile of what passing runs used, plus one, and
874
- * never below the current cap while runs still hit it. Pure and
875
- * deterministic; a proposal is for a person to review, and nothing here
876
- * changes a machine.
877
- *
878
- * @example
879
- * ```ts no-check
880
- * const [retries] = proposeCaps(traces, {
881
- * caps: [{ name: "retries", current: 3, failureState: "gaveUp", valueFromRun: countRetries }],
882
- * });
883
- * ```
625
+ * Compares two recordings and reports every effect that differs. `expected`
626
+ * is the original's recording, `actual` the port's. Effects that are the same
627
+ * on both sides are lined up first (longest common subsequence), so one extra
628
+ * call does not shift every effect after it; between them, effects of the same
629
+ * kind pair up and report what changed, and the rest are missing or extra.
884
630
  */
885
- function proposeCaps(traces, options) {
886
- const runs = [...traces];
887
- return options.caps.map((cap) => {
888
- let hits = 0;
889
- let max = null;
890
- const passing = [];
891
- for (const trace of runs) {
892
- const hit = hitCap(trace, cap);
893
- if (hit) hits++;
894
- const value = cap.valueFromRun(trace);
895
- if (value === void 0) continue;
896
- max = max === null ? value : Math.max(max, value);
897
- if (!hit) passing.push(value);
631
+ function compareRecordings(expected, actual, options = {}) {
632
+ const select = (effects) => normalize$1(effects.flatMap((effect, index) => {
633
+ if (options.kinds && !options.kinds.includes(effect.kind)) return [];
634
+ const mapped = options.normalize ? options.normalize(effect, index, effects) : effect;
635
+ return mapped ? [mapped] : [];
636
+ }));
637
+ const a = select(expected.effects);
638
+ const b = select(actual.effects);
639
+ const same = a.map((left) => b.map((right) => !differ(left, right, options)));
640
+ const lcs = Array.from({ length: a.length + 1 }, () => Array.from({ length: b.length + 1 }, () => 0));
641
+ for (let i = a.length - 1; i >= 0; i--) for (let j = b.length - 1; j >= 0; j--) lcs[i][j] = same[i][j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
642
+ const divergences = [];
643
+ let lefts = [];
644
+ let rights = [];
645
+ const flushGap = (at) => {
646
+ const unpairedRights = [...rights];
647
+ const changed = [];
648
+ for (const index of lefts) {
649
+ const left = a[index];
650
+ const k = unpairedRights.findIndex((j) => b[j].kind === left.kind);
651
+ if (k === -1) {
652
+ changed.push({
653
+ index,
654
+ reason: left.kind === "output" ? "output" : "missing-effect",
655
+ expected: left,
656
+ message: left.kind === "output" ? `The port never returned; the original returned ${JSON.stringify(left.value)}.` : `The port never made ${describe(left)}.`
657
+ });
658
+ continue;
659
+ }
660
+ const right = b[unpairedRights.splice(k, 1)[0]];
661
+ const difference = differ(left, right, options);
662
+ if (difference) changed.push({
663
+ index,
664
+ expected: left,
665
+ actual: right,
666
+ ...difference
667
+ });
898
668
  }
899
- passing.sort((a, b) => a - b);
900
- const p95 = passing.length ? passing[Math.ceil(.95 * passing.length) - 1] : null;
901
- const proposal = p95 === null ? null : hits > 0 && cap.current !== void 0 ? Math.max(p95 + 1, cap.current) : p95 + 1;
902
- return {
903
- name: cap.name,
904
- hits,
905
- passing: passing.length,
906
- max,
907
- p95,
908
- proposal
909
- };
910
- });
911
- }
912
- function hitCap(trace, cap) {
913
- for (const event of trace) {
914
- if (cap.failureState !== void 0 && event.type === "machine.transition" && statePaths(valueOf(event.snapshot)).some((path) => path === cap.failureState || path.startsWith(`${cap.failureState}.`))) return true;
915
- if (cap.failureOutput && event.type === "run.end" && event.status === "done") {
916
- if (cap.failureOutput(event.output)) return true;
669
+ for (const j of unpairedRights) {
670
+ const extra = b[j];
671
+ divergences.push({
672
+ index: lefts[0] ?? at,
673
+ reason: extra.kind === "output" ? "output" : "extra-effect",
674
+ actual: extra,
675
+ message: extra.kind === "output" ? `The original never returned; the port returned ${JSON.stringify(extra.value)}.` : `The port made ${describe(extra)}, which the original never made.`
676
+ });
917
677
  }
918
- }
919
- return false;
678
+ divergences.push(...changed);
679
+ lefts = [];
680
+ rights = [];
681
+ };
682
+ let i = 0;
683
+ let j = 0;
684
+ while (i < a.length || j < b.length) if (i < a.length && j < b.length && same[i][j]) {
685
+ flushGap(i);
686
+ i++;
687
+ j++;
688
+ } else if (j < b.length && (i === a.length || lcs[i][j + 1] >= lcs[i + 1][j])) rights.push(j++);
689
+ else lefts.push(i++);
690
+ flushGap(i);
691
+ divergences.sort((x, y) => x.index - y.index);
692
+ return divergences.length ? {
693
+ equivalent: false,
694
+ divergence: divergences[0],
695
+ divergences
696
+ } : {
697
+ equivalent: true,
698
+ divergences
699
+ };
920
700
  }
921
701
  //#endregion
922
- //#region src/decision-requests.ts
702
+ //#region src/crash-recovery.ts
703
+ var CrashError = class extends Error {
704
+ constructor() {
705
+ super("checkCrashRecovery: simulated crash");
706
+ this.name = "CrashError";
707
+ }
708
+ };
709
+ const DONE_ACTOR = "xstate.done.actor";
710
+ const ERROR_ACTOR = "xstate.error.actor";
923
711
  /**
924
- * The decisions a run asked for, read from its trace: each decision's first
925
- * `request.start`, with the states the run was in. A trace is one run's events,
926
- * live or serialized with `serializeTraceEvent`. Retries of the same decision
927
- * are left out, so each decision appears once.
712
+ * Runs `machine` once uninterrupted, then once per crash point: killed after
713
+ * each log append and while each effect is in flight, then resumed from the
714
+ * store's log by a fresh runtime. Each resumed run is checked against the
715
+ * uninterrupted one:
928
716
  *
929
- * Use the requests as eval rows (run a decider on each and compare its event
930
- * with a label) or as probe inputs for a policy. A serialized request keeps its
931
- * `options`, `metadata`, `system`, and `prompt`; event schemas do not survive
932
- * serialization, so replay closed choices (`options`) rather than open ones.
717
+ * - An effect whose completion was durable before the crash never runs again.
718
+ * Only an effect in flight at the crash may (execution is at-least-once).
719
+ * - Every effect of the uninterrupted run runs, and no other: a re-executed
720
+ * model call keeps its `callKey` and its request.
721
+ * - The run ends with the same status, output, state value, and context.
722
+ * - The resumed log replays to that state.
933
723
  *
934
- * @example
935
- * ```ts no-check
936
- * const rows = traces.flatMap((trace) => getDecisionRequests(trace, { name: "chooseResolution" }));
937
- * for (const { request } of rows) console.log((await decide(request)).event);
938
- * ```
724
+ * Model calls the uninterrupted run made are answered from its recording.
725
+ * Plain actors run live, so they must give the same result however many times
726
+ * they run (pass `reset` for fakes that keep state).
939
727
  */
940
- function getDecisionRequests(trace, options = {}) {
941
- const requests = [];
942
- let current = [];
943
- for (const event of trace) if (event.type === "machine.transition") current = statePaths(valueOf(event.snapshot));
944
- else if (event.type === "request.start" && isFirstDecision(event.request)) {
945
- if (options.name !== void 0 && event.request.name !== options.name) continue;
946
- requests.push({
947
- states: [...current],
948
- request: event.request
728
+ async function checkCrashRecovery(machine, options = {}) {
729
+ const timeout = options.timeout ?? 5e3;
730
+ options.reset?.();
731
+ const baselineWorld = newWorld(() => false);
732
+ const baselineStore = observedStore((options.store ?? require_event_log_store.createInMemoryEventLogStore)(), baselineWorld, () => false);
733
+ const baseline = await withTimeout(runTurns(machine, options, baselineWorld, baselineStore.store), timeout);
734
+ if (baseline === TIMED_OUT) throw new Error(`checkCrashRecovery: the uninterrupted run did not settle within ${timeout}ms`);
735
+ if (baselineWorld.unsent.length > 0) {
736
+ const keys = baselineWorld.unsent.map((index) => {
737
+ const { during, occurrence = 0 } = options.sendDuring[index];
738
+ return occurrence === 0 ? `'${during}'` : `'${during}' (start ${occurrence})`;
949
739
  });
740
+ throw new Error(`checkCrashRecovery: sendDuring names effects the uninterrupted run never started: ${keys.join(", ")}`);
950
741
  }
951
- return requests;
742
+ if (baselineWorld.undelivered > 0) throw new Error(`checkCrashRecovery: the uninterrupted run settled '${baseline.status}' with ${baselineWorld.undelivered} of \`events\` undelivered`);
743
+ const baselineEntries = await baselineStore.store.read(THREAD);
744
+ const baselineCounts = countBy(baselineWorld.effects);
745
+ const expected = outcomeOf(baseline);
746
+ const points = [];
747
+ for (const entries of /* @__PURE__ */ new Set([0, ...baselineStore.appends])) points.push({
748
+ kind: "log",
749
+ entries
750
+ });
751
+ const seen = /* @__PURE__ */ new Map();
752
+ baselineWorld.effects.forEach((effect, index) => {
753
+ const occurrence = seen.get(effect.key) ?? 0;
754
+ seen.set(effect.key, occurrence + 1);
755
+ points.push({
756
+ kind: "effect",
757
+ effect: {
758
+ kind: effect.kind,
759
+ key: effect.key
760
+ },
761
+ index,
762
+ occurrence
763
+ });
764
+ });
765
+ const failures = [];
766
+ const unreached = [];
767
+ const ctx = {
768
+ machine,
769
+ options,
770
+ timeout,
771
+ baselineCounts,
772
+ recorded: baselineWorld.answers,
773
+ expected,
774
+ sentAt: new Map([...baselineWorld.sentEntries].map(([index, entry]) => [index, entry.index]))
775
+ };
776
+ for (const point of points) {
777
+ let found = await crashAt(point, ctx);
778
+ for (let retry = 1; !Array.isArray(found) && retry < UNREACHED_RETRIES; retry++) found = await crashAt(point, ctx);
779
+ if (Array.isArray(found)) failures.push(...found);
780
+ else unreached.push({
781
+ point,
782
+ reason: found.unreached
783
+ });
784
+ }
785
+ return {
786
+ ok: failures.length === 0 && unreached.length === 0,
787
+ baseline: {
788
+ status: baseline.status,
789
+ ...baseline.status === "done" ? { output: baseline.output } : {},
790
+ entries: baselineEntries.length,
791
+ effects: baselineWorld.effects.map(({ kind, key }) => ({
792
+ kind,
793
+ key
794
+ }))
795
+ },
796
+ points,
797
+ unreached,
798
+ failures
799
+ };
952
800
  }
953
- //#endregion
954
- //#region src/repeat-runs.ts
955
- /**
956
- * Summarizes repeated runs of each scenario: how often each passed, whether
957
- * all its runs passed (pass^k), how many distinct state paths they took, and
958
- * where they first went different ways. Pure. Use it for runs collected
959
- * elsewhere, such as CI shards or replays; {@link repeatRuns} collects them and
960
- * calls it.
961
- *
962
- * A scenario that passes 3 of 5 runs and `divergedAt: 'deciding'` points at one
963
- * decision, not at a flaky agent.
964
- *
965
- * @example
966
- * ```ts no-check
967
- * const report = summarizeRepeats(shards.map((s) => ({ id: s.id, scenario: s, runs: s.runs })));
968
- * report.passHat(3); // the chance 3 runs of a scenario all pass
969
- * ```
970
- */
971
- function summarizeRepeats(groups) {
972
- const scenarios = groups.map((group) => summarize$1(group));
973
- const traces = groups.flatMap((group) => group.runs.flatMap((run) => run.trace ? [run.trace] : []));
974
- const most = Math.max(0, ...groups.map((group) => group.runs.length));
975
- const passHat = (j) => {
976
- if (!Number.isInteger(j) || j < 1 || j > most) throw new require_event_log.AgentError("pass-hat-out-of-range", `passHat: j must be a whole number from 1 to ${most}, got ${j}.`);
977
- const eligible = scenarios.filter((scenario) => scenario.runs.length >= j);
978
- return eligible.reduce((total, scenario) => total + allPass(scenario.passes, scenario.runs.length, j), 0) / eligible.length;
801
+ const THREAD = "crash-recovery";
802
+ const UNREACHED_RETRIES = 3;
803
+ const SKIPPED_TURN = "(no timer pending)";
804
+ const TIMED_OUT = Symbol("timed out");
805
+ async function crashAt(point, ctx) {
806
+ const { machine, options, timeout } = ctx;
807
+ const fail = (problem, message, key) => ({
808
+ point,
809
+ problem,
810
+ message,
811
+ ...key !== void 0 ? { key } : {}
812
+ });
813
+ options.reset?.();
814
+ let backing;
815
+ try {
816
+ backing = (options.store ?? require_event_log_store.createInMemoryEventLogStore)();
817
+ } catch (error) {
818
+ return [fail("threw", `the store factory threw: ${String(error)}`)];
819
+ }
820
+ const controller = new AbortController();
821
+ let runs = 0;
822
+ const crashed = newWorld((effect) => point.kind === "effect" && effect.key === point.effect.key && runs++ === point.occurrence, ctx.recorded);
823
+ const crash = () => {
824
+ if (crashed.crashed) return;
825
+ crashed.crashed = true;
826
+ controller.abort(new CrashError());
979
827
  };
980
- return {
981
- scenarios,
982
- passHat,
983
- traces
828
+ let missed = point.kind === "log" && point.entries === 0 ? void 0 : point.kind === "log" ? "the log never reached that length" : "the run never started that effect";
829
+ const onEffect = crashed.onEffect;
830
+ crashed.onEffect = (effect) => {
831
+ const dies = onEffect(effect);
832
+ if (dies) {
833
+ missed = void 0;
834
+ queueMicrotask(crash);
835
+ }
836
+ return dies;
984
837
  };
838
+ const dying = observedStore(backing, crashed, (accepted) => {
839
+ if (point.kind !== "log" || accepted < point.entries) return false;
840
+ missed = accepted === point.entries ? void 0 : `one write took the log to ${accepted} entries`;
841
+ crash();
842
+ return true;
843
+ });
844
+ if (point.kind === "log" && point.entries === 0) dying.freeze();
845
+ let died;
846
+ try {
847
+ died = await withTimeout(runTurns(machine, options, crashed, dying.store, void 0, controller.signal), timeout);
848
+ } catch (error) {
849
+ if (!(error instanceof CrashError)) {
850
+ crash();
851
+ return [fail("threw", `the crashed run threw: ${String(error)}`)];
852
+ }
853
+ }
854
+ crash();
855
+ if (died === TIMED_OUT) return [fail("stuck", `the run did not stop within ${timeout}ms after the crash`)];
856
+ if (missed !== void 0) return { unreached: missed };
857
+ const resumed = newWorld(() => false, ctx.recorded);
858
+ const resumeController = new AbortController();
859
+ let result;
860
+ try {
861
+ result = await withTimeout(runTurns(machine, options, resumed, backing, {
862
+ crashed,
863
+ sentAt: ctx.sentAt
864
+ }, resumeController.signal), timeout);
865
+ } catch (error) {
866
+ resumed.crashed = true;
867
+ resumeController.abort(new CrashError());
868
+ return [fail("threw", `the resume threw: ${String(error)}`)];
869
+ }
870
+ if (result === TIMED_OUT) {
871
+ resumed.crashed = true;
872
+ resumeController.abort(new CrashError());
873
+ return [fail("stuck", `the resume did not settle within ${timeout}ms`)];
874
+ }
875
+ const failures = [];
876
+ for (const index of resumed.unsentAfterCrash) {
877
+ const { during, occurrence = 0 } = options.sendDuring[index];
878
+ failures.push(fail("undelivered", `the sendDuring event for '${during}' (start ${occurrence}) was lost in the crash and never sent again`));
879
+ }
880
+ const total = countBy([...crashed.effects, ...resumed.effects]);
881
+ const inFlight = countBy(crashed.effects.filter((effect) => !effect.completed));
882
+ for (const [key, count] of total) {
883
+ const expected = ctx.baselineCounts.get(key) ?? 0;
884
+ const allowed = expected + (inFlight.get(key) ?? 0);
885
+ if (expected === 0) failures.push(fail("unexpected", `'${key}' ran ${count}x; the uninterrupted run never made it`, key));
886
+ else if (count > allowed) failures.push(fail("reran", `'${key}' ran ${count}x, at most ${allowed} expected: its completion was durable before the crash`, key));
887
+ }
888
+ for (const [key, expected] of ctx.baselineCounts) {
889
+ const count = total.get(key) ?? 0;
890
+ if (count < expected) failures.push(fail("skipped", `'${key}' ran ${count}x, ${expected} expected`, key));
891
+ }
892
+ const crashedRequests = new Map(crashed.effects.filter((effect) => effect.kind === "model").map((effect) => [effect.key, effect.request]));
893
+ for (const effect of resumed.effects) {
894
+ if (effect.kind !== "model" || !crashedRequests.has(effect.key)) continue;
895
+ const before = crashedRequests.get(effect.key);
896
+ if (before === void 0 || effect.request === void 0) continue;
897
+ if (before !== effect.request) failures.push(fail("request-changed", `'${effect.key}' re-executed with a different request`, effect.key));
898
+ }
899
+ const actual = outcomeOf(result);
900
+ if (actual !== ctx.expected) failures.push(fail("outcome", `resumed run ended ${actual}, expected ${ctx.expected}`));
901
+ const log = await backing.read(THREAD);
902
+ try {
903
+ const replayed = require_event_log.replay(options.actors !== void 0 ? machine.provide({ actors: options.actors }) : machine, log);
904
+ const fromLog = stableStringify(require_effects.toJsonValue({
905
+ value: replayed.snapshot.value,
906
+ context: replayed.snapshot.context
907
+ }));
908
+ const live = stableStringify(require_effects.toJsonValue({
909
+ value: result.snapshot.value,
910
+ context: result.snapshot.context
911
+ }));
912
+ if (fromLog !== live) failures.push(fail("replay", `the log replays to ${fromLog}, the resumed run ended at ${live}`));
913
+ } catch (error) {
914
+ failures.push(fail("replay", `the log does not replay: ${String(error)}`));
915
+ }
916
+ return failures;
985
917
  }
986
918
  /**
987
- * Runs every scenario `k` times and summarizes the runs with
988
- * {@link summarizeRepeats}. Runs are sequential, scenario by scenario. A run
989
- * that throws fails, with its message as `error` and no trace.
990
- *
991
- * With a live model, one run per scenario says little: τ-bench found pass^k
992
- * falls well below the single-run rate. Report pass^k next to it.
993
- *
994
- * @example
995
- * ```ts no-check
996
- * const report = await repeatRuns({
997
- * scenarios,
998
- * k: 5,
999
- * run: async (scenario) => {
1000
- * const trace: AgentTraceEvent[] = [];
1001
- * const result = await runToQuiescence(
1002
- * createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
1003
- * { input: scenario.input },
1004
- * );
1005
- * return { trace, outcome: String(result.snapshot.value) };
1006
- * },
1007
- * passed: (scenario, { outcome }) => outcome === scenario.expected,
1008
- * });
1009
- * report.passHat(5);
1010
- * ```
919
+ * Runs the turns: start from `input`, then one host event per idle settle,
920
+ * with `sendDuring` events sent as their effects start and host timers fired
921
+ * when the run is idle. Resuming, what the crashed run's log holds is skipped.
1011
922
  */
1012
- async function repeatRuns(options) {
1013
- if (!Number.isInteger(options.k) || options.k < 1) throw new require_event_log.AgentError("invalid-repeat-count", `repeatRuns: k must be a whole number of 1 or more, got ${options.k}.`);
1014
- const groups = [];
1015
- for (const scenario of options.scenarios) {
1016
- const runs = [];
1017
- for (let attempt = 0; attempt < options.k; attempt++) try {
1018
- const result = await options.run(scenario, attempt);
1019
- runs.push({
1020
- passed: options.passed(scenario, result),
1021
- trace: result.trace,
1022
- ...result.outcome === void 0 ? {} : { outcome: result.outcome }
1023
- });
1024
- } catch (error) {
1025
- runs.push({
1026
- passed: false,
1027
- error: error instanceof Error ? error.message : String(error)
1028
- });
923
+ async function runTurns(machine, options, world, store, resuming, signal) {
924
+ const tracked = trackActors(machine, world);
925
+ const executors = trackExecutors(options.executors ?? {}, world);
926
+ const actors = options.actors !== void 0 ? Object.fromEntries(Object.entries(options.actors).map(([key, logic]) => [key, require_effects.isStateMachineLogic(logic) ? trackActors(logic, world) : trackLogic(logic, world)])) : void 0;
927
+ const before = resuming?.crashed;
928
+ const log = before !== void 0 ? await store.read(THREAD) : [];
929
+ const durable = new Set(log.map((entry) => entry.id));
930
+ const events = options.events ?? [];
931
+ let next = 0;
932
+ if (before !== void 0) while (next < before.hostEntries.length && (before.hostEntries[next] === SKIPPED_TURN || durable.has(before.hostEntries[next]))) next++;
933
+ const sends = options.sendDuring ?? [];
934
+ const unsent = /* @__PURE__ */ new Set();
935
+ const redeliver = [];
936
+ const sendAt = /* @__PURE__ */ new Map();
937
+ const waitsFor = /* @__PURE__ */ new Map();
938
+ sends.forEach(({ during, occurrence = 0 }, index) => {
939
+ if (resuming === void 0) {
940
+ waitsFor.set(index, occurrence);
941
+ unsent.add(index);
942
+ return;
1029
943
  }
1030
- groups.push({
1031
- id: scenario.id,
1032
- scenario,
1033
- runs
944
+ const entry = before.sentEntries.get(index);
945
+ if (entry !== void 0 && durable.has(entry.id)) return;
946
+ const starts = before.effects.filter((effect) => effect.key === during);
947
+ if (starts[occurrence]?.completed !== true) {
948
+ waitsFor.set(index, occurrence - starts.filter((effect) => effect.completed).length);
949
+ unsent.add(index);
950
+ return;
951
+ }
952
+ const at = resuming.sentAt.get(index);
953
+ if (at === void 0) return;
954
+ if (at <= log.length) redeliver.push(index);
955
+ else sendAt.set(index, at);
956
+ });
957
+ const awaitingSends = [];
958
+ let live;
959
+ const send = (index) => {
960
+ const { event } = sends[index];
961
+ awaitingSends.push({
962
+ index,
963
+ json: require_event_log.canonicalEventJson(event)
964
+ });
965
+ live?.send(event);
966
+ };
967
+ const startsSeen = /* @__PURE__ */ new Map();
968
+ const onEffect = world.onEffect;
969
+ world.onEffect = (effect) => {
970
+ const nth = startsSeen.get(effect.key) ?? 0;
971
+ startsSeen.set(effect.key, nth + 1);
972
+ for (const index of unsent) {
973
+ if (sends[index].during !== effect.key || waitsFor.get(index) !== nth) continue;
974
+ unsent.delete(index);
975
+ send(index);
976
+ }
977
+ return onEffect(effect);
978
+ };
979
+ const pendingTimers = /* @__PURE__ */ new Map();
980
+ world.clock = {
981
+ now: before?.clock.now ?? 0,
982
+ due: pendingTimers
983
+ };
984
+ const dueBefore = new Map(before?.clock.due ?? []);
985
+ const timers = options.timers === "host" ? {
986
+ schedule: ({ id, delay }) => {
987
+ if (!pendingTimers.has(id)) {
988
+ const restored = dueBefore.get(id);
989
+ dueBefore.delete(id);
990
+ pendingTimers.set(id, restored ?? world.clock.now + delay);
991
+ }
992
+ },
993
+ cancel: (id) => {
994
+ pendingTimers.delete(id);
995
+ }
996
+ } : void 0;
997
+ /** `track`: the turn delivers one of `events` (a host event, or a timer's turn), whose entry is recorded. */
998
+ const turn = (event, track) => {
999
+ let awaiting = event !== void 0 && track === "host" ? require_event_log.canonicalEventJson(event) : void 0;
1000
+ let awaitingTimer = track === "timer";
1001
+ live = require_effects.createAgentRuntime(tracked, {
1002
+ ...options.runtime,
1003
+ onEvent: (entry) => {
1004
+ options.runtime?.onEvent?.(entry);
1005
+ for (const [index, at] of sendAt) {
1006
+ if (entry.index + 1 < at) continue;
1007
+ sendAt.delete(index);
1008
+ send(index);
1009
+ }
1010
+ if (entry.actorPath !== void 0) return;
1011
+ const json = require_event_log.canonicalEventJson(entry.event);
1012
+ if (awaiting !== void 0 && json === awaiting) {
1013
+ world.hostEntries.push(entry.id);
1014
+ awaiting = void 0;
1015
+ return;
1016
+ }
1017
+ if (awaitingTimer && entry.event.type === "xstate.timer") {
1018
+ world.hostEntries.push(entry.id);
1019
+ awaitingTimer = false;
1020
+ return;
1021
+ }
1022
+ const sent = awaitingSends.findIndex((item) => item.json === json);
1023
+ if (sent !== -1) {
1024
+ world.sentEntries.set(awaitingSends[sent].index, {
1025
+ id: entry.id,
1026
+ index: entry.index
1027
+ });
1028
+ awaitingSends.splice(sent, 1);
1029
+ }
1030
+ },
1031
+ executors,
1032
+ ...actors !== void 0 ? { actors } : {},
1033
+ store,
1034
+ threadId: THREAD,
1035
+ ...timers !== void 0 ? { timers } : {},
1036
+ ...signal !== void 0 ? { signal } : {}
1034
1037
  });
1038
+ for (const index of redeliver.splice(0)) send(index);
1039
+ return require_effects.runToQuiescence(live, {
1040
+ ...options.input !== void 0 ? { input: options.input } : {},
1041
+ ...event !== void 0 ? { event } : {}
1042
+ });
1043
+ };
1044
+ const soonestTimer = () => {
1045
+ let soonest;
1046
+ for (const timer of pendingTimers) if (soonest === void 0 || timer[1] < soonest[1]) soonest = timer;
1047
+ if (soonest === void 0) return void 0;
1048
+ pendingTimers.delete(soonest[0]);
1049
+ world.clock.now = soonest[1];
1050
+ return {
1051
+ type: "xstate.timer",
1052
+ id: soonest[0]
1053
+ };
1054
+ };
1055
+ let result = await turn();
1056
+ while (result.status === "idle" && !world.crashed) {
1057
+ let event;
1058
+ let host = true;
1059
+ let marked = false;
1060
+ if (next < events.length) {
1061
+ event = events[next++];
1062
+ if (event.type === "xstate.timer" && !("id" in event)) {
1063
+ event = soonestTimer();
1064
+ host = false;
1065
+ marked = true;
1066
+ if (event === void 0) {
1067
+ world.hostEntries.push(SKIPPED_TURN);
1068
+ continue;
1069
+ }
1070
+ }
1071
+ } else {
1072
+ event = soonestTimer();
1073
+ host = false;
1074
+ }
1075
+ if (event === void 0) break;
1076
+ result = await turn(event, host ? "host" : marked ? "timer" : void 0);
1077
+ await result.drain?.();
1035
1078
  }
1036
- return summarizeRepeats(groups);
1079
+ await result.drain?.();
1080
+ world.unsent = [...unsent];
1081
+ world.unsentAfterCrash = resuming !== void 0 ? [...unsent, ...sendAt.keys()] : [];
1082
+ world.undelivered = events.length - next;
1083
+ return result;
1037
1084
  }
1038
- function summarize$1(group) {
1039
- const passes = group.runs.filter((run) => run.passed).length;
1040
- const traces = group.runs.flatMap((run) => run.trace ? [run.trace] : []);
1041
- const paths = new Set(traces.map((trace) => traceSteps(trace).filter((step) => !step.ignored).map((step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`).join(" ")));
1042
- const reference = traces.find((trace) => trace.some((event) => event.type === "machine.transition"));
1043
- let earliest;
1044
- for (const trace of reference ? traces : []) {
1045
- if (trace === reference) continue;
1046
- const divergence = firstDivergence(trace, reference);
1047
- if (divergence && (!earliest || divergence.index < earliest.index)) earliest = divergence;
1048
- }
1085
+ function newWorld(crashesAt, recorded) {
1049
1086
  return {
1050
- ...group,
1051
- passes,
1052
- passAny: passes > 0,
1053
- passAll: group.runs.length > 0 && passes === group.runs.length,
1054
- distinctPaths: paths.size,
1055
- ...earliest ? { divergedAt: earliest.state } : {}
1087
+ effects: [],
1088
+ hostEntries: [],
1089
+ sentEntries: /* @__PURE__ */ new Map(),
1090
+ unsent: [],
1091
+ unsentAfterCrash: [],
1092
+ undelivered: 0,
1093
+ clock: {
1094
+ now: 0,
1095
+ due: /* @__PURE__ */ new Map()
1096
+ },
1097
+ answers: /* @__PURE__ */ new Map(),
1098
+ ...recorded !== void 0 ? { recorded } : {},
1099
+ crashed: false,
1100
+ onEffect: (effect) => crashesAt(effect)
1056
1101
  };
1057
1102
  }
1058
- /** C(passes, j) / C(runs, j), as a product so large counts don't overflow. */
1059
- function allPass(passes, runs, j) {
1060
- let chance = 1;
1061
- for (let i = 0; i < j; i++) chance *= Math.max(0, passes - i) / (runs - i);
1062
- return chance;
1063
- }
1064
- //#endregion
1065
- //#region src/judge-agreement.ts
1066
1103
  /**
1067
- * Measures how well a judge agrees with people on the same cases: accuracy,
1068
- * per-label recall, the confusion matrix, and the rows that disagree. With
1069
- * `positive`, also the TPR and TNR, the recommended view for binary judges
1070
- * (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
1071
- * Pure.
1072
- *
1073
- * Build rows from confirmed labels only; draft labels gate nothing. Group by
1074
- * decision state or request name so each narrow judge is aligned on its own.
1075
- * A row with no group key counts in the totals and in no group. Run it on a
1076
- * dev split and report the held-out split separately.
1077
- *
1078
- * Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
1079
- *
1080
- * @example
1081
- * ```ts no-check
1082
- * const report = judgeAgreement(
1083
- * labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
1084
- * { positive: 'fail' },
1085
- * );
1086
- * report.binary; // { tpr: 0.9, tnr: 0.8 }
1087
- * report.correct?.(0.3); // estimated true failure rate in production
1088
- * ```
1104
+ * `store`, recording each append's resulting length and marking the effects
1105
+ * whose completion an accepted entry carries. `afterAppend` returns true to
1106
+ * freeze it: every later append rejects, as if the process died.
1089
1107
  */
1090
- function judgeAgreement(rows, options = {}) {
1091
- if (rows.length === 0) throw new require_event_log.AgentError("empty-agreement-rows", "judgeAgreement: rows must contain at least one row.");
1092
- const report = summarize(rows, options.positive);
1093
- const groupOf = options.groupBy ?? ((row) => row.group);
1094
- if (options.groupBy || rows.some((row) => row.group !== void 0)) {
1095
- const grouped = /* @__PURE__ */ new Map();
1096
- for (const row of rows) {
1097
- const key = groupOf(row);
1098
- if (key === void 0) continue;
1099
- const members = grouped.get(key);
1100
- if (members) members.push(row);
1101
- else grouped.set(key, [row]);
1108
+ function observedStore(store, world, afterAppend) {
1109
+ const appends = [];
1110
+ let frozen = false;
1111
+ return {
1112
+ store: {
1113
+ ...store,
1114
+ read: (threadId, options) => store.read(threadId, options),
1115
+ length: (threadId) => store.length(threadId),
1116
+ fork: (input) => store.fork(input),
1117
+ async append(input) {
1118
+ if (frozen || world.crashed) throw new CrashError();
1119
+ await store.append(input);
1120
+ for (const entry of input.entries) markCompleted(entry, world.effects);
1121
+ const accepted = input.expectedIndex + input.entries.length;
1122
+ appends.push(accepted);
1123
+ if (afterAppend(accepted)) frozen = true;
1124
+ }
1125
+ },
1126
+ appends,
1127
+ freeze: () => {
1128
+ frozen = true;
1102
1129
  }
1103
- const groups = Object.create(null);
1104
- for (const [key, members] of grouped) groups[key] = summarize(members, options.positive);
1105
- report.groups = groups;
1130
+ };
1131
+ }
1132
+ /**
1133
+ * Marks the call an accepted entry completes: a decision's chosen event names
1134
+ * its call (`site#n`); a completion completes the oldest open effect at its
1135
+ * invoke site, with the other attempts of the same model call.
1136
+ */
1137
+ function markCompleted(entry, effects) {
1138
+ if (entry.decidedBy !== void 0) {
1139
+ const call = entry.decidedBy;
1140
+ for (const effect of effects) if (effect.key === call || effect.key.startsWith(`${call}.`)) effect.completed = true;
1141
+ return;
1106
1142
  }
1107
- return report;
1143
+ if (entry.event.type !== DONE_ACTOR && entry.event.type !== ERROR_ACTOR) return;
1144
+ const actorId = entry.event.actorId;
1145
+ if (typeof actorId !== "string") return;
1146
+ const site = require_event_log.agentCallSite(entry.actorPath ?? [], actorId);
1147
+ const oldest = effects.find((effect) => effect.site === site && !effect.completed);
1148
+ if (oldest === void 0) return;
1149
+ const call = oldest.kind === "model" ? callOfModelKey(oldest.key) : void 0;
1150
+ for (const effect of effects) if (effect === oldest || call !== void 0 && effect.site === site && callOfModelKey(effect.key) === call) effect.completed = true;
1151
+ }
1152
+ function countBy(effects) {
1153
+ const counts = /* @__PURE__ */ new Map();
1154
+ for (const { key } of effects) counts.set(key, (counts.get(key) ?? 0) + 1);
1155
+ return counts;
1156
+ }
1157
+ function outcomeOf(result) {
1158
+ const snapshot = result.snapshot;
1159
+ return stableStringify(require_effects.toJsonValue({
1160
+ status: result.status,
1161
+ ...result.status === "done" ? { output: result.output } : {},
1162
+ ...result.status === "error" ? { cause: result.cause } : {},
1163
+ value: snapshot.value,
1164
+ context: snapshot.context
1165
+ }));
1108
1166
  }
1109
- function summarize(rows, positive) {
1110
- const perLabel = Object.create(null);
1111
- const confusion = Object.create(null);
1112
- const disagreements = [];
1113
- let agreed = 0;
1114
- for (const { id, human, judge } of rows) {
1115
- const label = perLabel[human] ??= {
1116
- human: 0,
1117
- agreed: 0
1167
+ async function withTimeout(promise, ms) {
1168
+ let timer;
1169
+ try {
1170
+ return await Promise.race([promise, new Promise((resolve) => {
1171
+ timer = setTimeout(() => resolve(TIMED_OUT), ms);
1172
+ })]);
1173
+ } finally {
1174
+ clearTimeout(timer);
1175
+ }
1176
+ }
1177
+ /** `callKey` without its execution id: the same call has the same key in every run. */
1178
+ function modelKey(callKey) {
1179
+ return callKey.slice(callKey.indexOf(":") + 1);
1180
+ }
1181
+ function siteOfModelKey(key) {
1182
+ const hash = key.lastIndexOf("#");
1183
+ return hash === -1 ? key : key.slice(0, hash);
1184
+ }
1185
+ /** `site#n` of a model key: the call, without its attempt or strategy suffix. */
1186
+ function callOfModelKey(key) {
1187
+ const hash = key.lastIndexOf("#");
1188
+ if (hash === -1) return key;
1189
+ const dot = key.indexOf(".", hash);
1190
+ return dot === -1 ? key : key.slice(0, dot);
1191
+ }
1192
+ function trackExecutors(executors, world) {
1193
+ const tracked = {};
1194
+ for (const [slot, executor] of Object.entries(executors)) {
1195
+ if (typeof executor !== "function") continue;
1196
+ tracked[slot] = async (request, info) => {
1197
+ if (world.crashed) return never(info?.signal);
1198
+ const key = info?.callKey !== void 0 ? modelKey(info.callKey) : `${slot}:?`;
1199
+ const effect = {
1200
+ kind: "model",
1201
+ key,
1202
+ site: siteOfModelKey(key),
1203
+ completed: false,
1204
+ request: requestFingerprint(request)
1205
+ };
1206
+ world.effects.push(effect);
1207
+ const dies = world.onEffect(effect);
1208
+ const recorded = world.recorded?.get(key);
1209
+ let result;
1210
+ if (recorded !== void 0) {
1211
+ await new Promise((resolve) => setTimeout(resolve, recorded.ms));
1212
+ result = recorded.result;
1213
+ } else {
1214
+ const started = Date.now();
1215
+ result = await executor(request, info);
1216
+ if (info?.callKey !== void 0) world.answers.set(key, {
1217
+ result,
1218
+ ms: Date.now() - started
1219
+ });
1220
+ }
1221
+ if (dies || world.crashed) return never(info?.signal);
1222
+ return result;
1118
1223
  };
1119
- const row = confusion[human] ??= Object.create(null);
1120
- label.human++;
1121
- row[judge] = (row[judge] ?? 0) + 1;
1122
- if (human === judge) {
1123
- label.agreed++;
1124
- agreed++;
1125
- } else disagreements.push({
1126
- id,
1127
- human,
1128
- judge
1129
- });
1130
1224
  }
1131
- const summary = {
1132
- total: rows.length,
1133
- accuracy: agreed / rows.length,
1134
- perLabel,
1135
- confusion,
1136
- disagreements
1137
- };
1138
- if (positive === void 0) return summary;
1139
- let positives = 0;
1140
- let truePositives = 0;
1141
- let negatives = 0;
1142
- let trueNegatives = 0;
1143
- for (const { human, judge } of rows) if (human === positive) {
1144
- positives++;
1145
- if (judge === positive) truePositives++;
1146
- } else {
1147
- negatives++;
1148
- if (judge !== positive) trueNegatives++;
1225
+ return tracked;
1226
+ }
1227
+ const VOLATILE_REQUEST_KEYS = /* @__PURE__ */ new Set(["runId"]);
1228
+ function requestFingerprint(request) {
1229
+ try {
1230
+ const json = require_effects.toJsonValue(request);
1231
+ if (json && typeof json === "object" && !Array.isArray(json)) for (const key of VOLATILE_REQUEST_KEYS) delete json[key];
1232
+ return stableStringify(json);
1233
+ } catch {
1234
+ return;
1149
1235
  }
1150
- const tpr = positives === 0 ? void 0 : truePositives / positives;
1151
- const tnr = negatives === 0 ? void 0 : trueNegatives / negatives;
1152
- summary.binary = {
1153
- tpr,
1154
- tnr
1155
- };
1156
- if (tpr !== void 0 && tnr !== void 0 && tpr + tnr > 1) summary.correct = (p) => Math.min(1, Math.max(0, (p + tnr - 1) / (tpr + tnr - 1)));
1157
- return summary;
1158
1236
  }
1159
- //#endregion
1160
- //#region src/compare-runs.ts
1237
+ function never(signal) {
1238
+ return new Promise((_, reject) => {
1239
+ if (signal?.aborted) reject(signal.reason);
1240
+ signal?.addEventListener("abort", () => reject(signal.reason), { once: true });
1241
+ });
1242
+ }
1243
+ /** `machine` with every plain actor source, at any depth, tracked. */
1244
+ function trackActors(machine, world, ancestors = /* @__PURE__ */ new Set()) {
1245
+ if (ancestors.has(machine)) return machine;
1246
+ const visited = /* @__PURE__ */ new Set([...ancestors, machine]);
1247
+ const sources = machine.sources?.actors ?? {};
1248
+ const wrapped = {};
1249
+ for (const [key, logic] of Object.entries(sources)) {
1250
+ if (require_event_log.isTextLogic(logic) || require_decision.isDecisionLogic(logic) || require_act.isActLogic(logic)) continue;
1251
+ if (require_effects.isStateMachineLogic(logic)) {
1252
+ const inner = trackActors(logic, world, visited);
1253
+ if (inner !== logic) wrapped[key] = inner;
1254
+ continue;
1255
+ }
1256
+ if (typeof logic?.transition === "function") wrapped[key] = trackLogic(logic, world);
1257
+ }
1258
+ return Object.keys(wrapped).length > 0 ? machine.provide({ actors: wrapped }) : machine;
1259
+ }
1260
+ /** The invoke site of an actor: its path's last id, under its parents' path. */
1261
+ function actorSite(self) {
1262
+ const path = [];
1263
+ for (let at = self; at?._parent !== void 0; at = at._parent) path.unshift(String(at.id));
1264
+ return require_event_log.agentCallSite(path.slice(0, -1), path.at(-1) ?? "");
1265
+ }
1161
1266
  /**
1162
- * Runs every scenario through a baseline and a candidate implementation and
1163
- * labels each difference, so a port can show where it behaves like the code it
1164
- * replaces and where it differs on purpose.
1165
- *
1166
- * Parity is outcome-level: the terminal outcome and the effects in the world,
1167
- * not the sequence of tool calls, since a machine usually reorders steps. A
1168
- * candidate that broke no rule where the baseline broke one is intended, even
1169
- * when the outcome matches. A difference also counts as intended when only the
1170
- * candidate reached the scenario's `expected` outcome. Every other difference,
1171
- * and every candidate rule break, is a regression for a person to review.
1172
- *
1173
- * Runs are sequential, baseline first. A run that throws is recorded with
1174
- * outcome `'threw'` and the error message as its `detail`. Parity holds for
1175
- * these scenarios and these runs only; with a live model, run it more than once.
1176
- *
1177
- * @example
1178
- * ```ts no-check
1179
- * const report = await compareRuns({
1180
- * scenarios,
1181
- * baseline: async (s) => toParityRun(await runRawLoop(s)),
1182
- * candidate: async (s) => toParityRun(await runDesk(s)),
1183
- * });
1184
- * expect(report.ok).toBe(true);
1185
- * ```
1267
+ * `logic`, recording each effect it starts (`@xstate.effect`: an async
1268
+ * actor's run, a callback's body). After the crash, effects do not start.
1186
1269
  */
1187
- async function compareRuns(options) {
1188
- const scenarios = [];
1189
- const counts = {
1190
- same: 0,
1191
- intended: 0,
1192
- regression: 0
1193
- };
1194
- for (const scenario of options.scenarios) {
1195
- const baseline = await settle$1(() => options.baseline(scenario));
1196
- const candidate = await settle$1(() => options.candidate(scenario));
1197
- const { label, reason } = labelRun(scenario.expected, baseline, candidate);
1198
- counts[label]++;
1199
- scenarios.push({
1200
- id: scenario.id,
1201
- scenario,
1202
- label,
1203
- reason,
1204
- baseline,
1205
- candidate
1206
- });
1207
- }
1208
- return {
1209
- scenarios,
1210
- counts,
1211
- ok: counts.regression === 0
1270
+ function trackLogic(logic, world) {
1271
+ if (require_event_log.isTextLogic(logic) || require_decision.isDecisionLogic(logic) || require_act.isActLogic(logic)) return logic;
1272
+ const track = (result, self) => {
1273
+ const [snapshot, effects] = result;
1274
+ if (!Array.isArray(effects) || !effects.some((e) => e?.type === "@xstate.effect")) return result;
1275
+ return [snapshot, effects.map((effect) => {
1276
+ if (effect?.type !== "@xstate.effect") return effect;
1277
+ const exec = effect.exec;
1278
+ const tracked = (...args) => {
1279
+ if (world.crashed) return;
1280
+ const site = actorSite(self);
1281
+ const started = {
1282
+ kind: "actor",
1283
+ key: `actor:${site}`,
1284
+ site,
1285
+ completed: false
1286
+ };
1287
+ world.effects.push(started);
1288
+ world.onEffect(started);
1289
+ return exec(...args);
1290
+ };
1291
+ return {
1292
+ ...effect,
1293
+ exec: tracked,
1294
+ action: tracked
1295
+ };
1296
+ })];
1212
1297
  };
1298
+ const overrides = { transition: { value: (...args) => track(logic.transition(...args), args[2].self) } };
1299
+ if (typeof logic.initialTransition === "function") overrides["initialTransition"] = { value: (...args) => track(logic.initialTransition(...args), args[1].self) };
1300
+ if (typeof logic.start === "function") overrides["start"] = { value: (snapshot, scope, startOptions) => logic.start(snapshot, new Proxy(scope, { get(target, property) {
1301
+ if (property === "actionExecutor") return (effect) => {
1302
+ const [, [tracked]] = track([void 0, [effect]], target.self);
1303
+ target.actionExecutor(tracked);
1304
+ };
1305
+ return Reflect.get(target, property, target);
1306
+ } }), startOptions) };
1307
+ return Object.create(logic, overrides);
1213
1308
  }
1214
- async function settle$1(run) {
1215
- try {
1216
- return await run();
1217
- } catch (error) {
1218
- return {
1219
- outcome: "threw",
1220
- detail: error instanceof Error ? error.message : String(error)
1309
+ //#endregion
1310
+ //#region src/trajectory.ts
1311
+ /**
1312
+ * Classifies one item. Event-shaped things (log entries, events) become events;
1313
+ * everything else is a state value. A snapshot is unwrapped to its `value`.
1314
+ * @internal
1315
+ */
1316
+ function normalize(item) {
1317
+ if (require_event_log.isRecord(item)) {
1318
+ const nested = item["event"];
1319
+ if (require_event_log.isRecord(nested) && typeof nested["type"] === "string") return {
1320
+ kind: "event",
1321
+ event: nested
1322
+ };
1323
+ if (typeof item["type"] === "string") return {
1324
+ kind: "event",
1325
+ event: item
1326
+ };
1327
+ if ("value" in item) return {
1328
+ kind: "state",
1329
+ value: item["value"]
1221
1330
  };
1222
1331
  }
1223
- }
1224
- function labelRun(expected, baseline, candidate) {
1225
- if (candidate.violations?.length) return {
1226
- label: "regression",
1227
- reason: `candidate broke a rule: ${list(candidate.violations)}`
1228
- };
1229
- if (candidate.outcome === "threw") return {
1230
- label: "regression",
1231
- reason: `candidate threw: ${String(candidate.detail)}`
1232
- };
1233
- if (baseline.violations?.length) return {
1234
- label: "intended",
1235
- reason: `baseline broke a rule: ${list(baseline.violations)}`
1236
- };
1237
- const sameOutcome = baseline.outcome === candidate.outcome;
1238
- if (sameOutcome && structurallyEqual(baseline.effects, candidate.effects)) return {
1239
- label: "same",
1240
- reason: ""
1241
- };
1242
- if (expected !== void 0 && candidate.outcome === expected && baseline.outcome !== expected) return {
1243
- label: "intended",
1244
- reason: `only the candidate reached the expected '${expected}'`
1245
- };
1246
1332
  return {
1247
- label: "regression",
1248
- reason: sameOutcome ? "effects differ" : `outcome differs: ${baseline.outcome} → ${candidate.outcome}`
1333
+ kind: "state",
1334
+ value: item
1249
1335
  };
1250
1336
  }
1251
- function list(items) {
1252
- return items.map((item) => typeof item === "string" ? item : JSON.stringify(item)).join(", ");
1337
+ /** Every leaf of a state value as a dot path (`{ review: 'editing' }` → `review.editing`). @internal */
1338
+ function statePaths$1(value) {
1339
+ if (typeof value === "string") return [value];
1340
+ if (!require_event_log.isRecord(value)) return [];
1341
+ return Object.entries(value).flatMap(([key, child]) => statePaths$1(child).map((path) => `${key}.${path}`));
1253
1342
  }
1254
- function structurallyEqual(a, b) {
1343
+ /**
1344
+ * A string expectation matches a state value when it names one of its leaves or
1345
+ * an ancestor of one, so `'review'` matches `{ review: 'editing' }`. @internal
1346
+ */
1347
+ function stateMatchesPath(path, value) {
1348
+ return statePaths$1(value).some((leaf) => leaf === path || leaf.startsWith(`${path}.`));
1349
+ }
1350
+ function deepEqual(a, b) {
1255
1351
  if (Object.is(a, b)) return true;
1256
- if (a instanceof Date || b instanceof Date) return a instanceof Date && b instanceof Date && a.getTime() === b.getTime();
1257
- if (a instanceof Map || b instanceof Map) return a instanceof Map && b instanceof Map && structurallyEqual([...a], [...b]);
1258
- if (a instanceof Set || b instanceof Set) return a instanceof Set && b instanceof Set && structurallyEqual([...a], [...b]);
1259
- if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => structurallyEqual(item, b[index]));
1352
+ if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => deepEqual(item, b[index]));
1260
1353
  if (!require_event_log.isRecord(a) || !require_event_log.isRecord(b)) return false;
1261
1354
  const keys = Object.keys(a);
1262
- return keys.length === Object.keys(b).length && keys.every((key) => Object.hasOwn(b, key) && structurallyEqual(a[key], b[key]));
1355
+ return keys.length === Object.keys(b).length && keys.every((key) => deepEqual(a[key], b[key]));
1263
1356
  }
1264
- //#endregion
1265
- //#region src/equivalence.ts
1266
1357
  /**
1267
- * Verified equivalence — prove a port behaves like the code it replaces.
1358
+ * Normalizes ONE expected item into a predicate over actual items, so the
1359
+ * expectation is classified once instead of once per candidate:
1268
1360
  *
1269
- * A port is equivalent when, given the same model answers and the same tool
1270
- * results, it produces the same OBSERVABLE EFFECTS: the same number of model
1271
- * calls, the same tool calls with the same arguments, and the same output.
1272
- * Prompts, internal steps, and states are free to differ.
1361
+ * - A string expectation matches an event's `type`, or a state value it names.
1362
+ * - An event expectation matches an event with the same `type` whose other
1363
+ * declared keys deep-equal, so payload details are opt-in.
1364
+ * - A state-value expectation matches structurally.
1365
+ * @internal
1366
+ */
1367
+ function matcherFor(expected) {
1368
+ if (typeof expected === "string") return (actual) => {
1369
+ const actualItem = normalize(actual);
1370
+ return actualItem.kind === "event" ? actualItem.event.type === expected : stateMatchesPath(expected, actualItem.value);
1371
+ };
1372
+ const expectedItem = normalize(expected);
1373
+ switch (expectedItem.kind) {
1374
+ case "event": return (actual) => {
1375
+ const actualItem = normalize(actual);
1376
+ return actualItem.kind === "event" && Object.keys(expectedItem.event).every((key) => deepEqual(expectedItem.event[key], actualItem.event[key]));
1377
+ };
1378
+ case "state": return (actual) => {
1379
+ const actualItem = normalize(actual);
1380
+ return actualItem.kind === "state" && deepEqual(expectedItem.value, actualItem.value);
1381
+ };
1382
+ }
1383
+ }
1384
+ /**
1385
+ * Matches a run's trajectory against an expected one.
1273
1386
  *
1274
- * 1. Record the original: wrap its model (`recorder.model`, an AI SDK
1275
- * `LanguageModelV3`, or `recorder.openai`, an `openai` client) and its tools
1276
- * (`recorder.tool` / `recorder.tools`), run a scenario, call
1277
- * `recorder.output(result)`, and save `recorder.recording`.
1278
- * 2. Replay it on the port: `createRecorder({ replay: recording })` answers
1279
- * every model call and tool call from the recording, with no API key and no
1280
- * side effects, and records what the port did.
1281
- * 3. `compareRecordings(original, port)` reports the first effect that differs.
1387
+ * Both trajectories may be state values or events collected from
1388
+ * `onTransition`;
1389
+ * items are compared by shape, so `['prompting', 'drafting']` scores a state
1390
+ * path and `['PROMPT_SUBMITTED', 'SEND']` scores an event log with the same
1391
+ * call.
1282
1392
  *
1283
- * A port that changes how it calls the model on purpose (a model-picked action
1284
- * that became `agent.decide`, a stream that became `doGenerate`) still replays:
1285
- * `translate` turns the recorded answer into one that fits, `fallback` answers
1286
- * the calls the original never made, and `compareRecordings(…, { kinds })`
1287
- * compares only tool calls and output.
1393
+ * Semantics are an ordered subsequence: every expected item must appear in
1394
+ * `actual`, in order, with gaps allowed. Pass `{ exact: true }` to require
1395
+ * equality instead.
1288
1396
  *
1289
- * Any other client (`@anthropic-ai/sdk`, `@google/genai`, a framework's own
1290
- * model client) records through `recorder.client` or `recorder.fn`: each call
1291
- * is a `call` effect, answered in replay by name and order.
1397
+ * @example Trajectory assertions in a test
1398
+ * ```ts
1399
+ * const statePath: unknown[] = [];
1400
+ * const result = await runToQuiescence(
1401
+ * createAgentRuntime(machine, {
1402
+ * executors,
1403
+ * onTransition: (snapshot) => statePath.push(snapshot.value),
1404
+ * }),
1405
+ * { input },
1406
+ * );
1292
1407
  *
1293
- * Provider-free: every seam is wrapped structurally, so this module imports
1294
- * neither `ai` nor `openai`.
1295
- * @module
1408
+ * const path = matchesTrajectory(statePath, ['prompting', 'drafting', 'sent']);
1409
+ * expect(path.matched, JSON.stringify(path.firstMiss)).toBe(true);
1410
+ * expect(matchesTrajectory(events, ['PROMPT_SUBMITTED', 'SEND']).matched).toBe(true);
1411
+ * ```
1412
+ *
1413
+ * @example Partial credit in an eval scorer
1414
+ * ```ts
1415
+ * const match = matchesTrajectory(output.statePath, expected.statePath);
1416
+ * return { name: 'state_path', score: match.score, metadata: { ...match.firstMiss } };
1417
+ * ```
1296
1418
  */
1419
+ function matchesTrajectory(actual, expected, options = {}) {
1420
+ const expectedCount = expected.length;
1421
+ if (expectedCount === 0) throw new require_event_log.AgentError("empty-trajectory", "matchesTrajectory: expected trajectory must contain at least one item.");
1422
+ const exact = options.exact === true;
1423
+ let cursor = 0;
1424
+ let matchedCount = 0;
1425
+ let firstMiss;
1426
+ for (let index = 0; index < expectedCount; index++) {
1427
+ const want = expected[index];
1428
+ const matches = matcherFor(want);
1429
+ const searchedFrom = exact ? index : cursor;
1430
+ let found = -1;
1431
+ for (let at = searchedFrom; at < actual.length; at++) {
1432
+ if (matches(actual[at])) {
1433
+ found = at;
1434
+ break;
1435
+ }
1436
+ if (exact) break;
1437
+ }
1438
+ if (found === -1) {
1439
+ firstMiss = {
1440
+ index,
1441
+ expected: want,
1442
+ searchedFrom
1443
+ };
1444
+ break;
1445
+ }
1446
+ matchedCount++;
1447
+ cursor = found + 1;
1448
+ }
1449
+ const matched = !firstMiss && (!exact || actual.length === expectedCount);
1450
+ const denominator = exact ? Math.max(expectedCount, actual.length) : expectedCount;
1451
+ return {
1452
+ matched,
1453
+ matchedCount,
1454
+ expectedCount,
1455
+ score: matched ? 1 : denominator === 0 ? 1 : matchedCount / denominator,
1456
+ ...firstMiss ? { firstMiss } : {}
1457
+ };
1458
+ }
1459
+ //#endregion
1460
+ //#region src/seam.ts
1461
+ /** The only own keys an executor result carries. @internal */
1462
+ const TEXT_RESULT_KEYS = /* @__PURE__ */ new Set([
1463
+ "result",
1464
+ "messages",
1465
+ "usage",
1466
+ "raw"
1467
+ ]);
1297
1468
  /**
1298
- * Thrown in replay when the port asks for something the recording cannot
1299
- * answer: a model call past the last recorded one, or a tool call the original
1300
- * never made. The effect is still recorded, so {@link compareRecordings}
1301
- * reports it.
1469
+ * True when a scripted entry is the executor result rather than the output
1470
+ * value: it owns a `result` key and owns NOTHING outside the result's own
1471
+ * vocabulary. Bare `'result' in value` would swallow an output object's
1472
+ * siblings (`{ result: 'draft', confidence: 0.9 }` would lose `confidence`)
1473
+ * and would also match an inherited `result`. @internal
1302
1474
  */
1303
- var RecordingDivergenceError = class extends Error {
1304
- name = "RecordingDivergenceError";
1305
- };
1306
- function toRecordedError(error) {
1307
- if (!(error instanceof Error)) return {
1308
- name: "Error",
1309
- message: String(error)
1310
- };
1311
- const recorded = {
1312
- name: error.name,
1313
- message: error.message
1314
- };
1315
- if (Object.prototype.propertyIsEnumerable.call(error, "name")) recorded.ownName = true;
1316
- const code = error.code;
1317
- if (code !== void 0) recorded.code = toJson(code);
1318
- const data = Object.fromEntries(Object.entries(error).filter(([key, value]) => key !== "code" && (typeof value === "number" || typeof value === "boolean")));
1319
- if (Object.keys(data).length) recorded.data = data;
1320
- const markers = /* @__PURE__ */ new Set();
1321
- for (let holder = error; holder; holder = Object.getPrototypeOf(holder)) for (const symbol of Object.getOwnPropertySymbols(holder)) {
1322
- const key = Symbol.keyFor(symbol);
1323
- if (key !== void 0 && error[symbol] === true) markers.add(key);
1324
- }
1325
- if (markers.size) recorded.markers = [...markers];
1326
- return recorded;
1327
- }
1328
- function fromRecordedError({ data, markers, name, ownName, ...fields }) {
1329
- const error = Object.assign(new Error(fields.message), fields, data);
1330
- if (ownName || name !== "Error") Object.defineProperty(error, "name", {
1331
- value: name,
1332
- writable: true,
1333
- configurable: true,
1334
- enumerable: ownName ?? false
1335
- });
1336
- for (const marker of markers ?? []) error[Symbol.for(marker)] = true;
1337
- return error;
1338
- }
1339
- /** What two runs compare of an error: its data and markers may differ. */
1340
- function errorIdentity$1(error) {
1341
- return error && {
1342
- name: error.name,
1343
- message: error.message,
1344
- code: error.code
1345
- };
1475
+ function isTextResult(value) {
1476
+ return Object.hasOwn(value, "result") && Object.keys(value).every((key) => TEXT_RESULT_KEYS.has(key));
1346
1477
  }
1347
- /** A JSON-safe copy: Dates become ISO strings, Errors become {@link RecordedError}. */
1348
- function toJson(value) {
1349
- if (value === void 0) return void 0;
1350
- return JSON.parse(JSON.stringify(value, function(key, item) {
1351
- const raw = this[key];
1352
- if (raw instanceof Date) return raw.toISOString();
1353
- if (item instanceof Error) return toRecordedError(item);
1354
- if (typeof item === "bigint") return item.toString();
1355
- if (item instanceof Uint8Array) return Array.from(item);
1356
- return item;
1357
- }));
1478
+ /**
1479
+ * Resolves ONE scripted entry to an executor result: a function entry is
1480
+ * called with the request, and the value is taken as the executor result only
1481
+ * when it is one (see {@link isTextResult}). @internal
1482
+ */
1483
+ async function resolveSeamScriptEntry(entry, request, info) {
1484
+ const value = typeof entry === "function" ? await entry(request, info) : entry;
1485
+ return require_event_log.isRecord(value) && isTextResult(value) ? value : { result: value };
1358
1486
  }
1359
- /** Restores the `Date` timestamps the AI SDK puts on responses. */
1360
- function reviveTimestamps(value) {
1361
- if (value && typeof value === "object") {
1362
- for (const holder of [value, value.response]) if (holder && typeof holder["timestamp"] === "string") holder["timestamp"] = new Date(holder["timestamp"]);
1363
- }
1364
- return value;
1487
+ /** Names a pending text request in an error message. @internal */
1488
+ function describeText(request) {
1489
+ return request.name ? `'${request.name}' (model '${request.model}')` : `(model '${request.model}')`;
1365
1490
  }
1366
- function omit(value, keys) {
1367
- if (!value || typeof value !== "object") return value;
1368
- return Object.fromEntries(Object.entries(value).filter(([key]) => !keys.includes(key)));
1491
+ /**
1492
+ * Stream semantics with no model: the whole text lands as one chunk. @internal
1493
+ */
1494
+ function emitScriptedChunk(result, info) {
1495
+ const text = require_event_log.isRecord(result) ? result["result"] : void 0;
1496
+ if (typeof text === "string") info?.onChunk?.(text);
1369
1497
  }
1370
1498
  /**
1371
- * Key-order-independent JSON, for equality.
1499
+ * The seam's own answer, for scoring: the executor result's `result`.
1372
1500
  * @internal
1373
1501
  */
1374
- function stableStringify(value) {
1375
- if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
1376
- if (value && typeof value === "object") return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`;
1377
- return JSON.stringify(value) ?? "undefined";
1378
- }
1379
- function jsonEqual(a, b) {
1380
- return stableStringify(toJson(a)) === stableStringify(toJson(b));
1381
- }
1382
- function streamFrom(parts) {
1383
- return new ReadableStream({ start(controller) {
1384
- for (const part of parts) controller.enqueue(reviveTimestamps(structuredClone(part)));
1385
- controller.close();
1386
- } });
1387
- }
1388
- async function* iterate(chunks) {
1389
- for (const chunk of chunks) yield structuredClone(chunk);
1502
+ async function seamOutputOf(result, request) {
1503
+ if (!require_event_log.isRecord(result)) return;
1504
+ return await require_event_log.normalizeGeneratorResult(result, `seam '${request.name ?? request.model}'`);
1390
1505
  }
1391
1506
  /**
1392
- * Creates a {@link Recorder}. With no options it records: every wrapped call
1393
- * runs for real. With `replay` it answers from a recording instead.
1507
+ * Runs a machine end to end with one model call under test, and slices the run
1508
+ * at that call.
1509
+ *
1510
+ * The `before`/`after` slices are the point: `after.statePath` is the branch
1511
+ * the seam chose and `after.events` is the same question against the durable
1512
+ * log, both ready for `matchesTrajectory`. The state slice splits where the
1513
+ * state path stood when the seam answered; the event slice splits at the
1514
+ * seam's own effect completion (the first `xstate.done.*`/`xstate.error.*`
1515
+ * entry appended after the call was made).
1516
+ *
1517
+ * @example No API key: the seam is scripted too, so the whole thing runs offline.
1518
+ * ```ts
1519
+ * const run = await runSeam(emailDrafter, {
1520
+ * scripts: { writeFollowUps: [followUps], emailDrafter: [draft] },
1521
+ * seam: { request: 'writeFollowUps' },
1522
+ * respond: ({ state }) => (state === 'prompting' ? { type: 'PROMPT_SUBMITTED', prompt } : null),
1523
+ * });
1524
+ *
1525
+ * matchesTrajectory(run.after.statePath, ['needsMoreInfo', 'drafting']);
1526
+ * ```
1527
+ *
1528
+ * @example A candidate prompt at the seam, scored against the same rows.
1529
+ * ```ts
1530
+ * const { generateText } = createAiSdkExecutors({ models });
1531
+ * const run = await runSeam(emailDrafter, { ...row, candidate: generateText });
1532
+ * ```
1394
1533
  */
1395
- function createRecorder(options = {}) {
1396
- const effects = [];
1397
- const replay = options.replay?.effects;
1398
- const replayedModels = replay?.filter((effect) => effect.kind === "model");
1399
- const usedModels = /* @__PURE__ */ new Set();
1400
- const usedTools = /* @__PURE__ */ new Set();
1401
- const usedCalls = /* @__PURE__ */ new Set();
1402
- const callCounts = /* @__PURE__ */ new Map();
1403
- const normalizeEffect = (effect, list) => options.normalize?.(effect, list.indexOf(effect), list) ?? effect;
1404
- let modelCalls = 0;
1405
- let toolsInFlight = 0;
1406
- /**
1407
- * The earliest unused recorded call that fits this one: one `translate`
1408
- * adapts, or one with the same method and shape. Not by position, so a call
1409
- * the port added or dropped does not shift every answer after it.
1410
- */
1411
- function nextReplayedModel(effect) {
1412
- for (const recorded of replayedModels) {
1413
- if (usedModels.has(recorded)) continue;
1414
- const translated = options.translate?.(structuredClone(recorded), effect);
1415
- if (translated) {
1416
- effect.translated = true;
1417
- return {
1418
- recorded,
1419
- answer: translated
1420
- };
1421
- }
1422
- if (recorded.method === effect.method && modelCallsMatch(options.modelCalls, recorded, effect)) return {
1423
- recorded,
1424
- answer: recorded
1425
- };
1426
- }
1427
- }
1428
- function noAnswerMessage(effect) {
1429
- const unused = replayedModels.filter((recorded) => !usedModels.has(recorded));
1430
- return unused.length ? `Model call #${effect.call} (${effect.method}, ${JSON.stringify(modelCallShape(effect))}) fits no unused recorded call: ${unused.map((recorded) => `#${recorded.call} (${recorded.method}, ${JSON.stringify(modelCallShape(recorded))})`).join(", ")}. A recorded answer only fits a call of the same method and shape; pass \`translate\` to adapt one.` : `Model call #${effect.call} was never made by the recorded run (it made ${replayedModels.length}).`;
1431
- }
1534
+ async function runSeam(machine, options) {
1535
+ const { seam, candidate } = options;
1536
+ const queues = new Map(Object.entries(options.scripts ?? {}).map(([key, answers]) => [key, [...answers]]));
1537
+ const statePath = [];
1538
+ let calls = 0;
1539
+ let seamMatches = 0;
1540
+ let seamOutput;
1541
+ let seamUsage;
1542
+ const callLog = [];
1543
+ let seamReached = false;
1544
+ let callsBeforeSeam = -1;
1545
+ let seamStateAt = 0;
1546
+ let seamEventAt = 0;
1547
+ let liveEvents = 0;
1548
+ const queueKeyOf = (request) => request.name !== void 0 && queues.has(request.name) ? request.name : request.model;
1432
1549
  /**
1433
- * Records a model call around `run`, or answers it from the recording.
1434
- * `fallback` runs it on the stand-in when the recording has no answer.
1550
+ * Consumes this request's slot in the call plan, or resolves `undefined` when
1551
+ * its queue is dry. With `repeatLast`, the last entry is replayed instead of
1552
+ * running dry.
1435
1553
  */
1436
- async function modelCall(seam, method, request, streamed, run, fallback) {
1437
- const effect = {
1438
- kind: "model",
1439
- seam,
1440
- method,
1441
- call: modelCalls++,
1442
- request: toJson(omit(request, [
1443
- "abortSignal",
1444
- "headers",
1445
- "signal"
1446
- ]))
1554
+ const takeScriptedSlot = async (request, info) => {
1555
+ const queue = queues.get(queueKeyOf(request));
1556
+ if (!queue?.length) return;
1557
+ return resolveSeamScriptEntry(options.repeatLast && queue.length === 1 ? queue[0] : queue.shift(), request, info);
1558
+ };
1559
+ const scriptedAnswer = async (request, info) => {
1560
+ const scripted = await takeScriptedSlot(request, info);
1561
+ if (!scripted) throw new require_event_log.AgentError("seam-script-exhausted", `runSeam: no scripted answer left for request ${describeText(request)}. Add an entry to \`scripts.${queueKeyOf(request)}\`, or pass \`repeatLast: true\` to replay its last entry down a longer branch.`);
1562
+ return scripted;
1563
+ };
1564
+ const route = async (request, info) => {
1565
+ const callIndex = calls++;
1566
+ const isSeam = request.name === seam.request && seamMatches++ === (seam.occurrence ?? 0);
1567
+ const ledger = (source) => {
1568
+ callLog.push({
1569
+ ...request.name !== void 0 ? { name: request.name } : {},
1570
+ model: request.model,
1571
+ key: queueKeyOf(request),
1572
+ source,
1573
+ seam: isSeam
1574
+ });
1447
1575
  };
1448
- effects.push(effect);
1449
- if (replay) {
1450
- const replayed = nextReplayedModel(effect);
1451
- if (replayed) {
1452
- const { recorded, answer } = replayed;
1453
- usedModels.add(recorded);
1454
- effect.replayed = recorded.call;
1455
- if (answer.error) {
1456
- effect.error = answer.error;
1457
- throw fromRecordedError(answer.error);
1458
- }
1459
- effect.response = answer.response;
1460
- effect.stream = answer.stream;
1461
- if (seam === "openai") return streamed ? iterate(answer.stream ?? []) : structuredClone(answer.response);
1462
- return streamed ? {
1463
- ...reviveTimestamps(structuredClone(answer.response) ?? {}),
1464
- stream: streamFrom(answer.stream ?? [])
1465
- } : reviveTimestamps(structuredClone(answer.response));
1466
- }
1467
- if (!fallback) {
1468
- const error = new RecordingDivergenceError(noAnswerMessage(effect));
1469
- effect.error = toRecordedError(error);
1470
- throw error;
1471
- }
1472
- effect.unrecorded = true;
1473
- run = fallback;
1474
- }
1475
- let result;
1476
- try {
1477
- result = await run();
1478
- } catch (error) {
1479
- effect.error = toRecordedError(error);
1480
- throw error;
1481
- }
1482
- if (!streamed) {
1483
- effect.response = toJson(result);
1576
+ if (isSeam && candidate) {
1577
+ await takeScriptedSlot(request, info);
1578
+ seamReached = true;
1579
+ callsBeforeSeam = callIndex;
1580
+ seamStateAt = statePath.length;
1581
+ seamEventAt = liveEvents;
1582
+ const result = await candidate(request, info);
1583
+ seamOutput = await seamOutputOf(result, request);
1584
+ seamUsage = require_event_log.getCallUsage(result);
1585
+ ledger("candidate");
1484
1586
  return result;
1485
1587
  }
1486
- const parts = [];
1487
- effect.stream = parts;
1488
- if (seam === "openai") {
1489
- const source = result;
1490
- return (async function* () {
1491
- for await (const chunk of source) {
1492
- parts.push(toJson(chunk));
1493
- yield chunk;
1494
- }
1495
- })();
1588
+ const scripted = await scriptedAnswer(request, info);
1589
+ ledger("script");
1590
+ if (!isSeam) return scripted;
1591
+ seamReached = true;
1592
+ callsBeforeSeam = callIndex;
1593
+ seamStateAt = statePath.length;
1594
+ seamEventAt = liveEvents;
1595
+ seamOutput = scripted.result;
1596
+ seamUsage = scripted.usage;
1597
+ return scripted;
1598
+ };
1599
+ const executors = {
1600
+ ...options.executors,
1601
+ generateText: route,
1602
+ streamText: async (request, info) => {
1603
+ const result = await route(request, info);
1604
+ emitScriptedChunk(result, info);
1605
+ return result;
1496
1606
  }
1497
- const { stream, ...rest } = result;
1498
- effect.response = toJson(rest);
1499
- return {
1500
- ...rest,
1501
- stream: stream.pipeThrough(new TransformStream({ transform(part, controller) {
1502
- parts.push(toJson(part));
1503
- controller.enqueue(part);
1504
- } }))
1505
- };
1506
- }
1507
- /** `inputOf` picks the recorded input from the call's arguments. */
1508
- function tool(name, fn, inputOf = (args) => args.length === 1 ? args[0] : args) {
1509
- return ((...args) => {
1510
- const input = toJson(inputOf(args));
1511
- const effect = {
1512
- kind: "tool",
1513
- name,
1514
- step: modelCalls,
1515
- input,
1516
- ...toolsInFlight > 0 ? { concurrent: true } : {}
1517
- };
1518
- effects.push(effect);
1519
- return track(call(effect, args));
1607
+ };
1608
+ let snapshot;
1609
+ let event;
1610
+ const events = [];
1611
+ let result;
1612
+ const maxTurns = options.maxTurns ?? 12;
1613
+ for (let turn = 0; turn <= maxTurns; turn++) {
1614
+ result = await require_effects.runToQuiescence(require_effects.createAgentRuntime(machine, {
1615
+ ...options.actors ? { actors: options.actors } : {},
1616
+ executors,
1617
+ onTransition: (next, causedBy) => {
1618
+ if (turn > 0 && causedBy.type === "@xstate.init") return;
1619
+ events.push(causedBy);
1620
+ liveEvents = events.length;
1621
+ statePath.push(next.value);
1622
+ }
1623
+ }), {
1624
+ ...snapshot ? { snapshot } : { input: options.input },
1625
+ ...event ? { event } : {}
1520
1626
  });
1521
- function call(effect, args) {
1522
- const { input } = effect;
1523
- if (replay && options.tools !== "live") {
1524
- const wanted = normalizeEffect(effect, effects).input;
1525
- const matches = (candidate) => candidate.kind === "tool" && !usedTools.has(candidate) && candidate.name === name && jsonEqual(normalizeEffect(candidate, replay).input, wanted);
1526
- const recorded = replay.find(matches);
1527
- if (!recorded && options.fallback?.tools) {
1528
- effect.unrecorded = true;
1529
- return runLive(effect, args);
1530
- }
1531
- const settle = (value) => {
1532
- if (recorded?.sync) return value();
1533
- try {
1534
- return Promise.resolve(value());
1535
- } catch (error) {
1536
- return Promise.reject(error);
1537
- }
1538
- };
1539
- return settle(() => {
1540
- if (!recorded) {
1541
- const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)}; the recorded run made no such call.`);
1542
- effect.error = toRecordedError(error);
1543
- throw error;
1544
- }
1545
- usedTools.add(recorded);
1546
- if (recorded.sync) effect.sync = true;
1547
- if (recorded.error) {
1548
- effect.error = recorded.error;
1549
- throw fromRecordedError(recorded.error);
1550
- }
1551
- effect.output = recorded.output;
1552
- return structuredClone(recorded.output);
1553
- });
1627
+ if (result.status !== "idle" || turn === maxTurns) break;
1628
+ const next = await options.respond?.({
1629
+ snapshot: result.snapshot,
1630
+ state: result.snapshot.value,
1631
+ meta: require_event_log.getStateMeta(result.snapshot),
1632
+ turn,
1633
+ result
1634
+ });
1635
+ if (!next) break;
1636
+ event = next;
1637
+ snapshot = result.persist();
1638
+ }
1639
+ let splitAt = events.length;
1640
+ if (seamReached) {
1641
+ splitAt = seamEventAt;
1642
+ for (let index = seamEventAt; index < events.length; index++) {
1643
+ const type = events[index].type;
1644
+ if (type.startsWith("xstate.done") || type.startsWith("xstate.error")) {
1645
+ splitAt = index;
1646
+ break;
1554
1647
  }
1555
- return runLive(effect, args);
1556
1648
  }
1557
- function runLive(effect, args) {
1558
- let output;
1559
- try {
1560
- output = fn(...args);
1561
- } catch (error) {
1562
- effect.sync = true;
1563
- effect.error = toRecordedError(error);
1564
- throw error;
1565
- }
1566
- if (!(output instanceof Promise)) {
1567
- effect.sync = true;
1568
- effect.output = toJson(output);
1569
- return output;
1570
- }
1571
- return output.then((value) => {
1572
- effect.output = toJson(value);
1573
- return value;
1574
- }, (error) => {
1575
- effect.error = toRecordedError(error);
1576
- throw error;
1577
- });
1649
+ }
1650
+ const splitStateAt = seamReached ? seamStateAt : statePath.length;
1651
+ return {
1652
+ result,
1653
+ seamOutput,
1654
+ ...seamUsage !== void 0 ? { seamUsage } : {},
1655
+ callsBeforeSeam,
1656
+ calls: callLog,
1657
+ before: {
1658
+ statePath: statePath.slice(0, splitStateAt),
1659
+ events: events.slice(0, splitAt)
1660
+ },
1661
+ after: {
1662
+ statePath: statePath.slice(splitStateAt),
1663
+ events: events.slice(splitAt)
1664
+ }
1665
+ };
1666
+ }
1667
+ //#endregion
1668
+ //#region src/internal/trace-states.ts
1669
+ /** The `value` of a trace event's `from` or `snapshot`. @internal */
1670
+ function valueOf(holder) {
1671
+ return require_event_log.isRecord(holder) ? holder.value : void 0;
1672
+ }
1673
+ /** A decision request on its first attempt (no failed attempts yet). @internal */
1674
+ function isFirstDecision(request) {
1675
+ return require_event_log.isRecord(request) && request.kind === "decision" && (!Array.isArray(request.attempts) || request.attempts.length === 0);
1676
+ }
1677
+ /**
1678
+ * Every active leaf of a state value as a dot path:
1679
+ * `'a'` → `['a']`; `{ a: 'b' }` → `['a.b']`; `{ p: { x: 'one', y: {} } }` → `['p.x.one', 'p.y']`.
1680
+ * @internal
1681
+ */
1682
+ function statePaths(value, prefix = "") {
1683
+ if (typeof value === "string") return [prefix + value];
1684
+ if (!require_event_log.isRecord(value)) return [];
1685
+ const entries = Object.entries(value);
1686
+ if (entries.length === 0) return prefix ? [prefix.slice(0, -1)] : [];
1687
+ return entries.flatMap(([key, child]) => statePaths(child, `${prefix}${key}.`));
1688
+ }
1689
+ /**
1690
+ * A trace's `machine.transition` events as steps, skipping `@xstate.init`.
1691
+ * A first-attempt decision's `request.start` names the decision on the step
1692
+ * that follows it, skipping `@agent.usage` steps. Reads live and serialized traces alike.
1693
+ * @internal
1694
+ */
1695
+ function traceSteps(trace) {
1696
+ const steps = [];
1697
+ let decision;
1698
+ for (const event of trace) {
1699
+ if (event.type === "request.start" && isFirstDecision(event.request)) {
1700
+ const name = event.request.name;
1701
+ decision = typeof name === "string" ? name : void 0;
1702
+ continue;
1578
1703
  }
1704
+ if (event.type !== "machine.transition") continue;
1705
+ const type = require_event_log.isRecord(event.event) ? event.event.type : void 0;
1706
+ if (type === "@xstate.init") continue;
1707
+ const fromValue = valueOf(event.from);
1708
+ const toValue = valueOf(event.snapshot);
1709
+ steps.push({
1710
+ from: statePaths(fromValue),
1711
+ fromValue,
1712
+ event: typeof type === "string" ? type : "",
1713
+ payload: event.event,
1714
+ to: statePaths(toValue),
1715
+ toValue,
1716
+ ignored: event.ignored === true,
1717
+ ...decision === void 0 || type === "@agent.usage" ? {} : { decision }
1718
+ });
1719
+ if (type !== "@agent.usage") decision = void 0;
1579
1720
  }
1580
- function fn(name, run, callOptions = {}) {
1581
- const { redact, serialize = toJson, revive } = callOptions;
1582
- return (async (...args) => {
1583
- const passed = [...args];
1584
- while (passed.length && passed.at(-1) === void 0) passed.pop();
1585
- const call = callCounts.get(name) ?? 0;
1586
- callCounts.set(name, call + 1);
1587
- const effect = {
1588
- kind: "call",
1589
- name,
1590
- call,
1591
- args: toJson(redact ? redact(passed) : passed)
1592
- };
1593
- effects.push(effect);
1594
- if (replay) {
1595
- const recorded = replay.find((candidate) => candidate.kind === "call" && candidate.name === name && !usedCalls.has(candidate));
1596
- if (!recorded) {
1597
- const error = new RecordingDivergenceError(`Call #${call} to '${name}' was never made by the recorded run.`);
1598
- effect.error = toRecordedError(error);
1599
- throw error;
1600
- }
1601
- usedCalls.add(recorded);
1602
- if (recorded.error) {
1603
- effect.error = recorded.error;
1604
- throw fromRecordedError(recorded.error);
1605
- }
1606
- if (recorded.stream) {
1607
- effect.stream = recorded.stream;
1608
- const chunks = structuredClone(recorded.stream);
1609
- return revive ? revive(chunks) : iterate(chunks);
1721
+ return steps;
1722
+ }
1723
+ //#endregion
1724
+ //#region src/fold-traces.ts
1725
+ /**
1726
+ * Folds many runs' traces into per-state stats: how many runs visited each
1727
+ * state, how many of those failed, how many options the guards withheld
1728
+ * at decisions made there, how many events it ignored, and how many failures
1729
+ * `blame` traced back to it. Pure; reads only:
1730
+ *
1731
+ * - `machine.transition`: the run visited `from.value` and `snapshot.value`;
1732
+ * with `ignored`, each of those states ignored one event.
1733
+ * - `request.start` of a decision, first attempt only: its `request.withheld`
1734
+ * counts toward the state the run was in, the `from.value` of the
1735
+ * transition its choice causes.
1736
+ *
1737
+ * A trace is one run's events, live (`AgentTraceEvent`) or serialized
1738
+ * (`serializeTraceEvent`). A host-driven `decideNext` traces nothing; push
1739
+ * `{ type: 'request.start', request: result.request }` onto the trace before
1740
+ * sending its event. Parallel states count once per active leaf.
1741
+ *
1742
+ * `failures` counts a failed run against every state it visited, so states
1743
+ * after a bad decision rank as high as the decision. Sort by `blamed` to see
1744
+ * where failures start.
1745
+ *
1746
+ * @example
1747
+ * ```ts no-check
1748
+ * const stats = foldTraces(traces, {
1749
+ * failed: (trace) => trace.at(-1)?.type === 'run.end' && trace.at(-1).status === 'error',
1750
+ * });
1751
+ * ```
1752
+ */
1753
+ function foldTraces(traces, options) {
1754
+ const stats = /* @__PURE__ */ new Map();
1755
+ const statsFor = (state) => {
1756
+ let entry = stats.get(state);
1757
+ if (!entry) {
1758
+ entry = {
1759
+ state,
1760
+ visits: 0,
1761
+ failures: 0,
1762
+ withheld: 0,
1763
+ ignored: 0,
1764
+ blamed: 0
1765
+ };
1766
+ stats.set(state, entry);
1767
+ }
1768
+ return entry;
1769
+ };
1770
+ for (const trace of traces) {
1771
+ const visited = /* @__PURE__ */ new Set();
1772
+ let current = [];
1773
+ const visit = (state) => {
1774
+ statsFor(state);
1775
+ visited.add(state);
1776
+ };
1777
+ for (const event of trace) if (event.type === "machine.transition") {
1778
+ for (const state of statePaths(valueOf(event.from))) visit(state);
1779
+ current = statePaths(valueOf(event.snapshot));
1780
+ for (const state of current) visit(state);
1781
+ if (event.ignored === true) for (const state of current) statsFor(state).ignored++;
1782
+ } else if (event.type === "request.start" && isFirstDecision(event.request)) {
1783
+ const withheld = event.request.withheld;
1784
+ if (!Array.isArray(withheld)) continue;
1785
+ for (const state of current) statsFor(state).withheld += withheld.length;
1786
+ }
1787
+ const failed = options.failed(trace);
1788
+ for (const state of visited) {
1789
+ const entry = statsFor(state);
1790
+ entry.visits++;
1791
+ if (failed) entry.failures++;
1792
+ }
1793
+ const blamed = failed ? options.blame?.(trace) : void 0;
1794
+ if (blamed !== void 0) statsFor(blamed).blamed++;
1795
+ }
1796
+ return [...stats.values()];
1797
+ }
1798
+ //#endregion
1799
+ //#region src/trace-coverage.ts
1800
+ /** Mirrors xstate@6.0.0-alpha.69 `matchesEventDescriptor` (src/utils.ts), which the package does not export. */
1801
+ function matchesDescriptor(eventType, descriptor) {
1802
+ if (descriptor === eventType || descriptor === "*") return true;
1803
+ if (!descriptor.endsWith(".*")) return false;
1804
+ const tokens = descriptor.split(".");
1805
+ const eventTokens = eventType.split(".");
1806
+ for (let index = 0; index < tokens.length; index++) {
1807
+ if (tokens[index] === "*") return index === tokens.length - 1;
1808
+ if (tokens[index] !== eventTokens[index]) return false;
1809
+ }
1810
+ return true;
1811
+ }
1812
+ /** Whether a definition's `matches` pattern (e.g. an invoke's `actorId`) accepts the traced event. */
1813
+ function matchesPattern(definition, event) {
1814
+ const pattern = definition.matches;
1815
+ if (!require_event_log.isRecord(pattern) || !require_event_log.isRecord(event)) return true;
1816
+ return Object.entries(pattern).every(([key, value]) => Object.is(event[key], value));
1817
+ }
1818
+ /**
1819
+ * The descriptor XState would try first at `node` for `event`, guards aside:
1820
+ * the exact key, then partial and wildcard keys longest first, as in
1821
+ * xstate@6.0.0-alpha.69 `getCandidates` (src/stateUtils.ts).
1822
+ */
1823
+ function candidateAt(node, type, event) {
1824
+ const keys = [...node.transitions.keys()].filter((key) => key !== type && matchesDescriptor(type, key)).sort((a, b) => b.length - a.length);
1825
+ if (node.transitions.has(type)) keys.unshift(type);
1826
+ return keys.find((key) => node.transitions.get(key).some((d) => matchesPattern(d, event)));
1827
+ }
1828
+ const pathOf = (node) => node.path.join(".");
1829
+ const sameLeaves = (a, b) => statePaths(a).sort().join("|") === statePaths(b).sort().join("|");
1830
+ /**
1831
+ * The leaves a trace passed through inside its macrosteps: states an `always`
1832
+ * transition left before the step ended, which no traced snapshot holds.
1833
+ * Re-derived with XState's pure `getMicrosteps` from the snapshot before each
1834
+ * step (the previous step's, or the fresh start's microsteps from its input).
1835
+ * A step that cannot be recomputed, or whose recomputation ends elsewhere
1836
+ * than the traced snapshot, adds none. Only `value` and `context` are rebuilt,
1837
+ * so a guard that reads live children can take another route to the same
1838
+ * leaves, and that route is credited.
1839
+ */
1840
+ function passedThrough(machine, trace) {
1841
+ const leaves = /* @__PURE__ */ new Set();
1842
+ let resumed = false;
1843
+ let previous;
1844
+ for (const event of trace) {
1845
+ if (event.type === "run.start") resumed = event.snapshot !== void 0;
1846
+ if (event.type !== "machine.transition" || !require_event_log.isRecord(event.event)) continue;
1847
+ const snapshot = require_event_log.isRecord(event.snapshot) ? event.snapshot : void 0;
1848
+ try {
1849
+ const microsteps = event.event.type === "@xstate.init" ? resumed ? [] : (0, xstate.getInitialMicrosteps)(machine, event.event.input) : previous && sameLeaves(previous.value, valueOf(event.from)) ? (0, xstate.getMicrosteps)(machine, machine.resolveState({
1850
+ value: previous.value,
1851
+ context: previous.context
1852
+ }), event.event) : [];
1853
+ const last = microsteps.at(-1)?.[0];
1854
+ if (last && snapshot && sameLeaves(last.value, snapshot.value)) for (const [step] of microsteps) for (const leaf of statePaths(step.value)) leaves.add(leaf);
1855
+ } catch {}
1856
+ previous = snapshot;
1857
+ }
1858
+ return leaves;
1859
+ }
1860
+ const ratioOf = (visited, total) => total === 0 ? 1 : visited / total;
1861
+ /**
1862
+ * Measures how much of a machine a set of runs exercised: which atomic and
1863
+ * final states they visited, and which handlers they took. A handler is a
1864
+ * `(declaring state, event descriptor)` pair from `on`, invoke
1865
+ * `onDone`/`onError`, and `after`, with the targets it declares and the
1866
+ * leaves the runs landed in after it. Pure; reads only `machine.transition`
1867
+ * events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
1868
+ *
1869
+ * A step credits one handler per active leaf: the nearest node, from the leaf
1870
+ * up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
1871
+ * event, preferred in XState's order. Guards are not replayed, so a step
1872
+ * whose nearest handler was rejected and fell through to an ancestor is
1873
+ * credited to the nearest one. XState keys invoke and `after` handlers by
1874
+ * bare type (`xstate.done.actor`), one per state however many invokes; a
1875
+ * definition's `matches` (an invoke's `actorId`) must accept the traced event
1876
+ * for its state to be credited. Ignored steps credit no handler. `always`
1877
+ * handlers are listed as `unobservable`, never as unvisited; a state one
1878
+ * passes through is visited, re-derived from each step's microsteps.
1879
+ *
1880
+ * @example
1881
+ * ```ts no-check
1882
+ * const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
1883
+ * expect(report.unvisitedHandlers).toEqual([]);
1884
+ * ```
1885
+ */
1886
+ function traceCoverage(machine, traces, options = {}) {
1887
+ const nodes = /* @__PURE__ */ new Map();
1888
+ const states = [];
1889
+ const handlers = [];
1890
+ const handlerAt = /* @__PURE__ */ new Map();
1891
+ const unobservable = [];
1892
+ const walk = (node) => {
1893
+ const state = pathOf(node);
1894
+ nodes.set(state, node);
1895
+ if (node.type === "atomic" || node.type === "final") states.push({
1896
+ state,
1897
+ runs: 0
1898
+ });
1899
+ const byEvent = /* @__PURE__ */ new Map();
1900
+ for (const [event, definitions] of node.transitions) {
1901
+ const dynamic = definitions.some((d) => typeof d.to === "function");
1902
+ const declared = [...new Set(definitions.flatMap((d) => (d.target ?? []).map(pathOf)))];
1903
+ const handler = {
1904
+ state,
1905
+ event,
1906
+ runs: 0,
1907
+ targets: {
1908
+ declared: dynamic ? "dynamic" : declared,
1909
+ seen: [],
1910
+ unreached: []
1610
1911
  }
1611
- effect.result = recorded.result;
1612
- const result = structuredClone(recorded.result);
1613
- return revive ? revive(result) : result;
1614
- }
1615
- let result;
1616
- try {
1617
- result = await run(...args);
1618
- } catch (error) {
1619
- effect.error = toRecordedError(error);
1620
- throw error;
1912
+ };
1913
+ handlers.push(handler);
1914
+ byEvent.set(event, handler);
1915
+ }
1916
+ handlerAt.set(node, byEvent);
1917
+ if (node.always?.length) unobservable.push({
1918
+ state,
1919
+ event: "always"
1920
+ });
1921
+ for (const child of Object.values(node.states)) walk(child);
1922
+ };
1923
+ walk(machine.root);
1924
+ const stateRuns = new Map(states.map((row) => [row.state, row]));
1925
+ const seen = /* @__PURE__ */ new Map();
1926
+ for (const trace of traces) {
1927
+ const steps = traceSteps(trace);
1928
+ const visited = new Set(steps[0]?.from);
1929
+ for (const event of trace) {
1930
+ if (event.type !== "machine.transition" || !require_event_log.isRecord(event.event)) continue;
1931
+ if (event.event.type !== "@xstate.init" || !require_event_log.isRecord(event.snapshot)) continue;
1932
+ for (const leaf of statePaths(event.snapshot.value)) visited.add(leaf);
1933
+ }
1934
+ if (unobservable.length > 0) for (const leaf of passedThrough(machine, trace)) visited.add(leaf);
1935
+ const credited = /* @__PURE__ */ new Set();
1936
+ for (const step of steps) {
1937
+ for (const leaf of step.to) visited.add(leaf);
1938
+ if (step.ignored) continue;
1939
+ const hits = /* @__PURE__ */ new Set();
1940
+ for (const leaf of step.from) for (let node = nodes.get(leaf); node; node = node.parent) {
1941
+ if (node.type === "final") continue;
1942
+ const event = candidateAt(node, step.event, step.payload);
1943
+ if (event === void 0) continue;
1944
+ hits.add(handlerAt.get(node).get(event));
1945
+ break;
1621
1946
  }
1622
- if (!require_effects.isAsyncIterable(result)) {
1623
- effect.result = toJson(serialize(result));
1624
- return result;
1947
+ for (const handler of hits) {
1948
+ credited.add(handler);
1949
+ let leaves = seen.get(handler);
1950
+ if (!leaves) seen.set(handler, leaves = /* @__PURE__ */ new Set());
1951
+ for (const leaf of step.to) leaves.add(leaf);
1625
1952
  }
1626
- const chunks = [];
1627
- effect.stream = chunks;
1628
- const source = result;
1629
- return (async function* () {
1630
- try {
1631
- for await (const chunk of source) {
1632
- chunks.push(toJson(serialize(chunk)));
1633
- yield chunk;
1634
- }
1635
- } catch (error) {
1636
- effect.error = toRecordedError(error);
1637
- throw error;
1638
- }
1639
- })();
1953
+ }
1954
+ for (const state of visited) {
1955
+ const row = stateRuns.get(state);
1956
+ if (row) row.runs++;
1957
+ }
1958
+ for (const handler of credited) handler.runs++;
1959
+ }
1960
+ for (const handler of handlers) {
1961
+ const leaves = [...seen.get(handler) ?? []];
1962
+ handler.targets.seen = leaves;
1963
+ const { declared } = handler.targets;
1964
+ if (declared === "dynamic") continue;
1965
+ handler.targets.unreached = declared.filter((target) => !leaves.some((leaf) => leaf === target || leaf.startsWith(`${target}.`)));
1966
+ }
1967
+ const reachableComplete = options.reachable ? require_verify.isExplorationComplete(options.reachable) : void 0;
1968
+ let unreachable;
1969
+ if (options.reachable && reachableComplete) {
1970
+ const reached = new Set(options.reachable.reachedStates.flatMap((value) => statePaths(value)));
1971
+ unreachable = states.filter((row) => !reached.has(row.state) && row.runs === 0).map((row) => row.state);
1972
+ }
1973
+ const counted = unreachable ? states.filter((row) => !unreachable.includes(row.state)) : states;
1974
+ const unvisitedStates = counted.filter((row) => row.runs === 0).map((row) => row.state);
1975
+ const unvisitedHandlers = handlers.filter((handler) => handler.runs === 0).map(({ state, event }) => ({
1976
+ state,
1977
+ event
1978
+ }));
1979
+ return {
1980
+ states,
1981
+ handlers,
1982
+ unvisitedStates,
1983
+ unvisitedHandlers,
1984
+ unobservable,
1985
+ ...unreachable ? { unreachable } : {},
1986
+ ...reachableComplete === void 0 ? {} : { reachableComplete },
1987
+ ratio: {
1988
+ states: ratioOf(counted.length - unvisitedStates.length, counted.length),
1989
+ handlers: ratioOf(handlers.length - unvisitedHandlers.length, handlers.length)
1990
+ }
1991
+ };
1992
+ }
1993
+ //#endregion
1994
+ //#region src/first-divergence.ts
1995
+ const defaultKey = (step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`;
1996
+ const isErrorLike = (value) => require_event_log.isError(value) || require_event_log.isRecord(value) && typeof value.name === "string" && typeof value.message === "string";
1997
+ const normalizeErrors = (value, seen = /* @__PURE__ */ new WeakSet()) => {
1998
+ if (value === null || typeof value !== "object") return value;
1999
+ if (seen.has(value)) return "[circular]";
2000
+ seen.add(value);
2001
+ try {
2002
+ if (isErrorLike(value) && (require_event_log.isError(value) || "stack" in value || "code" in value)) {
2003
+ const { name, message, code, cause } = value;
2004
+ return {
2005
+ name,
2006
+ message,
2007
+ ...typeof code === "string" ? { code } : {},
2008
+ ...cause !== void 0 ? { cause: normalizeErrors(cause, seen) } : {}
2009
+ };
2010
+ }
2011
+ if (Array.isArray(value)) return value.map((item) => normalizeErrors(item, seen));
2012
+ if (!require_event_log.isPlainObject(value)) return value;
2013
+ const copy = Object.create(null);
2014
+ for (const key of Object.keys(value)) copy[key] = normalizeErrors(value[key], seen);
2015
+ return copy;
2016
+ } finally {
2017
+ seen.delete(value);
2018
+ }
2019
+ };
2020
+ const payloadKey = (step) => {
2021
+ if (!require_event_log.isRecord(step.payload)) return `${defaultKey(step)}|${require_event_log.canonicalEventJson(step.payload)}`;
2022
+ const { sessionId: _sessionId, ...payload } = step.payload;
2023
+ return `${defaultKey(step)}|${require_event_log.canonicalEventJson(normalizeErrors(payload))}`;
2024
+ };
2025
+ const isTrace = (items) => items.length === 0 || items.some((item) => require_event_log.isRecord(item) && (item.type === "machine.transition" || item.type === "run.start" || item.type === "run.end"));
2026
+ /**
2027
+ * Finds the first step where a run departs from a reference, so a failure is
2028
+ * blamed on the state it started in rather than on every state the run went
2029
+ * through afterwards.
2030
+ *
2031
+ * - Against a reference **trace**, such as a passing run of the same
2032
+ * scenario, steps are compared in order by `key`. The first step that
2033
+ * differs, or the point where either trace ends early, is the divergence.
2034
+ * - Against an expected **trajectory** (`TrajectoryItem[]`, as for
2035
+ * {@link matchesTrajectory}), the run is laid out as its initial state, then
2036
+ * each step's event and new state. The first expected item it never reaches
2037
+ * is the divergence, blamed on the step after the last item it did reach. A
2038
+ * run that goes on past the trajectory has not diverged.
2039
+ *
2040
+ * An empty reference is a trace with no steps, so any step diverges from it.
2041
+ * A trace is one run's events, live or serialized. Steps the machine ignored
2042
+ * changed nothing and are left out on both sides. Returns `undefined` when the
2043
+ * run does not diverge.
2044
+ *
2045
+ * @example
2046
+ * ```ts no-check
2047
+ * const stats = foldTraces(traces, {
2048
+ * failed,
2049
+ * blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
2050
+ * });
2051
+ * ```
2052
+ */
2053
+ function firstDivergence(actual, reference, options = {}) {
2054
+ const steps = traceSteps(actual).filter((step) => !step.ignored);
2055
+ return isTrace(reference) ? againstTrace(steps, traceSteps(reference).filter((step) => !step.ignored), options.key === "payload" ? payloadKey : options.key ?? defaultKey) : againstTrajectory(actual, steps, reference);
2056
+ }
2057
+ function diverged(steps, index, expected, initial = steps[0]?.fromValue) {
2058
+ const step = steps[index];
2059
+ return {
2060
+ index,
2061
+ state: (step ? step.from[0] : steps.at(-1)?.to[0] ?? statePaths(initial)[0]) ?? "",
2062
+ ...step?.decision === void 0 ? {} : { decision: step.decision },
2063
+ ...step ? { actual: step } : {},
2064
+ ...expected === void 0 ? {} : { expected }
2065
+ };
2066
+ }
2067
+ function againstTrace(steps, reference, key) {
2068
+ const length = Math.max(steps.length, reference.length);
2069
+ for (let index = 0; index < length; index++) {
2070
+ const step = steps[index];
2071
+ const want = reference[index];
2072
+ if (step && want && key(step) === key(want)) continue;
2073
+ return diverged(steps, index, want, steps[0]?.fromValue ?? reference[0]?.fromValue);
2074
+ }
2075
+ }
2076
+ function againstTrajectory(trace, steps, expected) {
2077
+ const initial = initialValue(trace, steps);
2078
+ const timeline = initial === void 0 ? [] : [{ value: initial }];
2079
+ const stepAt = initial === void 0 ? [] : [0];
2080
+ steps.forEach((step, index) => {
2081
+ timeline.push(step.payload);
2082
+ stepAt.push(index);
2083
+ if (JSON.stringify(step.toValue) !== JSON.stringify(step.fromValue)) {
2084
+ timeline.push({ value: step.toValue });
2085
+ stepAt.push(index);
2086
+ }
2087
+ });
2088
+ const match = matchesTrajectory(timeline, expected);
2089
+ if (!match.firstMiss) return void 0;
2090
+ const at = match.firstMiss.searchedFrom;
2091
+ return diverged(steps, at < timeline.length ? stepAt[at] ?? steps.length : steps.length, match.firstMiss.expected, initial);
2092
+ }
2093
+ function initialValue(trace, steps) {
2094
+ for (const event of trace) if (event.type === "machine.transition" && require_event_log.isRecord(event.snapshot)) return event.snapshot.value;
2095
+ return steps[0]?.fromValue;
2096
+ }
2097
+ //#endregion
2098
+ //#region src/propose-from-traces.ts
2099
+ /**
2100
+ * Groups the approval verdicts in many runs' traces by `key` and proposes an
2101
+ * `allow` or `deny` rule for each. Pure and deterministic; a proposal is for
2102
+ * a person to review, and nothing here changes a machine.
2103
+ *
2104
+ * Reads `machine.transition` events: a verdict is a transition, not ignored,
2105
+ * out of `approvalState` on an approve or reject event. Rows come in the
2106
+ * order keys were first seen.
2107
+ *
2108
+ * @example
2109
+ * ```ts no-check
2110
+ * const rows = proposeApprovalRules(traces, {
2111
+ * approvalState: "awaitingApproval",
2112
+ * approveEvents: ["APPROVE"],
2113
+ * rejectEvents: ["REJECT"],
2114
+ * key: ({ snapshot }) => `${snapshot.context.pending.tool}:${snapshot.context.pending.path}`,
2115
+ * });
2116
+ * ```
2117
+ */
2118
+ function proposeApprovalRules(traces, options) {
2119
+ const rows = /* @__PURE__ */ new Map();
2120
+ const rowFor = (key) => {
2121
+ let row = rows.get(key);
2122
+ if (!row) {
2123
+ row = {
2124
+ key,
2125
+ approved: 0,
2126
+ rejected: 0,
2127
+ violations: 0,
2128
+ runs: [],
2129
+ proposal: null,
2130
+ support: 0
2131
+ };
2132
+ rows.set(key, row);
2133
+ }
2134
+ return row;
2135
+ };
2136
+ const inState = (value) => statePaths(value).some((path) => path === options.approvalState || path.startsWith(`${options.approvalState}.`));
2137
+ let index = 0;
2138
+ for (const trace of traces) {
2139
+ const run = index++;
2140
+ const seen = /* @__PURE__ */ new Set();
2141
+ let snapshot;
2142
+ for (const event of trace) {
2143
+ if (event.type !== "machine.transition") continue;
2144
+ const before = snapshot;
2145
+ snapshot = event.snapshot;
2146
+ const type = require_event_log.isRecord(event.event) ? event.event.type : void 0;
2147
+ if (event.ignored === true || typeof type !== "string" || !inState(valueOf(event.from))) continue;
2148
+ const approve = options.approveEvents.includes(type);
2149
+ if (!approve && !options.rejectEvents.includes(type)) continue;
2150
+ const key = options.key({
2151
+ event: event.event,
2152
+ snapshot: before,
2153
+ trace
2154
+ });
2155
+ if (key === void 0) continue;
2156
+ const row = rowFor(key);
2157
+ if (approve) row.approved++;
2158
+ else row.rejected++;
2159
+ if (!seen.has(key)) {
2160
+ seen.add(key);
2161
+ row.runs.push(run);
2162
+ }
2163
+ }
2164
+ for (const key of options.violations?.(trace) ?? []) {
2165
+ const row = rows.get(key);
2166
+ if (row) row.violations++;
2167
+ }
2168
+ }
2169
+ for (const row of rows.values()) {
2170
+ row.support = row.runs.length;
2171
+ row.proposal = row.approved > 0 && row.rejected === 0 && row.violations === 0 ? "allow" : row.rejected > 0 && row.approved === 0 ? "deny" : null;
2172
+ }
2173
+ return [...rows.values()];
2174
+ }
2175
+ /**
2176
+ * Proposes a value for each cap (retry count, steps, batch size) from many
2177
+ * runs' traces: the 95th percentile of what passing runs used, plus one, and
2178
+ * never below the current cap while runs still hit it. Pure and
2179
+ * deterministic; a proposal is for a person to review, and nothing here
2180
+ * changes a machine.
2181
+ *
2182
+ * @example
2183
+ * ```ts no-check
2184
+ * const [retries] = proposeCaps(traces, {
2185
+ * caps: [{ name: "retries", current: 3, failureState: "gaveUp", valueFromRun: countRetries }],
2186
+ * });
2187
+ * ```
2188
+ */
2189
+ function proposeCaps(traces, options) {
2190
+ const runs = [...traces];
2191
+ return options.caps.map((cap) => {
2192
+ let hits = 0;
2193
+ let max = null;
2194
+ const passing = [];
2195
+ for (const trace of runs) {
2196
+ const hit = hitCap(trace, cap);
2197
+ if (hit) hits++;
2198
+ const value = cap.valueFromRun(trace);
2199
+ if (value === void 0) continue;
2200
+ max = max === null ? value : Math.max(max, value);
2201
+ if (!hit) passing.push(value);
2202
+ }
2203
+ passing.sort((a, b) => a - b);
2204
+ const p95 = passing.length ? passing[Math.ceil(.95 * passing.length) - 1] : null;
2205
+ const proposal = p95 === null ? null : hits > 0 && cap.current !== void 0 ? Math.max(p95 + 1, cap.current) : p95 + 1;
2206
+ return {
2207
+ name: cap.name,
2208
+ hits,
2209
+ passing: passing.length,
2210
+ max,
2211
+ p95,
2212
+ proposal
2213
+ };
2214
+ });
2215
+ }
2216
+ function hitCap(trace, cap) {
2217
+ for (const event of trace) {
2218
+ if (cap.failureState !== void 0 && event.type === "machine.transition" && statePaths(valueOf(event.snapshot)).some((path) => path === cap.failureState || path.startsWith(`${cap.failureState}.`))) return true;
2219
+ if (cap.failureOutput && event.type === "run.end" && event.status === "done") {
2220
+ if (cap.failureOutput(event.output)) return true;
2221
+ }
2222
+ }
2223
+ return false;
2224
+ }
2225
+ //#endregion
2226
+ //#region src/decision-requests.ts
2227
+ /**
2228
+ * The decisions a run asked for, read from its trace: each decision's first
2229
+ * `request.start`, with the states the run was in. A trace is one run's events,
2230
+ * live or serialized with `serializeTraceEvent`. Retries of the same decision
2231
+ * are left out, so each decision appears once.
2232
+ *
2233
+ * Use the requests as eval rows (run a decider on each and compare its event
2234
+ * with a label) or as probe inputs for a policy. A serialized request keeps its
2235
+ * `options`, `metadata`, `system`, and `prompt`; event schemas do not survive
2236
+ * serialization, so replay closed choices (`options`) rather than open ones.
2237
+ *
2238
+ * @example
2239
+ * ```ts no-check
2240
+ * const rows = traces.flatMap((trace) => getDecisionRequests(trace, { name: "chooseResolution" }));
2241
+ * for (const { request } of rows) console.log((await decide(request)).event);
2242
+ * ```
2243
+ */
2244
+ function getDecisionRequests(trace, options = {}) {
2245
+ const requests = [];
2246
+ let current = [];
2247
+ for (const event of trace) if (event.type === "machine.transition") current = statePaths(valueOf(event.snapshot));
2248
+ else if (event.type === "request.start" && isFirstDecision(event.request)) {
2249
+ if (options.name !== void 0 && event.request.name !== options.name) continue;
2250
+ requests.push({
2251
+ states: [...current],
2252
+ request: event.request
2253
+ });
2254
+ }
2255
+ return requests;
2256
+ }
2257
+ //#endregion
2258
+ //#region src/repeat-runs.ts
2259
+ /**
2260
+ * Summarizes repeated runs of each scenario: how often each passed, whether
2261
+ * all its runs passed (pass^k), how many distinct state paths they took, and
2262
+ * where they first went different ways. Pure. Use it for runs collected
2263
+ * elsewhere, such as CI shards or replays; {@link repeatRuns} collects them and
2264
+ * calls it.
2265
+ *
2266
+ * A scenario that passes 3 of 5 runs and `divergedAt: 'deciding'` points at one
2267
+ * decision, not at a flaky agent.
2268
+ *
2269
+ * @example
2270
+ * ```ts no-check
2271
+ * const report = summarizeRepeats(shards.map((s) => ({ id: s.id, scenario: s, runs: s.runs })));
2272
+ * report.passHat(3); // the chance 3 runs of a scenario all pass
2273
+ * ```
2274
+ */
2275
+ function summarizeRepeats(groups) {
2276
+ const scenarios = groups.map((group) => summarize$1(group));
2277
+ const traces = groups.flatMap((group) => group.runs.flatMap((run) => run.trace ? [run.trace] : []));
2278
+ const most = Math.max(0, ...groups.map((group) => group.runs.length));
2279
+ const passHat = (j) => {
2280
+ if (!Number.isInteger(j) || j < 1 || j > most) throw new require_event_log.AgentError("pass-hat-out-of-range", `passHat: j must be a whole number from 1 to ${most}, got ${j}.`);
2281
+ const eligible = scenarios.filter((scenario) => scenario.runs.length >= j);
2282
+ return eligible.reduce((total, scenario) => total + allPass(scenario.passes, scenario.runs.length, j), 0) / eligible.length;
2283
+ };
2284
+ return {
2285
+ scenarios,
2286
+ passHat,
2287
+ traces
2288
+ };
2289
+ }
2290
+ /**
2291
+ * Runs every scenario `k` times and summarizes the runs with
2292
+ * {@link summarizeRepeats}. Runs are sequential, scenario by scenario. A run
2293
+ * that throws fails, with its message as `error` and no trace.
2294
+ *
2295
+ * With a live model, one run per scenario says little: τ-bench found pass^k
2296
+ * falls well below the single-run rate. Report pass^k next to it.
2297
+ *
2298
+ * @example
2299
+ * ```ts no-check
2300
+ * const report = await repeatRuns({
2301
+ * scenarios,
2302
+ * k: 5,
2303
+ * run: async (scenario) => {
2304
+ * const trace: AgentTraceEvent[] = [];
2305
+ * const result = await runToQuiescence(
2306
+ * createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
2307
+ * { input: scenario.input },
2308
+ * );
2309
+ * return { trace, outcome: String(result.snapshot.value) };
2310
+ * },
2311
+ * passed: (scenario, { outcome }) => outcome === scenario.expected,
2312
+ * });
2313
+ * report.passHat(5);
2314
+ * ```
2315
+ */
2316
+ async function repeatRuns(options) {
2317
+ if (!Number.isInteger(options.k) || options.k < 1) throw new require_event_log.AgentError("invalid-repeat-count", `repeatRuns: k must be a whole number of 1 or more, got ${options.k}.`);
2318
+ const groups = [];
2319
+ for (const scenario of options.scenarios) {
2320
+ const runs = [];
2321
+ for (let attempt = 0; attempt < options.k; attempt++) try {
2322
+ const result = await options.run(scenario, attempt);
2323
+ runs.push({
2324
+ passed: options.passed(scenario, result),
2325
+ trace: result.trace,
2326
+ ...result.outcome === void 0 ? {} : { outcome: result.outcome }
2327
+ });
2328
+ } catch (error) {
2329
+ runs.push({
2330
+ passed: false,
2331
+ error: error instanceof Error ? error.message : String(error)
2332
+ });
2333
+ }
2334
+ groups.push({
2335
+ id: scenario.id,
2336
+ scenario,
2337
+ runs
1640
2338
  });
1641
2339
  }
1642
- /** Counts an async tool call as running until it settles. */
1643
- function track(result) {
1644
- if (result instanceof Promise) {
1645
- toolsInFlight++;
1646
- const settled = () => {
1647
- toolsInFlight--;
1648
- };
1649
- result.then(settled, settled);
1650
- }
1651
- return result;
2340
+ return summarizeRepeats(groups);
2341
+ }
2342
+ function summarize$1(group) {
2343
+ const passes = group.runs.filter((run) => run.passed).length;
2344
+ const traces = group.runs.flatMap((run) => run.trace ? [run.trace] : []);
2345
+ const paths = new Set(traces.map((trace) => traceSteps(trace).filter((step) => !step.ignored).map((step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`).join(" ")));
2346
+ const reference = traces.find((trace) => trace.some((event) => event.type === "machine.transition"));
2347
+ let earliest;
2348
+ for (const trace of reference ? traces : []) {
2349
+ if (trace === reference) continue;
2350
+ const divergence = firstDivergence(trace, reference);
2351
+ if (divergence && (!earliest || divergence.index < earliest.index)) earliest = divergence;
1652
2352
  }
1653
2353
  return {
1654
- model(model) {
1655
- const base = model;
1656
- const stand = options.fallback?.model;
1657
- const wrapped = Object.create(base);
1658
- for (const method of [
1659
- "doGenerate",
1660
- "doStream",
1661
- "doDecide",
1662
- "doEvaluate"
1663
- ]) {
1664
- if (typeof base[method] !== "function") continue;
1665
- const invoke = (target, callOptions) => target[method].call(target, callOptions);
1666
- wrapped[method] = (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0);
1667
- }
1668
- return wrapped;
1669
- },
1670
- openai(client) {
1671
- const wrapCreate = (resource, method) => new Proxy(resource, { get(target, key, receiver) {
1672
- if (key !== "create") return Reflect.get(target, key, receiver);
1673
- return (params, requestOptions) => {
1674
- let api;
1675
- const create = (resource) => () => {
1676
- api = Reflect.get(resource, "create").call(resource, params, requestOptions);
1677
- return Promise.resolve(api);
1678
- };
1679
- const stand = fallbackResource(method);
1680
- const promise = modelCall("openai", method, params, params?.stream === true, create(target), stand && create(stand));
1681
- return Object.assign(promise, { withResponse: async () => {
1682
- const data = await promise;
1683
- const meta = await api?.withResponse?.();
1684
- return {
1685
- data,
1686
- response: meta?.response ?? new Response(null),
1687
- request_id: meta?.request_id ?? null
1688
- };
1689
- } });
1690
- };
1691
- } });
1692
- const fallbackResource = (method) => {
1693
- let value = options.fallback?.openai;
1694
- for (const key of method.split(".").slice(0, -1)) value = value?.[key];
1695
- return value && typeof value === "object" ? value : void 0;
1696
- };
1697
- const wrapped = /* @__PURE__ */ new Map();
1698
- const wrap = (key, make) => {
1699
- if (!wrapped.has(key)) wrapped.set(key, make());
1700
- return wrapped.get(key);
1701
- };
1702
- return new Proxy(client, { get(target, key, receiver) {
1703
- const value = Reflect.get(target, key, receiver);
1704
- if (key === "responses" && value) return wrap(key, () => wrapCreate(value, "responses.create"));
1705
- if (key === "chat" && value) return wrap(key, () => new Proxy(value, { get(chat, chatKey, chatReceiver) {
1706
- const inner = Reflect.get(chat, chatKey, chatReceiver);
1707
- return chatKey === "completions" && inner ? wrap("chat.completions", () => wrapCreate(inner, "chat.completions.create")) : inner;
1708
- } }));
1709
- return value;
1710
- } });
1711
- },
1712
- tool: (name, fn) => tool(name, fn),
1713
- fn,
1714
- client(name, client, { methods, ...callOptions }) {
1715
- const paths = methods.map((method) => method.split("."));
1716
- const startsWith = (path, prefix) => prefix.every((key, index) => path[index] === key);
1717
- const wrapAt = (target, prefix) => {
1718
- const wrapped = /* @__PURE__ */ new Map();
1719
- return new Proxy(target, { get(object, key, receiver) {
1720
- if (typeof key !== "string") return Reflect.get(object, key, receiver);
1721
- const path = [...prefix, key];
1722
- const method = paths.some((p) => p.length === path.length && startsWith(p, path));
1723
- if (!method && !paths.some((p) => startsWith(p, path))) return Reflect.get(object, key, receiver);
1724
- if (!wrapped.has(key)) {
1725
- const value = Reflect.get(object, key, receiver);
1726
- wrapped.set(key, method ? fn(`${name}.${path.join(".")}`, (...args) => value.apply(object, args), callOptions) : wrapAt(value && typeof value === "object" ? value : {}, path));
1727
- }
1728
- return wrapped.get(key);
1729
- } });
1730
- };
1731
- return wrapAt(client, []);
1732
- },
1733
- tools(tools) {
1734
- return Object.fromEntries(Object.entries(tools).map(([name, definition]) => [name, definition.execute ? {
1735
- ...definition,
1736
- execute: tool(name, definition.execute, (args) => args[0])
1737
- } : definition]));
1738
- },
1739
- output(value) {
1740
- effects.push({
1741
- kind: "output",
1742
- value: toJson(value)
1743
- });
1744
- },
1745
- get recording() {
1746
- return {
1747
- version: 1,
1748
- effects: toJson(effects)
1749
- };
1750
- }
2354
+ ...group,
2355
+ passes,
2356
+ passAny: passes > 0,
2357
+ passAll: group.runs.length > 0 && passes === group.runs.length,
2358
+ distinctPaths: paths.size,
2359
+ ...earliest ? { divergedAt: earliest.state } : {}
1751
2360
  };
1752
2361
  }
1753
- /** True when two model calls offer the same tool names and response format. */
1754
- function sameModelCallShape(expected, actual) {
1755
- return jsonEqual(modelCallShape(expected), modelCallShape(actual));
1756
- }
1757
- function modelCallsMatch(match, expected, actual) {
1758
- if (match === "none") return true;
1759
- if (typeof match === "function") return match(expected, actual);
1760
- return sameModelCallShape(expected, actual);
1761
- }
1762
- function shapeMessage(expected, actual) {
1763
- return `Model call #${expected.call} changed shape: ${JSON.stringify(modelCallShape(expected))} → ${JSON.stringify(modelCallShape(actual))}. A recorded answer only fits a call of the same shape.`;
2362
+ /** C(passes, j) / C(runs, j), as a product so large counts don't overflow. */
2363
+ function allPass(passes, runs, j) {
2364
+ let chance = 1;
2365
+ for (let i = 0; i < j; i++) chance *= Math.max(0, passes - i) / (runs - i);
2366
+ return chance;
1764
2367
  }
2368
+ //#endregion
2369
+ //#region src/judge-agreement.ts
1765
2370
  /**
1766
- * Offered tool names and the response format, or a decision's questions and
1767
- * their options: what a recorded answer depends on.
2371
+ * Measures how well a judge agrees with people on the same cases: accuracy,
2372
+ * per-label recall, the confusion matrix, and the rows that disagree. With
2373
+ * `positive`, also the TPR and TNR, the recommended view for binary judges
2374
+ * (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
2375
+ * Pure.
2376
+ *
2377
+ * Build rows from confirmed labels only; draft labels gate nothing. Group by
2378
+ * decision state or request name so each narrow judge is aligned on its own.
2379
+ * A row with no group key counts in the totals and in no group. Run it on a
2380
+ * dev split and report the held-out split separately.
2381
+ *
2382
+ * Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
2383
+ *
2384
+ * @example
2385
+ * ```ts no-check
2386
+ * const report = judgeAgreement(
2387
+ * labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
2388
+ * { positive: 'fail' },
2389
+ * );
2390
+ * report.binary; // { tpr: 0.9, tnr: 0.8 }
2391
+ * report.correct?.(0.3); // estimated true failure rate in production
2392
+ * ```
1768
2393
  */
1769
- function modelCallShape(effect) {
1770
- const request = effect.request ?? {};
1771
- const questions = request["questions"];
1772
- if (questions && typeof questions === "object") return { questions: Object.entries(questions).map(([name, question]) => {
1773
- const criteria = question["criteria"];
1774
- if (question["type"] === "choice" && criteria && typeof criteria === "object") return [
1775
- name,
1776
- question["type"],
1777
- Object.keys(criteria).sort()
1778
- ];
1779
- if (question["type"] === "score" && Array.isArray(criteria)) return [
1780
- name,
1781
- question["type"],
1782
- criteria.length
1783
- ];
1784
- return [name, question["type"]];
1785
- }).sort((x, y) => String(x[0]).localeCompare(String(y[0]))) };
1786
- return {
1787
- tools: (Array.isArray(request["tools"]) ? request["tools"] : []).map((tool) => tool["name"] ?? tool["function"]?.["name"]).sort(),
1788
- format: request["responseFormat"]?.["type"] ?? request["response_format"]?.["type"] ?? (request["text"]?.["format"])?.["type"] ?? "text"
2394
+ function judgeAgreement(rows, options = {}) {
2395
+ if (rows.length === 0) throw new require_event_log.AgentError("empty-agreement-rows", "judgeAgreement: rows must contain at least one row.");
2396
+ const report = summarize(rows, options.positive);
2397
+ const groupOf = options.groupBy ?? ((row) => row.group);
2398
+ if (options.groupBy || rows.some((row) => row.group !== void 0)) {
2399
+ const grouped = /* @__PURE__ */ new Map();
2400
+ for (const row of rows) {
2401
+ const key = groupOf(row);
2402
+ if (key === void 0) continue;
2403
+ const members = grouped.get(key);
2404
+ if (members) members.push(row);
2405
+ else grouped.set(key, [row]);
2406
+ }
2407
+ const groups = Object.create(null);
2408
+ for (const [key, members] of grouped) groups[key] = summarize(members, options.positive);
2409
+ report.groups = groups;
2410
+ }
2411
+ return report;
2412
+ }
2413
+ function summarize(rows, positive) {
2414
+ const perLabel = Object.create(null);
2415
+ const confusion = Object.create(null);
2416
+ const disagreements = [];
2417
+ let agreed = 0;
2418
+ for (const { id, human, judge } of rows) {
2419
+ const label = perLabel[human] ??= {
2420
+ human: 0,
2421
+ agreed: 0
2422
+ };
2423
+ const row = confusion[human] ??= Object.create(null);
2424
+ label.human++;
2425
+ row[judge] = (row[judge] ?? 0) + 1;
2426
+ if (human === judge) {
2427
+ label.agreed++;
2428
+ agreed++;
2429
+ } else disagreements.push({
2430
+ id,
2431
+ human,
2432
+ judge
2433
+ });
2434
+ }
2435
+ const summary = {
2436
+ total: rows.length,
2437
+ accuracy: agreed / rows.length,
2438
+ perLabel,
2439
+ confusion,
2440
+ disagreements
2441
+ };
2442
+ if (positive === void 0) return summary;
2443
+ let positives = 0;
2444
+ let truePositives = 0;
2445
+ let negatives = 0;
2446
+ let trueNegatives = 0;
2447
+ for (const { human, judge } of rows) if (human === positive) {
2448
+ positives++;
2449
+ if (judge === positive) truePositives++;
2450
+ } else {
2451
+ negatives++;
2452
+ if (judge !== positive) trueNegatives++;
2453
+ }
2454
+ const tpr = positives === 0 ? void 0 : truePositives / positives;
2455
+ const tnr = negatives === 0 ? void 0 : trueNegatives / negatives;
2456
+ summary.binary = {
2457
+ tpr,
2458
+ tnr
1789
2459
  };
2460
+ if (tpr !== void 0 && tnr !== void 0 && tpr + tnr > 1) summary.correct = (p) => Math.min(1, Math.max(0, (p + tnr - 1) / (tpr + tnr - 1)));
2461
+ return summary;
1790
2462
  }
2463
+ //#endregion
2464
+ //#region src/compare-runs.ts
1791
2465
  /**
1792
- * Puts each group of concurrent tool calls in a stable order, so parallel tool
1793
- * calls that finished in a different order still compare equal. A tool call
1794
- * that started after the previous one settled begins a new group, so
1795
- * sequential calls keep their order.
2466
+ * Runs every scenario through a baseline and a candidate implementation and
2467
+ * labels each difference, so a port can show where it behaves like the code it
2468
+ * replaces and where it differs on purpose.
2469
+ *
2470
+ * Parity is outcome-level: the terminal outcome and the effects in the world,
2471
+ * not the sequence of tool calls, since a machine usually reorders steps. A
2472
+ * candidate that broke no rule where the baseline broke one is intended, even
2473
+ * when the outcome matches. A difference also counts as intended when only the
2474
+ * candidate reached the scenario's `expected` outcome. Every other difference,
2475
+ * and every candidate rule break, is a regression for a person to review.
2476
+ *
2477
+ * Runs are sequential, baseline first. A run that throws is recorded with
2478
+ * outcome `'threw'` and the error message as its `detail`. Parity holds for
2479
+ * these scenarios and these runs only; with a live model, run it more than once.
2480
+ *
2481
+ * @example
2482
+ * ```ts no-check
2483
+ * const report = await compareRuns({
2484
+ * scenarios,
2485
+ * baseline: async (s) => toParityRun(await runRawLoop(s)),
2486
+ * candidate: async (s) => toParityRun(await runDesk(s)),
2487
+ * });
2488
+ * expect(report.ok).toBe(true);
2489
+ * ```
1796
2490
  */
1797
- function normalize(effects) {
1798
- const result = [];
1799
- let batch = [];
1800
- const flush = () => {
1801
- result.push(...batch.sort((a, b) => stableStringify([a.name, a.input]).localeCompare(stableStringify([b.name, b.input]))));
1802
- batch = [];
2491
+ async function compareRuns(options) {
2492
+ const scenarios = [];
2493
+ const counts = {
2494
+ same: 0,
2495
+ intended: 0,
2496
+ regression: 0
1803
2497
  };
1804
- for (const effect of effects) {
1805
- if (effect.kind !== "tool" || !effect.concurrent) flush();
1806
- if (effect.kind === "tool") batch.push(effect);
1807
- else result.push(effect);
2498
+ for (const scenario of options.scenarios) {
2499
+ const baseline = await settle$1(() => options.baseline(scenario));
2500
+ const candidate = await settle$1(() => options.candidate(scenario));
2501
+ const { label, reason } = labelRun(scenario.expected, baseline, candidate);
2502
+ counts[label]++;
2503
+ scenarios.push({
2504
+ id: scenario.id,
2505
+ scenario,
2506
+ label,
2507
+ reason,
2508
+ baseline,
2509
+ candidate
2510
+ });
1808
2511
  }
1809
- flush();
1810
- return result;
2512
+ return {
2513
+ scenarios,
2514
+ counts,
2515
+ ok: counts.regression === 0
2516
+ };
1811
2517
  }
1812
- function describe(effect) {
1813
- switch (effect.kind) {
1814
- case "model": return `model call #${effect.call} (${effect.method})`;
1815
- case "tool": return `tool '${effect.name}' with ${JSON.stringify(effect.input)}`;
1816
- case "call": return `call #${effect.call} to '${effect.name}' with ${JSON.stringify(effect.args)}`;
1817
- case "output": return `output ${JSON.stringify(effect.value)}`;
2518
+ async function settle$1(run) {
2519
+ try {
2520
+ return await run();
2521
+ } catch (error) {
2522
+ return {
2523
+ outcome: "threw",
2524
+ detail: error instanceof Error ? error.message : String(error)
2525
+ };
1818
2526
  }
1819
2527
  }
1820
- /** Why two effects differ, or `undefined` when they are the same. */
1821
- function differ(left, right, options) {
1822
- if (left.kind !== right.kind) return {
1823
- reason: "kind",
1824
- message: `Expected ${describe(left)}; the port made ${describe(right)}.`
2528
+ function labelRun(expected, baseline, candidate) {
2529
+ if (candidate.violations?.length) return {
2530
+ label: "regression",
2531
+ reason: `candidate broke a rule: ${list(candidate.violations)}`
1825
2532
  };
1826
- if (left.kind === "model" && right.kind === "model") {
1827
- if (right.translated && right.replayed === left.call) return void 0;
1828
- if (left.method !== right.method) return {
1829
- reason: "model-method",
1830
- message: `Model call #${left.call} used ${left.method}; the port used ${right.method}.`
1831
- };
1832
- if (!modelCallsMatch(options.modelCalls, left, right)) return {
1833
- reason: "model-call-shape",
1834
- message: shapeMessage(left, right)
1835
- };
1836
- if (!jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) {
1837
- const outcome = (error) => error ? `failed with ${error.name}: ${error.message}` : "succeeded";
1838
- return {
1839
- reason: "model-error",
1840
- message: `Model call #${left.call} ${outcome(left.error)}; in the port it ${outcome(right.error)}.`
1841
- };
1842
- }
1843
- return;
1844
- }
1845
- if (left.kind === "tool" && right.kind === "tool") {
1846
- if (left.name !== right.name) return {
1847
- reason: "tool-name",
1848
- message: `Expected ${describe(left)}; the port made ${describe(right)}.`
1849
- };
1850
- if (!jsonEqual(left.input, right.input)) return {
1851
- reason: "tool-input",
1852
- message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.input)}.`
1853
- };
1854
- if (!jsonEqual(left.output, right.output) || !jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) return {
1855
- reason: "tool-result",
1856
- message: `Tool '${left.name}' returned a different result.`
1857
- };
1858
- return;
1859
- }
1860
- if (left.kind === "call" && right.kind === "call") {
1861
- if (left.name !== right.name) return {
1862
- reason: "call-name",
1863
- message: `Expected ${describe(left)}; the port made ${describe(right)}.`
1864
- };
1865
- if (!jsonEqual(left.args, right.args)) return {
1866
- reason: "call-args",
1867
- message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.args)}.`
1868
- };
1869
- if (!jsonEqual(left.result, right.result) || !jsonEqual(left.stream, right.stream) || !jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) return {
1870
- reason: "call-result",
1871
- message: `Call #${left.call} to '${left.name}' returned a different result.`
1872
- };
1873
- return;
1874
- }
1875
- if (left.kind === "output" && right.kind === "output" && !jsonEqual(left.value, right.value)) return {
1876
- reason: "output",
1877
- message: `Expected ${describe(left)}; the port returned ${JSON.stringify(right.value)}.`
2533
+ if (candidate.outcome === "threw") return {
2534
+ label: "regression",
2535
+ reason: `candidate threw: ${String(candidate.detail)}`
1878
2536
  };
1879
- }
1880
- /**
1881
- * Compares two recordings and reports every effect that differs. `expected`
1882
- * is the original's recording, `actual` the port's. Effects that are the same
1883
- * on both sides are lined up first (longest common subsequence), so one extra
1884
- * call does not shift every effect after it; between them, effects of the same
1885
- * kind pair up and report what changed, and the rest are missing or extra.
1886
- */
1887
- function compareRecordings(expected, actual, options = {}) {
1888
- const select = (effects) => normalize(effects.flatMap((effect, index) => {
1889
- if (options.kinds && !options.kinds.includes(effect.kind)) return [];
1890
- const mapped = options.normalize ? options.normalize(effect, index, effects) : effect;
1891
- return mapped ? [mapped] : [];
1892
- }));
1893
- const a = select(expected.effects);
1894
- const b = select(actual.effects);
1895
- const same = a.map((left) => b.map((right) => !differ(left, right, options)));
1896
- const lcs = Array.from({ length: a.length + 1 }, () => Array.from({ length: b.length + 1 }, () => 0));
1897
- for (let i = a.length - 1; i >= 0; i--) for (let j = b.length - 1; j >= 0; j--) lcs[i][j] = same[i][j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
1898
- const divergences = [];
1899
- let lefts = [];
1900
- let rights = [];
1901
- const flushGap = (at) => {
1902
- const unpairedRights = [...rights];
1903
- const changed = [];
1904
- for (const index of lefts) {
1905
- const left = a[index];
1906
- const k = unpairedRights.findIndex((j) => b[j].kind === left.kind);
1907
- if (k === -1) {
1908
- changed.push({
1909
- index,
1910
- reason: left.kind === "output" ? "output" : "missing-effect",
1911
- expected: left,
1912
- message: left.kind === "output" ? `The port never returned; the original returned ${JSON.stringify(left.value)}.` : `The port never made ${describe(left)}.`
1913
- });
1914
- continue;
1915
- }
1916
- const right = b[unpairedRights.splice(k, 1)[0]];
1917
- const difference = differ(left, right, options);
1918
- if (difference) changed.push({
1919
- index,
1920
- expected: left,
1921
- actual: right,
1922
- ...difference
1923
- });
1924
- }
1925
- for (const j of unpairedRights) {
1926
- const extra = b[j];
1927
- divergences.push({
1928
- index: lefts[0] ?? at,
1929
- reason: extra.kind === "output" ? "output" : "extra-effect",
1930
- actual: extra,
1931
- message: extra.kind === "output" ? `The original never returned; the port returned ${JSON.stringify(extra.value)}.` : `The port made ${describe(extra)}, which the original never made.`
1932
- });
1933
- }
1934
- divergences.push(...changed);
1935
- lefts = [];
1936
- rights = [];
2537
+ if (baseline.violations?.length) return {
2538
+ label: "intended",
2539
+ reason: `baseline broke a rule: ${list(baseline.violations)}`
1937
2540
  };
1938
- let i = 0;
1939
- let j = 0;
1940
- while (i < a.length || j < b.length) if (i < a.length && j < b.length && same[i][j]) {
1941
- flushGap(i);
1942
- i++;
1943
- j++;
1944
- } else if (j < b.length && (i === a.length || lcs[i][j + 1] >= lcs[i + 1][j])) rights.push(j++);
1945
- else lefts.push(i++);
1946
- flushGap(i);
1947
- divergences.sort((x, y) => x.index - y.index);
1948
- return divergences.length ? {
1949
- equivalent: false,
1950
- divergence: divergences[0],
1951
- divergences
1952
- } : {
1953
- equivalent: true,
1954
- divergences
2541
+ const sameOutcome = baseline.outcome === candidate.outcome;
2542
+ if (sameOutcome && structurallyEqual(baseline.effects, candidate.effects)) return {
2543
+ label: "same",
2544
+ reason: ""
2545
+ };
2546
+ if (expected !== void 0 && candidate.outcome === expected && baseline.outcome !== expected) return {
2547
+ label: "intended",
2548
+ reason: `only the candidate reached the expected '${expected}'`
2549
+ };
2550
+ return {
2551
+ label: "regression",
2552
+ reason: sameOutcome ? "effects differ" : `outcome differs: ${baseline.outcome} → ${candidate.outcome}`
1955
2553
  };
1956
2554
  }
2555
+ function list(items) {
2556
+ return items.map((item) => typeof item === "string" ? item : JSON.stringify(item)).join(", ");
2557
+ }
2558
+ function structurallyEqual(a, b) {
2559
+ if (Object.is(a, b)) return true;
2560
+ if (a instanceof Date || b instanceof Date) return a instanceof Date && b instanceof Date && a.getTime() === b.getTime();
2561
+ if (a instanceof Map || b instanceof Map) return a instanceof Map && b instanceof Map && structurallyEqual([...a], [...b]);
2562
+ if (a instanceof Set || b instanceof Set) return a instanceof Set && b instanceof Set && structurallyEqual([...a], [...b]);
2563
+ if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => structurallyEqual(item, b[index]));
2564
+ if (!require_event_log.isRecord(a) || !require_event_log.isRecord(b)) return false;
2565
+ const keys = Object.keys(a);
2566
+ return keys.length === Object.keys(b).length && keys.every((key) => Object.hasOwn(b, key) && structurallyEqual(a[key], b[key]));
2567
+ }
1957
2568
  //#endregion
1958
2569
  //#region src/check-parity.ts
1959
2570
  /**
@@ -2063,6 +2674,7 @@ exports.AgentUnknownStateError = require_verify.AgentUnknownStateError;
2063
2674
  exports.RecordingDivergenceError = RecordingDivergenceError;
2064
2675
  exports.assertAgentMachine = require_verify.assertAgentMachine;
2065
2676
  exports.canReach = require_verify.canReach;
2677
+ exports.checkCrashRecovery = checkCrashRecovery;
2066
2678
  exports.checkParity = checkParity;
2067
2679
  exports.compareRecordings = compareRecordings;
2068
2680
  exports.compareRuns = compareRuns;