@statelyai/agent 2.0.0-alpha.35 → 2.0.0-alpha.37

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/CHANGELOG.md +138 -0
  2. package/dist/act-B9hLQls6.mjs +19 -0
  3. package/dist/act-CrX0JwWY.cjs +42 -0
  4. package/dist/ai-sdk.cjs +63 -7
  5. package/dist/ai-sdk.d.cts +13 -3
  6. package/dist/ai-sdk.d.mts +13 -3
  7. package/dist/ai-sdk.mjs +63 -7
  8. package/dist/{decision-BEsATcr1.cjs → decision-CMeuNZ1g.cjs} +262 -4
  9. package/dist/{decision-COkJNjOW.mjs → decision-m5hGmHpF.mjs} +239 -5
  10. package/dist/decision-object-B5zGSPFp.d.mts +15 -0
  11. package/dist/decision-object-DBcLWVvv.d.cts +15 -0
  12. package/dist/{run-agent-Ct_GZw_f.mjs → effects-BT_cSY0j.mjs} +1003 -109
  13. package/dist/{run-agent-s5D32w99.cjs → effects-DDseYtJk.cjs} +1037 -107
  14. package/dist/{event-log-C7jeH0y8.mjs → event-log-DkVlk3Rh.mjs} +30 -15
  15. package/dist/{event-log-BWSSLUff.cjs → event-log-Dkc-yIYu.cjs} +41 -14
  16. package/dist/{event-log-store-BzXehP7R.d.cts → event-log-store-BBS8so8a.d.mts} +2 -2
  17. package/dist/{event-log-store-W49TCJXi.mjs → event-log-store-Brx9tOHt.mjs} +1 -1
  18. package/dist/{event-log-store-CleWpsp5.cjs → event-log-store-Cl-YWDnq.cjs} +1 -1
  19. package/dist/{event-log-store-CWjN6lmM.d.mts → event-log-store-bc4ICGcS.d.cts} +2 -2
  20. package/dist/index.cjs +660 -286
  21. package/dist/index.d.cts +168 -9
  22. package/dist/index.d.mts +168 -9
  23. package/dist/index.mjs +639 -271
  24. package/dist/internal-C3Lqu538.d.cts +43 -0
  25. package/dist/internal-DUcdhk-i.d.mts +43 -0
  26. package/dist/log.cjs +2 -2
  27. package/dist/log.d.cts +1 -1
  28. package/dist/log.d.mts +1 -1
  29. package/dist/log.mjs +2 -2
  30. package/dist/machines.cjs +457 -41
  31. package/dist/machines.d.cts +114 -43
  32. package/dist/machines.d.mts +114 -43
  33. package/dist/machines.mjs +457 -42
  34. package/dist/{model-source-DmhTMK39.cjs → model-source-B4WQ8Idp.cjs} +1 -1
  35. package/dist/{model-source-CK3F4YuU.mjs → model-source-kjGwqAV8.mjs} +1 -1
  36. package/dist/openai.cjs +149 -36
  37. package/dist/openai.d.cts +28 -10
  38. package/dist/openai.d.mts +28 -10
  39. package/dist/openai.mjs +149 -36
  40. package/dist/otel.d.cts +1 -1
  41. package/dist/otel.d.mts +1 -1
  42. package/dist/{run-agent-BGdx5913.d.mts → run-agent-30mH2moE.d.cts} +91 -9
  43. package/dist/{run-agent-B1-Eqviq.d.cts → run-agent-DlPzBWj-.d.mts} +91 -9
  44. package/dist/{setup-agent-05uw4dPZ.mjs → setup-agent-BDmNr6PC.mjs} +9 -3
  45. package/dist/{setup-agent-BtfeFfF6.d.cts → setup-agent-BaNyz6Q3.d.cts} +4 -4
  46. package/dist/{setup-agent-C6nqzv_X.d.mts → setup-agent-C1CG_nfX.d.mts} +4 -4
  47. package/dist/{setup-agent-CXEBURdo.cjs → setup-agent-CmEToB8y.cjs} +9 -3
  48. package/dist/tanstack-ai.cjs +62 -4
  49. package/dist/tanstack-ai.d.cts +18 -6
  50. package/dist/tanstack-ai.d.mts +18 -6
  51. package/dist/tanstack-ai.mjs +62 -4
  52. package/dist/testing.cjs +2493 -1502
  53. package/dist/testing.d.cts +360 -256
  54. package/dist/testing.d.mts +360 -256
  55. package/dist/testing.mjs +2489 -1503
  56. package/dist/{text-logic-CV_4KGr6.d.cts → text-logic-BqxalgMr.d.mts} +87 -4
  57. package/dist/{text-logic-Ckh-rJFV.d.mts → text-logic-CSf9bqpP.d.cts} +87 -4
  58. package/dist/{types-Dp5_Ru4R.d.cts → types-DQUOOaZO.d.cts} +1 -1
  59. package/dist/{types-Dp5_Ru4R.d.mts → types-DQUOOaZO.d.mts} +1 -1
  60. package/dist/{usage-BDiJKyv9.d.cts → usage-Bm5nvpow.d.cts} +6 -1
  61. package/dist/{usage-DOoVUGVq.d.mts → usage-BnikdOca.d.mts} +6 -1
  62. package/dist/utils-7lSa4H-i.d.cts +403 -0
  63. package/dist/utils-CBbKw2fM.d.mts +403 -0
  64. package/dist/validate.cjs +44 -12
  65. package/dist/validate.d.cts +6 -6
  66. package/dist/validate.d.mts +6 -6
  67. package/dist/validate.mjs +44 -12
  68. package/dist/verify-ChyX6jq0.mjs +2306 -0
  69. package/dist/verify-CwlombbC.cjs +2359 -0
  70. package/dist/{verify-BQpE8hUz.d.cts → verify-DJHfEfm6.d.cts} +213 -14
  71. package/dist/{verify-CqeuEmIk.d.mts → verify-i9Hiwc5g.d.mts} +213 -14
  72. package/dist/with-model-calls-B_WLmOSl.cjs +248 -0
  73. package/dist/with-model-calls-BkOwdroS.mjs +213 -0
  74. package/docs/act.md +102 -0
  75. package/docs/advanced.md +105 -0
  76. package/docs/any-stack.md +62 -0
  77. package/docs/choosing-a-run-mode.md +55 -0
  78. package/docs/debugging.md +29 -0
  79. package/docs/decisions.md +473 -0
  80. package/docs/environment-and-policy.md +188 -0
  81. package/docs/evals.md +509 -0
  82. package/docs/event-log.md +312 -0
  83. package/docs/from-a-loop.md +528 -0
  84. package/docs/generate-machines.md +110 -0
  85. package/docs/hosts.md +414 -0
  86. package/docs/human-in-the-loop.md +177 -0
  87. package/docs/index.md +48 -0
  88. package/docs/judgments.md +281 -0
  89. package/docs/langgraph-comparison.md +198 -0
  90. package/docs/machines-as-data.md +454 -0
  91. package/docs/machines-presets.md +198 -0
  92. package/docs/machines.md +497 -0
  93. package/docs/messages.md +67 -0
  94. package/docs/meta.json +52 -0
  95. package/docs/models-and-providers.md +268 -0
  96. package/docs/multi-agent.md +68 -0
  97. package/docs/observability.md +109 -0
  98. package/docs/patterns.md +31 -0
  99. package/docs/persistence.md +207 -0
  100. package/docs/quickstart.md +121 -0
  101. package/docs/roadmap.md +22 -0
  102. package/docs/scope.md +75 -0
  103. package/docs/statechart-policy-examples.md +69 -0
  104. package/docs/steps.md +56 -0
  105. package/docs/strangler-fig.md +323 -0
  106. package/docs/text-requests.md +250 -0
  107. package/docs/thinking-in-state-machines.md +324 -0
  108. package/docs/tools.md +249 -0
  109. package/docs/upgrading.md +37 -0
  110. package/docs/usage-and-budgets.md +518 -0
  111. package/docs/validate-and-repair.md +253 -0
  112. package/docs/verify.md +492 -0
  113. package/docs/xstate-4-and-5-apps.md +161 -0
  114. package/llms.txt +53 -0
  115. package/package.json +10 -3
  116. package/readme.md +30 -15
  117. package/skills/add-evals/SKILL.md +3 -1
  118. package/skills/add-traces/SKILL.md +6 -1
  119. package/skills/build-machine/SKILL.md +84 -0
  120. package/skills/generate-machine/SKILL.md +1 -1
  121. package/skills/graph-engineering/SKILL.md +13 -3
  122. package/skills/improve-machine/SKILL.md +21 -2
  123. package/skills/loop-to-machine/SKILL.md +147 -192
  124. package/skills/loop-to-machine/reference.md +260 -0
  125. package/dist/utils-BnXdWih3.d.mts +0 -108
  126. package/dist/utils-ebVpCYoF.d.cts +0 -108
  127. package/dist/verify-Bpahmr34.mjs +0 -1241
  128. package/dist/verify-DCLZa9R0.cjs +0 -1288
  129. package/dist/with-model-calls-BXYdBdcu.cjs +0 -45
  130. package/dist/with-model-calls-Bv4V4EHC.mjs +0 -34
package/dist/testing.mjs CHANGED
@@ -1,1685 +1,2671 @@
1
- import { F as getCallUsage, L as normalizeGeneratorResult, d as canonicalEventJson, dt as isPlainObject, ft as isRecord, pt as AgentError, tt as getStateMeta, ut as isError } from "./event-log-C7jeH0y8.mjs";
2
- import { p as runToQuiescence, s as createAgentRuntime } from "./run-agent-Ct_GZw_f.mjs";
3
- import { a as explorePaths, c as simulateAgent, i as canReach, n as AgentUnknownStateError, o as isExplorationComplete, r as assertAgentMachine, s as lintAgentMachine, t as AgentLintError } from "./verify-Bpahmr34.mjs";
1
+ import { L as getCallUsage, R as isTextLogic, S as replay, c as agentCallSite, d as canonicalEventJson, ft as isError, ht as AgentError, mt as isRecord, pt as isPlainObject, rt as getStateMeta, z as normalizeGeneratorResult } from "./event-log-DkVlk3Rh.mjs";
2
+ import { s as isDecisionLogic } from "./decision-m5hGmHpF.mjs";
3
+ import { i as isActLogic } from "./act-B9hLQls6.mjs";
4
+ import { _ as runToQuiescence, d as createAgentRuntime, h as isStateMachineLogic, n as mapEffects, t as isAsyncIterable, y as toJsonValue } from "./effects-BT_cSY0j.mjs";
5
+ import { n as createInMemoryEventLogStore } from "./event-log-store-Brx9tOHt.mjs";
6
+ import { a as explorePaths, c as simulateAgent, i as canReach, l as textOutput, n as AgentUnknownStateError, o as isExplorationComplete, r as assertAgentMachine, s as lintAgentMachine, t as AgentLintError } from "./verify-ChyX6jq0.mjs";
4
7
  import { getInitialMicrosteps, getMicrosteps } from "xstate";
5
- //#region src/trajectory.ts
6
- /**
7
- * Classifies one item. Event-shaped things (log entries, events) become events;
8
- * everything else is a state value. A snapshot is unwrapped to its `value`.
9
- * @internal
10
- */
11
- function normalize$1(item) {
12
- if (isRecord(item)) {
13
- const nested = item["event"];
14
- if (isRecord(nested) && typeof nested["type"] === "string") return {
15
- kind: "event",
16
- event: nested
17
- };
18
- if (typeof item["type"] === "string") return {
19
- kind: "event",
20
- event: item
21
- };
22
- if ("value" in item) return {
23
- kind: "state",
24
- value: item["value"]
25
- };
26
- }
27
- return {
28
- kind: "state",
29
- value: item
30
- };
31
- }
32
- /** Every leaf of a state value as a dot path (`{ review: 'editing' }` → `review.editing`). @internal */
33
- function statePaths$1(value) {
34
- if (typeof value === "string") return [value];
35
- if (!isRecord(value)) return [];
36
- return Object.entries(value).flatMap(([key, child]) => statePaths$1(child).map((path) => `${key}.${path}`));
37
- }
38
- /**
39
- * A string expectation matches a state value when it names one of its leaves or
40
- * an ancestor of one, so `'review'` matches `{ review: 'editing' }`. @internal
41
- */
42
- function stateMatchesPath(path, value) {
43
- return statePaths$1(value).some((leaf) => leaf === path || leaf.startsWith(`${path}.`));
44
- }
45
- function deepEqual(a, b) {
46
- if (Object.is(a, b)) return true;
47
- if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => deepEqual(item, b[index]));
48
- if (!isRecord(a) || !isRecord(b)) return false;
49
- const keys = Object.keys(a);
50
- return keys.length === Object.keys(b).length && keys.every((key) => deepEqual(a[key], b[key]));
51
- }
52
- /**
53
- * Normalizes ONE expected item into a predicate over actual items, so the
54
- * expectation is classified once instead of once per candidate:
55
- *
56
- * - A string expectation matches an event's `type`, or a state value it names.
57
- * - An event expectation matches an event with the same `type` whose other
58
- * declared keys deep-equal, so payload details are opt-in.
59
- * - A state-value expectation matches structurally.
60
- * @internal
61
- */
62
- function matcherFor(expected) {
63
- if (typeof expected === "string") return (actual) => {
64
- const actualItem = normalize$1(actual);
65
- return actualItem.kind === "event" ? actualItem.event.type === expected : stateMatchesPath(expected, actualItem.value);
66
- };
67
- const expectedItem = normalize$1(expected);
68
- switch (expectedItem.kind) {
69
- case "event": return (actual) => {
70
- const actualItem = normalize$1(actual);
71
- return actualItem.kind === "event" && Object.keys(expectedItem.event).every((key) => deepEqual(expectedItem.event[key], actualItem.event[key]));
72
- };
73
- case "state": return (actual) => {
74
- const actualItem = normalize$1(actual);
75
- return actualItem.kind === "state" && deepEqual(expectedItem.value, actualItem.value);
76
- };
77
- }
78
- }
8
+ //#region src/equivalence.ts
79
9
  /**
80
- * Matches a run's trajectory against an expected one.
10
+ * Verified equivalence — prove a port behaves like the code it replaces.
81
11
  *
82
- * Both trajectories may be state values or events collected from
83
- * `onTransition`;
84
- * items are compared by shape, so `['prompting', 'drafting']` scores a state
85
- * path and `['PROMPT_SUBMITTED', 'SEND']` scores an event log with the same
86
- * call.
12
+ * A port is equivalent when, given the same model answers and the same tool
13
+ * results, it produces the same OBSERVABLE EFFECTS: the same number of model
14
+ * calls, the same tool calls with the same arguments, and the same output.
15
+ * Prompts, internal steps, and states are free to differ.
87
16
  *
88
- * Semantics are an ordered subsequence: every expected item must appear in
89
- * `actual`, in order, with gaps allowed. Pass `{ exact: true }` to require
90
- * equality instead.
17
+ * 1. Record the original: wrap its model (`recorder.model`, an AI SDK
18
+ * `LanguageModelV3`, or `recorder.openai`, an `openai` client) and its tools
19
+ * (`recorder.tool` / `recorder.tools`), run a scenario, call
20
+ * `recorder.output(result)`, and save `recorder.recording`.
21
+ * 2. Replay it on the port: `createRecorder({ replay: recording })` answers
22
+ * every model call and tool call from the recording, with no API key and no
23
+ * side effects, and records what the port did.
24
+ * 3. `compareRecordings(original, port)` reports the first effect that differs.
91
25
  *
92
- * @example Trajectory assertions in a test
93
- * ```ts
94
- * const statePath: unknown[] = [];
95
- * const result = await runToQuiescence(
96
- * createAgentRuntime(machine, {
97
- * executors,
98
- * onTransition: (snapshot) => statePath.push(snapshot.value),
99
- * }),
100
- * { input },
101
- * );
26
+ * A port that changes how it calls the model on purpose (a model-picked action
27
+ * that became `agent.decide`, a stream that became `doGenerate`) still replays:
28
+ * `translate` turns the recorded answer into one that fits, `fallback` answers
29
+ * the calls the original never made, and `compareRecordings(…, { kinds })`
30
+ * compares only tool calls and output.
102
31
  *
103
- * const path = matchesTrajectory(statePath, ['prompting', 'drafting', 'sent']);
104
- * expect(path.matched, JSON.stringify(path.firstMiss)).toBe(true);
105
- * expect(matchesTrajectory(events, ['PROMPT_SUBMITTED', 'SEND']).matched).toBe(true);
106
- * ```
32
+ * Any other client (`@anthropic-ai/sdk`, `@google/genai`, a framework's own
33
+ * model client) records through `recorder.client` or `recorder.fn`: each call
34
+ * is a `call` effect, answered in replay by name and order.
107
35
  *
108
- * @example Partial credit in an eval scorer
109
- * ```ts
110
- * const match = matchesTrajectory(output.statePath, expected.statePath);
111
- * return { name: 'state_path', score: match.score, metadata: { ...match.firstMiss } };
112
- * ```
36
+ * Provider-free: every seam is wrapped structurally, so this module imports
37
+ * neither `ai` nor `openai`.
38
+ * @module
113
39
  */
114
- function matchesTrajectory(actual, expected, options = {}) {
115
- const expectedCount = expected.length;
116
- if (expectedCount === 0) throw new AgentError("empty-trajectory", "matchesTrajectory: expected trajectory must contain at least one item.");
117
- const exact = options.exact === true;
118
- let cursor = 0;
119
- let matchedCount = 0;
120
- let firstMiss;
121
- for (let index = 0; index < expectedCount; index++) {
122
- const want = expected[index];
123
- const matches = matcherFor(want);
124
- const searchedFrom = exact ? index : cursor;
125
- let found = -1;
126
- for (let at = searchedFrom; at < actual.length; at++) {
127
- if (matches(actual[at])) {
128
- found = at;
129
- break;
130
- }
131
- if (exact) break;
132
- }
133
- if (found === -1) {
134
- firstMiss = {
135
- index,
136
- expected: want,
137
- searchedFrom
138
- };
139
- break;
140
- }
141
- matchedCount++;
142
- cursor = found + 1;
143
- }
144
- const matched = !firstMiss && (!exact || actual.length === expectedCount);
145
- const denominator = exact ? Math.max(expectedCount, actual.length) : expectedCount;
146
- return {
147
- matched,
148
- matchedCount,
149
- expectedCount,
150
- score: matched ? 1 : denominator === 0 ? 1 : matchedCount / denominator,
151
- ...firstMiss ? { firstMiss } : {}
152
- };
153
- }
154
- //#endregion
155
- //#region src/seam.ts
156
- /** The only own keys an executor result carries. @internal */
157
- const TEXT_RESULT_KEYS = /* @__PURE__ */ new Set([
158
- "result",
159
- "messages",
160
- "usage",
161
- "raw"
162
- ]);
163
40
  /**
164
- * True when a scripted entry is the executor result rather than the output
165
- * value: it owns a `result` key and owns NOTHING outside the result's own
166
- * vocabulary. Bare `'result' in value` would swallow an output object's
167
- * siblings (`{ result: 'draft', confidence: 0.9 }` would lose `confidence`)
168
- * and would also match an inherited `result`. @internal
41
+ * Thrown in replay when the port asks for something the recording cannot
42
+ * answer: a model call past the last recorded one, or a tool call the original
43
+ * never made. The effect is still recorded, so {@link compareRecordings}
44
+ * reports it.
169
45
  */
170
- function isTextResult(value) {
171
- return Object.hasOwn(value, "result") && Object.keys(value).every((key) => TEXT_RESULT_KEYS.has(key));
46
+ var RecordingDivergenceError = class extends Error {
47
+ name = "RecordingDivergenceError";
48
+ };
49
+ function toRecordedError(error) {
50
+ if (!(error instanceof Error)) return {
51
+ name: "Error",
52
+ message: String(error)
53
+ };
54
+ const recorded = {
55
+ name: error.name,
56
+ message: error.message
57
+ };
58
+ if (Object.prototype.propertyIsEnumerable.call(error, "name")) recorded.ownName = true;
59
+ const code = error.code;
60
+ if (code !== void 0) recorded.code = toJson(code);
61
+ const data = Object.fromEntries(Object.entries(error).filter(([key, value]) => key !== "code" && (typeof value === "number" || typeof value === "boolean")));
62
+ if (Object.keys(data).length) recorded.data = data;
63
+ const markers = /* @__PURE__ */ new Set();
64
+ for (let holder = error; holder; holder = Object.getPrototypeOf(holder)) for (const symbol of Object.getOwnPropertySymbols(holder)) {
65
+ const key = Symbol.keyFor(symbol);
66
+ if (key !== void 0 && error[symbol] === true) markers.add(key);
67
+ }
68
+ if (markers.size) recorded.markers = [...markers];
69
+ return recorded;
172
70
  }
173
- /**
174
- * Resolves ONE scripted entry to an executor result: a function entry is
175
- * called with the request, and the value is taken as the executor result only
176
- * when it is one (see {@link isTextResult}). @internal
177
- */
178
- async function resolveSeamScriptEntry(entry, request, info) {
179
- const value = typeof entry === "function" ? await entry(request, info) : entry;
180
- return isRecord(value) && isTextResult(value) ? value : { result: value };
71
+ function fromRecordedError({ data, markers, name, ownName, ...fields }) {
72
+ const error = Object.assign(new Error(fields.message), fields, data);
73
+ if (ownName || name !== "Error") Object.defineProperty(error, "name", {
74
+ value: name,
75
+ writable: true,
76
+ configurable: true,
77
+ enumerable: ownName ?? false
78
+ });
79
+ for (const marker of markers ?? []) error[Symbol.for(marker)] = true;
80
+ return error;
181
81
  }
182
- /** Names a pending text request in an error message. @internal */
183
- function describeText(request) {
184
- return request.name ? `'${request.name}' (model '${request.model}')` : `(model '${request.model}')`;
82
+ /** What two runs compare of an error: its data and markers may differ. */
83
+ function errorIdentity$1(error) {
84
+ return error && {
85
+ name: error.name,
86
+ message: error.message,
87
+ code: error.code
88
+ };
185
89
  }
186
- /**
187
- * Stream semantics with no model: the whole text lands as one chunk. @internal
188
- */
189
- function emitScriptedChunk(result, info) {
190
- const text = isRecord(result) ? result["result"] : void 0;
191
- if (typeof text === "string") info?.onChunk?.(text);
90
+ /** A JSON-safe copy: Dates become ISO strings, Errors become {@link RecordedError}. */
91
+ function toJson(value) {
92
+ if (value === void 0) return void 0;
93
+ return JSON.parse(JSON.stringify(value, function(key, item) {
94
+ const raw = this[key];
95
+ if (raw instanceof Date) return raw.toISOString();
96
+ if (item instanceof Error) return toRecordedError(item);
97
+ if (typeof item === "bigint") return item.toString();
98
+ if (item instanceof Uint8Array) return Array.from(item);
99
+ return item;
100
+ }));
101
+ }
102
+ /** Restores the `Date` timestamps the AI SDK puts on responses. */
103
+ function reviveTimestamps(value) {
104
+ if (value && typeof value === "object") {
105
+ for (const holder of [value, value.response]) if (holder && typeof holder["timestamp"] === "string") holder["timestamp"] = new Date(holder["timestamp"]);
106
+ }
107
+ return value;
108
+ }
109
+ function omit(value, keys) {
110
+ if (!value || typeof value !== "object") return value;
111
+ return Object.fromEntries(Object.entries(value).filter(([key]) => !keys.includes(key)));
192
112
  }
193
113
  /**
194
- * The seam's own answer, for scoring: the executor result's `result`.
114
+ * Key-order-independent JSON, for equality.
195
115
  * @internal
196
116
  */
197
- async function seamOutputOf(result, request) {
198
- if (!isRecord(result)) return;
199
- return await normalizeGeneratorResult(result, `seam '${request.name ?? request.model}'`);
117
+ function stableStringify(value) {
118
+ if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
119
+ if (value && typeof value === "object") return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`;
120
+ return JSON.stringify(value) ?? "undefined";
121
+ }
122
+ function jsonEqual(a, b) {
123
+ return stableStringify(toJson(a)) === stableStringify(toJson(b));
124
+ }
125
+ function streamFrom(parts) {
126
+ return new ReadableStream({ start(controller) {
127
+ for (const part of parts) controller.enqueue(reviveTimestamps(structuredClone(part)));
128
+ controller.close();
129
+ } });
130
+ }
131
+ async function* iterate(chunks) {
132
+ for (const chunk of chunks) yield structuredClone(chunk);
200
133
  }
201
134
  /**
202
- * Runs a machine end to end with one model call under test, and slices the run
203
- * at that call.
204
- *
205
- * The `before`/`after` slices are the point: `after.statePath` is the branch
206
- * the seam chose and `after.events` is the same question against the durable
207
- * log, both ready for `matchesTrajectory`. The state slice splits where the
208
- * state path stood when the seam answered; the event slice splits at the
209
- * seam's own effect completion (the first `xstate.done.*`/`xstate.error.*`
210
- * entry appended after the call was made).
211
- *
212
- * @example No API key: the seam is scripted too, so the whole thing runs offline.
213
- * ```ts
214
- * const run = await runSeam(emailDrafter, {
215
- * scripts: { writeFollowUps: [followUps], emailDrafter: [draft] },
216
- * seam: { request: 'writeFollowUps' },
217
- * respond: ({ state }) => (state === 'prompting' ? { type: 'PROMPT_SUBMITTED', prompt } : null),
218
- * });
135
+ * Creates a {@link Recorder}. With no options it records: every wrapped call
136
+ * runs for real. With `replay` it answers from a recording instead.
137
+ */
138
+ function createRecorder(options = {}) {
139
+ const effects = [];
140
+ const replay = options.replay?.effects;
141
+ const replayedModels = replay?.filter((effect) => effect.kind === "model");
142
+ const usedModels = /* @__PURE__ */ new Set();
143
+ const usedTools = /* @__PURE__ */ new Set();
144
+ const usedCalls = /* @__PURE__ */ new Set();
145
+ const callCounts = /* @__PURE__ */ new Map();
146
+ const normalizeEffect = (effect, list) => options.normalize?.(effect, list.indexOf(effect), list) ?? effect;
147
+ let modelCalls = 0;
148
+ let toolsInFlight = 0;
149
+ /**
150
+ * The earliest unused recorded call that fits this one: one `translate`
151
+ * adapts, or one with the same method and shape. Not by position, so a call
152
+ * the port added or dropped does not shift every answer after it.
153
+ */
154
+ function nextReplayedModel(effect) {
155
+ for (const recorded of replayedModels) {
156
+ if (usedModels.has(recorded)) continue;
157
+ const translated = options.translate?.(structuredClone(recorded), effect);
158
+ if (translated) {
159
+ effect.translated = true;
160
+ return {
161
+ recorded,
162
+ answer: translated
163
+ };
164
+ }
165
+ if (recorded.method === effect.method && modelCallsMatch(options.modelCalls, recorded, effect)) return {
166
+ recorded,
167
+ answer: recorded
168
+ };
169
+ }
170
+ }
171
+ function noAnswerMessage(effect) {
172
+ const unused = replayedModels.filter((recorded) => !usedModels.has(recorded));
173
+ return unused.length ? `Model call #${effect.call} (${effect.method}, ${JSON.stringify(modelCallShape(effect))}) fits no unused recorded call: ${unused.map((recorded) => `#${recorded.call} (${recorded.method}, ${JSON.stringify(modelCallShape(recorded))})`).join(", ")}. A recorded answer only fits a call of the same method and shape; pass \`translate\` to adapt one.` : `Model call #${effect.call} was never made by the recorded run (it made ${replayedModels.length}).`;
174
+ }
175
+ /**
176
+ * Records a model call around `run`, or answers it from the recording.
177
+ * `fallback` runs it on the stand-in when the recording has no answer.
178
+ */
179
+ async function modelCall(seam, method, request, streamed, run, fallback) {
180
+ const effect = {
181
+ kind: "model",
182
+ seam,
183
+ method,
184
+ call: modelCalls++,
185
+ request: toJson(omit(request, [
186
+ "abortSignal",
187
+ "headers",
188
+ "signal"
189
+ ]))
190
+ };
191
+ effects.push(effect);
192
+ if (replay) {
193
+ const replayed = nextReplayedModel(effect);
194
+ if (replayed) {
195
+ const { recorded, answer } = replayed;
196
+ usedModels.add(recorded);
197
+ effect.replayed = recorded.call;
198
+ if (answer.error) {
199
+ effect.error = answer.error;
200
+ throw fromRecordedError(answer.error);
201
+ }
202
+ effect.response = answer.response;
203
+ effect.stream = answer.stream;
204
+ if (seam === "openai") return streamed ? iterate(answer.stream ?? []) : structuredClone(answer.response);
205
+ return streamed ? {
206
+ ...reviveTimestamps(structuredClone(answer.response) ?? {}),
207
+ stream: streamFrom(answer.stream ?? [])
208
+ } : reviveTimestamps(structuredClone(answer.response));
209
+ }
210
+ if (!fallback) {
211
+ const error = new RecordingDivergenceError(noAnswerMessage(effect));
212
+ effect.error = toRecordedError(error);
213
+ throw error;
214
+ }
215
+ effect.unrecorded = true;
216
+ run = fallback;
217
+ }
218
+ let result;
219
+ try {
220
+ result = await run();
221
+ } catch (error) {
222
+ effect.error = toRecordedError(error);
223
+ throw error;
224
+ }
225
+ if (!streamed) {
226
+ effect.response = toJson(result);
227
+ return result;
228
+ }
229
+ const parts = [];
230
+ effect.stream = parts;
231
+ if (seam === "openai") {
232
+ const source = result;
233
+ return (async function* () {
234
+ for await (const chunk of source) {
235
+ parts.push(toJson(chunk));
236
+ yield chunk;
237
+ }
238
+ })();
239
+ }
240
+ const { stream, ...rest } = result;
241
+ effect.response = toJson(rest);
242
+ return {
243
+ ...rest,
244
+ stream: stream.pipeThrough(new TransformStream({ transform(part, controller) {
245
+ parts.push(toJson(part));
246
+ controller.enqueue(part);
247
+ } }))
248
+ };
249
+ }
250
+ /** `inputOf` picks the recorded input from the call's arguments. */
251
+ function tool(name, fn, inputOf = (args) => args.length === 1 ? args[0] : args) {
252
+ return ((...args) => {
253
+ const input = toJson(inputOf(args));
254
+ const effect = {
255
+ kind: "tool",
256
+ name,
257
+ step: modelCalls,
258
+ input,
259
+ ...toolsInFlight > 0 ? { concurrent: true } : {}
260
+ };
261
+ effects.push(effect);
262
+ return track(call(effect, args));
263
+ });
264
+ function call(effect, args) {
265
+ const { input } = effect;
266
+ if (replay && options.tools !== "live") {
267
+ const wanted = normalizeEffect(effect, effects).input;
268
+ const matches = (candidate) => candidate.kind === "tool" && !usedTools.has(candidate) && candidate.name === name && jsonEqual(normalizeEffect(candidate, replay).input, wanted);
269
+ const recorded = replay.find(matches);
270
+ if (!recorded && options.fallback?.tools) {
271
+ effect.unrecorded = true;
272
+ return runLive(effect, args);
273
+ }
274
+ const settle = (value) => {
275
+ if (recorded?.sync) return value();
276
+ try {
277
+ return Promise.resolve(value());
278
+ } catch (error) {
279
+ return Promise.reject(error);
280
+ }
281
+ };
282
+ return settle(() => {
283
+ if (!recorded) {
284
+ const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)}; the recorded run made no such call.`);
285
+ effect.error = toRecordedError(error);
286
+ throw error;
287
+ }
288
+ usedTools.add(recorded);
289
+ if (recorded.sync) effect.sync = true;
290
+ if (recorded.error) {
291
+ effect.error = recorded.error;
292
+ throw fromRecordedError(recorded.error);
293
+ }
294
+ effect.output = recorded.output;
295
+ return structuredClone(recorded.output);
296
+ });
297
+ }
298
+ return runLive(effect, args);
299
+ }
300
+ function runLive(effect, args) {
301
+ let output;
302
+ try {
303
+ output = fn(...args);
304
+ } catch (error) {
305
+ effect.sync = true;
306
+ effect.error = toRecordedError(error);
307
+ throw error;
308
+ }
309
+ if (!(output instanceof Promise)) {
310
+ effect.sync = true;
311
+ effect.output = toJson(output);
312
+ return output;
313
+ }
314
+ return output.then((value) => {
315
+ effect.output = toJson(value);
316
+ return value;
317
+ }, (error) => {
318
+ effect.error = toRecordedError(error);
319
+ throw error;
320
+ });
321
+ }
322
+ }
323
+ function fn(name, run, callOptions = {}) {
324
+ const { redact, serialize = toJson, revive } = callOptions;
325
+ return (async (...args) => {
326
+ const passed = [...args];
327
+ while (passed.length && passed.at(-1) === void 0) passed.pop();
328
+ const call = callCounts.get(name) ?? 0;
329
+ callCounts.set(name, call + 1);
330
+ const effect = {
331
+ kind: "call",
332
+ name,
333
+ call,
334
+ args: toJson(redact ? redact(passed) : passed)
335
+ };
336
+ effects.push(effect);
337
+ if (replay) {
338
+ const recorded = replay.find((candidate) => candidate.kind === "call" && candidate.name === name && !usedCalls.has(candidate));
339
+ if (!recorded) {
340
+ const error = new RecordingDivergenceError(`Call #${call} to '${name}' was never made by the recorded run.`);
341
+ effect.error = toRecordedError(error);
342
+ throw error;
343
+ }
344
+ usedCalls.add(recorded);
345
+ if (recorded.error) {
346
+ effect.error = recorded.error;
347
+ throw fromRecordedError(recorded.error);
348
+ }
349
+ if (recorded.stream) {
350
+ effect.stream = recorded.stream;
351
+ const chunks = structuredClone(recorded.stream);
352
+ return revive ? revive(chunks) : iterate(chunks);
353
+ }
354
+ effect.result = recorded.result;
355
+ const result = structuredClone(recorded.result);
356
+ return revive ? revive(result) : result;
357
+ }
358
+ let result;
359
+ try {
360
+ result = await run(...args);
361
+ } catch (error) {
362
+ effect.error = toRecordedError(error);
363
+ throw error;
364
+ }
365
+ if (!isAsyncIterable(result)) {
366
+ effect.result = toJson(serialize(result));
367
+ return result;
368
+ }
369
+ const chunks = [];
370
+ effect.stream = chunks;
371
+ const source = result;
372
+ return (async function* () {
373
+ try {
374
+ for await (const chunk of source) {
375
+ chunks.push(toJson(serialize(chunk)));
376
+ yield chunk;
377
+ }
378
+ } catch (error) {
379
+ effect.error = toRecordedError(error);
380
+ throw error;
381
+ }
382
+ })();
383
+ });
384
+ }
385
+ /** Counts an async tool call as running until it settles. */
386
+ function track(result) {
387
+ if (result instanceof Promise) {
388
+ toolsInFlight++;
389
+ const settled = () => {
390
+ toolsInFlight--;
391
+ };
392
+ result.then(settled, settled);
393
+ }
394
+ return result;
395
+ }
396
+ return {
397
+ model(model) {
398
+ const base = model;
399
+ const stand = options.fallback?.model;
400
+ const wrapped = Object.create(base);
401
+ for (const method of [
402
+ "doGenerate",
403
+ "doStream",
404
+ "doDecide",
405
+ "doEvaluate"
406
+ ]) {
407
+ if (typeof base[method] !== "function") continue;
408
+ const invoke = (target, callOptions) => target[method].call(target, callOptions);
409
+ wrapped[method] = (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0);
410
+ }
411
+ return wrapped;
412
+ },
413
+ openai(client) {
414
+ const wrapCreate = (resource, method) => new Proxy(resource, { get(target, key, receiver) {
415
+ if (key !== "create") return Reflect.get(target, key, receiver);
416
+ return (params, requestOptions) => {
417
+ let api;
418
+ const create = (resource) => () => {
419
+ api = Reflect.get(resource, "create").call(resource, params, requestOptions);
420
+ return Promise.resolve(api);
421
+ };
422
+ const stand = fallbackResource(method);
423
+ const promise = modelCall("openai", method, params, params?.stream === true, create(target), stand && create(stand));
424
+ return Object.assign(promise, { withResponse: async () => {
425
+ const data = await promise;
426
+ const meta = await api?.withResponse?.();
427
+ return {
428
+ data,
429
+ response: meta?.response ?? new Response(null),
430
+ request_id: meta?.request_id ?? null
431
+ };
432
+ } });
433
+ };
434
+ } });
435
+ const fallbackResource = (method) => {
436
+ let value = options.fallback?.openai;
437
+ for (const key of method.split(".").slice(0, -1)) value = value?.[key];
438
+ return value && typeof value === "object" ? value : void 0;
439
+ };
440
+ const wrapped = /* @__PURE__ */ new Map();
441
+ const wrap = (key, make) => {
442
+ if (!wrapped.has(key)) wrapped.set(key, make());
443
+ return wrapped.get(key);
444
+ };
445
+ return new Proxy(client, { get(target, key, receiver) {
446
+ const value = Reflect.get(target, key, receiver);
447
+ if (key === "responses" && value) return wrap(key, () => wrapCreate(value, "responses.create"));
448
+ if (key === "chat" && value) return wrap(key, () => new Proxy(value, { get(chat, chatKey, chatReceiver) {
449
+ const inner = Reflect.get(chat, chatKey, chatReceiver);
450
+ return chatKey === "completions" && inner ? wrap("chat.completions", () => wrapCreate(inner, "chat.completions.create")) : inner;
451
+ } }));
452
+ return value;
453
+ } });
454
+ },
455
+ tool: (name, fn) => tool(name, fn),
456
+ fn,
457
+ client(name, client, { methods, ...callOptions }) {
458
+ const paths = methods.map((method) => method.split("."));
459
+ const startsWith = (path, prefix) => prefix.every((key, index) => path[index] === key);
460
+ const wrapAt = (target, prefix) => {
461
+ const wrapped = /* @__PURE__ */ new Map();
462
+ return new Proxy(target, { get(object, key, receiver) {
463
+ if (typeof key !== "string") return Reflect.get(object, key, receiver);
464
+ const path = [...prefix, key];
465
+ const method = paths.some((p) => p.length === path.length && startsWith(p, path));
466
+ if (!method && !paths.some((p) => startsWith(p, path))) return Reflect.get(object, key, receiver);
467
+ if (!wrapped.has(key)) {
468
+ const value = Reflect.get(object, key, receiver);
469
+ wrapped.set(key, method ? fn(`${name}.${path.join(".")}`, (...args) => value.apply(object, args), callOptions) : wrapAt(value && typeof value === "object" ? value : {}, path));
470
+ }
471
+ return wrapped.get(key);
472
+ } });
473
+ };
474
+ return wrapAt(client, []);
475
+ },
476
+ tools(tools) {
477
+ return Object.fromEntries(Object.entries(tools).map(([name, definition]) => [name, definition.execute ? {
478
+ ...definition,
479
+ execute: tool(name, definition.execute, (args) => args[0])
480
+ } : definition]));
481
+ },
482
+ output(value) {
483
+ effects.push({
484
+ kind: "output",
485
+ value: toJson(value)
486
+ });
487
+ },
488
+ get recording() {
489
+ return {
490
+ version: 1,
491
+ effects: toJson(effects)
492
+ };
493
+ }
494
+ };
495
+ }
496
+ /** True when two model calls offer the same tool names and response format. */
497
+ function sameModelCallShape(expected, actual) {
498
+ return jsonEqual(modelCallShape(expected), modelCallShape(actual));
499
+ }
500
+ function modelCallsMatch(match, expected, actual) {
501
+ if (match === "none") return true;
502
+ if (typeof match === "function") return match(expected, actual);
503
+ return sameModelCallShape(expected, actual);
504
+ }
505
+ function shapeMessage(expected, actual) {
506
+ return `Model call #${expected.call} changed shape: ${JSON.stringify(modelCallShape(expected))} → ${JSON.stringify(modelCallShape(actual))}. A recorded answer only fits a call of the same shape.`;
507
+ }
508
+ /**
509
+ * Offered tool names and the response format, or a decision's questions and
510
+ * their options: what a recorded answer depends on.
511
+ */
512
+ function modelCallShape(effect) {
513
+ const request = effect.request ?? {};
514
+ const questions = request["questions"];
515
+ if (questions && typeof questions === "object") return { questions: Object.entries(questions).map(([name, question]) => {
516
+ const criteria = question["criteria"];
517
+ if (question["type"] === "choice" && criteria && typeof criteria === "object") return [
518
+ name,
519
+ question["type"],
520
+ Object.keys(criteria).sort()
521
+ ];
522
+ if (question["type"] === "score" && Array.isArray(criteria)) return [
523
+ name,
524
+ question["type"],
525
+ criteria.length
526
+ ];
527
+ return [name, question["type"]];
528
+ }).sort((x, y) => String(x[0]).localeCompare(String(y[0]))) };
529
+ return {
530
+ tools: (Array.isArray(request["tools"]) ? request["tools"] : []).map((tool) => tool["name"] ?? tool["function"]?.["name"]).sort(),
531
+ format: request["responseFormat"]?.["type"] ?? request["response_format"]?.["type"] ?? (request["text"]?.["format"])?.["type"] ?? "text"
532
+ };
533
+ }
534
+ /**
535
+ * Puts each group of concurrent tool calls in a stable order, so parallel tool
536
+ * calls that finished in a different order still compare equal. A tool call
537
+ * that started after the previous one settled begins a new group, so
538
+ * sequential calls keep their order.
539
+ */
540
+ function normalize$1(effects) {
541
+ const result = [];
542
+ let batch = [];
543
+ const flush = () => {
544
+ result.push(...batch.sort((a, b) => stableStringify([a.name, a.input]).localeCompare(stableStringify([b.name, b.input]))));
545
+ batch = [];
546
+ };
547
+ for (const effect of effects) {
548
+ if (effect.kind !== "tool" || !effect.concurrent) flush();
549
+ if (effect.kind === "tool") batch.push(effect);
550
+ else result.push(effect);
551
+ }
552
+ flush();
553
+ return result;
554
+ }
555
+ function describe(effect) {
556
+ switch (effect.kind) {
557
+ case "model": return `model call #${effect.call} (${effect.method})`;
558
+ case "tool": return `tool '${effect.name}' with ${JSON.stringify(effect.input)}`;
559
+ case "call": return `call #${effect.call} to '${effect.name}' with ${JSON.stringify(effect.args)}`;
560
+ case "output": return `output ${JSON.stringify(effect.value)}`;
561
+ }
562
+ }
563
+ /** Why two effects differ, or `undefined` when they are the same. */
564
+ function differ(left, right, options) {
565
+ if (left.kind !== right.kind) return {
566
+ reason: "kind",
567
+ message: `Expected ${describe(left)}; the port made ${describe(right)}.`
568
+ };
569
+ if (left.kind === "model" && right.kind === "model") {
570
+ if (right.translated && right.replayed === left.call) return void 0;
571
+ if (left.method !== right.method) return {
572
+ reason: "model-method",
573
+ message: `Model call #${left.call} used ${left.method}; the port used ${right.method}.`
574
+ };
575
+ if (!modelCallsMatch(options.modelCalls, left, right)) return {
576
+ reason: "model-call-shape",
577
+ message: shapeMessage(left, right)
578
+ };
579
+ if (!jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) {
580
+ const outcome = (error) => error ? `failed with ${error.name}: ${error.message}` : "succeeded";
581
+ return {
582
+ reason: "model-error",
583
+ message: `Model call #${left.call} ${outcome(left.error)}; in the port it ${outcome(right.error)}.`
584
+ };
585
+ }
586
+ return;
587
+ }
588
+ if (left.kind === "tool" && right.kind === "tool") {
589
+ if (left.name !== right.name) return {
590
+ reason: "tool-name",
591
+ message: `Expected ${describe(left)}; the port made ${describe(right)}.`
592
+ };
593
+ if (!jsonEqual(left.input, right.input)) return {
594
+ reason: "tool-input",
595
+ message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.input)}.`
596
+ };
597
+ if (!jsonEqual(left.output, right.output) || !jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) return {
598
+ reason: "tool-result",
599
+ message: `Tool '${left.name}' returned a different result.`
600
+ };
601
+ return;
602
+ }
603
+ if (left.kind === "call" && right.kind === "call") {
604
+ if (left.name !== right.name) return {
605
+ reason: "call-name",
606
+ message: `Expected ${describe(left)}; the port made ${describe(right)}.`
607
+ };
608
+ if (!jsonEqual(left.args, right.args)) return {
609
+ reason: "call-args",
610
+ message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.args)}.`
611
+ };
612
+ if (!jsonEqual(left.result, right.result) || !jsonEqual(left.stream, right.stream) || !jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) return {
613
+ reason: "call-result",
614
+ message: `Call #${left.call} to '${left.name}' returned a different result.`
615
+ };
616
+ return;
617
+ }
618
+ if (left.kind === "output" && right.kind === "output" && !jsonEqual(left.value, right.value)) return {
619
+ reason: "output",
620
+ message: `Expected ${describe(left)}; the port returned ${JSON.stringify(right.value)}.`
621
+ };
622
+ }
623
+ /**
624
+ * Compares two recordings and reports every effect that differs. `expected`
625
+ * is the original's recording, `actual` the port's. Effects that are the same
626
+ * on both sides are lined up first (longest common subsequence), so one extra
627
+ * call does not shift every effect after it; between them, effects of the same
628
+ * kind pair up and report what changed, and the rest are missing or extra.
629
+ */
630
+ function compareRecordings(expected, actual, options = {}) {
631
+ const select = (effects) => normalize$1(effects.flatMap((effect, index) => {
632
+ if (options.kinds && !options.kinds.includes(effect.kind)) return [];
633
+ const mapped = options.normalize ? options.normalize(effect, index, effects) : effect;
634
+ return mapped ? [mapped] : [];
635
+ }));
636
+ const a = select(expected.effects);
637
+ const b = select(actual.effects);
638
+ const same = a.map((left) => b.map((right) => !differ(left, right, options)));
639
+ const lcs = Array.from({ length: a.length + 1 }, () => Array.from({ length: b.length + 1 }, () => 0));
640
+ for (let i = a.length - 1; i >= 0; i--) for (let j = b.length - 1; j >= 0; j--) lcs[i][j] = same[i][j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
641
+ const divergences = [];
642
+ let lefts = [];
643
+ let rights = [];
644
+ const flushGap = (at) => {
645
+ const unpairedRights = [...rights];
646
+ const changed = [];
647
+ for (const index of lefts) {
648
+ const left = a[index];
649
+ const k = unpairedRights.findIndex((j) => b[j].kind === left.kind);
650
+ if (k === -1) {
651
+ changed.push({
652
+ index,
653
+ reason: left.kind === "output" ? "output" : "missing-effect",
654
+ expected: left,
655
+ message: left.kind === "output" ? `The port never returned; the original returned ${JSON.stringify(left.value)}.` : `The port never made ${describe(left)}.`
656
+ });
657
+ continue;
658
+ }
659
+ const right = b[unpairedRights.splice(k, 1)[0]];
660
+ const difference = differ(left, right, options);
661
+ if (difference) changed.push({
662
+ index,
663
+ expected: left,
664
+ actual: right,
665
+ ...difference
666
+ });
667
+ }
668
+ for (const j of unpairedRights) {
669
+ const extra = b[j];
670
+ divergences.push({
671
+ index: lefts[0] ?? at,
672
+ reason: extra.kind === "output" ? "output" : "extra-effect",
673
+ actual: extra,
674
+ message: extra.kind === "output" ? `The original never returned; the port returned ${JSON.stringify(extra.value)}.` : `The port made ${describe(extra)}, which the original never made.`
675
+ });
676
+ }
677
+ divergences.push(...changed);
678
+ lefts = [];
679
+ rights = [];
680
+ };
681
+ let i = 0;
682
+ let j = 0;
683
+ while (i < a.length || j < b.length) if (i < a.length && j < b.length && same[i][j]) {
684
+ flushGap(i);
685
+ i++;
686
+ j++;
687
+ } else if (j < b.length && (i === a.length || lcs[i][j + 1] >= lcs[i + 1][j])) rights.push(j++);
688
+ else lefts.push(i++);
689
+ flushGap(i);
690
+ divergences.sort((x, y) => x.index - y.index);
691
+ return divergences.length ? {
692
+ equivalent: false,
693
+ divergence: divergences[0],
694
+ divergences
695
+ } : {
696
+ equivalent: true,
697
+ divergences
698
+ };
699
+ }
700
+ //#endregion
701
+ //#region src/crash-recovery.ts
702
+ var CrashError = class extends Error {
703
+ constructor() {
704
+ super("checkCrashRecovery: simulated crash");
705
+ this.name = "CrashError";
706
+ }
707
+ };
708
+ const DONE_ACTOR = "xstate.done.actor";
709
+ const ERROR_ACTOR = "xstate.error.actor";
710
+ /**
711
+ * Runs `machine` once uninterrupted, then once per crash point: killed after
712
+ * each log append and while each effect is in flight, then resumed from the
713
+ * store's log by a fresh runtime. Each resumed run is checked against the
714
+ * uninterrupted one:
219
715
  *
220
- * matchesTrajectory(run.after.statePath, ['needsMoreInfo', 'drafting']);
221
- * ```
716
+ * - An effect whose completion was durable before the crash never runs again.
717
+ * Only an effect in flight at the crash may (execution is at-least-once).
718
+ * - Every effect of the uninterrupted run runs, and no other: a re-executed
719
+ * model call keeps its `callKey` and its request.
720
+ * - The run ends with the same status, output, state value, and context.
721
+ * - The resumed log replays to that state.
222
722
  *
223
- * @example A candidate prompt at the seam, scored against the same rows.
224
- * ```ts
225
- * const { generateText } = createAiSdkExecutors({ models });
226
- * const run = await runSeam(emailDrafter, { ...row, candidate: generateText });
227
- * ```
723
+ * Model calls the uninterrupted run made are answered from its recording.
724
+ * Plain actors run live, so they must give the same result however many times
725
+ * they run (pass `reset` for fakes that keep state).
228
726
  */
229
- async function runSeam(machine, options) {
230
- const { seam, candidate } = options;
231
- const queues = new Map(Object.entries(options.scripts ?? {}).map(([key, answers]) => [key, [...answers]]));
232
- const statePath = [];
233
- let calls = 0;
234
- let seamMatches = 0;
235
- let seamOutput;
236
- let seamUsage;
237
- const callLog = [];
238
- let seamReached = false;
239
- let callsBeforeSeam = -1;
240
- let seamStateAt = 0;
241
- let seamEventAt = 0;
242
- let liveEvents = 0;
243
- const queueKeyOf = (request) => request.name !== void 0 && queues.has(request.name) ? request.name : request.model;
244
- /**
245
- * Consumes this request's slot in the call plan, or resolves `undefined` when
246
- * its queue is dry. With `repeatLast`, the last entry is replayed instead of
247
- * running dry.
248
- */
249
- const takeScriptedSlot = async (request, info) => {
250
- const queue = queues.get(queueKeyOf(request));
251
- if (!queue?.length) return;
252
- return resolveSeamScriptEntry(options.repeatLast && queue.length === 1 ? queue[0] : queue.shift(), request, info);
727
+ async function checkCrashRecovery(machine, options = {}) {
728
+ const timeout = options.timeout ?? 5e3;
729
+ options.reset?.();
730
+ const baselineWorld = newWorld(() => false);
731
+ const baselineStore = observedStore((options.store ?? createInMemoryEventLogStore)(), baselineWorld, () => false);
732
+ const baseline = await withTimeout(runTurns(machine, options, baselineWorld, baselineStore.store), timeout);
733
+ if (baseline === TIMED_OUT) throw new Error(`checkCrashRecovery: the uninterrupted run did not settle within ${timeout}ms`);
734
+ if (baselineWorld.unsent.length > 0) {
735
+ const keys = baselineWorld.unsent.map((index) => {
736
+ const { during, occurrence = 0 } = options.sendDuring[index];
737
+ return occurrence === 0 ? `'${during}'` : `'${during}' (start ${occurrence})`;
738
+ });
739
+ throw new Error(`checkCrashRecovery: sendDuring names effects the uninterrupted run never started: ${keys.join(", ")}`);
740
+ }
741
+ if (baselineWorld.undelivered > 0) throw new Error(`checkCrashRecovery: the uninterrupted run settled '${baseline.status}' with ${baselineWorld.undelivered} of \`events\` undelivered`);
742
+ const baselineEntries = await baselineStore.store.read(THREAD);
743
+ const baselineCounts = countBy(baselineWorld.effects);
744
+ const expected = outcomeOf(baseline);
745
+ const points = [];
746
+ for (const entries of /* @__PURE__ */ new Set([0, ...baselineStore.appends])) points.push({
747
+ kind: "log",
748
+ entries
749
+ });
750
+ const seen = /* @__PURE__ */ new Map();
751
+ baselineWorld.effects.forEach((effect, index) => {
752
+ const occurrence = seen.get(effect.key) ?? 0;
753
+ seen.set(effect.key, occurrence + 1);
754
+ points.push({
755
+ kind: "effect",
756
+ effect: {
757
+ kind: effect.kind,
758
+ key: effect.key
759
+ },
760
+ index,
761
+ occurrence
762
+ });
763
+ });
764
+ const failures = [];
765
+ const unreached = [];
766
+ const ctx = {
767
+ machine,
768
+ options,
769
+ timeout,
770
+ baselineCounts,
771
+ recorded: baselineWorld.answers,
772
+ expected,
773
+ sentAt: new Map([...baselineWorld.sentEntries].map(([index, entry]) => [index, entry.index]))
253
774
  };
254
- const scriptedAnswer = async (request, info) => {
255
- const scripted = await takeScriptedSlot(request, info);
256
- if (!scripted) throw new AgentError("seam-script-exhausted", `runSeam: no scripted answer left for request ${describeText(request)}. Add an entry to \`scripts.${queueKeyOf(request)}\`, or pass \`repeatLast: true\` to replay its last entry down a longer branch.`);
257
- return scripted;
775
+ for (const point of points) {
776
+ let found = await crashAt(point, ctx);
777
+ for (let retry = 1; !Array.isArray(found) && retry < UNREACHED_RETRIES; retry++) found = await crashAt(point, ctx);
778
+ if (Array.isArray(found)) failures.push(...found);
779
+ else unreached.push({
780
+ point,
781
+ reason: found.unreached
782
+ });
783
+ }
784
+ return {
785
+ ok: failures.length === 0 && unreached.length === 0,
786
+ baseline: {
787
+ status: baseline.status,
788
+ ...baseline.status === "done" ? { output: baseline.output } : {},
789
+ entries: baselineEntries.length,
790
+ effects: baselineWorld.effects.map(({ kind, key }) => ({
791
+ kind,
792
+ key
793
+ }))
794
+ },
795
+ points,
796
+ unreached,
797
+ failures
258
798
  };
259
- const route = async (request, info) => {
260
- const callIndex = calls++;
261
- const isSeam = request.name === seam.request && seamMatches++ === (seam.occurrence ?? 0);
262
- const ledger = (source) => {
263
- callLog.push({
264
- ...request.name !== void 0 ? { name: request.name } : {},
265
- model: request.model,
266
- key: queueKeyOf(request),
267
- source,
268
- seam: isSeam
269
- });
270
- };
271
- if (isSeam && candidate) {
272
- await takeScriptedSlot(request, info);
273
- seamReached = true;
274
- callsBeforeSeam = callIndex;
275
- seamStateAt = statePath.length;
276
- seamEventAt = liveEvents;
277
- const result = await candidate(request, info);
278
- seamOutput = await seamOutputOf(result, request);
279
- seamUsage = getCallUsage(result);
280
- ledger("candidate");
281
- return result;
282
- }
283
- const scripted = await scriptedAnswer(request, info);
284
- ledger("script");
285
- if (!isSeam) return scripted;
286
- seamReached = true;
287
- callsBeforeSeam = callIndex;
288
- seamStateAt = statePath.length;
289
- seamEventAt = liveEvents;
290
- seamOutput = scripted.result;
291
- seamUsage = scripted.usage;
292
- return scripted;
799
+ }
800
+ const THREAD = "crash-recovery";
801
+ const UNREACHED_RETRIES = 3;
802
+ const SKIPPED_TURN = "(no timer pending)";
803
+ const TIMED_OUT = Symbol("timed out");
804
+ async function crashAt(point, ctx) {
805
+ const { machine, options, timeout } = ctx;
806
+ const fail = (problem, message, key) => ({
807
+ point,
808
+ problem,
809
+ message,
810
+ ...key !== void 0 ? { key } : {}
811
+ });
812
+ options.reset?.();
813
+ let backing;
814
+ try {
815
+ backing = (options.store ?? createInMemoryEventLogStore)();
816
+ } catch (error) {
817
+ return [fail("threw", `the store factory threw: ${String(error)}`)];
818
+ }
819
+ const controller = new AbortController();
820
+ let runs = 0;
821
+ const crashed = newWorld((effect) => point.kind === "effect" && effect.key === point.effect.key && runs++ === point.occurrence, ctx.recorded);
822
+ const crash = () => {
823
+ if (crashed.crashed) return;
824
+ crashed.crashed = true;
825
+ controller.abort(new CrashError());
293
826
  };
294
- const executors = {
295
- ...options.executors,
296
- generateText: route,
297
- streamText: async (request, info) => {
298
- const result = await route(request, info);
299
- emitScriptedChunk(result, info);
300
- return result;
827
+ let missed = point.kind === "log" && point.entries === 0 ? void 0 : point.kind === "log" ? "the log never reached that length" : "the run never started that effect";
828
+ const onEffect = crashed.onEffect;
829
+ crashed.onEffect = (effect) => {
830
+ const dies = onEffect(effect);
831
+ if (dies) {
832
+ missed = void 0;
833
+ queueMicrotask(crash);
301
834
  }
835
+ return dies;
302
836
  };
303
- let snapshot;
304
- let event;
305
- const events = [];
837
+ const dying = observedStore(backing, crashed, (accepted) => {
838
+ if (point.kind !== "log" || accepted < point.entries) return false;
839
+ missed = accepted === point.entries ? void 0 : `one write took the log to ${accepted} entries`;
840
+ crash();
841
+ return true;
842
+ });
843
+ if (point.kind === "log" && point.entries === 0) dying.freeze();
844
+ let died;
845
+ try {
846
+ died = await withTimeout(runTurns(machine, options, crashed, dying.store, void 0, controller.signal), timeout);
847
+ } catch (error) {
848
+ if (!(error instanceof CrashError)) {
849
+ crash();
850
+ return [fail("threw", `the crashed run threw: ${String(error)}`)];
851
+ }
852
+ }
853
+ crash();
854
+ if (died === TIMED_OUT) return [fail("stuck", `the run did not stop within ${timeout}ms after the crash`)];
855
+ if (missed !== void 0) return { unreached: missed };
856
+ const resumed = newWorld(() => false, ctx.recorded);
857
+ const resumeController = new AbortController();
306
858
  let result;
307
- const maxTurns = options.maxTurns ?? 12;
308
- for (let turn = 0; turn <= maxTurns; turn++) {
309
- result = await runToQuiescence(createAgentRuntime(machine, {
310
- ...options.actors ? { actors: options.actors } : {},
311
- executors,
312
- onTransition: (next, causedBy) => {
313
- if (turn > 0 && causedBy.type === "@xstate.init") return;
314
- events.push(causedBy);
315
- liveEvents = events.length;
316
- statePath.push(next.value);
859
+ try {
860
+ result = await withTimeout(runTurns(machine, options, resumed, backing, {
861
+ crashed,
862
+ sentAt: ctx.sentAt
863
+ }, resumeController.signal), timeout);
864
+ } catch (error) {
865
+ resumed.crashed = true;
866
+ resumeController.abort(new CrashError());
867
+ return [fail("threw", `the resume threw: ${String(error)}`)];
868
+ }
869
+ if (result === TIMED_OUT) {
870
+ resumed.crashed = true;
871
+ resumeController.abort(new CrashError());
872
+ return [fail("stuck", `the resume did not settle within ${timeout}ms`)];
873
+ }
874
+ const failures = [];
875
+ for (const index of resumed.unsentAfterCrash) {
876
+ const { during, occurrence = 0 } = options.sendDuring[index];
877
+ failures.push(fail("undelivered", `the sendDuring event for '${during}' (start ${occurrence}) was lost in the crash and never sent again`));
878
+ }
879
+ const total = countBy([...crashed.effects, ...resumed.effects]);
880
+ const inFlight = countBy(crashed.effects.filter((effect) => !effect.completed));
881
+ for (const [key, count] of total) {
882
+ const expected = ctx.baselineCounts.get(key) ?? 0;
883
+ const allowed = expected + (inFlight.get(key) ?? 0);
884
+ if (expected === 0) failures.push(fail("unexpected", `'${key}' ran ${count}x; the uninterrupted run never made it`, key));
885
+ else if (count > allowed) failures.push(fail("reran", `'${key}' ran ${count}x, at most ${allowed} expected: its completion was durable before the crash`, key));
886
+ }
887
+ for (const [key, expected] of ctx.baselineCounts) {
888
+ const count = total.get(key) ?? 0;
889
+ if (count < expected) failures.push(fail("skipped", `'${key}' ran ${count}x, ${expected} expected`, key));
890
+ }
891
+ const crashedRequests = new Map(crashed.effects.filter((effect) => effect.kind === "model").map((effect) => [effect.key, effect.request]));
892
+ for (const effect of resumed.effects) {
893
+ if (effect.kind !== "model" || !crashedRequests.has(effect.key)) continue;
894
+ const before = crashedRequests.get(effect.key);
895
+ if (before === void 0 || effect.request === void 0) continue;
896
+ if (before !== effect.request) failures.push(fail("request-changed", `'${effect.key}' re-executed with a different request`, effect.key));
897
+ }
898
+ const actual = outcomeOf(result);
899
+ if (actual !== ctx.expected) failures.push(fail("outcome", `resumed run ended ${actual}, expected ${ctx.expected}`));
900
+ const log = await backing.read(THREAD);
901
+ try {
902
+ const replayed = replay(options.actors !== void 0 ? machine.provide({ actors: options.actors }) : machine, log);
903
+ const fromLog = stableStringify(toJsonValue({
904
+ value: replayed.snapshot.value,
905
+ context: replayed.snapshot.context
906
+ }));
907
+ const live = stableStringify(toJsonValue({
908
+ value: result.snapshot.value,
909
+ context: result.snapshot.context
910
+ }));
911
+ if (fromLog !== live) failures.push(fail("replay", `the log replays to ${fromLog}, the resumed run ended at ${live}`));
912
+ } catch (error) {
913
+ failures.push(fail("replay", `the log does not replay: ${String(error)}`));
914
+ }
915
+ return failures;
916
+ }
917
+ /**
918
+ * Runs the turns: start from `input`, then one host event per idle settle,
919
+ * with `sendDuring` events sent as their effects start and host timers fired
920
+ * when the run is idle. Resuming, what the crashed run's log holds is skipped.
921
+ */
922
+ async function runTurns(machine, options, world, store, resuming, signal) {
923
+ const tracked = trackActors(machine, world);
924
+ const executors = trackExecutors(options.executors ?? {}, world);
925
+ const actors = options.actors !== void 0 ? Object.fromEntries(Object.entries(options.actors).map(([key, logic]) => [key, isStateMachineLogic(logic) ? trackActors(logic, world) : trackLogic(logic, world)])) : void 0;
926
+ const before = resuming?.crashed;
927
+ const log = before !== void 0 ? await store.read(THREAD) : [];
928
+ const durable = new Set(log.map((entry) => entry.id));
929
+ const events = options.events ?? [];
930
+ let next = 0;
931
+ if (before !== void 0) while (next < before.hostEntries.length && (before.hostEntries[next] === SKIPPED_TURN || durable.has(before.hostEntries[next]))) next++;
932
+ const sends = options.sendDuring ?? [];
933
+ const unsent = /* @__PURE__ */ new Set();
934
+ const redeliver = [];
935
+ const sendAt = /* @__PURE__ */ new Map();
936
+ const waitsFor = /* @__PURE__ */ new Map();
937
+ sends.forEach(({ during, occurrence = 0 }, index) => {
938
+ if (resuming === void 0) {
939
+ waitsFor.set(index, occurrence);
940
+ unsent.add(index);
941
+ return;
942
+ }
943
+ const entry = before.sentEntries.get(index);
944
+ if (entry !== void 0 && durable.has(entry.id)) return;
945
+ const starts = before.effects.filter((effect) => effect.key === during);
946
+ if (starts[occurrence]?.completed !== true) {
947
+ waitsFor.set(index, occurrence - starts.filter((effect) => effect.completed).length);
948
+ unsent.add(index);
949
+ return;
950
+ }
951
+ const at = resuming.sentAt.get(index);
952
+ if (at === void 0) return;
953
+ if (at <= log.length) redeliver.push(index);
954
+ else sendAt.set(index, at);
955
+ });
956
+ const awaitingSends = [];
957
+ let live;
958
+ const send = (index) => {
959
+ const { event } = sends[index];
960
+ awaitingSends.push({
961
+ index,
962
+ json: canonicalEventJson(event)
963
+ });
964
+ live?.send(event);
965
+ };
966
+ const startsSeen = /* @__PURE__ */ new Map();
967
+ const onEffect = world.onEffect;
968
+ world.onEffect = (effect) => {
969
+ const nth = startsSeen.get(effect.key) ?? 0;
970
+ startsSeen.set(effect.key, nth + 1);
971
+ for (const index of unsent) {
972
+ if (sends[index].during !== effect.key || waitsFor.get(index) !== nth) continue;
973
+ unsent.delete(index);
974
+ send(index);
975
+ }
976
+ return onEffect(effect);
977
+ };
978
+ const pendingTimers = /* @__PURE__ */ new Map();
979
+ world.clock = {
980
+ now: before?.clock.now ?? 0,
981
+ due: pendingTimers
982
+ };
983
+ const dueBefore = new Map(before?.clock.due ?? []);
984
+ const timers = options.timers === "host" ? {
985
+ schedule: ({ id, delay }) => {
986
+ if (!pendingTimers.has(id)) {
987
+ const restored = dueBefore.get(id);
988
+ dueBefore.delete(id);
989
+ pendingTimers.set(id, restored ?? world.clock.now + delay);
317
990
  }
318
- }), {
319
- ...snapshot ? { snapshot } : { input: options.input },
320
- ...event ? { event } : {}
991
+ },
992
+ cancel: (id) => {
993
+ pendingTimers.delete(id);
994
+ }
995
+ } : void 0;
996
+ /** `track`: the turn delivers one of `events` (a host event, or a timer's turn), whose entry is recorded. */
997
+ const turn = (event, track) => {
998
+ let awaiting = event !== void 0 && track === "host" ? canonicalEventJson(event) : void 0;
999
+ let awaitingTimer = track === "timer";
1000
+ live = createAgentRuntime(tracked, {
1001
+ ...options.runtime,
1002
+ onEvent: (entry) => {
1003
+ options.runtime?.onEvent?.(entry);
1004
+ for (const [index, at] of sendAt) {
1005
+ if (entry.index + 1 < at) continue;
1006
+ sendAt.delete(index);
1007
+ send(index);
1008
+ }
1009
+ if (entry.actorPath !== void 0) return;
1010
+ const json = canonicalEventJson(entry.event);
1011
+ if (awaiting !== void 0 && json === awaiting) {
1012
+ world.hostEntries.push(entry.id);
1013
+ awaiting = void 0;
1014
+ return;
1015
+ }
1016
+ if (awaitingTimer && entry.event.type === "xstate.timer") {
1017
+ world.hostEntries.push(entry.id);
1018
+ awaitingTimer = false;
1019
+ return;
1020
+ }
1021
+ const sent = awaitingSends.findIndex((item) => item.json === json);
1022
+ if (sent !== -1) {
1023
+ world.sentEntries.set(awaitingSends[sent].index, {
1024
+ id: entry.id,
1025
+ index: entry.index
1026
+ });
1027
+ awaitingSends.splice(sent, 1);
1028
+ }
1029
+ },
1030
+ executors,
1031
+ ...actors !== void 0 ? { actors } : {},
1032
+ store,
1033
+ threadId: THREAD,
1034
+ ...timers !== void 0 ? { timers } : {},
1035
+ ...signal !== void 0 ? { signal } : {}
321
1036
  });
322
- if (result.status !== "idle" || turn === maxTurns) break;
323
- const next = await options.respond?.({
324
- snapshot: result.snapshot,
325
- state: result.snapshot.value,
326
- meta: getStateMeta(result.snapshot),
327
- turn,
328
- result
1037
+ for (const index of redeliver.splice(0)) send(index);
1038
+ return runToQuiescence(live, {
1039
+ ...options.input !== void 0 ? { input: options.input } : {},
1040
+ ...event !== void 0 ? { event } : {}
329
1041
  });
330
- if (!next) break;
331
- event = next;
332
- snapshot = result.persist();
333
- }
334
- let splitAt = events.length;
335
- if (seamReached) {
336
- splitAt = seamEventAt;
337
- for (let index = seamEventAt; index < events.length; index++) {
338
- const type = events[index].type;
339
- if (type.startsWith("xstate.done") || type.startsWith("xstate.error")) {
340
- splitAt = index;
341
- break;
1042
+ };
1043
+ const soonestTimer = () => {
1044
+ let soonest;
1045
+ for (const timer of pendingTimers) if (soonest === void 0 || timer[1] < soonest[1]) soonest = timer;
1046
+ if (soonest === void 0) return void 0;
1047
+ pendingTimers.delete(soonest[0]);
1048
+ world.clock.now = soonest[1];
1049
+ return {
1050
+ type: "xstate.timer",
1051
+ id: soonest[0]
1052
+ };
1053
+ };
1054
+ let result = await turn();
1055
+ while (result.status === "idle" && !world.crashed) {
1056
+ let event;
1057
+ let host = true;
1058
+ let marked = false;
1059
+ if (next < events.length) {
1060
+ event = events[next++];
1061
+ if (event.type === "xstate.timer" && !("id" in event)) {
1062
+ event = soonestTimer();
1063
+ host = false;
1064
+ marked = true;
1065
+ if (event === void 0) {
1066
+ world.hostEntries.push(SKIPPED_TURN);
1067
+ continue;
1068
+ }
342
1069
  }
1070
+ } else {
1071
+ event = soonestTimer();
1072
+ host = false;
343
1073
  }
1074
+ if (event === void 0) break;
1075
+ result = await turn(event, host ? "host" : marked ? "timer" : void 0);
1076
+ await result.drain?.();
344
1077
  }
345
- const splitStateAt = seamReached ? seamStateAt : statePath.length;
1078
+ await result.drain?.();
1079
+ world.unsent = [...unsent];
1080
+ world.unsentAfterCrash = resuming !== void 0 ? [...unsent, ...sendAt.keys()] : [];
1081
+ world.undelivered = events.length - next;
1082
+ return result;
1083
+ }
1084
+ function newWorld(crashesAt, recorded) {
346
1085
  return {
347
- result,
348
- seamOutput,
349
- ...seamUsage !== void 0 ? { seamUsage } : {},
350
- callsBeforeSeam,
351
- calls: callLog,
352
- before: {
353
- statePath: statePath.slice(0, splitStateAt),
354
- events: events.slice(0, splitAt)
1086
+ effects: [],
1087
+ hostEntries: [],
1088
+ sentEntries: /* @__PURE__ */ new Map(),
1089
+ unsent: [],
1090
+ unsentAfterCrash: [],
1091
+ undelivered: 0,
1092
+ clock: {
1093
+ now: 0,
1094
+ due: /* @__PURE__ */ new Map()
355
1095
  },
356
- after: {
357
- statePath: statePath.slice(splitStateAt),
358
- events: events.slice(splitAt)
359
- }
1096
+ answers: /* @__PURE__ */ new Map(),
1097
+ ...recorded !== void 0 ? { recorded } : {},
1098
+ crashed: false,
1099
+ onEffect: (effect) => crashesAt(effect)
360
1100
  };
361
1101
  }
362
- //#endregion
363
- //#region src/internal/trace-states.ts
364
- /** The `value` of a trace event's `from` or `snapshot`. @internal */
365
- function valueOf(holder) {
366
- return isRecord(holder) ? holder.value : void 0;
367
- }
368
- /** A decision request on its first attempt (no failed attempts yet). @internal */
369
- function isFirstDecision(request) {
370
- return isRecord(request) && request.kind === "decision" && (!Array.isArray(request.attempts) || request.attempts.length === 0);
371
- }
372
1102
  /**
373
- * Every active leaf of a state value as a dot path:
374
- * `'a'` → `['a']`; `{ a: 'b' }` → `['a.b']`; `{ p: { x: 'one', y: {} } }` → `['p.x.one', 'p.y']`.
375
- * @internal
1103
+ * `store`, recording each append's resulting length and marking the effects
1104
+ * whose completion an accepted entry carries. `afterAppend` returns true to
1105
+ * freeze it: every later append rejects, as if the process died.
376
1106
  */
377
- function statePaths(value, prefix = "") {
378
- if (typeof value === "string") return [prefix + value];
379
- if (!isRecord(value)) return [];
380
- const entries = Object.entries(value);
381
- if (entries.length === 0) return prefix ? [prefix.slice(0, -1)] : [];
382
- return entries.flatMap(([key, child]) => statePaths(child, `${prefix}${key}.`));
1107
+ function observedStore(store, world, afterAppend) {
1108
+ const appends = [];
1109
+ let frozen = false;
1110
+ return {
1111
+ store: {
1112
+ ...store,
1113
+ read: (threadId, options) => store.read(threadId, options),
1114
+ length: (threadId) => store.length(threadId),
1115
+ fork: (input) => store.fork(input),
1116
+ async append(input) {
1117
+ if (frozen || world.crashed) throw new CrashError();
1118
+ await store.append(input);
1119
+ for (const entry of input.entries) markCompleted(entry, world.effects);
1120
+ const accepted = input.expectedIndex + input.entries.length;
1121
+ appends.push(accepted);
1122
+ if (afterAppend(accepted)) frozen = true;
1123
+ }
1124
+ },
1125
+ appends,
1126
+ freeze: () => {
1127
+ frozen = true;
1128
+ }
1129
+ };
383
1130
  }
384
1131
  /**
385
- * A trace's `machine.transition` events as steps, skipping `@xstate.init`.
386
- * A first-attempt decision's `request.start` names the decision on the step
387
- * that follows it, skipping `@agent.usage` steps. Reads live and serialized traces alike.
388
- * @internal
1132
+ * Marks the call an accepted entry completes: a decision's chosen event names
1133
+ * its call (`site#n`); a completion completes the oldest open effect at its
1134
+ * invoke site, with the other attempts of the same model call.
389
1135
  */
390
- function traceSteps(trace) {
391
- const steps = [];
392
- let decision;
393
- for (const event of trace) {
394
- if (event.type === "request.start" && isFirstDecision(event.request)) {
395
- const name = event.request.name;
396
- decision = typeof name === "string" ? name : void 0;
1136
+ function markCompleted(entry, effects) {
1137
+ if (entry.decidedBy !== void 0) {
1138
+ const call = entry.decidedBy;
1139
+ for (const effect of effects) if (effect.key === call || effect.key.startsWith(`${call}.`)) effect.completed = true;
1140
+ return;
1141
+ }
1142
+ if (entry.event.type !== DONE_ACTOR && entry.event.type !== ERROR_ACTOR) return;
1143
+ const actorId = entry.event.actorId;
1144
+ if (typeof actorId !== "string") return;
1145
+ const site = agentCallSite(entry.actorPath ?? [], actorId);
1146
+ const oldest = effects.find((effect) => effect.site === site && !effect.completed);
1147
+ if (oldest === void 0) return;
1148
+ const call = oldest.kind === "model" ? callOfModelKey(oldest.key) : void 0;
1149
+ for (const effect of effects) if (effect === oldest || call !== void 0 && effect.site === site && callOfModelKey(effect.key) === call) effect.completed = true;
1150
+ }
1151
+ function countBy(effects) {
1152
+ const counts = /* @__PURE__ */ new Map();
1153
+ for (const { key } of effects) counts.set(key, (counts.get(key) ?? 0) + 1);
1154
+ return counts;
1155
+ }
1156
+ function outcomeOf(result) {
1157
+ const snapshot = result.snapshot;
1158
+ return stableStringify(toJsonValue({
1159
+ status: result.status,
1160
+ ...result.status === "done" ? { output: result.output } : {},
1161
+ ...result.status === "error" ? { cause: result.cause } : {},
1162
+ value: snapshot.value,
1163
+ context: snapshot.context
1164
+ }));
1165
+ }
1166
+ async function withTimeout(promise, ms) {
1167
+ let timer;
1168
+ try {
1169
+ return await Promise.race([promise, new Promise((resolve) => {
1170
+ timer = setTimeout(() => resolve(TIMED_OUT), ms);
1171
+ })]);
1172
+ } finally {
1173
+ clearTimeout(timer);
1174
+ }
1175
+ }
1176
+ /** `callKey` without its execution id: the same call has the same key in every run. */
1177
+ function modelKey(callKey) {
1178
+ return callKey.slice(callKey.indexOf(":") + 1);
1179
+ }
1180
+ function siteOfModelKey(key) {
1181
+ const hash = key.lastIndexOf("#");
1182
+ return hash === -1 ? key : key.slice(0, hash);
1183
+ }
1184
+ /** `site#n` of a model key: the call, without its attempt or strategy suffix. */
1185
+ function callOfModelKey(key) {
1186
+ const hash = key.lastIndexOf("#");
1187
+ if (hash === -1) return key;
1188
+ const dot = key.indexOf(".", hash);
1189
+ return dot === -1 ? key : key.slice(0, dot);
1190
+ }
1191
+ function trackExecutors(executors, world) {
1192
+ const tracked = {};
1193
+ for (const [slot, executor] of Object.entries(executors)) {
1194
+ if (typeof executor !== "function") continue;
1195
+ tracked[slot] = async (request, info) => {
1196
+ if (world.crashed) return never(info?.signal);
1197
+ const key = info?.callKey !== void 0 ? modelKey(info.callKey) : `${slot}:?`;
1198
+ const effect = {
1199
+ kind: "model",
1200
+ key,
1201
+ site: siteOfModelKey(key),
1202
+ completed: false,
1203
+ request: requestFingerprint(request)
1204
+ };
1205
+ world.effects.push(effect);
1206
+ const dies = world.onEffect(effect);
1207
+ const recorded = world.recorded?.get(key);
1208
+ let result;
1209
+ if (recorded !== void 0) {
1210
+ await new Promise((resolve) => setTimeout(resolve, recorded.ms));
1211
+ result = recorded.result;
1212
+ } else {
1213
+ const started = Date.now();
1214
+ result = await executor(request, info);
1215
+ if (info?.callKey !== void 0) world.answers.set(key, {
1216
+ result,
1217
+ ms: Date.now() - started
1218
+ });
1219
+ }
1220
+ if (dies || world.crashed) return never(info?.signal);
1221
+ return result;
1222
+ };
1223
+ }
1224
+ return tracked;
1225
+ }
1226
+ const VOLATILE_REQUEST_KEYS = /* @__PURE__ */ new Set(["runId"]);
1227
+ function requestFingerprint(request) {
1228
+ try {
1229
+ const json = toJsonValue(request);
1230
+ if (json && typeof json === "object" && !Array.isArray(json)) for (const key of VOLATILE_REQUEST_KEYS) delete json[key];
1231
+ return stableStringify(json);
1232
+ } catch {
1233
+ return;
1234
+ }
1235
+ }
1236
+ function never(signal) {
1237
+ return new Promise((_, reject) => {
1238
+ if (signal?.aborted) reject(signal.reason);
1239
+ signal?.addEventListener("abort", () => reject(signal.reason), { once: true });
1240
+ });
1241
+ }
1242
+ /** `machine` with every plain actor source, at any depth, tracked. */
1243
+ function trackActors(machine, world, ancestors = /* @__PURE__ */ new Set()) {
1244
+ if (ancestors.has(machine)) return machine;
1245
+ const visited = /* @__PURE__ */ new Set([...ancestors, machine]);
1246
+ const sources = machine.sources?.actors ?? {};
1247
+ const wrapped = {};
1248
+ for (const [key, logic] of Object.entries(sources)) {
1249
+ if (isTextLogic(logic) || isDecisionLogic(logic) || isActLogic(logic)) continue;
1250
+ if (isStateMachineLogic(logic)) {
1251
+ const inner = trackActors(logic, world, visited);
1252
+ if (inner !== logic) wrapped[key] = inner;
397
1253
  continue;
398
1254
  }
399
- if (event.type !== "machine.transition") continue;
400
- const type = isRecord(event.event) ? event.event.type : void 0;
401
- if (type === "@xstate.init") continue;
402
- const fromValue = valueOf(event.from);
403
- const toValue = valueOf(event.snapshot);
404
- steps.push({
405
- from: statePaths(fromValue),
406
- fromValue,
407
- event: typeof type === "string" ? type : "",
408
- payload: event.event,
409
- to: statePaths(toValue),
410
- toValue,
411
- ignored: event.ignored === true,
412
- ...decision === void 0 || type === "@agent.usage" ? {} : { decision }
413
- });
414
- if (type !== "@agent.usage") decision = void 0;
1255
+ if (typeof logic?.transition === "function") wrapped[key] = trackLogic(logic, world);
415
1256
  }
416
- return steps;
1257
+ return Object.keys(wrapped).length > 0 ? machine.provide({ actors: wrapped }) : machine;
1258
+ }
1259
+ /** The invoke site of an actor: its path's last id, under its parents' path. */
1260
+ function actorSite(self) {
1261
+ const path = [];
1262
+ for (let at = self; at?._parent !== void 0; at = at._parent) path.unshift(String(at.id));
1263
+ return agentCallSite(path.slice(0, -1), path.at(-1) ?? "");
417
1264
  }
418
- //#endregion
419
- //#region src/fold-traces.ts
420
1265
  /**
421
- * Folds many runs' traces into per-state stats: how many runs visited each
422
- * state, how many of those failed, how many options the guards withheld
423
- * at decisions made there, how many events it ignored, and how many failures
424
- * `blame` traced back to it. Pure; reads only:
425
- *
426
- * - `machine.transition`: the run visited `from.value` and `snapshot.value`;
427
- * with `ignored`, each of those states ignored one event.
428
- * - `request.start` of a decision, first attempt only: its `request.withheld`
429
- * counts toward the state the run was in, the `from.value` of the
430
- * transition its choice causes.
431
- *
432
- * A trace is one run's events, live (`AgentTraceEvent`) or serialized
433
- * (`serializeTraceEvent`). A host-driven `decideNext` traces nothing; push
434
- * `{ type: 'request.start', request: result.request }` onto the trace before
435
- * sending its event. Parallel states count once per active leaf.
436
- *
437
- * `failures` counts a failed run against every state it visited, so states
438
- * after a bad decision rank as high as the decision. Sort by `blamed` to see
439
- * where failures start.
440
- *
441
- * @example
442
- * ```ts no-check
443
- * const stats = foldTraces(traces, {
444
- * failed: (trace) => trace.at(-1)?.type === 'run.end' && trace.at(-1).status === 'error',
445
- * });
446
- * ```
1266
+ * `logic`, recording each effect it starts (`@xstate.effect`: an async
1267
+ * actor's run, a callback's body). After the crash, effects do not start.
447
1268
  */
448
- function foldTraces(traces, options) {
449
- const stats = /* @__PURE__ */ new Map();
450
- const statsFor = (state) => {
451
- let entry = stats.get(state);
452
- if (!entry) {
453
- entry = {
454
- state,
455
- visits: 0,
456
- failures: 0,
457
- withheld: 0,
458
- ignored: 0,
459
- blamed: 0
1269
+ function trackLogic(logic, world) {
1270
+ if (isTextLogic(logic) || isDecisionLogic(logic) || isActLogic(logic)) return logic;
1271
+ const track = (result, self) => {
1272
+ const [snapshot, effects] = result;
1273
+ if (!Array.isArray(effects) || !effects.some((e) => e?.type === "@xstate.effect")) return result;
1274
+ return [snapshot, effects.map((effect) => {
1275
+ if (effect?.type !== "@xstate.effect") return effect;
1276
+ const exec = effect.exec;
1277
+ const tracked = (...args) => {
1278
+ if (world.crashed) return;
1279
+ const site = actorSite(self);
1280
+ const started = {
1281
+ kind: "actor",
1282
+ key: `actor:${site}`,
1283
+ site,
1284
+ completed: false
1285
+ };
1286
+ world.effects.push(started);
1287
+ world.onEffect(started);
1288
+ return exec(...args);
460
1289
  };
461
- stats.set(state, entry);
462
- }
463
- return entry;
1290
+ return {
1291
+ ...effect,
1292
+ exec: tracked,
1293
+ action: tracked
1294
+ };
1295
+ })];
464
1296
  };
465
- for (const trace of traces) {
466
- const visited = /* @__PURE__ */ new Set();
467
- let current = [];
468
- const visit = (state) => {
469
- statsFor(state);
470
- visited.add(state);
1297
+ const overrides = { transition: { value: (...args) => track(logic.transition(...args), args[2].self) } };
1298
+ if (typeof logic.initialTransition === "function") overrides["initialTransition"] = { value: (...args) => track(logic.initialTransition(...args), args[1].self) };
1299
+ if (typeof logic.start === "function") overrides["start"] = { value: (snapshot, scope, startOptions) => logic.start(snapshot, new Proxy(scope, { get(target, property) {
1300
+ if (property === "actionExecutor") return (effect) => {
1301
+ const [, [tracked]] = track([void 0, [effect]], target.self);
1302
+ target.actionExecutor(tracked);
471
1303
  };
472
- for (const event of trace) if (event.type === "machine.transition") {
473
- for (const state of statePaths(valueOf(event.from))) visit(state);
474
- current = statePaths(valueOf(event.snapshot));
475
- for (const state of current) visit(state);
476
- if (event.ignored === true) for (const state of current) statsFor(state).ignored++;
477
- } else if (event.type === "request.start" && isFirstDecision(event.request)) {
478
- const withheld = event.request.withheld;
479
- if (!Array.isArray(withheld)) continue;
480
- for (const state of current) statsFor(state).withheld += withheld.length;
481
- }
482
- const failed = options.failed(trace);
483
- for (const state of visited) {
484
- const entry = statsFor(state);
485
- entry.visits++;
486
- if (failed) entry.failures++;
487
- }
488
- const blamed = failed ? options.blame?.(trace) : void 0;
489
- if (blamed !== void 0) statsFor(blamed).blamed++;
490
- }
491
- return [...stats.values()];
1304
+ return Reflect.get(target, property, target);
1305
+ } }), startOptions) };
1306
+ return Object.create(logic, overrides);
492
1307
  }
493
1308
  //#endregion
494
- //#region src/trace-coverage.ts
495
- /** Mirrors xstate@6.0.0-alpha.69 `matchesEventDescriptor` (src/utils.ts), which the package does not export. */
496
- function matchesDescriptor(eventType, descriptor) {
497
- if (descriptor === eventType || descriptor === "*") return true;
498
- if (!descriptor.endsWith(".*")) return false;
499
- const tokens = descriptor.split(".");
500
- const eventTokens = eventType.split(".");
501
- for (let index = 0; index < tokens.length; index++) {
502
- if (tokens[index] === "*") return index === tokens.length - 1;
503
- if (tokens[index] !== eventTokens[index]) return false;
1309
+ //#region src/trajectory.ts
1310
+ /**
1311
+ * Classifies one item. Event-shaped things (log entries, events) become events;
1312
+ * everything else is a state value. A snapshot is unwrapped to its `value`.
1313
+ * @internal
1314
+ */
1315
+ function normalize(item) {
1316
+ if (isRecord(item)) {
1317
+ const nested = item["event"];
1318
+ if (isRecord(nested) && typeof nested["type"] === "string") return {
1319
+ kind: "event",
1320
+ event: nested
1321
+ };
1322
+ if (typeof item["type"] === "string") return {
1323
+ kind: "event",
1324
+ event: item
1325
+ };
1326
+ if ("value" in item) return {
1327
+ kind: "state",
1328
+ value: item["value"]
1329
+ };
504
1330
  }
505
- return true;
1331
+ return {
1332
+ kind: "state",
1333
+ value: item
1334
+ };
506
1335
  }
507
- /** Whether a definition's `matches` pattern (e.g. an invoke's `actorId`) accepts the traced event. */
508
- function matchesPattern(definition, event) {
509
- const pattern = definition.matches;
510
- if (!isRecord(pattern) || !isRecord(event)) return true;
511
- return Object.entries(pattern).every(([key, value]) => Object.is(event[key], value));
1336
+ /** Every leaf of a state value as a dot path (`{ review: 'editing' }` → `review.editing`). @internal */
1337
+ function statePaths$1(value) {
1338
+ if (typeof value === "string") return [value];
1339
+ if (!isRecord(value)) return [];
1340
+ return Object.entries(value).flatMap(([key, child]) => statePaths$1(child).map((path) => `${key}.${path}`));
512
1341
  }
513
1342
  /**
514
- * The descriptor XState would try first at `node` for `event`, guards aside:
515
- * the exact key, then partial and wildcard keys longest first, as in
516
- * xstate@6.0.0-alpha.69 `getCandidates` (src/stateUtils.ts).
1343
+ * A string expectation matches a state value when it names one of its leaves or
1344
+ * an ancestor of one, so `'review'` matches `{ review: 'editing' }`. @internal
517
1345
  */
518
- function candidateAt(node, type, event) {
519
- const keys = [...node.transitions.keys()].filter((key) => key !== type && matchesDescriptor(type, key)).sort((a, b) => b.length - a.length);
520
- if (node.transitions.has(type)) keys.unshift(type);
521
- return keys.find((key) => node.transitions.get(key).some((d) => matchesPattern(d, event)));
1346
+ function stateMatchesPath(path, value) {
1347
+ return statePaths$1(value).some((leaf) => leaf === path || leaf.startsWith(`${path}.`));
1348
+ }
1349
+ function deepEqual(a, b) {
1350
+ if (Object.is(a, b)) return true;
1351
+ if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => deepEqual(item, b[index]));
1352
+ if (!isRecord(a) || !isRecord(b)) return false;
1353
+ const keys = Object.keys(a);
1354
+ return keys.length === Object.keys(b).length && keys.every((key) => deepEqual(a[key], b[key]));
522
1355
  }
523
- const pathOf = (node) => node.path.join(".");
524
- const sameLeaves = (a, b) => statePaths(a).sort().join("|") === statePaths(b).sort().join("|");
525
1356
  /**
526
- * The leaves a trace passed through inside its macrosteps: states an `always`
527
- * transition left before the step ended, which no traced snapshot holds.
528
- * Re-derived with XState's pure `getMicrosteps` from the snapshot before each
529
- * step (the previous step's, or the fresh start's microsteps from its input).
530
- * A step that cannot be recomputed, or whose recomputation ends elsewhere
531
- * than the traced snapshot, adds none. Only `value` and `context` are rebuilt,
532
- * so a guard that reads live children can take another route to the same
533
- * leaves, and that route is credited.
1357
+ * Normalizes ONE expected item into a predicate over actual items, so the
1358
+ * expectation is classified once instead of once per candidate:
1359
+ *
1360
+ * - A string expectation matches an event's `type`, or a state value it names.
1361
+ * - An event expectation matches an event with the same `type` whose other
1362
+ * declared keys deep-equal, so payload details are opt-in.
1363
+ * - A state-value expectation matches structurally.
1364
+ * @internal
534
1365
  */
535
- function passedThrough(machine, trace) {
536
- const leaves = /* @__PURE__ */ new Set();
537
- let resumed = false;
538
- let previous;
539
- for (const event of trace) {
540
- if (event.type === "run.start") resumed = event.snapshot !== void 0;
541
- if (event.type !== "machine.transition" || !isRecord(event.event)) continue;
542
- const snapshot = isRecord(event.snapshot) ? event.snapshot : void 0;
543
- try {
544
- const microsteps = event.event.type === "@xstate.init" ? resumed ? [] : getInitialMicrosteps(machine, event.event.input) : previous && sameLeaves(previous.value, valueOf(event.from)) ? getMicrosteps(machine, machine.resolveState({
545
- value: previous.value,
546
- context: previous.context
547
- }), event.event) : [];
548
- const last = microsteps.at(-1)?.[0];
549
- if (last && snapshot && sameLeaves(last.value, snapshot.value)) for (const [step] of microsteps) for (const leaf of statePaths(step.value)) leaves.add(leaf);
550
- } catch {}
551
- previous = snapshot;
1366
+ function matcherFor(expected) {
1367
+ if (typeof expected === "string") return (actual) => {
1368
+ const actualItem = normalize(actual);
1369
+ return actualItem.kind === "event" ? actualItem.event.type === expected : stateMatchesPath(expected, actualItem.value);
1370
+ };
1371
+ const expectedItem = normalize(expected);
1372
+ switch (expectedItem.kind) {
1373
+ case "event": return (actual) => {
1374
+ const actualItem = normalize(actual);
1375
+ return actualItem.kind === "event" && Object.keys(expectedItem.event).every((key) => deepEqual(expectedItem.event[key], actualItem.event[key]));
1376
+ };
1377
+ case "state": return (actual) => {
1378
+ const actualItem = normalize(actual);
1379
+ return actualItem.kind === "state" && deepEqual(expectedItem.value, actualItem.value);
1380
+ };
552
1381
  }
553
- return leaves;
554
1382
  }
555
- const ratioOf = (visited, total) => total === 0 ? 1 : visited / total;
556
1383
  /**
557
- * Measures how much of a machine a set of runs exercised: which atomic and
558
- * final states they visited, and which handlers they took. A handler is a
559
- * `(declaring state, event descriptor)` pair from `on`, invoke
560
- * `onDone`/`onError`, and `after`, with the targets it declares and the
561
- * leaves the runs landed in after it. Pure; reads only `machine.transition`
562
- * events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
1384
+ * Matches a run's trajectory against an expected one.
563
1385
  *
564
- * A step credits one handler per active leaf: the nearest node, from the leaf
565
- * up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
566
- * event, preferred in XState's order. Guards are not replayed, so a step
567
- * whose nearest handler was rejected and fell through to an ancestor is
568
- * credited to the nearest one. XState keys invoke and `after` handlers by
569
- * bare type (`xstate.done.actor`), one per state however many invokes; a
570
- * definition's `matches` (an invoke's `actorId`) must accept the traced event
571
- * for its state to be credited. Ignored steps credit no handler. `always`
572
- * handlers are listed as `unobservable`, never as unvisited; a state one
573
- * passes through is visited, re-derived from each step's microsteps.
1386
+ * Both trajectories may be state values or events collected from
1387
+ * `onTransition`;
1388
+ * items are compared by shape, so `['prompting', 'drafting']` scores a state
1389
+ * path and `['PROMPT_SUBMITTED', 'SEND']` scores an event log with the same
1390
+ * call.
574
1391
  *
575
- * @example
576
- * ```ts no-check
577
- * const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
578
- * expect(report.unvisitedHandlers).toEqual([]);
1392
+ * Semantics are an ordered subsequence: every expected item must appear in
1393
+ * `actual`, in order, with gaps allowed. Pass `{ exact: true }` to require
1394
+ * equality instead.
1395
+ *
1396
+ * @example Trajectory assertions in a test
1397
+ * ```ts
1398
+ * const statePath: unknown[] = [];
1399
+ * const result = await runToQuiescence(
1400
+ * createAgentRuntime(machine, {
1401
+ * executors,
1402
+ * onTransition: (snapshot) => statePath.push(snapshot.value),
1403
+ * }),
1404
+ * { input },
1405
+ * );
1406
+ *
1407
+ * const path = matchesTrajectory(statePath, ['prompting', 'drafting', 'sent']);
1408
+ * expect(path.matched, JSON.stringify(path.firstMiss)).toBe(true);
1409
+ * expect(matchesTrajectory(events, ['PROMPT_SUBMITTED', 'SEND']).matched).toBe(true);
1410
+ * ```
1411
+ *
1412
+ * @example Partial credit in an eval scorer
1413
+ * ```ts
1414
+ * const match = matchesTrajectory(output.statePath, expected.statePath);
1415
+ * return { name: 'state_path', score: match.score, metadata: { ...match.firstMiss } };
579
1416
  * ```
580
1417
  */
581
- function traceCoverage(machine, traces, options = {}) {
582
- const nodes = /* @__PURE__ */ new Map();
583
- const states = [];
584
- const handlers = [];
585
- const handlerAt = /* @__PURE__ */ new Map();
586
- const unobservable = [];
587
- const walk = (node) => {
588
- const state = pathOf(node);
589
- nodes.set(state, node);
590
- if (node.type === "atomic" || node.type === "final") states.push({
591
- state,
592
- runs: 0
593
- });
594
- const byEvent = /* @__PURE__ */ new Map();
595
- for (const [event, definitions] of node.transitions) {
596
- const dynamic = definitions.some((d) => typeof d.to === "function");
597
- const declared = [...new Set(definitions.flatMap((d) => (d.target ?? []).map(pathOf)))];
598
- const handler = {
599
- state,
600
- event,
601
- runs: 0,
602
- targets: {
603
- declared: dynamic ? "dynamic" : declared,
604
- seen: [],
605
- unreached: []
606
- }
607
- };
608
- handlers.push(handler);
609
- byEvent.set(event, handler);
610
- }
611
- handlerAt.set(node, byEvent);
612
- if (node.always?.length) unobservable.push({
613
- state,
614
- event: "always"
615
- });
616
- for (const child of Object.values(node.states)) walk(child);
617
- };
618
- walk(machine.root);
619
- const stateRuns = new Map(states.map((row) => [row.state, row]));
620
- const seen = /* @__PURE__ */ new Map();
621
- for (const trace of traces) {
622
- const steps = traceSteps(trace);
623
- const visited = new Set(steps[0]?.from);
624
- for (const event of trace) {
625
- if (event.type !== "machine.transition" || !isRecord(event.event)) continue;
626
- if (event.event.type !== "@xstate.init" || !isRecord(event.snapshot)) continue;
627
- for (const leaf of statePaths(event.snapshot.value)) visited.add(leaf);
628
- }
629
- if (unobservable.length > 0) for (const leaf of passedThrough(machine, trace)) visited.add(leaf);
630
- const credited = /* @__PURE__ */ new Set();
631
- for (const step of steps) {
632
- for (const leaf of step.to) visited.add(leaf);
633
- if (step.ignored) continue;
634
- const hits = /* @__PURE__ */ new Set();
635
- for (const leaf of step.from) for (let node = nodes.get(leaf); node; node = node.parent) {
636
- if (node.type === "final") continue;
637
- const event = candidateAt(node, step.event, step.payload);
638
- if (event === void 0) continue;
639
- hits.add(handlerAt.get(node).get(event));
1418
+ function matchesTrajectory(actual, expected, options = {}) {
1419
+ const expectedCount = expected.length;
1420
+ if (expectedCount === 0) throw new AgentError("empty-trajectory", "matchesTrajectory: expected trajectory must contain at least one item.");
1421
+ const exact = options.exact === true;
1422
+ let cursor = 0;
1423
+ let matchedCount = 0;
1424
+ let firstMiss;
1425
+ for (let index = 0; index < expectedCount; index++) {
1426
+ const want = expected[index];
1427
+ const matches = matcherFor(want);
1428
+ const searchedFrom = exact ? index : cursor;
1429
+ let found = -1;
1430
+ for (let at = searchedFrom; at < actual.length; at++) {
1431
+ if (matches(actual[at])) {
1432
+ found = at;
640
1433
  break;
641
1434
  }
642
- for (const handler of hits) {
643
- credited.add(handler);
644
- let leaves = seen.get(handler);
645
- if (!leaves) seen.set(handler, leaves = /* @__PURE__ */ new Set());
646
- for (const leaf of step.to) leaves.add(leaf);
647
- }
1435
+ if (exact) break;
648
1436
  }
649
- for (const state of visited) {
650
- const row = stateRuns.get(state);
651
- if (row) row.runs++;
1437
+ if (found === -1) {
1438
+ firstMiss = {
1439
+ index,
1440
+ expected: want,
1441
+ searchedFrom
1442
+ };
1443
+ break;
652
1444
  }
653
- for (const handler of credited) handler.runs++;
654
- }
655
- for (const handler of handlers) {
656
- const leaves = [...seen.get(handler) ?? []];
657
- handler.targets.seen = leaves;
658
- const { declared } = handler.targets;
659
- if (declared === "dynamic") continue;
660
- handler.targets.unreached = declared.filter((target) => !leaves.some((leaf) => leaf === target || leaf.startsWith(`${target}.`)));
661
- }
662
- const reachableComplete = options.reachable ? isExplorationComplete(options.reachable) : void 0;
663
- let unreachable;
664
- if (options.reachable && reachableComplete) {
665
- const reached = new Set(options.reachable.reachedStates.flatMap((value) => statePaths(value)));
666
- unreachable = states.filter((row) => !reached.has(row.state) && row.runs === 0).map((row) => row.state);
1445
+ matchedCount++;
1446
+ cursor = found + 1;
667
1447
  }
668
- const counted = unreachable ? states.filter((row) => !unreachable.includes(row.state)) : states;
669
- const unvisitedStates = counted.filter((row) => row.runs === 0).map((row) => row.state);
670
- const unvisitedHandlers = handlers.filter((handler) => handler.runs === 0).map(({ state, event }) => ({
671
- state,
672
- event
673
- }));
1448
+ const matched = !firstMiss && (!exact || actual.length === expectedCount);
1449
+ const denominator = exact ? Math.max(expectedCount, actual.length) : expectedCount;
674
1450
  return {
675
- states,
676
- handlers,
677
- unvisitedStates,
678
- unvisitedHandlers,
679
- unobservable,
680
- ...unreachable ? { unreachable } : {},
681
- ...reachableComplete === void 0 ? {} : { reachableComplete },
682
- ratio: {
683
- states: ratioOf(counted.length - unvisitedStates.length, counted.length),
684
- handlers: ratioOf(handlers.length - unvisitedHandlers.length, handlers.length)
685
- }
1451
+ matched,
1452
+ matchedCount,
1453
+ expectedCount,
1454
+ score: matched ? 1 : denominator === 0 ? 1 : matchedCount / denominator,
1455
+ ...firstMiss ? { firstMiss } : {}
686
1456
  };
687
1457
  }
688
- //#endregion
689
- //#region src/first-divergence.ts
690
- const defaultKey = (step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`;
691
- const isErrorLike = (value) => isError(value) || isRecord(value) && typeof value.name === "string" && typeof value.message === "string";
692
- const normalizeErrors = (value, seen = /* @__PURE__ */ new WeakSet()) => {
693
- if (value === null || typeof value !== "object") return value;
694
- if (seen.has(value)) return "[circular]";
695
- seen.add(value);
696
- try {
697
- if (isErrorLike(value) && (isError(value) || "stack" in value || "code" in value)) {
698
- const { name, message, code, cause } = value;
699
- return {
700
- name,
701
- message,
702
- ...typeof code === "string" ? { code } : {},
703
- ...cause !== void 0 ? { cause: normalizeErrors(cause, seen) } : {}
704
- };
705
- }
706
- if (Array.isArray(value)) return value.map((item) => normalizeErrors(item, seen));
707
- if (!isPlainObject(value)) return value;
708
- const copy = Object.create(null);
709
- for (const key of Object.keys(value)) copy[key] = normalizeErrors(value[key], seen);
710
- return copy;
711
- } finally {
712
- seen.delete(value);
713
- }
714
- };
715
- const payloadKey = (step) => {
716
- if (!isRecord(step.payload)) return `${defaultKey(step)}|${canonicalEventJson(step.payload)}`;
717
- const { sessionId: _sessionId, ...payload } = step.payload;
718
- return `${defaultKey(step)}|${canonicalEventJson(normalizeErrors(payload))}`;
719
- };
720
- const isTrace = (items) => items.length === 0 || items.some((item) => isRecord(item) && (item.type === "machine.transition" || item.type === "run.start" || item.type === "run.end"));
1458
+ //#endregion
1459
+ //#region src/seam.ts
1460
+ /** The only own keys an executor result carries. @internal */
1461
+ const TEXT_RESULT_KEYS = /* @__PURE__ */ new Set([
1462
+ "result",
1463
+ "messages",
1464
+ "usage",
1465
+ "raw"
1466
+ ]);
1467
+ /**
1468
+ * True when a scripted entry is the executor result rather than the output
1469
+ * value: it owns a `result` key and owns NOTHING outside the result's own
1470
+ * vocabulary. Bare `'result' in value` would swallow an output object's
1471
+ * siblings (`{ result: 'draft', confidence: 0.9 }` would lose `confidence`)
1472
+ * and would also match an inherited `result`. @internal
1473
+ */
1474
+ function isTextResult(value) {
1475
+ return Object.hasOwn(value, "result") && Object.keys(value).every((key) => TEXT_RESULT_KEYS.has(key));
1476
+ }
1477
+ /**
1478
+ * Resolves ONE scripted entry to an executor result: a function entry is
1479
+ * called with the request, and the value is taken as the executor result only
1480
+ * when it is one (see {@link isTextResult}). @internal
1481
+ */
1482
+ async function resolveSeamScriptEntry(entry, request, info) {
1483
+ const value = typeof entry === "function" ? await entry(request, info) : entry;
1484
+ return isRecord(value) && isTextResult(value) ? value : { result: value };
1485
+ }
1486
+ /** Names a pending text request in an error message. @internal */
1487
+ function describeText(request) {
1488
+ return request.name ? `'${request.name}' (model '${request.model}')` : `(model '${request.model}')`;
1489
+ }
721
1490
  /**
722
- * Finds the first step where a run departs from a reference, so a failure is
723
- * blamed on the state it started in rather than on every state the run went
724
- * through afterwards.
725
- *
726
- * - Against a reference **trace**, such as a passing run of the same
727
- * scenario, steps are compared in order by `key`. The first step that
728
- * differs, or the point where either trace ends early, is the divergence.
729
- * - Against an expected **trajectory** (`TrajectoryItem[]`, as for
730
- * {@link matchesTrajectory}), the run is laid out as its initial state, then
731
- * each step's event and new state. The first expected item it never reaches
732
- * is the divergence, blamed on the step after the last item it did reach. A
733
- * run that goes on past the trajectory has not diverged.
1491
+ * Stream semantics with no model: the whole text lands as one chunk. @internal
1492
+ */
1493
+ function emitScriptedChunk(result, info) {
1494
+ const text = isRecord(result) ? result["result"] : void 0;
1495
+ if (typeof text === "string") info?.onChunk?.(text);
1496
+ }
1497
+ /**
1498
+ * The seam's own answer, for scoring: the executor result's `result`.
1499
+ * @internal
1500
+ */
1501
+ async function seamOutputOf(result, request) {
1502
+ if (!isRecord(result)) return;
1503
+ return await normalizeGeneratorResult(result, `seam '${request.name ?? request.model}'`);
1504
+ }
1505
+ /**
1506
+ * Runs a machine end to end with one model call under test, and slices the run
1507
+ * at that call.
734
1508
  *
735
- * An empty reference is a trace with no steps, so any step diverges from it.
736
- * A trace is one run's events, live or serialized. Steps the machine ignored
737
- * changed nothing and are left out on both sides. Returns `undefined` when the
738
- * run does not diverge.
1509
+ * The `before`/`after` slices are the point: `after.statePath` is the branch
1510
+ * the seam chose and `after.events` is the same question against the durable
1511
+ * log, both ready for `matchesTrajectory`. The state slice splits where the
1512
+ * state path stood when the seam answered; the event slice splits at the
1513
+ * seam's own effect completion (the first `xstate.done.*`/`xstate.error.*`
1514
+ * entry appended after the call was made).
739
1515
  *
740
- * @example
741
- * ```ts no-check
742
- * const stats = foldTraces(traces, {
743
- * failed,
744
- * blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
1516
+ * @example No API key: the seam is scripted too, so the whole thing runs offline.
1517
+ * ```ts
1518
+ * const run = await runSeam(emailDrafter, {
1519
+ * scripts: { writeFollowUps: [followUps], emailDrafter: [draft] },
1520
+ * seam: { request: 'writeFollowUps' },
1521
+ * respond: ({ state }) => (state === 'prompting' ? { type: 'PROMPT_SUBMITTED', prompt } : null),
745
1522
  * });
1523
+ *
1524
+ * matchesTrajectory(run.after.statePath, ['needsMoreInfo', 'drafting']);
1525
+ * ```
1526
+ *
1527
+ * @example A candidate prompt at the seam, scored against the same rows.
1528
+ * ```ts
1529
+ * const { generateText } = createAiSdkExecutors({ models });
1530
+ * const run = await runSeam(emailDrafter, { ...row, candidate: generateText });
746
1531
  * ```
747
1532
  */
748
- function firstDivergence(actual, reference, options = {}) {
749
- const steps = traceSteps(actual).filter((step) => !step.ignored);
750
- return isTrace(reference) ? againstTrace(steps, traceSteps(reference).filter((step) => !step.ignored), options.key === "payload" ? payloadKey : options.key ?? defaultKey) : againstTrajectory(actual, steps, reference);
751
- }
752
- function diverged(steps, index, expected, initial = steps[0]?.fromValue) {
753
- const step = steps[index];
754
- return {
755
- index,
756
- state: (step ? step.from[0] : steps.at(-1)?.to[0] ?? statePaths(initial)[0]) ?? "",
757
- ...step?.decision === void 0 ? {} : { decision: step.decision },
758
- ...step ? { actual: step } : {},
759
- ...expected === void 0 ? {} : { expected }
1533
+ async function runSeam(machine, options) {
1534
+ const { seam, candidate } = options;
1535
+ const queues = new Map(Object.entries(options.scripts ?? {}).map(([key, answers]) => [key, [...answers]]));
1536
+ const statePath = [];
1537
+ let calls = 0;
1538
+ let seamMatches = 0;
1539
+ let seamOutput;
1540
+ let seamUsage;
1541
+ const callLog = [];
1542
+ let seamReached = false;
1543
+ let callsBeforeSeam = -1;
1544
+ let seamStateAt = 0;
1545
+ let seamEventAt = 0;
1546
+ let liveEvents = 0;
1547
+ const queueKeyOf = (request) => request.name !== void 0 && queues.has(request.name) ? request.name : request.model;
1548
+ /**
1549
+ * Consumes this request's slot in the call plan, or resolves `undefined` when
1550
+ * its queue is dry. With `repeatLast`, the last entry is replayed instead of
1551
+ * running dry.
1552
+ */
1553
+ const takeScriptedSlot = async (request, info) => {
1554
+ const queue = queues.get(queueKeyOf(request));
1555
+ if (!queue?.length) return;
1556
+ return resolveSeamScriptEntry(options.repeatLast && queue.length === 1 ? queue[0] : queue.shift(), request, info);
760
1557
  };
761
- }
762
- function againstTrace(steps, reference, key) {
763
- const length = Math.max(steps.length, reference.length);
764
- for (let index = 0; index < length; index++) {
765
- const step = steps[index];
766
- const want = reference[index];
767
- if (step && want && key(step) === key(want)) continue;
768
- return diverged(steps, index, want, steps[0]?.fromValue ?? reference[0]?.fromValue);
1558
+ const scriptedAnswer = async (request, info) => {
1559
+ const scripted = await takeScriptedSlot(request, info);
1560
+ if (!scripted) throw new AgentError("seam-script-exhausted", `runSeam: no scripted answer left for request ${describeText(request)}. Add an entry to \`scripts.${queueKeyOf(request)}\`, or pass \`repeatLast: true\` to replay its last entry down a longer branch.`);
1561
+ return scripted;
1562
+ };
1563
+ const route = async (request, info) => {
1564
+ const callIndex = calls++;
1565
+ const isSeam = request.name === seam.request && seamMatches++ === (seam.occurrence ?? 0);
1566
+ const ledger = (source) => {
1567
+ callLog.push({
1568
+ ...request.name !== void 0 ? { name: request.name } : {},
1569
+ model: request.model,
1570
+ key: queueKeyOf(request),
1571
+ source,
1572
+ seam: isSeam
1573
+ });
1574
+ };
1575
+ if (isSeam && candidate) {
1576
+ await takeScriptedSlot(request, info);
1577
+ seamReached = true;
1578
+ callsBeforeSeam = callIndex;
1579
+ seamStateAt = statePath.length;
1580
+ seamEventAt = liveEvents;
1581
+ const result = await candidate(request, info);
1582
+ seamOutput = await seamOutputOf(result, request);
1583
+ seamUsage = getCallUsage(result);
1584
+ ledger("candidate");
1585
+ return result;
1586
+ }
1587
+ const scripted = await scriptedAnswer(request, info);
1588
+ ledger("script");
1589
+ if (!isSeam) return scripted;
1590
+ seamReached = true;
1591
+ callsBeforeSeam = callIndex;
1592
+ seamStateAt = statePath.length;
1593
+ seamEventAt = liveEvents;
1594
+ seamOutput = scripted.result;
1595
+ seamUsage = scripted.usage;
1596
+ return scripted;
1597
+ };
1598
+ const executors = {
1599
+ ...options.executors,
1600
+ generateText: route,
1601
+ streamText: async (request, info) => {
1602
+ const result = await route(request, info);
1603
+ emitScriptedChunk(result, info);
1604
+ return result;
1605
+ }
1606
+ };
1607
+ let snapshot;
1608
+ let event;
1609
+ const events = [];
1610
+ let result;
1611
+ const maxTurns = options.maxTurns ?? 12;
1612
+ for (let turn = 0; turn <= maxTurns; turn++) {
1613
+ result = await runToQuiescence(createAgentRuntime(machine, {
1614
+ ...options.actors ? { actors: options.actors } : {},
1615
+ executors,
1616
+ onTransition: (next, causedBy) => {
1617
+ if (turn > 0 && causedBy.type === "@xstate.init") return;
1618
+ events.push(causedBy);
1619
+ liveEvents = events.length;
1620
+ statePath.push(next.value);
1621
+ }
1622
+ }), {
1623
+ ...snapshot ? { snapshot } : { input: options.input },
1624
+ ...event ? { event } : {}
1625
+ });
1626
+ if (result.status !== "idle" || turn === maxTurns) break;
1627
+ const next = await options.respond?.({
1628
+ snapshot: result.snapshot,
1629
+ state: result.snapshot.value,
1630
+ meta: getStateMeta(result.snapshot),
1631
+ turn,
1632
+ result
1633
+ });
1634
+ if (!next) break;
1635
+ event = next;
1636
+ snapshot = result.persist();
769
1637
  }
770
- }
771
- function againstTrajectory(trace, steps, expected) {
772
- const initial = initialValue(trace, steps);
773
- const timeline = initial === void 0 ? [] : [{ value: initial }];
774
- const stepAt = initial === void 0 ? [] : [0];
775
- steps.forEach((step, index) => {
776
- timeline.push(step.payload);
777
- stepAt.push(index);
778
- if (JSON.stringify(step.toValue) !== JSON.stringify(step.fromValue)) {
779
- timeline.push({ value: step.toValue });
780
- stepAt.push(index);
1638
+ let splitAt = events.length;
1639
+ if (seamReached) {
1640
+ splitAt = seamEventAt;
1641
+ for (let index = seamEventAt; index < events.length; index++) {
1642
+ const type = events[index].type;
1643
+ if (type.startsWith("xstate.done") || type.startsWith("xstate.error")) {
1644
+ splitAt = index;
1645
+ break;
1646
+ }
1647
+ }
1648
+ }
1649
+ const splitStateAt = seamReached ? seamStateAt : statePath.length;
1650
+ return {
1651
+ result,
1652
+ seamOutput,
1653
+ ...seamUsage !== void 0 ? { seamUsage } : {},
1654
+ callsBeforeSeam,
1655
+ calls: callLog,
1656
+ before: {
1657
+ statePath: statePath.slice(0, splitStateAt),
1658
+ events: events.slice(0, splitAt)
1659
+ },
1660
+ after: {
1661
+ statePath: statePath.slice(splitStateAt),
1662
+ events: events.slice(splitAt)
781
1663
  }
782
- });
783
- const match = matchesTrajectory(timeline, expected);
784
- if (!match.firstMiss) return void 0;
785
- const at = match.firstMiss.searchedFrom;
786
- return diverged(steps, at < timeline.length ? stepAt[at] ?? steps.length : steps.length, match.firstMiss.expected, initial);
787
- }
788
- function initialValue(trace, steps) {
789
- for (const event of trace) if (event.type === "machine.transition" && isRecord(event.snapshot)) return event.snapshot.value;
790
- return steps[0]?.fromValue;
1664
+ };
791
1665
  }
792
1666
  //#endregion
793
- //#region src/decision-requests.ts
1667
+ //#region src/internal/trace-states.ts
1668
+ /** The `value` of a trace event's `from` or `snapshot`. @internal */
1669
+ function valueOf(holder) {
1670
+ return isRecord(holder) ? holder.value : void 0;
1671
+ }
1672
+ /** A decision request on its first attempt (no failed attempts yet). @internal */
1673
+ function isFirstDecision(request) {
1674
+ return isRecord(request) && request.kind === "decision" && (!Array.isArray(request.attempts) || request.attempts.length === 0);
1675
+ }
794
1676
  /**
795
- * The decisions a run asked for, read from its trace: each decision's first
796
- * `request.start`, with the states the run was in. A trace is one run's events,
797
- * live or serialized with `serializeTraceEvent`. Retries of the same decision
798
- * are left out, so each decision appears once.
799
- *
800
- * Use the requests as eval rows (run a decider on each and compare its event
801
- * with a label) or as probe inputs for a policy. A serialized request keeps its
802
- * `options`, `metadata`, `system`, and `prompt`; event schemas do not survive
803
- * serialization, so replay closed choices (`options`) rather than open ones.
804
- *
805
- * @example
806
- * ```ts no-check
807
- * const rows = traces.flatMap((trace) => getDecisionRequests(trace, { name: "chooseResolution" }));
808
- * for (const { request } of rows) console.log((await decide(request)).event);
809
- * ```
1677
+ * Every active leaf of a state value as a dot path:
1678
+ * `'a'` → `['a']`; `{ a: 'b' }` → `['a.b']`; `{ p: { x: 'one', y: {} } }` → `['p.x.one', 'p.y']`.
1679
+ * @internal
810
1680
  */
811
- function getDecisionRequests(trace, options = {}) {
812
- const requests = [];
813
- let current = [];
814
- for (const event of trace) if (event.type === "machine.transition") current = statePaths(valueOf(event.snapshot));
815
- else if (event.type === "request.start" && isFirstDecision(event.request)) {
816
- if (options.name !== void 0 && event.request.name !== options.name) continue;
817
- requests.push({
818
- states: [...current],
819
- request: event.request
1681
+ function statePaths(value, prefix = "") {
1682
+ if (typeof value === "string") return [prefix + value];
1683
+ if (!isRecord(value)) return [];
1684
+ const entries = Object.entries(value);
1685
+ if (entries.length === 0) return prefix ? [prefix.slice(0, -1)] : [];
1686
+ return entries.flatMap(([key, child]) => statePaths(child, `${prefix}${key}.`));
1687
+ }
1688
+ /**
1689
+ * A trace's `machine.transition` events as steps, skipping `@xstate.init`.
1690
+ * A first-attempt decision's `request.start` names the decision on the step
1691
+ * that follows it, skipping `@agent.usage` steps. Reads live and serialized traces alike.
1692
+ * @internal
1693
+ */
1694
+ function traceSteps(trace) {
1695
+ const steps = [];
1696
+ let decision;
1697
+ for (const event of trace) {
1698
+ if (event.type === "request.start" && isFirstDecision(event.request)) {
1699
+ const name = event.request.name;
1700
+ decision = typeof name === "string" ? name : void 0;
1701
+ continue;
1702
+ }
1703
+ if (event.type !== "machine.transition") continue;
1704
+ const type = isRecord(event.event) ? event.event.type : void 0;
1705
+ if (type === "@xstate.init") continue;
1706
+ const fromValue = valueOf(event.from);
1707
+ const toValue = valueOf(event.snapshot);
1708
+ steps.push({
1709
+ from: statePaths(fromValue),
1710
+ fromValue,
1711
+ event: typeof type === "string" ? type : "",
1712
+ payload: event.event,
1713
+ to: statePaths(toValue),
1714
+ toValue,
1715
+ ignored: event.ignored === true,
1716
+ ...decision === void 0 || type === "@agent.usage" ? {} : { decision }
820
1717
  });
1718
+ if (type !== "@agent.usage") decision = void 0;
821
1719
  }
822
- return requests;
1720
+ return steps;
823
1721
  }
824
1722
  //#endregion
825
- //#region src/repeat-runs.ts
1723
+ //#region src/fold-traces.ts
826
1724
  /**
827
- * Summarizes repeated runs of each scenario: how often each passed, whether
828
- * all its runs passed (pass^k), how many distinct state paths they took, and
829
- * where they first went different ways. Pure. Use it for runs collected
830
- * elsewhere, such as CI shards or replays; {@link repeatRuns} collects them and
831
- * calls it.
1725
+ * Folds many runs' traces into per-state stats: how many runs visited each
1726
+ * state, how many of those failed, how many options the guards withheld
1727
+ * at decisions made there, how many events it ignored, and how many failures
1728
+ * `blame` traced back to it. Pure; reads only:
832
1729
  *
833
- * A scenario that passes 3 of 5 runs and `divergedAt: 'deciding'` points at one
834
- * decision, not at a flaky agent.
1730
+ * - `machine.transition`: the run visited `from.value` and `snapshot.value`;
1731
+ * with `ignored`, each of those states ignored one event.
1732
+ * - `request.start` of a decision, first attempt only: its `request.withheld`
1733
+ * counts toward the state the run was in, the `from.value` of the
1734
+ * transition its choice causes.
835
1735
  *
836
- * @example
837
- * ```ts no-check
838
- * const report = summarizeRepeats(shards.map((s) => ({ id: s.id, scenario: s, runs: s.runs })));
839
- * report.passHat(3); // the chance 3 runs of a scenario all pass
840
- * ```
841
- */
842
- function summarizeRepeats(groups) {
843
- const scenarios = groups.map((group) => summarize$1(group));
844
- const traces = groups.flatMap((group) => group.runs.flatMap((run) => run.trace ? [run.trace] : []));
845
- const most = Math.max(0, ...groups.map((group) => group.runs.length));
846
- const passHat = (j) => {
847
- if (!Number.isInteger(j) || j < 1 || j > most) throw new AgentError("pass-hat-out-of-range", `passHat: j must be a whole number from 1 to ${most}, got ${j}.`);
848
- const eligible = scenarios.filter((scenario) => scenario.runs.length >= j);
849
- return eligible.reduce((total, scenario) => total + allPass(scenario.passes, scenario.runs.length, j), 0) / eligible.length;
850
- };
851
- return {
852
- scenarios,
853
- passHat,
854
- traces
855
- };
856
- }
857
- /**
858
- * Runs every scenario `k` times and summarizes the runs with
859
- * {@link summarizeRepeats}. Runs are sequential, scenario by scenario. A run
860
- * that throws fails, with its message as `error` and no trace.
1736
+ * A trace is one run's events, live (`AgentTraceEvent`) or serialized
1737
+ * (`serializeTraceEvent`). A host-driven `decideNext` traces nothing; push
1738
+ * `{ type: 'request.start', request: result.request }` onto the trace before
1739
+ * sending its event. Parallel states count once per active leaf.
861
1740
  *
862
- * With a live model, one run per scenario says little: τ-bench found pass^k
863
- * falls well below the single-run rate. Report pass^k next to it.
1741
+ * `failures` counts a failed run against every state it visited, so states
1742
+ * after a bad decision rank as high as the decision. Sort by `blamed` to see
1743
+ * where failures start.
864
1744
  *
865
1745
  * @example
866
1746
  * ```ts no-check
867
- * const report = await repeatRuns({
868
- * scenarios,
869
- * k: 5,
870
- * run: async (scenario) => {
871
- * const trace: AgentTraceEvent[] = [];
872
- * const result = await runToQuiescence(
873
- * createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
874
- * { input: scenario.input },
875
- * );
876
- * return { trace, outcome: String(result.snapshot.value) };
877
- * },
878
- * passed: (scenario, { outcome }) => outcome === scenario.expected,
1747
+ * const stats = foldTraces(traces, {
1748
+ * failed: (trace) => trace.at(-1)?.type === 'run.end' && trace.at(-1).status === 'error',
879
1749
  * });
880
- * report.passHat(5);
881
1750
  * ```
882
1751
  */
883
- async function repeatRuns(options) {
884
- if (!Number.isInteger(options.k) || options.k < 1) throw new AgentError("invalid-repeat-count", `repeatRuns: k must be a whole number of 1 or more, got ${options.k}.`);
885
- const groups = [];
886
- for (const scenario of options.scenarios) {
887
- const runs = [];
888
- for (let attempt = 0; attempt < options.k; attempt++) try {
889
- const result = await options.run(scenario, attempt);
890
- runs.push({
891
- passed: options.passed(scenario, result),
892
- trace: result.trace,
893
- ...result.outcome === void 0 ? {} : { outcome: result.outcome }
894
- });
895
- } catch (error) {
896
- runs.push({
897
- passed: false,
898
- error: error instanceof Error ? error.message : String(error)
899
- });
1752
+ function foldTraces(traces, options) {
1753
+ const stats = /* @__PURE__ */ new Map();
1754
+ const statsFor = (state) => {
1755
+ let entry = stats.get(state);
1756
+ if (!entry) {
1757
+ entry = {
1758
+ state,
1759
+ visits: 0,
1760
+ failures: 0,
1761
+ withheld: 0,
1762
+ ignored: 0,
1763
+ blamed: 0
1764
+ };
1765
+ stats.set(state, entry);
900
1766
  }
901
- groups.push({
902
- id: scenario.id,
903
- scenario,
904
- runs
905
- });
1767
+ return entry;
1768
+ };
1769
+ for (const trace of traces) {
1770
+ const visited = /* @__PURE__ */ new Set();
1771
+ let current = [];
1772
+ const visit = (state) => {
1773
+ statsFor(state);
1774
+ visited.add(state);
1775
+ };
1776
+ for (const event of trace) if (event.type === "machine.transition") {
1777
+ for (const state of statePaths(valueOf(event.from))) visit(state);
1778
+ current = statePaths(valueOf(event.snapshot));
1779
+ for (const state of current) visit(state);
1780
+ if (event.ignored === true) for (const state of current) statsFor(state).ignored++;
1781
+ } else if (event.type === "request.start" && isFirstDecision(event.request)) {
1782
+ const withheld = event.request.withheld;
1783
+ if (!Array.isArray(withheld)) continue;
1784
+ for (const state of current) statsFor(state).withheld += withheld.length;
1785
+ }
1786
+ const failed = options.failed(trace);
1787
+ for (const state of visited) {
1788
+ const entry = statsFor(state);
1789
+ entry.visits++;
1790
+ if (failed) entry.failures++;
1791
+ }
1792
+ const blamed = failed ? options.blame?.(trace) : void 0;
1793
+ if (blamed !== void 0) statsFor(blamed).blamed++;
906
1794
  }
907
- return summarizeRepeats(groups);
1795
+ return [...stats.values()];
908
1796
  }
909
- function summarize$1(group) {
910
- const passes = group.runs.filter((run) => run.passed).length;
911
- const traces = group.runs.flatMap((run) => run.trace ? [run.trace] : []);
912
- const paths = new Set(traces.map((trace) => traceSteps(trace).filter((step) => !step.ignored).map((step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`).join(" ")));
913
- const reference = traces.find((trace) => trace.some((event) => event.type === "machine.transition"));
914
- let earliest;
915
- for (const trace of reference ? traces : []) {
916
- if (trace === reference) continue;
917
- const divergence = firstDivergence(trace, reference);
918
- if (divergence && (!earliest || divergence.index < earliest.index)) earliest = divergence;
1797
+ //#endregion
1798
+ //#region src/trace-coverage.ts
1799
+ /** Mirrors xstate@6.0.0-alpha.69 `matchesEventDescriptor` (src/utils.ts), which the package does not export. */
1800
+ function matchesDescriptor(eventType, descriptor) {
1801
+ if (descriptor === eventType || descriptor === "*") return true;
1802
+ if (!descriptor.endsWith(".*")) return false;
1803
+ const tokens = descriptor.split(".");
1804
+ const eventTokens = eventType.split(".");
1805
+ for (let index = 0; index < tokens.length; index++) {
1806
+ if (tokens[index] === "*") return index === tokens.length - 1;
1807
+ if (tokens[index] !== eventTokens[index]) return false;
919
1808
  }
920
- return {
921
- ...group,
922
- passes,
923
- passAny: passes > 0,
924
- passAll: group.runs.length > 0 && passes === group.runs.length,
925
- distinctPaths: paths.size,
926
- ...earliest ? { divergedAt: earliest.state } : {}
927
- };
1809
+ return true;
1810
+ }
1811
+ /** Whether a definition's `matches` pattern (e.g. an invoke's `actorId`) accepts the traced event. */
1812
+ function matchesPattern(definition, event) {
1813
+ const pattern = definition.matches;
1814
+ if (!isRecord(pattern) || !isRecord(event)) return true;
1815
+ return Object.entries(pattern).every(([key, value]) => Object.is(event[key], value));
1816
+ }
1817
+ /**
1818
+ * The descriptor XState would try first at `node` for `event`, guards aside:
1819
+ * the exact key, then partial and wildcard keys longest first, as in
1820
+ * xstate@6.0.0-alpha.69 `getCandidates` (src/stateUtils.ts).
1821
+ */
1822
+ function candidateAt(node, type, event) {
1823
+ const keys = [...node.transitions.keys()].filter((key) => key !== type && matchesDescriptor(type, key)).sort((a, b) => b.length - a.length);
1824
+ if (node.transitions.has(type)) keys.unshift(type);
1825
+ return keys.find((key) => node.transitions.get(key).some((d) => matchesPattern(d, event)));
928
1826
  }
929
- /** C(passes, j) / C(runs, j), as a product so large counts don't overflow. */
930
- function allPass(passes, runs, j) {
931
- let chance = 1;
932
- for (let i = 0; i < j; i++) chance *= Math.max(0, passes - i) / (runs - i);
933
- return chance;
1827
+ const pathOf = (node) => node.path.join(".");
1828
+ const sameLeaves = (a, b) => statePaths(a).sort().join("|") === statePaths(b).sort().join("|");
1829
+ /**
1830
+ * The leaves a trace passed through inside its macrosteps: states an `always`
1831
+ * transition left before the step ended, which no traced snapshot holds.
1832
+ * Re-derived with XState's pure `getMicrosteps` from the snapshot before each
1833
+ * step (the previous step's, or the fresh start's microsteps from its input).
1834
+ * A step that cannot be recomputed, or whose recomputation ends elsewhere
1835
+ * than the traced snapshot, adds none. Only `value` and `context` are rebuilt,
1836
+ * so a guard that reads live children can take another route to the same
1837
+ * leaves, and that route is credited.
1838
+ */
1839
+ function passedThrough(machine, trace) {
1840
+ const leaves = /* @__PURE__ */ new Set();
1841
+ let resumed = false;
1842
+ let previous;
1843
+ for (const event of trace) {
1844
+ if (event.type === "run.start") resumed = event.snapshot !== void 0;
1845
+ if (event.type !== "machine.transition" || !isRecord(event.event)) continue;
1846
+ const snapshot = isRecord(event.snapshot) ? event.snapshot : void 0;
1847
+ try {
1848
+ const microsteps = event.event.type === "@xstate.init" ? resumed ? [] : getInitialMicrosteps(machine, event.event.input) : previous && sameLeaves(previous.value, valueOf(event.from)) ? getMicrosteps(machine, machine.resolveState({
1849
+ value: previous.value,
1850
+ context: previous.context
1851
+ }), event.event) : [];
1852
+ const last = microsteps.at(-1)?.[0];
1853
+ if (last && snapshot && sameLeaves(last.value, snapshot.value)) for (const [step] of microsteps) for (const leaf of statePaths(step.value)) leaves.add(leaf);
1854
+ } catch {}
1855
+ previous = snapshot;
1856
+ }
1857
+ return leaves;
934
1858
  }
935
- //#endregion
936
- //#region src/judge-agreement.ts
1859
+ const ratioOf = (visited, total) => total === 0 ? 1 : visited / total;
937
1860
  /**
938
- * Measures how well a judge agrees with people on the same cases: accuracy,
939
- * per-label recall, the confusion matrix, and the rows that disagree. With
940
- * `positive`, also the TPR and TNR, the recommended view for binary judges
941
- * (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
942
- * Pure.
943
- *
944
- * Build rows from confirmed labels only; draft labels gate nothing. Group by
945
- * decision state or request name so each narrow judge is aligned on its own.
946
- * A row with no group key counts in the totals and in no group. Run it on a
947
- * dev split and report the held-out split separately.
1861
+ * Measures how much of a machine a set of runs exercised: which atomic and
1862
+ * final states they visited, and which handlers they took. A handler is a
1863
+ * `(declaring state, event descriptor)` pair from `on`, invoke
1864
+ * `onDone`/`onError`, and `after`, with the targets it declares and the
1865
+ * leaves the runs landed in after it. Pure; reads only `machine.transition`
1866
+ * events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
948
1867
  *
949
- * Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
1868
+ * A step credits one handler per active leaf: the nearest node, from the leaf
1869
+ * up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
1870
+ * event, preferred in XState's order. Guards are not replayed, so a step
1871
+ * whose nearest handler was rejected and fell through to an ancestor is
1872
+ * credited to the nearest one. XState keys invoke and `after` handlers by
1873
+ * bare type (`xstate.done.actor`), one per state however many invokes; a
1874
+ * definition's `matches` (an invoke's `actorId`) must accept the traced event
1875
+ * for its state to be credited. Ignored steps credit no handler. `always`
1876
+ * handlers are listed as `unobservable`, never as unvisited; a state one
1877
+ * passes through is visited, re-derived from each step's microsteps.
950
1878
  *
951
1879
  * @example
952
1880
  * ```ts no-check
953
- * const report = judgeAgreement(
954
- * labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
955
- * { positive: 'fail' },
956
- * );
957
- * report.binary; // { tpr: 0.9, tnr: 0.8 }
958
- * report.correct?.(0.3); // estimated true failure rate in production
1881
+ * const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
1882
+ * expect(report.unvisitedHandlers).toEqual([]);
959
1883
  * ```
960
1884
  */
961
- function judgeAgreement(rows, options = {}) {
962
- if (rows.length === 0) throw new AgentError("empty-agreement-rows", "judgeAgreement: rows must contain at least one row.");
963
- const report = summarize(rows, options.positive);
964
- const groupOf = options.groupBy ?? ((row) => row.group);
965
- if (options.groupBy || rows.some((row) => row.group !== void 0)) {
966
- const grouped = /* @__PURE__ */ new Map();
967
- for (const row of rows) {
968
- const key = groupOf(row);
969
- if (key === void 0) continue;
970
- const members = grouped.get(key);
971
- if (members) members.push(row);
972
- else grouped.set(key, [row]);
1885
+ function traceCoverage(machine, traces, options = {}) {
1886
+ const nodes = /* @__PURE__ */ new Map();
1887
+ const states = [];
1888
+ const handlers = [];
1889
+ const handlerAt = /* @__PURE__ */ new Map();
1890
+ const unobservable = [];
1891
+ const walk = (node) => {
1892
+ const state = pathOf(node);
1893
+ nodes.set(state, node);
1894
+ if (node.type === "atomic" || node.type === "final") states.push({
1895
+ state,
1896
+ runs: 0
1897
+ });
1898
+ const byEvent = /* @__PURE__ */ new Map();
1899
+ for (const [event, definitions] of node.transitions) {
1900
+ const dynamic = definitions.some((d) => typeof d.to === "function");
1901
+ const declared = [...new Set(definitions.flatMap((d) => (d.target ?? []).map(pathOf)))];
1902
+ const handler = {
1903
+ state,
1904
+ event,
1905
+ runs: 0,
1906
+ targets: {
1907
+ declared: dynamic ? "dynamic" : declared,
1908
+ seen: [],
1909
+ unreached: []
1910
+ }
1911
+ };
1912
+ handlers.push(handler);
1913
+ byEvent.set(event, handler);
973
1914
  }
974
- const groups = Object.create(null);
975
- for (const [key, members] of grouped) groups[key] = summarize(members, options.positive);
976
- report.groups = groups;
977
- }
978
- return report;
979
- }
980
- function summarize(rows, positive) {
981
- const perLabel = Object.create(null);
982
- const confusion = Object.create(null);
983
- const disagreements = [];
984
- let agreed = 0;
985
- for (const { id, human, judge } of rows) {
986
- const label = perLabel[human] ??= {
987
- human: 0,
988
- agreed: 0
989
- };
990
- const row = confusion[human] ??= Object.create(null);
991
- label.human++;
992
- row[judge] = (row[judge] ?? 0) + 1;
993
- if (human === judge) {
994
- label.agreed++;
995
- agreed++;
996
- } else disagreements.push({
997
- id,
998
- human,
999
- judge
1915
+ handlerAt.set(node, byEvent);
1916
+ if (node.always?.length) unobservable.push({
1917
+ state,
1918
+ event: "always"
1000
1919
  });
1920
+ for (const child of Object.values(node.states)) walk(child);
1921
+ };
1922
+ walk(machine.root);
1923
+ const stateRuns = new Map(states.map((row) => [row.state, row]));
1924
+ const seen = /* @__PURE__ */ new Map();
1925
+ for (const trace of traces) {
1926
+ const steps = traceSteps(trace);
1927
+ const visited = new Set(steps[0]?.from);
1928
+ for (const event of trace) {
1929
+ if (event.type !== "machine.transition" || !isRecord(event.event)) continue;
1930
+ if (event.event.type !== "@xstate.init" || !isRecord(event.snapshot)) continue;
1931
+ for (const leaf of statePaths(event.snapshot.value)) visited.add(leaf);
1932
+ }
1933
+ if (unobservable.length > 0) for (const leaf of passedThrough(machine, trace)) visited.add(leaf);
1934
+ const credited = /* @__PURE__ */ new Set();
1935
+ for (const step of steps) {
1936
+ for (const leaf of step.to) visited.add(leaf);
1937
+ if (step.ignored) continue;
1938
+ const hits = /* @__PURE__ */ new Set();
1939
+ for (const leaf of step.from) for (let node = nodes.get(leaf); node; node = node.parent) {
1940
+ if (node.type === "final") continue;
1941
+ const event = candidateAt(node, step.event, step.payload);
1942
+ if (event === void 0) continue;
1943
+ hits.add(handlerAt.get(node).get(event));
1944
+ break;
1945
+ }
1946
+ for (const handler of hits) {
1947
+ credited.add(handler);
1948
+ let leaves = seen.get(handler);
1949
+ if (!leaves) seen.set(handler, leaves = /* @__PURE__ */ new Set());
1950
+ for (const leaf of step.to) leaves.add(leaf);
1951
+ }
1952
+ }
1953
+ for (const state of visited) {
1954
+ const row = stateRuns.get(state);
1955
+ if (row) row.runs++;
1956
+ }
1957
+ for (const handler of credited) handler.runs++;
1001
1958
  }
1002
- const summary = {
1003
- total: rows.length,
1004
- accuracy: agreed / rows.length,
1005
- perLabel,
1006
- confusion,
1007
- disagreements
1959
+ for (const handler of handlers) {
1960
+ const leaves = [...seen.get(handler) ?? []];
1961
+ handler.targets.seen = leaves;
1962
+ const { declared } = handler.targets;
1963
+ if (declared === "dynamic") continue;
1964
+ handler.targets.unreached = declared.filter((target) => !leaves.some((leaf) => leaf === target || leaf.startsWith(`${target}.`)));
1965
+ }
1966
+ const reachableComplete = options.reachable ? isExplorationComplete(options.reachable) : void 0;
1967
+ let unreachable;
1968
+ if (options.reachable && reachableComplete) {
1969
+ const reached = new Set(options.reachable.reachedStates.flatMap((value) => statePaths(value)));
1970
+ unreachable = states.filter((row) => !reached.has(row.state) && row.runs === 0).map((row) => row.state);
1971
+ }
1972
+ const counted = unreachable ? states.filter((row) => !unreachable.includes(row.state)) : states;
1973
+ const unvisitedStates = counted.filter((row) => row.runs === 0).map((row) => row.state);
1974
+ const unvisitedHandlers = handlers.filter((handler) => handler.runs === 0).map(({ state, event }) => ({
1975
+ state,
1976
+ event
1977
+ }));
1978
+ return {
1979
+ states,
1980
+ handlers,
1981
+ unvisitedStates,
1982
+ unvisitedHandlers,
1983
+ unobservable,
1984
+ ...unreachable ? { unreachable } : {},
1985
+ ...reachableComplete === void 0 ? {} : { reachableComplete },
1986
+ ratio: {
1987
+ states: ratioOf(counted.length - unvisitedStates.length, counted.length),
1988
+ handlers: ratioOf(handlers.length - unvisitedHandlers.length, handlers.length)
1989
+ }
1008
1990
  };
1009
- if (positive === void 0) return summary;
1010
- let positives = 0;
1011
- let truePositives = 0;
1012
- let negatives = 0;
1013
- let trueNegatives = 0;
1014
- for (const { human, judge } of rows) if (human === positive) {
1015
- positives++;
1016
- if (judge === positive) truePositives++;
1017
- } else {
1018
- negatives++;
1019
- if (judge !== positive) trueNegatives++;
1991
+ }
1992
+ //#endregion
1993
+ //#region src/first-divergence.ts
1994
+ const defaultKey = (step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`;
1995
+ const isErrorLike = (value) => isError(value) || isRecord(value) && typeof value.name === "string" && typeof value.message === "string";
1996
+ const normalizeErrors = (value, seen = /* @__PURE__ */ new WeakSet()) => {
1997
+ if (value === null || typeof value !== "object") return value;
1998
+ if (seen.has(value)) return "[circular]";
1999
+ seen.add(value);
2000
+ try {
2001
+ if (isErrorLike(value) && (isError(value) || "stack" in value || "code" in value)) {
2002
+ const { name, message, code, cause } = value;
2003
+ return {
2004
+ name,
2005
+ message,
2006
+ ...typeof code === "string" ? { code } : {},
2007
+ ...cause !== void 0 ? { cause: normalizeErrors(cause, seen) } : {}
2008
+ };
2009
+ }
2010
+ if (Array.isArray(value)) return value.map((item) => normalizeErrors(item, seen));
2011
+ if (!isPlainObject(value)) return value;
2012
+ const copy = Object.create(null);
2013
+ for (const key of Object.keys(value)) copy[key] = normalizeErrors(value[key], seen);
2014
+ return copy;
2015
+ } finally {
2016
+ seen.delete(value);
1020
2017
  }
1021
- const tpr = positives === 0 ? void 0 : truePositives / positives;
1022
- const tnr = negatives === 0 ? void 0 : trueNegatives / negatives;
1023
- summary.binary = {
1024
- tpr,
1025
- tnr
1026
- };
1027
- if (tpr !== void 0 && tnr !== void 0 && tpr + tnr > 1) summary.correct = (p) => Math.min(1, Math.max(0, (p + tnr - 1) / (tpr + tnr - 1)));
1028
- return summary;
1029
- }
1030
- //#endregion
1031
- //#region src/compare-runs.ts
2018
+ };
2019
+ const payloadKey = (step) => {
2020
+ if (!isRecord(step.payload)) return `${defaultKey(step)}|${canonicalEventJson(step.payload)}`;
2021
+ const { sessionId: _sessionId, ...payload } = step.payload;
2022
+ return `${defaultKey(step)}|${canonicalEventJson(normalizeErrors(payload))}`;
2023
+ };
2024
+ const isTrace = (items) => items.length === 0 || items.some((item) => isRecord(item) && (item.type === "machine.transition" || item.type === "run.start" || item.type === "run.end"));
1032
2025
  /**
1033
- * Runs every scenario through a baseline and a candidate implementation and
1034
- * labels each difference, so a port can show where it behaves like the code it
1035
- * replaces and where it differs on purpose.
2026
+ * Finds the first step where a run departs from a reference, so a failure is
2027
+ * blamed on the state it started in rather than on every state the run went
2028
+ * through afterwards.
1036
2029
  *
1037
- * Parity is outcome-level: the terminal outcome and the effects in the world,
1038
- * not the sequence of tool calls, since a machine usually reorders steps. A
1039
- * candidate that broke no rule where the baseline broke one is intended, even
1040
- * when the outcome matches. A difference also counts as intended when only the
1041
- * candidate reached the scenario's `expected` outcome. Every other difference,
1042
- * and every candidate rule break, is a regression for a person to review.
2030
+ * - Against a reference **trace**, such as a passing run of the same
2031
+ * scenario, steps are compared in order by `key`. The first step that
2032
+ * differs, or the point where either trace ends early, is the divergence.
2033
+ * - Against an expected **trajectory** (`TrajectoryItem[]`, as for
2034
+ * {@link matchesTrajectory}), the run is laid out as its initial state, then
2035
+ * each step's event and new state. The first expected item it never reaches
2036
+ * is the divergence, blamed on the step after the last item it did reach. A
2037
+ * run that goes on past the trajectory has not diverged.
1043
2038
  *
1044
- * Runs are sequential, baseline first. A run that throws is recorded with
1045
- * outcome `'threw'` and the error message as its `detail`. Parity holds for
1046
- * these scenarios and these runs only; with a live model, run it more than once.
2039
+ * An empty reference is a trace with no steps, so any step diverges from it.
2040
+ * A trace is one run's events, live or serialized. Steps the machine ignored
2041
+ * changed nothing and are left out on both sides. Returns `undefined` when the
2042
+ * run does not diverge.
1047
2043
  *
1048
2044
  * @example
1049
2045
  * ```ts no-check
1050
- * const report = await compareRuns({
1051
- * scenarios,
1052
- * baseline: async (s) => toParityRun(await runRawLoop(s)),
1053
- * candidate: async (s) => toParityRun(await runDesk(s)),
2046
+ * const stats = foldTraces(traces, {
2047
+ * failed,
2048
+ * blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
1054
2049
  * });
1055
- * expect(report.ok).toBe(true);
1056
2050
  * ```
1057
2051
  */
1058
- async function compareRuns(options) {
1059
- const scenarios = [];
1060
- const counts = {
1061
- same: 0,
1062
- intended: 0,
1063
- regression: 0
1064
- };
1065
- for (const scenario of options.scenarios) {
1066
- const baseline = await settle(() => options.baseline(scenario));
1067
- const candidate = await settle(() => options.candidate(scenario));
1068
- const { label, reason } = labelRun(scenario.expected, baseline, candidate);
1069
- counts[label]++;
1070
- scenarios.push({
1071
- id: scenario.id,
1072
- scenario,
1073
- label,
1074
- reason,
1075
- baseline,
1076
- candidate
1077
- });
1078
- }
1079
- return {
1080
- scenarios,
1081
- counts,
1082
- ok: counts.regression === 0
1083
- };
1084
- }
1085
- async function settle(run) {
1086
- try {
1087
- return await run();
1088
- } catch (error) {
1089
- return {
1090
- outcome: "threw",
1091
- detail: error instanceof Error ? error.message : String(error)
1092
- };
1093
- }
2052
+ function firstDivergence(actual, reference, options = {}) {
2053
+ const steps = traceSteps(actual).filter((step) => !step.ignored);
2054
+ return isTrace(reference) ? againstTrace(steps, traceSteps(reference).filter((step) => !step.ignored), options.key === "payload" ? payloadKey : options.key ?? defaultKey) : againstTrajectory(actual, steps, reference);
1094
2055
  }
1095
- function labelRun(expected, baseline, candidate) {
1096
- if (candidate.violations?.length) return {
1097
- label: "regression",
1098
- reason: `candidate broke a rule: ${list(candidate.violations)}`
1099
- };
1100
- if (candidate.outcome === "threw") return {
1101
- label: "regression",
1102
- reason: `candidate threw: ${String(candidate.detail)}`
1103
- };
1104
- if (baseline.violations?.length) return {
1105
- label: "intended",
1106
- reason: `baseline broke a rule: ${list(baseline.violations)}`
1107
- };
1108
- const sameOutcome = baseline.outcome === candidate.outcome;
1109
- if (sameOutcome && structurallyEqual(baseline.effects, candidate.effects)) return {
1110
- label: "same",
1111
- reason: ""
1112
- };
1113
- if (expected !== void 0 && candidate.outcome === expected && baseline.outcome !== expected) return {
1114
- label: "intended",
1115
- reason: `only the candidate reached the expected '${expected}'`
1116
- };
2056
+ function diverged(steps, index, expected, initial = steps[0]?.fromValue) {
2057
+ const step = steps[index];
1117
2058
  return {
1118
- label: "regression",
1119
- reason: sameOutcome ? "effects differ" : `outcome differs: ${baseline.outcome} → ${candidate.outcome}`
1120
- };
1121
- }
1122
- function list(items) {
1123
- return items.map((item) => typeof item === "string" ? item : JSON.stringify(item)).join(", ");
1124
- }
1125
- function structurallyEqual(a, b) {
1126
- if (Object.is(a, b)) return true;
1127
- if (a instanceof Date || b instanceof Date) return a instanceof Date && b instanceof Date && a.getTime() === b.getTime();
1128
- if (a instanceof Map || b instanceof Map) return a instanceof Map && b instanceof Map && structurallyEqual([...a], [...b]);
1129
- if (a instanceof Set || b instanceof Set) return a instanceof Set && b instanceof Set && structurallyEqual([...a], [...b]);
1130
- if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => structurallyEqual(item, b[index]));
1131
- if (!isRecord(a) || !isRecord(b)) return false;
1132
- const keys = Object.keys(a);
1133
- return keys.length === Object.keys(b).length && keys.every((key) => Object.hasOwn(b, key) && structurallyEqual(a[key], b[key]));
1134
- }
1135
- //#endregion
1136
- //#region src/equivalence.ts
1137
- /**
1138
- * Thrown in replay when the port asks for something the recording cannot
1139
- * answer: a model call past the last recorded one, or a tool call the original
1140
- * never made. The effect is still recorded, so {@link compareRecordings}
1141
- * reports it.
1142
- */
1143
- var RecordingDivergenceError = class extends Error {
1144
- name = "RecordingDivergenceError";
1145
- };
1146
- function toRecordedError(error) {
1147
- if (!(error instanceof Error)) return {
1148
- name: "Error",
1149
- message: String(error)
1150
- };
1151
- const recorded = {
1152
- name: error.name,
1153
- message: error.message
1154
- };
1155
- const code = error.code;
1156
- if (code !== void 0) recorded.code = toJson(code);
1157
- const data = Object.fromEntries(Object.entries(error).filter(([key, value]) => key !== "code" && (typeof value === "number" || typeof value === "boolean")));
1158
- if (Object.keys(data).length) recorded.data = data;
1159
- const markers = /* @__PURE__ */ new Set();
1160
- for (let holder = error; holder; holder = Object.getPrototypeOf(holder)) for (const symbol of Object.getOwnPropertySymbols(holder)) {
1161
- const key = Symbol.keyFor(symbol);
1162
- if (key !== void 0 && error[symbol] === true) markers.add(key);
1163
- }
1164
- if (markers.size) recorded.markers = [...markers];
1165
- return recorded;
1166
- }
1167
- function fromRecordedError({ data, markers, ...fields }) {
1168
- const error = Object.assign(new Error(fields.message), fields, data);
1169
- for (const marker of markers ?? []) error[Symbol.for(marker)] = true;
1170
- return error;
1171
- }
1172
- /** What two runs compare of an error: its data and markers may differ. */
1173
- function errorIdentity(error) {
1174
- return error && {
1175
- name: error.name,
1176
- message: error.message,
1177
- code: error.code
2059
+ index,
2060
+ state: (step ? step.from[0] : steps.at(-1)?.to[0] ?? statePaths(initial)[0]) ?? "",
2061
+ ...step?.decision === void 0 ? {} : { decision: step.decision },
2062
+ ...step ? { actual: step } : {},
2063
+ ...expected === void 0 ? {} : { expected }
1178
2064
  };
1179
2065
  }
1180
- /** A JSON-safe copy: Dates become ISO strings, Errors become {@link RecordedError}. */
1181
- function toJson(value) {
1182
- if (value === void 0) return void 0;
1183
- return JSON.parse(JSON.stringify(value, function(key, item) {
1184
- const raw = this[key];
1185
- if (raw instanceof Date) return raw.toISOString();
1186
- if (item instanceof Error) return toRecordedError(item);
1187
- if (typeof item === "bigint") return item.toString();
1188
- if (item instanceof Uint8Array) return Array.from(item);
1189
- return item;
1190
- }));
1191
- }
1192
- /** Restores the `Date` timestamps the AI SDK puts on responses. */
1193
- function reviveTimestamps(value) {
1194
- if (value && typeof value === "object") {
1195
- for (const holder of [value, value.response]) if (holder && typeof holder["timestamp"] === "string") holder["timestamp"] = new Date(holder["timestamp"]);
2066
+ function againstTrace(steps, reference, key) {
2067
+ const length = Math.max(steps.length, reference.length);
2068
+ for (let index = 0; index < length; index++) {
2069
+ const step = steps[index];
2070
+ const want = reference[index];
2071
+ if (step && want && key(step) === key(want)) continue;
2072
+ return diverged(steps, index, want, steps[0]?.fromValue ?? reference[0]?.fromValue);
1196
2073
  }
1197
- return value;
1198
- }
1199
- function omit(value, keys) {
1200
- if (!value || typeof value !== "object") return value;
1201
- return Object.fromEntries(Object.entries(value).filter(([key]) => !keys.includes(key)));
1202
- }
1203
- /** Key-order-independent JSON, for equality. */
1204
- function stableStringify(value) {
1205
- if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
1206
- if (value && typeof value === "object") return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`;
1207
- return JSON.stringify(value) ?? "undefined";
1208
- }
1209
- function jsonEqual(a, b) {
1210
- return stableStringify(toJson(a)) === stableStringify(toJson(b));
1211
2074
  }
1212
- function streamFrom(parts) {
1213
- return new ReadableStream({ start(controller) {
1214
- for (const part of parts) controller.enqueue(reviveTimestamps(structuredClone(part)));
1215
- controller.close();
1216
- } });
2075
+ function againstTrajectory(trace, steps, expected) {
2076
+ const initial = initialValue(trace, steps);
2077
+ const timeline = initial === void 0 ? [] : [{ value: initial }];
2078
+ const stepAt = initial === void 0 ? [] : [0];
2079
+ steps.forEach((step, index) => {
2080
+ timeline.push(step.payload);
2081
+ stepAt.push(index);
2082
+ if (JSON.stringify(step.toValue) !== JSON.stringify(step.fromValue)) {
2083
+ timeline.push({ value: step.toValue });
2084
+ stepAt.push(index);
2085
+ }
2086
+ });
2087
+ const match = matchesTrajectory(timeline, expected);
2088
+ if (!match.firstMiss) return void 0;
2089
+ const at = match.firstMiss.searchedFrom;
2090
+ return diverged(steps, at < timeline.length ? stepAt[at] ?? steps.length : steps.length, match.firstMiss.expected, initial);
1217
2091
  }
1218
- async function* iterate(chunks) {
1219
- for (const chunk of chunks) yield structuredClone(chunk);
2092
+ function initialValue(trace, steps) {
2093
+ for (const event of trace) if (event.type === "machine.transition" && isRecord(event.snapshot)) return event.snapshot.value;
2094
+ return steps[0]?.fromValue;
1220
2095
  }
2096
+ //#endregion
2097
+ //#region src/propose-from-traces.ts
1221
2098
  /**
1222
- * Creates a {@link Recorder}. With no options it records: every wrapped call
1223
- * runs for real. With `replay` it answers from a recording instead.
2099
+ * Groups the approval verdicts in many runs' traces by `key` and proposes an
2100
+ * `allow` or `deny` rule for each. Pure and deterministic; a proposal is for
2101
+ * a person to review, and nothing here changes a machine.
2102
+ *
2103
+ * Reads `machine.transition` events: a verdict is a transition, not ignored,
2104
+ * out of `approvalState` on an approve or reject event. Rows come in the
2105
+ * order keys were first seen.
2106
+ *
2107
+ * @example
2108
+ * ```ts no-check
2109
+ * const rows = proposeApprovalRules(traces, {
2110
+ * approvalState: "awaitingApproval",
2111
+ * approveEvents: ["APPROVE"],
2112
+ * rejectEvents: ["REJECT"],
2113
+ * key: ({ snapshot }) => `${snapshot.context.pending.tool}:${snapshot.context.pending.path}`,
2114
+ * });
2115
+ * ```
1224
2116
  */
1225
- function createRecorder(options = {}) {
1226
- const effects = [];
1227
- const replay = options.replay?.effects;
1228
- const replayedModels = replay?.filter((effect) => effect.kind === "model");
1229
- const usedModels = /* @__PURE__ */ new Set();
1230
- const usedTools = /* @__PURE__ */ new Set();
1231
- const normalizeEffect = (effect, list) => options.normalize?.(effect, list.indexOf(effect), list) ?? effect;
1232
- let modelCalls = 0;
1233
- let toolsInFlight = 0;
1234
- /**
1235
- * The earliest unused recorded call that fits this one: one `translate`
1236
- * adapts, or one with the same method and shape. Not by position, so a call
1237
- * the port added or dropped does not shift every answer after it.
1238
- */
1239
- function nextReplayedModel(effect) {
1240
- for (const recorded of replayedModels) {
1241
- if (usedModels.has(recorded)) continue;
1242
- const translated = options.translate?.(structuredClone(recorded), effect);
1243
- if (translated) {
1244
- effect.translated = true;
1245
- return {
1246
- recorded,
1247
- answer: translated
1248
- };
1249
- }
1250
- if (recorded.method === effect.method && modelCallsMatch(options.modelCalls, recorded, effect)) return {
1251
- recorded,
1252
- answer: recorded
2117
+ function proposeApprovalRules(traces, options) {
2118
+ const rows = /* @__PURE__ */ new Map();
2119
+ const rowFor = (key) => {
2120
+ let row = rows.get(key);
2121
+ if (!row) {
2122
+ row = {
2123
+ key,
2124
+ approved: 0,
2125
+ rejected: 0,
2126
+ violations: 0,
2127
+ runs: [],
2128
+ proposal: null,
2129
+ support: 0
1253
2130
  };
2131
+ rows.set(key, row);
1254
2132
  }
1255
- }
1256
- function noAnswerMessage(effect) {
1257
- const unused = replayedModels.filter((recorded) => !usedModels.has(recorded));
1258
- return unused.length ? `Model call #${effect.call} (${effect.method}, ${JSON.stringify(modelCallShape(effect))}) fits no unused recorded call: ${unused.map((recorded) => `#${recorded.call} (${recorded.method}, ${JSON.stringify(modelCallShape(recorded))})`).join(", ")}. A recorded answer only fits a call of the same method and shape; pass \`translate\` to adapt one.` : `Model call #${effect.call} was never made by the recorded run (it made ${replayedModels.length}).`;
1259
- }
1260
- /**
1261
- * Records a model call around `run`, or answers it from the recording.
1262
- * `fallback` runs it on the stand-in when the recording has no answer.
1263
- */
1264
- async function modelCall(seam, method, request, streamed, run, fallback) {
1265
- const effect = {
1266
- kind: "model",
1267
- seam,
1268
- method,
1269
- call: modelCalls++,
1270
- request: toJson(omit(request, [
1271
- "abortSignal",
1272
- "headers",
1273
- "signal"
1274
- ]))
1275
- };
1276
- effects.push(effect);
1277
- if (replay) {
1278
- const replayed = nextReplayedModel(effect);
1279
- if (replayed) {
1280
- const { recorded, answer } = replayed;
1281
- usedModels.add(recorded);
1282
- effect.replayed = recorded.call;
1283
- if (answer.error) {
1284
- effect.error = answer.error;
1285
- throw fromRecordedError(answer.error);
1286
- }
1287
- effect.response = answer.response;
1288
- effect.stream = answer.stream;
1289
- if (seam === "openai") return streamed ? iterate(answer.stream ?? []) : structuredClone(answer.response);
1290
- return streamed ? {
1291
- ...reviveTimestamps(structuredClone(answer.response) ?? {}),
1292
- stream: streamFrom(answer.stream ?? [])
1293
- } : reviveTimestamps(structuredClone(answer.response));
1294
- }
1295
- if (!fallback) {
1296
- const error = new RecordingDivergenceError(noAnswerMessage(effect));
1297
- effect.error = toRecordedError(error);
1298
- throw error;
2133
+ return row;
2134
+ };
2135
+ const inState = (value) => statePaths(value).some((path) => path === options.approvalState || path.startsWith(`${options.approvalState}.`));
2136
+ let index = 0;
2137
+ for (const trace of traces) {
2138
+ const run = index++;
2139
+ const seen = /* @__PURE__ */ new Set();
2140
+ let snapshot;
2141
+ for (const event of trace) {
2142
+ if (event.type !== "machine.transition") continue;
2143
+ const before = snapshot;
2144
+ snapshot = event.snapshot;
2145
+ const type = isRecord(event.event) ? event.event.type : void 0;
2146
+ if (event.ignored === true || typeof type !== "string" || !inState(valueOf(event.from))) continue;
2147
+ const approve = options.approveEvents.includes(type);
2148
+ if (!approve && !options.rejectEvents.includes(type)) continue;
2149
+ const key = options.key({
2150
+ event: event.event,
2151
+ snapshot: before,
2152
+ trace
2153
+ });
2154
+ if (key === void 0) continue;
2155
+ const row = rowFor(key);
2156
+ if (approve) row.approved++;
2157
+ else row.rejected++;
2158
+ if (!seen.has(key)) {
2159
+ seen.add(key);
2160
+ row.runs.push(run);
1299
2161
  }
1300
- effect.unrecorded = true;
1301
- run = fallback;
1302
2162
  }
1303
- let result;
1304
- try {
1305
- result = await run();
1306
- } catch (error) {
1307
- effect.error = toRecordedError(error);
1308
- throw error;
1309
- }
1310
- if (!streamed) {
1311
- effect.response = toJson(result);
1312
- return result;
2163
+ for (const key of options.violations?.(trace) ?? []) {
2164
+ const row = rows.get(key);
2165
+ if (row) row.violations++;
1313
2166
  }
1314
- const parts = [];
1315
- effect.stream = parts;
1316
- if (seam === "openai") {
1317
- const source = result;
1318
- return (async function* () {
1319
- for await (const chunk of source) {
1320
- parts.push(toJson(chunk));
1321
- yield chunk;
1322
- }
1323
- })();
2167
+ }
2168
+ for (const row of rows.values()) {
2169
+ row.support = row.runs.length;
2170
+ row.proposal = row.approved > 0 && row.rejected === 0 && row.violations === 0 ? "allow" : row.rejected > 0 && row.approved === 0 ? "deny" : null;
2171
+ }
2172
+ return [...rows.values()];
2173
+ }
2174
+ /**
2175
+ * Proposes a value for each cap (retry count, steps, batch size) from many
2176
+ * runs' traces: the 95th percentile of what passing runs used, plus one, and
2177
+ * never below the current cap while runs still hit it. Pure and
2178
+ * deterministic; a proposal is for a person to review, and nothing here
2179
+ * changes a machine.
2180
+ *
2181
+ * @example
2182
+ * ```ts no-check
2183
+ * const [retries] = proposeCaps(traces, {
2184
+ * caps: [{ name: "retries", current: 3, failureState: "gaveUp", valueFromRun: countRetries }],
2185
+ * });
2186
+ * ```
2187
+ */
2188
+ function proposeCaps(traces, options) {
2189
+ const runs = [...traces];
2190
+ return options.caps.map((cap) => {
2191
+ let hits = 0;
2192
+ let max = null;
2193
+ const passing = [];
2194
+ for (const trace of runs) {
2195
+ const hit = hitCap(trace, cap);
2196
+ if (hit) hits++;
2197
+ const value = cap.valueFromRun(trace);
2198
+ if (value === void 0) continue;
2199
+ max = max === null ? value : Math.max(max, value);
2200
+ if (!hit) passing.push(value);
1324
2201
  }
1325
- const { stream, ...rest } = result;
1326
- effect.response = toJson(rest);
2202
+ passing.sort((a, b) => a - b);
2203
+ const p95 = passing.length ? passing[Math.ceil(.95 * passing.length) - 1] : null;
2204
+ const proposal = p95 === null ? null : hits > 0 && cap.current !== void 0 ? Math.max(p95 + 1, cap.current) : p95 + 1;
1327
2205
  return {
1328
- ...rest,
1329
- stream: stream.pipeThrough(new TransformStream({ transform(part, controller) {
1330
- parts.push(toJson(part));
1331
- controller.enqueue(part);
1332
- } }))
2206
+ name: cap.name,
2207
+ hits,
2208
+ passing: passing.length,
2209
+ max,
2210
+ p95,
2211
+ proposal
1333
2212
  };
2213
+ });
2214
+ }
2215
+ function hitCap(trace, cap) {
2216
+ for (const event of trace) {
2217
+ if (cap.failureState !== void 0 && event.type === "machine.transition" && statePaths(valueOf(event.snapshot)).some((path) => path === cap.failureState || path.startsWith(`${cap.failureState}.`))) return true;
2218
+ if (cap.failureOutput && event.type === "run.end" && event.status === "done") {
2219
+ if (cap.failureOutput(event.output)) return true;
2220
+ }
1334
2221
  }
1335
- /** `inputOf` picks the recorded input from the call's arguments. */
1336
- function tool(name, fn, inputOf = (args) => args.length === 1 ? args[0] : args) {
1337
- return ((...args) => {
1338
- const input = toJson(inputOf(args));
1339
- const effect = {
1340
- kind: "tool",
1341
- name,
1342
- step: modelCalls,
1343
- input,
1344
- ...toolsInFlight > 0 ? { concurrent: true } : {}
1345
- };
1346
- effects.push(effect);
1347
- return track(call(effect, args));
2222
+ return false;
2223
+ }
2224
+ //#endregion
2225
+ //#region src/decision-requests.ts
2226
+ /**
2227
+ * The decisions a run asked for, read from its trace: each decision's first
2228
+ * `request.start`, with the states the run was in. A trace is one run's events,
2229
+ * live or serialized with `serializeTraceEvent`. Retries of the same decision
2230
+ * are left out, so each decision appears once.
2231
+ *
2232
+ * Use the requests as eval rows (run a decider on each and compare its event
2233
+ * with a label) or as probe inputs for a policy. A serialized request keeps its
2234
+ * `options`, `metadata`, `system`, and `prompt`; event schemas do not survive
2235
+ * serialization, so replay closed choices (`options`) rather than open ones.
2236
+ *
2237
+ * @example
2238
+ * ```ts no-check
2239
+ * const rows = traces.flatMap((trace) => getDecisionRequests(trace, { name: "chooseResolution" }));
2240
+ * for (const { request } of rows) console.log((await decide(request)).event);
2241
+ * ```
2242
+ */
2243
+ function getDecisionRequests(trace, options = {}) {
2244
+ const requests = [];
2245
+ let current = [];
2246
+ for (const event of trace) if (event.type === "machine.transition") current = statePaths(valueOf(event.snapshot));
2247
+ else if (event.type === "request.start" && isFirstDecision(event.request)) {
2248
+ if (options.name !== void 0 && event.request.name !== options.name) continue;
2249
+ requests.push({
2250
+ states: [...current],
2251
+ request: event.request
1348
2252
  });
1349
- function call(effect, args) {
1350
- const { input } = effect;
1351
- if (replay && options.tools !== "live") {
1352
- const wanted = normalizeEffect(effect, effects).input;
1353
- const matches = (candidate) => candidate.kind === "tool" && !usedTools.has(candidate) && candidate.name === name && jsonEqual(normalizeEffect(candidate, replay).input, wanted);
1354
- const recorded = replay.find(matches);
1355
- if (!recorded && options.fallback?.tools) {
1356
- effect.unrecorded = true;
1357
- return runLive(effect, args);
1358
- }
1359
- const settle = (value) => {
1360
- if (recorded?.sync) return value();
1361
- try {
1362
- return Promise.resolve(value());
1363
- } catch (error) {
1364
- return Promise.reject(error);
1365
- }
1366
- };
1367
- return settle(() => {
1368
- if (!recorded) {
1369
- const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)}; the recorded run made no such call.`);
1370
- effect.error = toRecordedError(error);
1371
- throw error;
1372
- }
1373
- usedTools.add(recorded);
1374
- if (recorded.sync) effect.sync = true;
1375
- if (recorded.error) {
1376
- effect.error = recorded.error;
1377
- throw fromRecordedError(recorded.error);
1378
- }
1379
- effect.output = recorded.output;
1380
- return structuredClone(recorded.output);
1381
- });
1382
- }
1383
- return runLive(effect, args);
1384
- }
1385
- function runLive(effect, args) {
1386
- let output;
1387
- try {
1388
- output = fn(...args);
1389
- } catch (error) {
1390
- effect.sync = true;
1391
- effect.error = toRecordedError(error);
1392
- throw error;
1393
- }
1394
- if (!(output instanceof Promise)) {
1395
- effect.sync = true;
1396
- effect.output = toJson(output);
1397
- return output;
1398
- }
1399
- return output.then((value) => {
1400
- effect.output = toJson(value);
1401
- return value;
1402
- }, (error) => {
1403
- effect.error = toRecordedError(error);
1404
- throw error;
2253
+ }
2254
+ return requests;
2255
+ }
2256
+ //#endregion
2257
+ //#region src/repeat-runs.ts
2258
+ /**
2259
+ * Summarizes repeated runs of each scenario: how often each passed, whether
2260
+ * all its runs passed (pass^k), how many distinct state paths they took, and
2261
+ * where they first went different ways. Pure. Use it for runs collected
2262
+ * elsewhere, such as CI shards or replays; {@link repeatRuns} collects them and
2263
+ * calls it.
2264
+ *
2265
+ * A scenario that passes 3 of 5 runs and `divergedAt: 'deciding'` points at one
2266
+ * decision, not at a flaky agent.
2267
+ *
2268
+ * @example
2269
+ * ```ts no-check
2270
+ * const report = summarizeRepeats(shards.map((s) => ({ id: s.id, scenario: s, runs: s.runs })));
2271
+ * report.passHat(3); // the chance 3 runs of a scenario all pass
2272
+ * ```
2273
+ */
2274
+ function summarizeRepeats(groups) {
2275
+ const scenarios = groups.map((group) => summarize$1(group));
2276
+ const traces = groups.flatMap((group) => group.runs.flatMap((run) => run.trace ? [run.trace] : []));
2277
+ const most = Math.max(0, ...groups.map((group) => group.runs.length));
2278
+ const passHat = (j) => {
2279
+ if (!Number.isInteger(j) || j < 1 || j > most) throw new AgentError("pass-hat-out-of-range", `passHat: j must be a whole number from 1 to ${most}, got ${j}.`);
2280
+ const eligible = scenarios.filter((scenario) => scenario.runs.length >= j);
2281
+ return eligible.reduce((total, scenario) => total + allPass(scenario.passes, scenario.runs.length, j), 0) / eligible.length;
2282
+ };
2283
+ return {
2284
+ scenarios,
2285
+ passHat,
2286
+ traces
2287
+ };
2288
+ }
2289
+ /**
2290
+ * Runs every scenario `k` times and summarizes the runs with
2291
+ * {@link summarizeRepeats}. Runs are sequential, scenario by scenario. A run
2292
+ * that throws fails, with its message as `error` and no trace.
2293
+ *
2294
+ * With a live model, one run per scenario says little: τ-bench found pass^k
2295
+ * falls well below the single-run rate. Report pass^k next to it.
2296
+ *
2297
+ * @example
2298
+ * ```ts no-check
2299
+ * const report = await repeatRuns({
2300
+ * scenarios,
2301
+ * k: 5,
2302
+ * run: async (scenario) => {
2303
+ * const trace: AgentTraceEvent[] = [];
2304
+ * const result = await runToQuiescence(
2305
+ * createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
2306
+ * { input: scenario.input },
2307
+ * );
2308
+ * return { trace, outcome: String(result.snapshot.value) };
2309
+ * },
2310
+ * passed: (scenario, { outcome }) => outcome === scenario.expected,
2311
+ * });
2312
+ * report.passHat(5);
2313
+ * ```
2314
+ */
2315
+ async function repeatRuns(options) {
2316
+ if (!Number.isInteger(options.k) || options.k < 1) throw new AgentError("invalid-repeat-count", `repeatRuns: k must be a whole number of 1 or more, got ${options.k}.`);
2317
+ const groups = [];
2318
+ for (const scenario of options.scenarios) {
2319
+ const runs = [];
2320
+ for (let attempt = 0; attempt < options.k; attempt++) try {
2321
+ const result = await options.run(scenario, attempt);
2322
+ runs.push({
2323
+ passed: options.passed(scenario, result),
2324
+ trace: result.trace,
2325
+ ...result.outcome === void 0 ? {} : { outcome: result.outcome }
2326
+ });
2327
+ } catch (error) {
2328
+ runs.push({
2329
+ passed: false,
2330
+ error: error instanceof Error ? error.message : String(error)
1405
2331
  });
1406
2332
  }
2333
+ groups.push({
2334
+ id: scenario.id,
2335
+ scenario,
2336
+ runs
2337
+ });
1407
2338
  }
1408
- /** Counts an async tool call as running until it settles. */
1409
- function track(result) {
1410
- if (result instanceof Promise) {
1411
- toolsInFlight++;
1412
- const settled = () => {
1413
- toolsInFlight--;
1414
- };
1415
- result.then(settled, settled);
1416
- }
1417
- return result;
2339
+ return summarizeRepeats(groups);
2340
+ }
2341
+ function summarize$1(group) {
2342
+ const passes = group.runs.filter((run) => run.passed).length;
2343
+ const traces = group.runs.flatMap((run) => run.trace ? [run.trace] : []);
2344
+ const paths = new Set(traces.map((trace) => traceSteps(trace).filter((step) => !step.ignored).map((step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`).join(" ")));
2345
+ const reference = traces.find((trace) => trace.some((event) => event.type === "machine.transition"));
2346
+ let earliest;
2347
+ for (const trace of reference ? traces : []) {
2348
+ if (trace === reference) continue;
2349
+ const divergence = firstDivergence(trace, reference);
2350
+ if (divergence && (!earliest || divergence.index < earliest.index)) earliest = divergence;
1418
2351
  }
1419
2352
  return {
1420
- model(model) {
1421
- const base = model;
1422
- const stand = options.fallback?.model;
1423
- const wrapped = Object.create(base);
1424
- for (const method of [
1425
- "doGenerate",
1426
- "doStream",
1427
- "doDecide",
1428
- "doEvaluate"
1429
- ]) {
1430
- if (typeof base[method] !== "function") continue;
1431
- const invoke = (target, callOptions) => target[method].call(target, callOptions);
1432
- wrapped[method] = (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0);
1433
- }
1434
- return wrapped;
1435
- },
1436
- openai(client) {
1437
- const wrapCreate = (resource, method) => new Proxy(resource, { get(target, key, receiver) {
1438
- if (key !== "create") return Reflect.get(target, key, receiver);
1439
- return (params, requestOptions) => {
1440
- let api;
1441
- const create = (resource) => () => {
1442
- api = Reflect.get(resource, "create").call(resource, params, requestOptions);
1443
- return Promise.resolve(api);
1444
- };
1445
- const stand = fallbackResource(method);
1446
- const promise = modelCall("openai", method, params, params?.stream === true, create(target), stand && create(stand));
1447
- return Object.assign(promise, { withResponse: async () => {
1448
- const data = await promise;
1449
- const meta = await api?.withResponse?.();
1450
- return {
1451
- data,
1452
- response: meta?.response ?? new Response(null),
1453
- request_id: meta?.request_id ?? null
1454
- };
1455
- } });
1456
- };
1457
- } });
1458
- const fallbackResource = (method) => {
1459
- let value = options.fallback?.openai;
1460
- for (const key of method.split(".").slice(0, -1)) value = value?.[key];
1461
- return value && typeof value === "object" ? value : void 0;
1462
- };
1463
- const wrapped = /* @__PURE__ */ new Map();
1464
- const wrap = (key, make) => {
1465
- if (!wrapped.has(key)) wrapped.set(key, make());
1466
- return wrapped.get(key);
1467
- };
1468
- return new Proxy(client, { get(target, key, receiver) {
1469
- const value = Reflect.get(target, key, receiver);
1470
- if (key === "responses" && value) return wrap(key, () => wrapCreate(value, "responses.create"));
1471
- if (key === "chat" && value) return wrap(key, () => new Proxy(value, { get(chat, chatKey, chatReceiver) {
1472
- const inner = Reflect.get(chat, chatKey, chatReceiver);
1473
- return chatKey === "completions" && inner ? wrap("chat.completions", () => wrapCreate(inner, "chat.completions.create")) : inner;
1474
- } }));
1475
- return value;
1476
- } });
1477
- },
1478
- tool: (name, fn) => tool(name, fn),
1479
- tools(tools) {
1480
- return Object.fromEntries(Object.entries(tools).map(([name, definition]) => [name, definition.execute ? {
1481
- ...definition,
1482
- execute: tool(name, definition.execute, (args) => args[0])
1483
- } : definition]));
1484
- },
1485
- output(value) {
1486
- effects.push({
1487
- kind: "output",
1488
- value: toJson(value)
1489
- });
1490
- },
1491
- get recording() {
1492
- return {
1493
- version: 1,
1494
- effects: toJson(effects)
1495
- };
1496
- }
2353
+ ...group,
2354
+ passes,
2355
+ passAny: passes > 0,
2356
+ passAll: group.runs.length > 0 && passes === group.runs.length,
2357
+ distinctPaths: paths.size,
2358
+ ...earliest ? { divergedAt: earliest.state } : {}
1497
2359
  };
1498
2360
  }
1499
- /** True when two model calls offer the same tool names and response format. */
1500
- function sameModelCallShape(expected, actual) {
1501
- return jsonEqual(modelCallShape(expected), modelCallShape(actual));
1502
- }
1503
- function modelCallsMatch(match, expected, actual) {
1504
- if (match === "none") return true;
1505
- if (typeof match === "function") return match(expected, actual);
1506
- return sameModelCallShape(expected, actual);
1507
- }
1508
- function shapeMessage(expected, actual) {
1509
- return `Model call #${expected.call} changed shape: ${JSON.stringify(modelCallShape(expected))} → ${JSON.stringify(modelCallShape(actual))}. A recorded answer only fits a call of the same shape.`;
2361
+ /** C(passes, j) / C(runs, j), as a product so large counts don't overflow. */
2362
+ function allPass(passes, runs, j) {
2363
+ let chance = 1;
2364
+ for (let i = 0; i < j; i++) chance *= Math.max(0, passes - i) / (runs - i);
2365
+ return chance;
1510
2366
  }
2367
+ //#endregion
2368
+ //#region src/judge-agreement.ts
1511
2369
  /**
1512
- * Offered tool names and the response format, or a decision's questions and
1513
- * their options: what a recorded answer depends on.
2370
+ * Measures how well a judge agrees with people on the same cases: accuracy,
2371
+ * per-label recall, the confusion matrix, and the rows that disagree. With
2372
+ * `positive`, also the TPR and TNR, the recommended view for binary judges
2373
+ * (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
2374
+ * Pure.
2375
+ *
2376
+ * Build rows from confirmed labels only; draft labels gate nothing. Group by
2377
+ * decision state or request name so each narrow judge is aligned on its own.
2378
+ * A row with no group key counts in the totals and in no group. Run it on a
2379
+ * dev split and report the held-out split separately.
2380
+ *
2381
+ * Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
2382
+ *
2383
+ * @example
2384
+ * ```ts no-check
2385
+ * const report = judgeAgreement(
2386
+ * labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
2387
+ * { positive: 'fail' },
2388
+ * );
2389
+ * report.binary; // { tpr: 0.9, tnr: 0.8 }
2390
+ * report.correct?.(0.3); // estimated true failure rate in production
2391
+ * ```
1514
2392
  */
1515
- function modelCallShape(effect) {
1516
- const request = effect.request ?? {};
1517
- const questions = request["questions"];
1518
- if (questions && typeof questions === "object") return { questions: Object.entries(questions).map(([name, question]) => {
1519
- const criteria = question["criteria"];
1520
- if (question["type"] === "choice" && criteria && typeof criteria === "object") return [
1521
- name,
1522
- question["type"],
1523
- Object.keys(criteria).sort()
1524
- ];
1525
- if (question["type"] === "score" && Array.isArray(criteria)) return [
1526
- name,
1527
- question["type"],
1528
- criteria.length
1529
- ];
1530
- return [name, question["type"]];
1531
- }).sort((x, y) => String(x[0]).localeCompare(String(y[0]))) };
1532
- return {
1533
- tools: (Array.isArray(request["tools"]) ? request["tools"] : []).map((tool) => tool["name"] ?? tool["function"]?.["name"]).sort(),
1534
- format: request["responseFormat"]?.["type"] ?? request["response_format"]?.["type"] ?? (request["text"]?.["format"])?.["type"] ?? "text"
2393
+ function judgeAgreement(rows, options = {}) {
2394
+ if (rows.length === 0) throw new AgentError("empty-agreement-rows", "judgeAgreement: rows must contain at least one row.");
2395
+ const report = summarize(rows, options.positive);
2396
+ const groupOf = options.groupBy ?? ((row) => row.group);
2397
+ if (options.groupBy || rows.some((row) => row.group !== void 0)) {
2398
+ const grouped = /* @__PURE__ */ new Map();
2399
+ for (const row of rows) {
2400
+ const key = groupOf(row);
2401
+ if (key === void 0) continue;
2402
+ const members = grouped.get(key);
2403
+ if (members) members.push(row);
2404
+ else grouped.set(key, [row]);
2405
+ }
2406
+ const groups = Object.create(null);
2407
+ for (const [key, members] of grouped) groups[key] = summarize(members, options.positive);
2408
+ report.groups = groups;
2409
+ }
2410
+ return report;
2411
+ }
2412
+ function summarize(rows, positive) {
2413
+ const perLabel = Object.create(null);
2414
+ const confusion = Object.create(null);
2415
+ const disagreements = [];
2416
+ let agreed = 0;
2417
+ for (const { id, human, judge } of rows) {
2418
+ const label = perLabel[human] ??= {
2419
+ human: 0,
2420
+ agreed: 0
2421
+ };
2422
+ const row = confusion[human] ??= Object.create(null);
2423
+ label.human++;
2424
+ row[judge] = (row[judge] ?? 0) + 1;
2425
+ if (human === judge) {
2426
+ label.agreed++;
2427
+ agreed++;
2428
+ } else disagreements.push({
2429
+ id,
2430
+ human,
2431
+ judge
2432
+ });
2433
+ }
2434
+ const summary = {
2435
+ total: rows.length,
2436
+ accuracy: agreed / rows.length,
2437
+ perLabel,
2438
+ confusion,
2439
+ disagreements
2440
+ };
2441
+ if (positive === void 0) return summary;
2442
+ let positives = 0;
2443
+ let truePositives = 0;
2444
+ let negatives = 0;
2445
+ let trueNegatives = 0;
2446
+ for (const { human, judge } of rows) if (human === positive) {
2447
+ positives++;
2448
+ if (judge === positive) truePositives++;
2449
+ } else {
2450
+ negatives++;
2451
+ if (judge !== positive) trueNegatives++;
2452
+ }
2453
+ const tpr = positives === 0 ? void 0 : truePositives / positives;
2454
+ const tnr = negatives === 0 ? void 0 : trueNegatives / negatives;
2455
+ summary.binary = {
2456
+ tpr,
2457
+ tnr
1535
2458
  };
2459
+ if (tpr !== void 0 && tnr !== void 0 && tpr + tnr > 1) summary.correct = (p) => Math.min(1, Math.max(0, (p + tnr - 1) / (tpr + tnr - 1)));
2460
+ return summary;
1536
2461
  }
2462
+ //#endregion
2463
+ //#region src/compare-runs.ts
1537
2464
  /**
1538
- * Puts each group of concurrent tool calls in a stable order, so parallel tool
1539
- * calls that finished in a different order still compare equal. A tool call
1540
- * that started after the previous one settled begins a new group, so
1541
- * sequential calls keep their order.
2465
+ * Runs every scenario through a baseline and a candidate implementation and
2466
+ * labels each difference, so a port can show where it behaves like the code it
2467
+ * replaces and where it differs on purpose.
2468
+ *
2469
+ * Parity is outcome-level: the terminal outcome and the effects in the world,
2470
+ * not the sequence of tool calls, since a machine usually reorders steps. A
2471
+ * candidate that broke no rule where the baseline broke one is intended, even
2472
+ * when the outcome matches. A difference also counts as intended when only the
2473
+ * candidate reached the scenario's `expected` outcome. Every other difference,
2474
+ * and every candidate rule break, is a regression for a person to review.
2475
+ *
2476
+ * Runs are sequential, baseline first. A run that throws is recorded with
2477
+ * outcome `'threw'` and the error message as its `detail`. Parity holds for
2478
+ * these scenarios and these runs only; with a live model, run it more than once.
2479
+ *
2480
+ * @example
2481
+ * ```ts no-check
2482
+ * const report = await compareRuns({
2483
+ * scenarios,
2484
+ * baseline: async (s) => toParityRun(await runRawLoop(s)),
2485
+ * candidate: async (s) => toParityRun(await runDesk(s)),
2486
+ * });
2487
+ * expect(report.ok).toBe(true);
2488
+ * ```
1542
2489
  */
1543
- function normalize(effects) {
1544
- const result = [];
1545
- let batch = [];
1546
- const flush = () => {
1547
- result.push(...batch.sort((a, b) => stableStringify([a.name, a.input]).localeCompare(stableStringify([b.name, b.input]))));
1548
- batch = [];
2490
+ async function compareRuns(options) {
2491
+ const scenarios = [];
2492
+ const counts = {
2493
+ same: 0,
2494
+ intended: 0,
2495
+ regression: 0
1549
2496
  };
1550
- for (const effect of effects) {
1551
- if (effect.kind !== "tool" || !effect.concurrent) flush();
1552
- if (effect.kind === "tool") batch.push(effect);
1553
- else result.push(effect);
2497
+ for (const scenario of options.scenarios) {
2498
+ const baseline = await settle$1(() => options.baseline(scenario));
2499
+ const candidate = await settle$1(() => options.candidate(scenario));
2500
+ const { label, reason } = labelRun(scenario.expected, baseline, candidate);
2501
+ counts[label]++;
2502
+ scenarios.push({
2503
+ id: scenario.id,
2504
+ scenario,
2505
+ label,
2506
+ reason,
2507
+ baseline,
2508
+ candidate
2509
+ });
1554
2510
  }
1555
- flush();
1556
- return result;
2511
+ return {
2512
+ scenarios,
2513
+ counts,
2514
+ ok: counts.regression === 0
2515
+ };
1557
2516
  }
1558
- function describe(effect) {
1559
- switch (effect.kind) {
1560
- case "model": return `model call #${effect.call} (${effect.method})`;
1561
- case "tool": return `tool '${effect.name}' with ${JSON.stringify(effect.input)}`;
1562
- case "output": return `output ${JSON.stringify(effect.value)}`;
2517
+ async function settle$1(run) {
2518
+ try {
2519
+ return await run();
2520
+ } catch (error) {
2521
+ return {
2522
+ outcome: "threw",
2523
+ detail: error instanceof Error ? error.message : String(error)
2524
+ };
1563
2525
  }
1564
2526
  }
1565
- /** Why two effects differ, or `undefined` when they are the same. */
1566
- function differ(left, right, options) {
1567
- if (left.kind !== right.kind) return {
1568
- reason: "kind",
1569
- message: `Expected ${describe(left)}; the port made ${describe(right)}.`
2527
+ function labelRun(expected, baseline, candidate) {
2528
+ if (candidate.violations?.length) return {
2529
+ label: "regression",
2530
+ reason: `candidate broke a rule: ${list(candidate.violations)}`
1570
2531
  };
1571
- if (left.kind === "model" && right.kind === "model") {
1572
- if (right.translated && right.replayed === left.call) return void 0;
1573
- if (left.method !== right.method) return {
1574
- reason: "model-method",
1575
- message: `Model call #${left.call} used ${left.method}; the port used ${right.method}.`
1576
- };
1577
- if (!modelCallsMatch(options.modelCalls, left, right)) return {
1578
- reason: "model-call-shape",
1579
- message: shapeMessage(left, right)
1580
- };
1581
- if (!jsonEqual(errorIdentity(left.error), errorIdentity(right.error))) {
1582
- const outcome = (error) => error ? `failed with ${error.name}: ${error.message}` : "succeeded";
1583
- return {
1584
- reason: "model-error",
1585
- message: `Model call #${left.call} ${outcome(left.error)}; in the port it ${outcome(right.error)}.`
1586
- };
1587
- }
1588
- return;
2532
+ if (candidate.outcome === "threw") return {
2533
+ label: "regression",
2534
+ reason: `candidate threw: ${String(candidate.detail)}`
2535
+ };
2536
+ if (baseline.violations?.length) return {
2537
+ label: "intended",
2538
+ reason: `baseline broke a rule: ${list(baseline.violations)}`
2539
+ };
2540
+ const sameOutcome = baseline.outcome === candidate.outcome;
2541
+ if (sameOutcome && structurallyEqual(baseline.effects, candidate.effects)) return {
2542
+ label: "same",
2543
+ reason: ""
2544
+ };
2545
+ if (expected !== void 0 && candidate.outcome === expected && baseline.outcome !== expected) return {
2546
+ label: "intended",
2547
+ reason: `only the candidate reached the expected '${expected}'`
2548
+ };
2549
+ return {
2550
+ label: "regression",
2551
+ reason: sameOutcome ? "effects differ" : `outcome differs: ${baseline.outcome} → ${candidate.outcome}`
2552
+ };
2553
+ }
2554
+ function list(items) {
2555
+ return items.map((item) => typeof item === "string" ? item : JSON.stringify(item)).join(", ");
2556
+ }
2557
+ function structurallyEqual(a, b) {
2558
+ if (Object.is(a, b)) return true;
2559
+ if (a instanceof Date || b instanceof Date) return a instanceof Date && b instanceof Date && a.getTime() === b.getTime();
2560
+ if (a instanceof Map || b instanceof Map) return a instanceof Map && b instanceof Map && structurallyEqual([...a], [...b]);
2561
+ if (a instanceof Set || b instanceof Set) return a instanceof Set && b instanceof Set && structurallyEqual([...a], [...b]);
2562
+ if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => structurallyEqual(item, b[index]));
2563
+ if (!isRecord(a) || !isRecord(b)) return false;
2564
+ const keys = Object.keys(a);
2565
+ return keys.length === Object.keys(b).length && keys.every((key) => Object.hasOwn(b, key) && structurallyEqual(a[key], b[key]));
2566
+ }
2567
+ //#endregion
2568
+ //#region src/check-parity.ts
2569
+ /**
2570
+ * `checkParity` — one call that records a workflow, replays its port, and
2571
+ * compares their effects, per scenario.
2572
+ * @module
2573
+ */
2574
+ /** Each function effect wrapped with `recorder.fn`, by its key. */
2575
+ function recordEffects(recorder, effects) {
2576
+ return mapEffects(effects, (name, effect) => recorder.fn(name, effect));
2577
+ }
2578
+ async function settle(run, recorder, normalize) {
2579
+ try {
2580
+ const output = await run();
2581
+ recorder.output(normalize(output));
2582
+ return { output };
2583
+ } catch (error) {
2584
+ return { error };
1589
2585
  }
1590
- if (left.kind === "tool" && right.kind === "tool") {
1591
- if (left.name !== right.name) return {
1592
- reason: "tool-name",
1593
- message: `Expected ${describe(left)}; the port made ${describe(right)}.`
1594
- };
1595
- if (!jsonEqual(left.input, right.input)) return {
1596
- reason: "tool-input",
1597
- message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.input)}.`
1598
- };
1599
- if (!jsonEqual(left.output, right.output) || !jsonEqual(errorIdentity(left.error), errorIdentity(right.error))) return {
1600
- reason: "tool-result",
1601
- message: `Tool '${left.name}' returned a different result.`
1602
- };
1603
- return;
2586
+ }
2587
+ function errorIdentity(outcome, normalize) {
2588
+ if (!("error" in outcome)) return void 0;
2589
+ const error = normalize ? normalize(outcome.error) : outcome.error;
2590
+ if (error instanceof Error) return `${error.name}: ${error.message}`;
2591
+ if (!normalize) return String(error);
2592
+ try {
2593
+ return stableStringify(toJsonValue(error) ?? null);
2594
+ } catch {
2595
+ return String(error);
1604
2596
  }
1605
- if (left.kind === "output" && right.kind === "output" && !jsonEqual(left.value, right.value)) return {
1606
- reason: "output",
1607
- message: `Expected ${describe(left)}; the port returned ${JSON.stringify(right.value)}.`
1608
- };
1609
2597
  }
1610
2598
  /**
1611
- * Compares two recordings and reports every effect that differs. `expected`
1612
- * is the original's recording, `actual` the port's. Effects that are the same
1613
- * on both sides are lined up first (longest common subsequence), so one extra
1614
- * call does not shift every effect after it; between them, effects of the same
1615
- * kind pair up and report what changed, and the rest are missing or extra.
2599
+ * Checks a port against the workflow it replaces, scenario by scenario: runs
2600
+ * `before` on fresh `effects` under `createRecorder`, replays `after` against
2601
+ * that recording (no effect really runs), and compares the two with
2602
+ * `compareRecordings`. Effects are compared by name, call order, arguments
2603
+ * and results; the output by value.
2604
+ *
2605
+ * Never throws for a port that misbehaves: an `after` that throws (a
2606
+ * `RecordingDivergenceError` for a call the original never made, or its own
2607
+ * error) is a row with `after: { error }`. When both sides throw, they match
2608
+ * on the error's name and message, after `normalizeError` when given.
2609
+ *
2610
+ * @example
2611
+ * ```ts
2612
+ * const report = await checkParity({
2613
+ * scenarios: [{ id: "refund", input: ticket }],
2614
+ * effects: () => createFakeEffects(),
2615
+ * before: (input, effects) => legacyRunTicket(input, effects),
2616
+ * after: (input, effects) => step1.run(input, { effects }),
2617
+ * });
2618
+ * expect(report.equivalent).toBe(true);
2619
+ * ```
1616
2620
  */
1617
- function compareRecordings(expected, actual, options = {}) {
1618
- const select = (effects) => normalize(effects.flatMap((effect, index) => {
1619
- if (options.kinds && !options.kinds.includes(effect.kind)) return [];
1620
- const mapped = options.normalize ? options.normalize(effect, index, effects) : effect;
1621
- return mapped ? [mapped] : [];
1622
- }));
1623
- const a = select(expected.effects);
1624
- const b = select(actual.effects);
1625
- const same = a.map((left) => b.map((right) => !differ(left, right, options)));
1626
- const lcs = Array.from({ length: a.length + 1 }, () => Array.from({ length: b.length + 1 }, () => 0));
1627
- for (let i = a.length - 1; i >= 0; i--) for (let j = b.length - 1; j >= 0; j--) lcs[i][j] = same[i][j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
1628
- const divergences = [];
1629
- let lefts = [];
1630
- let rights = [];
1631
- const flushGap = (at) => {
1632
- const unpairedRights = [...rights];
1633
- const changed = [];
1634
- for (const index of lefts) {
1635
- const left = a[index];
1636
- const k = unpairedRights.findIndex((j) => b[j].kind === left.kind);
1637
- if (k === -1) {
1638
- changed.push({
1639
- index,
1640
- reason: "missing-effect",
1641
- expected: left,
1642
- message: `The port never made ${describe(left)}.`
1643
- });
1644
- continue;
1645
- }
1646
- const right = b[unpairedRights.splice(k, 1)[0]];
1647
- const difference = differ(left, right, options);
1648
- if (difference) changed.push({
1649
- index,
1650
- expected: left,
1651
- actual: right,
1652
- ...difference
2621
+ async function checkParity(options) {
2622
+ const rows = [];
2623
+ for (const scenario of options.scenarios) {
2624
+ const recorder = createRecorder();
2625
+ const effects = recordEffects(recorder, options.effects(scenario));
2626
+ const normalize = (output) => options.normalizeOutput ? options.normalizeOutput(output, scenario) : output;
2627
+ const before = await settle(() => options.before(scenario.input, effects), recorder, normalize);
2628
+ const recording = recorder.recording;
2629
+ const replayer = createRecorder({ replay: recording });
2630
+ const stand = recordEffects(replayer, options.effects(scenario));
2631
+ const after = await settle(() => options.after(scenario.input, stand), replayer, normalize);
2632
+ const replay = replayer.recording;
2633
+ const { divergences } = compareRecordings(recording, replay, options.compare);
2634
+ const normalizeError = options.normalizeError && ((error) => options.normalizeError(error, scenario));
2635
+ const thrownBefore = errorIdentity(before, normalizeError);
2636
+ const thrownAfter = errorIdentity(after, normalizeError);
2637
+ if (thrownBefore !== void 0 && thrownAfter !== void 0 && thrownBefore !== thrownAfter) divergences.push({
2638
+ index: recording.effects.length,
2639
+ reason: "output",
2640
+ message: `The original threw ${thrownBefore}; the port threw ${thrownAfter}.`
2641
+ });
2642
+ else if (thrownBefore === void 0 !== (thrownAfter === void 0)) {
2643
+ const returned = (thrownBefore === void 0 ? recording : replay).effects.filter((effect) => effect.kind === "output").at(-1);
2644
+ const value = JSON.stringify(returned?.kind === "output" ? returned.value : void 0);
2645
+ const message = thrownAfter !== void 0 ? `The original returned ${value}; the port threw ${thrownAfter}.` : `The original threw ${thrownBefore}; the port returned ${value}.`;
2646
+ const existing = divergences.find((d) => d.reason === "output" && d.expected?.kind === "output" !== (d.actual?.kind === "output"));
2647
+ if (existing) existing.message = message;
2648
+ else divergences.push({
2649
+ index: recording.effects.length,
2650
+ reason: "output",
2651
+ message
1653
2652
  });
1654
2653
  }
1655
- for (const j of unpairedRights) divergences.push({
1656
- index: lefts[0] ?? at,
1657
- reason: "extra-effect",
1658
- actual: b[j],
1659
- message: `The port made ${describe(b[j])}, which the original never made.`
2654
+ rows.push({
2655
+ id: scenario.id,
2656
+ scenario,
2657
+ equivalent: divergences.length === 0,
2658
+ divergences,
2659
+ before,
2660
+ after,
2661
+ recording,
2662
+ replay
1660
2663
  });
1661
- divergences.push(...changed);
1662
- lefts = [];
1663
- rights = [];
1664
- };
1665
- let i = 0;
1666
- let j = 0;
1667
- while (i < a.length || j < b.length) if (i < a.length && j < b.length && same[i][j]) {
1668
- flushGap(i);
1669
- i++;
1670
- j++;
1671
- } else if (j < b.length && (i === a.length || lcs[i][j + 1] >= lcs[i + 1][j])) rights.push(j++);
1672
- else lefts.push(i++);
1673
- flushGap(i);
1674
- divergences.sort((x, y) => x.index - y.index);
1675
- return divergences.length ? {
1676
- equivalent: false,
1677
- divergence: divergences[0],
1678
- divergences
1679
- } : {
1680
- equivalent: true,
1681
- divergences
2664
+ }
2665
+ return {
2666
+ equivalent: rows.every((row) => row.equivalent),
2667
+ scenarios: rows
1682
2668
  };
1683
2669
  }
1684
2670
  //#endregion
1685
- export { AgentLintError, AgentUnknownStateError, RecordingDivergenceError, assertAgentMachine, canReach, compareRecordings, compareRuns, createRecorder, explorePaths, firstDivergence, foldTraces, getDecisionRequests, judgeAgreement, lintAgentMachine, matchesTrajectory, repeatRuns, runSeam, sameModelCallShape, simulateAgent, summarizeRepeats, traceCoverage };
2671
+ export { AgentLintError, AgentUnknownStateError, RecordingDivergenceError, assertAgentMachine, canReach, checkCrashRecovery, checkParity, compareRecordings, compareRuns, createRecorder, explorePaths, firstDivergence, foldTraces, getDecisionRequests, judgeAgreement, lintAgentMachine, matchesTrajectory, proposeApprovalRules, proposeCaps, repeatRuns, runSeam, sameModelCallShape, simulateAgent, summarizeRepeats, textOutput, traceCoverage };