@statelyai/agent 2.0.0-alpha.37 → 2.0.0-alpha.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/CHANGELOG.md +51 -0
  2. package/dist/{act-CrX0JwWY.cjs → act-B0CVbmo4.cjs} +1 -1
  3. package/dist/{act-B9hLQls6.mjs → act-Bsz-hUfD.mjs} +1 -1
  4. package/dist/ai-sdk.cjs +4 -4
  5. package/dist/ai-sdk.d.cts +2 -3
  6. package/dist/ai-sdk.d.mts +2 -3
  7. package/dist/ai-sdk.mjs +4 -4
  8. package/dist/{decision-m5hGmHpF.mjs → decision-B41YBx5U.mjs} +1 -1
  9. package/dist/{decision-CMeuNZ1g.cjs → decision-D8bcIGXQ.cjs} +1 -1
  10. package/dist/{decision-object-DBcLWVvv.d.cts → decision-object-BB8XV5Ec.d.cts} +1 -1
  11. package/dist/{decision-object-B5zGSPFp.d.mts → decision-object-CH_sE694.d.mts} +1 -1
  12. package/dist/effects-B9IvUPVh.mjs +23 -0
  13. package/dist/effects-DrWcUGgc.cjs +34 -0
  14. package/dist/evals.cjs +810 -0
  15. package/dist/evals.d.cts +595 -0
  16. package/dist/evals.d.mts +595 -0
  17. package/dist/evals.mjs +801 -0
  18. package/dist/{event-log-DkVlk3Rh.mjs → event-log-CEq_bVXQ.mjs} +8 -4
  19. package/dist/{event-log-Dkc-yIYu.cjs → event-log-dSOnXHuG.cjs} +7 -15
  20. package/dist/{event-log-store-Brx9tOHt.mjs → event-log-store-DLAfNM34.mjs} +1 -1
  21. package/dist/{event-log-store-BBS8so8a.d.mts → event-log-store-DYun526m.d.mts} +15 -14
  22. package/dist/{event-log-store-Cl-YWDnq.cjs → event-log-store-Dey9ufA4.cjs} +1 -1
  23. package/dist/{event-log-store-bc4ICGcS.d.cts → event-log-store-aFXK6yZX.d.cts} +15 -14
  24. package/dist/index.cjs +136 -98
  25. package/dist/index.d.cts +73 -134
  26. package/dist/index.d.mts +73 -134
  27. package/dist/index.mjs +117 -75
  28. package/dist/internal-BzlpzSux.d.cts +14 -0
  29. package/dist/internal-BzlpzSux.d.mts +14 -0
  30. package/dist/log.cjs +2 -3
  31. package/dist/log.d.cts +2 -2
  32. package/dist/log.d.mts +2 -2
  33. package/dist/log.mjs +3 -3
  34. package/dist/machines.cjs +1 -719
  35. package/dist/machines.d.cts +2 -424
  36. package/dist/machines.d.mts +2 -424
  37. package/dist/machines.mjs +2 -709
  38. package/dist/{model-source-kjGwqAV8.mjs → model-source-BrsLDvHL.mjs} +1 -1
  39. package/dist/{model-source-B4WQ8Idp.cjs → model-source-YpxjmWuX.cjs} +1 -1
  40. package/dist/openai.cjs +3 -3
  41. package/dist/openai.d.cts +2 -3
  42. package/dist/openai.d.mts +2 -3
  43. package/dist/openai.mjs +3 -3
  44. package/dist/otel.d.cts +1 -1
  45. package/dist/otel.d.mts +1 -1
  46. package/dist/policy.cjs +77 -0
  47. package/dist/policy.d.cts +143 -0
  48. package/dist/policy.d.mts +143 -0
  49. package/dist/policy.mjs +75 -0
  50. package/dist/{effects-BT_cSY0j.mjs → run-agent-BE7DOWD6.mjs} +150 -184
  51. package/dist/{run-agent-DlPzBWj-.d.mts → run-agent-DHlq87G7.d.mts} +18 -34
  52. package/dist/{effects-DDseYtJk.cjs → run-agent-De3xVFGw.cjs} +149 -201
  53. package/dist/{run-agent-30mH2moE.d.cts → run-agent-Dq0oSFRm.d.cts} +18 -34
  54. package/dist/{setup-agent-BDmNr6PC.mjs → setup-agent-CLm6QVGy.mjs} +3 -3
  55. package/dist/{setup-agent-BaNyz6Q3.d.cts → setup-agent-Cv8Hk3mX.d.cts} +12 -11
  56. package/dist/{setup-agent-C1CG_nfX.d.mts → setup-agent-DgFAhaPJ.d.mts} +12 -11
  57. package/dist/{setup-agent-CmEToB8y.cjs → setup-agent-Dv5VpuHe.cjs} +3 -3
  58. package/dist/tanstack-ai.cjs +4 -4
  59. package/dist/tanstack-ai.d.cts +2 -3
  60. package/dist/tanstack-ai.d.mts +2 -3
  61. package/dist/tanstack-ai.mjs +4 -4
  62. package/dist/testing.cjs +122 -996
  63. package/dist/testing.d.cts +36 -695
  64. package/dist/testing.d.mts +36 -695
  65. package/dist/testing.mjs +112 -977
  66. package/dist/{text-logic-BqxalgMr.d.mts → text-logic-x17uLBwA.d.cts} +238 -2
  67. package/dist/{text-logic-CSf9bqpP.d.cts → text-logic-x17uLBwA.d.mts} +238 -2
  68. package/dist/trajectory-BFjXaooB.d.cts +96 -0
  69. package/dist/trajectory-BFjXaooB.d.mts +96 -0
  70. package/dist/trajectory-BUXj_1Ue.mjs +152 -0
  71. package/dist/trajectory-CoZSv4wF.cjs +157 -0
  72. package/dist/{usage-Bm5nvpow.d.cts → usage-B4B-TQSV.d.mts} +1 -1
  73. package/dist/{usage-BnikdOca.d.mts → usage-DvGcoK6x.d.cts} +1 -1
  74. package/dist/{utils-7lSa4H-i.d.cts → utils-B8k-YPYI.d.mts} +23 -17
  75. package/dist/{utils-CBbKw2fM.d.mts → utils-DAclF-kw.d.cts} +23 -17
  76. package/dist/validate.cjs +4 -4
  77. package/dist/validate.d.cts +2 -2
  78. package/dist/validate.d.mts +2 -2
  79. package/dist/validate.mjs +4 -4
  80. package/dist/{verify-DJHfEfm6.d.cts → verify-BM0sGmsf.d.cts} +42 -6
  81. package/dist/{verify-ChyX6jq0.mjs → verify-CWlg3j7B.mjs} +153 -45
  82. package/dist/{verify-CwlombbC.cjs → verify-Dek6gHCS.cjs} +153 -45
  83. package/dist/{verify-i9Hiwc5g.d.mts → verify-H3Pr2gNw.d.mts} +42 -6
  84. package/dist/{with-model-calls-B_WLmOSl.cjs → with-model-calls-D50Ca6qW.cjs} +2 -2
  85. package/dist/{with-model-calls-BkOwdroS.mjs → with-model-calls-uY-KKliA.mjs} +2 -2
  86. package/docs/advanced.md +5 -5
  87. package/docs/choosing-a-run-mode.md +7 -7
  88. package/docs/debugging.md +6 -8
  89. package/docs/decisions.md +6 -5
  90. package/docs/environment-and-policy.md +1 -1
  91. package/docs/evals.md +57 -43
  92. package/docs/event-log.md +14 -16
  93. package/docs/from-a-loop.md +3 -3
  94. package/docs/generate-machines.md +4 -4
  95. package/docs/hosts.md +22 -29
  96. package/docs/human-in-the-loop.md +5 -8
  97. package/docs/index.md +4 -4
  98. package/docs/judgments.md +5 -7
  99. package/docs/langgraph-comparison.md +6 -6
  100. package/docs/machines-as-data.md +12 -17
  101. package/docs/machines-presets.md +17 -120
  102. package/docs/machines.md +13 -13
  103. package/docs/models-and-providers.md +19 -25
  104. package/docs/multi-agent.md +1 -1
  105. package/docs/observability.md +6 -8
  106. package/docs/persistence.md +15 -25
  107. package/docs/quickstart.md +4 -4
  108. package/docs/statechart-policy-examples.md +1 -1
  109. package/docs/steps.md +5 -6
  110. package/docs/strangler-fig.md +72 -7
  111. package/docs/text-requests.md +10 -12
  112. package/docs/thinking-in-state-machines.md +8 -8
  113. package/docs/tools.md +10 -12
  114. package/docs/upgrading.md +1 -1
  115. package/docs/usage-and-budgets.md +51 -56
  116. package/docs/validate-and-repair.md +9 -14
  117. package/docs/verify.md +53 -7
  118. package/docs/xstate-4-and-5-apps.md +19 -28
  119. package/llms.txt +1 -1
  120. package/package.json +31 -6
  121. package/readme.md +32 -24
  122. package/dist/internal-C3Lqu538.d.cts +0 -43
  123. package/dist/internal-DUcdhk-i.d.mts +0 -43
  124. package/dist/types-DQUOOaZO.d.cts +0 -238
  125. package/dist/types-DQUOOaZO.d.mts +0 -238
  126. package/skills/add-evals/SKILL.md +0 -152
  127. package/skills/add-traces/SKILL.md +0 -152
  128. package/skills/build-machine/SKILL.md +0 -84
  129. package/skills/generate-machine/SKILL.md +0 -335
  130. package/skills/graph-engineering/SKILL.md +0 -43
  131. package/skills/improve-machine/SKILL.md +0 -261
  132. package/skills/improve-machine/reference.md +0 -101
  133. package/skills/loop-to-machine/SKILL.md +0 -205
  134. package/skills/loop-to-machine/policy.md +0 -154
  135. package/skills/loop-to-machine/reference.md +0 -260
package/dist/evals.cjs ADDED
@@ -0,0 +1,810 @@
1
+ Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
2
+ const require_event_log = require("./event-log-dSOnXHuG.cjs");
3
+ const require_verify = require("./verify-Dek6gHCS.cjs");
4
+ const require_trajectory = require("./trajectory-CoZSv4wF.cjs");
5
+ let xstate = require("xstate");
6
+ //#region src/internal/trace-states.ts
7
+ /** The `value` of a trace event's `from` or `snapshot`. @internal */
8
+ function valueOf(holder) {
9
+ return require_event_log.isRecord(holder) ? holder.value : void 0;
10
+ }
11
+ /** A decision request on its first attempt (no failed attempts yet). @internal */
12
+ function isFirstDecision(request) {
13
+ return require_event_log.isRecord(request) && request.kind === "decision" && (!Array.isArray(request.attempts) || request.attempts.length === 0);
14
+ }
15
+ /**
16
+ * Every active leaf of a state value as a dot path:
17
+ * `'a'` → `['a']`; `{ a: 'b' }` → `['a.b']`; `{ p: { x: 'one', y: {} } }` → `['p.x.one', 'p.y']`.
18
+ * @internal
19
+ */
20
+ function statePaths(value, prefix = "") {
21
+ if (typeof value === "string") return [prefix + value];
22
+ if (!require_event_log.isRecord(value)) return [];
23
+ const entries = Object.entries(value);
24
+ if (entries.length === 0) return prefix ? [prefix.slice(0, -1)] : [];
25
+ return entries.flatMap(([key, child]) => statePaths(child, `${prefix}${key}.`));
26
+ }
27
+ /**
28
+ * A trace's `machine.transition` events as steps, skipping `@xstate.init`.
29
+ * A first-attempt decision's `request.start` names the decision on the step
30
+ * that follows it, skipping `@agent.usage` steps. Reads live and serialized traces alike.
31
+ * @internal
32
+ */
33
+ function traceSteps(trace) {
34
+ const steps = [];
35
+ let decision;
36
+ for (const event of trace) {
37
+ if (event.type === "request.start" && isFirstDecision(event.request)) {
38
+ const name = event.request.name;
39
+ decision = typeof name === "string" ? name : void 0;
40
+ continue;
41
+ }
42
+ if (event.type !== "machine.transition") continue;
43
+ const type = require_event_log.isRecord(event.event) ? event.event.type : void 0;
44
+ if (type === "@xstate.init") continue;
45
+ const fromValue = valueOf(event.from);
46
+ const toValue = valueOf(event.snapshot);
47
+ steps.push({
48
+ from: statePaths(fromValue),
49
+ fromValue,
50
+ event: typeof type === "string" ? type : "",
51
+ payload: event.event,
52
+ to: statePaths(toValue),
53
+ toValue,
54
+ ignored: event.ignored === true,
55
+ ...decision === void 0 || type === "@agent.usage" ? {} : { decision }
56
+ });
57
+ if (type !== "@agent.usage") decision = void 0;
58
+ }
59
+ return steps;
60
+ }
61
+ //#endregion
62
+ //#region src/fold-traces.ts
63
+ /**
64
+ * Folds many runs' traces into per-state stats: how many runs visited each
65
+ * state, how many of those failed, how many options the guards withheld
66
+ * at decisions made there, how many events it ignored, and how many failures
67
+ * `blame` traced back to it. Pure; reads only:
68
+ *
69
+ * - `machine.transition`: the run visited `from.value` and `snapshot.value`;
70
+ * with `ignored`, each of those states ignored one event.
71
+ * - `request.start` of a decision, first attempt only: its `request.withheld`
72
+ * counts toward the state the run was in, the `from.value` of the
73
+ * transition its choice causes.
74
+ *
75
+ * A trace is one run's events, live (`AgentTraceEvent`) or serialized
76
+ * (`serializeTraceEvent`). A host-driven `decideNext` traces nothing; push
77
+ * `{ type: 'request.start', request: result.request }` onto the trace before
78
+ * sending its event. Parallel states count once per active leaf.
79
+ *
80
+ * `failures` counts a failed run against every state it visited, so states
81
+ * after a bad decision rank as high as the decision. Sort by `blamed` to see
82
+ * where failures start.
83
+ *
84
+ * @example
85
+ * ```ts no-check
86
+ * const stats = foldTraces(traces, {
87
+ * failed: (trace) => trace.at(-1)?.type === 'run.end' && trace.at(-1).status === 'error',
88
+ * });
89
+ * ```
90
+ */
91
+ function foldTraces(traces, options) {
92
+ const stats = /* @__PURE__ */ new Map();
93
+ const statsFor = (state) => {
94
+ let entry = stats.get(state);
95
+ if (!entry) {
96
+ entry = {
97
+ state,
98
+ visits: 0,
99
+ failures: 0,
100
+ withheld: 0,
101
+ ignored: 0,
102
+ blamed: 0
103
+ };
104
+ stats.set(state, entry);
105
+ }
106
+ return entry;
107
+ };
108
+ for (const trace of traces) {
109
+ const visited = /* @__PURE__ */ new Set();
110
+ let current = [];
111
+ const visit = (state) => {
112
+ statsFor(state);
113
+ visited.add(state);
114
+ };
115
+ for (const event of trace) if (event.type === "machine.transition") {
116
+ for (const state of statePaths(valueOf(event.from))) visit(state);
117
+ current = statePaths(valueOf(event.snapshot));
118
+ for (const state of current) visit(state);
119
+ if (event.ignored === true) for (const state of current) statsFor(state).ignored++;
120
+ } else if (event.type === "request.start" && isFirstDecision(event.request)) {
121
+ const withheld = event.request.withheld;
122
+ if (!Array.isArray(withheld)) continue;
123
+ for (const state of current) statsFor(state).withheld += withheld.length;
124
+ }
125
+ const failed = options.failed(trace);
126
+ for (const state of visited) {
127
+ const entry = statsFor(state);
128
+ entry.visits++;
129
+ if (failed) entry.failures++;
130
+ }
131
+ const blamed = failed ? options.blame?.(trace) : void 0;
132
+ if (blamed !== void 0) statsFor(blamed).blamed++;
133
+ }
134
+ return [...stats.values()];
135
+ }
136
+ //#endregion
137
+ //#region src/trace-coverage.ts
138
+ /** Mirrors xstate@6.0.0-alpha.69 `matchesEventDescriptor` (src/utils.ts), which the package does not export. */
139
+ function matchesDescriptor(eventType, descriptor) {
140
+ if (descriptor === eventType || descriptor === "*") return true;
141
+ if (!descriptor.endsWith(".*")) return false;
142
+ const tokens = descriptor.split(".");
143
+ const eventTokens = eventType.split(".");
144
+ for (let index = 0; index < tokens.length; index++) {
145
+ if (tokens[index] === "*") return index === tokens.length - 1;
146
+ if (tokens[index] !== eventTokens[index]) return false;
147
+ }
148
+ return true;
149
+ }
150
+ /** Whether a definition's `matches` pattern (e.g. an invoke's `actorId`) accepts the traced event. */
151
+ function matchesPattern(definition, event) {
152
+ const pattern = definition.matches;
153
+ if (!require_event_log.isRecord(pattern) || !require_event_log.isRecord(event)) return true;
154
+ return Object.entries(pattern).every(([key, value]) => Object.is(event[key], value));
155
+ }
156
+ /**
157
+ * The descriptor XState would try first at `node` for `event`, guards aside:
158
+ * the exact key, then partial and wildcard keys longest first, as in
159
+ * xstate@6.0.0-alpha.69 `getCandidates` (src/stateUtils.ts).
160
+ */
161
+ function candidateAt(node, type, event) {
162
+ const keys = [...node.transitions.keys()].filter((key) => key !== type && matchesDescriptor(type, key)).sort((a, b) => b.length - a.length);
163
+ if (node.transitions.has(type)) keys.unshift(type);
164
+ return keys.find((key) => node.transitions.get(key).some((d) => matchesPattern(d, event)));
165
+ }
166
+ const pathOf = (node) => node.path.join(".");
167
+ const sameLeaves = (a, b) => statePaths(a).sort().join("|") === statePaths(b).sort().join("|");
168
+ /**
169
+ * The leaves a trace passed through inside its macrosteps: states an `always`
170
+ * transition left before the step ended, which no traced snapshot holds.
171
+ * Re-derived with XState's pure `getMicrosteps` from the snapshot before each
172
+ * step (the previous step's, or the fresh start's microsteps from its input).
173
+ * A step that cannot be recomputed, or whose recomputation ends elsewhere
174
+ * than the traced snapshot, adds none. Only `value` and `context` are rebuilt,
175
+ * so a guard that reads live children can take another route to the same
176
+ * leaves, and that route is credited.
177
+ */
178
+ function passedThrough(machine, trace) {
179
+ const leaves = /* @__PURE__ */ new Set();
180
+ let resumed = false;
181
+ let previous;
182
+ for (const event of trace) {
183
+ if (event.type === "run.start") resumed = event.snapshot !== void 0;
184
+ if (event.type !== "machine.transition" || !require_event_log.isRecord(event.event)) continue;
185
+ const snapshot = require_event_log.isRecord(event.snapshot) ? event.snapshot : void 0;
186
+ try {
187
+ const microsteps = event.event.type === "@xstate.init" ? resumed ? [] : (0, xstate.getInitialMicrosteps)(machine, event.event.input) : previous && sameLeaves(previous.value, valueOf(event.from)) ? (0, xstate.getMicrosteps)(machine, machine.resolveState({
188
+ value: previous.value,
189
+ context: previous.context
190
+ }), event.event) : [];
191
+ const last = microsteps.at(-1)?.[0];
192
+ if (last && snapshot && sameLeaves(last.value, snapshot.value)) for (const [step] of microsteps) for (const leaf of statePaths(step.value)) leaves.add(leaf);
193
+ } catch {}
194
+ previous = snapshot;
195
+ }
196
+ return leaves;
197
+ }
198
+ const ratioOf = (visited, total) => total === 0 ? 1 : visited / total;
199
+ /**
200
+ * Measures how much of a machine a set of runs exercised: which atomic and
201
+ * final states they visited, and which handlers they took. A handler is a
202
+ * `(declaring state, event descriptor)` pair from `on`, invoke
203
+ * `onDone`/`onError`, and `after`, with the targets it declares and the
204
+ * leaves the runs landed in after it. Pure; reads only `machine.transition`
205
+ * events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
206
+ *
207
+ * A step credits one handler per active leaf: the nearest node, from the leaf
208
+ * up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
209
+ * event, preferred in XState's order. Guards are not replayed, so a step
210
+ * whose nearest handler was rejected and fell through to an ancestor is
211
+ * credited to the nearest one. XState keys invoke and `after` handlers by
212
+ * bare type (`xstate.done.actor`), one per state however many invokes; a
213
+ * definition's `matches` (an invoke's `actorId`) must accept the traced event
214
+ * for its state to be credited. Ignored steps credit no handler. `always`
215
+ * handlers are listed as `unobservable`, never as unvisited; a state one
216
+ * passes through is visited, re-derived from each step's microsteps.
217
+ *
218
+ * @example
219
+ * ```ts no-check
220
+ * const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
221
+ * expect(report.unvisitedHandlers).toEqual([]);
222
+ * ```
223
+ */
224
+ function traceCoverage(machine, traces, options = {}) {
225
+ const nodes = /* @__PURE__ */ new Map();
226
+ const states = [];
227
+ const handlers = [];
228
+ const handlerAt = /* @__PURE__ */ new Map();
229
+ const unobservable = [];
230
+ const walk = (node) => {
231
+ const state = pathOf(node);
232
+ nodes.set(state, node);
233
+ if (node.type === "atomic" || node.type === "final") states.push({
234
+ state,
235
+ runs: 0
236
+ });
237
+ const byEvent = /* @__PURE__ */ new Map();
238
+ for (const [event, definitions] of node.transitions) {
239
+ const dynamic = definitions.some((d) => typeof d.to === "function");
240
+ const declared = [...new Set(definitions.flatMap((d) => (d.target ?? []).map(pathOf)))];
241
+ const handler = {
242
+ state,
243
+ event,
244
+ runs: 0,
245
+ targets: {
246
+ declared: dynamic ? "dynamic" : declared,
247
+ seen: [],
248
+ unreached: []
249
+ }
250
+ };
251
+ handlers.push(handler);
252
+ byEvent.set(event, handler);
253
+ }
254
+ handlerAt.set(node, byEvent);
255
+ if (node.always?.length) unobservable.push({
256
+ state,
257
+ event: "always"
258
+ });
259
+ for (const child of Object.values(node.states)) walk(child);
260
+ };
261
+ walk(machine.root);
262
+ const stateRuns = new Map(states.map((row) => [row.state, row]));
263
+ const seen = /* @__PURE__ */ new Map();
264
+ for (const trace of traces) {
265
+ const steps = traceSteps(trace);
266
+ const visited = new Set(steps[0]?.from);
267
+ for (const event of trace) {
268
+ if (event.type !== "machine.transition" || !require_event_log.isRecord(event.event)) continue;
269
+ if (event.event.type !== "@xstate.init" || !require_event_log.isRecord(event.snapshot)) continue;
270
+ for (const leaf of statePaths(event.snapshot.value)) visited.add(leaf);
271
+ }
272
+ if (unobservable.length > 0) for (const leaf of passedThrough(machine, trace)) visited.add(leaf);
273
+ const credited = /* @__PURE__ */ new Set();
274
+ for (const step of steps) {
275
+ for (const leaf of step.to) visited.add(leaf);
276
+ if (step.ignored) continue;
277
+ const hits = /* @__PURE__ */ new Set();
278
+ for (const leaf of step.from) for (let node = nodes.get(leaf); node; node = node.parent) {
279
+ if (node.type === "final") continue;
280
+ const event = candidateAt(node, step.event, step.payload);
281
+ if (event === void 0) continue;
282
+ hits.add(handlerAt.get(node).get(event));
283
+ break;
284
+ }
285
+ for (const handler of hits) {
286
+ credited.add(handler);
287
+ let leaves = seen.get(handler);
288
+ if (!leaves) seen.set(handler, leaves = /* @__PURE__ */ new Set());
289
+ for (const leaf of step.to) leaves.add(leaf);
290
+ }
291
+ }
292
+ for (const state of visited) {
293
+ const row = stateRuns.get(state);
294
+ if (row) row.runs++;
295
+ }
296
+ for (const handler of credited) handler.runs++;
297
+ }
298
+ for (const handler of handlers) {
299
+ const leaves = [...seen.get(handler) ?? []];
300
+ handler.targets.seen = leaves;
301
+ const { declared } = handler.targets;
302
+ if (declared === "dynamic") continue;
303
+ handler.targets.unreached = declared.filter((target) => !leaves.some((leaf) => leaf === target || leaf.startsWith(`${target}.`)));
304
+ }
305
+ const reachableComplete = options.reachable ? require_verify.isExplorationComplete(options.reachable) : void 0;
306
+ let unreachable;
307
+ if (options.reachable && reachableComplete) {
308
+ const reached = new Set(options.reachable.reachedStates.flatMap((value) => statePaths(value)));
309
+ unreachable = states.filter((row) => !reached.has(row.state) && row.runs === 0).map((row) => row.state);
310
+ }
311
+ const counted = unreachable ? states.filter((row) => !unreachable.includes(row.state)) : states;
312
+ const unvisitedStates = counted.filter((row) => row.runs === 0).map((row) => row.state);
313
+ const unvisitedHandlers = handlers.filter((handler) => handler.runs === 0).map(({ state, event }) => ({
314
+ state,
315
+ event
316
+ }));
317
+ return {
318
+ states,
319
+ handlers,
320
+ unvisitedStates,
321
+ unvisitedHandlers,
322
+ unobservable,
323
+ ...unreachable ? { unreachable } : {},
324
+ ...reachableComplete === void 0 ? {} : { reachableComplete },
325
+ ratio: {
326
+ states: ratioOf(counted.length - unvisitedStates.length, counted.length),
327
+ handlers: ratioOf(handlers.length - unvisitedHandlers.length, handlers.length)
328
+ }
329
+ };
330
+ }
331
+ //#endregion
332
+ //#region src/first-divergence.ts
333
+ const defaultKey = (step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`;
334
+ const isErrorLike = (value) => require_event_log.isError(value) || require_event_log.isRecord(value) && typeof value.name === "string" && typeof value.message === "string";
335
+ const normalizeErrors = (value, seen = /* @__PURE__ */ new WeakSet()) => {
336
+ if (value === null || typeof value !== "object") return value;
337
+ if (seen.has(value)) return "[circular]";
338
+ seen.add(value);
339
+ try {
340
+ if (isErrorLike(value) && (require_event_log.isError(value) || "stack" in value || "code" in value)) {
341
+ const { name, message, code, cause } = value;
342
+ return {
343
+ name,
344
+ message,
345
+ ...typeof code === "string" ? { code } : {},
346
+ ...cause !== void 0 ? { cause: normalizeErrors(cause, seen) } : {}
347
+ };
348
+ }
349
+ if (Array.isArray(value)) return value.map((item) => normalizeErrors(item, seen));
350
+ if (!require_event_log.isPlainObject(value)) return value;
351
+ const copy = Object.create(null);
352
+ for (const key of Object.keys(value)) copy[key] = normalizeErrors(value[key], seen);
353
+ return copy;
354
+ } finally {
355
+ seen.delete(value);
356
+ }
357
+ };
358
+ const payloadKey = (step) => {
359
+ if (!require_event_log.isRecord(step.payload)) return `${defaultKey(step)}|${require_event_log.canonicalEventJson(step.payload)}`;
360
+ const { sessionId: _sessionId, ...payload } = step.payload;
361
+ return `${defaultKey(step)}|${require_event_log.canonicalEventJson(normalizeErrors(payload))}`;
362
+ };
363
+ const isTrace = (items) => items.length === 0 || items.some((item) => require_event_log.isRecord(item) && (item.type === "machine.transition" || item.type === "run.start" || item.type === "run.end"));
364
+ /**
365
+ * Finds the first step where a run departs from a reference, so a failure is
366
+ * blamed on the state it started in rather than on every state the run went
367
+ * through afterwards.
368
+ *
369
+ * - Against a reference **trace**, such as a passing run of the same
370
+ * scenario, steps are compared in order by `key`. The first step that
371
+ * differs, or the point where either trace ends early, is the divergence.
372
+ * - Against an expected **trajectory** (`TrajectoryItem[]`, as for
373
+ * {@link matchesTrajectory}), the run is laid out as its initial state, then
374
+ * each step's event and new state. The first expected item it never reaches
375
+ * is the divergence, blamed on the step after the last item it did reach. A
376
+ * run that goes on past the trajectory has not diverged.
377
+ *
378
+ * An empty reference is a trace with no steps, so any step diverges from it.
379
+ * A trace is one run's events, live or serialized. Steps the machine ignored
380
+ * changed nothing and are left out on both sides. Returns `undefined` when the
381
+ * run does not diverge.
382
+ *
383
+ * @example
384
+ * ```ts no-check
385
+ * const stats = foldTraces(traces, {
386
+ * failed,
387
+ * blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
388
+ * });
389
+ * ```
390
+ */
391
+ function firstDivergence(actual, reference, options = {}) {
392
+ const steps = traceSteps(actual).filter((step) => !step.ignored);
393
+ return isTrace(reference) ? againstTrace(steps, traceSteps(reference).filter((step) => !step.ignored), options.key === "payload" ? payloadKey : options.key ?? defaultKey) : againstTrajectory(actual, steps, reference);
394
+ }
395
+ function diverged(steps, index, expected, initial = steps[0]?.fromValue) {
396
+ const step = steps[index];
397
+ return {
398
+ index,
399
+ state: (step ? step.from[0] : steps.at(-1)?.to[0] ?? statePaths(initial)[0]) ?? "",
400
+ ...step?.decision === void 0 ? {} : { decision: step.decision },
401
+ ...step ? { actual: step } : {},
402
+ ...expected === void 0 ? {} : { expected }
403
+ };
404
+ }
405
+ function againstTrace(steps, reference, key) {
406
+ const length = Math.max(steps.length, reference.length);
407
+ for (let index = 0; index < length; index++) {
408
+ const step = steps[index];
409
+ const want = reference[index];
410
+ if (step && want && key(step) === key(want)) continue;
411
+ return diverged(steps, index, want, steps[0]?.fromValue ?? reference[0]?.fromValue);
412
+ }
413
+ }
414
+ function againstTrajectory(trace, steps, expected) {
415
+ const initial = initialValue(trace, steps);
416
+ const timeline = initial === void 0 ? [] : [{ value: initial }];
417
+ const stepAt = initial === void 0 ? [] : [0];
418
+ steps.forEach((step, index) => {
419
+ timeline.push(step.payload);
420
+ stepAt.push(index);
421
+ if (JSON.stringify(step.toValue) !== JSON.stringify(step.fromValue)) {
422
+ timeline.push({ value: step.toValue });
423
+ stepAt.push(index);
424
+ }
425
+ });
426
+ const match = require_trajectory.matchesTrajectory(timeline, expected);
427
+ if (!match.firstMiss) return void 0;
428
+ const at = match.firstMiss.searchedFrom;
429
+ return diverged(steps, at < timeline.length ? stepAt[at] ?? steps.length : steps.length, match.firstMiss.expected, initial);
430
+ }
431
+ function initialValue(trace, steps) {
432
+ for (const event of trace) if (event.type === "machine.transition" && require_event_log.isRecord(event.snapshot)) return event.snapshot.value;
433
+ return steps[0]?.fromValue;
434
+ }
435
+ //#endregion
436
+ //#region src/propose-from-traces.ts
437
+ /**
438
+ * Groups the approval verdicts in many runs' traces by `key` and proposes an
439
+ * `allow` or `deny` rule for each. Pure and deterministic; a proposal is for
440
+ * a person to review, and nothing here changes a machine.
441
+ *
442
+ * Reads `machine.transition` events: a verdict is a transition, not ignored,
443
+ * out of `approvalState` on an approve or reject event. Rows come in the
444
+ * order keys were first seen.
445
+ *
446
+ * @example
447
+ * ```ts no-check
448
+ * const rows = proposeApprovalRules(traces, {
449
+ * approvalState: "awaitingApproval",
450
+ * approveEvents: ["APPROVE"],
451
+ * rejectEvents: ["REJECT"],
452
+ * key: ({ snapshot }) => `${snapshot.context.pending.tool}:${snapshot.context.pending.path}`,
453
+ * });
454
+ * ```
455
+ */
456
+ function proposeApprovalRules(traces, options) {
457
+ const rows = /* @__PURE__ */ new Map();
458
+ const rowFor = (key) => {
459
+ let row = rows.get(key);
460
+ if (!row) {
461
+ row = {
462
+ key,
463
+ approved: 0,
464
+ rejected: 0,
465
+ violations: 0,
466
+ runs: [],
467
+ proposal: null,
468
+ support: 0
469
+ };
470
+ rows.set(key, row);
471
+ }
472
+ return row;
473
+ };
474
+ const inState = (value) => statePaths(value).some((path) => path === options.approvalState || path.startsWith(`${options.approvalState}.`));
475
+ let index = 0;
476
+ for (const trace of traces) {
477
+ const run = index++;
478
+ const seen = /* @__PURE__ */ new Set();
479
+ let snapshot;
480
+ for (const event of trace) {
481
+ if (event.type !== "machine.transition") continue;
482
+ const before = snapshot;
483
+ snapshot = event.snapshot;
484
+ const type = require_event_log.isRecord(event.event) ? event.event.type : void 0;
485
+ if (event.ignored === true || typeof type !== "string" || !inState(valueOf(event.from))) continue;
486
+ const approve = options.approveEvents.includes(type);
487
+ if (!approve && !options.rejectEvents.includes(type)) continue;
488
+ const key = options.key({
489
+ event: event.event,
490
+ snapshot: before,
491
+ trace
492
+ });
493
+ if (key === void 0) continue;
494
+ const row = rowFor(key);
495
+ if (approve) row.approved++;
496
+ else row.rejected++;
497
+ if (!seen.has(key)) {
498
+ seen.add(key);
499
+ row.runs.push(run);
500
+ }
501
+ }
502
+ for (const key of options.violations?.(trace) ?? []) {
503
+ const row = rows.get(key);
504
+ if (row) row.violations++;
505
+ }
506
+ }
507
+ for (const row of rows.values()) {
508
+ row.support = row.runs.length;
509
+ row.proposal = row.approved > 0 && row.rejected === 0 && row.violations === 0 ? "allow" : row.rejected > 0 && row.approved === 0 ? "deny" : null;
510
+ }
511
+ return [...rows.values()];
512
+ }
513
+ /**
514
+ * Proposes a value for each cap (retry count, steps, batch size) from many
515
+ * runs' traces: the 95th percentile of what passing runs used, plus one, and
516
+ * never below the current cap while runs still hit it. Pure and
517
+ * deterministic; a proposal is for a person to review, and nothing here
518
+ * changes a machine.
519
+ *
520
+ * @example
521
+ * ```ts no-check
522
+ * const [retries] = proposeCaps(traces, {
523
+ * caps: [{ name: "retries", current: 3, failureState: "gaveUp", valueFromRun: countRetries }],
524
+ * });
525
+ * ```
526
+ */
527
+ function proposeCaps(traces, options) {
528
+ const runs = [...traces];
529
+ return options.caps.map((cap) => {
530
+ let hits = 0;
531
+ let max = null;
532
+ const passing = [];
533
+ for (const trace of runs) {
534
+ const hit = hitCap(trace, cap);
535
+ if (hit) hits++;
536
+ const value = cap.valueFromRun(trace);
537
+ if (value === void 0) continue;
538
+ max = max === null ? value : Math.max(max, value);
539
+ if (!hit) passing.push(value);
540
+ }
541
+ passing.sort((a, b) => a - b);
542
+ const p95 = passing.length ? passing[Math.ceil(.95 * passing.length) - 1] : null;
543
+ const proposal = p95 === null ? null : hits > 0 && cap.current !== void 0 ? Math.max(p95 + 1, cap.current) : p95 + 1;
544
+ return {
545
+ name: cap.name,
546
+ hits,
547
+ passing: passing.length,
548
+ max,
549
+ p95,
550
+ proposal
551
+ };
552
+ });
553
+ }
554
+ function hitCap(trace, cap) {
555
+ for (const event of trace) {
556
+ if (cap.failureState !== void 0 && event.type === "machine.transition" && statePaths(valueOf(event.snapshot)).some((path) => path === cap.failureState || path.startsWith(`${cap.failureState}.`))) return true;
557
+ if (cap.failureOutput && event.type === "run.end" && event.status === "done") {
558
+ if (cap.failureOutput(event.output)) return true;
559
+ }
560
+ }
561
+ return false;
562
+ }
563
+ //#endregion
564
+ //#region src/decision-requests.ts
565
+ /**
566
+ * The decisions a run asked for, read from its trace: each decision's first
567
+ * `request.start`, with the states the run was in. A trace is one run's events,
568
+ * live or serialized with `serializeTraceEvent`. Retries of the same decision
569
+ * are left out, so each decision appears once.
570
+ *
571
+ * Use the requests as eval rows (run a decider on each and compare its event
572
+ * with a label) or as probe inputs for a policy. A serialized request keeps its
573
+ * `options`, `metadata`, `system`, and `prompt`; event schemas do not survive
574
+ * serialization, so replay closed choices (`options`) rather than open ones.
575
+ *
576
+ * @example
577
+ * ```ts no-check
578
+ * const rows = traces.flatMap((trace) => getDecisionRequests(trace, { name: "chooseResolution" }));
579
+ * for (const { request } of rows) console.log((await decide(request)).event);
580
+ * ```
581
+ */
582
+ function getDecisionRequests(trace, options = {}) {
583
+ const requests = [];
584
+ let current = [];
585
+ for (const event of trace) if (event.type === "machine.transition") current = statePaths(valueOf(event.snapshot));
586
+ else if (event.type === "request.start" && isFirstDecision(event.request)) {
587
+ if (options.name !== void 0 && event.request.name !== options.name) continue;
588
+ requests.push({
589
+ states: [...current],
590
+ request: event.request
591
+ });
592
+ }
593
+ return requests;
594
+ }
595
+ //#endregion
596
+ //#region src/repeat-runs.ts
597
+ /**
598
+ * Summarizes repeated runs of each scenario: how often each passed, whether
599
+ * all its runs passed (pass^k), how many distinct state paths they took, and
600
+ * where they first went different ways. Pure. Use it for runs collected
601
+ * elsewhere, such as CI shards or replays; {@link repeatRuns} collects them and
602
+ * calls it.
603
+ *
604
+ * A scenario that passes 3 of 5 runs and `divergedAt: 'deciding'` points at one
605
+ * decision, not at a flaky agent.
606
+ *
607
+ * @example
608
+ * ```ts no-check
609
+ * const report = summarizeRepeats(shards.map((s) => ({ id: s.id, scenario: s, runs: s.runs })));
610
+ * report.passHat(3); // the chance 3 runs of a scenario all pass
611
+ * ```
612
+ */
613
+ function summarizeRepeats(groups) {
614
+ const scenarios = groups.map((group) => summarize$1(group));
615
+ const traces = groups.flatMap((group) => group.runs.flatMap((run) => run.trace ? [run.trace] : []));
616
+ const most = Math.max(0, ...groups.map((group) => group.runs.length));
617
+ const passHat = (j) => {
618
+ if (!Number.isInteger(j) || j < 1 || j > most) throw new require_event_log.AgentError("pass-hat-out-of-range", `passHat: j must be a whole number from 1 to ${most}, got ${j}.`);
619
+ const eligible = scenarios.filter((scenario) => scenario.runs.length >= j);
620
+ return eligible.reduce((total, scenario) => total + allPass(scenario.passes, scenario.runs.length, j), 0) / eligible.length;
621
+ };
622
+ return {
623
+ scenarios,
624
+ passHat,
625
+ traces
626
+ };
627
+ }
628
+ /**
629
+ * Runs every scenario `k` times and summarizes the runs with
630
+ * {@link summarizeRepeats}. Runs are sequential, scenario by scenario. A run
631
+ * that throws fails, with its message as `error` and no trace.
632
+ *
633
+ * With a live model, one run per scenario says little: τ-bench found pass^k
634
+ * falls well below the single-run rate. Report pass^k next to it.
635
+ *
636
+ * @example
637
+ * ```ts no-check
638
+ * const report = await repeatRuns({
639
+ * scenarios,
640
+ * k: 5,
641
+ * run: async (scenario) => {
642
+ * const trace: AgentTraceEvent[] = [];
643
+ * const result = await runToQuiescence(
644
+ * createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
645
+ * { input: scenario.input },
646
+ * );
647
+ * return { trace, outcome: String(result.snapshot.value) };
648
+ * },
649
+ * passed: (scenario, { outcome }) => outcome === scenario.expected,
650
+ * });
651
+ * report.passHat(5);
652
+ * ```
653
+ */
654
+ async function repeatRuns(options) {
655
+ if (!Number.isInteger(options.k) || options.k < 1) throw new require_event_log.AgentError("invalid-repeat-count", `repeatRuns: k must be a whole number of 1 or more, got ${options.k}.`);
656
+ const groups = [];
657
+ for (const scenario of options.scenarios) {
658
+ const runs = [];
659
+ for (let attempt = 0; attempt < options.k; attempt++) try {
660
+ const result = await options.run(scenario, attempt);
661
+ runs.push({
662
+ passed: options.passed(scenario, result),
663
+ trace: result.trace,
664
+ ...result.outcome === void 0 ? {} : { outcome: result.outcome }
665
+ });
666
+ } catch (error) {
667
+ runs.push({
668
+ passed: false,
669
+ error: error instanceof Error ? error.message : String(error)
670
+ });
671
+ }
672
+ groups.push({
673
+ id: scenario.id,
674
+ scenario,
675
+ runs
676
+ });
677
+ }
678
+ return summarizeRepeats(groups);
679
+ }
680
+ function summarize$1(group) {
681
+ const passes = group.runs.filter((run) => run.passed).length;
682
+ const traces = group.runs.flatMap((run) => run.trace ? [run.trace] : []);
683
+ const paths = new Set(traces.map((trace) => traceSteps(trace).filter((step) => !step.ignored).map((step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`).join(" ")));
684
+ const reference = traces.find((trace) => trace.some((event) => event.type === "machine.transition"));
685
+ let earliest;
686
+ for (const trace of reference ? traces : []) {
687
+ if (trace === reference) continue;
688
+ const divergence = firstDivergence(trace, reference);
689
+ if (divergence && (!earliest || divergence.index < earliest.index)) earliest = divergence;
690
+ }
691
+ return {
692
+ ...group,
693
+ passes,
694
+ passAny: passes > 0,
695
+ passAll: group.runs.length > 0 && passes === group.runs.length,
696
+ distinctPaths: paths.size,
697
+ ...earliest ? { divergedAt: earliest.state } : {}
698
+ };
699
+ }
700
+ /** C(passes, j) / C(runs, j), as a product so large counts don't overflow. */
701
+ function allPass(passes, runs, j) {
702
+ let chance = 1;
703
+ for (let i = 0; i < j; i++) chance *= Math.max(0, passes - i) / (runs - i);
704
+ return chance;
705
+ }
706
+ //#endregion
707
+ //#region src/judge-agreement.ts
708
+ /**
709
+ * Measures how well a judge agrees with people on the same cases: accuracy,
710
+ * per-label recall, the confusion matrix, and the rows that disagree. With
711
+ * `positive`, also the TPR and TNR, the recommended view for binary judges
712
+ * (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
713
+ * Pure.
714
+ *
715
+ * Build rows from confirmed labels only; draft labels gate nothing. Group by
716
+ * decision state or request name so each narrow judge is aligned on its own.
717
+ * A row with no group key counts in the totals and in no group. Run it on a
718
+ * dev split and report the held-out split separately.
719
+ *
720
+ * Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
721
+ *
722
+ * @example
723
+ * ```ts no-check
724
+ * const report = judgeAgreement(
725
+ * labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
726
+ * { positive: 'fail' },
727
+ * );
728
+ * report.binary; // { tpr: 0.9, tnr: 0.8 }
729
+ * report.correct?.(0.3); // estimated true failure rate in production
730
+ * ```
731
+ */
732
+ function judgeAgreement(rows, options = {}) {
733
+ if (rows.length === 0) throw new require_event_log.AgentError("empty-agreement-rows", "judgeAgreement: rows must contain at least one row.");
734
+ const report = summarize(rows, options.positive);
735
+ const groupOf = options.groupBy ?? ((row) => row.group);
736
+ if (options.groupBy || rows.some((row) => row.group !== void 0)) {
737
+ const grouped = /* @__PURE__ */ new Map();
738
+ for (const row of rows) {
739
+ const key = groupOf(row);
740
+ if (key === void 0) continue;
741
+ const members = grouped.get(key);
742
+ if (members) members.push(row);
743
+ else grouped.set(key, [row]);
744
+ }
745
+ const groups = Object.create(null);
746
+ for (const [key, members] of grouped) groups[key] = summarize(members, options.positive);
747
+ report.groups = groups;
748
+ }
749
+ return report;
750
+ }
751
+ function summarize(rows, positive) {
752
+ const perLabel = Object.create(null);
753
+ const confusion = Object.create(null);
754
+ const disagreements = [];
755
+ let agreed = 0;
756
+ for (const { id, human, judge } of rows) {
757
+ const label = perLabel[human] ??= {
758
+ human: 0,
759
+ agreed: 0
760
+ };
761
+ const row = confusion[human] ??= Object.create(null);
762
+ label.human++;
763
+ row[judge] = (row[judge] ?? 0) + 1;
764
+ if (human === judge) {
765
+ label.agreed++;
766
+ agreed++;
767
+ } else disagreements.push({
768
+ id,
769
+ human,
770
+ judge
771
+ });
772
+ }
773
+ const summary = {
774
+ total: rows.length,
775
+ accuracy: agreed / rows.length,
776
+ perLabel,
777
+ confusion,
778
+ disagreements
779
+ };
780
+ if (positive === void 0) return summary;
781
+ let positives = 0;
782
+ let truePositives = 0;
783
+ let negatives = 0;
784
+ let trueNegatives = 0;
785
+ for (const { human, judge } of rows) if (human === positive) {
786
+ positives++;
787
+ if (judge === positive) truePositives++;
788
+ } else {
789
+ negatives++;
790
+ if (judge !== positive) trueNegatives++;
791
+ }
792
+ const tpr = positives === 0 ? void 0 : truePositives / positives;
793
+ const tnr = negatives === 0 ? void 0 : trueNegatives / negatives;
794
+ summary.binary = {
795
+ tpr,
796
+ tnr
797
+ };
798
+ if (tpr !== void 0 && tnr !== void 0 && tpr + tnr > 1) summary.correct = (p) => Math.min(1, Math.max(0, (p + tnr - 1) / (tpr + tnr - 1)));
799
+ return summary;
800
+ }
801
+ //#endregion
802
+ exports.firstDivergence = firstDivergence;
803
+ exports.foldTraces = foldTraces;
804
+ exports.getDecisionRequests = getDecisionRequests;
805
+ exports.judgeAgreement = judgeAgreement;
806
+ exports.proposeApprovalRules = proposeApprovalRules;
807
+ exports.proposeCaps = proposeCaps;
808
+ exports.repeatRuns = repeatRuns;
809
+ exports.summarizeRepeats = summarizeRepeats;
810
+ exports.traceCoverage = traceCoverage;