@statelyai/agent 2.0.0-alpha.37 → 2.0.0-alpha.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/CHANGELOG.md +51 -0
  2. package/dist/{act-CrX0JwWY.cjs → act-B0CVbmo4.cjs} +1 -1
  3. package/dist/{act-B9hLQls6.mjs → act-Bsz-hUfD.mjs} +1 -1
  4. package/dist/ai-sdk.cjs +4 -4
  5. package/dist/ai-sdk.d.cts +2 -3
  6. package/dist/ai-sdk.d.mts +2 -3
  7. package/dist/ai-sdk.mjs +4 -4
  8. package/dist/{decision-m5hGmHpF.mjs → decision-B41YBx5U.mjs} +1 -1
  9. package/dist/{decision-CMeuNZ1g.cjs → decision-D8bcIGXQ.cjs} +1 -1
  10. package/dist/{decision-object-DBcLWVvv.d.cts → decision-object-BB8XV5Ec.d.cts} +1 -1
  11. package/dist/{decision-object-B5zGSPFp.d.mts → decision-object-CH_sE694.d.mts} +1 -1
  12. package/dist/effects-B9IvUPVh.mjs +23 -0
  13. package/dist/effects-DrWcUGgc.cjs +34 -0
  14. package/dist/evals.cjs +810 -0
  15. package/dist/evals.d.cts +595 -0
  16. package/dist/evals.d.mts +595 -0
  17. package/dist/evals.mjs +801 -0
  18. package/dist/{event-log-DkVlk3Rh.mjs → event-log-CEq_bVXQ.mjs} +8 -4
  19. package/dist/{event-log-Dkc-yIYu.cjs → event-log-dSOnXHuG.cjs} +7 -15
  20. package/dist/{event-log-store-Brx9tOHt.mjs → event-log-store-DLAfNM34.mjs} +1 -1
  21. package/dist/{event-log-store-BBS8so8a.d.mts → event-log-store-DYun526m.d.mts} +15 -14
  22. package/dist/{event-log-store-Cl-YWDnq.cjs → event-log-store-Dey9ufA4.cjs} +1 -1
  23. package/dist/{event-log-store-bc4ICGcS.d.cts → event-log-store-aFXK6yZX.d.cts} +15 -14
  24. package/dist/index.cjs +136 -98
  25. package/dist/index.d.cts +73 -134
  26. package/dist/index.d.mts +73 -134
  27. package/dist/index.mjs +117 -75
  28. package/dist/internal-BzlpzSux.d.cts +14 -0
  29. package/dist/internal-BzlpzSux.d.mts +14 -0
  30. package/dist/log.cjs +2 -3
  31. package/dist/log.d.cts +2 -2
  32. package/dist/log.d.mts +2 -2
  33. package/dist/log.mjs +3 -3
  34. package/dist/machines.cjs +1 -719
  35. package/dist/machines.d.cts +2 -424
  36. package/dist/machines.d.mts +2 -424
  37. package/dist/machines.mjs +2 -709
  38. package/dist/{model-source-kjGwqAV8.mjs → model-source-BrsLDvHL.mjs} +1 -1
  39. package/dist/{model-source-B4WQ8Idp.cjs → model-source-YpxjmWuX.cjs} +1 -1
  40. package/dist/openai.cjs +3 -3
  41. package/dist/openai.d.cts +2 -3
  42. package/dist/openai.d.mts +2 -3
  43. package/dist/openai.mjs +3 -3
  44. package/dist/otel.d.cts +1 -1
  45. package/dist/otel.d.mts +1 -1
  46. package/dist/policy.cjs +77 -0
  47. package/dist/policy.d.cts +143 -0
  48. package/dist/policy.d.mts +143 -0
  49. package/dist/policy.mjs +75 -0
  50. package/dist/{effects-BT_cSY0j.mjs → run-agent-BE7DOWD6.mjs} +150 -184
  51. package/dist/{run-agent-DlPzBWj-.d.mts → run-agent-DHlq87G7.d.mts} +18 -34
  52. package/dist/{effects-DDseYtJk.cjs → run-agent-De3xVFGw.cjs} +149 -201
  53. package/dist/{run-agent-30mH2moE.d.cts → run-agent-Dq0oSFRm.d.cts} +18 -34
  54. package/dist/{setup-agent-BDmNr6PC.mjs → setup-agent-CLm6QVGy.mjs} +3 -3
  55. package/dist/{setup-agent-BaNyz6Q3.d.cts → setup-agent-Cv8Hk3mX.d.cts} +12 -11
  56. package/dist/{setup-agent-C1CG_nfX.d.mts → setup-agent-DgFAhaPJ.d.mts} +12 -11
  57. package/dist/{setup-agent-CmEToB8y.cjs → setup-agent-Dv5VpuHe.cjs} +3 -3
  58. package/dist/tanstack-ai.cjs +4 -4
  59. package/dist/tanstack-ai.d.cts +2 -3
  60. package/dist/tanstack-ai.d.mts +2 -3
  61. package/dist/tanstack-ai.mjs +4 -4
  62. package/dist/testing.cjs +122 -996
  63. package/dist/testing.d.cts +36 -695
  64. package/dist/testing.d.mts +36 -695
  65. package/dist/testing.mjs +112 -977
  66. package/dist/{text-logic-BqxalgMr.d.mts → text-logic-x17uLBwA.d.cts} +238 -2
  67. package/dist/{text-logic-CSf9bqpP.d.cts → text-logic-x17uLBwA.d.mts} +238 -2
  68. package/dist/trajectory-BFjXaooB.d.cts +96 -0
  69. package/dist/trajectory-BFjXaooB.d.mts +96 -0
  70. package/dist/trajectory-BUXj_1Ue.mjs +152 -0
  71. package/dist/trajectory-CoZSv4wF.cjs +157 -0
  72. package/dist/{usage-Bm5nvpow.d.cts → usage-B4B-TQSV.d.mts} +1 -1
  73. package/dist/{usage-BnikdOca.d.mts → usage-DvGcoK6x.d.cts} +1 -1
  74. package/dist/{utils-7lSa4H-i.d.cts → utils-B8k-YPYI.d.mts} +23 -17
  75. package/dist/{utils-CBbKw2fM.d.mts → utils-DAclF-kw.d.cts} +23 -17
  76. package/dist/validate.cjs +4 -4
  77. package/dist/validate.d.cts +2 -2
  78. package/dist/validate.d.mts +2 -2
  79. package/dist/validate.mjs +4 -4
  80. package/dist/{verify-DJHfEfm6.d.cts → verify-BM0sGmsf.d.cts} +42 -6
  81. package/dist/{verify-ChyX6jq0.mjs → verify-CWlg3j7B.mjs} +153 -45
  82. package/dist/{verify-CwlombbC.cjs → verify-Dek6gHCS.cjs} +153 -45
  83. package/dist/{verify-i9Hiwc5g.d.mts → verify-H3Pr2gNw.d.mts} +42 -6
  84. package/dist/{with-model-calls-B_WLmOSl.cjs → with-model-calls-D50Ca6qW.cjs} +2 -2
  85. package/dist/{with-model-calls-BkOwdroS.mjs → with-model-calls-uY-KKliA.mjs} +2 -2
  86. package/docs/advanced.md +5 -5
  87. package/docs/choosing-a-run-mode.md +7 -7
  88. package/docs/debugging.md +6 -8
  89. package/docs/decisions.md +6 -5
  90. package/docs/environment-and-policy.md +1 -1
  91. package/docs/evals.md +57 -43
  92. package/docs/event-log.md +14 -16
  93. package/docs/from-a-loop.md +3 -3
  94. package/docs/generate-machines.md +4 -4
  95. package/docs/hosts.md +22 -29
  96. package/docs/human-in-the-loop.md +5 -8
  97. package/docs/index.md +4 -4
  98. package/docs/judgments.md +5 -7
  99. package/docs/langgraph-comparison.md +6 -6
  100. package/docs/machines-as-data.md +12 -17
  101. package/docs/machines-presets.md +17 -120
  102. package/docs/machines.md +13 -13
  103. package/docs/models-and-providers.md +19 -25
  104. package/docs/multi-agent.md +1 -1
  105. package/docs/observability.md +6 -8
  106. package/docs/persistence.md +15 -25
  107. package/docs/quickstart.md +4 -4
  108. package/docs/statechart-policy-examples.md +1 -1
  109. package/docs/steps.md +5 -6
  110. package/docs/strangler-fig.md +72 -7
  111. package/docs/text-requests.md +10 -12
  112. package/docs/thinking-in-state-machines.md +8 -8
  113. package/docs/tools.md +10 -12
  114. package/docs/upgrading.md +1 -1
  115. package/docs/usage-and-budgets.md +51 -56
  116. package/docs/validate-and-repair.md +9 -14
  117. package/docs/verify.md +53 -7
  118. package/docs/xstate-4-and-5-apps.md +19 -28
  119. package/llms.txt +1 -1
  120. package/package.json +31 -6
  121. package/readme.md +32 -24
  122. package/dist/internal-C3Lqu538.d.cts +0 -43
  123. package/dist/internal-DUcdhk-i.d.mts +0 -43
  124. package/dist/types-DQUOOaZO.d.cts +0 -238
  125. package/dist/types-DQUOOaZO.d.mts +0 -238
  126. package/skills/add-evals/SKILL.md +0 -152
  127. package/skills/add-traces/SKILL.md +0 -152
  128. package/skills/build-machine/SKILL.md +0 -84
  129. package/skills/generate-machine/SKILL.md +0 -335
  130. package/skills/graph-engineering/SKILL.md +0 -43
  131. package/skills/improve-machine/SKILL.md +0 -261
  132. package/skills/improve-machine/reference.md +0 -101
  133. package/skills/loop-to-machine/SKILL.md +0 -205
  134. package/skills/loop-to-machine/policy.md +0 -154
  135. package/skills/loop-to-machine/reference.md +0 -260
package/dist/testing.mjs CHANGED
@@ -1,10 +1,11 @@
1
- import { L as getCallUsage, R as isTextLogic, S as replay, c as agentCallSite, d as canonicalEventJson, ft as isError, ht as AgentError, mt as isRecord, pt as isPlainObject, rt as getStateMeta, z as normalizeGeneratorResult } from "./event-log-DkVlk3Rh.mjs";
2
- import { s as isDecisionLogic } from "./decision-m5hGmHpF.mjs";
3
- import { i as isActLogic } from "./act-B9hLQls6.mjs";
4
- import { _ as runToQuiescence, d as createAgentRuntime, h as isStateMachineLogic, n as mapEffects, t as isAsyncIterable, y as toJsonValue } from "./effects-BT_cSY0j.mjs";
5
- import { n as createInMemoryEventLogStore } from "./event-log-store-Brx9tOHt.mjs";
6
- import { a as explorePaths, c as simulateAgent, i as canReach, l as textOutput, n as AgentUnknownStateError, o as isExplorationComplete, r as assertAgentMachine, s as lintAgentMachine, t as AgentLintError } from "./verify-ChyX6jq0.mjs";
7
- import { getInitialMicrosteps, getMicrosteps } from "xstate";
1
+ import { I as getCallUsage, L as isTextLogic, R as normalizeGeneratorResult, c as agentCallSite, d as canonicalEventJson, ft as isRecord, pt as AgentError, tt as getStateMeta, x as replay } from "./event-log-CEq_bVXQ.mjs";
2
+ import { s as isDecisionLogic } from "./decision-B41YBx5U.mjs";
3
+ import { i as isActLogic } from "./act-Bsz-hUfD.mjs";
4
+ import { n as createInMemoryEventLogStore } from "./event-log-store-DLAfNM34.mjs";
5
+ import { f as isStateMachineLogic, g as toJsonValue, l as createAgentRuntime, m as runToQuiescence } from "./run-agent-BE7DOWD6.mjs";
6
+ import { n as mapEffects, t as isAsyncIterable } from "./effects-B9IvUPVh.mjs";
7
+ import { a as explorePaths, c as simulateAgent, i as canReach, l as textOutput, n as AgentUnknownStateError, r as assertAgentMachine, s as lintAgentMachine, t as AgentLintError } from "./verify-CWlg3j7B.mjs";
8
+ import { t as matchesTrajectory } from "./trajectory-BUXj_1Ue.mjs";
8
9
  //#region src/equivalence.ts
9
10
  /**
10
11
  * Verified equivalence — prove a port behaves like the code it replaces.
@@ -46,11 +47,12 @@ import { getInitialMicrosteps, getMicrosteps } from "xstate";
46
47
  var RecordingDivergenceError = class extends Error {
47
48
  name = "RecordingDivergenceError";
48
49
  };
49
- function toRecordedError(error) {
50
+ function toRecordedError(error, path = /* @__PURE__ */ new Set()) {
50
51
  if (!(error instanceof Error)) return {
51
52
  name: "Error",
52
53
  message: String(error)
53
54
  };
55
+ path.add(error);
54
56
  const recorded = {
55
57
  name: error.name,
56
58
  message: error.message
@@ -66,10 +68,39 @@ function toRecordedError(error) {
66
68
  if (key !== void 0 && error[symbol] === true) markers.add(key);
67
69
  }
68
70
  if (markers.size) recorded.markers = [...markers];
71
+ if (error.cause instanceof Error && !path.has(error.cause)) recorded.cause = toRecordedError(error.cause, path);
72
+ if (error instanceof AggregateError && Array.isArray(error.errors)) recorded.errors = error.errors.flatMap((item) => {
73
+ if (!(item instanceof Error)) try {
74
+ return [{ value: toJson(item) }];
75
+ } catch {
76
+ try {
77
+ return [{ value: String(item) }];
78
+ } catch {
79
+ return [{ value: "[unserializable]" }];
80
+ }
81
+ }
82
+ return path.has(item) ? [] : [toRecordedError(item, path)];
83
+ });
84
+ path.delete(error);
69
85
  return recorded;
70
86
  }
71
- function fromRecordedError({ data, markers, name, ownName, ...fields }) {
72
- const error = Object.assign(new Error(fields.message), fields, data);
87
+ function fromRecordedError(recorded, reviveError) {
88
+ const { data, markers, name, ownName, cause, errors, ...fields } = recorded;
89
+ const revivedErrors = errors?.map((item) => "name" in item ? fromRecordedError(item, reviveError) : structuredClone(item.value));
90
+ const define = (target, key, value) => Object.defineProperty(target, key, {
91
+ value,
92
+ writable: true,
93
+ configurable: true
94
+ });
95
+ const custom = reviveError?.(structuredClone(recorded));
96
+ if (custom) {
97
+ if (cause && custom.cause === void 0) define(custom, "cause", fromRecordedError(cause, reviveError));
98
+ if (revivedErrors && custom instanceof AggregateError) define(custom, "errors", revivedErrors);
99
+ return custom;
100
+ }
101
+ const base = revivedErrors ? new AggregateError(revivedErrors, fields.message) : new Error(fields.message);
102
+ const error = Object.assign(base, fields, data);
103
+ if (cause) define(error, "cause", fromRecordedError(cause, reviveError));
73
104
  if (ownName || name !== "Error") Object.defineProperty(error, "name", {
74
105
  value: name,
75
106
  writable: true,
@@ -132,6 +163,25 @@ async function* iterate(chunks) {
132
163
  for (const chunk of chunks) yield structuredClone(chunk);
133
164
  }
134
165
  /**
166
+ * Yields what `revived` yields (awaited), if it is an async iterable, then
167
+ * throws the recorded `failure`, if any. Like the recording wrapper, it calls
168
+ * `onError` only for a failure the port reads to, or one its consumer throws
169
+ * in (`.throw()`); `.return()` ends it with no error.
170
+ */
171
+ async function* replayStream(revived, onError, failure) {
172
+ try {
173
+ const value = await revived;
174
+ if (isAsyncIterable(value)) yield* value;
175
+ } catch (error) {
176
+ onError(toRecordedError(error));
177
+ throw error;
178
+ }
179
+ if (failure) {
180
+ onError(failure.recorded);
181
+ throw failure.error;
182
+ }
183
+ }
184
+ /**
135
185
  * Creates a {@link Recorder}. With no options it records: every wrapped call
136
186
  * runs for real. With `replay` it answers from a recording instead.
137
187
  */
@@ -146,6 +196,7 @@ function createRecorder(options = {}) {
146
196
  const normalizeEffect = (effect, list) => options.normalize?.(effect, list.indexOf(effect), list) ?? effect;
147
197
  let modelCalls = 0;
148
198
  let toolsInFlight = 0;
199
+ const reviveRecordedError = (error) => fromRecordedError(error, options.reviveError);
149
200
  /**
150
201
  * The earliest unused recorded call that fits this one: one `translate`
151
202
  * adapts, or one with the same method and shape. Not by position, so a call
@@ -195,13 +246,23 @@ function createRecorder(options = {}) {
195
246
  const { recorded, answer } = replayed;
196
247
  usedModels.add(recorded);
197
248
  effect.replayed = recorded.call;
249
+ const recordError = (error) => {
250
+ effect.error = error;
251
+ };
252
+ if (answer.error && answer.stream && streamed && seam === "openai") {
253
+ effect.stream = answer.stream;
254
+ return replayStream(iterate(answer.stream), recordError, {
255
+ recorded: answer.error,
256
+ error: reviveRecordedError(answer.error)
257
+ });
258
+ }
198
259
  if (answer.error) {
199
260
  effect.error = answer.error;
200
- throw fromRecordedError(answer.error);
261
+ throw reviveRecordedError(answer.error);
201
262
  }
202
263
  effect.response = answer.response;
203
264
  effect.stream = answer.stream;
204
- if (seam === "openai") return streamed ? iterate(answer.stream ?? []) : structuredClone(answer.response);
265
+ if (seam === "openai") return streamed ? replayStream(iterate(answer.stream ?? []), recordError) : structuredClone(answer.response);
205
266
  return streamed ? {
206
267
  ...reviveTimestamps(structuredClone(answer.response) ?? {}),
207
268
  stream: streamFrom(answer.stream ?? [])
@@ -231,9 +292,14 @@ function createRecorder(options = {}) {
231
292
  if (seam === "openai") {
232
293
  const source = result;
233
294
  return (async function* () {
234
- for await (const chunk of source) {
235
- parts.push(toJson(chunk));
236
- yield chunk;
295
+ try {
296
+ for await (const chunk of source) {
297
+ parts.push(toJson(chunk));
298
+ yield chunk;
299
+ }
300
+ } catch (error) {
301
+ effect.error = toRecordedError(error);
302
+ throw error;
237
303
  }
238
304
  })();
239
305
  }
@@ -289,7 +355,7 @@ function createRecorder(options = {}) {
289
355
  if (recorded.sync) effect.sync = true;
290
356
  if (recorded.error) {
291
357
  effect.error = recorded.error;
292
- throw fromRecordedError(recorded.error);
358
+ throw reviveRecordedError(recorded.error);
293
359
  }
294
360
  effect.output = recorded.output;
295
361
  return structuredClone(recorded.output);
@@ -342,14 +408,23 @@ function createRecorder(options = {}) {
342
408
  throw error;
343
409
  }
344
410
  usedCalls.add(recorded);
345
- if (recorded.error) {
346
- effect.error = recorded.error;
347
- throw fromRecordedError(recorded.error);
348
- }
349
411
  if (recorded.stream) {
350
412
  effect.stream = recorded.stream;
351
413
  const chunks = structuredClone(recorded.stream);
352
- return revive ? revive(chunks) : iterate(chunks);
414
+ const revived = revive ? revive(chunks) : iterate(chunks);
415
+ const { error } = recorded;
416
+ if (!error && revive) return revived;
417
+ const recordError = (thrown) => {
418
+ effect.error = thrown;
419
+ };
420
+ return replayStream(revived, recordError, error && {
421
+ recorded: error,
422
+ error: reviveRecordedError(error)
423
+ });
424
+ }
425
+ if (recorded.error) {
426
+ effect.error = recorded.error;
427
+ throw reviveRecordedError(recorded.error);
353
428
  }
354
429
  effect.result = recorded.result;
355
430
  const result = structuredClone(recorded.result);
@@ -537,7 +612,7 @@ function modelCallShape(effect) {
537
612
  * that started after the previous one settled begins a new group, so
538
613
  * sequential calls keep their order.
539
614
  */
540
- function normalize$1(effects) {
615
+ function normalize(effects) {
541
616
  const result = [];
542
617
  let batch = [];
543
618
  const flush = () => {
@@ -628,7 +703,7 @@ function differ(left, right, options) {
628
703
  * kind pair up and report what changed, and the rest are missing or extra.
629
704
  */
630
705
  function compareRecordings(expected, actual, options = {}) {
631
- const select = (effects) => normalize$1(effects.flatMap((effect, index) => {
706
+ const select = (effects) => normalize(effects.flatMap((effect, index) => {
632
707
  if (options.kinds && !options.kinds.includes(effect.kind)) return [];
633
708
  const mapped = options.normalize ? options.normalize(effect, index, effects) : effect;
634
709
  return mapped ? [mapped] : [];
@@ -1306,156 +1381,6 @@ function trackLogic(logic, world) {
1306
1381
  return Object.create(logic, overrides);
1307
1382
  }
1308
1383
  //#endregion
1309
- //#region src/trajectory.ts
1310
- /**
1311
- * Classifies one item. Event-shaped things (log entries, events) become events;
1312
- * everything else is a state value. A snapshot is unwrapped to its `value`.
1313
- * @internal
1314
- */
1315
- function normalize(item) {
1316
- if (isRecord(item)) {
1317
- const nested = item["event"];
1318
- if (isRecord(nested) && typeof nested["type"] === "string") return {
1319
- kind: "event",
1320
- event: nested
1321
- };
1322
- if (typeof item["type"] === "string") return {
1323
- kind: "event",
1324
- event: item
1325
- };
1326
- if ("value" in item) return {
1327
- kind: "state",
1328
- value: item["value"]
1329
- };
1330
- }
1331
- return {
1332
- kind: "state",
1333
- value: item
1334
- };
1335
- }
1336
- /** Every leaf of a state value as a dot path (`{ review: 'editing' }` → `review.editing`). @internal */
1337
- function statePaths$1(value) {
1338
- if (typeof value === "string") return [value];
1339
- if (!isRecord(value)) return [];
1340
- return Object.entries(value).flatMap(([key, child]) => statePaths$1(child).map((path) => `${key}.${path}`));
1341
- }
1342
- /**
1343
- * A string expectation matches a state value when it names one of its leaves or
1344
- * an ancestor of one, so `'review'` matches `{ review: 'editing' }`. @internal
1345
- */
1346
- function stateMatchesPath(path, value) {
1347
- return statePaths$1(value).some((leaf) => leaf === path || leaf.startsWith(`${path}.`));
1348
- }
1349
- function deepEqual(a, b) {
1350
- if (Object.is(a, b)) return true;
1351
- if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => deepEqual(item, b[index]));
1352
- if (!isRecord(a) || !isRecord(b)) return false;
1353
- const keys = Object.keys(a);
1354
- return keys.length === Object.keys(b).length && keys.every((key) => deepEqual(a[key], b[key]));
1355
- }
1356
- /**
1357
- * Normalizes ONE expected item into a predicate over actual items, so the
1358
- * expectation is classified once instead of once per candidate:
1359
- *
1360
- * - A string expectation matches an event's `type`, or a state value it names.
1361
- * - An event expectation matches an event with the same `type` whose other
1362
- * declared keys deep-equal, so payload details are opt-in.
1363
- * - A state-value expectation matches structurally.
1364
- * @internal
1365
- */
1366
- function matcherFor(expected) {
1367
- if (typeof expected === "string") return (actual) => {
1368
- const actualItem = normalize(actual);
1369
- return actualItem.kind === "event" ? actualItem.event.type === expected : stateMatchesPath(expected, actualItem.value);
1370
- };
1371
- const expectedItem = normalize(expected);
1372
- switch (expectedItem.kind) {
1373
- case "event": return (actual) => {
1374
- const actualItem = normalize(actual);
1375
- return actualItem.kind === "event" && Object.keys(expectedItem.event).every((key) => deepEqual(expectedItem.event[key], actualItem.event[key]));
1376
- };
1377
- case "state": return (actual) => {
1378
- const actualItem = normalize(actual);
1379
- return actualItem.kind === "state" && deepEqual(expectedItem.value, actualItem.value);
1380
- };
1381
- }
1382
- }
1383
- /**
1384
- * Matches a run's trajectory against an expected one.
1385
- *
1386
- * Both trajectories may be state values or events collected from
1387
- * `onTransition`;
1388
- * items are compared by shape, so `['prompting', 'drafting']` scores a state
1389
- * path and `['PROMPT_SUBMITTED', 'SEND']` scores an event log with the same
1390
- * call.
1391
- *
1392
- * Semantics are an ordered subsequence: every expected item must appear in
1393
- * `actual`, in order, with gaps allowed. Pass `{ exact: true }` to require
1394
- * equality instead.
1395
- *
1396
- * @example Trajectory assertions in a test
1397
- * ```ts
1398
- * const statePath: unknown[] = [];
1399
- * const result = await runToQuiescence(
1400
- * createAgentRuntime(machine, {
1401
- * executors,
1402
- * onTransition: (snapshot) => statePath.push(snapshot.value),
1403
- * }),
1404
- * { input },
1405
- * );
1406
- *
1407
- * const path = matchesTrajectory(statePath, ['prompting', 'drafting', 'sent']);
1408
- * expect(path.matched, JSON.stringify(path.firstMiss)).toBe(true);
1409
- * expect(matchesTrajectory(events, ['PROMPT_SUBMITTED', 'SEND']).matched).toBe(true);
1410
- * ```
1411
- *
1412
- * @example Partial credit in an eval scorer
1413
- * ```ts
1414
- * const match = matchesTrajectory(output.statePath, expected.statePath);
1415
- * return { name: 'state_path', score: match.score, metadata: { ...match.firstMiss } };
1416
- * ```
1417
- */
1418
- function matchesTrajectory(actual, expected, options = {}) {
1419
- const expectedCount = expected.length;
1420
- if (expectedCount === 0) throw new AgentError("empty-trajectory", "matchesTrajectory: expected trajectory must contain at least one item.");
1421
- const exact = options.exact === true;
1422
- let cursor = 0;
1423
- let matchedCount = 0;
1424
- let firstMiss;
1425
- for (let index = 0; index < expectedCount; index++) {
1426
- const want = expected[index];
1427
- const matches = matcherFor(want);
1428
- const searchedFrom = exact ? index : cursor;
1429
- let found = -1;
1430
- for (let at = searchedFrom; at < actual.length; at++) {
1431
- if (matches(actual[at])) {
1432
- found = at;
1433
- break;
1434
- }
1435
- if (exact) break;
1436
- }
1437
- if (found === -1) {
1438
- firstMiss = {
1439
- index,
1440
- expected: want,
1441
- searchedFrom
1442
- };
1443
- break;
1444
- }
1445
- matchedCount++;
1446
- cursor = found + 1;
1447
- }
1448
- const matched = !firstMiss && (!exact || actual.length === expectedCount);
1449
- const denominator = exact ? Math.max(expectedCount, actual.length) : expectedCount;
1450
- return {
1451
- matched,
1452
- matchedCount,
1453
- expectedCount,
1454
- score: matched ? 1 : denominator === 0 ? 1 : matchedCount / denominator,
1455
- ...firstMiss ? { firstMiss } : {}
1456
- };
1457
- }
1458
- //#endregion
1459
1384
  //#region src/seam.ts
1460
1385
  /** The only own keys an executor result carries. @internal */
1461
1386
  const TEXT_RESULT_KEYS = /* @__PURE__ */ new Set([
@@ -1664,802 +1589,6 @@ async function runSeam(machine, options) {
1664
1589
  };
1665
1590
  }
1666
1591
  //#endregion
1667
- //#region src/internal/trace-states.ts
1668
- /** The `value` of a trace event's `from` or `snapshot`. @internal */
1669
- function valueOf(holder) {
1670
- return isRecord(holder) ? holder.value : void 0;
1671
- }
1672
- /** A decision request on its first attempt (no failed attempts yet). @internal */
1673
- function isFirstDecision(request) {
1674
- return isRecord(request) && request.kind === "decision" && (!Array.isArray(request.attempts) || request.attempts.length === 0);
1675
- }
1676
- /**
1677
- * Every active leaf of a state value as a dot path:
1678
- * `'a'` → `['a']`; `{ a: 'b' }` → `['a.b']`; `{ p: { x: 'one', y: {} } }` → `['p.x.one', 'p.y']`.
1679
- * @internal
1680
- */
1681
- function statePaths(value, prefix = "") {
1682
- if (typeof value === "string") return [prefix + value];
1683
- if (!isRecord(value)) return [];
1684
- const entries = Object.entries(value);
1685
- if (entries.length === 0) return prefix ? [prefix.slice(0, -1)] : [];
1686
- return entries.flatMap(([key, child]) => statePaths(child, `${prefix}${key}.`));
1687
- }
1688
- /**
1689
- * A trace's `machine.transition` events as steps, skipping `@xstate.init`.
1690
- * A first-attempt decision's `request.start` names the decision on the step
1691
- * that follows it, skipping `@agent.usage` steps. Reads live and serialized traces alike.
1692
- * @internal
1693
- */
1694
- function traceSteps(trace) {
1695
- const steps = [];
1696
- let decision;
1697
- for (const event of trace) {
1698
- if (event.type === "request.start" && isFirstDecision(event.request)) {
1699
- const name = event.request.name;
1700
- decision = typeof name === "string" ? name : void 0;
1701
- continue;
1702
- }
1703
- if (event.type !== "machine.transition") continue;
1704
- const type = isRecord(event.event) ? event.event.type : void 0;
1705
- if (type === "@xstate.init") continue;
1706
- const fromValue = valueOf(event.from);
1707
- const toValue = valueOf(event.snapshot);
1708
- steps.push({
1709
- from: statePaths(fromValue),
1710
- fromValue,
1711
- event: typeof type === "string" ? type : "",
1712
- payload: event.event,
1713
- to: statePaths(toValue),
1714
- toValue,
1715
- ignored: event.ignored === true,
1716
- ...decision === void 0 || type === "@agent.usage" ? {} : { decision }
1717
- });
1718
- if (type !== "@agent.usage") decision = void 0;
1719
- }
1720
- return steps;
1721
- }
1722
- //#endregion
1723
- //#region src/fold-traces.ts
1724
- /**
1725
- * Folds many runs' traces into per-state stats: how many runs visited each
1726
- * state, how many of those failed, how many options the guards withheld
1727
- * at decisions made there, how many events it ignored, and how many failures
1728
- * `blame` traced back to it. Pure; reads only:
1729
- *
1730
- * - `machine.transition`: the run visited `from.value` and `snapshot.value`;
1731
- * with `ignored`, each of those states ignored one event.
1732
- * - `request.start` of a decision, first attempt only: its `request.withheld`
1733
- * counts toward the state the run was in, the `from.value` of the
1734
- * transition its choice causes.
1735
- *
1736
- * A trace is one run's events, live (`AgentTraceEvent`) or serialized
1737
- * (`serializeTraceEvent`). A host-driven `decideNext` traces nothing; push
1738
- * `{ type: 'request.start', request: result.request }` onto the trace before
1739
- * sending its event. Parallel states count once per active leaf.
1740
- *
1741
- * `failures` counts a failed run against every state it visited, so states
1742
- * after a bad decision rank as high as the decision. Sort by `blamed` to see
1743
- * where failures start.
1744
- *
1745
- * @example
1746
- * ```ts no-check
1747
- * const stats = foldTraces(traces, {
1748
- * failed: (trace) => trace.at(-1)?.type === 'run.end' && trace.at(-1).status === 'error',
1749
- * });
1750
- * ```
1751
- */
1752
- function foldTraces(traces, options) {
1753
- const stats = /* @__PURE__ */ new Map();
1754
- const statsFor = (state) => {
1755
- let entry = stats.get(state);
1756
- if (!entry) {
1757
- entry = {
1758
- state,
1759
- visits: 0,
1760
- failures: 0,
1761
- withheld: 0,
1762
- ignored: 0,
1763
- blamed: 0
1764
- };
1765
- stats.set(state, entry);
1766
- }
1767
- return entry;
1768
- };
1769
- for (const trace of traces) {
1770
- const visited = /* @__PURE__ */ new Set();
1771
- let current = [];
1772
- const visit = (state) => {
1773
- statsFor(state);
1774
- visited.add(state);
1775
- };
1776
- for (const event of trace) if (event.type === "machine.transition") {
1777
- for (const state of statePaths(valueOf(event.from))) visit(state);
1778
- current = statePaths(valueOf(event.snapshot));
1779
- for (const state of current) visit(state);
1780
- if (event.ignored === true) for (const state of current) statsFor(state).ignored++;
1781
- } else if (event.type === "request.start" && isFirstDecision(event.request)) {
1782
- const withheld = event.request.withheld;
1783
- if (!Array.isArray(withheld)) continue;
1784
- for (const state of current) statsFor(state).withheld += withheld.length;
1785
- }
1786
- const failed = options.failed(trace);
1787
- for (const state of visited) {
1788
- const entry = statsFor(state);
1789
- entry.visits++;
1790
- if (failed) entry.failures++;
1791
- }
1792
- const blamed = failed ? options.blame?.(trace) : void 0;
1793
- if (blamed !== void 0) statsFor(blamed).blamed++;
1794
- }
1795
- return [...stats.values()];
1796
- }
1797
- //#endregion
1798
- //#region src/trace-coverage.ts
1799
- /** Mirrors xstate@6.0.0-alpha.69 `matchesEventDescriptor` (src/utils.ts), which the package does not export. */
1800
- function matchesDescriptor(eventType, descriptor) {
1801
- if (descriptor === eventType || descriptor === "*") return true;
1802
- if (!descriptor.endsWith(".*")) return false;
1803
- const tokens = descriptor.split(".");
1804
- const eventTokens = eventType.split(".");
1805
- for (let index = 0; index < tokens.length; index++) {
1806
- if (tokens[index] === "*") return index === tokens.length - 1;
1807
- if (tokens[index] !== eventTokens[index]) return false;
1808
- }
1809
- return true;
1810
- }
1811
- /** Whether a definition's `matches` pattern (e.g. an invoke's `actorId`) accepts the traced event. */
1812
- function matchesPattern(definition, event) {
1813
- const pattern = definition.matches;
1814
- if (!isRecord(pattern) || !isRecord(event)) return true;
1815
- return Object.entries(pattern).every(([key, value]) => Object.is(event[key], value));
1816
- }
1817
- /**
1818
- * The descriptor XState would try first at `node` for `event`, guards aside:
1819
- * the exact key, then partial and wildcard keys longest first, as in
1820
- * xstate@6.0.0-alpha.69 `getCandidates` (src/stateUtils.ts).
1821
- */
1822
- function candidateAt(node, type, event) {
1823
- const keys = [...node.transitions.keys()].filter((key) => key !== type && matchesDescriptor(type, key)).sort((a, b) => b.length - a.length);
1824
- if (node.transitions.has(type)) keys.unshift(type);
1825
- return keys.find((key) => node.transitions.get(key).some((d) => matchesPattern(d, event)));
1826
- }
1827
- const pathOf = (node) => node.path.join(".");
1828
- const sameLeaves = (a, b) => statePaths(a).sort().join("|") === statePaths(b).sort().join("|");
1829
- /**
1830
- * The leaves a trace passed through inside its macrosteps: states an `always`
1831
- * transition left before the step ended, which no traced snapshot holds.
1832
- * Re-derived with XState's pure `getMicrosteps` from the snapshot before each
1833
- * step (the previous step's, or the fresh start's microsteps from its input).
1834
- * A step that cannot be recomputed, or whose recomputation ends elsewhere
1835
- * than the traced snapshot, adds none. Only `value` and `context` are rebuilt,
1836
- * so a guard that reads live children can take another route to the same
1837
- * leaves, and that route is credited.
1838
- */
1839
- function passedThrough(machine, trace) {
1840
- const leaves = /* @__PURE__ */ new Set();
1841
- let resumed = false;
1842
- let previous;
1843
- for (const event of trace) {
1844
- if (event.type === "run.start") resumed = event.snapshot !== void 0;
1845
- if (event.type !== "machine.transition" || !isRecord(event.event)) continue;
1846
- const snapshot = isRecord(event.snapshot) ? event.snapshot : void 0;
1847
- try {
1848
- const microsteps = event.event.type === "@xstate.init" ? resumed ? [] : getInitialMicrosteps(machine, event.event.input) : previous && sameLeaves(previous.value, valueOf(event.from)) ? getMicrosteps(machine, machine.resolveState({
1849
- value: previous.value,
1850
- context: previous.context
1851
- }), event.event) : [];
1852
- const last = microsteps.at(-1)?.[0];
1853
- if (last && snapshot && sameLeaves(last.value, snapshot.value)) for (const [step] of microsteps) for (const leaf of statePaths(step.value)) leaves.add(leaf);
1854
- } catch {}
1855
- previous = snapshot;
1856
- }
1857
- return leaves;
1858
- }
1859
- const ratioOf = (visited, total) => total === 0 ? 1 : visited / total;
1860
- /**
1861
- * Measures how much of a machine a set of runs exercised: which atomic and
1862
- * final states they visited, and which handlers they took. A handler is a
1863
- * `(declaring state, event descriptor)` pair from `on`, invoke
1864
- * `onDone`/`onError`, and `after`, with the targets it declares and the
1865
- * leaves the runs landed in after it. Pure; reads only `machine.transition`
1866
- * events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
1867
- *
1868
- * A step credits one handler per active leaf: the nearest node, from the leaf
1869
- * up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
1870
- * event, preferred in XState's order. Guards are not replayed, so a step
1871
- * whose nearest handler was rejected and fell through to an ancestor is
1872
- * credited to the nearest one. XState keys invoke and `after` handlers by
1873
- * bare type (`xstate.done.actor`), one per state however many invokes; a
1874
- * definition's `matches` (an invoke's `actorId`) must accept the traced event
1875
- * for its state to be credited. Ignored steps credit no handler. `always`
1876
- * handlers are listed as `unobservable`, never as unvisited; a state one
1877
- * passes through is visited, re-derived from each step's microsteps.
1878
- *
1879
- * @example
1880
- * ```ts no-check
1881
- * const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
1882
- * expect(report.unvisitedHandlers).toEqual([]);
1883
- * ```
1884
- */
1885
- function traceCoverage(machine, traces, options = {}) {
1886
- const nodes = /* @__PURE__ */ new Map();
1887
- const states = [];
1888
- const handlers = [];
1889
- const handlerAt = /* @__PURE__ */ new Map();
1890
- const unobservable = [];
1891
- const walk = (node) => {
1892
- const state = pathOf(node);
1893
- nodes.set(state, node);
1894
- if (node.type === "atomic" || node.type === "final") states.push({
1895
- state,
1896
- runs: 0
1897
- });
1898
- const byEvent = /* @__PURE__ */ new Map();
1899
- for (const [event, definitions] of node.transitions) {
1900
- const dynamic = definitions.some((d) => typeof d.to === "function");
1901
- const declared = [...new Set(definitions.flatMap((d) => (d.target ?? []).map(pathOf)))];
1902
- const handler = {
1903
- state,
1904
- event,
1905
- runs: 0,
1906
- targets: {
1907
- declared: dynamic ? "dynamic" : declared,
1908
- seen: [],
1909
- unreached: []
1910
- }
1911
- };
1912
- handlers.push(handler);
1913
- byEvent.set(event, handler);
1914
- }
1915
- handlerAt.set(node, byEvent);
1916
- if (node.always?.length) unobservable.push({
1917
- state,
1918
- event: "always"
1919
- });
1920
- for (const child of Object.values(node.states)) walk(child);
1921
- };
1922
- walk(machine.root);
1923
- const stateRuns = new Map(states.map((row) => [row.state, row]));
1924
- const seen = /* @__PURE__ */ new Map();
1925
- for (const trace of traces) {
1926
- const steps = traceSteps(trace);
1927
- const visited = new Set(steps[0]?.from);
1928
- for (const event of trace) {
1929
- if (event.type !== "machine.transition" || !isRecord(event.event)) continue;
1930
- if (event.event.type !== "@xstate.init" || !isRecord(event.snapshot)) continue;
1931
- for (const leaf of statePaths(event.snapshot.value)) visited.add(leaf);
1932
- }
1933
- if (unobservable.length > 0) for (const leaf of passedThrough(machine, trace)) visited.add(leaf);
1934
- const credited = /* @__PURE__ */ new Set();
1935
- for (const step of steps) {
1936
- for (const leaf of step.to) visited.add(leaf);
1937
- if (step.ignored) continue;
1938
- const hits = /* @__PURE__ */ new Set();
1939
- for (const leaf of step.from) for (let node = nodes.get(leaf); node; node = node.parent) {
1940
- if (node.type === "final") continue;
1941
- const event = candidateAt(node, step.event, step.payload);
1942
- if (event === void 0) continue;
1943
- hits.add(handlerAt.get(node).get(event));
1944
- break;
1945
- }
1946
- for (const handler of hits) {
1947
- credited.add(handler);
1948
- let leaves = seen.get(handler);
1949
- if (!leaves) seen.set(handler, leaves = /* @__PURE__ */ new Set());
1950
- for (const leaf of step.to) leaves.add(leaf);
1951
- }
1952
- }
1953
- for (const state of visited) {
1954
- const row = stateRuns.get(state);
1955
- if (row) row.runs++;
1956
- }
1957
- for (const handler of credited) handler.runs++;
1958
- }
1959
- for (const handler of handlers) {
1960
- const leaves = [...seen.get(handler) ?? []];
1961
- handler.targets.seen = leaves;
1962
- const { declared } = handler.targets;
1963
- if (declared === "dynamic") continue;
1964
- handler.targets.unreached = declared.filter((target) => !leaves.some((leaf) => leaf === target || leaf.startsWith(`${target}.`)));
1965
- }
1966
- const reachableComplete = options.reachable ? isExplorationComplete(options.reachable) : void 0;
1967
- let unreachable;
1968
- if (options.reachable && reachableComplete) {
1969
- const reached = new Set(options.reachable.reachedStates.flatMap((value) => statePaths(value)));
1970
- unreachable = states.filter((row) => !reached.has(row.state) && row.runs === 0).map((row) => row.state);
1971
- }
1972
- const counted = unreachable ? states.filter((row) => !unreachable.includes(row.state)) : states;
1973
- const unvisitedStates = counted.filter((row) => row.runs === 0).map((row) => row.state);
1974
- const unvisitedHandlers = handlers.filter((handler) => handler.runs === 0).map(({ state, event }) => ({
1975
- state,
1976
- event
1977
- }));
1978
- return {
1979
- states,
1980
- handlers,
1981
- unvisitedStates,
1982
- unvisitedHandlers,
1983
- unobservable,
1984
- ...unreachable ? { unreachable } : {},
1985
- ...reachableComplete === void 0 ? {} : { reachableComplete },
1986
- ratio: {
1987
- states: ratioOf(counted.length - unvisitedStates.length, counted.length),
1988
- handlers: ratioOf(handlers.length - unvisitedHandlers.length, handlers.length)
1989
- }
1990
- };
1991
- }
1992
- //#endregion
1993
- //#region src/first-divergence.ts
1994
- const defaultKey = (step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`;
1995
- const isErrorLike = (value) => isError(value) || isRecord(value) && typeof value.name === "string" && typeof value.message === "string";
1996
- const normalizeErrors = (value, seen = /* @__PURE__ */ new WeakSet()) => {
1997
- if (value === null || typeof value !== "object") return value;
1998
- if (seen.has(value)) return "[circular]";
1999
- seen.add(value);
2000
- try {
2001
- if (isErrorLike(value) && (isError(value) || "stack" in value || "code" in value)) {
2002
- const { name, message, code, cause } = value;
2003
- return {
2004
- name,
2005
- message,
2006
- ...typeof code === "string" ? { code } : {},
2007
- ...cause !== void 0 ? { cause: normalizeErrors(cause, seen) } : {}
2008
- };
2009
- }
2010
- if (Array.isArray(value)) return value.map((item) => normalizeErrors(item, seen));
2011
- if (!isPlainObject(value)) return value;
2012
- const copy = Object.create(null);
2013
- for (const key of Object.keys(value)) copy[key] = normalizeErrors(value[key], seen);
2014
- return copy;
2015
- } finally {
2016
- seen.delete(value);
2017
- }
2018
- };
2019
- const payloadKey = (step) => {
2020
- if (!isRecord(step.payload)) return `${defaultKey(step)}|${canonicalEventJson(step.payload)}`;
2021
- const { sessionId: _sessionId, ...payload } = step.payload;
2022
- return `${defaultKey(step)}|${canonicalEventJson(normalizeErrors(payload))}`;
2023
- };
2024
- const isTrace = (items) => items.length === 0 || items.some((item) => isRecord(item) && (item.type === "machine.transition" || item.type === "run.start" || item.type === "run.end"));
2025
- /**
2026
- * Finds the first step where a run departs from a reference, so a failure is
2027
- * blamed on the state it started in rather than on every state the run went
2028
- * through afterwards.
2029
- *
2030
- * - Against a reference **trace**, such as a passing run of the same
2031
- * scenario, steps are compared in order by `key`. The first step that
2032
- * differs, or the point where either trace ends early, is the divergence.
2033
- * - Against an expected **trajectory** (`TrajectoryItem[]`, as for
2034
- * {@link matchesTrajectory}), the run is laid out as its initial state, then
2035
- * each step's event and new state. The first expected item it never reaches
2036
- * is the divergence, blamed on the step after the last item it did reach. A
2037
- * run that goes on past the trajectory has not diverged.
2038
- *
2039
- * An empty reference is a trace with no steps, so any step diverges from it.
2040
- * A trace is one run's events, live or serialized. Steps the machine ignored
2041
- * changed nothing and are left out on both sides. Returns `undefined` when the
2042
- * run does not diverge.
2043
- *
2044
- * @example
2045
- * ```ts no-check
2046
- * const stats = foldTraces(traces, {
2047
- * failed,
2048
- * blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
2049
- * });
2050
- * ```
2051
- */
2052
- function firstDivergence(actual, reference, options = {}) {
2053
- const steps = traceSteps(actual).filter((step) => !step.ignored);
2054
- return isTrace(reference) ? againstTrace(steps, traceSteps(reference).filter((step) => !step.ignored), options.key === "payload" ? payloadKey : options.key ?? defaultKey) : againstTrajectory(actual, steps, reference);
2055
- }
2056
- function diverged(steps, index, expected, initial = steps[0]?.fromValue) {
2057
- const step = steps[index];
2058
- return {
2059
- index,
2060
- state: (step ? step.from[0] : steps.at(-1)?.to[0] ?? statePaths(initial)[0]) ?? "",
2061
- ...step?.decision === void 0 ? {} : { decision: step.decision },
2062
- ...step ? { actual: step } : {},
2063
- ...expected === void 0 ? {} : { expected }
2064
- };
2065
- }
2066
- function againstTrace(steps, reference, key) {
2067
- const length = Math.max(steps.length, reference.length);
2068
- for (let index = 0; index < length; index++) {
2069
- const step = steps[index];
2070
- const want = reference[index];
2071
- if (step && want && key(step) === key(want)) continue;
2072
- return diverged(steps, index, want, steps[0]?.fromValue ?? reference[0]?.fromValue);
2073
- }
2074
- }
2075
- function againstTrajectory(trace, steps, expected) {
2076
- const initial = initialValue(trace, steps);
2077
- const timeline = initial === void 0 ? [] : [{ value: initial }];
2078
- const stepAt = initial === void 0 ? [] : [0];
2079
- steps.forEach((step, index) => {
2080
- timeline.push(step.payload);
2081
- stepAt.push(index);
2082
- if (JSON.stringify(step.toValue) !== JSON.stringify(step.fromValue)) {
2083
- timeline.push({ value: step.toValue });
2084
- stepAt.push(index);
2085
- }
2086
- });
2087
- const match = matchesTrajectory(timeline, expected);
2088
- if (!match.firstMiss) return void 0;
2089
- const at = match.firstMiss.searchedFrom;
2090
- return diverged(steps, at < timeline.length ? stepAt[at] ?? steps.length : steps.length, match.firstMiss.expected, initial);
2091
- }
2092
- function initialValue(trace, steps) {
2093
- for (const event of trace) if (event.type === "machine.transition" && isRecord(event.snapshot)) return event.snapshot.value;
2094
- return steps[0]?.fromValue;
2095
- }
2096
- //#endregion
2097
- //#region src/propose-from-traces.ts
2098
- /**
2099
- * Groups the approval verdicts in many runs' traces by `key` and proposes an
2100
- * `allow` or `deny` rule for each. Pure and deterministic; a proposal is for
2101
- * a person to review, and nothing here changes a machine.
2102
- *
2103
- * Reads `machine.transition` events: a verdict is a transition, not ignored,
2104
- * out of `approvalState` on an approve or reject event. Rows come in the
2105
- * order keys were first seen.
2106
- *
2107
- * @example
2108
- * ```ts no-check
2109
- * const rows = proposeApprovalRules(traces, {
2110
- * approvalState: "awaitingApproval",
2111
- * approveEvents: ["APPROVE"],
2112
- * rejectEvents: ["REJECT"],
2113
- * key: ({ snapshot }) => `${snapshot.context.pending.tool}:${snapshot.context.pending.path}`,
2114
- * });
2115
- * ```
2116
- */
2117
- function proposeApprovalRules(traces, options) {
2118
- const rows = /* @__PURE__ */ new Map();
2119
- const rowFor = (key) => {
2120
- let row = rows.get(key);
2121
- if (!row) {
2122
- row = {
2123
- key,
2124
- approved: 0,
2125
- rejected: 0,
2126
- violations: 0,
2127
- runs: [],
2128
- proposal: null,
2129
- support: 0
2130
- };
2131
- rows.set(key, row);
2132
- }
2133
- return row;
2134
- };
2135
- const inState = (value) => statePaths(value).some((path) => path === options.approvalState || path.startsWith(`${options.approvalState}.`));
2136
- let index = 0;
2137
- for (const trace of traces) {
2138
- const run = index++;
2139
- const seen = /* @__PURE__ */ new Set();
2140
- let snapshot;
2141
- for (const event of trace) {
2142
- if (event.type !== "machine.transition") continue;
2143
- const before = snapshot;
2144
- snapshot = event.snapshot;
2145
- const type = isRecord(event.event) ? event.event.type : void 0;
2146
- if (event.ignored === true || typeof type !== "string" || !inState(valueOf(event.from))) continue;
2147
- const approve = options.approveEvents.includes(type);
2148
- if (!approve && !options.rejectEvents.includes(type)) continue;
2149
- const key = options.key({
2150
- event: event.event,
2151
- snapshot: before,
2152
- trace
2153
- });
2154
- if (key === void 0) continue;
2155
- const row = rowFor(key);
2156
- if (approve) row.approved++;
2157
- else row.rejected++;
2158
- if (!seen.has(key)) {
2159
- seen.add(key);
2160
- row.runs.push(run);
2161
- }
2162
- }
2163
- for (const key of options.violations?.(trace) ?? []) {
2164
- const row = rows.get(key);
2165
- if (row) row.violations++;
2166
- }
2167
- }
2168
- for (const row of rows.values()) {
2169
- row.support = row.runs.length;
2170
- row.proposal = row.approved > 0 && row.rejected === 0 && row.violations === 0 ? "allow" : row.rejected > 0 && row.approved === 0 ? "deny" : null;
2171
- }
2172
- return [...rows.values()];
2173
- }
2174
- /**
2175
- * Proposes a value for each cap (retry count, steps, batch size) from many
2176
- * runs' traces: the 95th percentile of what passing runs used, plus one, and
2177
- * never below the current cap while runs still hit it. Pure and
2178
- * deterministic; a proposal is for a person to review, and nothing here
2179
- * changes a machine.
2180
- *
2181
- * @example
2182
- * ```ts no-check
2183
- * const [retries] = proposeCaps(traces, {
2184
- * caps: [{ name: "retries", current: 3, failureState: "gaveUp", valueFromRun: countRetries }],
2185
- * });
2186
- * ```
2187
- */
2188
- function proposeCaps(traces, options) {
2189
- const runs = [...traces];
2190
- return options.caps.map((cap) => {
2191
- let hits = 0;
2192
- let max = null;
2193
- const passing = [];
2194
- for (const trace of runs) {
2195
- const hit = hitCap(trace, cap);
2196
- if (hit) hits++;
2197
- const value = cap.valueFromRun(trace);
2198
- if (value === void 0) continue;
2199
- max = max === null ? value : Math.max(max, value);
2200
- if (!hit) passing.push(value);
2201
- }
2202
- passing.sort((a, b) => a - b);
2203
- const p95 = passing.length ? passing[Math.ceil(.95 * passing.length) - 1] : null;
2204
- const proposal = p95 === null ? null : hits > 0 && cap.current !== void 0 ? Math.max(p95 + 1, cap.current) : p95 + 1;
2205
- return {
2206
- name: cap.name,
2207
- hits,
2208
- passing: passing.length,
2209
- max,
2210
- p95,
2211
- proposal
2212
- };
2213
- });
2214
- }
2215
- function hitCap(trace, cap) {
2216
- for (const event of trace) {
2217
- if (cap.failureState !== void 0 && event.type === "machine.transition" && statePaths(valueOf(event.snapshot)).some((path) => path === cap.failureState || path.startsWith(`${cap.failureState}.`))) return true;
2218
- if (cap.failureOutput && event.type === "run.end" && event.status === "done") {
2219
- if (cap.failureOutput(event.output)) return true;
2220
- }
2221
- }
2222
- return false;
2223
- }
2224
- //#endregion
2225
- //#region src/decision-requests.ts
2226
- /**
2227
- * The decisions a run asked for, read from its trace: each decision's first
2228
- * `request.start`, with the states the run was in. A trace is one run's events,
2229
- * live or serialized with `serializeTraceEvent`. Retries of the same decision
2230
- * are left out, so each decision appears once.
2231
- *
2232
- * Use the requests as eval rows (run a decider on each and compare its event
2233
- * with a label) or as probe inputs for a policy. A serialized request keeps its
2234
- * `options`, `metadata`, `system`, and `prompt`; event schemas do not survive
2235
- * serialization, so replay closed choices (`options`) rather than open ones.
2236
- *
2237
- * @example
2238
- * ```ts no-check
2239
- * const rows = traces.flatMap((trace) => getDecisionRequests(trace, { name: "chooseResolution" }));
2240
- * for (const { request } of rows) console.log((await decide(request)).event);
2241
- * ```
2242
- */
2243
- function getDecisionRequests(trace, options = {}) {
2244
- const requests = [];
2245
- let current = [];
2246
- for (const event of trace) if (event.type === "machine.transition") current = statePaths(valueOf(event.snapshot));
2247
- else if (event.type === "request.start" && isFirstDecision(event.request)) {
2248
- if (options.name !== void 0 && event.request.name !== options.name) continue;
2249
- requests.push({
2250
- states: [...current],
2251
- request: event.request
2252
- });
2253
- }
2254
- return requests;
2255
- }
2256
- //#endregion
2257
- //#region src/repeat-runs.ts
2258
- /**
2259
- * Summarizes repeated runs of each scenario: how often each passed, whether
2260
- * all its runs passed (pass^k), how many distinct state paths they took, and
2261
- * where they first went different ways. Pure. Use it for runs collected
2262
- * elsewhere, such as CI shards or replays; {@link repeatRuns} collects them and
2263
- * calls it.
2264
- *
2265
- * A scenario that passes 3 of 5 runs and `divergedAt: 'deciding'` points at one
2266
- * decision, not at a flaky agent.
2267
- *
2268
- * @example
2269
- * ```ts no-check
2270
- * const report = summarizeRepeats(shards.map((s) => ({ id: s.id, scenario: s, runs: s.runs })));
2271
- * report.passHat(3); // the chance 3 runs of a scenario all pass
2272
- * ```
2273
- */
2274
- function summarizeRepeats(groups) {
2275
- const scenarios = groups.map((group) => summarize$1(group));
2276
- const traces = groups.flatMap((group) => group.runs.flatMap((run) => run.trace ? [run.trace] : []));
2277
- const most = Math.max(0, ...groups.map((group) => group.runs.length));
2278
- const passHat = (j) => {
2279
- if (!Number.isInteger(j) || j < 1 || j > most) throw new AgentError("pass-hat-out-of-range", `passHat: j must be a whole number from 1 to ${most}, got ${j}.`);
2280
- const eligible = scenarios.filter((scenario) => scenario.runs.length >= j);
2281
- return eligible.reduce((total, scenario) => total + allPass(scenario.passes, scenario.runs.length, j), 0) / eligible.length;
2282
- };
2283
- return {
2284
- scenarios,
2285
- passHat,
2286
- traces
2287
- };
2288
- }
2289
- /**
2290
- * Runs every scenario `k` times and summarizes the runs with
2291
- * {@link summarizeRepeats}. Runs are sequential, scenario by scenario. A run
2292
- * that throws fails, with its message as `error` and no trace.
2293
- *
2294
- * With a live model, one run per scenario says little: τ-bench found pass^k
2295
- * falls well below the single-run rate. Report pass^k next to it.
2296
- *
2297
- * @example
2298
- * ```ts no-check
2299
- * const report = await repeatRuns({
2300
- * scenarios,
2301
- * k: 5,
2302
- * run: async (scenario) => {
2303
- * const trace: AgentTraceEvent[] = [];
2304
- * const result = await runToQuiescence(
2305
- * createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
2306
- * { input: scenario.input },
2307
- * );
2308
- * return { trace, outcome: String(result.snapshot.value) };
2309
- * },
2310
- * passed: (scenario, { outcome }) => outcome === scenario.expected,
2311
- * });
2312
- * report.passHat(5);
2313
- * ```
2314
- */
2315
- async function repeatRuns(options) {
2316
- if (!Number.isInteger(options.k) || options.k < 1) throw new AgentError("invalid-repeat-count", `repeatRuns: k must be a whole number of 1 or more, got ${options.k}.`);
2317
- const groups = [];
2318
- for (const scenario of options.scenarios) {
2319
- const runs = [];
2320
- for (let attempt = 0; attempt < options.k; attempt++) try {
2321
- const result = await options.run(scenario, attempt);
2322
- runs.push({
2323
- passed: options.passed(scenario, result),
2324
- trace: result.trace,
2325
- ...result.outcome === void 0 ? {} : { outcome: result.outcome }
2326
- });
2327
- } catch (error) {
2328
- runs.push({
2329
- passed: false,
2330
- error: error instanceof Error ? error.message : String(error)
2331
- });
2332
- }
2333
- groups.push({
2334
- id: scenario.id,
2335
- scenario,
2336
- runs
2337
- });
2338
- }
2339
- return summarizeRepeats(groups);
2340
- }
2341
- function summarize$1(group) {
2342
- const passes = group.runs.filter((run) => run.passed).length;
2343
- const traces = group.runs.flatMap((run) => run.trace ? [run.trace] : []);
2344
- const paths = new Set(traces.map((trace) => traceSteps(trace).filter((step) => !step.ignored).map((step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`).join(" ")));
2345
- const reference = traces.find((trace) => trace.some((event) => event.type === "machine.transition"));
2346
- let earliest;
2347
- for (const trace of reference ? traces : []) {
2348
- if (trace === reference) continue;
2349
- const divergence = firstDivergence(trace, reference);
2350
- if (divergence && (!earliest || divergence.index < earliest.index)) earliest = divergence;
2351
- }
2352
- return {
2353
- ...group,
2354
- passes,
2355
- passAny: passes > 0,
2356
- passAll: group.runs.length > 0 && passes === group.runs.length,
2357
- distinctPaths: paths.size,
2358
- ...earliest ? { divergedAt: earliest.state } : {}
2359
- };
2360
- }
2361
- /** C(passes, j) / C(runs, j), as a product so large counts don't overflow. */
2362
- function allPass(passes, runs, j) {
2363
- let chance = 1;
2364
- for (let i = 0; i < j; i++) chance *= Math.max(0, passes - i) / (runs - i);
2365
- return chance;
2366
- }
2367
- //#endregion
2368
- //#region src/judge-agreement.ts
2369
- /**
2370
- * Measures how well a judge agrees with people on the same cases: accuracy,
2371
- * per-label recall, the confusion matrix, and the rows that disagree. With
2372
- * `positive`, also the TPR and TNR, the recommended view for binary judges
2373
- * (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
2374
- * Pure.
2375
- *
2376
- * Build rows from confirmed labels only; draft labels gate nothing. Group by
2377
- * decision state or request name so each narrow judge is aligned on its own.
2378
- * A row with no group key counts in the totals and in no group. Run it on a
2379
- * dev split and report the held-out split separately.
2380
- *
2381
- * Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
2382
- *
2383
- * @example
2384
- * ```ts no-check
2385
- * const report = judgeAgreement(
2386
- * labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
2387
- * { positive: 'fail' },
2388
- * );
2389
- * report.binary; // { tpr: 0.9, tnr: 0.8 }
2390
- * report.correct?.(0.3); // estimated true failure rate in production
2391
- * ```
2392
- */
2393
- function judgeAgreement(rows, options = {}) {
2394
- if (rows.length === 0) throw new AgentError("empty-agreement-rows", "judgeAgreement: rows must contain at least one row.");
2395
- const report = summarize(rows, options.positive);
2396
- const groupOf = options.groupBy ?? ((row) => row.group);
2397
- if (options.groupBy || rows.some((row) => row.group !== void 0)) {
2398
- const grouped = /* @__PURE__ */ new Map();
2399
- for (const row of rows) {
2400
- const key = groupOf(row);
2401
- if (key === void 0) continue;
2402
- const members = grouped.get(key);
2403
- if (members) members.push(row);
2404
- else grouped.set(key, [row]);
2405
- }
2406
- const groups = Object.create(null);
2407
- for (const [key, members] of grouped) groups[key] = summarize(members, options.positive);
2408
- report.groups = groups;
2409
- }
2410
- return report;
2411
- }
2412
- function summarize(rows, positive) {
2413
- const perLabel = Object.create(null);
2414
- const confusion = Object.create(null);
2415
- const disagreements = [];
2416
- let agreed = 0;
2417
- for (const { id, human, judge } of rows) {
2418
- const label = perLabel[human] ??= {
2419
- human: 0,
2420
- agreed: 0
2421
- };
2422
- const row = confusion[human] ??= Object.create(null);
2423
- label.human++;
2424
- row[judge] = (row[judge] ?? 0) + 1;
2425
- if (human === judge) {
2426
- label.agreed++;
2427
- agreed++;
2428
- } else disagreements.push({
2429
- id,
2430
- human,
2431
- judge
2432
- });
2433
- }
2434
- const summary = {
2435
- total: rows.length,
2436
- accuracy: agreed / rows.length,
2437
- perLabel,
2438
- confusion,
2439
- disagreements
2440
- };
2441
- if (positive === void 0) return summary;
2442
- let positives = 0;
2443
- let truePositives = 0;
2444
- let negatives = 0;
2445
- let trueNegatives = 0;
2446
- for (const { human, judge } of rows) if (human === positive) {
2447
- positives++;
2448
- if (judge === positive) truePositives++;
2449
- } else {
2450
- negatives++;
2451
- if (judge !== positive) trueNegatives++;
2452
- }
2453
- const tpr = positives === 0 ? void 0 : truePositives / positives;
2454
- const tnr = negatives === 0 ? void 0 : trueNegatives / negatives;
2455
- summary.binary = {
2456
- tpr,
2457
- tnr
2458
- };
2459
- if (tpr !== void 0 && tnr !== void 0 && tpr + tnr > 1) summary.correct = (p) => Math.min(1, Math.max(0, (p + tnr - 1) / (tpr + tnr - 1)));
2460
- return summary;
2461
- }
2462
- //#endregion
2463
1592
  //#region src/compare-runs.ts
2464
1593
  /**
2465
1594
  * Runs every scenario through a baseline and a candidate implementation and
@@ -2572,8 +1701,8 @@ function structurallyEqual(a, b) {
2572
1701
  * @module
2573
1702
  */
2574
1703
  /** Each function effect wrapped with `recorder.fn`, by its key. */
2575
- function recordEffects(recorder, effects) {
2576
- return mapEffects(effects, (name, effect) => recorder.fn(name, effect));
1704
+ function recordEffects(recorder, effects, recordOptions) {
1705
+ return mapEffects(effects, (name, effect) => recorder.fn(name, effect, recordOptions?.[name]));
2577
1706
  }
2578
1707
  async function settle(run, recorder, normalize) {
2579
1708
  try {
@@ -2621,14 +1750,20 @@ function errorIdentity(outcome, normalize) {
2621
1750
  async function checkParity(options) {
2622
1751
  const rows = [];
2623
1752
  for (const scenario of options.scenarios) {
1753
+ const setup = (side) => options.setup ? options.setup(scenario, side) : void 0;
2624
1754
  const recorder = createRecorder();
2625
- const effects = recordEffects(recorder, options.effects(scenario));
1755
+ const live = await setup("before");
1756
+ const effects = recordEffects(recorder, options.effects(scenario, live), options.recordOptions);
2626
1757
  const normalize = (output) => options.normalizeOutput ? options.normalizeOutput(output, scenario) : output;
2627
- const before = await settle(() => options.before(scenario.input, effects), recorder, normalize);
1758
+ const before = await settle(() => options.before(scenario.input, effects, live), recorder, normalize);
2628
1759
  const recording = recorder.recording;
2629
- const replayer = createRecorder({ replay: recording });
2630
- const stand = recordEffects(replayer, options.effects(scenario));
2631
- const after = await settle(() => options.after(scenario.input, stand), replayer, normalize);
1760
+ const replayer = createRecorder({
1761
+ replay: recording,
1762
+ reviveError: options.reviveError
1763
+ });
1764
+ const liveAfter = await setup("after");
1765
+ const stand = recordEffects(replayer, options.effects(scenario, liveAfter), options.recordOptions);
1766
+ const after = await settle(() => options.after(scenario.input, stand, liveAfter), replayer, normalize);
2632
1767
  const replay = replayer.recording;
2633
1768
  const { divergences } = compareRecordings(recording, replay, options.compare);
2634
1769
  const normalizeError = options.normalizeError && ((error) => options.normalizeError(error, scenario));
@@ -2668,4 +1803,4 @@ async function checkParity(options) {
2668
1803
  };
2669
1804
  }
2670
1805
  //#endregion
2671
- export { AgentLintError, AgentUnknownStateError, RecordingDivergenceError, assertAgentMachine, canReach, checkCrashRecovery, checkParity, compareRecordings, compareRuns, createRecorder, explorePaths, firstDivergence, foldTraces, getDecisionRequests, judgeAgreement, lintAgentMachine, matchesTrajectory, proposeApprovalRules, proposeCaps, repeatRuns, runSeam, sameModelCallShape, simulateAgent, summarizeRepeats, textOutput, traceCoverage };
1806
+ export { AgentLintError, AgentUnknownStateError, RecordingDivergenceError, assertAgentMachine, canReach, checkCrashRecovery, checkParity, compareRecordings, compareRuns, createRecorder, explorePaths, lintAgentMachine, matchesTrajectory, runSeam, sameModelCallShape, simulateAgent, textOutput };