@statelyai/agent 2.0.0-alpha.38 → 2.0.0-alpha.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/CHANGELOG.md +59 -0
  2. package/dist/{act-CrX0JwWY.cjs → act-B0CVbmo4.cjs} +1 -1
  3. package/dist/{act-B9hLQls6.mjs → act-Bsz-hUfD.mjs} +1 -1
  4. package/dist/ai-sdk.cjs +4 -4
  5. package/dist/ai-sdk.d.cts +2 -3
  6. package/dist/ai-sdk.d.mts +2 -3
  7. package/dist/ai-sdk.mjs +4 -4
  8. package/dist/{decision-m5hGmHpF.mjs → decision-B41YBx5U.mjs} +1 -1
  9. package/dist/{decision-CMeuNZ1g.cjs → decision-D8bcIGXQ.cjs} +1 -1
  10. package/dist/{decision-object-DBcLWVvv.d.cts → decision-object-BB8XV5Ec.d.cts} +1 -1
  11. package/dist/{decision-object-B5zGSPFp.d.mts → decision-object-CH_sE694.d.mts} +1 -1
  12. package/dist/effects-B9IvUPVh.mjs +23 -0
  13. package/dist/effects-DrWcUGgc.cjs +34 -0
  14. package/dist/evals.cjs +810 -0
  15. package/dist/evals.d.cts +595 -0
  16. package/dist/evals.d.mts +595 -0
  17. package/dist/evals.mjs +801 -0
  18. package/dist/{event-log-DkVlk3Rh.mjs → event-log-CEq_bVXQ.mjs} +8 -4
  19. package/dist/{event-log-Dkc-yIYu.cjs → event-log-dSOnXHuG.cjs} +7 -15
  20. package/dist/{event-log-store-Brx9tOHt.mjs → event-log-store-DLAfNM34.mjs} +1 -1
  21. package/dist/{event-log-store-BBS8so8a.d.mts → event-log-store-DYun526m.d.mts} +15 -14
  22. package/dist/{event-log-store-Cl-YWDnq.cjs → event-log-store-Dey9ufA4.cjs} +1 -1
  23. package/dist/{event-log-store-bc4ICGcS.d.cts → event-log-store-aFXK6yZX.d.cts} +15 -14
  24. package/dist/index.cjs +142 -97
  25. package/dist/index.d.cts +97 -141
  26. package/dist/index.d.mts +97 -141
  27. package/dist/index.mjs +123 -74
  28. package/dist/internal-BzlpzSux.d.cts +14 -0
  29. package/dist/internal-BzlpzSux.d.mts +14 -0
  30. package/dist/log.cjs +2 -3
  31. package/dist/log.d.cts +2 -2
  32. package/dist/log.d.mts +2 -2
  33. package/dist/log.mjs +3 -3
  34. package/dist/machine-input-0YTPcnCm.mjs +27 -0
  35. package/dist/machine-input-DU3owLUr.cjs +32 -0
  36. package/dist/machines.cjs +1 -719
  37. package/dist/machines.d.cts +2 -424
  38. package/dist/machines.d.mts +2 -424
  39. package/dist/machines.mjs +2 -709
  40. package/dist/{model-source-kjGwqAV8.mjs → model-source-BrsLDvHL.mjs} +1 -1
  41. package/dist/{model-source-B4WQ8Idp.cjs → model-source-YpxjmWuX.cjs} +1 -1
  42. package/dist/openai.cjs +3 -3
  43. package/dist/openai.d.cts +2 -3
  44. package/dist/openai.d.mts +2 -3
  45. package/dist/openai.mjs +3 -3
  46. package/dist/otel.d.cts +1 -1
  47. package/dist/otel.d.mts +1 -1
  48. package/dist/policy.cjs +77 -0
  49. package/dist/policy.d.cts +143 -0
  50. package/dist/policy.d.mts +143 -0
  51. package/dist/policy.mjs +75 -0
  52. package/dist/{run-agent-DlPzBWj-.d.mts → run-agent-BO9RSb7i.d.cts} +33 -40
  53. package/dist/{run-agent-30mH2moE.d.cts → run-agent-BxkeuAYR.d.mts} +33 -40
  54. package/dist/{effects-DDseYtJk.cjs → run-agent-COdR9FpM.cjs} +175 -228
  55. package/dist/{effects-BT_cSY0j.mjs → run-agent-D3gW9wfd.mjs} +176 -211
  56. package/dist/{setup-agent-BDmNr6PC.mjs → setup-agent-CLm6QVGy.mjs} +3 -3
  57. package/dist/{setup-agent-BaNyz6Q3.d.cts → setup-agent-Cv8Hk3mX.d.cts} +12 -11
  58. package/dist/{setup-agent-C1CG_nfX.d.mts → setup-agent-DgFAhaPJ.d.mts} +12 -11
  59. package/dist/{setup-agent-CmEToB8y.cjs → setup-agent-Dv5VpuHe.cjs} +3 -3
  60. package/dist/tanstack-ai.cjs +4 -4
  61. package/dist/tanstack-ai.d.cts +2 -3
  62. package/dist/tanstack-ai.d.mts +2 -3
  63. package/dist/tanstack-ai.mjs +4 -4
  64. package/dist/testing.cjs +122 -996
  65. package/dist/testing.d.cts +36 -695
  66. package/dist/testing.d.mts +36 -695
  67. package/dist/testing.mjs +112 -977
  68. package/dist/{text-logic-BqxalgMr.d.mts → text-logic-x17uLBwA.d.cts} +238 -2
  69. package/dist/{text-logic-CSf9bqpP.d.cts → text-logic-x17uLBwA.d.mts} +238 -2
  70. package/dist/trajectory-BFjXaooB.d.cts +96 -0
  71. package/dist/trajectory-BFjXaooB.d.mts +96 -0
  72. package/dist/trajectory-BUXj_1Ue.mjs +152 -0
  73. package/dist/trajectory-CoZSv4wF.cjs +157 -0
  74. package/dist/{usage-Bm5nvpow.d.cts → usage-B4B-TQSV.d.mts} +1 -1
  75. package/dist/{usage-BnikdOca.d.mts → usage-DvGcoK6x.d.cts} +1 -1
  76. package/dist/{utils-7lSa4H-i.d.cts → utils-B8k-YPYI.d.mts} +23 -17
  77. package/dist/{utils-CBbKw2fM.d.mts → utils-DAclF-kw.d.cts} +23 -17
  78. package/dist/validate.cjs +4 -4
  79. package/dist/validate.d.cts +2 -2
  80. package/dist/validate.d.mts +2 -2
  81. package/dist/validate.mjs +4 -4
  82. package/dist/{verify-DJHfEfm6.d.cts → verify-CIu4mqIZ.d.mts} +56 -6
  83. package/dist/{verify-i9Hiwc5g.d.mts → verify-CTHlmBbh.d.cts} +56 -6
  84. package/dist/{verify-ChyX6jq0.mjs → verify-Cacz6XSk.mjs} +170 -46
  85. package/dist/{verify-CwlombbC.cjs → verify-D8UDqU4l.cjs} +170 -46
  86. package/dist/{with-model-calls-B_WLmOSl.cjs → with-model-calls-D50Ca6qW.cjs} +2 -2
  87. package/dist/{with-model-calls-BkOwdroS.mjs → with-model-calls-uY-KKliA.mjs} +2 -2
  88. package/docs/advanced.md +5 -5
  89. package/docs/choosing-a-run-mode.md +7 -7
  90. package/docs/debugging.md +6 -8
  91. package/docs/decisions.md +5 -4
  92. package/docs/environment-and-policy.md +1 -1
  93. package/docs/evals.md +57 -43
  94. package/docs/event-log.md +13 -15
  95. package/docs/from-a-loop.md +3 -3
  96. package/docs/generate-machines.md +4 -4
  97. package/docs/hosts.md +25 -30
  98. package/docs/human-in-the-loop.md +5 -8
  99. package/docs/index.md +4 -4
  100. package/docs/judgments.md +5 -7
  101. package/docs/langgraph-comparison.md +6 -6
  102. package/docs/machines-as-data.md +7 -12
  103. package/docs/machines-presets.md +17 -120
  104. package/docs/machines.md +13 -13
  105. package/docs/models-and-providers.md +19 -25
  106. package/docs/multi-agent.md +1 -1
  107. package/docs/observability.md +6 -8
  108. package/docs/persistence.md +10 -20
  109. package/docs/quickstart.md +3 -3
  110. package/docs/statechart-policy-examples.md +1 -1
  111. package/docs/steps.md +5 -6
  112. package/docs/strangler-fig.md +105 -33
  113. package/docs/text-requests.md +10 -12
  114. package/docs/thinking-in-state-machines.md +8 -8
  115. package/docs/tools.md +10 -12
  116. package/docs/upgrading.md +1 -1
  117. package/docs/usage-and-budgets.md +51 -56
  118. package/docs/validate-and-repair.md +9 -14
  119. package/docs/verify.md +54 -7
  120. package/docs/xstate-4-and-5-apps.md +19 -28
  121. package/llms.txt +1 -1
  122. package/package.json +31 -6
  123. package/readme.md +32 -24
  124. package/dist/internal-C3Lqu538.d.cts +0 -43
  125. package/dist/internal-DUcdhk-i.d.mts +0 -43
  126. package/dist/types-DQUOOaZO.d.cts +0 -238
  127. package/dist/types-DQUOOaZO.d.mts +0 -238
  128. package/skills/add-evals/SKILL.md +0 -152
  129. package/skills/add-traces/SKILL.md +0 -152
  130. package/skills/build-machine/SKILL.md +0 -84
  131. package/skills/generate-machine/SKILL.md +0 -335
  132. package/skills/graph-engineering/SKILL.md +0 -43
  133. package/skills/improve-machine/SKILL.md +0 -261
  134. package/skills/improve-machine/reference.md +0 -101
  135. package/skills/loop-to-machine/SKILL.md +0 -205
  136. package/skills/loop-to-machine/policy.md +0 -154
  137. package/skills/loop-to-machine/reference.md +0 -260
package/dist/testing.cjs CHANGED
@@ -1,11 +1,12 @@
1
1
  Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
2
- const require_event_log = require("./event-log-Dkc-yIYu.cjs");
3
- const require_decision = require("./decision-CMeuNZ1g.cjs");
4
- const require_act = require("./act-CrX0JwWY.cjs");
5
- const require_effects = require("./effects-DDseYtJk.cjs");
6
- const require_event_log_store = require("./event-log-store-Cl-YWDnq.cjs");
7
- const require_verify = require("./verify-CwlombbC.cjs");
8
- let xstate = require("xstate");
2
+ const require_event_log = require("./event-log-dSOnXHuG.cjs");
3
+ const require_decision = require("./decision-D8bcIGXQ.cjs");
4
+ const require_act = require("./act-B0CVbmo4.cjs");
5
+ const require_event_log_store = require("./event-log-store-Dey9ufA4.cjs");
6
+ const require_run_agent = require("./run-agent-COdR9FpM.cjs");
7
+ const require_effects = require("./effects-DrWcUGgc.cjs");
8
+ const require_verify = require("./verify-D8UDqU4l.cjs");
9
+ const require_trajectory = require("./trajectory-CoZSv4wF.cjs");
9
10
  //#region src/equivalence.ts
10
11
  /**
11
12
  * Verified equivalence — prove a port behaves like the code it replaces.
@@ -47,11 +48,12 @@ let xstate = require("xstate");
47
48
  var RecordingDivergenceError = class extends Error {
48
49
  name = "RecordingDivergenceError";
49
50
  };
50
- function toRecordedError(error) {
51
+ function toRecordedError(error, path = /* @__PURE__ */ new Set()) {
51
52
  if (!(error instanceof Error)) return {
52
53
  name: "Error",
53
54
  message: String(error)
54
55
  };
56
+ path.add(error);
55
57
  const recorded = {
56
58
  name: error.name,
57
59
  message: error.message
@@ -67,10 +69,39 @@ function toRecordedError(error) {
67
69
  if (key !== void 0 && error[symbol] === true) markers.add(key);
68
70
  }
69
71
  if (markers.size) recorded.markers = [...markers];
72
+ if (error.cause instanceof Error && !path.has(error.cause)) recorded.cause = toRecordedError(error.cause, path);
73
+ if (error instanceof AggregateError && Array.isArray(error.errors)) recorded.errors = error.errors.flatMap((item) => {
74
+ if (!(item instanceof Error)) try {
75
+ return [{ value: toJson(item) }];
76
+ } catch {
77
+ try {
78
+ return [{ value: String(item) }];
79
+ } catch {
80
+ return [{ value: "[unserializable]" }];
81
+ }
82
+ }
83
+ return path.has(item) ? [] : [toRecordedError(item, path)];
84
+ });
85
+ path.delete(error);
70
86
  return recorded;
71
87
  }
72
- function fromRecordedError({ data, markers, name, ownName, ...fields }) {
73
- const error = Object.assign(new Error(fields.message), fields, data);
88
+ function fromRecordedError(recorded, reviveError) {
89
+ const { data, markers, name, ownName, cause, errors, ...fields } = recorded;
90
+ const revivedErrors = errors?.map((item) => "name" in item ? fromRecordedError(item, reviveError) : structuredClone(item.value));
91
+ const define = (target, key, value) => Object.defineProperty(target, key, {
92
+ value,
93
+ writable: true,
94
+ configurable: true
95
+ });
96
+ const custom = reviveError?.(structuredClone(recorded));
97
+ if (custom) {
98
+ if (cause && custom.cause === void 0) define(custom, "cause", fromRecordedError(cause, reviveError));
99
+ if (revivedErrors && custom instanceof AggregateError) define(custom, "errors", revivedErrors);
100
+ return custom;
101
+ }
102
+ const base = revivedErrors ? new AggregateError(revivedErrors, fields.message) : new Error(fields.message);
103
+ const error = Object.assign(base, fields, data);
104
+ if (cause) define(error, "cause", fromRecordedError(cause, reviveError));
74
105
  if (ownName || name !== "Error") Object.defineProperty(error, "name", {
75
106
  value: name,
76
107
  writable: true,
@@ -133,6 +164,25 @@ async function* iterate(chunks) {
133
164
  for (const chunk of chunks) yield structuredClone(chunk);
134
165
  }
135
166
  /**
167
+ * Yields what `revived` yields (awaited), if it is an async iterable, then
168
+ * throws the recorded `failure`, if any. Like the recording wrapper, it calls
169
+ * `onError` only for a failure the port reads to, or one its consumer throws
170
+ * in (`.throw()`); `.return()` ends it with no error.
171
+ */
172
+ async function* replayStream(revived, onError, failure) {
173
+ try {
174
+ const value = await revived;
175
+ if (require_effects.isAsyncIterable(value)) yield* value;
176
+ } catch (error) {
177
+ onError(toRecordedError(error));
178
+ throw error;
179
+ }
180
+ if (failure) {
181
+ onError(failure.recorded);
182
+ throw failure.error;
183
+ }
184
+ }
185
+ /**
136
186
  * Creates a {@link Recorder}. With no options it records: every wrapped call
137
187
  * runs for real. With `replay` it answers from a recording instead.
138
188
  */
@@ -147,6 +197,7 @@ function createRecorder(options = {}) {
147
197
  const normalizeEffect = (effect, list) => options.normalize?.(effect, list.indexOf(effect), list) ?? effect;
148
198
  let modelCalls = 0;
149
199
  let toolsInFlight = 0;
200
+ const reviveRecordedError = (error) => fromRecordedError(error, options.reviveError);
150
201
  /**
151
202
  * The earliest unused recorded call that fits this one: one `translate`
152
203
  * adapts, or one with the same method and shape. Not by position, so a call
@@ -196,13 +247,23 @@ function createRecorder(options = {}) {
196
247
  const { recorded, answer } = replayed;
197
248
  usedModels.add(recorded);
198
249
  effect.replayed = recorded.call;
250
+ const recordError = (error) => {
251
+ effect.error = error;
252
+ };
253
+ if (answer.error && answer.stream && streamed && seam === "openai") {
254
+ effect.stream = answer.stream;
255
+ return replayStream(iterate(answer.stream), recordError, {
256
+ recorded: answer.error,
257
+ error: reviveRecordedError(answer.error)
258
+ });
259
+ }
199
260
  if (answer.error) {
200
261
  effect.error = answer.error;
201
- throw fromRecordedError(answer.error);
262
+ throw reviveRecordedError(answer.error);
202
263
  }
203
264
  effect.response = answer.response;
204
265
  effect.stream = answer.stream;
205
- if (seam === "openai") return streamed ? iterate(answer.stream ?? []) : structuredClone(answer.response);
266
+ if (seam === "openai") return streamed ? replayStream(iterate(answer.stream ?? []), recordError) : structuredClone(answer.response);
206
267
  return streamed ? {
207
268
  ...reviveTimestamps(structuredClone(answer.response) ?? {}),
208
269
  stream: streamFrom(answer.stream ?? [])
@@ -232,9 +293,14 @@ function createRecorder(options = {}) {
232
293
  if (seam === "openai") {
233
294
  const source = result;
234
295
  return (async function* () {
235
- for await (const chunk of source) {
236
- parts.push(toJson(chunk));
237
- yield chunk;
296
+ try {
297
+ for await (const chunk of source) {
298
+ parts.push(toJson(chunk));
299
+ yield chunk;
300
+ }
301
+ } catch (error) {
302
+ effect.error = toRecordedError(error);
303
+ throw error;
238
304
  }
239
305
  })();
240
306
  }
@@ -290,7 +356,7 @@ function createRecorder(options = {}) {
290
356
  if (recorded.sync) effect.sync = true;
291
357
  if (recorded.error) {
292
358
  effect.error = recorded.error;
293
- throw fromRecordedError(recorded.error);
359
+ throw reviveRecordedError(recorded.error);
294
360
  }
295
361
  effect.output = recorded.output;
296
362
  return structuredClone(recorded.output);
@@ -343,14 +409,23 @@ function createRecorder(options = {}) {
343
409
  throw error;
344
410
  }
345
411
  usedCalls.add(recorded);
346
- if (recorded.error) {
347
- effect.error = recorded.error;
348
- throw fromRecordedError(recorded.error);
349
- }
350
412
  if (recorded.stream) {
351
413
  effect.stream = recorded.stream;
352
414
  const chunks = structuredClone(recorded.stream);
353
- return revive ? revive(chunks) : iterate(chunks);
415
+ const revived = revive ? revive(chunks) : iterate(chunks);
416
+ const { error } = recorded;
417
+ if (!error && revive) return revived;
418
+ const recordError = (thrown) => {
419
+ effect.error = thrown;
420
+ };
421
+ return replayStream(revived, recordError, error && {
422
+ recorded: error,
423
+ error: reviveRecordedError(error)
424
+ });
425
+ }
426
+ if (recorded.error) {
427
+ effect.error = recorded.error;
428
+ throw reviveRecordedError(recorded.error);
354
429
  }
355
430
  effect.result = recorded.result;
356
431
  const result = structuredClone(recorded.result);
@@ -538,7 +613,7 @@ function modelCallShape(effect) {
538
613
  * that started after the previous one settled begins a new group, so
539
614
  * sequential calls keep their order.
540
615
  */
541
- function normalize$1(effects) {
616
+ function normalize(effects) {
542
617
  const result = [];
543
618
  let batch = [];
544
619
  const flush = () => {
@@ -629,7 +704,7 @@ function differ(left, right, options) {
629
704
  * kind pair up and report what changed, and the rest are missing or extra.
630
705
  */
631
706
  function compareRecordings(expected, actual, options = {}) {
632
- const select = (effects) => normalize$1(effects.flatMap((effect, index) => {
707
+ const select = (effects) => normalize(effects.flatMap((effect, index) => {
633
708
  if (options.kinds && !options.kinds.includes(effect.kind)) return [];
634
709
  const mapped = options.normalize ? options.normalize(effect, index, effects) : effect;
635
710
  return mapped ? [mapped] : [];
@@ -901,11 +976,11 @@ async function crashAt(point, ctx) {
901
976
  const log = await backing.read(THREAD);
902
977
  try {
903
978
  const replayed = require_event_log.replay(options.actors !== void 0 ? machine.provide({ actors: options.actors }) : machine, log);
904
- const fromLog = stableStringify(require_effects.toJsonValue({
979
+ const fromLog = stableStringify(require_run_agent.toJsonValue({
905
980
  value: replayed.snapshot.value,
906
981
  context: replayed.snapshot.context
907
982
  }));
908
- const live = stableStringify(require_effects.toJsonValue({
983
+ const live = stableStringify(require_run_agent.toJsonValue({
909
984
  value: result.snapshot.value,
910
985
  context: result.snapshot.context
911
986
  }));
@@ -923,7 +998,7 @@ async function crashAt(point, ctx) {
923
998
  async function runTurns(machine, options, world, store, resuming, signal) {
924
999
  const tracked = trackActors(machine, world);
925
1000
  const executors = trackExecutors(options.executors ?? {}, world);
926
- const actors = options.actors !== void 0 ? Object.fromEntries(Object.entries(options.actors).map(([key, logic]) => [key, require_effects.isStateMachineLogic(logic) ? trackActors(logic, world) : trackLogic(logic, world)])) : void 0;
1001
+ const actors = options.actors !== void 0 ? Object.fromEntries(Object.entries(options.actors).map(([key, logic]) => [key, require_run_agent.isStateMachineLogic(logic) ? trackActors(logic, world) : trackLogic(logic, world)])) : void 0;
927
1002
  const before = resuming?.crashed;
928
1003
  const log = before !== void 0 ? await store.read(THREAD) : [];
929
1004
  const durable = new Set(log.map((entry) => entry.id));
@@ -998,7 +1073,7 @@ async function runTurns(machine, options, world, store, resuming, signal) {
998
1073
  const turn = (event, track) => {
999
1074
  let awaiting = event !== void 0 && track === "host" ? require_event_log.canonicalEventJson(event) : void 0;
1000
1075
  let awaitingTimer = track === "timer";
1001
- live = require_effects.createAgentRuntime(tracked, {
1076
+ live = require_run_agent.createAgentRuntime(tracked, {
1002
1077
  ...options.runtime,
1003
1078
  onEvent: (entry) => {
1004
1079
  options.runtime?.onEvent?.(entry);
@@ -1036,7 +1111,7 @@ async function runTurns(machine, options, world, store, resuming, signal) {
1036
1111
  ...signal !== void 0 ? { signal } : {}
1037
1112
  });
1038
1113
  for (const index of redeliver.splice(0)) send(index);
1039
- return require_effects.runToQuiescence(live, {
1114
+ return require_run_agent.runToQuiescence(live, {
1040
1115
  ...options.input !== void 0 ? { input: options.input } : {},
1041
1116
  ...event !== void 0 ? { event } : {}
1042
1117
  });
@@ -1156,7 +1231,7 @@ function countBy(effects) {
1156
1231
  }
1157
1232
  function outcomeOf(result) {
1158
1233
  const snapshot = result.snapshot;
1159
- return stableStringify(require_effects.toJsonValue({
1234
+ return stableStringify(require_run_agent.toJsonValue({
1160
1235
  status: result.status,
1161
1236
  ...result.status === "done" ? { output: result.output } : {},
1162
1237
  ...result.status === "error" ? { cause: result.cause } : {},
@@ -1227,7 +1302,7 @@ function trackExecutors(executors, world) {
1227
1302
  const VOLATILE_REQUEST_KEYS = /* @__PURE__ */ new Set(["runId"]);
1228
1303
  function requestFingerprint(request) {
1229
1304
  try {
1230
- const json = require_effects.toJsonValue(request);
1305
+ const json = require_run_agent.toJsonValue(request);
1231
1306
  if (json && typeof json === "object" && !Array.isArray(json)) for (const key of VOLATILE_REQUEST_KEYS) delete json[key];
1232
1307
  return stableStringify(json);
1233
1308
  } catch {
@@ -1248,7 +1323,7 @@ function trackActors(machine, world, ancestors = /* @__PURE__ */ new Set()) {
1248
1323
  const wrapped = {};
1249
1324
  for (const [key, logic] of Object.entries(sources)) {
1250
1325
  if (require_event_log.isTextLogic(logic) || require_decision.isDecisionLogic(logic) || require_act.isActLogic(logic)) continue;
1251
- if (require_effects.isStateMachineLogic(logic)) {
1326
+ if (require_run_agent.isStateMachineLogic(logic)) {
1252
1327
  const inner = trackActors(logic, world, visited);
1253
1328
  if (inner !== logic) wrapped[key] = inner;
1254
1329
  continue;
@@ -1307,156 +1382,6 @@ function trackLogic(logic, world) {
1307
1382
  return Object.create(logic, overrides);
1308
1383
  }
1309
1384
  //#endregion
1310
- //#region src/trajectory.ts
1311
- /**
1312
- * Classifies one item. Event-shaped things (log entries, events) become events;
1313
- * everything else is a state value. A snapshot is unwrapped to its `value`.
1314
- * @internal
1315
- */
1316
- function normalize(item) {
1317
- if (require_event_log.isRecord(item)) {
1318
- const nested = item["event"];
1319
- if (require_event_log.isRecord(nested) && typeof nested["type"] === "string") return {
1320
- kind: "event",
1321
- event: nested
1322
- };
1323
- if (typeof item["type"] === "string") return {
1324
- kind: "event",
1325
- event: item
1326
- };
1327
- if ("value" in item) return {
1328
- kind: "state",
1329
- value: item["value"]
1330
- };
1331
- }
1332
- return {
1333
- kind: "state",
1334
- value: item
1335
- };
1336
- }
1337
- /** Every leaf of a state value as a dot path (`{ review: 'editing' }` → `review.editing`). @internal */
1338
- function statePaths$1(value) {
1339
- if (typeof value === "string") return [value];
1340
- if (!require_event_log.isRecord(value)) return [];
1341
- return Object.entries(value).flatMap(([key, child]) => statePaths$1(child).map((path) => `${key}.${path}`));
1342
- }
1343
- /**
1344
- * A string expectation matches a state value when it names one of its leaves or
1345
- * an ancestor of one, so `'review'` matches `{ review: 'editing' }`. @internal
1346
- */
1347
- function stateMatchesPath(path, value) {
1348
- return statePaths$1(value).some((leaf) => leaf === path || leaf.startsWith(`${path}.`));
1349
- }
1350
- function deepEqual(a, b) {
1351
- if (Object.is(a, b)) return true;
1352
- if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => deepEqual(item, b[index]));
1353
- if (!require_event_log.isRecord(a) || !require_event_log.isRecord(b)) return false;
1354
- const keys = Object.keys(a);
1355
- return keys.length === Object.keys(b).length && keys.every((key) => deepEqual(a[key], b[key]));
1356
- }
1357
- /**
1358
- * Normalizes ONE expected item into a predicate over actual items, so the
1359
- * expectation is classified once instead of once per candidate:
1360
- *
1361
- * - A string expectation matches an event's `type`, or a state value it names.
1362
- * - An event expectation matches an event with the same `type` whose other
1363
- * declared keys deep-equal, so payload details are opt-in.
1364
- * - A state-value expectation matches structurally.
1365
- * @internal
1366
- */
1367
- function matcherFor(expected) {
1368
- if (typeof expected === "string") return (actual) => {
1369
- const actualItem = normalize(actual);
1370
- return actualItem.kind === "event" ? actualItem.event.type === expected : stateMatchesPath(expected, actualItem.value);
1371
- };
1372
- const expectedItem = normalize(expected);
1373
- switch (expectedItem.kind) {
1374
- case "event": return (actual) => {
1375
- const actualItem = normalize(actual);
1376
- return actualItem.kind === "event" && Object.keys(expectedItem.event).every((key) => deepEqual(expectedItem.event[key], actualItem.event[key]));
1377
- };
1378
- case "state": return (actual) => {
1379
- const actualItem = normalize(actual);
1380
- return actualItem.kind === "state" && deepEqual(expectedItem.value, actualItem.value);
1381
- };
1382
- }
1383
- }
1384
- /**
1385
- * Matches a run's trajectory against an expected one.
1386
- *
1387
- * Both trajectories may be state values or events collected from
1388
- * `onTransition`;
1389
- * items are compared by shape, so `['prompting', 'drafting']` scores a state
1390
- * path and `['PROMPT_SUBMITTED', 'SEND']` scores an event log with the same
1391
- * call.
1392
- *
1393
- * Semantics are an ordered subsequence: every expected item must appear in
1394
- * `actual`, in order, with gaps allowed. Pass `{ exact: true }` to require
1395
- * equality instead.
1396
- *
1397
- * @example Trajectory assertions in a test
1398
- * ```ts
1399
- * const statePath: unknown[] = [];
1400
- * const result = await runToQuiescence(
1401
- * createAgentRuntime(machine, {
1402
- * executors,
1403
- * onTransition: (snapshot) => statePath.push(snapshot.value),
1404
- * }),
1405
- * { input },
1406
- * );
1407
- *
1408
- * const path = matchesTrajectory(statePath, ['prompting', 'drafting', 'sent']);
1409
- * expect(path.matched, JSON.stringify(path.firstMiss)).toBe(true);
1410
- * expect(matchesTrajectory(events, ['PROMPT_SUBMITTED', 'SEND']).matched).toBe(true);
1411
- * ```
1412
- *
1413
- * @example Partial credit in an eval scorer
1414
- * ```ts
1415
- * const match = matchesTrajectory(output.statePath, expected.statePath);
1416
- * return { name: 'state_path', score: match.score, metadata: { ...match.firstMiss } };
1417
- * ```
1418
- */
1419
- function matchesTrajectory(actual, expected, options = {}) {
1420
- const expectedCount = expected.length;
1421
- if (expectedCount === 0) throw new require_event_log.AgentError("empty-trajectory", "matchesTrajectory: expected trajectory must contain at least one item.");
1422
- const exact = options.exact === true;
1423
- let cursor = 0;
1424
- let matchedCount = 0;
1425
- let firstMiss;
1426
- for (let index = 0; index < expectedCount; index++) {
1427
- const want = expected[index];
1428
- const matches = matcherFor(want);
1429
- const searchedFrom = exact ? index : cursor;
1430
- let found = -1;
1431
- for (let at = searchedFrom; at < actual.length; at++) {
1432
- if (matches(actual[at])) {
1433
- found = at;
1434
- break;
1435
- }
1436
- if (exact) break;
1437
- }
1438
- if (found === -1) {
1439
- firstMiss = {
1440
- index,
1441
- expected: want,
1442
- searchedFrom
1443
- };
1444
- break;
1445
- }
1446
- matchedCount++;
1447
- cursor = found + 1;
1448
- }
1449
- const matched = !firstMiss && (!exact || actual.length === expectedCount);
1450
- const denominator = exact ? Math.max(expectedCount, actual.length) : expectedCount;
1451
- return {
1452
- matched,
1453
- matchedCount,
1454
- expectedCount,
1455
- score: matched ? 1 : denominator === 0 ? 1 : matchedCount / denominator,
1456
- ...firstMiss ? { firstMiss } : {}
1457
- };
1458
- }
1459
- //#endregion
1460
1385
  //#region src/seam.ts
1461
1386
  /** The only own keys an executor result carries. @internal */
1462
1387
  const TEXT_RESULT_KEYS = /* @__PURE__ */ new Set([
@@ -1611,7 +1536,7 @@ async function runSeam(machine, options) {
1611
1536
  let result;
1612
1537
  const maxTurns = options.maxTurns ?? 12;
1613
1538
  for (let turn = 0; turn <= maxTurns; turn++) {
1614
- result = await require_effects.runToQuiescence(require_effects.createAgentRuntime(machine, {
1539
+ result = await require_run_agent.runToQuiescence(require_run_agent.createAgentRuntime(machine, {
1615
1540
  ...options.actors ? { actors: options.actors } : {},
1616
1541
  executors,
1617
1542
  onTransition: (next, causedBy) => {
@@ -1665,802 +1590,6 @@ async function runSeam(machine, options) {
1665
1590
  };
1666
1591
  }
1667
1592
  //#endregion
1668
- //#region src/internal/trace-states.ts
1669
- /** The `value` of a trace event's `from` or `snapshot`. @internal */
1670
- function valueOf(holder) {
1671
- return require_event_log.isRecord(holder) ? holder.value : void 0;
1672
- }
1673
- /** A decision request on its first attempt (no failed attempts yet). @internal */
1674
- function isFirstDecision(request) {
1675
- return require_event_log.isRecord(request) && request.kind === "decision" && (!Array.isArray(request.attempts) || request.attempts.length === 0);
1676
- }
1677
- /**
1678
- * Every active leaf of a state value as a dot path:
1679
- * `'a'` → `['a']`; `{ a: 'b' }` → `['a.b']`; `{ p: { x: 'one', y: {} } }` → `['p.x.one', 'p.y']`.
1680
- * @internal
1681
- */
1682
- function statePaths(value, prefix = "") {
1683
- if (typeof value === "string") return [prefix + value];
1684
- if (!require_event_log.isRecord(value)) return [];
1685
- const entries = Object.entries(value);
1686
- if (entries.length === 0) return prefix ? [prefix.slice(0, -1)] : [];
1687
- return entries.flatMap(([key, child]) => statePaths(child, `${prefix}${key}.`));
1688
- }
1689
- /**
1690
- * A trace's `machine.transition` events as steps, skipping `@xstate.init`.
1691
- * A first-attempt decision's `request.start` names the decision on the step
1692
- * that follows it, skipping `@agent.usage` steps. Reads live and serialized traces alike.
1693
- * @internal
1694
- */
1695
- function traceSteps(trace) {
1696
- const steps = [];
1697
- let decision;
1698
- for (const event of trace) {
1699
- if (event.type === "request.start" && isFirstDecision(event.request)) {
1700
- const name = event.request.name;
1701
- decision = typeof name === "string" ? name : void 0;
1702
- continue;
1703
- }
1704
- if (event.type !== "machine.transition") continue;
1705
- const type = require_event_log.isRecord(event.event) ? event.event.type : void 0;
1706
- if (type === "@xstate.init") continue;
1707
- const fromValue = valueOf(event.from);
1708
- const toValue = valueOf(event.snapshot);
1709
- steps.push({
1710
- from: statePaths(fromValue),
1711
- fromValue,
1712
- event: typeof type === "string" ? type : "",
1713
- payload: event.event,
1714
- to: statePaths(toValue),
1715
- toValue,
1716
- ignored: event.ignored === true,
1717
- ...decision === void 0 || type === "@agent.usage" ? {} : { decision }
1718
- });
1719
- if (type !== "@agent.usage") decision = void 0;
1720
- }
1721
- return steps;
1722
- }
1723
- //#endregion
1724
- //#region src/fold-traces.ts
1725
- /**
1726
- * Folds many runs' traces into per-state stats: how many runs visited each
1727
- * state, how many of those failed, how many options the guards withheld
1728
- * at decisions made there, how many events it ignored, and how many failures
1729
- * `blame` traced back to it. Pure; reads only:
1730
- *
1731
- * - `machine.transition`: the run visited `from.value` and `snapshot.value`;
1732
- * with `ignored`, each of those states ignored one event.
1733
- * - `request.start` of a decision, first attempt only: its `request.withheld`
1734
- * counts toward the state the run was in, the `from.value` of the
1735
- * transition its choice causes.
1736
- *
1737
- * A trace is one run's events, live (`AgentTraceEvent`) or serialized
1738
- * (`serializeTraceEvent`). A host-driven `decideNext` traces nothing; push
1739
- * `{ type: 'request.start', request: result.request }` onto the trace before
1740
- * sending its event. Parallel states count once per active leaf.
1741
- *
1742
- * `failures` counts a failed run against every state it visited, so states
1743
- * after a bad decision rank as high as the decision. Sort by `blamed` to see
1744
- * where failures start.
1745
- *
1746
- * @example
1747
- * ```ts no-check
1748
- * const stats = foldTraces(traces, {
1749
- * failed: (trace) => trace.at(-1)?.type === 'run.end' && trace.at(-1).status === 'error',
1750
- * });
1751
- * ```
1752
- */
1753
- function foldTraces(traces, options) {
1754
- const stats = /* @__PURE__ */ new Map();
1755
- const statsFor = (state) => {
1756
- let entry = stats.get(state);
1757
- if (!entry) {
1758
- entry = {
1759
- state,
1760
- visits: 0,
1761
- failures: 0,
1762
- withheld: 0,
1763
- ignored: 0,
1764
- blamed: 0
1765
- };
1766
- stats.set(state, entry);
1767
- }
1768
- return entry;
1769
- };
1770
- for (const trace of traces) {
1771
- const visited = /* @__PURE__ */ new Set();
1772
- let current = [];
1773
- const visit = (state) => {
1774
- statsFor(state);
1775
- visited.add(state);
1776
- };
1777
- for (const event of trace) if (event.type === "machine.transition") {
1778
- for (const state of statePaths(valueOf(event.from))) visit(state);
1779
- current = statePaths(valueOf(event.snapshot));
1780
- for (const state of current) visit(state);
1781
- if (event.ignored === true) for (const state of current) statsFor(state).ignored++;
1782
- } else if (event.type === "request.start" && isFirstDecision(event.request)) {
1783
- const withheld = event.request.withheld;
1784
- if (!Array.isArray(withheld)) continue;
1785
- for (const state of current) statsFor(state).withheld += withheld.length;
1786
- }
1787
- const failed = options.failed(trace);
1788
- for (const state of visited) {
1789
- const entry = statsFor(state);
1790
- entry.visits++;
1791
- if (failed) entry.failures++;
1792
- }
1793
- const blamed = failed ? options.blame?.(trace) : void 0;
1794
- if (blamed !== void 0) statsFor(blamed).blamed++;
1795
- }
1796
- return [...stats.values()];
1797
- }
1798
- //#endregion
1799
- //#region src/trace-coverage.ts
1800
- /** Mirrors xstate@6.0.0-alpha.69 `matchesEventDescriptor` (src/utils.ts), which the package does not export. */
1801
- function matchesDescriptor(eventType, descriptor) {
1802
- if (descriptor === eventType || descriptor === "*") return true;
1803
- if (!descriptor.endsWith(".*")) return false;
1804
- const tokens = descriptor.split(".");
1805
- const eventTokens = eventType.split(".");
1806
- for (let index = 0; index < tokens.length; index++) {
1807
- if (tokens[index] === "*") return index === tokens.length - 1;
1808
- if (tokens[index] !== eventTokens[index]) return false;
1809
- }
1810
- return true;
1811
- }
1812
- /** Whether a definition's `matches` pattern (e.g. an invoke's `actorId`) accepts the traced event. */
1813
- function matchesPattern(definition, event) {
1814
- const pattern = definition.matches;
1815
- if (!require_event_log.isRecord(pattern) || !require_event_log.isRecord(event)) return true;
1816
- return Object.entries(pattern).every(([key, value]) => Object.is(event[key], value));
1817
- }
1818
- /**
1819
- * The descriptor XState would try first at `node` for `event`, guards aside:
1820
- * the exact key, then partial and wildcard keys longest first, as in
1821
- * xstate@6.0.0-alpha.69 `getCandidates` (src/stateUtils.ts).
1822
- */
1823
- function candidateAt(node, type, event) {
1824
- const keys = [...node.transitions.keys()].filter((key) => key !== type && matchesDescriptor(type, key)).sort((a, b) => b.length - a.length);
1825
- if (node.transitions.has(type)) keys.unshift(type);
1826
- return keys.find((key) => node.transitions.get(key).some((d) => matchesPattern(d, event)));
1827
- }
1828
- const pathOf = (node) => node.path.join(".");
1829
- const sameLeaves = (a, b) => statePaths(a).sort().join("|") === statePaths(b).sort().join("|");
1830
- /**
1831
- * The leaves a trace passed through inside its macrosteps: states an `always`
1832
- * transition left before the step ended, which no traced snapshot holds.
1833
- * Re-derived with XState's pure `getMicrosteps` from the snapshot before each
1834
- * step (the previous step's, or the fresh start's microsteps from its input).
1835
- * A step that cannot be recomputed, or whose recomputation ends elsewhere
1836
- * than the traced snapshot, adds none. Only `value` and `context` are rebuilt,
1837
- * so a guard that reads live children can take another route to the same
1838
- * leaves, and that route is credited.
1839
- */
1840
- function passedThrough(machine, trace) {
1841
- const leaves = /* @__PURE__ */ new Set();
1842
- let resumed = false;
1843
- let previous;
1844
- for (const event of trace) {
1845
- if (event.type === "run.start") resumed = event.snapshot !== void 0;
1846
- if (event.type !== "machine.transition" || !require_event_log.isRecord(event.event)) continue;
1847
- const snapshot = require_event_log.isRecord(event.snapshot) ? event.snapshot : void 0;
1848
- try {
1849
- const microsteps = event.event.type === "@xstate.init" ? resumed ? [] : (0, xstate.getInitialMicrosteps)(machine, event.event.input) : previous && sameLeaves(previous.value, valueOf(event.from)) ? (0, xstate.getMicrosteps)(machine, machine.resolveState({
1850
- value: previous.value,
1851
- context: previous.context
1852
- }), event.event) : [];
1853
- const last = microsteps.at(-1)?.[0];
1854
- if (last && snapshot && sameLeaves(last.value, snapshot.value)) for (const [step] of microsteps) for (const leaf of statePaths(step.value)) leaves.add(leaf);
1855
- } catch {}
1856
- previous = snapshot;
1857
- }
1858
- return leaves;
1859
- }
1860
- const ratioOf = (visited, total) => total === 0 ? 1 : visited / total;
1861
- /**
1862
- * Measures how much of a machine a set of runs exercised: which atomic and
1863
- * final states they visited, and which handlers they took. A handler is a
1864
- * `(declaring state, event descriptor)` pair from `on`, invoke
1865
- * `onDone`/`onError`, and `after`, with the targets it declares and the
1866
- * leaves the runs landed in after it. Pure; reads only `machine.transition`
1867
- * events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
1868
- *
1869
- * A step credits one handler per active leaf: the nearest node, from the leaf
1870
- * up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
1871
- * event, preferred in XState's order. Guards are not replayed, so a step
1872
- * whose nearest handler was rejected and fell through to an ancestor is
1873
- * credited to the nearest one. XState keys invoke and `after` handlers by
1874
- * bare type (`xstate.done.actor`), one per state however many invokes; a
1875
- * definition's `matches` (an invoke's `actorId`) must accept the traced event
1876
- * for its state to be credited. Ignored steps credit no handler. `always`
1877
- * handlers are listed as `unobservable`, never as unvisited; a state one
1878
- * passes through is visited, re-derived from each step's microsteps.
1879
- *
1880
- * @example
1881
- * ```ts no-check
1882
- * const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
1883
- * expect(report.unvisitedHandlers).toEqual([]);
1884
- * ```
1885
- */
1886
- function traceCoverage(machine, traces, options = {}) {
1887
- const nodes = /* @__PURE__ */ new Map();
1888
- const states = [];
1889
- const handlers = [];
1890
- const handlerAt = /* @__PURE__ */ new Map();
1891
- const unobservable = [];
1892
- const walk = (node) => {
1893
- const state = pathOf(node);
1894
- nodes.set(state, node);
1895
- if (node.type === "atomic" || node.type === "final") states.push({
1896
- state,
1897
- runs: 0
1898
- });
1899
- const byEvent = /* @__PURE__ */ new Map();
1900
- for (const [event, definitions] of node.transitions) {
1901
- const dynamic = definitions.some((d) => typeof d.to === "function");
1902
- const declared = [...new Set(definitions.flatMap((d) => (d.target ?? []).map(pathOf)))];
1903
- const handler = {
1904
- state,
1905
- event,
1906
- runs: 0,
1907
- targets: {
1908
- declared: dynamic ? "dynamic" : declared,
1909
- seen: [],
1910
- unreached: []
1911
- }
1912
- };
1913
- handlers.push(handler);
1914
- byEvent.set(event, handler);
1915
- }
1916
- handlerAt.set(node, byEvent);
1917
- if (node.always?.length) unobservable.push({
1918
- state,
1919
- event: "always"
1920
- });
1921
- for (const child of Object.values(node.states)) walk(child);
1922
- };
1923
- walk(machine.root);
1924
- const stateRuns = new Map(states.map((row) => [row.state, row]));
1925
- const seen = /* @__PURE__ */ new Map();
1926
- for (const trace of traces) {
1927
- const steps = traceSteps(trace);
1928
- const visited = new Set(steps[0]?.from);
1929
- for (const event of trace) {
1930
- if (event.type !== "machine.transition" || !require_event_log.isRecord(event.event)) continue;
1931
- if (event.event.type !== "@xstate.init" || !require_event_log.isRecord(event.snapshot)) continue;
1932
- for (const leaf of statePaths(event.snapshot.value)) visited.add(leaf);
1933
- }
1934
- if (unobservable.length > 0) for (const leaf of passedThrough(machine, trace)) visited.add(leaf);
1935
- const credited = /* @__PURE__ */ new Set();
1936
- for (const step of steps) {
1937
- for (const leaf of step.to) visited.add(leaf);
1938
- if (step.ignored) continue;
1939
- const hits = /* @__PURE__ */ new Set();
1940
- for (const leaf of step.from) for (let node = nodes.get(leaf); node; node = node.parent) {
1941
- if (node.type === "final") continue;
1942
- const event = candidateAt(node, step.event, step.payload);
1943
- if (event === void 0) continue;
1944
- hits.add(handlerAt.get(node).get(event));
1945
- break;
1946
- }
1947
- for (const handler of hits) {
1948
- credited.add(handler);
1949
- let leaves = seen.get(handler);
1950
- if (!leaves) seen.set(handler, leaves = /* @__PURE__ */ new Set());
1951
- for (const leaf of step.to) leaves.add(leaf);
1952
- }
1953
- }
1954
- for (const state of visited) {
1955
- const row = stateRuns.get(state);
1956
- if (row) row.runs++;
1957
- }
1958
- for (const handler of credited) handler.runs++;
1959
- }
1960
- for (const handler of handlers) {
1961
- const leaves = [...seen.get(handler) ?? []];
1962
- handler.targets.seen = leaves;
1963
- const { declared } = handler.targets;
1964
- if (declared === "dynamic") continue;
1965
- handler.targets.unreached = declared.filter((target) => !leaves.some((leaf) => leaf === target || leaf.startsWith(`${target}.`)));
1966
- }
1967
- const reachableComplete = options.reachable ? require_verify.isExplorationComplete(options.reachable) : void 0;
1968
- let unreachable;
1969
- if (options.reachable && reachableComplete) {
1970
- const reached = new Set(options.reachable.reachedStates.flatMap((value) => statePaths(value)));
1971
- unreachable = states.filter((row) => !reached.has(row.state) && row.runs === 0).map((row) => row.state);
1972
- }
1973
- const counted = unreachable ? states.filter((row) => !unreachable.includes(row.state)) : states;
1974
- const unvisitedStates = counted.filter((row) => row.runs === 0).map((row) => row.state);
1975
- const unvisitedHandlers = handlers.filter((handler) => handler.runs === 0).map(({ state, event }) => ({
1976
- state,
1977
- event
1978
- }));
1979
- return {
1980
- states,
1981
- handlers,
1982
- unvisitedStates,
1983
- unvisitedHandlers,
1984
- unobservable,
1985
- ...unreachable ? { unreachable } : {},
1986
- ...reachableComplete === void 0 ? {} : { reachableComplete },
1987
- ratio: {
1988
- states: ratioOf(counted.length - unvisitedStates.length, counted.length),
1989
- handlers: ratioOf(handlers.length - unvisitedHandlers.length, handlers.length)
1990
- }
1991
- };
1992
- }
1993
- //#endregion
1994
- //#region src/first-divergence.ts
1995
- const defaultKey = (step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`;
1996
- const isErrorLike = (value) => require_event_log.isError(value) || require_event_log.isRecord(value) && typeof value.name === "string" && typeof value.message === "string";
1997
- const normalizeErrors = (value, seen = /* @__PURE__ */ new WeakSet()) => {
1998
- if (value === null || typeof value !== "object") return value;
1999
- if (seen.has(value)) return "[circular]";
2000
- seen.add(value);
2001
- try {
2002
- if (isErrorLike(value) && (require_event_log.isError(value) || "stack" in value || "code" in value)) {
2003
- const { name, message, code, cause } = value;
2004
- return {
2005
- name,
2006
- message,
2007
- ...typeof code === "string" ? { code } : {},
2008
- ...cause !== void 0 ? { cause: normalizeErrors(cause, seen) } : {}
2009
- };
2010
- }
2011
- if (Array.isArray(value)) return value.map((item) => normalizeErrors(item, seen));
2012
- if (!require_event_log.isPlainObject(value)) return value;
2013
- const copy = Object.create(null);
2014
- for (const key of Object.keys(value)) copy[key] = normalizeErrors(value[key], seen);
2015
- return copy;
2016
- } finally {
2017
- seen.delete(value);
2018
- }
2019
- };
2020
- const payloadKey = (step) => {
2021
- if (!require_event_log.isRecord(step.payload)) return `${defaultKey(step)}|${require_event_log.canonicalEventJson(step.payload)}`;
2022
- const { sessionId: _sessionId, ...payload } = step.payload;
2023
- return `${defaultKey(step)}|${require_event_log.canonicalEventJson(normalizeErrors(payload))}`;
2024
- };
2025
- const isTrace = (items) => items.length === 0 || items.some((item) => require_event_log.isRecord(item) && (item.type === "machine.transition" || item.type === "run.start" || item.type === "run.end"));
2026
- /**
2027
- * Finds the first step where a run departs from a reference, so a failure is
2028
- * blamed on the state it started in rather than on every state the run went
2029
- * through afterwards.
2030
- *
2031
- * - Against a reference **trace**, such as a passing run of the same
2032
- * scenario, steps are compared in order by `key`. The first step that
2033
- * differs, or the point where either trace ends early, is the divergence.
2034
- * - Against an expected **trajectory** (`TrajectoryItem[]`, as for
2035
- * {@link matchesTrajectory}), the run is laid out as its initial state, then
2036
- * each step's event and new state. The first expected item it never reaches
2037
- * is the divergence, blamed on the step after the last item it did reach. A
2038
- * run that goes on past the trajectory has not diverged.
2039
- *
2040
- * An empty reference is a trace with no steps, so any step diverges from it.
2041
- * A trace is one run's events, live or serialized. Steps the machine ignored
2042
- * changed nothing and are left out on both sides. Returns `undefined` when the
2043
- * run does not diverge.
2044
- *
2045
- * @example
2046
- * ```ts no-check
2047
- * const stats = foldTraces(traces, {
2048
- * failed,
2049
- * blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
2050
- * });
2051
- * ```
2052
- */
2053
- function firstDivergence(actual, reference, options = {}) {
2054
- const steps = traceSteps(actual).filter((step) => !step.ignored);
2055
- return isTrace(reference) ? againstTrace(steps, traceSteps(reference).filter((step) => !step.ignored), options.key === "payload" ? payloadKey : options.key ?? defaultKey) : againstTrajectory(actual, steps, reference);
2056
- }
2057
- function diverged(steps, index, expected, initial = steps[0]?.fromValue) {
2058
- const step = steps[index];
2059
- return {
2060
- index,
2061
- state: (step ? step.from[0] : steps.at(-1)?.to[0] ?? statePaths(initial)[0]) ?? "",
2062
- ...step?.decision === void 0 ? {} : { decision: step.decision },
2063
- ...step ? { actual: step } : {},
2064
- ...expected === void 0 ? {} : { expected }
2065
- };
2066
- }
2067
- function againstTrace(steps, reference, key) {
2068
- const length = Math.max(steps.length, reference.length);
2069
- for (let index = 0; index < length; index++) {
2070
- const step = steps[index];
2071
- const want = reference[index];
2072
- if (step && want && key(step) === key(want)) continue;
2073
- return diverged(steps, index, want, steps[0]?.fromValue ?? reference[0]?.fromValue);
2074
- }
2075
- }
2076
- function againstTrajectory(trace, steps, expected) {
2077
- const initial = initialValue(trace, steps);
2078
- const timeline = initial === void 0 ? [] : [{ value: initial }];
2079
- const stepAt = initial === void 0 ? [] : [0];
2080
- steps.forEach((step, index) => {
2081
- timeline.push(step.payload);
2082
- stepAt.push(index);
2083
- if (JSON.stringify(step.toValue) !== JSON.stringify(step.fromValue)) {
2084
- timeline.push({ value: step.toValue });
2085
- stepAt.push(index);
2086
- }
2087
- });
2088
- const match = matchesTrajectory(timeline, expected);
2089
- if (!match.firstMiss) return void 0;
2090
- const at = match.firstMiss.searchedFrom;
2091
- return diverged(steps, at < timeline.length ? stepAt[at] ?? steps.length : steps.length, match.firstMiss.expected, initial);
2092
- }
2093
- function initialValue(trace, steps) {
2094
- for (const event of trace) if (event.type === "machine.transition" && require_event_log.isRecord(event.snapshot)) return event.snapshot.value;
2095
- return steps[0]?.fromValue;
2096
- }
2097
- //#endregion
2098
- //#region src/propose-from-traces.ts
2099
- /**
2100
- * Groups the approval verdicts in many runs' traces by `key` and proposes an
2101
- * `allow` or `deny` rule for each. Pure and deterministic; a proposal is for
2102
- * a person to review, and nothing here changes a machine.
2103
- *
2104
- * Reads `machine.transition` events: a verdict is a transition, not ignored,
2105
- * out of `approvalState` on an approve or reject event. Rows come in the
2106
- * order keys were first seen.
2107
- *
2108
- * @example
2109
- * ```ts no-check
2110
- * const rows = proposeApprovalRules(traces, {
2111
- * approvalState: "awaitingApproval",
2112
- * approveEvents: ["APPROVE"],
2113
- * rejectEvents: ["REJECT"],
2114
- * key: ({ snapshot }) => `${snapshot.context.pending.tool}:${snapshot.context.pending.path}`,
2115
- * });
2116
- * ```
2117
- */
2118
- function proposeApprovalRules(traces, options) {
2119
- const rows = /* @__PURE__ */ new Map();
2120
- const rowFor = (key) => {
2121
- let row = rows.get(key);
2122
- if (!row) {
2123
- row = {
2124
- key,
2125
- approved: 0,
2126
- rejected: 0,
2127
- violations: 0,
2128
- runs: [],
2129
- proposal: null,
2130
- support: 0
2131
- };
2132
- rows.set(key, row);
2133
- }
2134
- return row;
2135
- };
2136
- const inState = (value) => statePaths(value).some((path) => path === options.approvalState || path.startsWith(`${options.approvalState}.`));
2137
- let index = 0;
2138
- for (const trace of traces) {
2139
- const run = index++;
2140
- const seen = /* @__PURE__ */ new Set();
2141
- let snapshot;
2142
- for (const event of trace) {
2143
- if (event.type !== "machine.transition") continue;
2144
- const before = snapshot;
2145
- snapshot = event.snapshot;
2146
- const type = require_event_log.isRecord(event.event) ? event.event.type : void 0;
2147
- if (event.ignored === true || typeof type !== "string" || !inState(valueOf(event.from))) continue;
2148
- const approve = options.approveEvents.includes(type);
2149
- if (!approve && !options.rejectEvents.includes(type)) continue;
2150
- const key = options.key({
2151
- event: event.event,
2152
- snapshot: before,
2153
- trace
2154
- });
2155
- if (key === void 0) continue;
2156
- const row = rowFor(key);
2157
- if (approve) row.approved++;
2158
- else row.rejected++;
2159
- if (!seen.has(key)) {
2160
- seen.add(key);
2161
- row.runs.push(run);
2162
- }
2163
- }
2164
- for (const key of options.violations?.(trace) ?? []) {
2165
- const row = rows.get(key);
2166
- if (row) row.violations++;
2167
- }
2168
- }
2169
- for (const row of rows.values()) {
2170
- row.support = row.runs.length;
2171
- row.proposal = row.approved > 0 && row.rejected === 0 && row.violations === 0 ? "allow" : row.rejected > 0 && row.approved === 0 ? "deny" : null;
2172
- }
2173
- return [...rows.values()];
2174
- }
2175
- /**
2176
- * Proposes a value for each cap (retry count, steps, batch size) from many
2177
- * runs' traces: the 95th percentile of what passing runs used, plus one, and
2178
- * never below the current cap while runs still hit it. Pure and
2179
- * deterministic; a proposal is for a person to review, and nothing here
2180
- * changes a machine.
2181
- *
2182
- * @example
2183
- * ```ts no-check
2184
- * const [retries] = proposeCaps(traces, {
2185
- * caps: [{ name: "retries", current: 3, failureState: "gaveUp", valueFromRun: countRetries }],
2186
- * });
2187
- * ```
2188
- */
2189
- function proposeCaps(traces, options) {
2190
- const runs = [...traces];
2191
- return options.caps.map((cap) => {
2192
- let hits = 0;
2193
- let max = null;
2194
- const passing = [];
2195
- for (const trace of runs) {
2196
- const hit = hitCap(trace, cap);
2197
- if (hit) hits++;
2198
- const value = cap.valueFromRun(trace);
2199
- if (value === void 0) continue;
2200
- max = max === null ? value : Math.max(max, value);
2201
- if (!hit) passing.push(value);
2202
- }
2203
- passing.sort((a, b) => a - b);
2204
- const p95 = passing.length ? passing[Math.ceil(.95 * passing.length) - 1] : null;
2205
- const proposal = p95 === null ? null : hits > 0 && cap.current !== void 0 ? Math.max(p95 + 1, cap.current) : p95 + 1;
2206
- return {
2207
- name: cap.name,
2208
- hits,
2209
- passing: passing.length,
2210
- max,
2211
- p95,
2212
- proposal
2213
- };
2214
- });
2215
- }
2216
- function hitCap(trace, cap) {
2217
- for (const event of trace) {
2218
- if (cap.failureState !== void 0 && event.type === "machine.transition" && statePaths(valueOf(event.snapshot)).some((path) => path === cap.failureState || path.startsWith(`${cap.failureState}.`))) return true;
2219
- if (cap.failureOutput && event.type === "run.end" && event.status === "done") {
2220
- if (cap.failureOutput(event.output)) return true;
2221
- }
2222
- }
2223
- return false;
2224
- }
2225
- //#endregion
2226
- //#region src/decision-requests.ts
2227
- /**
2228
- * The decisions a run asked for, read from its trace: each decision's first
2229
- * `request.start`, with the states the run was in. A trace is one run's events,
2230
- * live or serialized with `serializeTraceEvent`. Retries of the same decision
2231
- * are left out, so each decision appears once.
2232
- *
2233
- * Use the requests as eval rows (run a decider on each and compare its event
2234
- * with a label) or as probe inputs for a policy. A serialized request keeps its
2235
- * `options`, `metadata`, `system`, and `prompt`; event schemas do not survive
2236
- * serialization, so replay closed choices (`options`) rather than open ones.
2237
- *
2238
- * @example
2239
- * ```ts no-check
2240
- * const rows = traces.flatMap((trace) => getDecisionRequests(trace, { name: "chooseResolution" }));
2241
- * for (const { request } of rows) console.log((await decide(request)).event);
2242
- * ```
2243
- */
2244
- function getDecisionRequests(trace, options = {}) {
2245
- const requests = [];
2246
- let current = [];
2247
- for (const event of trace) if (event.type === "machine.transition") current = statePaths(valueOf(event.snapshot));
2248
- else if (event.type === "request.start" && isFirstDecision(event.request)) {
2249
- if (options.name !== void 0 && event.request.name !== options.name) continue;
2250
- requests.push({
2251
- states: [...current],
2252
- request: event.request
2253
- });
2254
- }
2255
- return requests;
2256
- }
2257
- //#endregion
2258
- //#region src/repeat-runs.ts
2259
- /**
2260
- * Summarizes repeated runs of each scenario: how often each passed, whether
2261
- * all its runs passed (pass^k), how many distinct state paths they took, and
2262
- * where they first went different ways. Pure. Use it for runs collected
2263
- * elsewhere, such as CI shards or replays; {@link repeatRuns} collects them and
2264
- * calls it.
2265
- *
2266
- * A scenario that passes 3 of 5 runs and `divergedAt: 'deciding'` points at one
2267
- * decision, not at a flaky agent.
2268
- *
2269
- * @example
2270
- * ```ts no-check
2271
- * const report = summarizeRepeats(shards.map((s) => ({ id: s.id, scenario: s, runs: s.runs })));
2272
- * report.passHat(3); // the chance 3 runs of a scenario all pass
2273
- * ```
2274
- */
2275
- function summarizeRepeats(groups) {
2276
- const scenarios = groups.map((group) => summarize$1(group));
2277
- const traces = groups.flatMap((group) => group.runs.flatMap((run) => run.trace ? [run.trace] : []));
2278
- const most = Math.max(0, ...groups.map((group) => group.runs.length));
2279
- const passHat = (j) => {
2280
- if (!Number.isInteger(j) || j < 1 || j > most) throw new require_event_log.AgentError("pass-hat-out-of-range", `passHat: j must be a whole number from 1 to ${most}, got ${j}.`);
2281
- const eligible = scenarios.filter((scenario) => scenario.runs.length >= j);
2282
- return eligible.reduce((total, scenario) => total + allPass(scenario.passes, scenario.runs.length, j), 0) / eligible.length;
2283
- };
2284
- return {
2285
- scenarios,
2286
- passHat,
2287
- traces
2288
- };
2289
- }
2290
- /**
2291
- * Runs every scenario `k` times and summarizes the runs with
2292
- * {@link summarizeRepeats}. Runs are sequential, scenario by scenario. A run
2293
- * that throws fails, with its message as `error` and no trace.
2294
- *
2295
- * With a live model, one run per scenario says little: τ-bench found pass^k
2296
- * falls well below the single-run rate. Report pass^k next to it.
2297
- *
2298
- * @example
2299
- * ```ts no-check
2300
- * const report = await repeatRuns({
2301
- * scenarios,
2302
- * k: 5,
2303
- * run: async (scenario) => {
2304
- * const trace: AgentTraceEvent[] = [];
2305
- * const result = await runToQuiescence(
2306
- * createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
2307
- * { input: scenario.input },
2308
- * );
2309
- * return { trace, outcome: String(result.snapshot.value) };
2310
- * },
2311
- * passed: (scenario, { outcome }) => outcome === scenario.expected,
2312
- * });
2313
- * report.passHat(5);
2314
- * ```
2315
- */
2316
- async function repeatRuns(options) {
2317
- if (!Number.isInteger(options.k) || options.k < 1) throw new require_event_log.AgentError("invalid-repeat-count", `repeatRuns: k must be a whole number of 1 or more, got ${options.k}.`);
2318
- const groups = [];
2319
- for (const scenario of options.scenarios) {
2320
- const runs = [];
2321
- for (let attempt = 0; attempt < options.k; attempt++) try {
2322
- const result = await options.run(scenario, attempt);
2323
- runs.push({
2324
- passed: options.passed(scenario, result),
2325
- trace: result.trace,
2326
- ...result.outcome === void 0 ? {} : { outcome: result.outcome }
2327
- });
2328
- } catch (error) {
2329
- runs.push({
2330
- passed: false,
2331
- error: error instanceof Error ? error.message : String(error)
2332
- });
2333
- }
2334
- groups.push({
2335
- id: scenario.id,
2336
- scenario,
2337
- runs
2338
- });
2339
- }
2340
- return summarizeRepeats(groups);
2341
- }
2342
- function summarize$1(group) {
2343
- const passes = group.runs.filter((run) => run.passed).length;
2344
- const traces = group.runs.flatMap((run) => run.trace ? [run.trace] : []);
2345
- const paths = new Set(traces.map((trace) => traceSteps(trace).filter((step) => !step.ignored).map((step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`).join(" ")));
2346
- const reference = traces.find((trace) => trace.some((event) => event.type === "machine.transition"));
2347
- let earliest;
2348
- for (const trace of reference ? traces : []) {
2349
- if (trace === reference) continue;
2350
- const divergence = firstDivergence(trace, reference);
2351
- if (divergence && (!earliest || divergence.index < earliest.index)) earliest = divergence;
2352
- }
2353
- return {
2354
- ...group,
2355
- passes,
2356
- passAny: passes > 0,
2357
- passAll: group.runs.length > 0 && passes === group.runs.length,
2358
- distinctPaths: paths.size,
2359
- ...earliest ? { divergedAt: earliest.state } : {}
2360
- };
2361
- }
2362
- /** C(passes, j) / C(runs, j), as a product so large counts don't overflow. */
2363
- function allPass(passes, runs, j) {
2364
- let chance = 1;
2365
- for (let i = 0; i < j; i++) chance *= Math.max(0, passes - i) / (runs - i);
2366
- return chance;
2367
- }
2368
- //#endregion
2369
- //#region src/judge-agreement.ts
2370
- /**
2371
- * Measures how well a judge agrees with people on the same cases: accuracy,
2372
- * per-label recall, the confusion matrix, and the rows that disagree. With
2373
- * `positive`, also the TPR and TNR, the recommended view for binary judges
2374
- * (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
2375
- * Pure.
2376
- *
2377
- * Build rows from confirmed labels only; draft labels gate nothing. Group by
2378
- * decision state or request name so each narrow judge is aligned on its own.
2379
- * A row with no group key counts in the totals and in no group. Run it on a
2380
- * dev split and report the held-out split separately.
2381
- *
2382
- * Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
2383
- *
2384
- * @example
2385
- * ```ts no-check
2386
- * const report = judgeAgreement(
2387
- * labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
2388
- * { positive: 'fail' },
2389
- * );
2390
- * report.binary; // { tpr: 0.9, tnr: 0.8 }
2391
- * report.correct?.(0.3); // estimated true failure rate in production
2392
- * ```
2393
- */
2394
- function judgeAgreement(rows, options = {}) {
2395
- if (rows.length === 0) throw new require_event_log.AgentError("empty-agreement-rows", "judgeAgreement: rows must contain at least one row.");
2396
- const report = summarize(rows, options.positive);
2397
- const groupOf = options.groupBy ?? ((row) => row.group);
2398
- if (options.groupBy || rows.some((row) => row.group !== void 0)) {
2399
- const grouped = /* @__PURE__ */ new Map();
2400
- for (const row of rows) {
2401
- const key = groupOf(row);
2402
- if (key === void 0) continue;
2403
- const members = grouped.get(key);
2404
- if (members) members.push(row);
2405
- else grouped.set(key, [row]);
2406
- }
2407
- const groups = Object.create(null);
2408
- for (const [key, members] of grouped) groups[key] = summarize(members, options.positive);
2409
- report.groups = groups;
2410
- }
2411
- return report;
2412
- }
2413
- function summarize(rows, positive) {
2414
- const perLabel = Object.create(null);
2415
- const confusion = Object.create(null);
2416
- const disagreements = [];
2417
- let agreed = 0;
2418
- for (const { id, human, judge } of rows) {
2419
- const label = perLabel[human] ??= {
2420
- human: 0,
2421
- agreed: 0
2422
- };
2423
- const row = confusion[human] ??= Object.create(null);
2424
- label.human++;
2425
- row[judge] = (row[judge] ?? 0) + 1;
2426
- if (human === judge) {
2427
- label.agreed++;
2428
- agreed++;
2429
- } else disagreements.push({
2430
- id,
2431
- human,
2432
- judge
2433
- });
2434
- }
2435
- const summary = {
2436
- total: rows.length,
2437
- accuracy: agreed / rows.length,
2438
- perLabel,
2439
- confusion,
2440
- disagreements
2441
- };
2442
- if (positive === void 0) return summary;
2443
- let positives = 0;
2444
- let truePositives = 0;
2445
- let negatives = 0;
2446
- let trueNegatives = 0;
2447
- for (const { human, judge } of rows) if (human === positive) {
2448
- positives++;
2449
- if (judge === positive) truePositives++;
2450
- } else {
2451
- negatives++;
2452
- if (judge !== positive) trueNegatives++;
2453
- }
2454
- const tpr = positives === 0 ? void 0 : truePositives / positives;
2455
- const tnr = negatives === 0 ? void 0 : trueNegatives / negatives;
2456
- summary.binary = {
2457
- tpr,
2458
- tnr
2459
- };
2460
- if (tpr !== void 0 && tnr !== void 0 && tpr + tnr > 1) summary.correct = (p) => Math.min(1, Math.max(0, (p + tnr - 1) / (tpr + tnr - 1)));
2461
- return summary;
2462
- }
2463
- //#endregion
2464
1593
  //#region src/compare-runs.ts
2465
1594
  /**
2466
1595
  * Runs every scenario through a baseline and a candidate implementation and
@@ -2573,8 +1702,8 @@ function structurallyEqual(a, b) {
2573
1702
  * @module
2574
1703
  */
2575
1704
  /** Each function effect wrapped with `recorder.fn`, by its key. */
2576
- function recordEffects(recorder, effects) {
2577
- return require_effects.mapEffects(effects, (name, effect) => recorder.fn(name, effect));
1705
+ function recordEffects(recorder, effects, recordOptions) {
1706
+ return require_effects.mapEffects(effects, (name, effect) => recorder.fn(name, effect, recordOptions?.[name]));
2578
1707
  }
2579
1708
  async function settle(run, recorder, normalize) {
2580
1709
  try {
@@ -2591,7 +1720,7 @@ function errorIdentity(outcome, normalize) {
2591
1720
  if (error instanceof Error) return `${error.name}: ${error.message}`;
2592
1721
  if (!normalize) return String(error);
2593
1722
  try {
2594
- return stableStringify(require_effects.toJsonValue(error) ?? null);
1723
+ return stableStringify(require_run_agent.toJsonValue(error) ?? null);
2595
1724
  } catch {
2596
1725
  return String(error);
2597
1726
  }
@@ -2622,14 +1751,20 @@ function errorIdentity(outcome, normalize) {
2622
1751
  async function checkParity(options) {
2623
1752
  const rows = [];
2624
1753
  for (const scenario of options.scenarios) {
1754
+ const setup = (side) => options.setup ? options.setup(scenario, side) : void 0;
2625
1755
  const recorder = createRecorder();
2626
- const effects = recordEffects(recorder, options.effects(scenario));
1756
+ const live = await setup("before");
1757
+ const effects = recordEffects(recorder, options.effects(scenario, live), options.recordOptions);
2627
1758
  const normalize = (output) => options.normalizeOutput ? options.normalizeOutput(output, scenario) : output;
2628
- const before = await settle(() => options.before(scenario.input, effects), recorder, normalize);
1759
+ const before = await settle(() => options.before(scenario.input, effects, live), recorder, normalize);
2629
1760
  const recording = recorder.recording;
2630
- const replayer = createRecorder({ replay: recording });
2631
- const stand = recordEffects(replayer, options.effects(scenario));
2632
- const after = await settle(() => options.after(scenario.input, stand), replayer, normalize);
1761
+ const replayer = createRecorder({
1762
+ replay: recording,
1763
+ reviveError: options.reviveError
1764
+ });
1765
+ const liveAfter = await setup("after");
1766
+ const stand = recordEffects(replayer, options.effects(scenario, liveAfter), options.recordOptions);
1767
+ const after = await settle(() => options.after(scenario.input, stand, liveAfter), replayer, normalize);
2633
1768
  const replay = replayer.recording;
2634
1769
  const { divergences } = compareRecordings(recording, replay, options.compare);
2635
1770
  const normalizeError = options.normalizeError && ((error) => options.normalizeError(error, scenario));
@@ -2680,18 +1815,9 @@ exports.compareRecordings = compareRecordings;
2680
1815
  exports.compareRuns = compareRuns;
2681
1816
  exports.createRecorder = createRecorder;
2682
1817
  exports.explorePaths = require_verify.explorePaths;
2683
- exports.firstDivergence = firstDivergence;
2684
- exports.foldTraces = foldTraces;
2685
- exports.getDecisionRequests = getDecisionRequests;
2686
- exports.judgeAgreement = judgeAgreement;
2687
1818
  exports.lintAgentMachine = require_verify.lintAgentMachine;
2688
- exports.matchesTrajectory = matchesTrajectory;
2689
- exports.proposeApprovalRules = proposeApprovalRules;
2690
- exports.proposeCaps = proposeCaps;
2691
- exports.repeatRuns = repeatRuns;
1819
+ exports.matchesTrajectory = require_trajectory.matchesTrajectory;
2692
1820
  exports.runSeam = runSeam;
2693
1821
  exports.sameModelCallShape = sameModelCallShape;
2694
1822
  exports.simulateAgent = require_verify.simulateAgent;
2695
- exports.summarizeRepeats = summarizeRepeats;
2696
1823
  exports.textOutput = require_verify.textOutput;
2697
- exports.traceCoverage = traceCoverage;