@statelyai/agent 2.0.0-alpha.37 → 2.0.0-alpha.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/CHANGELOG.md +51 -0
  2. package/dist/{act-CrX0JwWY.cjs → act-B0CVbmo4.cjs} +1 -1
  3. package/dist/{act-B9hLQls6.mjs → act-Bsz-hUfD.mjs} +1 -1
  4. package/dist/ai-sdk.cjs +4 -4
  5. package/dist/ai-sdk.d.cts +2 -3
  6. package/dist/ai-sdk.d.mts +2 -3
  7. package/dist/ai-sdk.mjs +4 -4
  8. package/dist/{decision-m5hGmHpF.mjs → decision-B41YBx5U.mjs} +1 -1
  9. package/dist/{decision-CMeuNZ1g.cjs → decision-D8bcIGXQ.cjs} +1 -1
  10. package/dist/{decision-object-DBcLWVvv.d.cts → decision-object-BB8XV5Ec.d.cts} +1 -1
  11. package/dist/{decision-object-B5zGSPFp.d.mts → decision-object-CH_sE694.d.mts} +1 -1
  12. package/dist/effects-B9IvUPVh.mjs +23 -0
  13. package/dist/effects-DrWcUGgc.cjs +34 -0
  14. package/dist/evals.cjs +810 -0
  15. package/dist/evals.d.cts +595 -0
  16. package/dist/evals.d.mts +595 -0
  17. package/dist/evals.mjs +801 -0
  18. package/dist/{event-log-DkVlk3Rh.mjs → event-log-CEq_bVXQ.mjs} +8 -4
  19. package/dist/{event-log-Dkc-yIYu.cjs → event-log-dSOnXHuG.cjs} +7 -15
  20. package/dist/{event-log-store-Brx9tOHt.mjs → event-log-store-DLAfNM34.mjs} +1 -1
  21. package/dist/{event-log-store-BBS8so8a.d.mts → event-log-store-DYun526m.d.mts} +15 -14
  22. package/dist/{event-log-store-Cl-YWDnq.cjs → event-log-store-Dey9ufA4.cjs} +1 -1
  23. package/dist/{event-log-store-bc4ICGcS.d.cts → event-log-store-aFXK6yZX.d.cts} +15 -14
  24. package/dist/index.cjs +136 -98
  25. package/dist/index.d.cts +73 -134
  26. package/dist/index.d.mts +73 -134
  27. package/dist/index.mjs +117 -75
  28. package/dist/internal-BzlpzSux.d.cts +14 -0
  29. package/dist/internal-BzlpzSux.d.mts +14 -0
  30. package/dist/log.cjs +2 -3
  31. package/dist/log.d.cts +2 -2
  32. package/dist/log.d.mts +2 -2
  33. package/dist/log.mjs +3 -3
  34. package/dist/machines.cjs +1 -719
  35. package/dist/machines.d.cts +2 -424
  36. package/dist/machines.d.mts +2 -424
  37. package/dist/machines.mjs +2 -709
  38. package/dist/{model-source-kjGwqAV8.mjs → model-source-BrsLDvHL.mjs} +1 -1
  39. package/dist/{model-source-B4WQ8Idp.cjs → model-source-YpxjmWuX.cjs} +1 -1
  40. package/dist/openai.cjs +3 -3
  41. package/dist/openai.d.cts +2 -3
  42. package/dist/openai.d.mts +2 -3
  43. package/dist/openai.mjs +3 -3
  44. package/dist/otel.d.cts +1 -1
  45. package/dist/otel.d.mts +1 -1
  46. package/dist/policy.cjs +77 -0
  47. package/dist/policy.d.cts +143 -0
  48. package/dist/policy.d.mts +143 -0
  49. package/dist/policy.mjs +75 -0
  50. package/dist/{effects-BT_cSY0j.mjs → run-agent-BE7DOWD6.mjs} +150 -184
  51. package/dist/{run-agent-DlPzBWj-.d.mts → run-agent-DHlq87G7.d.mts} +18 -34
  52. package/dist/{effects-DDseYtJk.cjs → run-agent-De3xVFGw.cjs} +149 -201
  53. package/dist/{run-agent-30mH2moE.d.cts → run-agent-Dq0oSFRm.d.cts} +18 -34
  54. package/dist/{setup-agent-BDmNr6PC.mjs → setup-agent-CLm6QVGy.mjs} +3 -3
  55. package/dist/{setup-agent-BaNyz6Q3.d.cts → setup-agent-Cv8Hk3mX.d.cts} +12 -11
  56. package/dist/{setup-agent-C1CG_nfX.d.mts → setup-agent-DgFAhaPJ.d.mts} +12 -11
  57. package/dist/{setup-agent-CmEToB8y.cjs → setup-agent-Dv5VpuHe.cjs} +3 -3
  58. package/dist/tanstack-ai.cjs +4 -4
  59. package/dist/tanstack-ai.d.cts +2 -3
  60. package/dist/tanstack-ai.d.mts +2 -3
  61. package/dist/tanstack-ai.mjs +4 -4
  62. package/dist/testing.cjs +122 -996
  63. package/dist/testing.d.cts +36 -695
  64. package/dist/testing.d.mts +36 -695
  65. package/dist/testing.mjs +112 -977
  66. package/dist/{text-logic-BqxalgMr.d.mts → text-logic-x17uLBwA.d.cts} +238 -2
  67. package/dist/{text-logic-CSf9bqpP.d.cts → text-logic-x17uLBwA.d.mts} +238 -2
  68. package/dist/trajectory-BFjXaooB.d.cts +96 -0
  69. package/dist/trajectory-BFjXaooB.d.mts +96 -0
  70. package/dist/trajectory-BUXj_1Ue.mjs +152 -0
  71. package/dist/trajectory-CoZSv4wF.cjs +157 -0
  72. package/dist/{usage-Bm5nvpow.d.cts → usage-B4B-TQSV.d.mts} +1 -1
  73. package/dist/{usage-BnikdOca.d.mts → usage-DvGcoK6x.d.cts} +1 -1
  74. package/dist/{utils-7lSa4H-i.d.cts → utils-B8k-YPYI.d.mts} +23 -17
  75. package/dist/{utils-CBbKw2fM.d.mts → utils-DAclF-kw.d.cts} +23 -17
  76. package/dist/validate.cjs +4 -4
  77. package/dist/validate.d.cts +2 -2
  78. package/dist/validate.d.mts +2 -2
  79. package/dist/validate.mjs +4 -4
  80. package/dist/{verify-DJHfEfm6.d.cts → verify-BM0sGmsf.d.cts} +42 -6
  81. package/dist/{verify-ChyX6jq0.mjs → verify-CWlg3j7B.mjs} +153 -45
  82. package/dist/{verify-CwlombbC.cjs → verify-Dek6gHCS.cjs} +153 -45
  83. package/dist/{verify-i9Hiwc5g.d.mts → verify-H3Pr2gNw.d.mts} +42 -6
  84. package/dist/{with-model-calls-B_WLmOSl.cjs → with-model-calls-D50Ca6qW.cjs} +2 -2
  85. package/dist/{with-model-calls-BkOwdroS.mjs → with-model-calls-uY-KKliA.mjs} +2 -2
  86. package/docs/advanced.md +5 -5
  87. package/docs/choosing-a-run-mode.md +7 -7
  88. package/docs/debugging.md +6 -8
  89. package/docs/decisions.md +6 -5
  90. package/docs/environment-and-policy.md +1 -1
  91. package/docs/evals.md +57 -43
  92. package/docs/event-log.md +14 -16
  93. package/docs/from-a-loop.md +3 -3
  94. package/docs/generate-machines.md +4 -4
  95. package/docs/hosts.md +22 -29
  96. package/docs/human-in-the-loop.md +5 -8
  97. package/docs/index.md +4 -4
  98. package/docs/judgments.md +5 -7
  99. package/docs/langgraph-comparison.md +6 -6
  100. package/docs/machines-as-data.md +12 -17
  101. package/docs/machines-presets.md +17 -120
  102. package/docs/machines.md +13 -13
  103. package/docs/models-and-providers.md +19 -25
  104. package/docs/multi-agent.md +1 -1
  105. package/docs/observability.md +6 -8
  106. package/docs/persistence.md +15 -25
  107. package/docs/quickstart.md +4 -4
  108. package/docs/statechart-policy-examples.md +1 -1
  109. package/docs/steps.md +5 -6
  110. package/docs/strangler-fig.md +72 -7
  111. package/docs/text-requests.md +10 -12
  112. package/docs/thinking-in-state-machines.md +8 -8
  113. package/docs/tools.md +10 -12
  114. package/docs/upgrading.md +1 -1
  115. package/docs/usage-and-budgets.md +51 -56
  116. package/docs/validate-and-repair.md +9 -14
  117. package/docs/verify.md +53 -7
  118. package/docs/xstate-4-and-5-apps.md +19 -28
  119. package/llms.txt +1 -1
  120. package/package.json +31 -6
  121. package/readme.md +32 -24
  122. package/dist/internal-C3Lqu538.d.cts +0 -43
  123. package/dist/internal-DUcdhk-i.d.mts +0 -43
  124. package/dist/types-DQUOOaZO.d.cts +0 -238
  125. package/dist/types-DQUOOaZO.d.mts +0 -238
  126. package/skills/add-evals/SKILL.md +0 -152
  127. package/skills/add-traces/SKILL.md +0 -152
  128. package/skills/build-machine/SKILL.md +0 -84
  129. package/skills/generate-machine/SKILL.md +0 -335
  130. package/skills/graph-engineering/SKILL.md +0 -43
  131. package/skills/improve-machine/SKILL.md +0 -261
  132. package/skills/improve-machine/reference.md +0 -101
  133. package/skills/loop-to-machine/SKILL.md +0 -205
  134. package/skills/loop-to-machine/policy.md +0 -154
  135. package/skills/loop-to-machine/reference.md +0 -260
@@ -1,8 +1,9 @@
1
- import { R as AgentExecutorDecisionRequest, c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-CSf9bqpP.cjs";
2
- import { p as AgentRuntimeOptions, s as AgentRunInit, u as AgentRunResult } from "./run-agent-30mH2moE.cjs";
3
- import { n as AgentEventLogStore } from "./event-log-store-bc4ICGcS.cjs";
4
- import { A as RecordedOutput, C as RecordableDecisionModel, D as RecordedEffect, E as RecordedCall, F as compareRecordings, I as createRecorder, L as sameModelCallShape, M as Recorder, N as Recording, O as RecordedError, P as RecordingDivergenceError, S as RecordClientOptions, T as RecordableModel, _ as EquivalenceReport, b as NormalizeEffect, g as EffectDivergenceReason, h as EffectDivergence, j as RecordedToolCall, k as RecordedModelCall, m as CreateRecorderOptions, p as CompareRecordingsOptions, t as MetaOfSnapshot, v as ModelAnswer, w as RecordableLanguageModel, x as RecordCallOptions, y as ModelCallMatch } from "./utils-7lSa4H-i.cjs";
5
- import { S as textOutput, _ as assertAgentMachine, a as AgentPathTerminal, b as lintAgentMachine, c as CanReachResult, d as ExploreSnapshot, f as LintAgentMachineOptions, g as SimulationTrailEntry, h as SimulationScript, i as AgentPathReport, l as CanReachTarget, m as SimulateAgentResult, n as AgentLintError, o as AgentUnknownStateError, p as SimulateAgentOptions, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as ExplorePathsOptions, v as canReach, x as simulateAgent, y as explorePaths } from "./verify-DJHfEfm6.cjs";
1
+ import { c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-x17uLBwA.cjs";
2
+ import { C as simulateAgent, S as lintAgentMachine, _ as SimulationScript, a as AgentPathTerminal, b as canReach, c as CanReachOrdering, d as ExplorePathsOptions, f as ExploreSnapshot, g as SimulateAgentResult, h as SimulateAgentOptions, i as AgentPathReport, l as CanReachResult, m as LintAgentMachineOptions, n as AgentLintError, o as AgentUnknownStateError, p as ExploreStep, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as CanReachTarget, v as SimulationTrailEntry, w as textOutput, x as explorePaths, y as assertAgentMachine } from "./verify-BM0sGmsf.cjs";
3
+ import { a as TrajectoryMatch, i as TrajectoryItem, n as TrajectoryEvent, o as TrajectoryMiss, r as TrajectoryEventEnvelope, s as matchesTrajectory, t as MatchTrajectoryOptions } from "./trajectory-BFjXaooB.cjs";
4
+ import { p as AgentRuntimeOptions, s as AgentRunInit, u as AgentRunResult } from "./run-agent-Dq0oSFRm.cjs";
5
+ import { n as AgentEventLogStore } from "./event-log-store-aFXK6yZX.cjs";
6
+ import { A as RecordedToolCall, C as RecordableLanguageModel, D as RecordedError, E as RecordedEffect, F as createRecorder, I as sameModelCallShape, M as Recording, N as RecordingDivergenceError, O as RecordedModelCall, P as compareRecordings, S as RecordableDecisionModel, T as RecordedCall, _ as ModelAnswer, b as RecordCallOptions, f as CompareRecordingsOptions, g as EquivalenceReport, h as EffectDivergenceReason, j as Recorder, k as RecordedOutput, m as EffectDivergence, p as CreateRecorderOptions, t as MetaOfSnapshot, v as ModelCallMatch, w as RecordableModel, x as RecordClientOptions, y as NormalizeEffect } from "./utils-DAclF-kw.cjs";
6
7
  import { AnyActorLogic, AnyStateMachine, EventFromLogic, InputFrom, SnapshotFrom, StateValue } from "xstate";
7
8
  //#region src/crash-recovery.d.ts
8
9
  /** Options for {@link checkCrashRecovery}. */
@@ -166,100 +167,6 @@ interface CrashRecoveryReport {
166
167
  */
167
168
  export declare function checkCrashRecovery<TMachine extends AnyStateMachine>(machine: TMachine, options?: CheckCrashRecoveryOptions<TMachine>): Promise<CrashRecoveryReport>;
168
169
  //#endregion
169
- //#region src/trajectory.d.ts
170
- /** An event-shaped trajectory item: anything with a string `type`. */
171
- interface TrajectoryEvent {
172
- type: string;
173
- [key: string]: unknown;
174
- }
175
- /** Any host-owned envelope around an event. */
176
- interface TrajectoryEventEnvelope {
177
- event: TrajectoryEvent;
178
- }
179
- /**
180
- * One item in a trajectory, on either side of the comparison:
181
- *
182
- * - a state value — `'drafting'`, a dot path `'review.editing'`, or the nested
183
- * object XState reports (`{ review: 'editing' }`),
184
- * - an event object, or just the event type as a string,
185
- * - a machine snapshot, whose `value` is used.
186
- */
187
- type TrajectoryItem = StateValue | TrajectoryEvent | TrajectoryEventEnvelope | {
188
- value: StateValue;
189
- };
190
- /** Options for {@link matchesTrajectory}. */
191
- interface MatchTrajectoryOptions {
192
- /**
193
- * Require the trajectories to be equal: same length, item for item. Default
194
- * `false` — expected items must appear in `actual` in order, gaps allowed.
195
- */
196
- exact?: boolean;
197
- }
198
- /** The first expected item that could not be found. */
199
- interface TrajectoryMiss {
200
- /** Its position in `expected`. */
201
- index: number;
202
- /** The expected item itself. */
203
- expected: TrajectoryItem;
204
- /** Where the search for it started in `actual` (everything before was already consumed). */
205
- searchedFrom: number;
206
- }
207
- /**
208
- * The result of {@link matchesTrajectory}: a boolean for tests, plus enough
209
- * detail for a scorer's partial credit and a failure message's diagnosis.
210
- */
211
- interface TrajectoryMatch {
212
- /** Every expected item was found, in order (and, in `exact` mode, nothing else). */
213
- matched: boolean;
214
- /** How many expected items were matched before the first miss. */
215
- matchedCount: number;
216
- /** `expected.length`, so a scorer can compute its own ratio. */
217
- expectedCount: number;
218
- /**
219
- * Partial credit, 0..1. `1` for an empty expectation. Denominator is
220
- * `max(expectedCount, actual.length)` in `exact` mode, `expectedCount` otherwise.
221
- */
222
- score: number;
223
- /** Absent when `matched`. */
224
- firstMiss?: TrajectoryMiss;
225
- }
226
- /**
227
- * Matches a run's trajectory against an expected one.
228
- *
229
- * Both trajectories may be state values or events collected from
230
- * `onTransition`;
231
- * items are compared by shape, so `['prompting', 'drafting']` scores a state
232
- * path and `['PROMPT_SUBMITTED', 'SEND']` scores an event log with the same
233
- * call.
234
- *
235
- * Semantics are an ordered subsequence: every expected item must appear in
236
- * `actual`, in order, with gaps allowed. Pass `{ exact: true }` to require
237
- * equality instead.
238
- *
239
- * @example Trajectory assertions in a test
240
- * ```ts
241
- * const statePath: unknown[] = [];
242
- * const result = await runToQuiescence(
243
- * createAgentRuntime(machine, {
244
- * executors,
245
- * onTransition: (snapshot) => statePath.push(snapshot.value),
246
- * }),
247
- * { input },
248
- * );
249
- *
250
- * const path = matchesTrajectory(statePath, ['prompting', 'drafting', 'sent']);
251
- * expect(path.matched, JSON.stringify(path.firstMiss)).toBe(true);
252
- * expect(matchesTrajectory(events, ['PROMPT_SUBMITTED', 'SEND']).matched).toBe(true);
253
- * ```
254
- *
255
- * @example Partial credit in an eval scorer
256
- * ```ts
257
- * const match = matchesTrajectory(output.statePath, expected.statePath);
258
- * return { name: 'state_path', score: match.score, metadata: { ...match.firstMiss } };
259
- * ```
260
- */
261
- export declare function matchesTrajectory(actual: readonly unknown[], expected: readonly TrajectoryItem[], options?: MatchTrajectoryOptions): TrajectoryMatch;
262
- //#endregion
263
170
  //#region src/seam.d.ts
264
171
  /**
265
172
  * One scripted answer in a {@link RunSeamOptions.scripts} queue: the request's
@@ -440,596 +347,6 @@ interface RunSeamResult<TMachine extends AnyStateMachine> {
440
347
  */
441
348
  export declare function runSeam<TMachine extends AnyStateMachine>(machine: TMachine, options: RunSeamOptions<TMachine>): Promise<RunSeamResult<TMachine>>;
442
349
  //#endregion
443
- //#region src/fold-traces.d.ts
444
- /** One state's numbers across many runs. See {@link foldTraces}. */
445
- interface StateStats {
446
- /** The state's dot-joined path from the root, e.g. `'deciding'` or `'work.review'`. */
447
- state: string;
448
- /** Runs that were in this state at least once. */
449
- visits: number;
450
- /** Of those runs, the ones `failed` flagged. */
451
- failures: number;
452
- /** Options withheld by the guards at decisions made in this state, summed over runs. */
453
- withheld: number;
454
- /** Events this state ignored (`machine.transition` with `ignored`), summed over runs. */
455
- ignored: number;
456
- /** Failed runs `blame` attributed to this state. 0 without `blame`. */
457
- blamed: number;
458
- }
459
- /** Options for {@link foldTraces}. */
460
- interface FoldTracesOptions<TTrace> {
461
- /** Whether a run failed, judged from its whole trace. */
462
- failed: (trace: TTrace) => boolean;
463
- /**
464
- * The state a failed run's failure started in, usually
465
- * `firstDivergence(trace, reference)?.state`. Called for failed runs only;
466
- * `undefined` blames no state.
467
- */
468
- blame?: (trace: TTrace) => string | undefined;
469
- }
470
- /**
471
- * Folds many runs' traces into per-state stats: how many runs visited each
472
- * state, how many of those failed, how many options the guards withheld
473
- * at decisions made there, how many events it ignored, and how many failures
474
- * `blame` traced back to it. Pure; reads only:
475
- *
476
- * - `machine.transition`: the run visited `from.value` and `snapshot.value`;
477
- * with `ignored`, each of those states ignored one event.
478
- * - `request.start` of a decision, first attempt only: its `request.withheld`
479
- * counts toward the state the run was in, the `from.value` of the
480
- * transition its choice causes.
481
- *
482
- * A trace is one run's events, live (`AgentTraceEvent`) or serialized
483
- * (`serializeTraceEvent`). A host-driven `decideNext` traces nothing; push
484
- * `{ type: 'request.start', request: result.request }` onto the trace before
485
- * sending its event. Parallel states count once per active leaf.
486
- *
487
- * `failures` counts a failed run against every state it visited, so states
488
- * after a bad decision rank as high as the decision. Sort by `blamed` to see
489
- * where failures start.
490
- *
491
- * @example
492
- * ```ts no-check
493
- * const stats = foldTraces(traces, {
494
- * failed: (trace) => trace.at(-1)?.type === 'run.end' && trace.at(-1).status === 'error',
495
- * });
496
- * ```
497
- */
498
- export declare function foldTraces<TTrace extends readonly {
499
- readonly type: string;
500
- }[]>(traces: Iterable<TTrace>, options: FoldTracesOptions<TTrace>): StateStats[];
501
- //#endregion
502
- //#region src/trace-coverage.d.ts
503
- /** One state's row in a {@link CoverageReport}. */
504
- interface StateCoverage {
505
- /** The state's dot-joined path from the root, e.g. `'deciding'` or `'work.review'`. */
506
- state: string;
507
- /** Runs that were in this state at least once. */
508
- runs: number;
509
- }
510
- /** A handler: the state node that declares it and its event descriptor. */
511
- interface HandlerRef {
512
- /** The declaring state's dot path; `''` for the machine root. */
513
- state: string;
514
- /** The descriptor as XState keys it: `'APPROVE'`, `'task.*'`, `'*'`, `'xstate.done.actor'`, `'xstate.error.actor'`, `'xstate.after'`. */
515
- event: string;
516
- }
517
- /** One handler's row in a {@link CoverageReport}. */
518
- interface HandlerCoverage extends HandlerRef {
519
- /** Runs in which this handler was credited at least once. */
520
- runs: number;
521
- targets: {
522
- /** Target state paths of the object transitions under this descriptor; `'dynamic'` when any is a function transition. */
523
- declared: string[] | "dynamic";
524
- /** Active leaves observed right after a step credited to this handler. */
525
- seen: string[];
526
- /** Declared targets that no seen leaf is in. Empty when `declared` is `'dynamic'`. */
527
- unreached: string[];
528
- };
529
- }
530
- /** The result of {@link traceCoverage}. */
531
- interface CoverageReport {
532
- /** Every atomic and final state, in document order, including `runs: 0`. */
533
- states: StateCoverage[];
534
- /** Every observable handler, in document order, including `runs: 0`. */
535
- handlers: HandlerCoverage[];
536
- /** States no run visited, excluding `unreachable` ones. */
537
- unvisitedStates: string[];
538
- /** Handlers no run was credited with. */
539
- unvisitedHandlers: HandlerRef[];
540
- /** `always` handlers: they fire inside a macrostep, which a trace does not record. */
541
- unobservable: HandlerRef[];
542
- /**
543
- * With a complete `reachable` report: states no explored path reached and
544
- * no trace visited. They leave the state denominator.
545
- */
546
- unreachable?: string[];
547
- /**
548
- * With `reachable`: whether the report explored every path. When `false`,
549
- * a state it missed may still be reachable, so `unreachable` is absent and
550
- * every state stays in the denominator.
551
- */
552
- reachableComplete?: boolean;
553
- /** Visited over total, for states and for handlers (unobservable excluded). 1 when the total is 0. */
554
- ratio: {
555
- states: number;
556
- handlers: number;
557
- };
558
- }
559
- /** Options for {@link traceCoverage}. */
560
- interface TraceCoverageOptions {
561
- /**
562
- * An `explorePaths` report. When it explored every path (no cap hit, every
563
- * payload expanded, no branch stopped early), states it never reached drop
564
- * out of the state denominator.
565
- */
566
- reachable?: AgentPathReport;
567
- }
568
- /**
569
- * Measures how much of a machine a set of runs exercised: which atomic and
570
- * final states they visited, and which handlers they took. A handler is a
571
- * `(declaring state, event descriptor)` pair from `on`, invoke
572
- * `onDone`/`onError`, and `after`, with the targets it declares and the
573
- * leaves the runs landed in after it. Pure; reads only `machine.transition`
574
- * events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
575
- *
576
- * A step credits one handler per active leaf: the nearest node, from the leaf
577
- * up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
578
- * event, preferred in XState's order. Guards are not replayed, so a step
579
- * whose nearest handler was rejected and fell through to an ancestor is
580
- * credited to the nearest one. XState keys invoke and `after` handlers by
581
- * bare type (`xstate.done.actor`), one per state however many invokes; a
582
- * definition's `matches` (an invoke's `actorId`) must accept the traced event
583
- * for its state to be credited. Ignored steps credit no handler. `always`
584
- * handlers are listed as `unobservable`, never as unvisited; a state one
585
- * passes through is visited, re-derived from each step's microsteps.
586
- *
587
- * @example
588
- * ```ts no-check
589
- * const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
590
- * expect(report.unvisitedHandlers).toEqual([]);
591
- * ```
592
- */
593
- export declare function traceCoverage<TTrace extends readonly {
594
- readonly type: string;
595
- }[]>(machine: AnyStateMachine, traces: Iterable<TTrace>, options?: TraceCoverageOptions): CoverageReport;
596
- //#endregion
597
- //#region src/internal/trace-states.d.ts
598
- /** One macrostep read from a trace: a `machine.transition` event. */
599
- interface TraceStep {
600
- /** Active leaves before the step. */
601
- from: string[];
602
- /** The state value before the step, as traced. */
603
- fromValue: unknown;
604
- /** The event's `type`. */
605
- event: string;
606
- /** The event object, as traced. */
607
- payload: unknown;
608
- /** Active leaves after the step. */
609
- to: string[];
610
- /** The state value after the step, as traced. */
611
- toValue: unknown;
612
- /** The state had no transition for a host event, so nothing changed. */
613
- ignored: boolean;
614
- /** The `name` of the first-attempt decision request that started since the previous step. */
615
- decision?: string;
616
- }
617
- //#endregion
618
- //#region src/first-divergence.d.ts
619
- /** Options for {@link firstDivergence}. */
620
- interface FirstDivergenceOptions {
621
- /**
622
- * Projects a step to the key two traces are compared by. Default
623
- * `from|event|to`. Override to leave out what may differ between equivalent
624
- * runs, such as a parallel region that only logs. Steps are still compared
625
- * in order. Trace references only.
626
- *
627
- * `"payload"` also compares the event's payload, such as an actor's output,
628
- * as key-order-independent JSON without its `sessionId`, and with errors as
629
- * `{ name, message, code?, cause? }` (an `Error`, or a serialized one with `stack` or
630
- * `code`). Use it for deterministic or mocked runs; live model outputs usually differ
631
- * between runs.
632
- */
633
- key?: ((step: TraceStep) => string) | "payload";
634
- }
635
- /** Where a run first departs from its reference. See {@link firstDivergence}. */
636
- interface Divergence {
637
- /** The step's index in the run's steps, or the step count when the run ended early. */
638
- index: number;
639
- /** The state the run was in when it departed: the first active leaf before that step. */
640
- state: string;
641
- /** The decision request that chose that step's event, when one did. */
642
- decision?: string;
643
- /** The run's step. Absent when the run ended early. */
644
- actual?: TraceStep;
645
- /** The reference's step, or the expected trajectory item the run never reached. Absent when the run went on past a reference trace. */
646
- expected?: TraceStep | TrajectoryItem;
647
- }
648
- /**
649
- * Finds the first step where a run departs from a reference, so a failure is
650
- * blamed on the state it started in rather than on every state the run went
651
- * through afterwards.
652
- *
653
- * - Against a reference **trace**, such as a passing run of the same
654
- * scenario, steps are compared in order by `key`. The first step that
655
- * differs, or the point where either trace ends early, is the divergence.
656
- * - Against an expected **trajectory** (`TrajectoryItem[]`, as for
657
- * {@link matchesTrajectory}), the run is laid out as its initial state, then
658
- * each step's event and new state. The first expected item it never reaches
659
- * is the divergence, blamed on the step after the last item it did reach. A
660
- * run that goes on past the trajectory has not diverged.
661
- *
662
- * An empty reference is a trace with no steps, so any step diverges from it.
663
- * A trace is one run's events, live or serialized. Steps the machine ignored
664
- * changed nothing and are left out on both sides. Returns `undefined` when the
665
- * run does not diverge.
666
- *
667
- * @example
668
- * ```ts no-check
669
- * const stats = foldTraces(traces, {
670
- * failed,
671
- * blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
672
- * });
673
- * ```
674
- */
675
- export declare function firstDivergence(actual: readonly {
676
- readonly type: string;
677
- }[], reference: readonly {
678
- readonly type: string;
679
- }[] | readonly TrajectoryItem[], options?: FirstDivergenceOptions): Divergence | undefined;
680
- //#endregion
681
- //#region src/propose-from-traces.d.ts
682
- /** A person's verdict as {@link proposeApprovalRules} hands it to `key`. */
683
- interface ApprovalVerdict<TTrace> {
684
- /** The approve or reject event, as traced. */
685
- event: Record<string, unknown>;
686
- /** The traced snapshot the verdict arrived in: the approval state, with the context that holds the pending call. */
687
- snapshot: unknown;
688
- /** The whole run. */
689
- trace: TTrace;
690
- }
691
- /** Options for {@link proposeApprovalRules}. */
692
- interface ProposeApprovalRulesOptions<TTrace> {
693
- /** The state's dot path, such as `'awaitingApproval'`. A verdict counts when the run was in it or in a state inside it. */
694
- approvalState: string;
695
- /** Event types that approve, such as `['APPROVE']`. */
696
- approveEvents: readonly string[];
697
- /** Event types that reject, such as `['REJECT']`. */
698
- rejectEvents: readonly string[];
699
- /**
700
- * The rule a verdict is evidence for: usually the tool name and a pattern
701
- * of its arguments, read from `snapshot.context`. `undefined` skips the verdict.
702
- */
703
- key: (verdict: ApprovalVerdict<TTrace>) => string | undefined;
704
- /**
705
- * The keys a run broke a rule for, from ground truth (a ledger checked
706
- * after the fact). Any violation blocks `allow` for that key.
707
- */
708
- violations?: (trace: TTrace) => Iterable<string>;
709
- }
710
- /** One key's row from {@link proposeApprovalRules}. */
711
- interface ApprovalRuleProposal {
712
- key: string;
713
- /** Approve verdicts, summed over runs. */
714
- approved: number;
715
- /** Reject verdicts, summed over runs. */
716
- rejected: number;
717
- /** Violations `violations` reported, summed over runs. 0 without `violations`. */
718
- violations: number;
719
- /** Indexes, in input order, of the runs with a verdict for this key. */
720
- runs: number[];
721
- /**
722
- * `'allow'`: approved, never rejected, no violation. `'deny'`: rejected,
723
- * never approved. `null`: mixed evidence.
724
- */
725
- proposal: "allow" | "deny" | null;
726
- /** Runs with a verdict for this key: `runs.length`. */
727
- support: number;
728
- }
729
- /**
730
- * Groups the approval verdicts in many runs' traces by `key` and proposes an
731
- * `allow` or `deny` rule for each. Pure and deterministic; a proposal is for
732
- * a person to review, and nothing here changes a machine.
733
- *
734
- * Reads `machine.transition` events: a verdict is a transition, not ignored,
735
- * out of `approvalState` on an approve or reject event. Rows come in the
736
- * order keys were first seen.
737
- *
738
- * @example
739
- * ```ts no-check
740
- * const rows = proposeApprovalRules(traces, {
741
- * approvalState: "awaitingApproval",
742
- * approveEvents: ["APPROVE"],
743
- * rejectEvents: ["REJECT"],
744
- * key: ({ snapshot }) => `${snapshot.context.pending.tool}:${snapshot.context.pending.path}`,
745
- * });
746
- * ```
747
- */
748
- export declare function proposeApprovalRules<TTrace extends readonly {
749
- readonly type: string;
750
- }[]>(traces: Iterable<TTrace>, options: ProposeApprovalRulesOptions<TTrace>): ApprovalRuleProposal[];
751
- /** One cap for {@link proposeCaps}. Give `failureState` or `failureOutput`. */
752
- interface CapSpec<TTrace> {
753
- /** Names the row, such as `'retries'`. */
754
- name: string;
755
- /** The cap's value now. A cap runs hit is never proposed below it. */
756
- current?: number;
757
- /** The run hit the cap when it entered this state (dot path), such as `'gaveUp'`. */
758
- failureState?: string;
759
- /** The run hit the cap when its `run.end` output satisfies this. */
760
- failureOutput?: (output: unknown) => boolean;
761
- /** How much of the cap the run used: retries made, steps taken, batch size. `undefined` leaves the run out. */
762
- valueFromRun: (trace: TTrace) => number | undefined;
763
- }
764
- /** One cap's row from {@link proposeCaps}. */
765
- interface CapProposal {
766
- name: string;
767
- /** Runs that hit the cap. */
768
- hits: number;
769
- /** Runs that did not, with a value. */
770
- passing: number;
771
- /** The largest value over all runs; `null` with none. */
772
- max: number | null;
773
- /** The 95th percentile (nearest rank) over passing runs; `null` with none. */
774
- p95: number | null;
775
- /** `p95 + 1`, and never below `current` when any run hit the cap; `null` with no passing run. */
776
- proposal: number | null;
777
- }
778
- /** Options for {@link proposeCaps}. */
779
- interface ProposeCapsOptions<TTrace> {
780
- caps: readonly CapSpec<TTrace>[];
781
- }
782
- /**
783
- * Proposes a value for each cap (retry count, steps, batch size) from many
784
- * runs' traces: the 95th percentile of what passing runs used, plus one, and
785
- * never below the current cap while runs still hit it. Pure and
786
- * deterministic; a proposal is for a person to review, and nothing here
787
- * changes a machine.
788
- *
789
- * @example
790
- * ```ts no-check
791
- * const [retries] = proposeCaps(traces, {
792
- * caps: [{ name: "retries", current: 3, failureState: "gaveUp", valueFromRun: countRetries }],
793
- * });
794
- * ```
795
- */
796
- export declare function proposeCaps<TTrace extends readonly {
797
- readonly type: string;
798
- }[]>(traces: Iterable<TTrace>, options: ProposeCapsOptions<TTrace>): CapProposal[];
799
- //#endregion
800
- //#region src/decision-requests.d.ts
801
- /** One decision a run asked for. See {@link getDecisionRequests}. */
802
- interface TracedDecisionRequest {
803
- /** The active states when the decision was asked, as dot paths (one per parallel region). */
804
- states: string[];
805
- /** The request as the executor received it, on its first attempt. */
806
- request: AgentExecutorDecisionRequest;
807
- }
808
- /** Options for {@link getDecisionRequests}. */
809
- interface GetDecisionRequestsOptions {
810
- /** Keep only requests with this `name` (the `agent.decide` input's `name`). */
811
- name?: string;
812
- }
813
- /**
814
- * The decisions a run asked for, read from its trace: each decision's first
815
- * `request.start`, with the states the run was in. A trace is one run's events,
816
- * live or serialized with `serializeTraceEvent`. Retries of the same decision
817
- * are left out, so each decision appears once.
818
- *
819
- * Use the requests as eval rows (run a decider on each and compare its event
820
- * with a label) or as probe inputs for a policy. A serialized request keeps its
821
- * `options`, `metadata`, `system`, and `prompt`; event schemas do not survive
822
- * serialization, so replay closed choices (`options`) rather than open ones.
823
- *
824
- * @example
825
- * ```ts no-check
826
- * const rows = traces.flatMap((trace) => getDecisionRequests(trace, { name: "chooseResolution" }));
827
- * for (const { request } of rows) console.log((await decide(request)).event);
828
- * ```
829
- */
830
- export declare function getDecisionRequests(trace: Iterable<{
831
- readonly type: string;
832
- }>, options?: GetDecisionRequestsOptions): TracedDecisionRequest[];
833
- //#endregion
834
- //#region src/repeat-runs.d.ts
835
- type AnyTrace = readonly {
836
- readonly type: string;
837
- }[];
838
- /** One run of a repeated scenario. See {@link summarizeRepeats}. */
839
- interface RepeatRun<TTrace extends AnyTrace = AnyTrace> {
840
- passed: boolean;
841
- /** The run's trace. Absent for a run that threw. */
842
- trace?: TTrace;
843
- /** How the run ended, for the report. Not compared. */
844
- outcome?: string;
845
- /** The message of a run that threw. */
846
- error?: string;
847
- }
848
- /** One scenario's runs. See {@link summarizeRepeats}. */
849
- interface RepeatGroup<TScenario, TTrace extends AnyTrace = AnyTrace> {
850
- id: string;
851
- scenario: TScenario;
852
- runs: readonly RepeatRun<TTrace>[];
853
- }
854
- /** One scenario's row in a {@link RepeatReport}. */
855
- interface RepeatResult<TScenario, TTrace extends AnyTrace = AnyTrace> extends RepeatGroup<TScenario, TTrace> {
856
- /** Runs that passed. */
857
- passes: number;
858
- /** At least one run passed: pass@k for this scenario. */
859
- passAny: boolean;
860
- /** Every run passed: pass^k for this scenario. */
861
- passAll: boolean;
862
- /** Distinct state paths the traced runs took. 1 when every run went the same way. */
863
- distinctPaths: number;
864
- /** The state where the runs first went different ways, judged against the first run whose trace has a transition. */
865
- divergedAt?: string;
866
- }
867
- /** The result of {@link summarizeRepeats} and {@link repeatRuns}. */
868
- interface RepeatReport<TScenario, TTrace extends AnyTrace = AnyTrace> {
869
- scenarios: RepeatResult<TScenario, TTrace>[];
870
- /**
871
- * The suite's pass^j: the chance that `j` runs of a scenario, drawn from the
872
- * ones recorded, all pass, averaged over scenarios. The unbiased estimate is
873
- * C(passes, j) / C(runs, j) per scenario. Scenarios with fewer than `j` runs
874
- * are left out. Throws when `j` is not a whole number from 1 to the most
875
- * runs any scenario has.
876
- */
877
- passHat: (j: number) => number;
878
- /** Every recorded trace, ready for `foldTraces` or `traceCoverage`. */
879
- traces: TTrace[];
880
- }
881
- /** Options for {@link repeatRuns}. */
882
- interface RepeatRunsOptions<TScenario extends {
883
- id: string;
884
- }, TTrace extends AnyTrace> {
885
- scenarios: readonly TScenario[];
886
- /** Runs per scenario. */
887
- k: number;
888
- /** Runs one scenario once and returns its trace. `attempt` counts from 0. */
889
- run: (scenario: TScenario, attempt: number) => Promise<{
890
- trace: TTrace;
891
- outcome?: string;
892
- }>;
893
- /** Whether a run passed, judged from its ground truth. */
894
- passed: (scenario: TScenario, result: {
895
- trace: TTrace;
896
- outcome?: string;
897
- }) => boolean;
898
- }
899
- /**
900
- * Summarizes repeated runs of each scenario: how often each passed, whether
901
- * all its runs passed (pass^k), how many distinct state paths they took, and
902
- * where they first went different ways. Pure. Use it for runs collected
903
- * elsewhere, such as CI shards or replays; {@link repeatRuns} collects them and
904
- * calls it.
905
- *
906
- * A scenario that passes 3 of 5 runs and `divergedAt: 'deciding'` points at one
907
- * decision, not at a flaky agent.
908
- *
909
- * @example
910
- * ```ts no-check
911
- * const report = summarizeRepeats(shards.map((s) => ({ id: s.id, scenario: s, runs: s.runs })));
912
- * report.passHat(3); // the chance 3 runs of a scenario all pass
913
- * ```
914
- */
915
- export declare function summarizeRepeats<TScenario, TTrace extends AnyTrace = AnyTrace>(groups: readonly RepeatGroup<TScenario, TTrace>[]): RepeatReport<TScenario, TTrace>;
916
- /**
917
- * Runs every scenario `k` times and summarizes the runs with
918
- * {@link summarizeRepeats}. Runs are sequential, scenario by scenario. A run
919
- * that throws fails, with its message as `error` and no trace.
920
- *
921
- * With a live model, one run per scenario says little: τ-bench found pass^k
922
- * falls well below the single-run rate. Report pass^k next to it.
923
- *
924
- * @example
925
- * ```ts no-check
926
- * const report = await repeatRuns({
927
- * scenarios,
928
- * k: 5,
929
- * run: async (scenario) => {
930
- * const trace: AgentTraceEvent[] = [];
931
- * const result = await runToQuiescence(
932
- * createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
933
- * { input: scenario.input },
934
- * );
935
- * return { trace, outcome: String(result.snapshot.value) };
936
- * },
937
- * passed: (scenario, { outcome }) => outcome === scenario.expected,
938
- * });
939
- * report.passHat(5);
940
- * ```
941
- */
942
- export declare function repeatRuns<TScenario extends {
943
- id: string;
944
- }, TTrace extends AnyTrace>(options: RepeatRunsOptions<TScenario, TTrace>): Promise<RepeatReport<TScenario, TTrace>>;
945
- //#endregion
946
- //#region src/judge-agreement.d.ts
947
- /** One labeled case: what a person said and what the judge said. See {@link judgeAgreement}. */
948
- interface AgreementRow {
949
- /** Identifies the case in `disagreements`. */
950
- id: string;
951
- /** The confirmed human label. */
952
- human: string;
953
- /** The judge's label for the same case. */
954
- judge: string;
955
- /** The group the row is aligned in, e.g. a decision state or request name. `groupBy` overrides it. */
956
- group?: string;
957
- }
958
- /** Agreement numbers for one set of rows. See {@link AgreementReport}. */
959
- interface AgreementSummary {
960
- /** Rows counted. */
961
- total: number;
962
- /** Share of rows where `judge` equals `human`. */
963
- accuracy: number;
964
- /** Per human label: rows with that label, and how many the judge matched. `agreed / human` is that label's recall. */
965
- perLabel: Record<string, {
966
- human: number;
967
- agreed: number;
968
- }>;
969
- /** Counts by human label, then judge label. */
970
- confusion: Record<string, Record<string, number>>;
971
- /** Rows where the judge differs from the person, in input order. */
972
- disagreements: {
973
- id: string;
974
- human: string;
975
- judge: string;
976
- }[];
977
- /**
978
- * With `positive` set: the true positive rate (judge says positive when the
979
- * person did) and true negative rate (judge says any other label when the
980
- * person did). A rate is `undefined` when no row has that human side.
981
- */
982
- binary?: {
983
- tpr: number | undefined;
984
- tnr: number | undefined;
985
- };
986
- /**
987
- * With `positive` set: estimates the true share of positives among
988
- * unlabeled rows from the share `p` the judge labeled positive,
989
- * θ = (p + TNR − 1) / (TPR + TNR − 1), clamped to [0, 1]. Omitted when
990
- * either rate is `undefined`, or when TPR + TNR ≤ 1: the judge is no better
991
- * than chance, so its labels carry no signal to correct.
992
- */
993
- correct?: (observedPositiveRate: number) => number;
994
- }
995
- /** The result of {@link judgeAgreement}. */
996
- interface AgreementReport extends AgreementSummary {
997
- /** The same numbers per group. Present when any row has `group` or `groupBy` is given. */
998
- groups?: Record<string, AgreementSummary>;
999
- }
1000
- /** Options for {@link judgeAgreement}. */
1001
- interface JudgeAgreementOptions<TRow extends AgreementRow> {
1002
- /** The positive label. Every other label is negative. Turns on `binary` and `correct`. */
1003
- positive?: string;
1004
- /** The row's group, overriding `row.group`. Return `undefined` to leave the row out of every group. */
1005
- groupBy?: (row: TRow) => string | undefined;
1006
- }
1007
- /**
1008
- * Measures how well a judge agrees with people on the same cases: accuracy,
1009
- * per-label recall, the confusion matrix, and the rows that disagree. With
1010
- * `positive`, also the TPR and TNR, the recommended view for binary judges
1011
- * (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
1012
- * Pure.
1013
- *
1014
- * Build rows from confirmed labels only; draft labels gate nothing. Group by
1015
- * decision state or request name so each narrow judge is aligned on its own.
1016
- * A row with no group key counts in the totals and in no group. Run it on a
1017
- * dev split and report the held-out split separately.
1018
- *
1019
- * Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
1020
- *
1021
- * @example
1022
- * ```ts no-check
1023
- * const report = judgeAgreement(
1024
- * labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
1025
- * { positive: 'fail' },
1026
- * );
1027
- * report.binary; // { tpr: 0.9, tnr: 0.8 }
1028
- * report.correct?.(0.3); // estimated true failure rate in production
1029
- * ```
1030
- */
1031
- export declare function judgeAgreement<TRow extends AgreementRow>(rows: readonly TRow[], options?: JudgeAgreementOptions<TRow>): AgreementReport;
1032
- //#endregion
1033
350
  //#region src/compare-runs.d.ts
1034
351
  /** One run of one scenario, reduced to what parity compares. See {@link compareRuns}. */
1035
352
  interface ParityRun {
@@ -1124,7 +441,20 @@ type CheckParityOutcome<TOutput = unknown> = {
1124
441
  error: unknown;
1125
442
  };
1126
443
  /** Options for {@link checkParity}. */
1127
- interface CheckParityOptions<TScenario extends CheckParityScenario, TEffects extends object, TOutput> {
444
+ type CheckParityOptions<TScenario extends CheckParityScenario, TEffects extends object, TOutput, TLive = undefined> = CheckParityBaseOptions<TScenario, TEffects, TOutput, TLive> & (undefined extends TLive ? {
445
+ setup?: CheckParitySetup<TScenario, TLive>;
446
+ } : {
447
+ setup: CheckParitySetup<TScenario, TLive>;
448
+ });
449
+ /**
450
+ * Builds the live objects one side of a scenario runs on (a class instance, a
451
+ * session), passed to `effects`, `before` and `after`. Called once for
452
+ * `before` and once for `after`, so a mutable object does not carry state from
453
+ * one side to the other.
454
+ */
455
+ type CheckParitySetup<TScenario extends CheckParityScenario, TLive> = (scenario: TScenario, side: "before" | "after") => TLive | PromiseLike<TLive>;
456
+ /** {@link CheckParityOptions} without `setup`. */
457
+ interface CheckParityBaseOptions<TScenario extends CheckParityScenario, TEffects extends object, TOutput, TLive = undefined> {
1128
458
  scenarios: readonly TScenario[];
1129
459
  /**
1130
460
  * Fresh effects for one scenario: fakes, stubs, or real clients. `before`
@@ -1133,11 +463,11 @@ interface CheckParityOptions<TScenario extends CheckParityScenario, TEffects ext
1133
463
  * replay stand-ins and never runs them. A class instance's prototype
1134
464
  * methods count, bound to the instance.
1135
465
  */
1136
- effects: (scenario: TScenario) => TEffects;
466
+ effects: (scenario: TScenario, live: TLive) => TEffects;
1137
467
  /** The original workflow. */
1138
- before: (input: TScenario["input"], effects: TEffects) => PromiseLike<TOutput>;
468
+ before: (input: TScenario["input"], effects: TEffects, live: TLive) => PromiseLike<TOutput>;
1139
469
  /** The port: a `wrapWorkflow` run (`(input, effects) => step.run(input, { effects })`) or any runner with the same signature. */
1140
- after: (input: TScenario["input"], effects: TEffects) => PromiseLike<unknown>;
470
+ after: (input: TScenario["input"], effects: TEffects, live: TLive) => PromiseLike<unknown>;
1141
471
  /** Passed to `compareRecordings`. */
1142
472
  compare?: CompareRecordingsOptions;
1143
473
  /**
@@ -1152,6 +482,17 @@ interface CheckParityOptions<TScenario extends CheckParityScenario, TEffects ext
1152
482
  * raw errors.
1153
483
  */
1154
484
  normalizeError?: (error: unknown, scenario: TScenario) => unknown;
485
+ /**
486
+ * Builds the error a recorded failure throws in the port's replay, so code
487
+ * that checks `instanceof` sees its own class. Passed to `createRecorder`.
488
+ */
489
+ reviveError?: CreateRecorderOptions["reviveError"];
490
+ /**
491
+ * `recorder.fn` options per effect, on both sides: `serialize` for a
492
+ * result that is not JSON, `revive` to rebuild it for the port, `redact`
493
+ * for arguments.
494
+ */
495
+ recordOptions?: { [K in keyof TEffects]?: RecordCallOptions; };
1155
496
  }
1156
497
  /** One scenario's row in a {@link CheckParityReport}. */
1157
498
  interface CheckParityRow<TScenario extends CheckParityScenario, TOutput> {
@@ -1196,6 +537,6 @@ interface CheckParityReport<TScenario extends CheckParityScenario, TOutput> {
1196
537
  * expect(report.equivalent).toBe(true);
1197
538
  * ```
1198
539
  */
1199
- export declare function checkParity<TScenario extends CheckParityScenario, TEffects extends object, TOutput>(options: CheckParityOptions<TScenario, TEffects, TOutput>): Promise<CheckParityReport<TScenario, TOutput>>;
540
+ export declare function checkParity<TScenario extends CheckParityScenario, TEffects extends object, TOutput, TLive = undefined>(options: CheckParityOptions<TScenario, TEffects, TOutput, TLive>): Promise<CheckParityReport<TScenario, TOutput>>;
1200
541
  //#endregion
1201
- export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AgreementReport, type AgreementRow, type AgreementSummary, type ApprovalRuleProposal, type ApprovalVerdict, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CapProposal, type CapSpec, type CheckCrashRecoveryOptions, type CheckParityOptions, type CheckParityOutcome, type CheckParityReport, type CheckParityRow, type CheckParityScenario, type CompareRecordingsOptions, type CompareRunsOptions, type CoverageReport, type CrashEffect, type CrashPoint, type CrashRecoveryFailure, type CrashRecoveryProblem, type CrashRecoveryReport, type CreateRecorderOptions, type Divergence, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type FirstDivergenceOptions, type FoldTracesOptions, type GetDecisionRequestsOptions, type HandlerCoverage, type HandlerRef, type JudgeAgreementOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelAnswer, type ModelCallMatch, type NormalizeEffect, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type ProposeApprovalRulesOptions, type ProposeCapsOptions, type RecordCallOptions, type RecordClientOptions, type RecordableDecisionModel, type RecordableLanguageModel, type RecordableModel, type RecordedCall, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, RecordingDivergenceError, type RepeatGroup, type RepeatReport, type RepeatResult, type RepeatRun, type RepeatRunsOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateCoverage, type StateStats, type TraceCoverageOptions, type TraceStep, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryEventEnvelope, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, compareRecordings, createRecorder, explorePaths, lintAgentMachine, sameModelCallShape, simulateAgent, textOutput };
542
+ export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AssertAgentMachineOptions, type CanReachOrdering, type CanReachResult, type CanReachTarget, type CheckCrashRecoveryOptions, type CheckParityBaseOptions, type CheckParityOptions, type CheckParityOutcome, type CheckParityReport, type CheckParityRow, type CheckParityScenario, type CheckParitySetup, type CompareRecordingsOptions, type CompareRunsOptions, type CrashEffect, type CrashPoint, type CrashRecoveryFailure, type CrashRecoveryProblem, type CrashRecoveryReport, type CreateRecorderOptions, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type ExploreStep, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelAnswer, type ModelCallMatch, type NormalizeEffect, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordCallOptions, type RecordClientOptions, type RecordableDecisionModel, type RecordableLanguageModel, type RecordableModel, type RecordedCall, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, RecordingDivergenceError, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type TrajectoryEvent, type TrajectoryEventEnvelope, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, compareRecordings, createRecorder, explorePaths, lintAgentMachine, matchesTrajectory, sameModelCallShape, simulateAgent, textOutput };