@statelyai/agent 2.0.0-alpha.35 → 2.0.0-alpha.37

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/CHANGELOG.md +138 -0
  2. package/dist/act-B9hLQls6.mjs +19 -0
  3. package/dist/act-CrX0JwWY.cjs +42 -0
  4. package/dist/ai-sdk.cjs +63 -7
  5. package/dist/ai-sdk.d.cts +13 -3
  6. package/dist/ai-sdk.d.mts +13 -3
  7. package/dist/ai-sdk.mjs +63 -7
  8. package/dist/{decision-BEsATcr1.cjs → decision-CMeuNZ1g.cjs} +262 -4
  9. package/dist/{decision-COkJNjOW.mjs → decision-m5hGmHpF.mjs} +239 -5
  10. package/dist/decision-object-B5zGSPFp.d.mts +15 -0
  11. package/dist/decision-object-DBcLWVvv.d.cts +15 -0
  12. package/dist/{run-agent-Ct_GZw_f.mjs → effects-BT_cSY0j.mjs} +1003 -109
  13. package/dist/{run-agent-s5D32w99.cjs → effects-DDseYtJk.cjs} +1037 -107
  14. package/dist/{event-log-C7jeH0y8.mjs → event-log-DkVlk3Rh.mjs} +30 -15
  15. package/dist/{event-log-BWSSLUff.cjs → event-log-Dkc-yIYu.cjs} +41 -14
  16. package/dist/{event-log-store-BzXehP7R.d.cts → event-log-store-BBS8so8a.d.mts} +2 -2
  17. package/dist/{event-log-store-W49TCJXi.mjs → event-log-store-Brx9tOHt.mjs} +1 -1
  18. package/dist/{event-log-store-CleWpsp5.cjs → event-log-store-Cl-YWDnq.cjs} +1 -1
  19. package/dist/{event-log-store-CWjN6lmM.d.mts → event-log-store-bc4ICGcS.d.cts} +2 -2
  20. package/dist/index.cjs +660 -286
  21. package/dist/index.d.cts +168 -9
  22. package/dist/index.d.mts +168 -9
  23. package/dist/index.mjs +639 -271
  24. package/dist/internal-C3Lqu538.d.cts +43 -0
  25. package/dist/internal-DUcdhk-i.d.mts +43 -0
  26. package/dist/log.cjs +2 -2
  27. package/dist/log.d.cts +1 -1
  28. package/dist/log.d.mts +1 -1
  29. package/dist/log.mjs +2 -2
  30. package/dist/machines.cjs +457 -41
  31. package/dist/machines.d.cts +114 -43
  32. package/dist/machines.d.mts +114 -43
  33. package/dist/machines.mjs +457 -42
  34. package/dist/{model-source-DmhTMK39.cjs → model-source-B4WQ8Idp.cjs} +1 -1
  35. package/dist/{model-source-CK3F4YuU.mjs → model-source-kjGwqAV8.mjs} +1 -1
  36. package/dist/openai.cjs +149 -36
  37. package/dist/openai.d.cts +28 -10
  38. package/dist/openai.d.mts +28 -10
  39. package/dist/openai.mjs +149 -36
  40. package/dist/otel.d.cts +1 -1
  41. package/dist/otel.d.mts +1 -1
  42. package/dist/{run-agent-BGdx5913.d.mts → run-agent-30mH2moE.d.cts} +91 -9
  43. package/dist/{run-agent-B1-Eqviq.d.cts → run-agent-DlPzBWj-.d.mts} +91 -9
  44. package/dist/{setup-agent-05uw4dPZ.mjs → setup-agent-BDmNr6PC.mjs} +9 -3
  45. package/dist/{setup-agent-BtfeFfF6.d.cts → setup-agent-BaNyz6Q3.d.cts} +4 -4
  46. package/dist/{setup-agent-C6nqzv_X.d.mts → setup-agent-C1CG_nfX.d.mts} +4 -4
  47. package/dist/{setup-agent-CXEBURdo.cjs → setup-agent-CmEToB8y.cjs} +9 -3
  48. package/dist/tanstack-ai.cjs +62 -4
  49. package/dist/tanstack-ai.d.cts +18 -6
  50. package/dist/tanstack-ai.d.mts +18 -6
  51. package/dist/tanstack-ai.mjs +62 -4
  52. package/dist/testing.cjs +2493 -1502
  53. package/dist/testing.d.cts +360 -256
  54. package/dist/testing.d.mts +360 -256
  55. package/dist/testing.mjs +2489 -1503
  56. package/dist/{text-logic-CV_4KGr6.d.cts → text-logic-BqxalgMr.d.mts} +87 -4
  57. package/dist/{text-logic-Ckh-rJFV.d.mts → text-logic-CSf9bqpP.d.cts} +87 -4
  58. package/dist/{types-Dp5_Ru4R.d.cts → types-DQUOOaZO.d.cts} +1 -1
  59. package/dist/{types-Dp5_Ru4R.d.mts → types-DQUOOaZO.d.mts} +1 -1
  60. package/dist/{usage-BDiJKyv9.d.cts → usage-Bm5nvpow.d.cts} +6 -1
  61. package/dist/{usage-DOoVUGVq.d.mts → usage-BnikdOca.d.mts} +6 -1
  62. package/dist/utils-7lSa4H-i.d.cts +403 -0
  63. package/dist/utils-CBbKw2fM.d.mts +403 -0
  64. package/dist/validate.cjs +44 -12
  65. package/dist/validate.d.cts +6 -6
  66. package/dist/validate.d.mts +6 -6
  67. package/dist/validate.mjs +44 -12
  68. package/dist/verify-ChyX6jq0.mjs +2306 -0
  69. package/dist/verify-CwlombbC.cjs +2359 -0
  70. package/dist/{verify-BQpE8hUz.d.cts → verify-DJHfEfm6.d.cts} +213 -14
  71. package/dist/{verify-CqeuEmIk.d.mts → verify-i9Hiwc5g.d.mts} +213 -14
  72. package/dist/with-model-calls-B_WLmOSl.cjs +248 -0
  73. package/dist/with-model-calls-BkOwdroS.mjs +213 -0
  74. package/docs/act.md +102 -0
  75. package/docs/advanced.md +105 -0
  76. package/docs/any-stack.md +62 -0
  77. package/docs/choosing-a-run-mode.md +55 -0
  78. package/docs/debugging.md +29 -0
  79. package/docs/decisions.md +473 -0
  80. package/docs/environment-and-policy.md +188 -0
  81. package/docs/evals.md +509 -0
  82. package/docs/event-log.md +312 -0
  83. package/docs/from-a-loop.md +528 -0
  84. package/docs/generate-machines.md +110 -0
  85. package/docs/hosts.md +414 -0
  86. package/docs/human-in-the-loop.md +177 -0
  87. package/docs/index.md +48 -0
  88. package/docs/judgments.md +281 -0
  89. package/docs/langgraph-comparison.md +198 -0
  90. package/docs/machines-as-data.md +454 -0
  91. package/docs/machines-presets.md +198 -0
  92. package/docs/machines.md +497 -0
  93. package/docs/messages.md +67 -0
  94. package/docs/meta.json +52 -0
  95. package/docs/models-and-providers.md +268 -0
  96. package/docs/multi-agent.md +68 -0
  97. package/docs/observability.md +109 -0
  98. package/docs/patterns.md +31 -0
  99. package/docs/persistence.md +207 -0
  100. package/docs/quickstart.md +121 -0
  101. package/docs/roadmap.md +22 -0
  102. package/docs/scope.md +75 -0
  103. package/docs/statechart-policy-examples.md +69 -0
  104. package/docs/steps.md +56 -0
  105. package/docs/strangler-fig.md +323 -0
  106. package/docs/text-requests.md +250 -0
  107. package/docs/thinking-in-state-machines.md +324 -0
  108. package/docs/tools.md +249 -0
  109. package/docs/upgrading.md +37 -0
  110. package/docs/usage-and-budgets.md +518 -0
  111. package/docs/validate-and-repair.md +253 -0
  112. package/docs/verify.md +492 -0
  113. package/docs/xstate-4-and-5-apps.md +161 -0
  114. package/llms.txt +53 -0
  115. package/package.json +10 -3
  116. package/readme.md +30 -15
  117. package/skills/add-evals/SKILL.md +3 -1
  118. package/skills/add-traces/SKILL.md +6 -1
  119. package/skills/build-machine/SKILL.md +84 -0
  120. package/skills/generate-machine/SKILL.md +1 -1
  121. package/skills/graph-engineering/SKILL.md +13 -3
  122. package/skills/improve-machine/SKILL.md +21 -2
  123. package/skills/loop-to-machine/SKILL.md +147 -192
  124. package/skills/loop-to-machine/reference.md +260 -0
  125. package/dist/utils-BnXdWih3.d.mts +0 -108
  126. package/dist/utils-ebVpCYoF.d.cts +0 -108
  127. package/dist/verify-Bpahmr34.mjs +0 -1241
  128. package/dist/verify-DCLZa9R0.cjs +0 -1288
  129. package/dist/with-model-calls-BXYdBdcu.cjs +0 -45
  130. package/dist/with-model-calls-Bv4V4EHC.mjs +0 -34
@@ -1,8 +1,171 @@
1
- import { F as AgentExecutorDecisionRequest, c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-Ckh-rJFV.mjs";
2
- import { f as AgentRuntimeOptions, l as AgentRunResult, o as AgentRunInit } from "./run-agent-BGdx5913.mjs";
3
- import { t as MetaOfSnapshot } from "./utils-BnXdWih3.mjs";
4
- import { _ as assertAgentMachine, a as AgentPathTerminal, b as lintAgentMachine, c as CanReachResult, d as ExploreSnapshot, f as LintAgentMachineOptions, g as SimulationTrailEntry, h as SimulationScript, i as AgentPathReport, l as CanReachTarget, m as SimulateAgentResult, n as AgentLintError, o as AgentUnknownStateError, p as SimulateAgentOptions, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as ExplorePathsOptions, v as canReach, x as simulateAgent, y as explorePaths } from "./verify-CqeuEmIk.mjs";
5
- import { AnyStateMachine, EventFromLogic, InputFrom, SnapshotFrom, StateValue } from "xstate";
1
+ import { R as AgentExecutorDecisionRequest, c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-BqxalgMr.mjs";
2
+ import { p as AgentRuntimeOptions, s as AgentRunInit, u as AgentRunResult } from "./run-agent-DlPzBWj-.mjs";
3
+ import { n as AgentEventLogStore } from "./event-log-store-BBS8so8a.mjs";
4
+ import { A as RecordedOutput, C as RecordableDecisionModel, D as RecordedEffect, E as RecordedCall, F as compareRecordings, I as createRecorder, L as sameModelCallShape, M as Recorder, N as Recording, O as RecordedError, P as RecordingDivergenceError, S as RecordClientOptions, T as RecordableModel, _ as EquivalenceReport, b as NormalizeEffect, g as EffectDivergenceReason, h as EffectDivergence, j as RecordedToolCall, k as RecordedModelCall, m as CreateRecorderOptions, p as CompareRecordingsOptions, t as MetaOfSnapshot, v as ModelAnswer, w as RecordableLanguageModel, x as RecordCallOptions, y as ModelCallMatch } from "./utils-CBbKw2fM.mjs";
5
+ import { S as textOutput, _ as assertAgentMachine, a as AgentPathTerminal, b as lintAgentMachine, c as CanReachResult, d as ExploreSnapshot, f as LintAgentMachineOptions, g as SimulationTrailEntry, h as SimulationScript, i as AgentPathReport, l as CanReachTarget, m as SimulateAgentResult, n as AgentLintError, o as AgentUnknownStateError, p as SimulateAgentOptions, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as ExplorePathsOptions, v as canReach, x as simulateAgent, y as explorePaths } from "./verify-i9Hiwc5g.mjs";
6
+ import { AnyActorLogic, AnyStateMachine, EventFromLogic, InputFrom, SnapshotFrom, StateValue } from "xstate";
7
+ //#region src/crash-recovery.d.ts
8
+ /** Options for {@link checkCrashRecovery}. */
9
+ interface CheckCrashRecoveryOptions<TMachine extends AnyStateMachine> {
10
+ /** Machine input for the run. */
11
+ input?: unknown;
12
+ /**
13
+ * Host events, delivered one per turn: each time the run settles `idle`,
14
+ * the next one resumes it. The run ends when it settles `done` or `error`,
15
+ * or `idle` with none left. An uninterrupted run that ends with some left
16
+ * throws. With `timers: "host"`, an id-less `{ type: "xstate.timer" }`
17
+ * here is a timer's turn: the pending timer due soonest fires.
18
+ */
19
+ events?: ReadonlyArray<EventFromLogic<TMachine> | {
20
+ type: "xstate.timer";
21
+ }>;
22
+ /**
23
+ * Host events sent while the run is busy: each one goes out when the effect
24
+ * `during` starts (its first start, or its `occurrence`-th) (a key as {@link CrashEffect.key} names it,
25
+ * such as `draft#1` or `actor:fetch:0`), so it lands while that effect is
26
+ * in flight. After a crash, one the log does not hold is sent again, as a
27
+ * host retrying its delivery would: as its effect starts again, or, when
28
+ * that effect's completion was durable, where the uninterrupted log holds
29
+ * the event. One whose
30
+ * effect the uninterrupted run never starts throws.
31
+ */
32
+ sendDuring?: ReadonlyArray<{
33
+ during: string;
34
+ /** Which start of `during` to send it at, from 0, as {@link CrashPoint}'s `occurrence`. Default 0. */
35
+ occurrence?: number;
36
+ event: EventFromLogic<TMachine>;
37
+ }>;
38
+ /**
39
+ * `"host"` hands root `after` timers to the harness as a host scheduler
40
+ * (see `AgentTimerScheduler`) instead of arming them in process. The run
41
+ * settles `idle` with them pending: an id-less `{ type: "xstate.timer" }`
42
+ * in `events` fires the one due soonest, and once `events` run out, the
43
+ * pending ones fire in due order. Default `"in-process"`.
44
+ */
45
+ timers?: "in-process" | "host";
46
+ /**
47
+ * Model executors. Each call is recorded by its `callKey`. They run in the
48
+ * uninterrupted run only: after a crash, a call it made gets its recorded
49
+ * answer, so a fake may answer by call order.
50
+ */
51
+ executors?: Partial<AgentRequestExecutors>;
52
+ /** Actor sources, as `createAgentRuntime`'s `actors`. */
53
+ actors?: Record<string, AnyActorLogic>;
54
+ /**
55
+ * A fresh, empty store for one run: the baseline, or one crash and its
56
+ * resume. Default: an in-memory store. Pass your own adapter to check that
57
+ * an entry it acknowledged is read back after the crash.
58
+ */
59
+ store?: () => AgentEventLogStore;
60
+ /** Called before the baseline and before each crashed run, to reset fakes the executors or actors keep. Not called before the resume: like a real dependency, a fake keeps what the crashed run did to it. */
61
+ reset?: () => void;
62
+ /** Other runtime options (`maxModelCalls`, `verification`, …), shared by every run. */
63
+ runtime?: Omit<AgentRuntimeOptions<TMachine>, "executors" | "actors" | "store" | "threadId" | "signal" | "timers" | "abortWith">;
64
+ /**
65
+ * Milliseconds one run (the baseline, a crashed run, or its resume) may take.
66
+ * Default 5000. An uninterrupted run that times out throws, so no point
67
+ * runs after it. A crashed run or resume that times out is aborted and
68
+ * starts no more effects, but an effect already running that ignores its
69
+ * abort signal can still finish after the next point's `reset`.
70
+ */
71
+ timeout?: number;
72
+ }
73
+ /**
74
+ * One effect the run started: a model call (keyed by its `callKey` without the
75
+ * execution id, which differs per run) or a plain actor (keyed by its invoke
76
+ * site, as `actor:<site>`).
77
+ */
78
+ interface CrashEffect {
79
+ kind: "model" | "actor";
80
+ key: string;
81
+ }
82
+ /** Where a run was killed. */
83
+ type CrashPoint = {
84
+ /** After the store accepted `entries` entries: the next append never lands. */
85
+ kind: "log";
86
+ entries: number;
87
+ } | {
88
+ /** While `effect` was in flight: it ran, but its result was never recorded. */
89
+ kind: "effect";
90
+ effect: CrashEffect;
91
+ /** The effect's position among the baseline's effects, from 0. */
92
+ index: number;
93
+ /** Which run of `effect.key` it is, from 0: the run dies at that one, whatever order concurrent effects start in. */
94
+ occurrence: number;
95
+ };
96
+ /** What went wrong after a crash. */
97
+ type CrashRecoveryProblem =
98
+ /** An effect whose completion was durable ran again. */
99
+ "reran" |
100
+ /** An effect the uninterrupted run made never ran. */
101
+ "skipped" |
102
+ /** An effect ran that the uninterrupted run never made (a model call under a new key). */
103
+ "unexpected" |
104
+ /** A model call re-executed under its key with a different request. */
105
+ "request-changed" |
106
+ /** The resumed run ended with a different status, output, state, or context. */
107
+ "outcome" |
108
+ /** The final log does not replay to the resumed run's state. */
109
+ "replay" |
110
+ /** A `sendDuring` event the crash lost, which the resume never sent again. */
111
+ "undelivered" |
112
+ /** The crashed run or its resume never settled within `timeout`. */
113
+ "stuck" |
114
+ /** The crashed run, its resume, or the store factory threw. */
115
+ "threw";
116
+ /** One problem at one crash point. */
117
+ interface CrashRecoveryFailure {
118
+ point: CrashPoint;
119
+ problem: CrashRecoveryProblem;
120
+ message: string;
121
+ /** The effect key, for effect problems. */
122
+ key?: string;
123
+ }
124
+ /** What {@link checkCrashRecovery} found. */
125
+ interface CrashRecoveryReport {
126
+ /** True when every crash point was reached and recovered: no failures, nothing unreached. */
127
+ ok: boolean;
128
+ /** The uninterrupted run. */
129
+ baseline: {
130
+ status: AgentRunResult<AnyStateMachine>["status"];
131
+ output?: unknown;
132
+ /** Entries in its log. */
133
+ entries: number;
134
+ /** Effects it started, in start order. */
135
+ effects: CrashEffect[];
136
+ };
137
+ /** Every point a run was killed at, in order. */
138
+ points: CrashPoint[];
139
+ /**
140
+ * Points no run died at, with why: the crashed run never started that
141
+ * effect, or one write took the log past that length. Their resumes were
142
+ * not checked.
143
+ */
144
+ unreached: Array<{
145
+ point: CrashPoint;
146
+ reason: string;
147
+ }>;
148
+ failures: CrashRecoveryFailure[];
149
+ }
150
+ /**
151
+ * Runs `machine` once uninterrupted, then once per crash point: killed after
152
+ * each log append and while each effect is in flight, then resumed from the
153
+ * store's log by a fresh runtime. Each resumed run is checked against the
154
+ * uninterrupted one:
155
+ *
156
+ * - An effect whose completion was durable before the crash never runs again.
157
+ * Only an effect in flight at the crash may (execution is at-least-once).
158
+ * - Every effect of the uninterrupted run runs, and no other: a re-executed
159
+ * model call keeps its `callKey` and its request.
160
+ * - The run ends with the same status, output, state value, and context.
161
+ * - The resumed log replays to that state.
162
+ *
163
+ * Model calls the uninterrupted run made are answered from its recording.
164
+ * Plain actors run live, so they must give the same result however many times
165
+ * they run (pass `reset` for fakes that keep state).
166
+ */
167
+ export declare function checkCrashRecovery<TMachine extends AnyStateMachine>(machine: TMachine, options?: CheckCrashRecoveryOptions<TMachine>): Promise<CrashRecoveryReport>;
168
+ //#endregion
6
169
  //#region src/trajectory.d.ts
7
170
  /** An event-shaped trajectory item: anything with a string `type`. */
8
171
  interface TrajectoryEvent {
@@ -515,6 +678,125 @@ export declare function firstDivergence(actual: readonly {
515
678
  readonly type: string;
516
679
  }[] | readonly TrajectoryItem[], options?: FirstDivergenceOptions): Divergence | undefined;
517
680
  //#endregion
681
+ //#region src/propose-from-traces.d.ts
682
+ /** A person's verdict as {@link proposeApprovalRules} hands it to `key`. */
683
+ interface ApprovalVerdict<TTrace> {
684
+ /** The approve or reject event, as traced. */
685
+ event: Record<string, unknown>;
686
+ /** The traced snapshot the verdict arrived in: the approval state, with the context that holds the pending call. */
687
+ snapshot: unknown;
688
+ /** The whole run. */
689
+ trace: TTrace;
690
+ }
691
+ /** Options for {@link proposeApprovalRules}. */
692
+ interface ProposeApprovalRulesOptions<TTrace> {
693
+ /** The state's dot path, such as `'awaitingApproval'`. A verdict counts when the run was in it or in a state inside it. */
694
+ approvalState: string;
695
+ /** Event types that approve, such as `['APPROVE']`. */
696
+ approveEvents: readonly string[];
697
+ /** Event types that reject, such as `['REJECT']`. */
698
+ rejectEvents: readonly string[];
699
+ /**
700
+ * The rule a verdict is evidence for: usually the tool name and a pattern
701
+ * of its arguments, read from `snapshot.context`. `undefined` skips the verdict.
702
+ */
703
+ key: (verdict: ApprovalVerdict<TTrace>) => string | undefined;
704
+ /**
705
+ * The keys a run broke a rule for, from ground truth (a ledger checked
706
+ * after the fact). Any violation blocks `allow` for that key.
707
+ */
708
+ violations?: (trace: TTrace) => Iterable<string>;
709
+ }
710
+ /** One key's row from {@link proposeApprovalRules}. */
711
+ interface ApprovalRuleProposal {
712
+ key: string;
713
+ /** Approve verdicts, summed over runs. */
714
+ approved: number;
715
+ /** Reject verdicts, summed over runs. */
716
+ rejected: number;
717
+ /** Violations `violations` reported, summed over runs. 0 without `violations`. */
718
+ violations: number;
719
+ /** Indexes, in input order, of the runs with a verdict for this key. */
720
+ runs: number[];
721
+ /**
722
+ * `'allow'`: approved, never rejected, no violation. `'deny'`: rejected,
723
+ * never approved. `null`: mixed evidence.
724
+ */
725
+ proposal: "allow" | "deny" | null;
726
+ /** Runs with a verdict for this key: `runs.length`. */
727
+ support: number;
728
+ }
729
+ /**
730
+ * Groups the approval verdicts in many runs' traces by `key` and proposes an
731
+ * `allow` or `deny` rule for each. Pure and deterministic; a proposal is for
732
+ * a person to review, and nothing here changes a machine.
733
+ *
734
+ * Reads `machine.transition` events: a verdict is a transition, not ignored,
735
+ * out of `approvalState` on an approve or reject event. Rows come in the
736
+ * order keys were first seen.
737
+ *
738
+ * @example
739
+ * ```ts no-check
740
+ * const rows = proposeApprovalRules(traces, {
741
+ * approvalState: "awaitingApproval",
742
+ * approveEvents: ["APPROVE"],
743
+ * rejectEvents: ["REJECT"],
744
+ * key: ({ snapshot }) => `${snapshot.context.pending.tool}:${snapshot.context.pending.path}`,
745
+ * });
746
+ * ```
747
+ */
748
+ export declare function proposeApprovalRules<TTrace extends readonly {
749
+ readonly type: string;
750
+ }[]>(traces: Iterable<TTrace>, options: ProposeApprovalRulesOptions<TTrace>): ApprovalRuleProposal[];
751
+ /** One cap for {@link proposeCaps}. Give `failureState` or `failureOutput`. */
752
+ interface CapSpec<TTrace> {
753
+ /** Names the row, such as `'retries'`. */
754
+ name: string;
755
+ /** The cap's value now. A cap runs hit is never proposed below it. */
756
+ current?: number;
757
+ /** The run hit the cap when it entered this state (dot path), such as `'gaveUp'`. */
758
+ failureState?: string;
759
+ /** The run hit the cap when its `run.end` output satisfies this. */
760
+ failureOutput?: (output: unknown) => boolean;
761
+ /** How much of the cap the run used: retries made, steps taken, batch size. `undefined` leaves the run out. */
762
+ valueFromRun: (trace: TTrace) => number | undefined;
763
+ }
764
+ /** One cap's row from {@link proposeCaps}. */
765
+ interface CapProposal {
766
+ name: string;
767
+ /** Runs that hit the cap. */
768
+ hits: number;
769
+ /** Runs that did not, with a value. */
770
+ passing: number;
771
+ /** The largest value over all runs; `null` with none. */
772
+ max: number | null;
773
+ /** The 95th percentile (nearest rank) over passing runs; `null` with none. */
774
+ p95: number | null;
775
+ /** `p95 + 1`, and never below `current` when any run hit the cap; `null` with no passing run. */
776
+ proposal: number | null;
777
+ }
778
+ /** Options for {@link proposeCaps}. */
779
+ interface ProposeCapsOptions<TTrace> {
780
+ caps: readonly CapSpec<TTrace>[];
781
+ }
782
+ /**
783
+ * Proposes a value for each cap (retry count, steps, batch size) from many
784
+ * runs' traces: the 95th percentile of what passing runs used, plus one, and
785
+ * never below the current cap while runs still hit it. Pure and
786
+ * deterministic; a proposal is for a person to review, and nothing here
787
+ * changes a machine.
788
+ *
789
+ * @example
790
+ * ```ts no-check
791
+ * const [retries] = proposeCaps(traces, {
792
+ * caps: [{ name: "retries", current: 3, failureState: "gaveUp", valueFromRun: countRetries }],
793
+ * });
794
+ * ```
795
+ */
796
+ export declare function proposeCaps<TTrace extends readonly {
797
+ readonly type: string;
798
+ }[]>(traces: Iterable<TTrace>, options: ProposeCapsOptions<TTrace>): CapProposal[];
799
+ //#endregion
518
800
  //#region src/decision-requests.d.ts
519
801
  /** One decision a run asked for. See {@link getDecisionRequests}. */
520
802
  interface TracedDecisionRequest {
@@ -829,269 +1111,91 @@ export declare function compareRuns<TScenario extends {
829
1111
  expected?: string;
830
1112
  }>(options: CompareRunsOptions<TScenario>): Promise<ParityReport<TScenario>>;
831
1113
  //#endregion
832
- //#region src/equivalence.d.ts
833
- /**
834
- * Verified equivalence — prove a port behaves like the code it replaces.
835
- *
836
- * A port is equivalent when, given the same model answers and the same tool
837
- * results, it produces the same OBSERVABLE EFFECTS: the same number of model
838
- * calls, the same tool calls with the same arguments, and the same output.
839
- * Prompts, internal steps, and states are free to differ.
840
- *
841
- * 1. Record the original: wrap its model (`recorder.model`, an AI SDK
842
- * `LanguageModelV3`, or `recorder.openai`, an `openai` client) and its tools
843
- * (`recorder.tool` / `recorder.tools`), run a scenario, call
844
- * `recorder.output(result)`, and save `recorder.recording`.
845
- * 2. Replay it on the port: `createRecorder({ replay: recording })` answers
846
- * every model call and tool call from the recording, with no API key and no
847
- * side effects, and records what the port did.
848
- * 3. `compareRecordings(original, port)` reports the first effect that differs.
849
- *
850
- * A port that changes how it calls the model on purpose (a model-picked action
851
- * that became `agent.decide`, a stream that became `doGenerate`) still replays:
852
- * `translate` turns the recorded answer into one that fits, `fallback` answers
853
- * the calls the original never made, and `compareRecordings(…, { kinds })`
854
- * compares only tool calls and output.
855
- *
856
- * Provider-free: both seams are wrapped structurally, so this module imports
857
- * neither `ai` nor `openai`.
858
- * @module
859
- */
860
- /** An error as recorded: JSON-safe. */
861
- interface RecordedError {
862
- name: string;
863
- message: string;
864
- code?: unknown;
865
- /** The error's own number and boolean fields, such as `statusCode` and
866
- * `isRetryable`; replay restores them. */
867
- data?: Record<string, number | boolean>;
868
- /** `Symbol.for` keys set to `true` on the error, such as the AI SDK's
869
- * `vercel.ai.error.AI_APICallError`; replay restores them, so checks like
870
- * `APICallError.isInstance` still pass. */
871
- markers?: string[];
872
- }
873
- /** One model call. `call` is its 0-based position among model calls. */
874
- interface RecordedModelCall {
875
- kind: "model";
876
- /** Which wrapper saw it. */
877
- seam: "ai-sdk" | "openai";
878
- /** `doGenerate`/`doStream`/`doDecide`, or `chat.completions.create`/`responses.create`. */
879
- method: string;
880
- call: number;
881
- /** The call options or params, JSON-safe (abort signals and headers dropped). */
882
- request: unknown;
883
- /** The non-streaming result. */
884
- response?: unknown;
885
- /** The stream's parts or chunks, in order. */
886
- stream?: unknown[];
887
- error?: RecordedError;
888
- /** In replay, the recorded call (its `call`) whose answer served this one. */
889
- replayed?: number;
890
- /** In replay, `translate` adapted the recorded answer to this call. */
891
- translated?: true;
892
- /** In replay, no recorded call fit, so `fallback` answered it. */
893
- unrecorded?: true;
894
- }
895
- /** One tool call. `step` is the number of model calls made before it. */
896
- interface RecordedToolCall {
897
- kind: "tool";
898
- name: string;
899
- step: number;
900
- input: unknown;
901
- output?: unknown;
902
- error?: RecordedError;
903
- /** The tool returned without a Promise; replay answers synchronously too. */
904
- sync?: true;
905
- /** Started while an earlier tool call was still running. Concurrent calls
906
- * may finish in either order; sequential ones must keep theirs. */
907
- concurrent?: true;
908
- /** In replay, no recorded call matched, so `fallback.tools` ran the tool. */
909
- unrecorded?: true;
910
- }
911
- /** The run's final result. */
912
- interface RecordedOutput {
913
- kind: "output";
914
- value: unknown;
915
- }
916
- type RecordedEffect = RecordedModelCall | RecordedToolCall | RecordedOutput;
917
- /** A run's effects, in order. JSON-serializable: save it as a golden file. */
918
- interface Recording {
919
- version: 1;
920
- effects: RecordedEffect[];
1114
+ //#region src/check-parity.d.ts
1115
+ /** One scenario: an id and the workflow's input. Extra fields are yours. */
1116
+ interface CheckParityScenario<TInput = unknown> {
1117
+ id: string;
1118
+ input: TInput;
921
1119
  }
922
- /** Options for {@link createRecorder}. Omit for record mode. */
923
- interface CreateRecorderOptions {
924
- /** Answer model and tool calls from this recording instead of running them. */
925
- replay?: Recording;
926
- /**
927
- * In replay, `"replay"` (default) returns each tool's recorded result
928
- * without running it; `"live"` runs the wrapped tool (your stub) and records
929
- * what it returned.
930
- */
931
- tools?: "replay" | "live";
932
- /**
933
- * In replay, which model calls a recorded answer may serve. `"shape"`
934
- * (default) refuses a call whose offered tools or response format changed,
935
- * since the recorded answer may not fit it; see {@link ModelCallMatch}.
936
- */
937
- modelCalls?: ModelCallMatch;
938
- /**
939
- * In replay, maps the recorded and the new tool call before matching them,
940
- * like {@link CompareRecordingsOptions.normalize}: redact a generated id in a
941
- * tool's input so the recorded result still answers it. A dropped
942
- * (`undefined`) effect matches as recorded.
943
- */
944
- normalize?: NormalizeEffect;
945
- /**
946
- * In replay, adapts a recorded answer to a model call you changed on
947
- * purpose. Called with each unused recorded call, in order, until it
948
- * returns an answer; that answer serves the call regardless of method and
949
- * shape, and {@link compareRecordings} counts the pair as the same call.
950
- * Return `undefined` to leave a recorded call to the usual matching.
951
- *
952
- * @example
953
- * ```ts
954
- * // The original streamed; the port generates.
955
- * translate: (recorded, actual) =>
956
- * actual.method === "doGenerate" && recorded.method === "doStream"
957
- * ? { response: toGenerateResult(recorded.stream) }
958
- * : undefined,
959
- * ```
960
- */
961
- translate?: (recorded: RecordedModelCall, actual: RecordedModelCall) => ModelAnswer | undefined;
1120
+ /** How one side of a scenario ended: what it returned, or what it threw. */
1121
+ type CheckParityOutcome<TOutput = unknown> = {
1122
+ output: TOutput;
1123
+ } | {
1124
+ error: unknown;
1125
+ };
1126
+ /** Options for {@link checkParity}. */
1127
+ interface CheckParityOptions<TScenario extends CheckParityScenario, TEffects extends object, TOutput> {
1128
+ scenarios: readonly TScenario[];
962
1129
  /**
963
- * In replay, answers what the recording cannot, instead of throwing
964
- * {@link RecordingDivergenceError}: a model call no recorded call fits (the
965
- * port's retry or an added call) goes to `model` (AI SDK seam) or `openai`
966
- * (client seam), and with `tools: true` a tool call no recorded call matches
967
- * runs the wrapped tool. Such effects are marked `unrecorded` and still
968
- * count as extra in {@link compareRecordings}.
1130
+ * Fresh effects for one scenario: fakes, stubs, or real clients. `before`
1131
+ * runs on them for real, recorded, so a write effect writes once per
1132
+ * scenario: point it at an isolated fixture, not production. `after` gets
1133
+ * replay stand-ins and never runs them. A class instance's prototype
1134
+ * methods count, bound to the instance.
969
1135
  */
970
- fallback?: {
971
- model?: RecordableModel;
972
- openai?: object;
973
- tools?: boolean;
974
- };
975
- }
976
- /** A model call's answer: the result, the stream's parts, or an error. */
977
- interface ModelAnswer {
978
- response?: unknown;
979
- stream?: unknown[];
980
- error?: RecordedError;
981
- }
982
- /**
983
- * Maps an effect before matching or comparing; `undefined` drops it.
984
- * `index` and `effects` are its position in the whole recording, so a repeat
985
- * of an earlier effect (an idempotent write made again) can be dropped.
986
- */
987
- type NormalizeEffect = (effect: RecordedEffect, index: number, effects: readonly RecordedEffect[]) => RecordedEffect | undefined;
988
- /**
989
- * When two model calls count as the same: `"shape"` compares offered tool
990
- * names and response format, `"none"` only that a call happened, and a
991
- * function decides per call (allow the changes you made on purpose and fall
992
- * back to {@link sameModelCallShape}).
993
- */
994
- type ModelCallMatch = "shape" | "none" | ((expected: RecordedModelCall, actual: RecordedModelCall) => boolean);
995
- /** The structural slice of an AI SDK `LanguageModelV3` the recorder wraps. */
996
- interface RecordableLanguageModel {
997
- doGenerate(options: never): PromiseLike<unknown>;
998
- doStream(options: never): PromiseLike<{
999
- stream: ReadableStream<unknown>;
1000
- }>;
1001
- }
1002
- /** The structural slice of an AI SDK decision model (`experimental_decide`). */
1003
- interface RecordableDecisionModel {
1004
- doDecide(options: never): PromiseLike<unknown>;
1005
- }
1006
- /** An AI SDK model the recorder can wrap. */
1007
- type RecordableModel = RecordableLanguageModel | RecordableDecisionModel;
1008
- type AnyFunction = (...args: never[]) => unknown;
1009
- /** Records (or replays) the effects of one run. */
1010
- interface Recorder {
1136
+ effects: (scenario: TScenario) => TEffects;
1137
+ /** The original workflow. */
1138
+ before: (input: TScenario["input"], effects: TEffects) => PromiseLike<TOutput>;
1139
+ /** The port: a `wrapWorkflow` run (`(input, effects) => step.run(input, { effects })`) or any runner with the same signature. */
1140
+ after: (input: TScenario["input"], effects: TEffects) => PromiseLike<unknown>;
1141
+ /** Passed to `compareRecordings`. */
1142
+ compare?: CompareRecordingsOptions;
1011
1143
  /**
1012
- * Wraps an AI SDK `LanguageModelV3` (`openai("gpt-…")`, `anthropic(…)`) or
1013
- * decision model (`doDecide`), keeping its type. In replay the model is
1014
- * never called, so a provider model built with no API key works.
1144
+ * Maps each side's output before it is compared: drop or pin what differs
1145
+ * run to run (a duration, a generated id). Rows keep the raw outputs.
1015
1146
  */
1016
- model<TModel extends RecordableModel>(model: TModel): TModel;
1147
+ normalizeOutput?: (output: unknown, scenario: TScenario) => unknown;
1017
1148
  /**
1018
- * Wraps an `openai` client's `chat.completions.create` and
1019
- * `responses.create`, keeping its type. A streamed result is an async
1020
- * iterable only. In replay the client is never called, so
1021
- * `new OpenAI({ apiKey: "replay" })` works.
1149
+ * Maps each side's thrown error before it is compared: drop what differs
1150
+ * run to run (an embedded stack line, a generated id). An `Error` result
1151
+ * matches on name and message, anything else on its JSON. Rows keep the
1152
+ * raw errors.
1022
1153
  */
1023
- openai<TClient extends object>(client: TClient): TClient;
1024
- /** Wraps one tool function. Its arguments are the recorded input. */
1025
- tool<TFn extends AnyFunction>(name: string, fn: TFn): TFn;
1026
- /** Wraps every `execute` in an AI SDK tool set, named by key. */
1027
- tools<TTools extends Record<string, {
1028
- execute?: AnyFunction;
1029
- }>>(tools: TTools): TTools;
1030
- /** Records the run's final result. */
1031
- output(value: unknown): void;
1032
- /** What this run did so far. */
1033
- readonly recording: Recording;
1034
- }
1035
- /**
1036
- * Thrown in replay when the port asks for something the recording cannot
1037
- * answer: a model call past the last recorded one, or a tool call the original
1038
- * never made. The effect is still recorded, so {@link compareRecordings}
1039
- * reports it.
1040
- */
1041
- export declare class RecordingDivergenceError extends Error {
1042
- name: string;
1154
+ normalizeError?: (error: unknown, scenario: TScenario) => unknown;
1043
1155
  }
1044
- /**
1045
- * Creates a {@link Recorder}. With no options it records: every wrapped call
1046
- * runs for real. With `replay` it answers from a recording instead.
1047
- */
1048
- export declare function createRecorder(options?: CreateRecorderOptions): Recorder;
1049
- /** Why two effects differ. */
1050
- type EffectDivergenceReason = "missing-effect" | "extra-effect" | "kind" | "model-method" | "model-call-shape" | "model-error" | "tool-name" | "tool-input" | "tool-result" | "output";
1051
- /** The first effect where two recordings differ. */
1052
- interface EffectDivergence {
1053
- /** Position in the compared (tool-order-normalized) effect lists. */
1054
- index: number;
1055
- reason: EffectDivergenceReason;
1056
- expected?: RecordedEffect;
1057
- actual?: RecordedEffect;
1058
- /** One sentence, for test failures. */
1059
- message: string;
1060
- }
1061
- /** The result of {@link compareRecordings}. */
1062
- interface EquivalenceReport {
1156
+ /** One scenario's row in a {@link CheckParityReport}. */
1157
+ interface CheckParityRow<TScenario extends CheckParityScenario, TOutput> {
1158
+ id: string;
1159
+ scenario: TScenario;
1160
+ /** No divergence, and both sides returned the same value or threw the same error. */
1063
1161
  equivalent: boolean;
1064
- /** The first divergence, by position in the original's effects. */
1065
- divergence?: EffectDivergence;
1066
- /** Every divergence, in order. Effects both runs made line up between them. */
1067
1162
  divergences: EffectDivergence[];
1163
+ before: CheckParityOutcome<TOutput>;
1164
+ after: CheckParityOutcome;
1165
+ /** The golden recording of `before`: save it as a fixture. */
1166
+ recording: Recording;
1167
+ /** What `after` did in replay. */
1168
+ replay: Recording;
1068
1169
  }
1069
- /** Options for {@link compareRecordings}. */
1070
- interface CompareRecordingsOptions {
1071
- /** When two model calls count as the same. Default `"shape"`. */
1072
- modelCalls?: ModelCallMatch;
1073
- /**
1074
- * Maps each effect before comparing, on both sides: redact values that
1075
- * differ on every run (generated ids, timestamps), project a tool's input
1076
- * to the fields that matter, or return `undefined` to drop the effect.
1077
- */
1078
- normalize?: NormalizeEffect;
1079
- /**
1080
- * Which kinds of effect to compare. Default all. `["tool", "output"]`
1081
- * compares a port that calls the model differently (another protocol, a
1082
- * structured draft instead of a tool loop) by what it did, not how it asked.
1083
- */
1084
- kinds?: readonly RecordedEffect["kind"][];
1170
+ /** The result of {@link checkParity}. */
1171
+ interface CheckParityReport<TScenario extends CheckParityScenario, TOutput> {
1172
+ /** Every scenario is equivalent. */
1173
+ equivalent: boolean;
1174
+ scenarios: CheckParityRow<TScenario, TOutput>[];
1085
1175
  }
1086
- /** True when two model calls offer the same tool names and response format. */
1087
- export declare function sameModelCallShape(expected: RecordedModelCall, actual: RecordedModelCall): boolean;
1088
1176
  /**
1089
- * Compares two recordings and reports every effect that differs. `expected`
1090
- * is the original's recording, `actual` the port's. Effects that are the same
1091
- * on both sides are lined up first (longest common subsequence), so one extra
1092
- * call does not shift every effect after it; between them, effects of the same
1093
- * kind pair up and report what changed, and the rest are missing or extra.
1177
+ * Checks a port against the workflow it replaces, scenario by scenario: runs
1178
+ * `before` on fresh `effects` under `createRecorder`, replays `after` against
1179
+ * that recording (no effect really runs), and compares the two with
1180
+ * `compareRecordings`. Effects are compared by name, call order, arguments
1181
+ * and results; the output by value.
1182
+ *
1183
+ * Never throws for a port that misbehaves: an `after` that throws (a
1184
+ * `RecordingDivergenceError` for a call the original never made, or its own
1185
+ * error) is a row with `after: { error }`. When both sides throw, they match
1186
+ * on the error's name and message, after `normalizeError` when given.
1187
+ *
1188
+ * @example
1189
+ * ```ts
1190
+ * const report = await checkParity({
1191
+ * scenarios: [{ id: "refund", input: ticket }],
1192
+ * effects: () => createFakeEffects(),
1193
+ * before: (input, effects) => legacyRunTicket(input, effects),
1194
+ * after: (input, effects) => step1.run(input, { effects }),
1195
+ * });
1196
+ * expect(report.equivalent).toBe(true);
1197
+ * ```
1094
1198
  */
1095
- export declare function compareRecordings(expected: Recording, actual: Recording, options?: CompareRecordingsOptions): EquivalenceReport;
1199
+ export declare function checkParity<TScenario extends CheckParityScenario, TEffects extends object, TOutput>(options: CheckParityOptions<TScenario, TEffects, TOutput>): Promise<CheckParityReport<TScenario, TOutput>>;
1096
1200
  //#endregion
1097
- export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AgreementReport, type AgreementRow, type AgreementSummary, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRecordingsOptions, type CompareRunsOptions, type CoverageReport, type CreateRecorderOptions, type Divergence, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type FirstDivergenceOptions, type FoldTracesOptions, type GetDecisionRequestsOptions, type HandlerCoverage, type HandlerRef, type JudgeAgreementOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelAnswer, type ModelCallMatch, type NormalizeEffect, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordableDecisionModel, type RecordableLanguageModel, type RecordableModel, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, type RepeatGroup, type RepeatReport, type RepeatResult, type RepeatRun, type RepeatRunsOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateCoverage, type StateStats, type TraceCoverageOptions, type TraceStep, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };
1201
+ export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AgreementReport, type AgreementRow, type AgreementSummary, type ApprovalRuleProposal, type ApprovalVerdict, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CapProposal, type CapSpec, type CheckCrashRecoveryOptions, type CheckParityOptions, type CheckParityOutcome, type CheckParityReport, type CheckParityRow, type CheckParityScenario, type CompareRecordingsOptions, type CompareRunsOptions, type CoverageReport, type CrashEffect, type CrashPoint, type CrashRecoveryFailure, type CrashRecoveryProblem, type CrashRecoveryReport, type CreateRecorderOptions, type Divergence, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type FirstDivergenceOptions, type FoldTracesOptions, type GetDecisionRequestsOptions, type HandlerCoverage, type HandlerRef, type JudgeAgreementOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelAnswer, type ModelCallMatch, type NormalizeEffect, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type ProposeApprovalRulesOptions, type ProposeCapsOptions, type RecordCallOptions, type RecordClientOptions, type RecordableDecisionModel, type RecordableLanguageModel, type RecordableModel, type RecordedCall, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, RecordingDivergenceError, type RepeatGroup, type RepeatReport, type RepeatResult, type RepeatRun, type RepeatRunsOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateCoverage, type StateStats, type TraceCoverageOptions, type TraceStep, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryEventEnvelope, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, compareRecordings, createRecorder, explorePaths, lintAgentMachine, sameModelCallShape, simulateAgent, textOutput };