@statelyai/agent 2.0.0-alpha.38 → 2.0.0-alpha.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/CHANGELOG.md +45 -0
  2. package/dist/{act-CrX0JwWY.cjs → act-B0CVbmo4.cjs} +1 -1
  3. package/dist/{act-B9hLQls6.mjs → act-Bsz-hUfD.mjs} +1 -1
  4. package/dist/ai-sdk.cjs +4 -4
  5. package/dist/ai-sdk.d.cts +2 -3
  6. package/dist/ai-sdk.d.mts +2 -3
  7. package/dist/ai-sdk.mjs +4 -4
  8. package/dist/{decision-m5hGmHpF.mjs → decision-B41YBx5U.mjs} +1 -1
  9. package/dist/{decision-CMeuNZ1g.cjs → decision-D8bcIGXQ.cjs} +1 -1
  10. package/dist/{decision-object-DBcLWVvv.d.cts → decision-object-BB8XV5Ec.d.cts} +1 -1
  11. package/dist/{decision-object-B5zGSPFp.d.mts → decision-object-CH_sE694.d.mts} +1 -1
  12. package/dist/effects-B9IvUPVh.mjs +23 -0
  13. package/dist/effects-DrWcUGgc.cjs +34 -0
  14. package/dist/evals.cjs +810 -0
  15. package/dist/evals.d.cts +595 -0
  16. package/dist/evals.d.mts +595 -0
  17. package/dist/evals.mjs +801 -0
  18. package/dist/{event-log-DkVlk3Rh.mjs → event-log-CEq_bVXQ.mjs} +8 -4
  19. package/dist/{event-log-Dkc-yIYu.cjs → event-log-dSOnXHuG.cjs} +7 -15
  20. package/dist/{event-log-store-Brx9tOHt.mjs → event-log-store-DLAfNM34.mjs} +1 -1
  21. package/dist/{event-log-store-BBS8so8a.d.mts → event-log-store-DYun526m.d.mts} +15 -14
  22. package/dist/{event-log-store-Cl-YWDnq.cjs → event-log-store-Dey9ufA4.cjs} +1 -1
  23. package/dist/{event-log-store-bc4ICGcS.d.cts → event-log-store-aFXK6yZX.d.cts} +15 -14
  24. package/dist/index.cjs +131 -93
  25. package/dist/index.d.cts +73 -134
  26. package/dist/index.d.mts +73 -134
  27. package/dist/index.mjs +112 -70
  28. package/dist/internal-BzlpzSux.d.cts +14 -0
  29. package/dist/internal-BzlpzSux.d.mts +14 -0
  30. package/dist/log.cjs +2 -3
  31. package/dist/log.d.cts +2 -2
  32. package/dist/log.d.mts +2 -2
  33. package/dist/log.mjs +3 -3
  34. package/dist/machines.cjs +1 -719
  35. package/dist/machines.d.cts +2 -424
  36. package/dist/machines.d.mts +2 -424
  37. package/dist/machines.mjs +2 -709
  38. package/dist/{model-source-kjGwqAV8.mjs → model-source-BrsLDvHL.mjs} +1 -1
  39. package/dist/{model-source-B4WQ8Idp.cjs → model-source-YpxjmWuX.cjs} +1 -1
  40. package/dist/openai.cjs +3 -3
  41. package/dist/openai.d.cts +2 -3
  42. package/dist/openai.d.mts +2 -3
  43. package/dist/openai.mjs +3 -3
  44. package/dist/otel.d.cts +1 -1
  45. package/dist/otel.d.mts +1 -1
  46. package/dist/policy.cjs +77 -0
  47. package/dist/policy.d.cts +143 -0
  48. package/dist/policy.d.mts +143 -0
  49. package/dist/policy.mjs +75 -0
  50. package/dist/{effects-BT_cSY0j.mjs → run-agent-BE7DOWD6.mjs} +150 -184
  51. package/dist/{run-agent-DlPzBWj-.d.mts → run-agent-DHlq87G7.d.mts} +18 -34
  52. package/dist/{effects-DDseYtJk.cjs → run-agent-De3xVFGw.cjs} +149 -201
  53. package/dist/{run-agent-30mH2moE.d.cts → run-agent-Dq0oSFRm.d.cts} +18 -34
  54. package/dist/{setup-agent-BDmNr6PC.mjs → setup-agent-CLm6QVGy.mjs} +3 -3
  55. package/dist/{setup-agent-BaNyz6Q3.d.cts → setup-agent-Cv8Hk3mX.d.cts} +12 -11
  56. package/dist/{setup-agent-C1CG_nfX.d.mts → setup-agent-DgFAhaPJ.d.mts} +12 -11
  57. package/dist/{setup-agent-CmEToB8y.cjs → setup-agent-Dv5VpuHe.cjs} +3 -3
  58. package/dist/tanstack-ai.cjs +4 -4
  59. package/dist/tanstack-ai.d.cts +2 -3
  60. package/dist/tanstack-ai.d.mts +2 -3
  61. package/dist/tanstack-ai.mjs +4 -4
  62. package/dist/testing.cjs +122 -996
  63. package/dist/testing.d.cts +36 -695
  64. package/dist/testing.d.mts +36 -695
  65. package/dist/testing.mjs +112 -977
  66. package/dist/{text-logic-BqxalgMr.d.mts → text-logic-x17uLBwA.d.cts} +238 -2
  67. package/dist/{text-logic-CSf9bqpP.d.cts → text-logic-x17uLBwA.d.mts} +238 -2
  68. package/dist/trajectory-BFjXaooB.d.cts +96 -0
  69. package/dist/trajectory-BFjXaooB.d.mts +96 -0
  70. package/dist/trajectory-BUXj_1Ue.mjs +152 -0
  71. package/dist/trajectory-CoZSv4wF.cjs +157 -0
  72. package/dist/{usage-Bm5nvpow.d.cts → usage-B4B-TQSV.d.mts} +1 -1
  73. package/dist/{usage-BnikdOca.d.mts → usage-DvGcoK6x.d.cts} +1 -1
  74. package/dist/{utils-7lSa4H-i.d.cts → utils-B8k-YPYI.d.mts} +23 -17
  75. package/dist/{utils-CBbKw2fM.d.mts → utils-DAclF-kw.d.cts} +23 -17
  76. package/dist/validate.cjs +4 -4
  77. package/dist/validate.d.cts +2 -2
  78. package/dist/validate.d.mts +2 -2
  79. package/dist/validate.mjs +4 -4
  80. package/dist/{verify-DJHfEfm6.d.cts → verify-BM0sGmsf.d.cts} +42 -6
  81. package/dist/{verify-ChyX6jq0.mjs → verify-CWlg3j7B.mjs} +153 -45
  82. package/dist/{verify-CwlombbC.cjs → verify-Dek6gHCS.cjs} +153 -45
  83. package/dist/{verify-i9Hiwc5g.d.mts → verify-H3Pr2gNw.d.mts} +42 -6
  84. package/dist/{with-model-calls-B_WLmOSl.cjs → with-model-calls-D50Ca6qW.cjs} +2 -2
  85. package/dist/{with-model-calls-BkOwdroS.mjs → with-model-calls-uY-KKliA.mjs} +2 -2
  86. package/docs/advanced.md +5 -5
  87. package/docs/choosing-a-run-mode.md +7 -7
  88. package/docs/debugging.md +6 -8
  89. package/docs/decisions.md +5 -4
  90. package/docs/environment-and-policy.md +1 -1
  91. package/docs/evals.md +57 -43
  92. package/docs/event-log.md +13 -15
  93. package/docs/from-a-loop.md +3 -3
  94. package/docs/generate-machines.md +4 -4
  95. package/docs/hosts.md +20 -27
  96. package/docs/human-in-the-loop.md +5 -8
  97. package/docs/index.md +4 -4
  98. package/docs/judgments.md +5 -7
  99. package/docs/langgraph-comparison.md +6 -6
  100. package/docs/machines-as-data.md +7 -12
  101. package/docs/machines-presets.md +17 -120
  102. package/docs/machines.md +13 -13
  103. package/docs/models-and-providers.md +19 -25
  104. package/docs/multi-agent.md +1 -1
  105. package/docs/observability.md +6 -8
  106. package/docs/persistence.md +10 -20
  107. package/docs/quickstart.md +3 -3
  108. package/docs/statechart-policy-examples.md +1 -1
  109. package/docs/steps.md +5 -6
  110. package/docs/strangler-fig.md +72 -7
  111. package/docs/text-requests.md +10 -12
  112. package/docs/thinking-in-state-machines.md +8 -8
  113. package/docs/tools.md +10 -12
  114. package/docs/upgrading.md +1 -1
  115. package/docs/usage-and-budgets.md +51 -56
  116. package/docs/validate-and-repair.md +9 -14
  117. package/docs/verify.md +53 -7
  118. package/docs/xstate-4-and-5-apps.md +19 -28
  119. package/llms.txt +1 -1
  120. package/package.json +31 -6
  121. package/readme.md +32 -24
  122. package/dist/internal-C3Lqu538.d.cts +0 -43
  123. package/dist/internal-DUcdhk-i.d.mts +0 -43
  124. package/dist/types-DQUOOaZO.d.cts +0 -238
  125. package/dist/types-DQUOOaZO.d.mts +0 -238
  126. package/skills/add-evals/SKILL.md +0 -152
  127. package/skills/add-traces/SKILL.md +0 -152
  128. package/skills/build-machine/SKILL.md +0 -84
  129. package/skills/generate-machine/SKILL.md +0 -335
  130. package/skills/graph-engineering/SKILL.md +0 -43
  131. package/skills/improve-machine/SKILL.md +0 -261
  132. package/skills/improve-machine/reference.md +0 -101
  133. package/skills/loop-to-machine/SKILL.md +0 -205
  134. package/skills/loop-to-machine/policy.md +0 -154
  135. package/skills/loop-to-machine/reference.md +0 -260
@@ -0,0 +1,595 @@
1
+ import { R as AgentExecutorDecisionRequest } from "./text-logic-x17uLBwA.cjs";
2
+ import { i as AgentPathReport } from "./verify-BM0sGmsf.cjs";
3
+ import { i as TrajectoryItem } from "./trajectory-BFjXaooB.cjs";
4
+ import { AnyStateMachine } from "xstate";
5
+ //#region src/fold-traces.d.ts
6
+ /** One state's numbers across many runs. See {@link foldTraces}. */
7
+ interface StateStats {
8
+ /** The state's dot-joined path from the root, e.g. `'deciding'` or `'work.review'`. */
9
+ state: string;
10
+ /** Runs that were in this state at least once. */
11
+ visits: number;
12
+ /** Of those runs, the ones `failed` flagged. */
13
+ failures: number;
14
+ /** Options withheld by the guards at decisions made in this state, summed over runs. */
15
+ withheld: number;
16
+ /** Events this state ignored (`machine.transition` with `ignored`), summed over runs. */
17
+ ignored: number;
18
+ /** Failed runs `blame` attributed to this state. 0 without `blame`. */
19
+ blamed: number;
20
+ }
21
+ /** Options for {@link foldTraces}. */
22
+ interface FoldTracesOptions<TTrace> {
23
+ /** Whether a run failed, judged from its whole trace. */
24
+ failed: (trace: TTrace) => boolean;
25
+ /**
26
+ * The state a failed run's failure started in, usually
27
+ * `firstDivergence(trace, reference)?.state`. Called for failed runs only;
28
+ * `undefined` blames no state.
29
+ */
30
+ blame?: (trace: TTrace) => string | undefined;
31
+ }
32
+ /**
33
+ * Folds many runs' traces into per-state stats: how many runs visited each
34
+ * state, how many of those failed, how many options the guards withheld
35
+ * at decisions made there, how many events it ignored, and how many failures
36
+ * `blame` traced back to it. Pure; reads only:
37
+ *
38
+ * - `machine.transition`: the run visited `from.value` and `snapshot.value`;
39
+ * with `ignored`, each of those states ignored one event.
40
+ * - `request.start` of a decision, first attempt only: its `request.withheld`
41
+ * counts toward the state the run was in, the `from.value` of the
42
+ * transition its choice causes.
43
+ *
44
+ * A trace is one run's events, live (`AgentTraceEvent`) or serialized
45
+ * (`serializeTraceEvent`). A host-driven `decideNext` traces nothing; push
46
+ * `{ type: 'request.start', request: result.request }` onto the trace before
47
+ * sending its event. Parallel states count once per active leaf.
48
+ *
49
+ * `failures` counts a failed run against every state it visited, so states
50
+ * after a bad decision rank as high as the decision. Sort by `blamed` to see
51
+ * where failures start.
52
+ *
53
+ * @example
54
+ * ```ts no-check
55
+ * const stats = foldTraces(traces, {
56
+ * failed: (trace) => trace.at(-1)?.type === 'run.end' && trace.at(-1).status === 'error',
57
+ * });
58
+ * ```
59
+ */
60
+ export declare function foldTraces<TTrace extends readonly {
61
+ readonly type: string;
62
+ }[]>(traces: Iterable<TTrace>, options: FoldTracesOptions<TTrace>): StateStats[];
63
+ //#endregion
64
+ //#region src/trace-coverage.d.ts
65
+ /** One state's row in a {@link CoverageReport}. */
66
+ interface StateCoverage {
67
+ /** The state's dot-joined path from the root, e.g. `'deciding'` or `'work.review'`. */
68
+ state: string;
69
+ /** Runs that were in this state at least once. */
70
+ runs: number;
71
+ }
72
+ /** A handler: the state node that declares it and its event descriptor. */
73
+ interface HandlerRef {
74
+ /** The declaring state's dot path; `''` for the machine root. */
75
+ state: string;
76
+ /** The descriptor as XState keys it: `'APPROVE'`, `'task.*'`, `'*'`, `'xstate.done.actor'`, `'xstate.error.actor'`, `'xstate.after'`. */
77
+ event: string;
78
+ }
79
+ /** One handler's row in a {@link CoverageReport}. */
80
+ interface HandlerCoverage extends HandlerRef {
81
+ /** Runs in which this handler was credited at least once. */
82
+ runs: number;
83
+ targets: {
84
+ /** Target state paths of the object transitions under this descriptor; `'dynamic'` when any is a function transition. */
85
+ declared: string[] | "dynamic";
86
+ /** Active leaves observed right after a step credited to this handler. */
87
+ seen: string[];
88
+ /** Declared targets that no seen leaf is in. Empty when `declared` is `'dynamic'`. */
89
+ unreached: string[];
90
+ };
91
+ }
92
+ /** The result of {@link traceCoverage}. */
93
+ interface CoverageReport {
94
+ /** Every atomic and final state, in document order, including `runs: 0`. */
95
+ states: StateCoverage[];
96
+ /** Every observable handler, in document order, including `runs: 0`. */
97
+ handlers: HandlerCoverage[];
98
+ /** States no run visited, excluding `unreachable` ones. */
99
+ unvisitedStates: string[];
100
+ /** Handlers no run was credited with. */
101
+ unvisitedHandlers: HandlerRef[];
102
+ /** `always` handlers: they fire inside a macrostep, which a trace does not record. */
103
+ unobservable: HandlerRef[];
104
+ /**
105
+ * With a complete `reachable` report: states no explored path reached and
106
+ * no trace visited. They leave the state denominator.
107
+ */
108
+ unreachable?: string[];
109
+ /**
110
+ * With `reachable`: whether the report explored every path. When `false`,
111
+ * a state it missed may still be reachable, so `unreachable` is absent and
112
+ * every state stays in the denominator.
113
+ */
114
+ reachableComplete?: boolean;
115
+ /** Visited over total, for states and for handlers (unobservable excluded). 1 when the total is 0. */
116
+ ratio: {
117
+ states: number;
118
+ handlers: number;
119
+ };
120
+ }
121
+ /** Options for {@link traceCoverage}. */
122
+ interface TraceCoverageOptions {
123
+ /**
124
+ * An `explorePaths` report. When it explored every path (no cap hit, every
125
+ * payload expanded, no branch stopped early), states it never reached drop
126
+ * out of the state denominator.
127
+ */
128
+ reachable?: AgentPathReport;
129
+ }
130
+ /**
131
+ * Measures how much of a machine a set of runs exercised: which atomic and
132
+ * final states they visited, and which handlers they took. A handler is a
133
+ * `(declaring state, event descriptor)` pair from `on`, invoke
134
+ * `onDone`/`onError`, and `after`, with the targets it declares and the
135
+ * leaves the runs landed in after it. Pure; reads only `machine.transition`
136
+ * events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
137
+ *
138
+ * A step credits one handler per active leaf: the nearest node, from the leaf
139
+ * up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
140
+ * event, preferred in XState's order. Guards are not replayed, so a step
141
+ * whose nearest handler was rejected and fell through to an ancestor is
142
+ * credited to the nearest one. XState keys invoke and `after` handlers by
143
+ * bare type (`xstate.done.actor`), one per state however many invokes; a
144
+ * definition's `matches` (an invoke's `actorId`) must accept the traced event
145
+ * for its state to be credited. Ignored steps credit no handler. `always`
146
+ * handlers are listed as `unobservable`, never as unvisited; a state one
147
+ * passes through is visited, re-derived from each step's microsteps.
148
+ *
149
+ * @example
150
+ * ```ts no-check
151
+ * const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
152
+ * expect(report.unvisitedHandlers).toEqual([]);
153
+ * ```
154
+ */
155
+ export declare function traceCoverage<TTrace extends readonly {
156
+ readonly type: string;
157
+ }[]>(machine: AnyStateMachine, traces: Iterable<TTrace>, options?: TraceCoverageOptions): CoverageReport;
158
+ //#endregion
159
+ //#region src/internal/trace-states.d.ts
160
+ /** One macrostep read from a trace: a `machine.transition` event. */
161
+ interface TraceStep {
162
+ /** Active leaves before the step. */
163
+ from: string[];
164
+ /** The state value before the step, as traced. */
165
+ fromValue: unknown;
166
+ /** The event's `type`. */
167
+ event: string;
168
+ /** The event object, as traced. */
169
+ payload: unknown;
170
+ /** Active leaves after the step. */
171
+ to: string[];
172
+ /** The state value after the step, as traced. */
173
+ toValue: unknown;
174
+ /** The state had no transition for a host event, so nothing changed. */
175
+ ignored: boolean;
176
+ /** The `name` of the first-attempt decision request that started since the previous step. */
177
+ decision?: string;
178
+ }
179
+ //#endregion
180
+ //#region src/first-divergence.d.ts
181
+ /** Options for {@link firstDivergence}. */
182
+ interface FirstDivergenceOptions {
183
+ /**
184
+ * Projects a step to the key two traces are compared by. Default
185
+ * `from|event|to`. Override to leave out what may differ between equivalent
186
+ * runs, such as a parallel region that only logs. Steps are still compared
187
+ * in order. Trace references only.
188
+ *
189
+ * `"payload"` also compares the event's payload, such as an actor's output,
190
+ * as key-order-independent JSON without its `sessionId`, and with errors as
191
+ * `{ name, message, code?, cause? }` (an `Error`, or a serialized one with `stack` or
192
+ * `code`). Use it for deterministic or mocked runs; live model outputs usually differ
193
+ * between runs.
194
+ */
195
+ key?: ((step: TraceStep) => string) | "payload";
196
+ }
197
+ /** Where a run first departs from its reference. See {@link firstDivergence}. */
198
+ interface Divergence {
199
+ /** The step's index in the run's steps, or the step count when the run ended early. */
200
+ index: number;
201
+ /** The state the run was in when it departed: the first active leaf before that step. */
202
+ state: string;
203
+ /** The decision request that chose that step's event, when one did. */
204
+ decision?: string;
205
+ /** The run's step. Absent when the run ended early. */
206
+ actual?: TraceStep;
207
+ /** The reference's step, or the expected trajectory item the run never reached. Absent when the run went on past a reference trace. */
208
+ expected?: TraceStep | TrajectoryItem;
209
+ }
210
+ /**
211
+ * Finds the first step where a run departs from a reference, so a failure is
212
+ * blamed on the state it started in rather than on every state the run went
213
+ * through afterwards.
214
+ *
215
+ * - Against a reference **trace**, such as a passing run of the same
216
+ * scenario, steps are compared in order by `key`. The first step that
217
+ * differs, or the point where either trace ends early, is the divergence.
218
+ * - Against an expected **trajectory** (`TrajectoryItem[]`, as for
219
+ * {@link matchesTrajectory}), the run is laid out as its initial state, then
220
+ * each step's event and new state. The first expected item it never reaches
221
+ * is the divergence, blamed on the step after the last item it did reach. A
222
+ * run that goes on past the trajectory has not diverged.
223
+ *
224
+ * An empty reference is a trace with no steps, so any step diverges from it.
225
+ * A trace is one run's events, live or serialized. Steps the machine ignored
226
+ * changed nothing and are left out on both sides. Returns `undefined` when the
227
+ * run does not diverge.
228
+ *
229
+ * @example
230
+ * ```ts no-check
231
+ * const stats = foldTraces(traces, {
232
+ * failed,
233
+ * blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
234
+ * });
235
+ * ```
236
+ */
237
+ export declare function firstDivergence(actual: readonly {
238
+ readonly type: string;
239
+ }[], reference: readonly {
240
+ readonly type: string;
241
+ }[] | readonly TrajectoryItem[], options?: FirstDivergenceOptions): Divergence | undefined;
242
+ //#endregion
243
+ //#region src/propose-from-traces.d.ts
244
+ /** A person's verdict as {@link proposeApprovalRules} hands it to `key`. */
245
+ interface ApprovalVerdict<TTrace> {
246
+ /** The approve or reject event, as traced. */
247
+ event: Record<string, unknown>;
248
+ /** The traced snapshot the verdict arrived in: the approval state, with the context that holds the pending call. */
249
+ snapshot: unknown;
250
+ /** The whole run. */
251
+ trace: TTrace;
252
+ }
253
+ /** Options for {@link proposeApprovalRules}. */
254
+ interface ProposeApprovalRulesOptions<TTrace> {
255
+ /** The state's dot path, such as `'awaitingApproval'`. A verdict counts when the run was in it or in a state inside it. */
256
+ approvalState: string;
257
+ /** Event types that approve, such as `['APPROVE']`. */
258
+ approveEvents: readonly string[];
259
+ /** Event types that reject, such as `['REJECT']`. */
260
+ rejectEvents: readonly string[];
261
+ /**
262
+ * The rule a verdict is evidence for: usually the tool name and a pattern
263
+ * of its arguments, read from `snapshot.context`. `undefined` skips the verdict.
264
+ */
265
+ key: (verdict: ApprovalVerdict<TTrace>) => string | undefined;
266
+ /**
267
+ * The keys a run broke a rule for, from ground truth (a ledger checked
268
+ * after the fact). Any violation blocks `allow` for that key.
269
+ */
270
+ violations?: (trace: TTrace) => Iterable<string>;
271
+ }
272
+ /** One key's row from {@link proposeApprovalRules}. */
273
+ interface ApprovalRuleProposal {
274
+ key: string;
275
+ /** Approve verdicts, summed over runs. */
276
+ approved: number;
277
+ /** Reject verdicts, summed over runs. */
278
+ rejected: number;
279
+ /** Violations `violations` reported, summed over runs. 0 without `violations`. */
280
+ violations: number;
281
+ /** Indexes, in input order, of the runs with a verdict for this key. */
282
+ runs: number[];
283
+ /**
284
+ * `'allow'`: approved, never rejected, no violation. `'deny'`: rejected,
285
+ * never approved. `null`: mixed evidence.
286
+ */
287
+ proposal: "allow" | "deny" | null;
288
+ /** Runs with a verdict for this key: `runs.length`. */
289
+ support: number;
290
+ }
291
+ /**
292
+ * Groups the approval verdicts in many runs' traces by `key` and proposes an
293
+ * `allow` or `deny` rule for each. Pure and deterministic; a proposal is for
294
+ * a person to review, and nothing here changes a machine.
295
+ *
296
+ * Reads `machine.transition` events: a verdict is a transition, not ignored,
297
+ * out of `approvalState` on an approve or reject event. Rows come in the
298
+ * order keys were first seen.
299
+ *
300
+ * @example
301
+ * ```ts no-check
302
+ * const rows = proposeApprovalRules(traces, {
303
+ * approvalState: "awaitingApproval",
304
+ * approveEvents: ["APPROVE"],
305
+ * rejectEvents: ["REJECT"],
306
+ * key: ({ snapshot }) => `${snapshot.context.pending.tool}:${snapshot.context.pending.path}`,
307
+ * });
308
+ * ```
309
+ */
310
+ export declare function proposeApprovalRules<TTrace extends readonly {
311
+ readonly type: string;
312
+ }[]>(traces: Iterable<TTrace>, options: ProposeApprovalRulesOptions<TTrace>): ApprovalRuleProposal[];
313
+ /** One cap for {@link proposeCaps}. Give `failureState` or `failureOutput`. */
314
+ interface CapSpec<TTrace> {
315
+ /** Names the row, such as `'retries'`. */
316
+ name: string;
317
+ /** The cap's value now. A cap runs hit is never proposed below it. */
318
+ current?: number;
319
+ /** The run hit the cap when it entered this state (dot path), such as `'gaveUp'`. */
320
+ failureState?: string;
321
+ /** The run hit the cap when its `run.end` output satisfies this. */
322
+ failureOutput?: (output: unknown) => boolean;
323
+ /** How much of the cap the run used: retries made, steps taken, batch size. `undefined` leaves the run out. */
324
+ valueFromRun: (trace: TTrace) => number | undefined;
325
+ }
326
+ /** One cap's row from {@link proposeCaps}. */
327
+ interface CapProposal {
328
+ name: string;
329
+ /** Runs that hit the cap. */
330
+ hits: number;
331
+ /** Runs that did not, with a value. */
332
+ passing: number;
333
+ /** The largest value over all runs; `null` with none. */
334
+ max: number | null;
335
+ /** The 95th percentile (nearest rank) over passing runs; `null` with none. */
336
+ p95: number | null;
337
+ /** `p95 + 1`, and never below `current` when any run hit the cap; `null` with no passing run. */
338
+ proposal: number | null;
339
+ }
340
+ /** Options for {@link proposeCaps}. */
341
+ interface ProposeCapsOptions<TTrace> {
342
+ caps: readonly CapSpec<TTrace>[];
343
+ }
344
+ /**
345
+ * Proposes a value for each cap (retry count, steps, batch size) from many
346
+ * runs' traces: the 95th percentile of what passing runs used, plus one, and
347
+ * never below the current cap while runs still hit it. Pure and
348
+ * deterministic; a proposal is for a person to review, and nothing here
349
+ * changes a machine.
350
+ *
351
+ * @example
352
+ * ```ts no-check
353
+ * const [retries] = proposeCaps(traces, {
354
+ * caps: [{ name: "retries", current: 3, failureState: "gaveUp", valueFromRun: countRetries }],
355
+ * });
356
+ * ```
357
+ */
358
+ export declare function proposeCaps<TTrace extends readonly {
359
+ readonly type: string;
360
+ }[]>(traces: Iterable<TTrace>, options: ProposeCapsOptions<TTrace>): CapProposal[];
361
+ //#endregion
362
+ //#region src/decision-requests.d.ts
363
+ /** One decision a run asked for. See {@link getDecisionRequests}. */
364
+ interface TracedDecisionRequest {
365
+ /** The active states when the decision was asked, as dot paths (one per parallel region). */
366
+ states: string[];
367
+ /** The request as the executor received it, on its first attempt. */
368
+ request: AgentExecutorDecisionRequest;
369
+ }
370
+ /** Options for {@link getDecisionRequests}. */
371
+ interface GetDecisionRequestsOptions {
372
+ /** Keep only requests with this `name` (the `agent.decide` input's `name`). */
373
+ name?: string;
374
+ }
375
+ /**
376
+ * The decisions a run asked for, read from its trace: each decision's first
377
+ * `request.start`, with the states the run was in. A trace is one run's events,
378
+ * live or serialized with `serializeTraceEvent`. Retries of the same decision
379
+ * are left out, so each decision appears once.
380
+ *
381
+ * Use the requests as eval rows (run a decider on each and compare its event
382
+ * with a label) or as probe inputs for a policy. A serialized request keeps its
383
+ * `options`, `metadata`, `system`, and `prompt`; event schemas do not survive
384
+ * serialization, so replay closed choices (`options`) rather than open ones.
385
+ *
386
+ * @example
387
+ * ```ts no-check
388
+ * const rows = traces.flatMap((trace) => getDecisionRequests(trace, { name: "chooseResolution" }));
389
+ * for (const { request } of rows) console.log((await decide(request)).event);
390
+ * ```
391
+ */
392
+ export declare function getDecisionRequests(trace: Iterable<{
393
+ readonly type: string;
394
+ }>, options?: GetDecisionRequestsOptions): TracedDecisionRequest[];
395
+ //#endregion
396
+ //#region src/repeat-runs.d.ts
397
+ type AnyTrace = readonly {
398
+ readonly type: string;
399
+ }[];
400
+ /** One run of a repeated scenario. See {@link summarizeRepeats}. */
401
+ interface RepeatRun<TTrace extends AnyTrace = AnyTrace> {
402
+ passed: boolean;
403
+ /** The run's trace. Absent for a run that threw. */
404
+ trace?: TTrace;
405
+ /** How the run ended, for the report. Not compared. */
406
+ outcome?: string;
407
+ /** The message of a run that threw. */
408
+ error?: string;
409
+ }
410
+ /** One scenario's runs. See {@link summarizeRepeats}. */
411
+ interface RepeatGroup<TScenario, TTrace extends AnyTrace = AnyTrace> {
412
+ id: string;
413
+ scenario: TScenario;
414
+ runs: readonly RepeatRun<TTrace>[];
415
+ }
416
+ /** One scenario's row in a {@link RepeatReport}. */
417
+ interface RepeatResult<TScenario, TTrace extends AnyTrace = AnyTrace> extends RepeatGroup<TScenario, TTrace> {
418
+ /** Runs that passed. */
419
+ passes: number;
420
+ /** At least one run passed: pass@k for this scenario. */
421
+ passAny: boolean;
422
+ /** Every run passed: pass^k for this scenario. */
423
+ passAll: boolean;
424
+ /** Distinct state paths the traced runs took. 1 when every run went the same way. */
425
+ distinctPaths: number;
426
+ /** The state where the runs first went different ways, judged against the first run whose trace has a transition. */
427
+ divergedAt?: string;
428
+ }
429
+ /** The result of {@link summarizeRepeats} and {@link repeatRuns}. */
430
+ interface RepeatReport<TScenario, TTrace extends AnyTrace = AnyTrace> {
431
+ scenarios: RepeatResult<TScenario, TTrace>[];
432
+ /**
433
+ * The suite's pass^j: the chance that `j` runs of a scenario, drawn from the
434
+ * ones recorded, all pass, averaged over scenarios. The unbiased estimate is
435
+ * C(passes, j) / C(runs, j) per scenario. Scenarios with fewer than `j` runs
436
+ * are left out. Throws when `j` is not a whole number from 1 to the most
437
+ * runs any scenario has.
438
+ */
439
+ passHat: (j: number) => number;
440
+ /** Every recorded trace, ready for `foldTraces` or `traceCoverage`. */
441
+ traces: TTrace[];
442
+ }
443
+ /** Options for {@link repeatRuns}. */
444
+ interface RepeatRunsOptions<TScenario extends {
445
+ id: string;
446
+ }, TTrace extends AnyTrace> {
447
+ scenarios: readonly TScenario[];
448
+ /** Runs per scenario. */
449
+ k: number;
450
+ /** Runs one scenario once and returns its trace. `attempt` counts from 0. */
451
+ run: (scenario: TScenario, attempt: number) => Promise<{
452
+ trace: TTrace;
453
+ outcome?: string;
454
+ }>;
455
+ /** Whether a run passed, judged from its ground truth. */
456
+ passed: (scenario: TScenario, result: {
457
+ trace: TTrace;
458
+ outcome?: string;
459
+ }) => boolean;
460
+ }
461
+ /**
462
+ * Summarizes repeated runs of each scenario: how often each passed, whether
463
+ * all its runs passed (pass^k), how many distinct state paths they took, and
464
+ * where they first went different ways. Pure. Use it for runs collected
465
+ * elsewhere, such as CI shards or replays; {@link repeatRuns} collects them and
466
+ * calls it.
467
+ *
468
+ * A scenario that passes 3 of 5 runs and `divergedAt: 'deciding'` points at one
469
+ * decision, not at a flaky agent.
470
+ *
471
+ * @example
472
+ * ```ts no-check
473
+ * const report = summarizeRepeats(shards.map((s) => ({ id: s.id, scenario: s, runs: s.runs })));
474
+ * report.passHat(3); // the chance 3 runs of a scenario all pass
475
+ * ```
476
+ */
477
+ export declare function summarizeRepeats<TScenario, TTrace extends AnyTrace = AnyTrace>(groups: readonly RepeatGroup<TScenario, TTrace>[]): RepeatReport<TScenario, TTrace>;
478
+ /**
479
+ * Runs every scenario `k` times and summarizes the runs with
480
+ * {@link summarizeRepeats}. Runs are sequential, scenario by scenario. A run
481
+ * that throws fails, with its message as `error` and no trace.
482
+ *
483
+ * With a live model, one run per scenario says little: τ-bench found pass^k
484
+ * falls well below the single-run rate. Report pass^k next to it.
485
+ *
486
+ * @example
487
+ * ```ts no-check
488
+ * const report = await repeatRuns({
489
+ * scenarios,
490
+ * k: 5,
491
+ * run: async (scenario) => {
492
+ * const trace: AgentTraceEvent[] = [];
493
+ * const result = await runToQuiescence(
494
+ * createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
495
+ * { input: scenario.input },
496
+ * );
497
+ * return { trace, outcome: String(result.snapshot.value) };
498
+ * },
499
+ * passed: (scenario, { outcome }) => outcome === scenario.expected,
500
+ * });
501
+ * report.passHat(5);
502
+ * ```
503
+ */
504
+ export declare function repeatRuns<TScenario extends {
505
+ id: string;
506
+ }, TTrace extends AnyTrace>(options: RepeatRunsOptions<TScenario, TTrace>): Promise<RepeatReport<TScenario, TTrace>>;
507
+ //#endregion
508
+ //#region src/judge-agreement.d.ts
509
+ /** One labeled case: what a person said and what the judge said. See {@link judgeAgreement}. */
510
+ interface AgreementRow {
511
+ /** Identifies the case in `disagreements`. */
512
+ id: string;
513
+ /** The confirmed human label. */
514
+ human: string;
515
+ /** The judge's label for the same case. */
516
+ judge: string;
517
+ /** The group the row is aligned in, e.g. a decision state or request name. `groupBy` overrides it. */
518
+ group?: string;
519
+ }
520
+ /** Agreement numbers for one set of rows. See {@link AgreementReport}. */
521
+ interface AgreementSummary {
522
+ /** Rows counted. */
523
+ total: number;
524
+ /** Share of rows where `judge` equals `human`. */
525
+ accuracy: number;
526
+ /** Per human label: rows with that label, and how many the judge matched. `agreed / human` is that label's recall. */
527
+ perLabel: Record<string, {
528
+ human: number;
529
+ agreed: number;
530
+ }>;
531
+ /** Counts by human label, then judge label. */
532
+ confusion: Record<string, Record<string, number>>;
533
+ /** Rows where the judge differs from the person, in input order. */
534
+ disagreements: {
535
+ id: string;
536
+ human: string;
537
+ judge: string;
538
+ }[];
539
+ /**
540
+ * With `positive` set: the true positive rate (judge says positive when the
541
+ * person did) and true negative rate (judge says any other label when the
542
+ * person did). A rate is `undefined` when no row has that human side.
543
+ */
544
+ binary?: {
545
+ tpr: number | undefined;
546
+ tnr: number | undefined;
547
+ };
548
+ /**
549
+ * With `positive` set: estimates the true share of positives among
550
+ * unlabeled rows from the share `p` the judge labeled positive,
551
+ * θ = (p + TNR − 1) / (TPR + TNR − 1), clamped to [0, 1]. Omitted when
552
+ * either rate is `undefined`, or when TPR + TNR ≤ 1: the judge is no better
553
+ * than chance, so its labels carry no signal to correct.
554
+ */
555
+ correct?: (observedPositiveRate: number) => number;
556
+ }
557
+ /** The result of {@link judgeAgreement}. */
558
+ interface AgreementReport extends AgreementSummary {
559
+ /** The same numbers per group. Present when any row has `group` or `groupBy` is given. */
560
+ groups?: Record<string, AgreementSummary>;
561
+ }
562
+ /** Options for {@link judgeAgreement}. */
563
+ interface JudgeAgreementOptions<TRow extends AgreementRow> {
564
+ /** The positive label. Every other label is negative. Turns on `binary` and `correct`. */
565
+ positive?: string;
566
+ /** The row's group, overriding `row.group`. Return `undefined` to leave the row out of every group. */
567
+ groupBy?: (row: TRow) => string | undefined;
568
+ }
569
+ /**
570
+ * Measures how well a judge agrees with people on the same cases: accuracy,
571
+ * per-label recall, the confusion matrix, and the rows that disagree. With
572
+ * `positive`, also the TPR and TNR, the recommended view for binary judges
573
+ * (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
574
+ * Pure.
575
+ *
576
+ * Build rows from confirmed labels only; draft labels gate nothing. Group by
577
+ * decision state or request name so each narrow judge is aligned on its own.
578
+ * A row with no group key counts in the totals and in no group. Run it on a
579
+ * dev split and report the held-out split separately.
580
+ *
581
+ * Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
582
+ *
583
+ * @example
584
+ * ```ts no-check
585
+ * const report = judgeAgreement(
586
+ * labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
587
+ * { positive: 'fail' },
588
+ * );
589
+ * report.binary; // { tpr: 0.9, tnr: 0.8 }
590
+ * report.correct?.(0.3); // estimated true failure rate in production
591
+ * ```
592
+ */
593
+ export declare function judgeAgreement<TRow extends AgreementRow>(rows: readonly TRow[], options?: JudgeAgreementOptions<TRow>): AgreementReport;
594
+ //#endregion
595
+ export type { AgreementReport, AgreementRow, AgreementSummary, ApprovalRuleProposal, ApprovalVerdict, CapProposal, CapSpec, CoverageReport, Divergence, FirstDivergenceOptions, FoldTracesOptions, GetDecisionRequestsOptions, HandlerCoverage, HandlerRef, JudgeAgreementOptions, ProposeApprovalRulesOptions, ProposeCapsOptions, RepeatGroup, RepeatReport, RepeatResult, RepeatRun, RepeatRunsOptions, StateCoverage, StateStats, TraceCoverageOptions, TraceStep, TracedDecisionRequest };