@statelyai/agent 2.0.0-alpha.38 → 2.0.0-alpha.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -0
- package/dist/{act-CrX0JwWY.cjs → act-B0CVbmo4.cjs} +1 -1
- package/dist/{act-B9hLQls6.mjs → act-Bsz-hUfD.mjs} +1 -1
- package/dist/ai-sdk.cjs +4 -4
- package/dist/ai-sdk.d.cts +2 -3
- package/dist/ai-sdk.d.mts +2 -3
- package/dist/ai-sdk.mjs +4 -4
- package/dist/{decision-m5hGmHpF.mjs → decision-B41YBx5U.mjs} +1 -1
- package/dist/{decision-CMeuNZ1g.cjs → decision-D8bcIGXQ.cjs} +1 -1
- package/dist/{decision-object-DBcLWVvv.d.cts → decision-object-BB8XV5Ec.d.cts} +1 -1
- package/dist/{decision-object-B5zGSPFp.d.mts → decision-object-CH_sE694.d.mts} +1 -1
- package/dist/effects-B9IvUPVh.mjs +23 -0
- package/dist/effects-DrWcUGgc.cjs +34 -0
- package/dist/evals.cjs +810 -0
- package/dist/evals.d.cts +595 -0
- package/dist/evals.d.mts +595 -0
- package/dist/evals.mjs +801 -0
- package/dist/{event-log-DkVlk3Rh.mjs → event-log-CEq_bVXQ.mjs} +8 -4
- package/dist/{event-log-Dkc-yIYu.cjs → event-log-dSOnXHuG.cjs} +7 -15
- package/dist/{event-log-store-Brx9tOHt.mjs → event-log-store-DLAfNM34.mjs} +1 -1
- package/dist/{event-log-store-BBS8so8a.d.mts → event-log-store-DYun526m.d.mts} +15 -14
- package/dist/{event-log-store-Cl-YWDnq.cjs → event-log-store-Dey9ufA4.cjs} +1 -1
- package/dist/{event-log-store-bc4ICGcS.d.cts → event-log-store-aFXK6yZX.d.cts} +15 -14
- package/dist/index.cjs +142 -97
- package/dist/index.d.cts +97 -141
- package/dist/index.d.mts +97 -141
- package/dist/index.mjs +123 -74
- package/dist/internal-BzlpzSux.d.cts +14 -0
- package/dist/internal-BzlpzSux.d.mts +14 -0
- package/dist/log.cjs +2 -3
- package/dist/log.d.cts +2 -2
- package/dist/log.d.mts +2 -2
- package/dist/log.mjs +3 -3
- package/dist/machine-input-0YTPcnCm.mjs +27 -0
- package/dist/machine-input-DU3owLUr.cjs +32 -0
- package/dist/machines.cjs +1 -719
- package/dist/machines.d.cts +2 -424
- package/dist/machines.d.mts +2 -424
- package/dist/machines.mjs +2 -709
- package/dist/{model-source-kjGwqAV8.mjs → model-source-BrsLDvHL.mjs} +1 -1
- package/dist/{model-source-B4WQ8Idp.cjs → model-source-YpxjmWuX.cjs} +1 -1
- package/dist/openai.cjs +3 -3
- package/dist/openai.d.cts +2 -3
- package/dist/openai.d.mts +2 -3
- package/dist/openai.mjs +3 -3
- package/dist/otel.d.cts +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/policy.cjs +77 -0
- package/dist/policy.d.cts +143 -0
- package/dist/policy.d.mts +143 -0
- package/dist/policy.mjs +75 -0
- package/dist/{run-agent-DlPzBWj-.d.mts → run-agent-BO9RSb7i.d.cts} +33 -40
- package/dist/{run-agent-30mH2moE.d.cts → run-agent-BxkeuAYR.d.mts} +33 -40
- package/dist/{effects-DDseYtJk.cjs → run-agent-COdR9FpM.cjs} +175 -228
- package/dist/{effects-BT_cSY0j.mjs → run-agent-D3gW9wfd.mjs} +176 -211
- package/dist/{setup-agent-BDmNr6PC.mjs → setup-agent-CLm6QVGy.mjs} +3 -3
- package/dist/{setup-agent-BaNyz6Q3.d.cts → setup-agent-Cv8Hk3mX.d.cts} +12 -11
- package/dist/{setup-agent-C1CG_nfX.d.mts → setup-agent-DgFAhaPJ.d.mts} +12 -11
- package/dist/{setup-agent-CmEToB8y.cjs → setup-agent-Dv5VpuHe.cjs} +3 -3
- package/dist/tanstack-ai.cjs +4 -4
- package/dist/tanstack-ai.d.cts +2 -3
- package/dist/tanstack-ai.d.mts +2 -3
- package/dist/tanstack-ai.mjs +4 -4
- package/dist/testing.cjs +122 -996
- package/dist/testing.d.cts +36 -695
- package/dist/testing.d.mts +36 -695
- package/dist/testing.mjs +112 -977
- package/dist/{text-logic-BqxalgMr.d.mts → text-logic-x17uLBwA.d.cts} +238 -2
- package/dist/{text-logic-CSf9bqpP.d.cts → text-logic-x17uLBwA.d.mts} +238 -2
- package/dist/trajectory-BFjXaooB.d.cts +96 -0
- package/dist/trajectory-BFjXaooB.d.mts +96 -0
- package/dist/trajectory-BUXj_1Ue.mjs +152 -0
- package/dist/trajectory-CoZSv4wF.cjs +157 -0
- package/dist/{usage-Bm5nvpow.d.cts → usage-B4B-TQSV.d.mts} +1 -1
- package/dist/{usage-BnikdOca.d.mts → usage-DvGcoK6x.d.cts} +1 -1
- package/dist/{utils-7lSa4H-i.d.cts → utils-B8k-YPYI.d.mts} +23 -17
- package/dist/{utils-CBbKw2fM.d.mts → utils-DAclF-kw.d.cts} +23 -17
- package/dist/validate.cjs +4 -4
- package/dist/validate.d.cts +2 -2
- package/dist/validate.d.mts +2 -2
- package/dist/validate.mjs +4 -4
- package/dist/{verify-DJHfEfm6.d.cts → verify-CIu4mqIZ.d.mts} +56 -6
- package/dist/{verify-i9Hiwc5g.d.mts → verify-CTHlmBbh.d.cts} +56 -6
- package/dist/{verify-ChyX6jq0.mjs → verify-Cacz6XSk.mjs} +170 -46
- package/dist/{verify-CwlombbC.cjs → verify-D8UDqU4l.cjs} +170 -46
- package/dist/{with-model-calls-B_WLmOSl.cjs → with-model-calls-D50Ca6qW.cjs} +2 -2
- package/dist/{with-model-calls-BkOwdroS.mjs → with-model-calls-uY-KKliA.mjs} +2 -2
- package/docs/advanced.md +5 -5
- package/docs/choosing-a-run-mode.md +7 -7
- package/docs/debugging.md +6 -8
- package/docs/decisions.md +5 -4
- package/docs/environment-and-policy.md +1 -1
- package/docs/evals.md +57 -43
- package/docs/event-log.md +13 -15
- package/docs/from-a-loop.md +3 -3
- package/docs/generate-machines.md +4 -4
- package/docs/hosts.md +25 -30
- package/docs/human-in-the-loop.md +5 -8
- package/docs/index.md +4 -4
- package/docs/judgments.md +5 -7
- package/docs/langgraph-comparison.md +6 -6
- package/docs/machines-as-data.md +7 -12
- package/docs/machines-presets.md +17 -120
- package/docs/machines.md +13 -13
- package/docs/models-and-providers.md +19 -25
- package/docs/multi-agent.md +1 -1
- package/docs/observability.md +6 -8
- package/docs/persistence.md +10 -20
- package/docs/quickstart.md +3 -3
- package/docs/statechart-policy-examples.md +1 -1
- package/docs/steps.md +5 -6
- package/docs/strangler-fig.md +105 -33
- package/docs/text-requests.md +10 -12
- package/docs/thinking-in-state-machines.md +8 -8
- package/docs/tools.md +10 -12
- package/docs/upgrading.md +1 -1
- package/docs/usage-and-budgets.md +51 -56
- package/docs/validate-and-repair.md +9 -14
- package/docs/verify.md +54 -7
- package/docs/xstate-4-and-5-apps.md +19 -28
- package/llms.txt +1 -1
- package/package.json +31 -6
- package/readme.md +32 -24
- package/dist/internal-C3Lqu538.d.cts +0 -43
- package/dist/internal-DUcdhk-i.d.mts +0 -43
- package/dist/types-DQUOOaZO.d.cts +0 -238
- package/dist/types-DQUOOaZO.d.mts +0 -238
- package/skills/add-evals/SKILL.md +0 -152
- package/skills/add-traces/SKILL.md +0 -152
- package/skills/build-machine/SKILL.md +0 -84
- package/skills/generate-machine/SKILL.md +0 -335
- package/skills/graph-engineering/SKILL.md +0 -43
- package/skills/improve-machine/SKILL.md +0 -261
- package/skills/improve-machine/reference.md +0 -101
- package/skills/loop-to-machine/SKILL.md +0 -205
- package/skills/loop-to-machine/policy.md +0 -154
- package/skills/loop-to-machine/reference.md +0 -260
package/dist/evals.d.cts
ADDED
|
@@ -0,0 +1,595 @@
|
|
|
1
|
+
import { R as AgentExecutorDecisionRequest } from "./text-logic-x17uLBwA.cjs";
|
|
2
|
+
import { i as AgentPathReport } from "./verify-CTHlmBbh.cjs";
|
|
3
|
+
import { i as TrajectoryItem } from "./trajectory-BFjXaooB.cjs";
|
|
4
|
+
import { AnyStateMachine } from "xstate";
|
|
5
|
+
//#region src/fold-traces.d.ts
|
|
6
|
+
/** One state's numbers across many runs. See {@link foldTraces}. */
|
|
7
|
+
interface StateStats {
|
|
8
|
+
/** The state's dot-joined path from the root, e.g. `'deciding'` or `'work.review'`. */
|
|
9
|
+
state: string;
|
|
10
|
+
/** Runs that were in this state at least once. */
|
|
11
|
+
visits: number;
|
|
12
|
+
/** Of those runs, the ones `failed` flagged. */
|
|
13
|
+
failures: number;
|
|
14
|
+
/** Options withheld by the guards at decisions made in this state, summed over runs. */
|
|
15
|
+
withheld: number;
|
|
16
|
+
/** Events this state ignored (`machine.transition` with `ignored`), summed over runs. */
|
|
17
|
+
ignored: number;
|
|
18
|
+
/** Failed runs `blame` attributed to this state. 0 without `blame`. */
|
|
19
|
+
blamed: number;
|
|
20
|
+
}
|
|
21
|
+
/** Options for {@link foldTraces}. */
|
|
22
|
+
interface FoldTracesOptions<TTrace> {
|
|
23
|
+
/** Whether a run failed, judged from its whole trace. */
|
|
24
|
+
failed: (trace: TTrace) => boolean;
|
|
25
|
+
/**
|
|
26
|
+
* The state a failed run's failure started in, usually
|
|
27
|
+
* `firstDivergence(trace, reference)?.state`. Called for failed runs only;
|
|
28
|
+
* `undefined` blames no state.
|
|
29
|
+
*/
|
|
30
|
+
blame?: (trace: TTrace) => string | undefined;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Folds many runs' traces into per-state stats: how many runs visited each
|
|
34
|
+
* state, how many of those failed, how many options the guards withheld
|
|
35
|
+
* at decisions made there, how many events it ignored, and how many failures
|
|
36
|
+
* `blame` traced back to it. Pure; reads only:
|
|
37
|
+
*
|
|
38
|
+
* - `machine.transition`: the run visited `from.value` and `snapshot.value`;
|
|
39
|
+
* with `ignored`, each of those states ignored one event.
|
|
40
|
+
* - `request.start` of a decision, first attempt only: its `request.withheld`
|
|
41
|
+
* counts toward the state the run was in, the `from.value` of the
|
|
42
|
+
* transition its choice causes.
|
|
43
|
+
*
|
|
44
|
+
* A trace is one run's events, live (`AgentTraceEvent`) or serialized
|
|
45
|
+
* (`serializeTraceEvent`). A host-driven `decideNext` traces nothing; push
|
|
46
|
+
* `{ type: 'request.start', request: result.request }` onto the trace before
|
|
47
|
+
* sending its event. Parallel states count once per active leaf.
|
|
48
|
+
*
|
|
49
|
+
* `failures` counts a failed run against every state it visited, so states
|
|
50
|
+
* after a bad decision rank as high as the decision. Sort by `blamed` to see
|
|
51
|
+
* where failures start.
|
|
52
|
+
*
|
|
53
|
+
* @example
|
|
54
|
+
* ```ts no-check
|
|
55
|
+
* const stats = foldTraces(traces, {
|
|
56
|
+
* failed: (trace) => trace.at(-1)?.type === 'run.end' && trace.at(-1).status === 'error',
|
|
57
|
+
* });
|
|
58
|
+
* ```
|
|
59
|
+
*/
|
|
60
|
+
export declare function foldTraces<TTrace extends readonly {
|
|
61
|
+
readonly type: string;
|
|
62
|
+
}[]>(traces: Iterable<TTrace>, options: FoldTracesOptions<TTrace>): StateStats[];
|
|
63
|
+
//#endregion
|
|
64
|
+
//#region src/trace-coverage.d.ts
|
|
65
|
+
/** One state's row in a {@link CoverageReport}. */
|
|
66
|
+
interface StateCoverage {
|
|
67
|
+
/** The state's dot-joined path from the root, e.g. `'deciding'` or `'work.review'`. */
|
|
68
|
+
state: string;
|
|
69
|
+
/** Runs that were in this state at least once. */
|
|
70
|
+
runs: number;
|
|
71
|
+
}
|
|
72
|
+
/** A handler: the state node that declares it and its event descriptor. */
|
|
73
|
+
interface HandlerRef {
|
|
74
|
+
/** The declaring state's dot path; `''` for the machine root. */
|
|
75
|
+
state: string;
|
|
76
|
+
/** The descriptor as XState keys it: `'APPROVE'`, `'task.*'`, `'*'`, `'xstate.done.actor'`, `'xstate.error.actor'`, `'xstate.after'`. */
|
|
77
|
+
event: string;
|
|
78
|
+
}
|
|
79
|
+
/** One handler's row in a {@link CoverageReport}. */
|
|
80
|
+
interface HandlerCoverage extends HandlerRef {
|
|
81
|
+
/** Runs in which this handler was credited at least once. */
|
|
82
|
+
runs: number;
|
|
83
|
+
targets: {
|
|
84
|
+
/** Target state paths of the object transitions under this descriptor; `'dynamic'` when any is a function transition. */
|
|
85
|
+
declared: string[] | "dynamic";
|
|
86
|
+
/** Active leaves observed right after a step credited to this handler. */
|
|
87
|
+
seen: string[];
|
|
88
|
+
/** Declared targets that no seen leaf is in. Empty when `declared` is `'dynamic'`. */
|
|
89
|
+
unreached: string[];
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
/** The result of {@link traceCoverage}. */
|
|
93
|
+
interface CoverageReport {
|
|
94
|
+
/** Every atomic and final state, in document order, including `runs: 0`. */
|
|
95
|
+
states: StateCoverage[];
|
|
96
|
+
/** Every observable handler, in document order, including `runs: 0`. */
|
|
97
|
+
handlers: HandlerCoverage[];
|
|
98
|
+
/** States no run visited, excluding `unreachable` ones. */
|
|
99
|
+
unvisitedStates: string[];
|
|
100
|
+
/** Handlers no run was credited with. */
|
|
101
|
+
unvisitedHandlers: HandlerRef[];
|
|
102
|
+
/** `always` handlers: they fire inside a macrostep, which a trace does not record. */
|
|
103
|
+
unobservable: HandlerRef[];
|
|
104
|
+
/**
|
|
105
|
+
* With a complete `reachable` report: states no explored path reached and
|
|
106
|
+
* no trace visited. They leave the state denominator.
|
|
107
|
+
*/
|
|
108
|
+
unreachable?: string[];
|
|
109
|
+
/**
|
|
110
|
+
* With `reachable`: whether the report explored every path. When `false`,
|
|
111
|
+
* a state it missed may still be reachable, so `unreachable` is absent and
|
|
112
|
+
* every state stays in the denominator.
|
|
113
|
+
*/
|
|
114
|
+
reachableComplete?: boolean;
|
|
115
|
+
/** Visited over total, for states and for handlers (unobservable excluded). 1 when the total is 0. */
|
|
116
|
+
ratio: {
|
|
117
|
+
states: number;
|
|
118
|
+
handlers: number;
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
/** Options for {@link traceCoverage}. */
|
|
122
|
+
interface TraceCoverageOptions {
|
|
123
|
+
/**
|
|
124
|
+
* An `explorePaths` report. When it explored every path (no cap hit, every
|
|
125
|
+
* payload expanded, no branch stopped early), states it never reached drop
|
|
126
|
+
* out of the state denominator.
|
|
127
|
+
*/
|
|
128
|
+
reachable?: AgentPathReport;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Measures how much of a machine a set of runs exercised: which atomic and
|
|
132
|
+
* final states they visited, and which handlers they took. A handler is a
|
|
133
|
+
* `(declaring state, event descriptor)` pair from `on`, invoke
|
|
134
|
+
* `onDone`/`onError`, and `after`, with the targets it declares and the
|
|
135
|
+
* leaves the runs landed in after it. Pure; reads only `machine.transition`
|
|
136
|
+
* events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
|
|
137
|
+
*
|
|
138
|
+
* A step credits one handler per active leaf: the nearest node, from the leaf
|
|
139
|
+
* up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
|
|
140
|
+
* event, preferred in XState's order. Guards are not replayed, so a step
|
|
141
|
+
* whose nearest handler was rejected and fell through to an ancestor is
|
|
142
|
+
* credited to the nearest one. XState keys invoke and `after` handlers by
|
|
143
|
+
* bare type (`xstate.done.actor`), one per state however many invokes; a
|
|
144
|
+
* definition's `matches` (an invoke's `actorId`) must accept the traced event
|
|
145
|
+
* for its state to be credited. Ignored steps credit no handler. `always`
|
|
146
|
+
* handlers are listed as `unobservable`, never as unvisited; a state one
|
|
147
|
+
* passes through is visited, re-derived from each step's microsteps.
|
|
148
|
+
*
|
|
149
|
+
* @example
|
|
150
|
+
* ```ts no-check
|
|
151
|
+
* const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
|
|
152
|
+
* expect(report.unvisitedHandlers).toEqual([]);
|
|
153
|
+
* ```
|
|
154
|
+
*/
|
|
155
|
+
export declare function traceCoverage<TTrace extends readonly {
|
|
156
|
+
readonly type: string;
|
|
157
|
+
}[]>(machine: AnyStateMachine, traces: Iterable<TTrace>, options?: TraceCoverageOptions): CoverageReport;
|
|
158
|
+
//#endregion
|
|
159
|
+
//#region src/internal/trace-states.d.ts
|
|
160
|
+
/** One macrostep read from a trace: a `machine.transition` event. */
|
|
161
|
+
interface TraceStep {
|
|
162
|
+
/** Active leaves before the step. */
|
|
163
|
+
from: string[];
|
|
164
|
+
/** The state value before the step, as traced. */
|
|
165
|
+
fromValue: unknown;
|
|
166
|
+
/** The event's `type`. */
|
|
167
|
+
event: string;
|
|
168
|
+
/** The event object, as traced. */
|
|
169
|
+
payload: unknown;
|
|
170
|
+
/** Active leaves after the step. */
|
|
171
|
+
to: string[];
|
|
172
|
+
/** The state value after the step, as traced. */
|
|
173
|
+
toValue: unknown;
|
|
174
|
+
/** The state had no transition for a host event, so nothing changed. */
|
|
175
|
+
ignored: boolean;
|
|
176
|
+
/** The `name` of the first-attempt decision request that started since the previous step. */
|
|
177
|
+
decision?: string;
|
|
178
|
+
}
|
|
179
|
+
//#endregion
|
|
180
|
+
//#region src/first-divergence.d.ts
|
|
181
|
+
/** Options for {@link firstDivergence}. */
|
|
182
|
+
interface FirstDivergenceOptions {
|
|
183
|
+
/**
|
|
184
|
+
* Projects a step to the key two traces are compared by. Default
|
|
185
|
+
* `from|event|to`. Override to leave out what may differ between equivalent
|
|
186
|
+
* runs, such as a parallel region that only logs. Steps are still compared
|
|
187
|
+
* in order. Trace references only.
|
|
188
|
+
*
|
|
189
|
+
* `"payload"` also compares the event's payload, such as an actor's output,
|
|
190
|
+
* as key-order-independent JSON without its `sessionId`, and with errors as
|
|
191
|
+
* `{ name, message, code?, cause? }` (an `Error`, or a serialized one with `stack` or
|
|
192
|
+
* `code`). Use it for deterministic or mocked runs; live model outputs usually differ
|
|
193
|
+
* between runs.
|
|
194
|
+
*/
|
|
195
|
+
key?: ((step: TraceStep) => string) | "payload";
|
|
196
|
+
}
|
|
197
|
+
/** Where a run first departs from its reference. See {@link firstDivergence}. */
|
|
198
|
+
interface Divergence {
|
|
199
|
+
/** The step's index in the run's steps, or the step count when the run ended early. */
|
|
200
|
+
index: number;
|
|
201
|
+
/** The state the run was in when it departed: the first active leaf before that step. */
|
|
202
|
+
state: string;
|
|
203
|
+
/** The decision request that chose that step's event, when one did. */
|
|
204
|
+
decision?: string;
|
|
205
|
+
/** The run's step. Absent when the run ended early. */
|
|
206
|
+
actual?: TraceStep;
|
|
207
|
+
/** The reference's step, or the expected trajectory item the run never reached. Absent when the run went on past a reference trace. */
|
|
208
|
+
expected?: TraceStep | TrajectoryItem;
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* Finds the first step where a run departs from a reference, so a failure is
|
|
212
|
+
* blamed on the state it started in rather than on every state the run went
|
|
213
|
+
* through afterwards.
|
|
214
|
+
*
|
|
215
|
+
* - Against a reference **trace**, such as a passing run of the same
|
|
216
|
+
* scenario, steps are compared in order by `key`. The first step that
|
|
217
|
+
* differs, or the point where either trace ends early, is the divergence.
|
|
218
|
+
* - Against an expected **trajectory** (`TrajectoryItem[]`, as for
|
|
219
|
+
* {@link matchesTrajectory}), the run is laid out as its initial state, then
|
|
220
|
+
* each step's event and new state. The first expected item it never reaches
|
|
221
|
+
* is the divergence, blamed on the step after the last item it did reach. A
|
|
222
|
+
* run that goes on past the trajectory has not diverged.
|
|
223
|
+
*
|
|
224
|
+
* An empty reference is a trace with no steps, so any step diverges from it.
|
|
225
|
+
* A trace is one run's events, live or serialized. Steps the machine ignored
|
|
226
|
+
* changed nothing and are left out on both sides. Returns `undefined` when the
|
|
227
|
+
* run does not diverge.
|
|
228
|
+
*
|
|
229
|
+
* @example
|
|
230
|
+
* ```ts no-check
|
|
231
|
+
* const stats = foldTraces(traces, {
|
|
232
|
+
* failed,
|
|
233
|
+
* blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
|
|
234
|
+
* });
|
|
235
|
+
* ```
|
|
236
|
+
*/
|
|
237
|
+
export declare function firstDivergence(actual: readonly {
|
|
238
|
+
readonly type: string;
|
|
239
|
+
}[], reference: readonly {
|
|
240
|
+
readonly type: string;
|
|
241
|
+
}[] | readonly TrajectoryItem[], options?: FirstDivergenceOptions): Divergence | undefined;
|
|
242
|
+
//#endregion
|
|
243
|
+
//#region src/propose-from-traces.d.ts
|
|
244
|
+
/** A person's verdict as {@link proposeApprovalRules} hands it to `key`. */
|
|
245
|
+
interface ApprovalVerdict<TTrace> {
|
|
246
|
+
/** The approve or reject event, as traced. */
|
|
247
|
+
event: Record<string, unknown>;
|
|
248
|
+
/** The traced snapshot the verdict arrived in: the approval state, with the context that holds the pending call. */
|
|
249
|
+
snapshot: unknown;
|
|
250
|
+
/** The whole run. */
|
|
251
|
+
trace: TTrace;
|
|
252
|
+
}
|
|
253
|
+
/** Options for {@link proposeApprovalRules}. */
|
|
254
|
+
interface ProposeApprovalRulesOptions<TTrace> {
|
|
255
|
+
/** The state's dot path, such as `'awaitingApproval'`. A verdict counts when the run was in it or in a state inside it. */
|
|
256
|
+
approvalState: string;
|
|
257
|
+
/** Event types that approve, such as `['APPROVE']`. */
|
|
258
|
+
approveEvents: readonly string[];
|
|
259
|
+
/** Event types that reject, such as `['REJECT']`. */
|
|
260
|
+
rejectEvents: readonly string[];
|
|
261
|
+
/**
|
|
262
|
+
* The rule a verdict is evidence for: usually the tool name and a pattern
|
|
263
|
+
* of its arguments, read from `snapshot.context`. `undefined` skips the verdict.
|
|
264
|
+
*/
|
|
265
|
+
key: (verdict: ApprovalVerdict<TTrace>) => string | undefined;
|
|
266
|
+
/**
|
|
267
|
+
* The keys a run broke a rule for, from ground truth (a ledger checked
|
|
268
|
+
* after the fact). Any violation blocks `allow` for that key.
|
|
269
|
+
*/
|
|
270
|
+
violations?: (trace: TTrace) => Iterable<string>;
|
|
271
|
+
}
|
|
272
|
+
/** One key's row from {@link proposeApprovalRules}. */
|
|
273
|
+
interface ApprovalRuleProposal {
|
|
274
|
+
key: string;
|
|
275
|
+
/** Approve verdicts, summed over runs. */
|
|
276
|
+
approved: number;
|
|
277
|
+
/** Reject verdicts, summed over runs. */
|
|
278
|
+
rejected: number;
|
|
279
|
+
/** Violations `violations` reported, summed over runs. 0 without `violations`. */
|
|
280
|
+
violations: number;
|
|
281
|
+
/** Indexes, in input order, of the runs with a verdict for this key. */
|
|
282
|
+
runs: number[];
|
|
283
|
+
/**
|
|
284
|
+
* `'allow'`: approved, never rejected, no violation. `'deny'`: rejected,
|
|
285
|
+
* never approved. `null`: mixed evidence.
|
|
286
|
+
*/
|
|
287
|
+
proposal: "allow" | "deny" | null;
|
|
288
|
+
/** Runs with a verdict for this key: `runs.length`. */
|
|
289
|
+
support: number;
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Groups the approval verdicts in many runs' traces by `key` and proposes an
|
|
293
|
+
* `allow` or `deny` rule for each. Pure and deterministic; a proposal is for
|
|
294
|
+
* a person to review, and nothing here changes a machine.
|
|
295
|
+
*
|
|
296
|
+
* Reads `machine.transition` events: a verdict is a transition, not ignored,
|
|
297
|
+
* out of `approvalState` on an approve or reject event. Rows come in the
|
|
298
|
+
* order keys were first seen.
|
|
299
|
+
*
|
|
300
|
+
* @example
|
|
301
|
+
* ```ts no-check
|
|
302
|
+
* const rows = proposeApprovalRules(traces, {
|
|
303
|
+
* approvalState: "awaitingApproval",
|
|
304
|
+
* approveEvents: ["APPROVE"],
|
|
305
|
+
* rejectEvents: ["REJECT"],
|
|
306
|
+
* key: ({ snapshot }) => `${snapshot.context.pending.tool}:${snapshot.context.pending.path}`,
|
|
307
|
+
* });
|
|
308
|
+
* ```
|
|
309
|
+
*/
|
|
310
|
+
export declare function proposeApprovalRules<TTrace extends readonly {
|
|
311
|
+
readonly type: string;
|
|
312
|
+
}[]>(traces: Iterable<TTrace>, options: ProposeApprovalRulesOptions<TTrace>): ApprovalRuleProposal[];
|
|
313
|
+
/** One cap for {@link proposeCaps}. Give `failureState` or `failureOutput`. */
|
|
314
|
+
interface CapSpec<TTrace> {
|
|
315
|
+
/** Names the row, such as `'retries'`. */
|
|
316
|
+
name: string;
|
|
317
|
+
/** The cap's value now. A cap runs hit is never proposed below it. */
|
|
318
|
+
current?: number;
|
|
319
|
+
/** The run hit the cap when it entered this state (dot path), such as `'gaveUp'`. */
|
|
320
|
+
failureState?: string;
|
|
321
|
+
/** The run hit the cap when its `run.end` output satisfies this. */
|
|
322
|
+
failureOutput?: (output: unknown) => boolean;
|
|
323
|
+
/** How much of the cap the run used: retries made, steps taken, batch size. `undefined` leaves the run out. */
|
|
324
|
+
valueFromRun: (trace: TTrace) => number | undefined;
|
|
325
|
+
}
|
|
326
|
+
/** One cap's row from {@link proposeCaps}. */
|
|
327
|
+
interface CapProposal {
|
|
328
|
+
name: string;
|
|
329
|
+
/** Runs that hit the cap. */
|
|
330
|
+
hits: number;
|
|
331
|
+
/** Runs that did not, with a value. */
|
|
332
|
+
passing: number;
|
|
333
|
+
/** The largest value over all runs; `null` with none. */
|
|
334
|
+
max: number | null;
|
|
335
|
+
/** The 95th percentile (nearest rank) over passing runs; `null` with none. */
|
|
336
|
+
p95: number | null;
|
|
337
|
+
/** `p95 + 1`, and never below `current` when any run hit the cap; `null` with no passing run. */
|
|
338
|
+
proposal: number | null;
|
|
339
|
+
}
|
|
340
|
+
/** Options for {@link proposeCaps}. */
|
|
341
|
+
interface ProposeCapsOptions<TTrace> {
|
|
342
|
+
caps: readonly CapSpec<TTrace>[];
|
|
343
|
+
}
|
|
344
|
+
/**
|
|
345
|
+
* Proposes a value for each cap (retry count, steps, batch size) from many
|
|
346
|
+
* runs' traces: the 95th percentile of what passing runs used, plus one, and
|
|
347
|
+
* never below the current cap while runs still hit it. Pure and
|
|
348
|
+
* deterministic; a proposal is for a person to review, and nothing here
|
|
349
|
+
* changes a machine.
|
|
350
|
+
*
|
|
351
|
+
* @example
|
|
352
|
+
* ```ts no-check
|
|
353
|
+
* const [retries] = proposeCaps(traces, {
|
|
354
|
+
* caps: [{ name: "retries", current: 3, failureState: "gaveUp", valueFromRun: countRetries }],
|
|
355
|
+
* });
|
|
356
|
+
* ```
|
|
357
|
+
*/
|
|
358
|
+
export declare function proposeCaps<TTrace extends readonly {
|
|
359
|
+
readonly type: string;
|
|
360
|
+
}[]>(traces: Iterable<TTrace>, options: ProposeCapsOptions<TTrace>): CapProposal[];
|
|
361
|
+
//#endregion
|
|
362
|
+
//#region src/decision-requests.d.ts
|
|
363
|
+
/** One decision a run asked for. See {@link getDecisionRequests}. */
|
|
364
|
+
interface TracedDecisionRequest {
|
|
365
|
+
/** The active states when the decision was asked, as dot paths (one per parallel region). */
|
|
366
|
+
states: string[];
|
|
367
|
+
/** The request as the executor received it, on its first attempt. */
|
|
368
|
+
request: AgentExecutorDecisionRequest;
|
|
369
|
+
}
|
|
370
|
+
/** Options for {@link getDecisionRequests}. */
|
|
371
|
+
interface GetDecisionRequestsOptions {
|
|
372
|
+
/** Keep only requests with this `name` (the `agent.decide` input's `name`). */
|
|
373
|
+
name?: string;
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
376
|
+
* The decisions a run asked for, read from its trace: each decision's first
|
|
377
|
+
* `request.start`, with the states the run was in. A trace is one run's events,
|
|
378
|
+
* live or serialized with `serializeTraceEvent`. Retries of the same decision
|
|
379
|
+
* are left out, so each decision appears once.
|
|
380
|
+
*
|
|
381
|
+
* Use the requests as eval rows (run a decider on each and compare its event
|
|
382
|
+
* with a label) or as probe inputs for a policy. A serialized request keeps its
|
|
383
|
+
* `options`, `metadata`, `system`, and `prompt`; event schemas do not survive
|
|
384
|
+
* serialization, so replay closed choices (`options`) rather than open ones.
|
|
385
|
+
*
|
|
386
|
+
* @example
|
|
387
|
+
* ```ts no-check
|
|
388
|
+
* const rows = traces.flatMap((trace) => getDecisionRequests(trace, { name: "chooseResolution" }));
|
|
389
|
+
* for (const { request } of rows) console.log((await decide(request)).event);
|
|
390
|
+
* ```
|
|
391
|
+
*/
|
|
392
|
+
export declare function getDecisionRequests(trace: Iterable<{
|
|
393
|
+
readonly type: string;
|
|
394
|
+
}>, options?: GetDecisionRequestsOptions): TracedDecisionRequest[];
|
|
395
|
+
//#endregion
|
|
396
|
+
//#region src/repeat-runs.d.ts
|
|
397
|
+
type AnyTrace = readonly {
|
|
398
|
+
readonly type: string;
|
|
399
|
+
}[];
|
|
400
|
+
/** One run of a repeated scenario. See {@link summarizeRepeats}. */
|
|
401
|
+
interface RepeatRun<TTrace extends AnyTrace = AnyTrace> {
|
|
402
|
+
passed: boolean;
|
|
403
|
+
/** The run's trace. Absent for a run that threw. */
|
|
404
|
+
trace?: TTrace;
|
|
405
|
+
/** How the run ended, for the report. Not compared. */
|
|
406
|
+
outcome?: string;
|
|
407
|
+
/** The message of a run that threw. */
|
|
408
|
+
error?: string;
|
|
409
|
+
}
|
|
410
|
+
/** One scenario's runs. See {@link summarizeRepeats}. */
|
|
411
|
+
interface RepeatGroup<TScenario, TTrace extends AnyTrace = AnyTrace> {
|
|
412
|
+
id: string;
|
|
413
|
+
scenario: TScenario;
|
|
414
|
+
runs: readonly RepeatRun<TTrace>[];
|
|
415
|
+
}
|
|
416
|
+
/** One scenario's row in a {@link RepeatReport}. */
|
|
417
|
+
interface RepeatResult<TScenario, TTrace extends AnyTrace = AnyTrace> extends RepeatGroup<TScenario, TTrace> {
|
|
418
|
+
/** Runs that passed. */
|
|
419
|
+
passes: number;
|
|
420
|
+
/** At least one run passed: pass@k for this scenario. */
|
|
421
|
+
passAny: boolean;
|
|
422
|
+
/** Every run passed: pass^k for this scenario. */
|
|
423
|
+
passAll: boolean;
|
|
424
|
+
/** Distinct state paths the traced runs took. 1 when every run went the same way. */
|
|
425
|
+
distinctPaths: number;
|
|
426
|
+
/** The state where the runs first went different ways, judged against the first run whose trace has a transition. */
|
|
427
|
+
divergedAt?: string;
|
|
428
|
+
}
|
|
429
|
+
/** The result of {@link summarizeRepeats} and {@link repeatRuns}. */
|
|
430
|
+
interface RepeatReport<TScenario, TTrace extends AnyTrace = AnyTrace> {
|
|
431
|
+
scenarios: RepeatResult<TScenario, TTrace>[];
|
|
432
|
+
/**
|
|
433
|
+
* The suite's pass^j: the chance that `j` runs of a scenario, drawn from the
|
|
434
|
+
* ones recorded, all pass, averaged over scenarios. The unbiased estimate is
|
|
435
|
+
* C(passes, j) / C(runs, j) per scenario. Scenarios with fewer than `j` runs
|
|
436
|
+
* are left out. Throws when `j` is not a whole number from 1 to the most
|
|
437
|
+
* runs any scenario has.
|
|
438
|
+
*/
|
|
439
|
+
passHat: (j: number) => number;
|
|
440
|
+
/** Every recorded trace, ready for `foldTraces` or `traceCoverage`. */
|
|
441
|
+
traces: TTrace[];
|
|
442
|
+
}
|
|
443
|
+
/** Options for {@link repeatRuns}. */
|
|
444
|
+
interface RepeatRunsOptions<TScenario extends {
|
|
445
|
+
id: string;
|
|
446
|
+
}, TTrace extends AnyTrace> {
|
|
447
|
+
scenarios: readonly TScenario[];
|
|
448
|
+
/** Runs per scenario. */
|
|
449
|
+
k: number;
|
|
450
|
+
/** Runs one scenario once and returns its trace. `attempt` counts from 0. */
|
|
451
|
+
run: (scenario: TScenario, attempt: number) => Promise<{
|
|
452
|
+
trace: TTrace;
|
|
453
|
+
outcome?: string;
|
|
454
|
+
}>;
|
|
455
|
+
/** Whether a run passed, judged from its ground truth. */
|
|
456
|
+
passed: (scenario: TScenario, result: {
|
|
457
|
+
trace: TTrace;
|
|
458
|
+
outcome?: string;
|
|
459
|
+
}) => boolean;
|
|
460
|
+
}
|
|
461
|
+
/**
|
|
462
|
+
* Summarizes repeated runs of each scenario: how often each passed, whether
|
|
463
|
+
* all its runs passed (pass^k), how many distinct state paths they took, and
|
|
464
|
+
* where they first went different ways. Pure. Use it for runs collected
|
|
465
|
+
* elsewhere, such as CI shards or replays; {@link repeatRuns} collects them and
|
|
466
|
+
* calls it.
|
|
467
|
+
*
|
|
468
|
+
* A scenario that passes 3 of 5 runs and `divergedAt: 'deciding'` points at one
|
|
469
|
+
* decision, not at a flaky agent.
|
|
470
|
+
*
|
|
471
|
+
* @example
|
|
472
|
+
* ```ts no-check
|
|
473
|
+
* const report = summarizeRepeats(shards.map((s) => ({ id: s.id, scenario: s, runs: s.runs })));
|
|
474
|
+
* report.passHat(3); // the chance 3 runs of a scenario all pass
|
|
475
|
+
* ```
|
|
476
|
+
*/
|
|
477
|
+
export declare function summarizeRepeats<TScenario, TTrace extends AnyTrace = AnyTrace>(groups: readonly RepeatGroup<TScenario, TTrace>[]): RepeatReport<TScenario, TTrace>;
|
|
478
|
+
/**
|
|
479
|
+
* Runs every scenario `k` times and summarizes the runs with
|
|
480
|
+
* {@link summarizeRepeats}. Runs are sequential, scenario by scenario. A run
|
|
481
|
+
* that throws fails, with its message as `error` and no trace.
|
|
482
|
+
*
|
|
483
|
+
* With a live model, one run per scenario says little: τ-bench found pass^k
|
|
484
|
+
* falls well below the single-run rate. Report pass^k next to it.
|
|
485
|
+
*
|
|
486
|
+
* @example
|
|
487
|
+
* ```ts no-check
|
|
488
|
+
* const report = await repeatRuns({
|
|
489
|
+
* scenarios,
|
|
490
|
+
* k: 5,
|
|
491
|
+
* run: async (scenario) => {
|
|
492
|
+
* const trace: AgentTraceEvent[] = [];
|
|
493
|
+
* const result = await runToQuiescence(
|
|
494
|
+
* createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
|
|
495
|
+
* { input: scenario.input },
|
|
496
|
+
* );
|
|
497
|
+
* return { trace, outcome: String(result.snapshot.value) };
|
|
498
|
+
* },
|
|
499
|
+
* passed: (scenario, { outcome }) => outcome === scenario.expected,
|
|
500
|
+
* });
|
|
501
|
+
* report.passHat(5);
|
|
502
|
+
* ```
|
|
503
|
+
*/
|
|
504
|
+
export declare function repeatRuns<TScenario extends {
|
|
505
|
+
id: string;
|
|
506
|
+
}, TTrace extends AnyTrace>(options: RepeatRunsOptions<TScenario, TTrace>): Promise<RepeatReport<TScenario, TTrace>>;
|
|
507
|
+
//#endregion
|
|
508
|
+
//#region src/judge-agreement.d.ts
|
|
509
|
+
/** One labeled case: what a person said and what the judge said. See {@link judgeAgreement}. */
|
|
510
|
+
interface AgreementRow {
|
|
511
|
+
/** Identifies the case in `disagreements`. */
|
|
512
|
+
id: string;
|
|
513
|
+
/** The confirmed human label. */
|
|
514
|
+
human: string;
|
|
515
|
+
/** The judge's label for the same case. */
|
|
516
|
+
judge: string;
|
|
517
|
+
/** The group the row is aligned in, e.g. a decision state or request name. `groupBy` overrides it. */
|
|
518
|
+
group?: string;
|
|
519
|
+
}
|
|
520
|
+
/** Agreement numbers for one set of rows. See {@link AgreementReport}. */
|
|
521
|
+
interface AgreementSummary {
|
|
522
|
+
/** Rows counted. */
|
|
523
|
+
total: number;
|
|
524
|
+
/** Share of rows where `judge` equals `human`. */
|
|
525
|
+
accuracy: number;
|
|
526
|
+
/** Per human label: rows with that label, and how many the judge matched. `agreed / human` is that label's recall. */
|
|
527
|
+
perLabel: Record<string, {
|
|
528
|
+
human: number;
|
|
529
|
+
agreed: number;
|
|
530
|
+
}>;
|
|
531
|
+
/** Counts by human label, then judge label. */
|
|
532
|
+
confusion: Record<string, Record<string, number>>;
|
|
533
|
+
/** Rows where the judge differs from the person, in input order. */
|
|
534
|
+
disagreements: {
|
|
535
|
+
id: string;
|
|
536
|
+
human: string;
|
|
537
|
+
judge: string;
|
|
538
|
+
}[];
|
|
539
|
+
/**
|
|
540
|
+
* With `positive` set: the true positive rate (judge says positive when the
|
|
541
|
+
* person did) and true negative rate (judge says any other label when the
|
|
542
|
+
* person did). A rate is `undefined` when no row has that human side.
|
|
543
|
+
*/
|
|
544
|
+
binary?: {
|
|
545
|
+
tpr: number | undefined;
|
|
546
|
+
tnr: number | undefined;
|
|
547
|
+
};
|
|
548
|
+
/**
|
|
549
|
+
* With `positive` set: estimates the true share of positives among
|
|
550
|
+
* unlabeled rows from the share `p` the judge labeled positive,
|
|
551
|
+
* θ = (p + TNR − 1) / (TPR + TNR − 1), clamped to [0, 1]. Omitted when
|
|
552
|
+
* either rate is `undefined`, or when TPR + TNR ≤ 1: the judge is no better
|
|
553
|
+
* than chance, so its labels carry no signal to correct.
|
|
554
|
+
*/
|
|
555
|
+
correct?: (observedPositiveRate: number) => number;
|
|
556
|
+
}
|
|
557
|
+
/** The result of {@link judgeAgreement}. */
|
|
558
|
+
interface AgreementReport extends AgreementSummary {
|
|
559
|
+
/** The same numbers per group. Present when any row has `group` or `groupBy` is given. */
|
|
560
|
+
groups?: Record<string, AgreementSummary>;
|
|
561
|
+
}
|
|
562
|
+
/** Options for {@link judgeAgreement}. */
|
|
563
|
+
interface JudgeAgreementOptions<TRow extends AgreementRow> {
|
|
564
|
+
/** The positive label. Every other label is negative. Turns on `binary` and `correct`. */
|
|
565
|
+
positive?: string;
|
|
566
|
+
/** The row's group, overriding `row.group`. Return `undefined` to leave the row out of every group. */
|
|
567
|
+
groupBy?: (row: TRow) => string | undefined;
|
|
568
|
+
}
|
|
569
|
+
/**
|
|
570
|
+
* Measures how well a judge agrees with people on the same cases: accuracy,
|
|
571
|
+
* per-label recall, the confusion matrix, and the rows that disagree. With
|
|
572
|
+
* `positive`, also the TPR and TNR, the recommended view for binary judges
|
|
573
|
+
* (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
|
|
574
|
+
* Pure.
|
|
575
|
+
*
|
|
576
|
+
* Build rows from confirmed labels only; draft labels gate nothing. Group by
|
|
577
|
+
* decision state or request name so each narrow judge is aligned on its own.
|
|
578
|
+
* A row with no group key counts in the totals and in no group. Run it on a
|
|
579
|
+
* dev split and report the held-out split separately.
|
|
580
|
+
*
|
|
581
|
+
* Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
|
|
582
|
+
*
|
|
583
|
+
* @example
|
|
584
|
+
* ```ts no-check
|
|
585
|
+
* const report = judgeAgreement(
|
|
586
|
+
* labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
|
|
587
|
+
* { positive: 'fail' },
|
|
588
|
+
* );
|
|
589
|
+
* report.binary; // { tpr: 0.9, tnr: 0.8 }
|
|
590
|
+
* report.correct?.(0.3); // estimated true failure rate in production
|
|
591
|
+
* ```
|
|
592
|
+
*/
|
|
593
|
+
export declare function judgeAgreement<TRow extends AgreementRow>(rows: readonly TRow[], options?: JudgeAgreementOptions<TRow>): AgreementReport;
|
|
594
|
+
//#endregion
|
|
595
|
+
export type { AgreementReport, AgreementRow, AgreementSummary, ApprovalRuleProposal, ApprovalVerdict, CapProposal, CapSpec, CoverageReport, Divergence, FirstDivergenceOptions, FoldTracesOptions, GetDecisionRequestsOptions, HandlerCoverage, HandlerRef, JudgeAgreementOptions, ProposeApprovalRulesOptions, ProposeCapsOptions, RepeatGroup, RepeatReport, RepeatResult, RepeatRun, RepeatRunsOptions, StateCoverage, StateStats, TraceCoverageOptions, TraceStep, TracedDecisionRequest };
|