@statelyai/agent 2.0.0-alpha.32 → 2.0.0-alpha.34

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/CHANGELOG.md +44 -0
  2. package/dist/ai-sdk.cjs +114 -63
  3. package/dist/ai-sdk.d.cts +11 -2
  4. package/dist/ai-sdk.d.mts +11 -2
  5. package/dist/ai-sdk.mjs +115 -64
  6. package/dist/{decision-BPdEaxPf.cjs → decision-BEsATcr1.cjs} +18 -2
  7. package/dist/{decision-CV7WiV5C.mjs → decision-COkJNjOW.mjs} +18 -2
  8. package/dist/{event-log-CKWeC7H-.cjs → event-log-BWSSLUff.cjs} +139 -10
  9. package/dist/{event-log-DAqsalT1.mjs → event-log-C7jeH0y8.mjs} +123 -12
  10. package/dist/{event-log-store-C2NjVlWU.d.mts → event-log-store-BzXehP7R.d.cts} +28 -3
  11. package/dist/{event-log-store-s73r1Hxf.d.cts → event-log-store-CWjN6lmM.d.mts} +28 -3
  12. package/dist/{event-log-store-DKyE7JKr.cjs → event-log-store-CleWpsp5.cjs} +1 -1
  13. package/dist/{event-log-store-BGehvVTb.mjs → event-log-store-W49TCJXi.mjs} +1 -1
  14. package/dist/index.cjs +5 -5
  15. package/dist/index.d.cts +7 -7
  16. package/dist/index.d.mts +7 -7
  17. package/dist/index.mjs +5 -5
  18. package/dist/log.cjs +2 -2
  19. package/dist/log.d.cts +1 -1
  20. package/dist/log.d.mts +1 -1
  21. package/dist/log.mjs +2 -2
  22. package/dist/machines.cjs +1 -1
  23. package/dist/machines.mjs +1 -1
  24. package/dist/{model-source-BilDEyqk.mjs → model-source-CK3F4YuU.mjs} +1 -1
  25. package/dist/{model-source-BQ97_l1-.cjs → model-source-DmhTMK39.cjs} +1 -1
  26. package/dist/openai.cjs +377 -95
  27. package/dist/openai.d.cts +55 -20
  28. package/dist/openai.d.mts +55 -20
  29. package/dist/openai.mjs +375 -96
  30. package/dist/otel.d.cts +1 -1
  31. package/dist/otel.d.mts +1 -1
  32. package/dist/{run-agent-B5QhabdY.d.cts → run-agent-B1-Eqviq.d.cts} +13 -5
  33. package/dist/{run-agent-Atu-wEk7.d.mts → run-agent-BGdx5913.d.mts} +13 -5
  34. package/dist/{run-agent-2UnSUhDb.mjs → run-agent-Ct_GZw_f.mjs} +124 -38
  35. package/dist/{run-agent-Cg0xd3K-.cjs → run-agent-s5D32w99.cjs} +123 -37
  36. package/dist/{setup-agent-CmhB-U5P.mjs → setup-agent-05uw4dPZ.mjs} +2 -2
  37. package/dist/{setup-agent-BiF1N3td.d.cts → setup-agent-BtfeFfF6.d.cts} +2 -2
  38. package/dist/{setup-agent-J9f_Wu5T.d.mts → setup-agent-C6nqzv_X.d.mts} +2 -2
  39. package/dist/{setup-agent-CwUJyqKB.cjs → setup-agent-CXEBURdo.cjs} +2 -2
  40. package/dist/tanstack-ai.cjs +91 -59
  41. package/dist/tanstack-ai.d.cts +13 -2
  42. package/dist/tanstack-ai.d.mts +13 -2
  43. package/dist/tanstack-ai.mjs +91 -59
  44. package/dist/testing.cjs +1026 -10
  45. package/dist/testing.d.cts +585 -5
  46. package/dist/testing.d.mts +585 -5
  47. package/dist/testing.mjs +1018 -11
  48. package/dist/{text-logic-CYP502VT.d.cts → text-logic-CV_4KGr6.d.cts} +28 -3
  49. package/dist/{text-logic-DuK9HIFM.d.mts → text-logic-Ckh-rJFV.d.mts} +28 -3
  50. package/dist/{usage-D7JKAxLc.d.mts → usage-BDiJKyv9.d.cts} +11 -1
  51. package/dist/{usage-DVw-646x.d.cts → usage-DOoVUGVq.d.mts} +11 -1
  52. package/dist/validate.cjs +3 -3
  53. package/dist/validate.d.cts +2 -2
  54. package/dist/validate.d.mts +2 -2
  55. package/dist/validate.mjs +3 -3
  56. package/dist/{verify-Dk3Vwnha.d.cts → verify-BQpE8hUz.d.cts} +1 -1
  57. package/dist/{verify-CoCvR1eM.mjs → verify-Bpahmr34.mjs} +14 -4
  58. package/dist/{verify-BVz4cURF.d.mts → verify-CqeuEmIk.d.mts} +1 -1
  59. package/dist/{verify-D_llRCTr.cjs → verify-DCLZa9R0.cjs} +19 -3
  60. package/dist/with-model-calls-BXYdBdcu.cjs +45 -0
  61. package/dist/with-model-calls-Bv4V4EHC.mjs +34 -0
  62. package/package.json +5 -5
  63. package/readme.md +14 -13
  64. package/skills/add-evals/SKILL.md +8 -2
  65. package/skills/improve-machine/SKILL.md +1 -1
  66. package/skills/loop-to-machine/SKILL.md +40 -2
@@ -1,7 +1,7 @@
1
- import { P as AgentExecutorDecisionRequest, c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-CYP502VT.cjs";
2
- import { f as AgentRuntimeOptions, l as AgentRunResult, o as AgentRunInit } from "./run-agent-B5QhabdY.cjs";
1
+ import { F as AgentExecutorDecisionRequest, c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-CV_4KGr6.cjs";
2
+ import { f as AgentRuntimeOptions, l as AgentRunResult, o as AgentRunInit } from "./run-agent-B1-Eqviq.cjs";
3
3
  import { t as MetaOfSnapshot } from "./utils-ebVpCYoF.cjs";
4
- import { _ as assertAgentMachine, a as AgentPathTerminal, b as lintAgentMachine, c as CanReachResult, d as ExploreSnapshot, f as LintAgentMachineOptions, g as SimulationTrailEntry, h as SimulationScript, i as AgentPathReport, l as CanReachTarget, m as SimulateAgentResult, n as AgentLintError, o as AgentUnknownStateError, p as SimulateAgentOptions, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as ExplorePathsOptions, v as canReach, x as simulateAgent, y as explorePaths } from "./verify-Dk3Vwnha.cjs";
4
+ import { _ as assertAgentMachine, a as AgentPathTerminal, b as lintAgentMachine, c as CanReachResult, d as ExploreSnapshot, f as LintAgentMachineOptions, g as SimulationTrailEntry, h as SimulationScript, i as AgentPathReport, l as CanReachTarget, m as SimulateAgentResult, n as AgentLintError, o as AgentUnknownStateError, p as SimulateAgentOptions, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as ExplorePathsOptions, v as canReach, x as simulateAgent, y as explorePaths } from "./verify-BQpE8hUz.cjs";
5
5
  import { AnyStateMachine, EventFromLogic, InputFrom, SnapshotFrom, StateValue } from "xstate";
6
6
  //#region src/trajectory.d.ts
7
7
  /** An event-shaped trajectory item: anything with a string `type`. */
@@ -290,16 +290,25 @@ interface StateStats {
290
290
  withheld: number;
291
291
  /** Events this state ignored (`machine.transition` with `ignored`), summed over runs. */
292
292
  ignored: number;
293
+ /** Failed runs `blame` attributed to this state. 0 without `blame`. */
294
+ blamed: number;
293
295
  }
294
296
  /** Options for {@link foldTraces}. */
295
297
  interface FoldTracesOptions<TTrace> {
296
298
  /** Whether a run failed, judged from its whole trace. */
297
299
  failed: (trace: TTrace) => boolean;
300
+ /**
301
+ * The state a failed run's failure started in, usually
302
+ * `firstDivergence(trace, reference)?.state`. Called for failed runs only;
303
+ * `undefined` blames no state.
304
+ */
305
+ blame?: (trace: TTrace) => string | undefined;
298
306
  }
299
307
  /**
300
308
  * Folds many runs' traces into per-state stats: how many runs visited each
301
309
  * state, how many of those failed, how many options the guards withheld
302
- * at decisions made there, and how many events it ignored. Pure; reads only:
310
+ * at decisions made there, how many events it ignored, and how many failures
311
+ * `blame` traced back to it. Pure; reads only:
303
312
  *
304
313
  * - `machine.transition`: the run visited `from.value` and `snapshot.value`;
305
314
  * with `ignored`, each of those states ignored one event.
@@ -312,6 +321,10 @@ interface FoldTracesOptions<TTrace> {
312
321
  * `{ type: 'request.start', request: result.request }` onto the trace before
313
322
  * sending its event. Parallel states count once per active leaf.
314
323
  *
324
+ * `failures` counts a failed run against every state it visited, so states
325
+ * after a bad decision rank as high as the decision. Sort by `blamed` to see
326
+ * where failures start.
327
+ *
315
328
  * @example
316
329
  * ```ts no-check
317
330
  * const stats = foldTraces(traces, {
@@ -323,6 +336,185 @@ export declare function foldTraces<TTrace extends readonly {
323
336
  readonly type: string;
324
337
  }[]>(traces: Iterable<TTrace>, options: FoldTracesOptions<TTrace>): StateStats[];
325
338
  //#endregion
339
+ //#region src/trace-coverage.d.ts
340
+ /** One state's row in a {@link CoverageReport}. */
341
+ interface StateCoverage {
342
+ /** The state's dot-joined path from the root, e.g. `'deciding'` or `'work.review'`. */
343
+ state: string;
344
+ /** Runs that were in this state at least once. */
345
+ runs: number;
346
+ }
347
+ /** A handler: the state node that declares it and its event descriptor. */
348
+ interface HandlerRef {
349
+ /** The declaring state's dot path; `''` for the machine root. */
350
+ state: string;
351
+ /** The descriptor as XState keys it: `'APPROVE'`, `'task.*'`, `'*'`, `'xstate.done.actor'`, `'xstate.error.actor'`, `'xstate.after'`. */
352
+ event: string;
353
+ }
354
+ /** One handler's row in a {@link CoverageReport}. */
355
+ interface HandlerCoverage extends HandlerRef {
356
+ /** Runs in which this handler was credited at least once. */
357
+ runs: number;
358
+ targets: {
359
+ /** Target state paths of the object transitions under this descriptor; `'dynamic'` when any is a function transition. */
360
+ declared: string[] | "dynamic";
361
+ /** Active leaves observed right after a step credited to this handler. */
362
+ seen: string[];
363
+ /** Declared targets that no seen leaf is in. Empty when `declared` is `'dynamic'`. */
364
+ unreached: string[];
365
+ };
366
+ }
367
+ /** The result of {@link traceCoverage}. */
368
+ interface CoverageReport {
369
+ /** Every atomic and final state, in document order, including `runs: 0`. */
370
+ states: StateCoverage[];
371
+ /** Every observable handler, in document order, including `runs: 0`. */
372
+ handlers: HandlerCoverage[];
373
+ /** States no run visited, excluding `unreachable` ones. */
374
+ unvisitedStates: string[];
375
+ /** Handlers no run was credited with. */
376
+ unvisitedHandlers: HandlerRef[];
377
+ /** `always` handlers: they fire inside a macrostep, which a trace does not record. */
378
+ unobservable: HandlerRef[];
379
+ /**
380
+ * With a complete `reachable` report: states no explored path reached and
381
+ * no trace visited. They leave the state denominator.
382
+ */
383
+ unreachable?: string[];
384
+ /**
385
+ * With `reachable`: whether the report explored every path. When `false`,
386
+ * a state it missed may still be reachable, so `unreachable` is absent and
387
+ * every state stays in the denominator.
388
+ */
389
+ reachableComplete?: boolean;
390
+ /** Visited over total, for states and for handlers (unobservable excluded). 1 when the total is 0. */
391
+ ratio: {
392
+ states: number;
393
+ handlers: number;
394
+ };
395
+ }
396
+ /** Options for {@link traceCoverage}. */
397
+ interface TraceCoverageOptions {
398
+ /**
399
+ * An `explorePaths` report. When it explored every path (no cap hit, every
400
+ * payload expanded, no branch stopped early), states it never reached drop
401
+ * out of the state denominator.
402
+ */
403
+ reachable?: AgentPathReport;
404
+ }
405
+ /**
406
+ * Measures how much of a machine a set of runs exercised: which atomic and
407
+ * final states they visited, and which handlers they took. A handler is a
408
+ * `(declaring state, event descriptor)` pair from `on`, invoke
409
+ * `onDone`/`onError`, and `after`, with the targets it declares and the
410
+ * leaves the runs landed in after it. Pure; reads only `machine.transition`
411
+ * events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
412
+ *
413
+ * A step credits one handler per active leaf: the nearest node, from the leaf
414
+ * up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
415
+ * event, preferred in XState's order. Guards are not replayed, so a step
416
+ * whose nearest handler was rejected and fell through to an ancestor is
417
+ * credited to the nearest one. XState keys invoke and `after` handlers by
418
+ * bare type (`xstate.done.actor`), one per state however many invokes; a
419
+ * definition's `matches` (an invoke's `actorId`) must accept the traced event
420
+ * for its state to be credited. Ignored steps credit no handler. `always`
421
+ * handlers are listed as `unobservable`, never as unvisited; a state one
422
+ * passes through is visited, re-derived from each step's microsteps.
423
+ *
424
+ * @example
425
+ * ```ts no-check
426
+ * const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
427
+ * expect(report.unvisitedHandlers).toEqual([]);
428
+ * ```
429
+ */
430
+ export declare function traceCoverage<TTrace extends readonly {
431
+ readonly type: string;
432
+ }[]>(machine: AnyStateMachine, traces: Iterable<TTrace>, options?: TraceCoverageOptions): CoverageReport;
433
+ //#endregion
434
+ //#region src/internal/trace-states.d.ts
435
+ /** One macrostep read from a trace: a `machine.transition` event. */
436
+ interface TraceStep {
437
+ /** Active leaves before the step. */
438
+ from: string[];
439
+ /** The state value before the step, as traced. */
440
+ fromValue: unknown;
441
+ /** The event's `type`. */
442
+ event: string;
443
+ /** The event object, as traced. */
444
+ payload: unknown;
445
+ /** Active leaves after the step. */
446
+ to: string[];
447
+ /** The state value after the step, as traced. */
448
+ toValue: unknown;
449
+ /** The state had no transition for a host event, so nothing changed. */
450
+ ignored: boolean;
451
+ /** The `name` of the first-attempt decision request that started since the previous step. */
452
+ decision?: string;
453
+ }
454
+ //#endregion
455
+ //#region src/first-divergence.d.ts
456
+ /** Options for {@link firstDivergence}. */
457
+ interface FirstDivergenceOptions {
458
+ /**
459
+ * Projects a step to the key two traces are compared by. Default
460
+ * `from|event|to`. Override to leave out what may differ between equivalent
461
+ * runs, such as a parallel region that only logs. Steps are still compared
462
+ * in order. Trace references only.
463
+ *
464
+ * `"payload"` also compares the event's payload, such as an actor's output,
465
+ * as key-order-independent JSON without its `sessionId`, and with errors as
466
+ * `{ name, message, code?, cause? }` (an `Error`, or a serialized one with `stack` or
467
+ * `code`). Use it for deterministic or mocked runs; live model outputs usually differ
468
+ * between runs.
469
+ */
470
+ key?: ((step: TraceStep) => string) | "payload";
471
+ }
472
+ /** Where a run first departs from its reference. See {@link firstDivergence}. */
473
+ interface Divergence {
474
+ /** The step's index in the run's steps, or the step count when the run ended early. */
475
+ index: number;
476
+ /** The state the run was in when it departed: the first active leaf before that step. */
477
+ state: string;
478
+ /** The decision request that chose that step's event, when one did. */
479
+ decision?: string;
480
+ /** The run's step. Absent when the run ended early. */
481
+ actual?: TraceStep;
482
+ /** The reference's step, or the expected trajectory item the run never reached. Absent when the run went on past a reference trace. */
483
+ expected?: TraceStep | TrajectoryItem;
484
+ }
485
+ /**
486
+ * Finds the first step where a run departs from a reference, so a failure is
487
+ * blamed on the state it started in rather than on every state the run went
488
+ * through afterwards.
489
+ *
490
+ * - Against a reference **trace**, such as a passing run of the same
491
+ * scenario, steps are compared in order by `key`. The first step that
492
+ * differs, or the point where either trace ends early, is the divergence.
493
+ * - Against an expected **trajectory** (`TrajectoryItem[]`, as for
494
+ * {@link matchesTrajectory}), the run is laid out as its initial state, then
495
+ * each step's event and new state. The first expected item it never reaches
496
+ * is the divergence, blamed on the step after the last item it did reach. A
497
+ * run that goes on past the trajectory has not diverged.
498
+ *
499
+ * An empty reference is a trace with no steps, so any step diverges from it.
500
+ * A trace is one run's events, live or serialized. Steps the machine ignored
501
+ * changed nothing and are left out on both sides. Returns `undefined` when the
502
+ * run does not diverge.
503
+ *
504
+ * @example
505
+ * ```ts no-check
506
+ * const stats = foldTraces(traces, {
507
+ * failed,
508
+ * blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
509
+ * });
510
+ * ```
511
+ */
512
+ export declare function firstDivergence(actual: readonly {
513
+ readonly type: string;
514
+ }[], reference: readonly {
515
+ readonly type: string;
516
+ }[] | readonly TrajectoryItem[], options?: FirstDivergenceOptions): Divergence | undefined;
517
+ //#endregion
326
518
  //#region src/decision-requests.d.ts
327
519
  /** One decision a run asked for. See {@link getDecisionRequests}. */
328
520
  interface TracedDecisionRequest {
@@ -357,6 +549,205 @@ export declare function getDecisionRequests(trace: Iterable<{
357
549
  readonly type: string;
358
550
  }>, options?: GetDecisionRequestsOptions): TracedDecisionRequest[];
359
551
  //#endregion
552
+ //#region src/repeat-runs.d.ts
553
+ type AnyTrace = readonly {
554
+ readonly type: string;
555
+ }[];
556
+ /** One run of a repeated scenario. See {@link summarizeRepeats}. */
557
+ interface RepeatRun<TTrace extends AnyTrace = AnyTrace> {
558
+ passed: boolean;
559
+ /** The run's trace. Absent for a run that threw. */
560
+ trace?: TTrace;
561
+ /** How the run ended, for the report. Not compared. */
562
+ outcome?: string;
563
+ /** The message of a run that threw. */
564
+ error?: string;
565
+ }
566
+ /** One scenario's runs. See {@link summarizeRepeats}. */
567
+ interface RepeatGroup<TScenario, TTrace extends AnyTrace = AnyTrace> {
568
+ id: string;
569
+ scenario: TScenario;
570
+ runs: readonly RepeatRun<TTrace>[];
571
+ }
572
+ /** One scenario's row in a {@link RepeatReport}. */
573
+ interface RepeatResult<TScenario, TTrace extends AnyTrace = AnyTrace> extends RepeatGroup<TScenario, TTrace> {
574
+ /** Runs that passed. */
575
+ passes: number;
576
+ /** At least one run passed: pass@k for this scenario. */
577
+ passAny: boolean;
578
+ /** Every run passed: pass^k for this scenario. */
579
+ passAll: boolean;
580
+ /** Distinct state paths the traced runs took. 1 when every run went the same way. */
581
+ distinctPaths: number;
582
+ /** The state where the runs first went different ways, judged against the first run whose trace has a transition. */
583
+ divergedAt?: string;
584
+ }
585
+ /** The result of {@link summarizeRepeats} and {@link repeatRuns}. */
586
+ interface RepeatReport<TScenario, TTrace extends AnyTrace = AnyTrace> {
587
+ scenarios: RepeatResult<TScenario, TTrace>[];
588
+ /**
589
+ * The suite's pass^j: the chance that `j` runs of a scenario, drawn from the
590
+ * ones recorded, all pass, averaged over scenarios. The unbiased estimate is
591
+ * C(passes, j) / C(runs, j) per scenario. Scenarios with fewer than `j` runs
592
+ * are left out. Throws when `j` is not a whole number from 1 to the most
593
+ * runs any scenario has.
594
+ */
595
+ passHat: (j: number) => number;
596
+ /** Every recorded trace, ready for `foldTraces` or `traceCoverage`. */
597
+ traces: TTrace[];
598
+ }
599
+ /** Options for {@link repeatRuns}. */
600
+ interface RepeatRunsOptions<TScenario extends {
601
+ id: string;
602
+ }, TTrace extends AnyTrace> {
603
+ scenarios: readonly TScenario[];
604
+ /** Runs per scenario. */
605
+ k: number;
606
+ /** Runs one scenario once and returns its trace. `attempt` counts from 0. */
607
+ run: (scenario: TScenario, attempt: number) => Promise<{
608
+ trace: TTrace;
609
+ outcome?: string;
610
+ }>;
611
+ /** Whether a run passed, judged from its ground truth. */
612
+ passed: (scenario: TScenario, result: {
613
+ trace: TTrace;
614
+ outcome?: string;
615
+ }) => boolean;
616
+ }
617
+ /**
618
+ * Summarizes repeated runs of each scenario: how often each passed, whether
619
+ * all its runs passed (pass^k), how many distinct state paths they took, and
620
+ * where they first went different ways. Pure. Use it for runs collected
621
+ * elsewhere, such as CI shards or replays; {@link repeatRuns} collects them and
622
+ * calls it.
623
+ *
624
+ * A scenario that passes 3 of 5 runs and `divergedAt: 'deciding'` points at one
625
+ * decision, not at a flaky agent.
626
+ *
627
+ * @example
628
+ * ```ts no-check
629
+ * const report = summarizeRepeats(shards.map((s) => ({ id: s.id, scenario: s, runs: s.runs })));
630
+ * report.passHat(3); // the chance 3 runs of a scenario all pass
631
+ * ```
632
+ */
633
+ export declare function summarizeRepeats<TScenario, TTrace extends AnyTrace = AnyTrace>(groups: readonly RepeatGroup<TScenario, TTrace>[]): RepeatReport<TScenario, TTrace>;
634
+ /**
635
+ * Runs every scenario `k` times and summarizes the runs with
636
+ * {@link summarizeRepeats}. Runs are sequential, scenario by scenario. A run
637
+ * that throws fails, with its message as `error` and no trace.
638
+ *
639
+ * With a live model, one run per scenario says little: τ-bench found pass^k
640
+ * falls well below the single-run rate. Report pass^k next to it.
641
+ *
642
+ * @example
643
+ * ```ts no-check
644
+ * const report = await repeatRuns({
645
+ * scenarios,
646
+ * k: 5,
647
+ * run: async (scenario) => {
648
+ * const trace: AgentTraceEvent[] = [];
649
+ * const result = await runToQuiescence(
650
+ * createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
651
+ * { input: scenario.input },
652
+ * );
653
+ * return { trace, outcome: String(result.snapshot.value) };
654
+ * },
655
+ * passed: (scenario, { outcome }) => outcome === scenario.expected,
656
+ * });
657
+ * report.passHat(5);
658
+ * ```
659
+ */
660
+ export declare function repeatRuns<TScenario extends {
661
+ id: string;
662
+ }, TTrace extends AnyTrace>(options: RepeatRunsOptions<TScenario, TTrace>): Promise<RepeatReport<TScenario, TTrace>>;
663
+ //#endregion
664
+ //#region src/judge-agreement.d.ts
665
+ /** One labeled case: what a person said and what the judge said. See {@link judgeAgreement}. */
666
+ interface AgreementRow {
667
+ /** Identifies the case in `disagreements`. */
668
+ id: string;
669
+ /** The confirmed human label. */
670
+ human: string;
671
+ /** The judge's label for the same case. */
672
+ judge: string;
673
+ /** The group the row is aligned in, e.g. a decision state or request name. `groupBy` overrides it. */
674
+ group?: string;
675
+ }
676
+ /** Agreement numbers for one set of rows. See {@link AgreementReport}. */
677
+ interface AgreementSummary {
678
+ /** Rows counted. */
679
+ total: number;
680
+ /** Share of rows where `judge` equals `human`. */
681
+ accuracy: number;
682
+ /** Per human label: rows with that label, and how many the judge matched. `agreed / human` is that label's recall. */
683
+ perLabel: Record<string, {
684
+ human: number;
685
+ agreed: number;
686
+ }>;
687
+ /** Counts by human label, then judge label. */
688
+ confusion: Record<string, Record<string, number>>;
689
+ /** Rows where the judge differs from the person, in input order. */
690
+ disagreements: {
691
+ id: string;
692
+ human: string;
693
+ judge: string;
694
+ }[];
695
+ /**
696
+ * With `positive` set: the true positive rate (judge says positive when the
697
+ * person did) and true negative rate (judge says any other label when the
698
+ * person did). A rate is `undefined` when no row has that human side.
699
+ */
700
+ binary?: {
701
+ tpr: number | undefined;
702
+ tnr: number | undefined;
703
+ };
704
+ /**
705
+ * With `positive` set: estimates the true share of positives among
706
+ * unlabeled rows from the share `p` the judge labeled positive,
707
+ * θ = (p + TNR − 1) / (TPR + TNR − 1), clamped to [0, 1]. Omitted when
708
+ * either rate is `undefined`, or when TPR + TNR ≤ 1: the judge is no better
709
+ * than chance, so its labels carry no signal to correct.
710
+ */
711
+ correct?: (observedPositiveRate: number) => number;
712
+ }
713
+ /** The result of {@link judgeAgreement}. */
714
+ interface AgreementReport extends AgreementSummary {
715
+ /** The same numbers per group. Present when any row has `group` or `groupBy` is given. */
716
+ groups?: Record<string, AgreementSummary>;
717
+ }
718
+ /** Options for {@link judgeAgreement}. */
719
+ interface JudgeAgreementOptions<TRow extends AgreementRow> {
720
+ /** The positive label. Every other label is negative. Turns on `binary` and `correct`. */
721
+ positive?: string;
722
+ /** The row's group, overriding `row.group`. Return `undefined` to leave the row out of every group. */
723
+ groupBy?: (row: TRow) => string | undefined;
724
+ }
725
+ /**
726
+ * Measures how well a judge agrees with people on the same cases: accuracy,
727
+ * per-label recall, the confusion matrix, and the rows that disagree. With
728
+ * `positive`, also the TPR and TNR, the recommended view for binary judges
729
+ * (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
730
+ * Pure.
731
+ *
732
+ * Build rows from confirmed labels only; draft labels gate nothing. Group by
733
+ * decision state or request name so each narrow judge is aligned on its own.
734
+ * A row with no group key counts in the totals and in no group. Run it on a
735
+ * dev split and report the held-out split separately.
736
+ *
737
+ * Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
738
+ *
739
+ * @example
740
+ * ```ts no-check
741
+ * const report = judgeAgreement(
742
+ * labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
743
+ * { positive: 'fail' },
744
+ * );
745
+ * report.binary; // { tpr: 0.9, tnr: 0.8 }
746
+ * report.correct?.(0.3); // estimated true failure rate in production
747
+ * ```
748
+ */
749
+ export declare function judgeAgreement<TRow extends AgreementRow>(rows: readonly TRow[], options?: JudgeAgreementOptions<TRow>): AgreementReport;
750
+ //#endregion
360
751
  //#region src/compare-runs.d.ts
361
752
  /** One run of one scenario, reduced to what parity compares. See {@link compareRuns}. */
362
753
  interface ParityRun {
@@ -438,4 +829,193 @@ export declare function compareRuns<TScenario extends {
438
829
  expected?: string;
439
830
  }>(options: CompareRunsOptions<TScenario>): Promise<ParityReport<TScenario>>;
440
831
  //#endregion
441
- export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRunsOptions, type ExplorePathsOptions, type ExploreSnapshot, type FoldTracesOptions, type GetDecisionRequestsOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateStats, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };
832
+ //#region src/equivalence.d.ts
833
+ /**
834
+ * Verified equivalence — prove a port behaves like the code it replaces.
835
+ *
836
+ * A port is equivalent when, given the same model answers and the same tool
837
+ * results, it produces the same OBSERVABLE EFFECTS: the same number of model
838
+ * calls, the same tool calls with the same arguments, and the same output.
839
+ * Prompts, internal steps, and states are free to differ.
840
+ *
841
+ * 1. Record the original: wrap its model (`recorder.model`, an AI SDK
842
+ * `LanguageModelV3`, or `recorder.openai`, an `openai` client) and its tools
843
+ * (`recorder.tool` / `recorder.tools`), run a scenario, call
844
+ * `recorder.output(result)`, and save `recorder.recording`.
845
+ * 2. Replay it on the port: `createRecorder({ replay: recording })` answers
846
+ * every model call and tool call from the recording, with no API key and no
847
+ * side effects, and records what the port did.
848
+ * 3. `compareRecordings(original, port)` reports the first effect that differs.
849
+ *
850
+ * Provider-free: both seams are wrapped structurally, so this module imports
851
+ * neither `ai` nor `openai`.
852
+ * @module
853
+ */
854
+ /** An error as recorded: JSON-safe. */
855
+ interface RecordedError {
856
+ name: string;
857
+ message: string;
858
+ code?: unknown;
859
+ }
860
+ /** One model call. `call` is its 0-based position among model calls. */
861
+ interface RecordedModelCall {
862
+ kind: "model";
863
+ /** Which wrapper saw it. */
864
+ seam: "ai-sdk" | "openai";
865
+ /** `doGenerate`/`doStream`, or `chat.completions.create`/`responses.create`. */
866
+ method: string;
867
+ call: number;
868
+ /** The call options or params, JSON-safe (abort signals and headers dropped). */
869
+ request: unknown;
870
+ /** The non-streaming result. */
871
+ response?: unknown;
872
+ /** The stream's parts or chunks, in order. */
873
+ stream?: unknown[];
874
+ error?: RecordedError;
875
+ }
876
+ /** One tool call. `step` is the number of model calls made before it. */
877
+ interface RecordedToolCall {
878
+ kind: "tool";
879
+ name: string;
880
+ step: number;
881
+ input: unknown;
882
+ output?: unknown;
883
+ error?: RecordedError;
884
+ /** The tool returned without a Promise; replay answers synchronously too. */
885
+ sync?: true;
886
+ /** Started while an earlier tool call was still running. Concurrent calls
887
+ * may finish in either order; sequential ones must keep theirs. */
888
+ concurrent?: true;
889
+ }
890
+ /** The run's final result. */
891
+ interface RecordedOutput {
892
+ kind: "output";
893
+ value: unknown;
894
+ }
895
+ type RecordedEffect = RecordedModelCall | RecordedToolCall | RecordedOutput;
896
+ /** A run's effects, in order. JSON-serializable: save it as a golden file. */
897
+ interface Recording {
898
+ version: 1;
899
+ effects: RecordedEffect[];
900
+ }
901
+ /** Options for {@link createRecorder}. Omit for record mode. */
902
+ interface CreateRecorderOptions {
903
+ /** Answer model and tool calls from this recording instead of running them. */
904
+ replay?: Recording;
905
+ /**
906
+ * In replay, `"replay"` (default) returns each tool's recorded result
907
+ * without running it; `"live"` runs the wrapped tool (your stub) and records
908
+ * what it returned.
909
+ */
910
+ tools?: "replay" | "live";
911
+ /**
912
+ * In replay, which model calls a recorded answer may serve. `"shape"`
913
+ * (default) refuses a call whose offered tools or response format changed,
914
+ * since the recorded answer may not fit it; see {@link ModelCallMatch}.
915
+ */
916
+ modelCalls?: ModelCallMatch;
917
+ /**
918
+ * In replay, maps the recorded and the new tool call before matching them,
919
+ * like {@link CompareRecordingsOptions.normalize}: redact a generated id in a
920
+ * tool's input so the recorded result still answers it.
921
+ */
922
+ normalize?: (effect: RecordedEffect) => RecordedEffect;
923
+ }
924
+ /**
925
+ * When two model calls count as the same: `"shape"` compares offered tool
926
+ * names and response format, `"none"` only that a call happened, and a
927
+ * function decides per call (allow the changes you made on purpose and fall
928
+ * back to {@link sameModelCallShape}).
929
+ */
930
+ type ModelCallMatch = "shape" | "none" | ((expected: RecordedModelCall, actual: RecordedModelCall) => boolean);
931
+ /** The structural slice of an AI SDK `LanguageModelV3` the recorder wraps. */
932
+ interface RecordableLanguageModel {
933
+ doGenerate(options: never): PromiseLike<unknown>;
934
+ doStream(options: never): PromiseLike<{
935
+ stream: ReadableStream<unknown>;
936
+ }>;
937
+ }
938
+ type AnyFunction = (...args: never[]) => unknown;
939
+ /** Records (or replays) the effects of one run. */
940
+ interface Recorder {
941
+ /**
942
+ * Wraps an AI SDK `LanguageModelV3` (`openai("gpt-…")`, `anthropic(…)`),
943
+ * keeping its type. In replay the model is never called, so a provider
944
+ * model built with no API key works.
945
+ */
946
+ model<TModel extends RecordableLanguageModel>(model: TModel): TModel;
947
+ /**
948
+ * Wraps an `openai` client's `chat.completions.create` and
949
+ * `responses.create`, keeping its type. A streamed result is an async
950
+ * iterable only. In replay the client is never called, so
951
+ * `new OpenAI({ apiKey: "replay" })` works.
952
+ */
953
+ openai<TClient extends object>(client: TClient): TClient;
954
+ /** Wraps one tool function. Its arguments are the recorded input. */
955
+ tool<TFn extends AnyFunction>(name: string, fn: TFn): TFn;
956
+ /** Wraps every `execute` in an AI SDK tool set, named by key. */
957
+ tools<TTools extends Record<string, {
958
+ execute?: AnyFunction;
959
+ }>>(tools: TTools): TTools;
960
+ /** Records the run's final result. */
961
+ output(value: unknown): void;
962
+ /** What this run did so far. */
963
+ readonly recording: Recording;
964
+ }
965
+ /**
966
+ * Thrown in replay when the port asks for something the recording cannot
967
+ * answer: a model call past the last recorded one, or a tool call the original
968
+ * never made. The effect is still recorded, so {@link compareRecordings}
969
+ * reports it.
970
+ */
971
+ export declare class RecordingDivergenceError extends Error {
972
+ name: string;
973
+ }
974
+ /**
975
+ * Creates a {@link Recorder}. With no options it records: every wrapped call
976
+ * runs for real. With `replay` it answers from a recording instead.
977
+ */
978
+ export declare function createRecorder(options?: CreateRecorderOptions): Recorder;
979
+ /** Why two effects differ. */
980
+ type EffectDivergenceReason = "missing-effect" | "extra-effect" | "kind" | "model-method" | "model-call-shape" | "model-error" | "tool-name" | "tool-input" | "tool-result" | "output";
981
+ /** The first effect where two recordings differ. */
982
+ interface EffectDivergence {
983
+ /** Position in the compared (tool-order-normalized) effect lists. */
984
+ index: number;
985
+ reason: EffectDivergenceReason;
986
+ expected?: RecordedEffect;
987
+ actual?: RecordedEffect;
988
+ /** One sentence, for test failures. */
989
+ message: string;
990
+ }
991
+ /** The result of {@link compareRecordings}. */
992
+ interface EquivalenceReport {
993
+ equivalent: boolean;
994
+ /** The first divergence, by position in the original's effects. */
995
+ divergence?: EffectDivergence;
996
+ /** Every divergence, in order. Effects both runs made line up between them. */
997
+ divergences: EffectDivergence[];
998
+ }
999
+ /** Options for {@link compareRecordings}. */
1000
+ interface CompareRecordingsOptions {
1001
+ /** When two model calls count as the same. Default `"shape"`. */
1002
+ modelCalls?: ModelCallMatch;
1003
+ /**
1004
+ * Maps each effect before comparing, on both sides: redact values that
1005
+ * differ on every run (generated ids, timestamps) or project a tool's input
1006
+ * to the fields that matter.
1007
+ */
1008
+ normalize?: (effect: RecordedEffect) => RecordedEffect;
1009
+ }
1010
+ /** True when two model calls offer the same tool names and response format. */
1011
+ export declare function sameModelCallShape(expected: RecordedModelCall, actual: RecordedModelCall): boolean;
1012
+ /**
1013
+ * Compares two recordings and reports every effect that differs. `expected`
1014
+ * is the original's recording, `actual` the port's. Effects that are the same
1015
+ * on both sides are lined up first (longest common subsequence), so one extra
1016
+ * call does not shift every effect after it; between them, effects of the same
1017
+ * kind pair up and report what changed, and the rest are missing or extra.
1018
+ */
1019
+ export declare function compareRecordings(expected: Recording, actual: Recording, options?: CompareRecordingsOptions): EquivalenceReport;
1020
+ //#endregion
1021
+ export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AgreementReport, type AgreementRow, type AgreementSummary, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRecordingsOptions, type CompareRunsOptions, type CoverageReport, type CreateRecorderOptions, type Divergence, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type FirstDivergenceOptions, type FoldTracesOptions, type GetDecisionRequestsOptions, type HandlerCoverage, type HandlerRef, type JudgeAgreementOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelCallMatch, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordableLanguageModel, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, type RepeatGroup, type RepeatReport, type RepeatResult, type RepeatRun, type RepeatRunsOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateCoverage, type StateStats, type TraceCoverageOptions, type TraceStep, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };