@statelyai/agent 2.0.0-alpha.31 → 2.0.0-alpha.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/dist/ai-sdk.cjs +3 -3
  3. package/dist/ai-sdk.d.cts +1 -1
  4. package/dist/ai-sdk.d.mts +1 -1
  5. package/dist/ai-sdk.mjs +3 -3
  6. package/dist/{decision-Iys2hzg3.cjs → decision-BPdEaxPf.cjs} +1 -1
  7. package/dist/{decision-DqO018Tp.mjs → decision-CV7WiV5C.mjs} +1 -1
  8. package/dist/{event-log-Bdww2l1D.cjs → event-log-CKWeC7H-.cjs} +22 -3
  9. package/dist/{event-log-DE430ZCa.mjs → event-log-DAqsalT1.mjs} +22 -3
  10. package/dist/{event-log-store-CCWdTPEP.mjs → event-log-store-BGehvVTb.mjs} +2 -2
  11. package/dist/{event-log-store-DTf_OzB-.d.cts → event-log-store-C2NjVlWU.d.mts} +13 -5
  12. package/dist/{event-log-store-BWMLi6oh.cjs → event-log-store-DKyE7JKr.cjs} +2 -2
  13. package/dist/{event-log-store-RziIobdW.d.mts → event-log-store-s73r1Hxf.d.cts} +13 -5
  14. package/dist/index.cjs +5 -5
  15. package/dist/index.d.cts +9 -8
  16. package/dist/index.d.mts +9 -8
  17. package/dist/index.mjs +5 -5
  18. package/dist/log.cjs +40 -5
  19. package/dist/log.d.cts +1 -1
  20. package/dist/log.d.mts +1 -1
  21. package/dist/log.mjs +40 -5
  22. package/dist/machines.cjs +1 -1
  23. package/dist/machines.mjs +1 -1
  24. package/dist/{model-source-DQ7KgJaT.cjs → model-source-BQ97_l1-.cjs} +1 -1
  25. package/dist/{model-source-CXsSZPq0.mjs → model-source-BilDEyqk.mjs} +1 -1
  26. package/dist/openai.cjs +2 -2
  27. package/dist/openai.d.cts +1 -1
  28. package/dist/openai.d.mts +1 -1
  29. package/dist/openai.mjs +2 -2
  30. package/dist/otel.d.cts +1 -1
  31. package/dist/otel.d.mts +1 -1
  32. package/dist/{run-agent-DWZ1uyBJ.mjs → run-agent-2UnSUhDb.mjs} +7 -5
  33. package/dist/{run-agent-D4MwS5GY.d.mts → run-agent-Atu-wEk7.d.mts} +3 -3
  34. package/dist/{run-agent-B5opVUxx.d.cts → run-agent-B5QhabdY.d.cts} +3 -3
  35. package/dist/{run-agent-tmk58M0x.cjs → run-agent-Cg0xd3K-.cjs} +7 -5
  36. package/dist/{setup-agent-yFFVXf5s.d.cts → setup-agent-BiF1N3td.d.cts} +2 -2
  37. package/dist/{setup-agent-BaffhCcM.mjs → setup-agent-CmhB-U5P.mjs} +2 -2
  38. package/dist/{setup-agent-D0L8Anl_.cjs → setup-agent-CwUJyqKB.cjs} +2 -2
  39. package/dist/{setup-agent-CPyg2ic7.d.mts → setup-agent-J9f_Wu5T.d.mts} +2 -2
  40. package/dist/tanstack-ai.cjs +3 -3
  41. package/dist/tanstack-ai.d.cts +1 -1
  42. package/dist/tanstack-ai.d.mts +1 -1
  43. package/dist/tanstack-ai.mjs +3 -3
  44. package/dist/testing.cjs +495 -5
  45. package/dist/testing.d.cts +389 -5
  46. package/dist/testing.d.mts +389 -5
  47. package/dist/testing.mjs +491 -6
  48. package/dist/{text-logic-j9hShB7d.d.cts → text-logic-CYP502VT.d.cts} +5 -3
  49. package/dist/{text-logic-BwdrDpO9.d.mts → text-logic-DuK9HIFM.d.mts} +5 -3
  50. package/dist/{usage-BYqv7VBw.d.cts → usage-D7JKAxLc.d.mts} +1 -1
  51. package/dist/{usage-Bl4nK8D-.d.mts → usage-DVw-646x.d.cts} +1 -1
  52. package/dist/validate.cjs +3 -3
  53. package/dist/validate.d.cts +2 -2
  54. package/dist/validate.d.mts +2 -2
  55. package/dist/validate.mjs +3 -3
  56. package/dist/{verify-tkymWz6X.d.mts → verify-BVz4cURF.d.mts} +1 -1
  57. package/dist/{verify-CS_xxvVU.cjs → verify-CS_WLRXn.cjs} +19 -3
  58. package/dist/{verify-BH5e0XLK.mjs → verify-DgB8P-Wx.mjs} +14 -4
  59. package/dist/{verify-CPKDPMwY.d.cts → verify-Dk3Vwnha.d.cts} +1 -1
  60. package/package.json +1 -1
  61. package/skills/add-evals/SKILL.md +8 -2
  62. package/skills/improve-machine/SKILL.md +1 -1
package/dist/testing.mjs CHANGED
@@ -1,6 +1,6 @@
1
- import { $ as getStateMeta, F as normalizeGeneratorResult, N as getCallUsage, lt as isRecord, ut as AgentError } from "./event-log-DE430ZCa.mjs";
2
- import { p as runToQuiescence, s as createAgentRuntime } from "./run-agent-DWZ1uyBJ.mjs";
3
- import { a as explorePaths, i as canReach, n as AgentUnknownStateError, o as lintAgentMachine, r as assertAgentMachine, s as simulateAgent, t as AgentLintError } from "./verify-BH5e0XLK.mjs";
1
+ import { $ as getStateMeta, F as normalizeGeneratorResult, N as getCallUsage, lt as isRecord, ut as AgentError } from "./event-log-DAqsalT1.mjs";
2
+ import { p as runToQuiescence, s as createAgentRuntime } from "./run-agent-2UnSUhDb.mjs";
3
+ import { a as explorePaths, c as simulateAgent, i as canReach, n as AgentUnknownStateError, o as isExplorationComplete, r as assertAgentMachine, s as lintAgentMachine, t as AgentLintError } from "./verify-DgB8P-Wx.mjs";
4
4
  //#region src/trajectory.ts
5
5
  /**
6
6
  * Classifies one item. Event-shaped things (log entries, events) become events;
@@ -380,12 +380,47 @@ function statePaths(value, prefix = "") {
380
380
  if (entries.length === 0) return prefix ? [prefix.slice(0, -1)] : [];
381
381
  return entries.flatMap(([key, child]) => statePaths(child, `${prefix}${key}.`));
382
382
  }
383
+ /**
384
+ * A trace's `machine.transition` events as steps, skipping `@xstate.init`.
385
+ * A first-attempt decision's `request.start` names the decision on the step
386
+ * that follows it. Reads live and serialized traces alike.
387
+ * @internal
388
+ */
389
+ function traceSteps(trace) {
390
+ const steps = [];
391
+ let decision;
392
+ for (const event of trace) {
393
+ if (event.type === "request.start" && isFirstDecision(event.request)) {
394
+ const name = event.request.name;
395
+ decision = typeof name === "string" ? name : void 0;
396
+ continue;
397
+ }
398
+ if (event.type !== "machine.transition") continue;
399
+ const type = isRecord(event.event) ? event.event.type : void 0;
400
+ if (type === "@xstate.init") continue;
401
+ const fromValue = valueOf(event.from);
402
+ const toValue = valueOf(event.snapshot);
403
+ steps.push({
404
+ from: statePaths(fromValue),
405
+ fromValue,
406
+ event: typeof type === "string" ? type : "",
407
+ payload: event.event,
408
+ to: statePaths(toValue),
409
+ toValue,
410
+ ignored: event.ignored === true,
411
+ ...decision === void 0 ? {} : { decision }
412
+ });
413
+ decision = void 0;
414
+ }
415
+ return steps;
416
+ }
383
417
  //#endregion
384
418
  //#region src/fold-traces.ts
385
419
  /**
386
420
  * Folds many runs' traces into per-state stats: how many runs visited each
387
421
  * state, how many of those failed, how many options the guards withheld
388
- * at decisions made there, and how many events it ignored. Pure; reads only:
422
+ * at decisions made there, how many events it ignored, and how many failures
423
+ * `blame` traced back to it. Pure; reads only:
389
424
  *
390
425
  * - `machine.transition`: the run visited `from.value` and `snapshot.value`;
391
426
  * with `ignored`, each of those states ignored one event.
@@ -398,6 +433,10 @@ function statePaths(value, prefix = "") {
398
433
  * `{ type: 'request.start', request: result.request }` onto the trace before
399
434
  * sending its event. Parallel states count once per active leaf.
400
435
  *
436
+ * `failures` counts a failed run against every state it visited, so states
437
+ * after a bad decision rank as high as the decision. Sort by `blamed` to see
438
+ * where failures start.
439
+ *
401
440
  * @example
402
441
  * ```ts no-check
403
442
  * const stats = foldTraces(traces, {
@@ -415,7 +454,8 @@ function foldTraces(traces, options) {
415
454
  visits: 0,
416
455
  failures: 0,
417
456
  withheld: 0,
418
- ignored: 0
457
+ ignored: 0,
458
+ blamed: 0
419
459
  };
420
460
  stats.set(state, entry);
421
461
  }
@@ -444,10 +484,249 @@ function foldTraces(traces, options) {
444
484
  entry.visits++;
445
485
  if (failed) entry.failures++;
446
486
  }
487
+ const blamed = failed ? options.blame?.(trace) : void 0;
488
+ if (blamed !== void 0) statsFor(blamed).blamed++;
447
489
  }
448
490
  return [...stats.values()];
449
491
  }
450
492
  //#endregion
493
+ //#region src/trace-coverage.ts
494
+ /** Mirrors xstate@6.0.0-alpha.69 `matchesEventDescriptor` (src/utils.ts), which the package does not export. */
495
+ function matchesDescriptor(eventType, descriptor) {
496
+ if (descriptor === eventType || descriptor === "*") return true;
497
+ if (!descriptor.endsWith(".*")) return false;
498
+ const tokens = descriptor.split(".");
499
+ const eventTokens = eventType.split(".");
500
+ for (let index = 0; index < tokens.length; index++) {
501
+ if (tokens[index] === "*") return index === tokens.length - 1;
502
+ if (tokens[index] !== eventTokens[index]) return false;
503
+ }
504
+ return true;
505
+ }
506
+ /** Whether a definition's `matches` pattern (e.g. an invoke's `actorId`) accepts the traced event. */
507
+ function matchesPattern(definition, event) {
508
+ const pattern = definition.matches;
509
+ if (!isRecord(pattern) || !isRecord(event)) return true;
510
+ return Object.entries(pattern).every(([key, value]) => Object.is(event[key], value));
511
+ }
512
+ /**
513
+ * The descriptor XState would try first at `node` for `event`, guards aside:
514
+ * the exact key, then partial and wildcard keys longest first, as in
515
+ * xstate@6.0.0-alpha.69 `getCandidates` (src/stateUtils.ts).
516
+ */
517
+ function candidateAt(node, type, event) {
518
+ const keys = [...node.transitions.keys()].filter((key) => key !== type && matchesDescriptor(type, key)).sort((a, b) => b.length - a.length);
519
+ if (node.transitions.has(type)) keys.unshift(type);
520
+ return keys.find((key) => node.transitions.get(key).some((d) => matchesPattern(d, event)));
521
+ }
522
+ const pathOf = (node) => node.path.join(".");
523
+ const ratioOf = (visited, total) => total === 0 ? 1 : visited / total;
524
+ /**
525
+ * Measures how much of a machine a set of runs exercised: which atomic and
526
+ * final states they visited, and which handlers they took. A handler is a
527
+ * `(declaring state, event descriptor)` pair from `on`, invoke
528
+ * `onDone`/`onError`, and `after`, with the targets it declares and the
529
+ * leaves the runs landed in after it. Pure; reads only `machine.transition`
530
+ * events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
531
+ *
532
+ * A step credits one handler per active leaf: the nearest node, from the leaf
533
+ * up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
534
+ * event, preferred in XState's order. Guards are not replayed, so a step
535
+ * whose nearest handler was rejected and fell through to an ancestor is
536
+ * credited to the nearest one. XState keys invoke and `after` handlers by
537
+ * bare type (`xstate.done.actor`), one per state however many invokes; a
538
+ * definition's `matches` (an invoke's `actorId`) must accept the traced event
539
+ * for its state to be credited. Ignored steps credit no handler. `always`
540
+ * handlers are listed as `unobservable`, never as unvisited.
541
+ *
542
+ * @example
543
+ * ```ts no-check
544
+ * const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
545
+ * expect(report.unvisitedHandlers).toEqual([]);
546
+ * ```
547
+ */
548
+ function traceCoverage(machine, traces, options = {}) {
549
+ const nodes = /* @__PURE__ */ new Map();
550
+ const states = [];
551
+ const handlers = [];
552
+ const handlerAt = /* @__PURE__ */ new Map();
553
+ const unobservable = [];
554
+ const walk = (node) => {
555
+ const state = pathOf(node);
556
+ nodes.set(state, node);
557
+ if (node.type === "atomic" || node.type === "final") states.push({
558
+ state,
559
+ runs: 0
560
+ });
561
+ const byEvent = /* @__PURE__ */ new Map();
562
+ for (const [event, definitions] of node.transitions) {
563
+ const dynamic = definitions.some((d) => typeof d.to === "function");
564
+ const declared = [...new Set(definitions.flatMap((d) => (d.target ?? []).map(pathOf)))];
565
+ const handler = {
566
+ state,
567
+ event,
568
+ runs: 0,
569
+ targets: {
570
+ declared: dynamic ? "dynamic" : declared,
571
+ seen: [],
572
+ unreached: []
573
+ }
574
+ };
575
+ handlers.push(handler);
576
+ byEvent.set(event, handler);
577
+ }
578
+ handlerAt.set(node, byEvent);
579
+ if (node.always?.length) unobservable.push({
580
+ state,
581
+ event: "always"
582
+ });
583
+ for (const child of Object.values(node.states)) walk(child);
584
+ };
585
+ walk(machine.root);
586
+ const stateRuns = new Map(states.map((row) => [row.state, row]));
587
+ const seen = /* @__PURE__ */ new Map();
588
+ for (const trace of traces) {
589
+ const steps = traceSteps(trace);
590
+ const visited = new Set(steps[0]?.from);
591
+ for (const event of trace) {
592
+ if (event.type !== "machine.transition" || !isRecord(event.event)) continue;
593
+ if (event.event.type !== "@xstate.init" || !isRecord(event.snapshot)) continue;
594
+ for (const leaf of statePaths(event.snapshot.value)) visited.add(leaf);
595
+ }
596
+ const credited = /* @__PURE__ */ new Set();
597
+ for (const step of steps) {
598
+ for (const leaf of step.to) visited.add(leaf);
599
+ if (step.ignored) continue;
600
+ const hits = /* @__PURE__ */ new Set();
601
+ for (const leaf of step.from) for (let node = nodes.get(leaf); node; node = node.parent) {
602
+ if (node.type === "final") continue;
603
+ const event = candidateAt(node, step.event, step.payload);
604
+ if (event === void 0) continue;
605
+ hits.add(handlerAt.get(node).get(event));
606
+ break;
607
+ }
608
+ for (const handler of hits) {
609
+ credited.add(handler);
610
+ let leaves = seen.get(handler);
611
+ if (!leaves) seen.set(handler, leaves = /* @__PURE__ */ new Set());
612
+ for (const leaf of step.to) leaves.add(leaf);
613
+ }
614
+ }
615
+ for (const state of visited) {
616
+ const row = stateRuns.get(state);
617
+ if (row) row.runs++;
618
+ }
619
+ for (const handler of credited) handler.runs++;
620
+ }
621
+ for (const handler of handlers) {
622
+ const leaves = [...seen.get(handler) ?? []];
623
+ handler.targets.seen = leaves;
624
+ const { declared } = handler.targets;
625
+ if (declared === "dynamic") continue;
626
+ handler.targets.unreached = declared.filter((target) => !leaves.some((leaf) => leaf === target || leaf.startsWith(`${target}.`)));
627
+ }
628
+ const reachableComplete = options.reachable ? isExplorationComplete(options.reachable) : void 0;
629
+ let unreachable;
630
+ if (options.reachable && reachableComplete) {
631
+ const reached = new Set(options.reachable.reachedStates.flatMap((value) => statePaths(value)));
632
+ unreachable = states.filter((row) => !reached.has(row.state) && row.runs === 0).map((row) => row.state);
633
+ }
634
+ const counted = unreachable ? states.filter((row) => !unreachable.includes(row.state)) : states;
635
+ const unvisitedStates = counted.filter((row) => row.runs === 0).map((row) => row.state);
636
+ const unvisitedHandlers = handlers.filter((handler) => handler.runs === 0).map(({ state, event }) => ({
637
+ state,
638
+ event
639
+ }));
640
+ return {
641
+ states,
642
+ handlers,
643
+ unvisitedStates,
644
+ unvisitedHandlers,
645
+ unobservable,
646
+ ...unreachable ? { unreachable } : {},
647
+ ...reachableComplete === void 0 ? {} : { reachableComplete },
648
+ ratio: {
649
+ states: ratioOf(counted.length - unvisitedStates.length, counted.length),
650
+ handlers: ratioOf(handlers.length - unvisitedHandlers.length, handlers.length)
651
+ }
652
+ };
653
+ }
654
+ //#endregion
655
+ //#region src/first-divergence.ts
656
+ const defaultKey = (step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`;
657
+ const isTrace = (items) => items.length === 0 || items.some((item) => isRecord(item) && (item.type === "machine.transition" || item.type === "run.start" || item.type === "run.end"));
658
+ /**
659
+ * Finds the first step where a run departs from a reference, so a failure is
660
+ * blamed on the state it started in rather than on every state the run went
661
+ * through afterwards.
662
+ *
663
+ * - Against a reference **trace**, such as a passing run of the same
664
+ * scenario, steps are compared in order by `key`. The first step that
665
+ * differs, or the point where either trace ends early, is the divergence.
666
+ * - Against an expected **trajectory** (`TrajectoryItem[]`, as for
667
+ * {@link matchesTrajectory}), the run is laid out as its initial state, then
668
+ * each step's event and new state. The first expected item it never reaches
669
+ * is the divergence, blamed on the step after the last item it did reach. A
670
+ * run that goes on past the trajectory has not diverged.
671
+ *
672
+ * An empty reference is a trace with no steps, so any step diverges from it.
673
+ * A trace is one run's events, live or serialized. Steps the machine ignored
674
+ * changed nothing and are left out on both sides. Returns `undefined` when the
675
+ * run does not diverge.
676
+ *
677
+ * @example
678
+ * ```ts no-check
679
+ * const stats = foldTraces(traces, {
680
+ * failed,
681
+ * blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
682
+ * });
683
+ * ```
684
+ */
685
+ function firstDivergence(actual, reference, options = {}) {
686
+ const steps = traceSteps(actual).filter((step) => !step.ignored);
687
+ return isTrace(reference) ? againstTrace(steps, traceSteps(reference).filter((step) => !step.ignored), options.key ?? defaultKey) : againstTrajectory(actual, steps, reference);
688
+ }
689
+ function diverged(steps, index, expected, initial = steps[0]?.fromValue) {
690
+ const step = steps[index];
691
+ return {
692
+ index,
693
+ state: (step ? step.from[0] : steps.at(-1)?.to[0] ?? statePaths(initial)[0]) ?? "",
694
+ ...step?.decision === void 0 ? {} : { decision: step.decision },
695
+ ...step ? { actual: step } : {},
696
+ ...expected === void 0 ? {} : { expected }
697
+ };
698
+ }
699
+ function againstTrace(steps, reference, key) {
700
+ const length = Math.max(steps.length, reference.length);
701
+ for (let index = 0; index < length; index++) {
702
+ const step = steps[index];
703
+ const want = reference[index];
704
+ if (step && want && key(step) === key(want)) continue;
705
+ return diverged(steps, index, want, steps[0]?.fromValue ?? reference[0]?.fromValue);
706
+ }
707
+ }
708
+ function againstTrajectory(trace, steps, expected) {
709
+ const initial = initialValue(trace, steps);
710
+ const timeline = initial === void 0 ? [] : [{ value: initial }];
711
+ const stepAt = initial === void 0 ? [] : [0];
712
+ steps.forEach((step, index) => {
713
+ timeline.push(step.payload);
714
+ stepAt.push(index);
715
+ if (JSON.stringify(step.toValue) !== JSON.stringify(step.fromValue)) {
716
+ timeline.push({ value: step.toValue });
717
+ stepAt.push(index);
718
+ }
719
+ });
720
+ const match = matchesTrajectory(timeline, expected);
721
+ if (!match.firstMiss) return void 0;
722
+ const at = match.firstMiss.searchedFrom;
723
+ return diverged(steps, at < timeline.length ? stepAt[at] ?? steps.length : steps.length, match.firstMiss.expected, initial);
724
+ }
725
+ function initialValue(trace, steps) {
726
+ for (const event of trace) if (event.type === "machine.transition" && isRecord(event.snapshot)) return event.snapshot.value;
727
+ return steps[0]?.fromValue;
728
+ }
729
+ //#endregion
451
730
  //#region src/decision-requests.ts
452
731
  /**
453
732
  * The decisions a run asked for, read from its trace: each decision's first
@@ -480,6 +759,212 @@ function getDecisionRequests(trace, options = {}) {
480
759
  return requests;
481
760
  }
482
761
  //#endregion
762
+ //#region src/repeat-runs.ts
763
+ /**
764
+ * Summarizes repeated runs of each scenario: how often each passed, whether
765
+ * all its runs passed (pass^k), how many distinct state paths they took, and
766
+ * where they first went different ways. Pure. Use it for runs collected
767
+ * elsewhere, such as CI shards or replays; {@link repeatRuns} collects them and
768
+ * calls it.
769
+ *
770
+ * A scenario that passes 3 of 5 runs and `divergedAt: 'deciding'` points at one
771
+ * decision, not at a flaky agent.
772
+ *
773
+ * @example
774
+ * ```ts no-check
775
+ * const report = summarizeRepeats(shards.map((s) => ({ id: s.id, scenario: s, runs: s.runs })));
776
+ * report.passHat(3); // the chance 3 runs of a scenario all pass
777
+ * ```
778
+ */
779
+ function summarizeRepeats(groups) {
780
+ const scenarios = groups.map((group) => summarize$1(group));
781
+ const traces = groups.flatMap((group) => group.runs.flatMap((run) => run.trace ? [run.trace] : []));
782
+ const most = Math.max(0, ...groups.map((group) => group.runs.length));
783
+ const passHat = (j) => {
784
+ if (!Number.isInteger(j) || j < 1 || j > most) throw new AgentError("pass-hat-out-of-range", `passHat: j must be a whole number from 1 to ${most}, got ${j}.`);
785
+ const eligible = scenarios.filter((scenario) => scenario.runs.length >= j);
786
+ return eligible.reduce((total, scenario) => total + allPass(scenario.passes, scenario.runs.length, j), 0) / eligible.length;
787
+ };
788
+ return {
789
+ scenarios,
790
+ passHat,
791
+ traces
792
+ };
793
+ }
794
+ /**
795
+ * Runs every scenario `k` times and summarizes the runs with
796
+ * {@link summarizeRepeats}. Runs are sequential, scenario by scenario. A run
797
+ * that throws fails, with its message as `error` and no trace.
798
+ *
799
+ * With a live model, one run per scenario says little: τ-bench found pass^k
800
+ * falls well below the single-run rate. Report pass^k next to it.
801
+ *
802
+ * @example
803
+ * ```ts no-check
804
+ * const report = await repeatRuns({
805
+ * scenarios,
806
+ * k: 5,
807
+ * run: async (scenario) => {
808
+ * const trace: AgentTraceEvent[] = [];
809
+ * const result = await runToQuiescence(
810
+ * createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
811
+ * { input: scenario.input },
812
+ * );
813
+ * return { trace, outcome: String(result.snapshot.value) };
814
+ * },
815
+ * passed: (scenario, { outcome }) => outcome === scenario.expected,
816
+ * });
817
+ * report.passHat(5);
818
+ * ```
819
+ */
820
+ async function repeatRuns(options) {
821
+ if (!Number.isInteger(options.k) || options.k < 1) throw new AgentError("invalid-repeat-count", `repeatRuns: k must be a whole number of 1 or more, got ${options.k}.`);
822
+ const groups = [];
823
+ for (const scenario of options.scenarios) {
824
+ const runs = [];
825
+ for (let attempt = 0; attempt < options.k; attempt++) try {
826
+ const result = await options.run(scenario, attempt);
827
+ runs.push({
828
+ passed: options.passed(scenario, result),
829
+ trace: result.trace,
830
+ ...result.outcome === void 0 ? {} : { outcome: result.outcome }
831
+ });
832
+ } catch (error) {
833
+ runs.push({
834
+ passed: false,
835
+ error: error instanceof Error ? error.message : String(error)
836
+ });
837
+ }
838
+ groups.push({
839
+ id: scenario.id,
840
+ scenario,
841
+ runs
842
+ });
843
+ }
844
+ return summarizeRepeats(groups);
845
+ }
846
+ function summarize$1(group) {
847
+ const passes = group.runs.filter((run) => run.passed).length;
848
+ const traces = group.runs.flatMap((run) => run.trace ? [run.trace] : []);
849
+ const paths = new Set(traces.map((trace) => traceSteps(trace).filter((step) => !step.ignored).map((step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`).join(" ")));
850
+ const reference = traces.find((trace) => trace.some((event) => event.type === "machine.transition"));
851
+ let earliest;
852
+ for (const trace of reference ? traces : []) {
853
+ if (trace === reference) continue;
854
+ const divergence = firstDivergence(trace, reference);
855
+ if (divergence && (!earliest || divergence.index < earliest.index)) earliest = divergence;
856
+ }
857
+ return {
858
+ ...group,
859
+ passes,
860
+ passAny: passes > 0,
861
+ passAll: group.runs.length > 0 && passes === group.runs.length,
862
+ distinctPaths: paths.size,
863
+ ...earliest ? { divergedAt: earliest.state } : {}
864
+ };
865
+ }
866
+ /** C(passes, j) / C(runs, j), as a product so large counts don't overflow. */
867
+ function allPass(passes, runs, j) {
868
+ let chance = 1;
869
+ for (let i = 0; i < j; i++) chance *= Math.max(0, passes - i) / (runs - i);
870
+ return chance;
871
+ }
872
+ //#endregion
873
+ //#region src/judge-agreement.ts
874
+ /**
875
+ * Measures how well a judge agrees with people on the same cases: accuracy,
876
+ * per-label recall, the confusion matrix, and the rows that disagree. With
877
+ * `positive`, also the TPR and TNR, the recommended view for binary judges
878
+ * (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
879
+ * Pure.
880
+ *
881
+ * Build rows from confirmed labels only; draft labels gate nothing. Group by
882
+ * decision state or request name so each narrow judge is aligned on its own.
883
+ * A row with no group key counts in the totals and in no group. Run it on a
884
+ * dev split and report the held-out split separately.
885
+ *
886
+ * Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
887
+ *
888
+ * @example
889
+ * ```ts no-check
890
+ * const report = judgeAgreement(
891
+ * labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
892
+ * { positive: 'fail' },
893
+ * );
894
+ * report.binary; // { tpr: 0.9, tnr: 0.8 }
895
+ * report.correct?.(0.3); // estimated true failure rate in production
896
+ * ```
897
+ */
898
+ function judgeAgreement(rows, options = {}) {
899
+ if (rows.length === 0) throw new AgentError("empty-agreement-rows", "judgeAgreement: rows must contain at least one row.");
900
+ const report = summarize(rows, options.positive);
901
+ const groupOf = options.groupBy ?? ((row) => row.group);
902
+ if (options.groupBy || rows.some((row) => row.group !== void 0)) {
903
+ const grouped = /* @__PURE__ */ new Map();
904
+ for (const row of rows) {
905
+ const key = groupOf(row);
906
+ if (key === void 0) continue;
907
+ const members = grouped.get(key);
908
+ if (members) members.push(row);
909
+ else grouped.set(key, [row]);
910
+ }
911
+ const groups = Object.create(null);
912
+ for (const [key, members] of grouped) groups[key] = summarize(members, options.positive);
913
+ report.groups = groups;
914
+ }
915
+ return report;
916
+ }
917
+ function summarize(rows, positive) {
918
+ const perLabel = Object.create(null);
919
+ const confusion = Object.create(null);
920
+ const disagreements = [];
921
+ let agreed = 0;
922
+ for (const { id, human, judge } of rows) {
923
+ const label = perLabel[human] ??= {
924
+ human: 0,
925
+ agreed: 0
926
+ };
927
+ const row = confusion[human] ??= Object.create(null);
928
+ label.human++;
929
+ row[judge] = (row[judge] ?? 0) + 1;
930
+ if (human === judge) {
931
+ label.agreed++;
932
+ agreed++;
933
+ } else disagreements.push({
934
+ id,
935
+ human,
936
+ judge
937
+ });
938
+ }
939
+ const summary = {
940
+ total: rows.length,
941
+ accuracy: agreed / rows.length,
942
+ perLabel,
943
+ confusion,
944
+ disagreements
945
+ };
946
+ if (positive === void 0) return summary;
947
+ let positives = 0;
948
+ let truePositives = 0;
949
+ let negatives = 0;
950
+ let trueNegatives = 0;
951
+ for (const { human, judge } of rows) if (human === positive) {
952
+ positives++;
953
+ if (judge === positive) truePositives++;
954
+ } else {
955
+ negatives++;
956
+ if (judge !== positive) trueNegatives++;
957
+ }
958
+ const tpr = positives === 0 ? void 0 : truePositives / positives;
959
+ const tnr = negatives === 0 ? void 0 : trueNegatives / negatives;
960
+ summary.binary = {
961
+ tpr,
962
+ tnr
963
+ };
964
+ if (tpr !== void 0 && tnr !== void 0 && tpr + tnr > 1) summary.correct = (p) => Math.min(1, Math.max(0, (p + tnr - 1) / (tpr + tnr - 1)));
965
+ return summary;
966
+ }
967
+ //#endregion
483
968
  //#region src/compare-runs.ts
484
969
  /**
485
970
  * Runs every scenario through a baseline and a candidate implementation and
@@ -585,4 +1070,4 @@ function structurallyEqual(a, b) {
585
1070
  return keys.length === Object.keys(b).length && keys.every((key) => Object.hasOwn(b, key) && structurallyEqual(a[key], b[key]));
586
1071
  }
587
1072
  //#endregion
588
- export { AgentLintError, AgentUnknownStateError, assertAgentMachine, canReach, compareRuns, explorePaths, foldTraces, getDecisionRequests, lintAgentMachine, matchesTrajectory, runSeam, simulateAgent };
1073
+ export { AgentLintError, AgentUnknownStateError, assertAgentMachine, canReach, compareRuns, explorePaths, firstDivergence, foldTraces, getDecisionRequests, judgeAgreement, lintAgentMachine, matchesTrajectory, repeatRuns, runSeam, simulateAgent, summarizeRepeats, traceCoverage };
@@ -818,9 +818,11 @@ interface AgentRequestExecutorInfo {
818
818
  * `a` being the number of prior failed attempts), so a retry never collides
819
819
  * with the rejected attempt it is replacing.
820
820
  *
821
- * It identifies the CALL SITE, not the request: a fork inherits its parent's
822
- * lineage id, so cache on `callKey` together with a fingerprint of the
823
- * request and reuse a cached result only when the request also matches.
821
+ * It identifies the CALL SITE, not the request: a version bridge keeps the
822
+ * lineage id, so a migrated machine can ask something else under the same
823
+ * key. Cache on `callKey` together with a fingerprint of the request and
824
+ * reuse a cached result only when the request also matches. A fork mints
825
+ * its own lineage id, so its keys never match its parent's.
824
826
  *
825
827
  * Undefined off the `createAgentRuntime` path, and for a log with no `executionId`.
826
828
  */
@@ -818,9 +818,11 @@ interface AgentRequestExecutorInfo {
818
818
  * `a` being the number of prior failed attempts), so a retry never collides
819
819
  * with the rejected attempt it is replacing.
820
820
  *
821
- * It identifies the CALL SITE, not the request: a fork inherits its parent's
822
- * lineage id, so cache on `callKey` together with a fingerprint of the
823
- * request and reuse a cached result only when the request also matches.
821
+ * It identifies the CALL SITE, not the request: a version bridge keeps the
822
+ * lineage id, so a migrated machine can ask something else under the same
823
+ * key. Cache on `callKey` together with a fingerprint of the request and
824
+ * reuse a cached result only when the request also matches. A fork mints
825
+ * its own lineage id, so its keys never match its parent's.
824
826
  *
825
827
  * Undefined off the `createAgentRuntime` path, and for a log with no `executionId`.
826
828
  */
@@ -1,4 +1,4 @@
1
- import { t as AgentCallUsage } from "./text-logic-j9hShB7d.cjs";
1
+ import { t as AgentCallUsage } from "./text-logic-DuK9HIFM.mjs";
2
2
  import { EventObject } from "xstate";
3
3
  //#region src/usage.d.ts
4
4
  /** Reserved event delivered after a model call reports token usage. */
@@ -1,4 +1,4 @@
1
- import { t as AgentCallUsage } from "./text-logic-BwdrDpO9.mjs";
1
+ import { t as AgentCallUsage } from "./text-logic-CYP502VT.cjs";
2
2
  import { EventObject } from "xstate";
3
3
  //#region src/usage.d.ts
4
4
  /** Reserved event delivered after a model call reports token usage. */
package/dist/validate.cjs CHANGED
@@ -21,9 +21,9 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
21
21
  enumerable: true
22
22
  }) : target, mod));
23
23
  //#endregion
24
- const require_event_log = require("./event-log-Bdww2l1D.cjs");
25
- const require_setup_agent = require("./setup-agent-D0L8Anl_.cjs");
26
- const require_verify = require("./verify-CS_xxvVU.cjs");
24
+ const require_event_log = require("./event-log-CKWeC7H-.cjs");
25
+ const require_setup_agent = require("./setup-agent-CwUJyqKB.cjs");
26
+ const require_verify = require("./verify-CS_WLRXn.cjs");
27
27
  let ajv_dist_2020_js = require("ajv/dist/2020.js");
28
28
  ajv_dist_2020_js = __toESM(ajv_dist_2020_js, 1);
29
29
  //#region schemas/agent-workflow.json
@@ -1,5 +1,5 @@
1
- import { d as AgentWorkflowConfig, g as FromConfigOptions, v as SchemaCompiler } from "./setup-agent-yFFVXf5s.cjs";
2
- import { u as ExplorePathsOptions } from "./verify-CPKDPMwY.cjs";
1
+ import { d as AgentWorkflowConfig, g as FromConfigOptions, v as SchemaCompiler } from "./setup-agent-BiF1N3td.cjs";
2
+ import { u as ExplorePathsOptions } from "./verify-Dk3Vwnha.cjs";
3
3
  import { AnyActorLogic, AnyStateMachine } from "xstate";
4
4
  //#region src/validate/validate-config.d.ts
5
5
  /**
@@ -1,5 +1,5 @@
1
- import { d as AgentWorkflowConfig, g as FromConfigOptions, v as SchemaCompiler } from "./setup-agent-CPyg2ic7.mjs";
2
- import { u as ExplorePathsOptions } from "./verify-tkymWz6X.mjs";
1
+ import { d as AgentWorkflowConfig, g as FromConfigOptions, v as SchemaCompiler } from "./setup-agent-J9f_Wu5T.mjs";
2
+ import { u as ExplorePathsOptions } from "./verify-BVz4cURF.mjs";
3
3
  import { AnyActorLogic, AnyStateMachine } from "xstate";
4
4
  //#region src/validate/validate-config.d.ts
5
5
  /**
package/dist/validate.mjs CHANGED
@@ -1,6 +1,6 @@
1
- import { D as STREAM_TEXT_ACTOR, E as GENERATE_TEXT_ACTOR, T as DECIDE_ACTOR } from "./event-log-DE430ZCa.mjs";
2
- import { i as setupAgentFromConfig } from "./setup-agent-BaffhCcM.mjs";
3
- import { a as explorePaths, o as lintAgentMachine } from "./verify-BH5e0XLK.mjs";
1
+ import { D as STREAM_TEXT_ACTOR, E as GENERATE_TEXT_ACTOR, T as DECIDE_ACTOR } from "./event-log-DAqsalT1.mjs";
2
+ import { i as setupAgentFromConfig } from "./setup-agent-CmhB-U5P.mjs";
3
+ import { a as explorePaths, s as lintAgentMachine } from "./verify-DgB8P-Wx.mjs";
4
4
  import Ajv2020 from "ajv/dist/2020.js";
5
5
  //#region schemas/agent-workflow.json
6
6
  var agent_workflow_default = {
@@ -1,5 +1,5 @@
1
1
  import { u as ChosenEvent } from "./types-Dp5_Ru4R.mjs";
2
- import { Q as AgentError } from "./text-logic-BwdrDpO9.mjs";
2
+ import { Q as AgentError } from "./text-logic-DuK9HIFM.mjs";
3
3
  import { AnyMachineSnapshot, AnyStateMachine } from "xstate";
4
4
  //#region src/verify.d.ts
5
5
  /** Severity of an {@link AgentLintDiagnostic}. `error` findings fail CI/the CLI; `warning`s are advisory. */