@statelyai/agent 2.0.0-alpha.31 → 2.0.0-alpha.33

package/dist/testing.d.mts

@@ -1,7 +1,7 @@
-import { P as AgentExecutorDecisionRequest, c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-BwdrDpO9.mjs";
-import { f as AgentRuntimeOptions, l as AgentRunResult, o as AgentRunInit } from "./run-agent-D4MwS5GY.mjs";
+import { P as AgentExecutorDecisionRequest, c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-DuK9HIFM.mjs";
+import { f as AgentRuntimeOptions, l as AgentRunResult, o as AgentRunInit } from "./run-agent-Atu-wEk7.mjs";
 import { t as MetaOfSnapshot } from "./utils-BnXdWih3.mjs";
-import { _ as assertAgentMachine, a as AgentPathTerminal, b as lintAgentMachine, c as CanReachResult, d as ExploreSnapshot, f as LintAgentMachineOptions, g as SimulationTrailEntry, h as SimulationScript, i as AgentPathReport, l as CanReachTarget, m as SimulateAgentResult, n as AgentLintError, o as AgentUnknownStateError, p as SimulateAgentOptions, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as ExplorePathsOptions, v as canReach, x as simulateAgent, y as explorePaths } from "./verify-tkymWz6X.mjs";
+import { _ as assertAgentMachine, a as AgentPathTerminal, b as lintAgentMachine, c as CanReachResult, d as ExploreSnapshot, f as LintAgentMachineOptions, g as SimulationTrailEntry, h as SimulationScript, i as AgentPathReport, l as CanReachTarget, m as SimulateAgentResult, n as AgentLintError, o as AgentUnknownStateError, p as SimulateAgentOptions, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as ExplorePathsOptions, v as canReach, x as simulateAgent, y as explorePaths } from "./verify-BVz4cURF.mjs";
 import { AnyStateMachine, EventFromLogic, InputFrom, SnapshotFrom, StateValue } from "xstate";
 //#region src/trajectory.d.ts
 /** An event-shaped trajectory item: anything with a string `type`. */
@@ -290,16 +290,25 @@ interface StateStats {
   withheld: number;
   /** Events this state ignored (`machine.transition` with `ignored`), summed over runs. */
   ignored: number;
+  /** Failed runs `blame` attributed to this state. 0 without `blame`. */
+  blamed: number;
 }
 /** Options for {@link foldTraces}. */
 interface FoldTracesOptions<TTrace> {
   /** Whether a run failed, judged from its whole trace. */
   failed: (trace: TTrace) => boolean;
+  /**
+   * The state a failed run's failure started in, usually
+   * `firstDivergence(trace, reference)?.state`. Called for failed runs only;
+   * `undefined` blames no state.
+   */
+  blame?: (trace: TTrace) => string | undefined;
 }
 /**
  * Folds many runs' traces into per-state stats: how many runs visited each
  * state, how many of those failed, how many options the guards withheld
- * at decisions made there, and how many events it ignored. Pure; reads only:
+ * at decisions made there, how many events it ignored, and how many failures
+ * `blame` traced back to it. Pure; reads only:
  *
  * - `machine.transition`: the run visited `from.value` and `snapshot.value`;
  *   with `ignored`, each of those states ignored one event.
@@ -312,6 +321,10 @@ interface FoldTracesOptions<TTrace> {
  * `{ type: 'request.start', request: result.request }` onto the trace before
  * sending its event. Parallel states count once per active leaf.
  *
+ * `failures` counts a failed run against every state it visited, so states
+ * after a bad decision rank as high as the decision. Sort by `blamed` to see
+ * where failures start.
+ *
  * @example
  * ```ts no-check
  * const stats = foldTraces(traces, {
@@ -323,6 +336,178 @@ export declare function foldTraces<TTrace extends readonly {
   readonly type: string;
 }[]>(traces: Iterable<TTrace>, options: FoldTracesOptions<TTrace>): StateStats[];
 //#endregion
+//#region src/trace-coverage.d.ts
+/** One state's row in a {@link CoverageReport}. */
+interface StateCoverage {
+  /** The state's dot-joined path from the root, e.g. `'deciding'` or `'work.review'`. */
+  state: string;
+  /** Runs that were in this state at least once. */
+  runs: number;
+}
+/** A handler: the state node that declares it and its event descriptor. */
+interface HandlerRef {
+  /** The declaring state's dot path; `''` for the machine root. */
+  state: string;
+  /** The descriptor as XState keys it: `'APPROVE'`, `'task.*'`, `'*'`, `'xstate.done.actor'`, `'xstate.error.actor'`, `'xstate.after'`. */
+  event: string;
+}
+/** One handler's row in a {@link CoverageReport}. */
+interface HandlerCoverage extends HandlerRef {
+  /** Runs in which this handler was credited at least once. */
+  runs: number;
+  targets: {
+    /** Target state paths of the object transitions under this descriptor; `'dynamic'` when any is a function transition. */
+    declared: string[] | "dynamic";
+    /** Active leaves observed right after a step credited to this handler. */
+    seen: string[];
+    /** Declared targets that no seen leaf is in. Empty when `declared` is `'dynamic'`. */
+    unreached: string[];
+  };
+}
+/** The result of {@link traceCoverage}. */
+interface CoverageReport {
+  /** Every atomic and final state, in document order, including `runs: 0`. */
+  states: StateCoverage[];
+  /** Every observable handler, in document order, including `runs: 0`. */
+  handlers: HandlerCoverage[];
+  /** States no run visited, excluding `unreachable` ones. */
+  unvisitedStates: string[];
+  /** Handlers no run was credited with. */
+  unvisitedHandlers: HandlerRef[];
+  /** `always` handlers: they fire inside a macrostep, which a trace does not record. */
+  unobservable: HandlerRef[];
+  /**
+   * With a complete `reachable` report: states no explored path reached and
+   * no trace visited. They leave the state denominator.
+   */
+  unreachable?: string[];
+  /**
+   * With `reachable`: whether the report explored every path. When `false`,
+   * a state it missed may still be reachable, so `unreachable` is absent and
+   * every state stays in the denominator.
+   */
+  reachableComplete?: boolean;
+  /** Visited over total, for states and for handlers (unobservable excluded). 1 when the total is 0. */
+  ratio: {
+    states: number;
+    handlers: number;
+  };
+}
+/** Options for {@link traceCoverage}. */
+interface TraceCoverageOptions {
+  /**
+   * An `explorePaths` report. When it explored every path (no cap hit, every
+   * payload expanded, no branch stopped early), states it never reached drop
+   * out of the state denominator.
+   */
+  reachable?: AgentPathReport;
+}
+/**
+ * Measures how much of a machine a set of runs exercised: which atomic and
+ * final states they visited, and which handlers they took. A handler is a
+ * `(declaring state, event descriptor)` pair from `on`, invoke
+ * `onDone`/`onError`, and `after`, with the targets it declares and the
+ * leaves the runs landed in after it. Pure; reads only `machine.transition`
+ * events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
+ *
+ * A step credits one handler per active leaf: the nearest node, from the leaf
+ * up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
+ * event, preferred in XState's order. Guards are not replayed, so a step
+ * whose nearest handler was rejected and fell through to an ancestor is
+ * credited to the nearest one. XState keys invoke and `after` handlers by
+ * bare type (`xstate.done.actor`), one per state however many invokes; a
+ * definition's `matches` (an invoke's `actorId`) must accept the traced event
+ * for its state to be credited. Ignored steps credit no handler. `always`
+ * handlers are listed as `unobservable`, never as unvisited.
+ *
+ * @example
+ * ```ts no-check
+ * const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
+ * expect(report.unvisitedHandlers).toEqual([]);
+ * ```
+ */
+export declare function traceCoverage<TTrace extends readonly {
+  readonly type: string;
+}[]>(machine: AnyStateMachine, traces: Iterable<TTrace>, options?: TraceCoverageOptions): CoverageReport;
+//#endregion
+//#region src/internal/trace-states.d.ts
+/** One macrostep read from a trace: a `machine.transition` event. */
+interface TraceStep {
+  /** Active leaves before the step. */
+  from: string[];
+  /** The state value before the step, as traced. */
+  fromValue: unknown;
+  /** The event's `type`. */
+  event: string;
+  /** The event object, as traced. */
+  payload: unknown;
+  /** Active leaves after the step. */
+  to: string[];
+  /** The state value after the step, as traced. */
+  toValue: unknown;
+  /** The state had no transition for a host event, so nothing changed. */
+  ignored: boolean;
+  /** The `name` of the first-attempt decision request that started since the previous step. */
+  decision?: string;
+}
+//#endregion
+//#region src/first-divergence.d.ts
+/** Options for {@link firstDivergence}. */
+interface FirstDivergenceOptions {
+  /**
+   * Projects a step to the key two traces are compared by. Default
+   * `from|event|to`. Override to leave out what may differ between equivalent
+   * runs, such as a parallel region that only logs. Steps are still compared
+   * in order. Trace references only.
+   */
+  key?: (step: TraceStep) => string;
+}
+/** Where a run first departs from its reference. See {@link firstDivergence}. */
+interface Divergence {
+  /** The step's index in the run's steps, or the step count when the run ended early. */
+  index: number;
+  /** The state the run was in when it departed: the first active leaf before that step. */
+  state: string;
+  /** The decision request that chose that step's event, when one did. */
+  decision?: string;
+  /** The run's step. Absent when the run ended early. */
+  actual?: TraceStep;
+  /** The reference's step, or the expected trajectory item the run never reached. Absent when the run went on past a reference trace. */
+  expected?: TraceStep | TrajectoryItem;
+}
+/**
+ * Finds the first step where a run departs from a reference, so a failure is
+ * blamed on the state it started in rather than on every state the run went
+ * through afterwards.
+ *
+ * - Against a reference **trace**, such as a passing run of the same
+ *   scenario, steps are compared in order by `key`. The first step that
+ *   differs, or the point where either trace ends early, is the divergence.
+ * - Against an expected **trajectory** (`TrajectoryItem[]`, as for
+ *   {@link matchesTrajectory}), the run is laid out as its initial state, then
+ *   each step's event and new state. The first expected item it never reaches
+ *   is the divergence, blamed on the step after the last item it did reach. A
+ *   run that goes on past the trajectory has not diverged.
+ *
+ * An empty reference is a trace with no steps, so any step diverges from it.
+ * A trace is one run's events, live or serialized. Steps the machine ignored
+ * changed nothing and are left out on both sides. Returns `undefined` when the
+ * run does not diverge.
+ *
+ * @example
+ * ```ts no-check
+ * const stats = foldTraces(traces, {
+ *   failed,
+ *   blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
+ * });
+ * ```
+ */
+export declare function firstDivergence(actual: readonly {
+  readonly type: string;
+}[], reference: readonly {
+  readonly type: string;
+}[] | readonly TrajectoryItem[], options?: FirstDivergenceOptions): Divergence | undefined;
+//#endregion
 //#region src/decision-requests.d.ts
 /** One decision a run asked for. See {@link getDecisionRequests}. */
 interface TracedDecisionRequest {
@@ -357,6 +542,205 @@ export declare function getDecisionRequests(trace: Iterable<{
   readonly type: string;
 }>, options?: GetDecisionRequestsOptions): TracedDecisionRequest[];
 //#endregion
+//#region src/repeat-runs.d.ts
+type AnyTrace = readonly {
+  readonly type: string;
+}[];
+/** One run of a repeated scenario. See {@link summarizeRepeats}. */
+interface RepeatRun<TTrace extends AnyTrace = AnyTrace> {
+  passed: boolean;
+  /** The run's trace. Absent for a run that threw. */
+  trace?: TTrace;
+  /** How the run ended, for the report. Not compared. */
+  outcome?: string;
+  /** The message of a run that threw. */
+  error?: string;
+}
+/** One scenario's runs. See {@link summarizeRepeats}. */
+interface RepeatGroup<TScenario, TTrace extends AnyTrace = AnyTrace> {
+  id: string;
+  scenario: TScenario;
+  runs: readonly RepeatRun<TTrace>[];
+}
+/** One scenario's row in a {@link RepeatReport}. */
+interface RepeatResult<TScenario, TTrace extends AnyTrace = AnyTrace> extends RepeatGroup<TScenario, TTrace> {
+  /** Runs that passed. */
+  passes: number;
+  /** At least one run passed: pass@k for this scenario. */
+  passAny: boolean;
+  /** Every run passed: pass^k for this scenario. */
+  passAll: boolean;
+  /** Distinct state paths the traced runs took. 1 when every run went the same way. */
+  distinctPaths: number;
+  /** The state where the runs first went different ways, judged against the first run whose trace has a transition. */
+  divergedAt?: string;
+}
+/** The result of {@link summarizeRepeats} and {@link repeatRuns}. */
+interface RepeatReport<TScenario, TTrace extends AnyTrace = AnyTrace> {
+  scenarios: RepeatResult<TScenario, TTrace>[];
+  /**
+   * The suite's pass^j: the chance that `j` runs of a scenario, drawn from the
+   * ones recorded, all pass, averaged over scenarios. The unbiased estimate is
+   * C(passes, j) / C(runs, j) per scenario. Scenarios with fewer than `j` runs
+   * are left out. Throws when `j` is not a whole number from 1 to the most
+   * runs any scenario has.
+   */
+  passHat: (j: number) => number;
+  /** Every recorded trace, ready for `foldTraces` or `traceCoverage`. */
+  traces: TTrace[];
+}
+/** Options for {@link repeatRuns}. */
+interface RepeatRunsOptions<TScenario extends {
+  id: string;
+}, TTrace extends AnyTrace> {
+  scenarios: readonly TScenario[];
+  /** Runs per scenario. */
+  k: number;
+  /** Runs one scenario once and returns its trace. `attempt` counts from 0. */
+  run: (scenario: TScenario, attempt: number) => Promise<{
+    trace: TTrace;
+    outcome?: string;
+  }>;
+  /** Whether a run passed, judged from its ground truth. */
+  passed: (scenario: TScenario, result: {
+    trace: TTrace;
+    outcome?: string;
+  }) => boolean;
+}
+/**
+ * Summarizes repeated runs of each scenario: how often each passed, whether
+ * all its runs passed (pass^k), how many distinct state paths they took, and
+ * where they first went different ways. Pure. Use it for runs collected
+ * elsewhere, such as CI shards or replays; {@link repeatRuns} collects them and
+ * calls it.
+ *
+ * A scenario that passes 3 of 5 runs and `divergedAt: 'deciding'` points at one
+ * decision, not at a flaky agent.
+ *
+ * @example
+ * ```ts no-check
+ * const report = summarizeRepeats(shards.map((s) => ({ id: s.id, scenario: s, runs: s.runs })));
+ * report.passHat(3); // the chance 3 runs of a scenario all pass
+ * ```
+ */
+export declare function summarizeRepeats<TScenario, TTrace extends AnyTrace = AnyTrace>(groups: readonly RepeatGroup<TScenario, TTrace>[]): RepeatReport<TScenario, TTrace>;
+/**
+ * Runs every scenario `k` times and summarizes the runs with
+ * {@link summarizeRepeats}. Runs are sequential, scenario by scenario. A run
+ * that throws fails, with its message as `error` and no trace.
+ *
+ * With a live model, one run per scenario says little: τ-bench found pass^k
+ * falls well below the single-run rate. Report pass^k next to it.
+ *
+ * @example
+ * ```ts no-check
+ * const report = await repeatRuns({
+ *   scenarios,
+ *   k: 5,
+ *   run: async (scenario) => {
+ *     const trace: AgentTraceEvent[] = [];
+ *     const result = await runToQuiescence(
+ *       createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
+ *       { input: scenario.input },
+ *     );
+ *     return { trace, outcome: String(result.snapshot.value) };
+ *   },
+ *   passed: (scenario, { outcome }) => outcome === scenario.expected,
+ * });
+ * report.passHat(5);
+ * ```
+ */
+export declare function repeatRuns<TScenario extends {
+  id: string;
+}, TTrace extends AnyTrace>(options: RepeatRunsOptions<TScenario, TTrace>): Promise<RepeatReport<TScenario, TTrace>>;
+//#endregion
+//#region src/judge-agreement.d.ts
+/** One labeled case: what a person said and what the judge said. See {@link judgeAgreement}. */
+interface AgreementRow {
+  /** Identifies the case in `disagreements`. */
+  id: string;
+  /** The confirmed human label. */
+  human: string;
+  /** The judge's label for the same case. */
+  judge: string;
+  /** The group the row is aligned in, e.g. a decision state or request name. `groupBy` overrides it. */
+  group?: string;
+}
+/** Agreement numbers for one set of rows. See {@link AgreementReport}. */
+interface AgreementSummary {
+  /** Rows counted. */
+  total: number;
+  /** Share of rows where `judge` equals `human`. */
+  accuracy: number;
+  /** Per human label: rows with that label, and how many the judge matched. `agreed / human` is that label's recall. */
+  perLabel: Record<string, {
+    human: number;
+    agreed: number;
+  }>;
+  /** Counts by human label, then judge label. */
+  confusion: Record<string, Record<string, number>>;
+  /** Rows where the judge differs from the person, in input order. */
+  disagreements: {
+    id: string;
+    human: string;
+    judge: string;
+  }[];
+  /**
+   * With `positive` set: the true positive rate (judge says positive when the
+   * person did) and true negative rate (judge says any other label when the
+   * person did). A rate is `undefined` when no row has that human side.
+   */
+  binary?: {
+    tpr: number | undefined;
+    tnr: number | undefined;
+  };
+  /**
+   * With `positive` set: estimates the true share of positives among
+   * unlabeled rows from the share `p` the judge labeled positive,
+   * θ = (p + TNR − 1) / (TPR + TNR − 1), clamped to [0, 1]. Omitted when
+   * either rate is `undefined`, or when TPR + TNR ≤ 1: the judge is no better
+   * than chance, so its labels carry no signal to correct.
+   */
+  correct?: (observedPositiveRate: number) => number;
+}
+/** The result of {@link judgeAgreement}. */
+interface AgreementReport extends AgreementSummary {
+  /** The same numbers per group. Present when any row has `group` or `groupBy` is given. */
+  groups?: Record<string, AgreementSummary>;
+}
+/** Options for {@link judgeAgreement}. */
+interface JudgeAgreementOptions<TRow extends AgreementRow> {
+  /** The positive label. Every other label is negative. Turns on `binary` and `correct`. */
+  positive?: string;
+  /** The row's group, overriding `row.group`. Return `undefined` to leave the row out of every group. */
+  groupBy?: (row: TRow) => string | undefined;
+}
+/**
+ * Measures how well a judge agrees with people on the same cases: accuracy,
+ * per-label recall, the confusion matrix, and the rows that disagree. With
+ * `positive`, also the TPR and TNR, the recommended view for binary judges
+ * (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
+ * Pure.
+ *
+ * Build rows from confirmed labels only; draft labels gate nothing. Group by
+ * decision state or request name so each narrow judge is aligned on its own.
+ * A row with no group key counts in the totals and in no group. Run it on a
+ * dev split and report the held-out split separately.
+ *
+ * Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
+ *
+ * @example
+ * ```ts no-check
+ * const report = judgeAgreement(
+ *   labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
+ *   { positive: 'fail' },
+ * );
+ * report.binary; // { tpr: 0.9, tnr: 0.8 }
+ * report.correct?.(0.3); // estimated true failure rate in production
+ * ```
+ */
+export declare function judgeAgreement<TRow extends AgreementRow>(rows: readonly TRow[], options?: JudgeAgreementOptions<TRow>): AgreementReport;
+//#endregion
 //#region src/compare-runs.d.ts
 /** One run of one scenario, reduced to what parity compares. See {@link compareRuns}. */
 interface ParityRun {
@@ -438,4 +822,4 @@ export declare function compareRuns<TScenario extends {
   expected?: string;
 }>(options: CompareRunsOptions<TScenario>): Promise<ParityReport<TScenario>>;
 //#endregion
-export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRunsOptions, type ExplorePathsOptions, type ExploreSnapshot, type FoldTracesOptions, type GetDecisionRequestsOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateStats, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };
+export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AgreementReport, type AgreementRow, type AgreementSummary, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRunsOptions, type CoverageReport, type Divergence, type ExplorePathsOptions, type ExploreSnapshot, type FirstDivergenceOptions, type FoldTracesOptions, type GetDecisionRequestsOptions, type HandlerCoverage, type HandlerRef, type JudgeAgreementOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RepeatGroup, type RepeatReport, type RepeatResult, type RepeatRun, type RepeatRunsOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateCoverage, type StateStats, type TraceCoverageOptions, type TraceStep, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };