@statelyai/agent 2.0.0-alpha.29 → 2.0.0-alpha.31

package/dist/testing.cjs

@@ -1,8 +1,7 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_event_log = require("./event-log-Dtb16p1p.cjs");
-const require_decision = require("./decision-oevZnkBe.cjs");
-const require_run_agent = require("./run-agent-cToZ3K42.cjs");
-const require_verify = require("./verify-Cqghu4gz.cjs");
+const require_event_log = require("./event-log-Bdww2l1D.cjs");
+const require_run_agent = require("./run-agent-tmk58M0x.cjs");
+const require_verify = require("./verify-CS_xxvVU.cjs");
 //#region src/trajectory.ts
 /**
 * Classifies one item. Event-shaped things (log entries, events) become events;
@@ -10,9 +9,9 @@ const require_verify = require("./verify-Cqghu4gz.cjs");
 * @internal
 */
 function normalize(item) {
-	if (require_decision.isRecord(item)) {
+	if (require_event_log.isRecord(item)) {
 		const nested = item["event"];
-		if (require_decision.isRecord(nested) && typeof nested["type"] === "string") return {
+		if (require_event_log.isRecord(nested) && typeof nested["type"] === "string") return {
 			kind: "event",
 			event: nested
 		};
@@ -33,7 +32,7 @@ function normalize(item) {
 /** Every leaf of a state value as a dot path (`{ review: 'editing' }` → `review.editing`). @internal */
 function statePaths$1(value) {
 	if (typeof value === "string") return [value];
-	if (!require_decision.isRecord(value)) return [];
+	if (!require_event_log.isRecord(value)) return [];
 	return Object.entries(value).flatMap(([key, child]) => statePaths$1(child).map((path) => `${key}.${path}`));
 }
 /**
@@ -46,7 +45,7 @@ function stateMatchesPath(path, value) {
 function deepEqual(a, b) {
 	if (Object.is(a, b)) return true;
 	if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => deepEqual(item, b[index]));
-	if (!require_decision.isRecord(a) || !require_decision.isRecord(b)) return false;
+	if (!require_event_log.isRecord(a) || !require_event_log.isRecord(b)) return false;
 	const keys = Object.keys(a);
 	return keys.length === Object.keys(b).length && keys.every((key) => deepEqual(a[key], b[key]));
 }
@@ -178,7 +177,7 @@ function isTextResult(value) {
 */
 async function resolveSeamScriptEntry(entry, request, info) {
 	const value = typeof entry === "function" ? await entry(request, info) : entry;
-	return require_decision.isRecord(value) && isTextResult(value) ? value : { result: value };
+	return require_event_log.isRecord(value) && isTextResult(value) ? value : { result: value };
 }
 /** Names a pending text request in an error message. @internal */
 function describeText(request) {
@@ -188,7 +187,7 @@ function describeText(request) {
 * Stream semantics with no model: the whole text lands as one chunk. @internal
 */
 function emitScriptedChunk(result, info) {
-	const text = require_decision.isRecord(result) ? result["result"] : void 0;
+	const text = require_event_log.isRecord(result) ? result["result"] : void 0;
 	if (typeof text === "string") info?.onChunk?.(text);
 }
 /**
@@ -196,7 +195,7 @@ function emitScriptedChunk(result, info) {
 * @internal
 */
 async function seamOutputOf(result, request) {
-	if (!require_decision.isRecord(result)) return;
+	if (!require_event_log.isRecord(result)) return;
 	return await require_event_log.normalizeGeneratorResult(result, `seam '${request.name ?? request.model}'`);
 }
 /**
@@ -321,7 +320,7 @@ async function runSeam(machine, options) {
 			...event ? { event } : {}
 		});
 		if (result.status !== "idle" || turn === maxTurns) break;
-		const next = options.respond?.({
+		const next = await options.respond?.({
 			snapshot: result.snapshot,
 			state: result.snapshot.value,
 			meta: require_event_log.getStateMeta(result.snapshot),
@@ -361,13 +360,36 @@ async function runSeam(machine, options) {
 	};
 }
 //#endregion
+//#region src/internal/trace-states.ts
+/** The `value` of a trace event's `from` or `snapshot`. @internal */
+function valueOf(holder) {
+	return require_event_log.isRecord(holder) ? holder.value : void 0;
+}
+/** A decision request on its first attempt (no failed attempts yet). @internal */
+function isFirstDecision(request) {
+	return require_event_log.isRecord(request) && request.kind === "decision" && (!Array.isArray(request.attempts) || request.attempts.length === 0);
+}
+/**
+* Every active leaf of a state value as a dot path:
+* `'a'` → `['a']`; `{ a: 'b' }` → `['a.b']`; `{ p: { x: 'one', y: {} } }` → `['p.x.one', 'p.y']`.
+* @internal
+*/
+function statePaths(value, prefix = "") {
+	if (typeof value === "string") return [prefix + value];
+	if (!require_event_log.isRecord(value)) return [];
+	const entries = Object.entries(value);
+	if (entries.length === 0) return prefix ? [prefix.slice(0, -1)] : [];
+	return entries.flatMap(([key, child]) => statePaths(child, `${prefix}${key}.`));
+}
+//#endregion
 //#region src/fold-traces.ts
 /**
 * Folds many runs' traces into per-state stats: how many runs visited each
-* state, how many of those failed, and how many options the guards withheld
-* at decisions made there. Pure; reads only:
+* state, how many of those failed, how many options the guards withheld
+* at decisions made there, and how many events it ignored. Pure; reads only:
 *
-* - `machine.transition`: the run visited `from.value` and `snapshot.value`.
+* - `machine.transition`: the run visited `from.value` and `snapshot.value`;
+*   with `ignored`, each of those states ignored one event.
 * - `request.start` of a decision, first attempt only: its `request.withheld`
 *   counts toward the state the run was in, the `from.value` of the
 *   transition its choice causes.
@@ -393,7 +415,8 @@ function foldTraces(traces, options) {
 				state,
 				visits: 0,
 				failures: 0,
-				withheld: 0
+				withheld: 0,
+				ignored: 0
 			};
 			stats.set(state, entry);
 		}
@@ -410,6 +433,7 @@ function foldTraces(traces, options) {
 			for (const state of statePaths(valueOf(event.from))) visit(state);
 			current = statePaths(valueOf(event.snapshot));
 			for (const state of current) visit(state);
+			if (event.ignored === true) for (const state of current) statsFor(state).ignored++;
 		} else if (event.type === "request.start" && isFirstDecision(event.request)) {
 			const withheld = event.request.withheld;
 			if (!Array.isArray(withheld)) continue;
@@ -424,26 +448,152 @@ function foldTraces(traces, options) {
 	}
 	return [...stats.values()];
 }
-function valueOf(holder) {
-	return require_decision.isRecord(holder) ? holder.value : void 0;
+//#endregion
+//#region src/decision-requests.ts
+/**
+* The decisions a run asked for, read from its trace: each decision's first
+* `request.start`, with the states the run was in. A trace is one run's events,
+* live or serialized with `serializeTraceEvent`. Retries of the same decision
+* are left out, so each decision appears once.
+*
+* Use the requests as eval rows (run a decider on each and compare its event
+* with a label) or as probe inputs for a policy. A serialized request keeps its
+* `options`, `metadata`, `system`, and `prompt`; event schemas do not survive
+* serialization, so replay closed choices (`options`) rather than open ones.
+*
+* @example
+* ```ts no-check
+* const rows = traces.flatMap((trace) => getDecisionRequests(trace, { name: "chooseResolution" }));
+* for (const { request } of rows) console.log((await decide(request)).event);
+* ```
+*/
+function getDecisionRequests(trace, options = {}) {
+	const requests = [];
+	let current = [];
+	for (const event of trace) if (event.type === "machine.transition") current = statePaths(valueOf(event.snapshot));
+	else if (event.type === "request.start" && isFirstDecision(event.request)) {
+		if (options.name !== void 0 && event.request.name !== options.name) continue;
+		requests.push({
+			states: [...current],
+			request: event.request
+		});
+	}
+	return requests;
 }
-function isFirstDecision(request) {
-	return require_decision.isRecord(request) && request.kind === "decision" && (!Array.isArray(request.attempts) || request.attempts.length === 0);
+//#endregion
+//#region src/compare-runs.ts
+/**
+* Runs every scenario through a baseline and a candidate implementation and
+* labels each difference, so a port can show where it behaves like the code it
+* replaces and where it differs on purpose.
+*
+* Parity is outcome-level: the terminal outcome and the effects in the world,
+* not the sequence of tool calls, since a machine usually reorders steps. A
+* candidate that broke no rule where the baseline broke one is intended, even
+* when the outcome matches. A difference also counts as intended when only the
+* candidate reached the scenario's `expected` outcome. Every other difference,
+* and every candidate rule break, is a regression for a person to review.
+*
+* Runs are sequential, baseline first. A run that throws is recorded with
+* outcome `'threw'` and the error message as its `detail`. Parity holds for
+* these scenarios and these runs only; with a live model, run it more than once.
+*
+* @example
+* ```ts no-check
+* const report = await compareRuns({
+*   scenarios,
+*   baseline: async (s) => toParityRun(await runRawLoop(s)),
+*   candidate: async (s) => toParityRun(await runDesk(s)),
+* });
+* expect(report.ok).toBe(true);
+* ```
+*/
+async function compareRuns(options) {
+	const scenarios = [];
+	const counts = {
+		same: 0,
+		intended: 0,
+		regression: 0
+	};
+	for (const scenario of options.scenarios) {
+		const baseline = await settle(() => options.baseline(scenario));
+		const candidate = await settle(() => options.candidate(scenario));
+		const { label, reason } = labelRun(scenario.expected, baseline, candidate);
+		counts[label]++;
+		scenarios.push({
+			id: scenario.id,
+			scenario,
+			label,
+			reason,
+			baseline,
+			candidate
+		});
+	}
+	return {
+		scenarios,
+		counts,
+		ok: counts.regression === 0
+	};
 }
-function statePaths(value, prefix = "") {
-	if (typeof value === "string") return [prefix + value];
-	if (!require_decision.isRecord(value)) return [];
-	const entries = Object.entries(value);
-	if (entries.length === 0) return prefix ? [prefix.slice(0, -1)] : [];
-	return entries.flatMap(([key, child]) => statePaths(child, `${prefix}${key}.`));
+async function settle(run) {
+	try {
+		return await run();
+	} catch (error) {
+		return {
+			outcome: "threw",
+			detail: error instanceof Error ? error.message : String(error)
+		};
+	}
+}
+function labelRun(expected, baseline, candidate) {
+	if (candidate.violations?.length) return {
+		label: "regression",
+		reason: `candidate broke a rule: ${list(candidate.violations)}`
+	};
+	if (candidate.outcome === "threw") return {
+		label: "regression",
+		reason: `candidate threw: ${String(candidate.detail)}`
+	};
+	if (baseline.violations?.length) return {
+		label: "intended",
+		reason: `baseline broke a rule: ${list(baseline.violations)}`
+	};
+	const sameOutcome = baseline.outcome === candidate.outcome;
+	if (sameOutcome && structurallyEqual(baseline.effects, candidate.effects)) return {
+		label: "same",
+		reason: ""
+	};
+	if (expected !== void 0 && candidate.outcome === expected && baseline.outcome !== expected) return {
+		label: "intended",
+		reason: `only the candidate reached the expected '${expected}'`
+	};
+	return {
+		label: "regression",
+		reason: sameOutcome ? "effects differ" : `outcome differs: ${baseline.outcome} → ${candidate.outcome}`
+	};
+}
+function list(items) {
+	return items.map((item) => typeof item === "string" ? item : JSON.stringify(item)).join(", ");
+}
+function structurallyEqual(a, b) {
+	if (Object.is(a, b)) return true;
+	if (a instanceof Date || b instanceof Date) return a instanceof Date && b instanceof Date && a.getTime() === b.getTime();
+	if (a instanceof Map || b instanceof Map) return a instanceof Map && b instanceof Map && structurallyEqual([...a], [...b]);
+	if (a instanceof Set || b instanceof Set) return a instanceof Set && b instanceof Set && structurallyEqual([...a], [...b]);
+	if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => structurallyEqual(item, b[index]));
+	if (!require_event_log.isRecord(a) || !require_event_log.isRecord(b)) return false;
+	const keys = Object.keys(a);
+	return keys.length === Object.keys(b).length && keys.every((key) => Object.hasOwn(b, key) && structurallyEqual(a[key], b[key]));
 }
 //#endregion
 exports.AgentLintError = require_verify.AgentLintError;
 exports.AgentUnknownStateError = require_verify.AgentUnknownStateError;
 exports.assertAgentMachine = require_verify.assertAgentMachine;
 exports.canReach = require_verify.canReach;
+exports.compareRuns = compareRuns;
 exports.explorePaths = require_verify.explorePaths;
 exports.foldTraces = foldTraces;
+exports.getDecisionRequests = getDecisionRequests;
 exports.lintAgentMachine = require_verify.lintAgentMachine;
 exports.matchesTrajectory = matchesTrajectory;
 exports.runSeam = runSeam;

package/dist/testing.d.cts

@@ -1,7 +1,7 @@
-import { c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-9fv1i9aK.cjs";
-import { f as AgentRuntimeOptions, l as AgentRunResult, o as AgentRunInit } from "./run-agent-C9ubX8Tx.cjs";
+import { P as AgentExecutorDecisionRequest, c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-j9hShB7d.cjs";
+import { f as AgentRuntimeOptions, l as AgentRunResult, o as AgentRunInit } from "./run-agent-B5opVUxx.cjs";
 import { t as MetaOfSnapshot } from "./utils-ebVpCYoF.cjs";
-import { _ as explorePaths, a as AgentPathTerminal, c as CanReachResult, d as SimulateAgentOptions, f as SimulateAgentResult, g as canReach, h as assertAgentMachine, i as AgentPathReport, l as ExplorePathsOptions, m as SimulationTrailEntry, n as AgentLintError, o as AgentUnknownStateError, p as SimulationScript, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as LintAgentMachineOptions, v as lintAgentMachine, y as simulateAgent } from "./verify-DYH_2VzU.cjs";
+import { _ as assertAgentMachine, a as AgentPathTerminal, b as lintAgentMachine, c as CanReachResult, d as ExploreSnapshot, f as LintAgentMachineOptions, g as SimulationTrailEntry, h as SimulationScript, i as AgentPathReport, l as CanReachTarget, m as SimulateAgentResult, n as AgentLintError, o as AgentUnknownStateError, p as SimulateAgentOptions, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as ExplorePathsOptions, v as canReach, x as simulateAgent, y as explorePaths } from "./verify-CPKDPMwY.cjs";
 import { AnyStateMachine, EventFromLogic, InputFrom, SnapshotFrom, StateValue } from "xstate";
 //#region src/trajectory.d.ts
 /** An event-shaped trajectory item: anything with a string `type`. */
@@ -204,10 +204,10 @@ interface RunSeamOptions<TMachine extends AnyStateMachine> {
   candidate?: AgentRequestExecutor;
   /**
    * The simulated user. Called at every idle pause; return the event to send,
-   * or `null`/`undefined` to stop the run there. Omitted, the run stops at the
-   * first idle pause.
+   * or `null`/`undefined` to stop the run there, directly or as a promise (a
+   * real async client). Omitted, the run stops at the first idle pause.
    */
-  respond?: (turn: SeamTurn<TMachine>) => EventFromLogic<TMachine> | null | undefined;
+  respond?: (turn: SeamTurn<TMachine>) => EventFromLogic<TMachine> | null | undefined | PromiseLike<EventFromLogic<TMachine> | null | undefined>;
   /** Maximum idle pauses to answer before stopping. Default `12`. */
   maxTurns?: number;
   /**
@@ -288,6 +288,8 @@ interface StateStats {
   failures: number;
   /** Options withheld by the guards at decisions made in this state, summed over runs. */
   withheld: number;
+  /** Events this state ignored (`machine.transition` with `ignored`), summed over runs. */
+  ignored: number;
 }
 /** Options for {@link foldTraces}. */
 interface FoldTracesOptions<TTrace> {
@@ -296,10 +298,11 @@ interface FoldTracesOptions<TTrace> {
 }
 /**
  * Folds many runs' traces into per-state stats: how many runs visited each
- * state, how many of those failed, and how many options the guards withheld
- * at decisions made there. Pure; reads only:
+ * state, how many of those failed, how many options the guards withheld
+ * at decisions made there, and how many events it ignored. Pure; reads only:
  *
- * - `machine.transition`: the run visited `from.value` and `snapshot.value`.
+ * - `machine.transition`: the run visited `from.value` and `snapshot.value`;
+ *   with `ignored`, each of those states ignored one event.
  * - `request.start` of a decision, first attempt only: its `request.withheld`
  *   counts toward the state the run was in, the `from.value` of the
  *   transition its choice causes.
@@ -320,4 +323,119 @@ export declare function foldTraces<TTrace extends readonly {
   readonly type: string;
 }[]>(traces: Iterable<TTrace>, options: FoldTracesOptions<TTrace>): StateStats[];
 //#endregion
-export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AssertAgentMachineOptions, type CanReachResult, type ExplorePathsOptions, type FoldTracesOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateStats, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };
+//#region src/decision-requests.d.ts
+/** One decision a run asked for. See {@link getDecisionRequests}. */
+interface TracedDecisionRequest {
+  /** The active states when the decision was asked, as dot paths (one per parallel region). */
+  states: string[];
+  /** The request as the executor received it, on its first attempt. */
+  request: AgentExecutorDecisionRequest;
+}
+/** Options for {@link getDecisionRequests}. */
+interface GetDecisionRequestsOptions {
+  /** Keep only requests with this `name` (the `agent.decide` input's `name`). */
+  name?: string;
+}
+/**
+ * The decisions a run asked for, read from its trace: each decision's first
+ * `request.start`, with the states the run was in. A trace is one run's events,
+ * live or serialized with `serializeTraceEvent`. Retries of the same decision
+ * are left out, so each decision appears once.
+ *
+ * Use the requests as eval rows (run a decider on each and compare its event
+ * with a label) or as probe inputs for a policy. A serialized request keeps its
+ * `options`, `metadata`, `system`, and `prompt`; event schemas do not survive
+ * serialization, so replay closed choices (`options`) rather than open ones.
+ *
+ * @example
+ * ```ts no-check
+ * const rows = traces.flatMap((trace) => getDecisionRequests(trace, { name: "chooseResolution" }));
+ * for (const { request } of rows) console.log((await decide(request)).event);
+ * ```
+ */
+export declare function getDecisionRequests(trace: Iterable<{
+  readonly type: string;
+}>, options?: GetDecisionRequestsOptions): TracedDecisionRequest[];
+//#endregion
+//#region src/compare-runs.d.ts
+/** One run of one scenario, reduced to what parity compares. See {@link compareRuns}. */
+interface ParityRun {
+  /** How the run ended: a final state name, or the label of the loop's return. `'threw'` marks a run that threw. */
+  outcome: string;
+  /** What the run changed in the world, compared structurally. Sort lists whose order does not matter. */
+  effects?: unknown;
+  /** The rules the run broke, from ground truth. Any entry marks the run as rule-breaking. */
+  violations?: readonly unknown[];
+  /** Anything else to keep in the report. Not compared. */
+  detail?: unknown;
+}
+/**
+ * How one scenario's candidate run relates to its baseline run.
+ *
+ * - `same`: same outcome, same effects, and neither run broke a rule.
+ * - `intended`: the candidate broke no rule, and the baseline broke one (even
+ *   with the same outcome and effects), or the baseline missed the scenario's
+ *   `expected` outcome that the candidate reached.
+ * - `regression`: the candidate broke a rule, or they differ for any other reason.
+ */
+type ParityLabel = "same" | "intended" | "regression";
+/** One scenario's row in a {@link ParityReport}. */
+interface ParityResult<TScenario> {
+  id: string;
+  scenario: TScenario;
+  label: ParityLabel;
+  /** Why the label was given, in a few words. Empty for `same`. */
+  reason: string;
+  baseline: ParityRun;
+  candidate: ParityRun;
+}
+/** The result of {@link compareRuns}. */
+interface ParityReport<TScenario> {
+  scenarios: ParityResult<TScenario>[];
+  counts: Record<ParityLabel, number>;
+  /** True when no scenario is a `regression`. */
+  ok: boolean;
+}
+/** Options for {@link compareRuns}. */
+interface CompareRunsOptions<TScenario extends {
+  id: string;
+  expected?: string;
+}> {
+  scenarios: readonly TScenario[];
+  /** Runs the existing implementation (the loop) on one scenario. */
+  baseline: (scenario: TScenario) => Promise<ParityRun>;
+  /** Runs the replacement (the machine) on one scenario. */
+  candidate: (scenario: TScenario) => Promise<ParityRun>;
+}
+/**
+ * Runs every scenario through a baseline and a candidate implementation and
+ * labels each difference, so a port can show where it behaves like the code it
+ * replaces and where it differs on purpose.
+ *
+ * Parity is outcome-level: the terminal outcome and the effects in the world,
+ * not the sequence of tool calls, since a machine usually reorders steps. A
+ * candidate that broke no rule where the baseline broke one is intended, even
+ * when the outcome matches. A difference also counts as intended when only the
+ * candidate reached the scenario's `expected` outcome. Every other difference,
+ * and every candidate rule break, is a regression for a person to review.
+ *
+ * Runs are sequential, baseline first. A run that throws is recorded with
+ * outcome `'threw'` and the error message as its `detail`. Parity holds for
+ * these scenarios and these runs only; with a live model, run it more than once.
+ *
+ * @example
+ * ```ts no-check
+ * const report = await compareRuns({
+ *   scenarios,
+ *   baseline: async (s) => toParityRun(await runRawLoop(s)),
+ *   candidate: async (s) => toParityRun(await runDesk(s)),
+ * });
+ * expect(report.ok).toBe(true);
+ * ```
+ */
+export declare function compareRuns<TScenario extends {
+  id: string;
+  expected?: string;
+}>(options: CompareRunsOptions<TScenario>): Promise<ParityReport<TScenario>>;
+//#endregion
+export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRunsOptions, type ExplorePathsOptions, type ExploreSnapshot, type FoldTracesOptions, type GetDecisionRequestsOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateStats, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };

package/dist/testing.d.mts

@@ -1,7 +1,7 @@
-import { c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-BDbGjYMh.mjs";
-import { f as AgentRuntimeOptions, l as AgentRunResult, o as AgentRunInit } from "./run-agent-CxSyZmIE.mjs";
+import { P as AgentExecutorDecisionRequest, c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-BwdrDpO9.mjs";
+import { f as AgentRuntimeOptions, l as AgentRunResult, o as AgentRunInit } from "./run-agent-D4MwS5GY.mjs";
 import { t as MetaOfSnapshot } from "./utils-BnXdWih3.mjs";
-import { _ as explorePaths, a as AgentPathTerminal, c as CanReachResult, d as SimulateAgentOptions, f as SimulateAgentResult, g as canReach, h as assertAgentMachine, i as AgentPathReport, l as ExplorePathsOptions, m as SimulationTrailEntry, n as AgentLintError, o as AgentUnknownStateError, p as SimulationScript, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as LintAgentMachineOptions, v as lintAgentMachine, y as simulateAgent } from "./verify-CBuYIpsa.mjs";
+import { _ as assertAgentMachine, a as AgentPathTerminal, b as lintAgentMachine, c as CanReachResult, d as ExploreSnapshot, f as LintAgentMachineOptions, g as SimulationTrailEntry, h as SimulationScript, i as AgentPathReport, l as CanReachTarget, m as SimulateAgentResult, n as AgentLintError, o as AgentUnknownStateError, p as SimulateAgentOptions, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as ExplorePathsOptions, v as canReach, x as simulateAgent, y as explorePaths } from "./verify-tkymWz6X.mjs";
 import { AnyStateMachine, EventFromLogic, InputFrom, SnapshotFrom, StateValue } from "xstate";
 //#region src/trajectory.d.ts
 /** An event-shaped trajectory item: anything with a string `type`. */
@@ -204,10 +204,10 @@ interface RunSeamOptions<TMachine extends AnyStateMachine> {
   candidate?: AgentRequestExecutor;
   /**
    * The simulated user. Called at every idle pause; return the event to send,
-   * or `null`/`undefined` to stop the run there. Omitted, the run stops at the
-   * first idle pause.
+   * or `null`/`undefined` to stop the run there, directly or as a promise (a
+   * real async client). Omitted, the run stops at the first idle pause.
    */
-  respond?: (turn: SeamTurn<TMachine>) => EventFromLogic<TMachine> | null | undefined;
+  respond?: (turn: SeamTurn<TMachine>) => EventFromLogic<TMachine> | null | undefined | PromiseLike<EventFromLogic<TMachine> | null | undefined>;
   /** Maximum idle pauses to answer before stopping. Default `12`. */
   maxTurns?: number;
   /**
@@ -288,6 +288,8 @@ interface StateStats {
   failures: number;
   /** Options withheld by the guards at decisions made in this state, summed over runs. */
   withheld: number;
+  /** Events this state ignored (`machine.transition` with `ignored`), summed over runs. */
+  ignored: number;
 }
 /** Options for {@link foldTraces}. */
 interface FoldTracesOptions<TTrace> {
@@ -296,10 +298,11 @@ interface FoldTracesOptions<TTrace> {
 }
 /**
  * Folds many runs' traces into per-state stats: how many runs visited each
- * state, how many of those failed, and how many options the guards withheld
- * at decisions made there. Pure; reads only:
+ * state, how many of those failed, how many options the guards withheld
+ * at decisions made there, and how many events it ignored. Pure; reads only:
  *
- * - `machine.transition`: the run visited `from.value` and `snapshot.value`.
+ * - `machine.transition`: the run visited `from.value` and `snapshot.value`;
+ *   with `ignored`, each of those states ignored one event.
  * - `request.start` of a decision, first attempt only: its `request.withheld`
  *   counts toward the state the run was in, the `from.value` of the
  *   transition its choice causes.
@@ -320,4 +323,119 @@ export declare function foldTraces<TTrace extends readonly {
   readonly type: string;
 }[]>(traces: Iterable<TTrace>, options: FoldTracesOptions<TTrace>): StateStats[];
 //#endregion
-export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AssertAgentMachineOptions, type CanReachResult, type ExplorePathsOptions, type FoldTracesOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateStats, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };
+//#region src/decision-requests.d.ts
+/** One decision a run asked for. See {@link getDecisionRequests}. */
+interface TracedDecisionRequest {
+  /** The active states when the decision was asked, as dot paths (one per parallel region). */
+  states: string[];
+  /** The request as the executor received it, on its first attempt. */
+  request: AgentExecutorDecisionRequest;
+}
+/** Options for {@link getDecisionRequests}. */
+interface GetDecisionRequestsOptions {
+  /** Keep only requests with this `name` (the `agent.decide` input's `name`). */
+  name?: string;
+}
+/**
+ * The decisions a run asked for, read from its trace: each decision's first
+ * `request.start`, with the states the run was in. A trace is one run's events,
+ * live or serialized with `serializeTraceEvent`. Retries of the same decision
+ * are left out, so each decision appears once.
+ *
+ * Use the requests as eval rows (run a decider on each and compare its event
+ * with a label) or as probe inputs for a policy. A serialized request keeps its
+ * `options`, `metadata`, `system`, and `prompt`; event schemas do not survive
+ * serialization, so replay closed choices (`options`) rather than open ones.
+ *
+ * @example
+ * ```ts no-check
+ * const rows = traces.flatMap((trace) => getDecisionRequests(trace, { name: "chooseResolution" }));
+ * for (const { request } of rows) console.log((await decide(request)).event);
+ * ```
+ */
+export declare function getDecisionRequests(trace: Iterable<{
+  readonly type: string;
+}>, options?: GetDecisionRequestsOptions): TracedDecisionRequest[];
+//#endregion
+//#region src/compare-runs.d.ts
+/** One run of one scenario, reduced to what parity compares. See {@link compareRuns}. */
+interface ParityRun {
+  /** How the run ended: a final state name, or the label of the loop's return. `'threw'` marks a run that threw. */
+  outcome: string;
+  /** What the run changed in the world, compared structurally. Sort lists whose order does not matter. */
+  effects?: unknown;
+  /** The rules the run broke, from ground truth. Any entry marks the run as rule-breaking. */
+  violations?: readonly unknown[];
+  /** Anything else to keep in the report. Not compared. */
+  detail?: unknown;
+}
+/**
+ * How one scenario's candidate run relates to its baseline run.
+ *
+ * - `same`: same outcome, same effects, and neither run broke a rule.
+ * - `intended`: the candidate broke no rule, and the baseline broke one (even
+ *   with the same outcome and effects), or the baseline missed the scenario's
+ *   `expected` outcome that the candidate reached.
+ * - `regression`: the candidate broke a rule, or they differ for any other reason.
+ */
+type ParityLabel = "same" | "intended" | "regression";
+/** One scenario's row in a {@link ParityReport}. */
+interface ParityResult<TScenario> {
+  id: string;
+  scenario: TScenario;
+  label: ParityLabel;
+  /** Why the label was given, in a few words. Empty for `same`. */
+  reason: string;
+  baseline: ParityRun;
+  candidate: ParityRun;
+}
+/** The result of {@link compareRuns}. */
+interface ParityReport<TScenario> {
+  scenarios: ParityResult<TScenario>[];
+  counts: Record<ParityLabel, number>;
+  /** True when no scenario is a `regression`. */
+  ok: boolean;
+}
+/** Options for {@link compareRuns}. */
+interface CompareRunsOptions<TScenario extends {
+  id: string;
+  expected?: string;
+}> {
+  scenarios: readonly TScenario[];
+  /** Runs the existing implementation (the loop) on one scenario. */
+  baseline: (scenario: TScenario) => Promise<ParityRun>;
+  /** Runs the replacement (the machine) on one scenario. */
+  candidate: (scenario: TScenario) => Promise<ParityRun>;
+}
+/**
+ * Runs every scenario through a baseline and a candidate implementation and
+ * labels each difference, so a port can show where it behaves like the code it
+ * replaces and where it differs on purpose.
+ *
+ * Parity is outcome-level: the terminal outcome and the effects in the world,
+ * not the sequence of tool calls, since a machine usually reorders steps. A
+ * candidate that broke no rule where the baseline broke one is intended, even
+ * when the outcome matches. A difference also counts as intended when only the
+ * candidate reached the scenario's `expected` outcome. Every other difference,
+ * and every candidate rule break, is a regression for a person to review.
+ *
+ * Runs are sequential, baseline first. A run that throws is recorded with
+ * outcome `'threw'` and the error message as its `detail`. Parity holds for
+ * these scenarios and these runs only; with a live model, run it more than once.
+ *
+ * @example
+ * ```ts no-check
+ * const report = await compareRuns({
+ *   scenarios,
+ *   baseline: async (s) => toParityRun(await runRawLoop(s)),
+ *   candidate: async (s) => toParityRun(await runDesk(s)),
+ * });
+ * expect(report.ok).toBe(true);
+ * ```
+ */
+export declare function compareRuns<TScenario extends {
+  id: string;
+  expected?: string;
+}>(options: CompareRunsOptions<TScenario>): Promise<ParityReport<TScenario>>;
+//#endregion
+export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRunsOptions, type ExplorePathsOptions, type ExploreSnapshot, type FoldTracesOptions, type GetDecisionRequestsOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateStats, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };