@statelyai/agent 2.0.0-alpha.32 → 2.0.0-alpha.33

package/dist/testing.mjs

@@ -1,6 +1,6 @@
 import { $ as getStateMeta, F as normalizeGeneratorResult, N as getCallUsage, lt as isRecord, ut as AgentError } from "./event-log-DAqsalT1.mjs";
 import { p as runToQuiescence, s as createAgentRuntime } from "./run-agent-2UnSUhDb.mjs";
-import { a as explorePaths, i as canReach, n as AgentUnknownStateError, o as lintAgentMachine, r as assertAgentMachine, s as simulateAgent, t as AgentLintError } from "./verify-CoCvR1eM.mjs";
+import { a as explorePaths, c as simulateAgent, i as canReach, n as AgentUnknownStateError, o as isExplorationComplete, r as assertAgentMachine, s as lintAgentMachine, t as AgentLintError } from "./verify-DgB8P-Wx.mjs";
 //#region src/trajectory.ts
 /**
 * Classifies one item. Event-shaped things (log entries, events) become events;
@@ -380,12 +380,47 @@ function statePaths(value, prefix = "") {
 	if (entries.length === 0) return prefix ? [prefix.slice(0, -1)] : [];
 	return entries.flatMap(([key, child]) => statePaths(child, `${prefix}${key}.`));
 }
+/**
+* A trace's `machine.transition` events as steps, skipping `@xstate.init`.
+* A first-attempt decision's `request.start` names the decision on the step
+* that follows it. Reads live and serialized traces alike.
+* @internal
+*/
+function traceSteps(trace) {
+	const steps = [];
+	let decision;
+	for (const event of trace) {
+		if (event.type === "request.start" && isFirstDecision(event.request)) {
+			const name = event.request.name;
+			decision = typeof name === "string" ? name : void 0;
+			continue;
+		}
+		if (event.type !== "machine.transition") continue;
+		const type = isRecord(event.event) ? event.event.type : void 0;
+		if (type === "@xstate.init") continue;
+		const fromValue = valueOf(event.from);
+		const toValue = valueOf(event.snapshot);
+		steps.push({
+			from: statePaths(fromValue),
+			fromValue,
+			event: typeof type === "string" ? type : "",
+			payload: event.event,
+			to: statePaths(toValue),
+			toValue,
+			ignored: event.ignored === true,
+			...decision === void 0 ? {} : { decision }
+		});
+		decision = void 0;
+	}
+	return steps;
+}
 //#endregion
 //#region src/fold-traces.ts
 /**
 * Folds many runs' traces into per-state stats: how many runs visited each
 * state, how many of those failed, how many options the guards withheld
-* at decisions made there, and how many events it ignored. Pure; reads only:
+* at decisions made there, how many events it ignored, and how many failures
+* `blame` traced back to it. Pure; reads only:
 *
 * - `machine.transition`: the run visited `from.value` and `snapshot.value`;
 *   with `ignored`, each of those states ignored one event.
@@ -398,6 +433,10 @@ function statePaths(value, prefix = "") {
 * `{ type: 'request.start', request: result.request }` onto the trace before
 * sending its event. Parallel states count once per active leaf.
 *
+* `failures` counts a failed run against every state it visited, so states
+* after a bad decision rank as high as the decision. Sort by `blamed` to see
+* where failures start.
+*
 * @example
 * ```ts no-check
 * const stats = foldTraces(traces, {
@@ -415,7 +454,8 @@ function foldTraces(traces, options) {
 				visits: 0,
 				failures: 0,
 				withheld: 0,
-				ignored: 0
+				ignored: 0,
+				blamed: 0
 			};
 			stats.set(state, entry);
 		}
@@ -444,10 +484,249 @@ function foldTraces(traces, options) {
 			entry.visits++;
 			if (failed) entry.failures++;
 		}
+		const blamed = failed ? options.blame?.(trace) : void 0;
+		if (blamed !== void 0) statsFor(blamed).blamed++;
 	}
 	return [...stats.values()];
 }
 //#endregion
+//#region src/trace-coverage.ts
+/** Mirrors xstate@6.0.0-alpha.69 `matchesEventDescriptor` (src/utils.ts), which the package does not export. */
+function matchesDescriptor(eventType, descriptor) {
+	if (descriptor === eventType || descriptor === "*") return true;
+	if (!descriptor.endsWith(".*")) return false;
+	const tokens = descriptor.split(".");
+	const eventTokens = eventType.split(".");
+	for (let index = 0; index < tokens.length; index++) {
+		if (tokens[index] === "*") return index === tokens.length - 1;
+		if (tokens[index] !== eventTokens[index]) return false;
+	}
+	return true;
+}
+/** Whether a definition's `matches` pattern (e.g. an invoke's `actorId`) accepts the traced event. */
+function matchesPattern(definition, event) {
+	const pattern = definition.matches;
+	if (!isRecord(pattern) || !isRecord(event)) return true;
+	return Object.entries(pattern).every(([key, value]) => Object.is(event[key], value));
+}
+/**
+* The descriptor XState would try first at `node` for `event`, guards aside:
+* the exact key, then partial and wildcard keys longest first, as in
+* xstate@6.0.0-alpha.69 `getCandidates` (src/stateUtils.ts).
+*/
+function candidateAt(node, type, event) {
+	const keys = [...node.transitions.keys()].filter((key) => key !== type && matchesDescriptor(type, key)).sort((a, b) => b.length - a.length);
+	if (node.transitions.has(type)) keys.unshift(type);
+	return keys.find((key) => node.transitions.get(key).some((d) => matchesPattern(d, event)));
+}
+const pathOf = (node) => node.path.join(".");
+const ratioOf = (visited, total) => total === 0 ? 1 : visited / total;
+/**
+* Measures how much of a machine a set of runs exercised: which atomic and
+* final states they visited, and which handlers they took. A handler is a
+* `(declaring state, event descriptor)` pair from `on`, invoke
+* `onDone`/`onError`, and `after`, with the targets it declares and the
+* leaves the runs landed in after it. Pure; reads only `machine.transition`
+* events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
+*
+* A step credits one handler per active leaf: the nearest node, from the leaf
+* up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
+* event, preferred in XState's order. Guards are not replayed, so a step
+* whose nearest handler was rejected and fell through to an ancestor is
+* credited to the nearest one. XState keys invoke and `after` handlers by
+* bare type (`xstate.done.actor`), one per state however many invokes; a
+* definition's `matches` (an invoke's `actorId`) must accept the traced event
+* for its state to be credited. Ignored steps credit no handler. `always`
+* handlers are listed as `unobservable`, never as unvisited.
+*
+* @example
+* ```ts no-check
+* const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
+* expect(report.unvisitedHandlers).toEqual([]);
+* ```
+*/
+function traceCoverage(machine, traces, options = {}) {
+	const nodes = /* @__PURE__ */ new Map();
+	const states = [];
+	const handlers = [];
+	const handlerAt = /* @__PURE__ */ new Map();
+	const unobservable = [];
+	const walk = (node) => {
+		const state = pathOf(node);
+		nodes.set(state, node);
+		if (node.type === "atomic" || node.type === "final") states.push({
+			state,
+			runs: 0
+		});
+		const byEvent = /* @__PURE__ */ new Map();
+		for (const [event, definitions] of node.transitions) {
+			const dynamic = definitions.some((d) => typeof d.to === "function");
+			const declared = [...new Set(definitions.flatMap((d) => (d.target ?? []).map(pathOf)))];
+			const handler = {
+				state,
+				event,
+				runs: 0,
+				targets: {
+					declared: dynamic ? "dynamic" : declared,
+					seen: [],
+					unreached: []
+				}
+			};
+			handlers.push(handler);
+			byEvent.set(event, handler);
+		}
+		handlerAt.set(node, byEvent);
+		if (node.always?.length) unobservable.push({
+			state,
+			event: "always"
+		});
+		for (const child of Object.values(node.states)) walk(child);
+	};
+	walk(machine.root);
+	const stateRuns = new Map(states.map((row) => [row.state, row]));
+	const seen = /* @__PURE__ */ new Map();
+	for (const trace of traces) {
+		const steps = traceSteps(trace);
+		const visited = new Set(steps[0]?.from);
+		for (const event of trace) {
+			if (event.type !== "machine.transition" || !isRecord(event.event)) continue;
+			if (event.event.type !== "@xstate.init" || !isRecord(event.snapshot)) continue;
+			for (const leaf of statePaths(event.snapshot.value)) visited.add(leaf);
+		}
+		const credited = /* @__PURE__ */ new Set();
+		for (const step of steps) {
+			for (const leaf of step.to) visited.add(leaf);
+			if (step.ignored) continue;
+			const hits = /* @__PURE__ */ new Set();
+			for (const leaf of step.from) for (let node = nodes.get(leaf); node; node = node.parent) {
+				if (node.type === "final") continue;
+				const event = candidateAt(node, step.event, step.payload);
+				if (event === void 0) continue;
+				hits.add(handlerAt.get(node).get(event));
+				break;
+			}
+			for (const handler of hits) {
+				credited.add(handler);
+				let leaves = seen.get(handler);
+				if (!leaves) seen.set(handler, leaves = /* @__PURE__ */ new Set());
+				for (const leaf of step.to) leaves.add(leaf);
+			}
+		}
+		for (const state of visited) {
+			const row = stateRuns.get(state);
+			if (row) row.runs++;
+		}
+		for (const handler of credited) handler.runs++;
+	}
+	for (const handler of handlers) {
+		const leaves = [...seen.get(handler) ?? []];
+		handler.targets.seen = leaves;
+		const { declared } = handler.targets;
+		if (declared === "dynamic") continue;
+		handler.targets.unreached = declared.filter((target) => !leaves.some((leaf) => leaf === target || leaf.startsWith(`${target}.`)));
+	}
+	const reachableComplete = options.reachable ? isExplorationComplete(options.reachable) : void 0;
+	let unreachable;
+	if (options.reachable && reachableComplete) {
+		const reached = new Set(options.reachable.reachedStates.flatMap((value) => statePaths(value)));
+		unreachable = states.filter((row) => !reached.has(row.state) && row.runs === 0).map((row) => row.state);
+	}
+	const counted = unreachable ? states.filter((row) => !unreachable.includes(row.state)) : states;
+	const unvisitedStates = counted.filter((row) => row.runs === 0).map((row) => row.state);
+	const unvisitedHandlers = handlers.filter((handler) => handler.runs === 0).map(({ state, event }) => ({
+		state,
+		event
+	}));
+	return {
+		states,
+		handlers,
+		unvisitedStates,
+		unvisitedHandlers,
+		unobservable,
+		...unreachable ? { unreachable } : {},
+		...reachableComplete === void 0 ? {} : { reachableComplete },
+		ratio: {
+			states: ratioOf(counted.length - unvisitedStates.length, counted.length),
+			handlers: ratioOf(handlers.length - unvisitedHandlers.length, handlers.length)
+		}
+	};
+}
+//#endregion
+//#region src/first-divergence.ts
+const defaultKey = (step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`;
+const isTrace = (items) => items.length === 0 || items.some((item) => isRecord(item) && (item.type === "machine.transition" || item.type === "run.start" || item.type === "run.end"));
+/**
+* Finds the first step where a run departs from a reference, so a failure is
+* blamed on the state it started in rather than on every state the run went
+* through afterwards.
+*
+* - Against a reference **trace**, such as a passing run of the same
+*   scenario, steps are compared in order by `key`. The first step that
+*   differs, or the point where either trace ends early, is the divergence.
+* - Against an expected **trajectory** (`TrajectoryItem[]`, as for
+*   {@link matchesTrajectory}), the run is laid out as its initial state, then
+*   each step's event and new state. The first expected item it never reaches
+*   is the divergence, blamed on the step after the last item it did reach. A
+*   run that goes on past the trajectory has not diverged.
+*
+* An empty reference is a trace with no steps, so any step diverges from it.
+* A trace is one run's events, live or serialized. Steps the machine ignored
+* changed nothing and are left out on both sides. Returns `undefined` when the
+* run does not diverge.
+*
+* @example
+* ```ts no-check
+* const stats = foldTraces(traces, {
+*   failed,
+*   blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
+* });
+* ```
+*/
+function firstDivergence(actual, reference, options = {}) {
+	const steps = traceSteps(actual).filter((step) => !step.ignored);
+	return isTrace(reference) ? againstTrace(steps, traceSteps(reference).filter((step) => !step.ignored), options.key ?? defaultKey) : againstTrajectory(actual, steps, reference);
+}
+function diverged(steps, index, expected, initial = steps[0]?.fromValue) {
+	const step = steps[index];
+	return {
+		index,
+		state: (step ? step.from[0] : steps.at(-1)?.to[0] ?? statePaths(initial)[0]) ?? "",
+		...step?.decision === void 0 ? {} : { decision: step.decision },
+		...step ? { actual: step } : {},
+		...expected === void 0 ? {} : { expected }
+	};
+}
+function againstTrace(steps, reference, key) {
+	const length = Math.max(steps.length, reference.length);
+	for (let index = 0; index < length; index++) {
+		const step = steps[index];
+		const want = reference[index];
+		if (step && want && key(step) === key(want)) continue;
+		return diverged(steps, index, want, steps[0]?.fromValue ?? reference[0]?.fromValue);
+	}
+}
+function againstTrajectory(trace, steps, expected) {
+	const initial = initialValue(trace, steps);
+	const timeline = initial === void 0 ? [] : [{ value: initial }];
+	const stepAt = initial === void 0 ? [] : [0];
+	steps.forEach((step, index) => {
+		timeline.push(step.payload);
+		stepAt.push(index);
+		if (JSON.stringify(step.toValue) !== JSON.stringify(step.fromValue)) {
+			timeline.push({ value: step.toValue });
+			stepAt.push(index);
+		}
+	});
+	const match = matchesTrajectory(timeline, expected);
+	if (!match.firstMiss) return void 0;
+	const at = match.firstMiss.searchedFrom;
+	return diverged(steps, at < timeline.length ? stepAt[at] ?? steps.length : steps.length, match.firstMiss.expected, initial);
+}
+function initialValue(trace, steps) {
+	for (const event of trace) if (event.type === "machine.transition" && isRecord(event.snapshot)) return event.snapshot.value;
+	return steps[0]?.fromValue;
+}
+//#endregion
 //#region src/decision-requests.ts
 /**
 * The decisions a run asked for, read from its trace: each decision's first
@@ -480,6 +759,212 @@ function getDecisionRequests(trace, options = {}) {
 	return requests;
 }
 //#endregion
+//#region src/repeat-runs.ts
+/**
+* Summarizes repeated runs of each scenario: how often each passed, whether
+* all its runs passed (pass^k), how many distinct state paths they took, and
+* where they first went different ways. Pure. Use it for runs collected
+* elsewhere, such as CI shards or replays; {@link repeatRuns} collects them and
+* calls it.
+*
+* A scenario that passes 3 of 5 runs and `divergedAt: 'deciding'` points at one
+* decision, not at a flaky agent.
+*
+* @example
+* ```ts no-check
+* const report = summarizeRepeats(shards.map((s) => ({ id: s.id, scenario: s, runs: s.runs })));
+* report.passHat(3); // the chance 3 runs of a scenario all pass
+* ```
+*/
+function summarizeRepeats(groups) {
+	const scenarios = groups.map((group) => summarize$1(group));
+	const traces = groups.flatMap((group) => group.runs.flatMap((run) => run.trace ? [run.trace] : []));
+	const most = Math.max(0, ...groups.map((group) => group.runs.length));
+	const passHat = (j) => {
+		if (!Number.isInteger(j) || j < 1 || j > most) throw new AgentError("pass-hat-out-of-range", `passHat: j must be a whole number from 1 to ${most}, got ${j}.`);
+		const eligible = scenarios.filter((scenario) => scenario.runs.length >= j);
+		return eligible.reduce((total, scenario) => total + allPass(scenario.passes, scenario.runs.length, j), 0) / eligible.length;
+	};
+	return {
+		scenarios,
+		passHat,
+		traces
+	};
+}
+/**
+* Runs every scenario `k` times and summarizes the runs with
+* {@link summarizeRepeats}. Runs are sequential, scenario by scenario. A run
+* that throws fails, with its message as `error` and no trace.
+*
+* With a live model, one run per scenario says little: τ-bench found pass^k
+* falls well below the single-run rate. Report pass^k next to it.
+*
+* @example
+* ```ts no-check
+* const report = await repeatRuns({
+*   scenarios,
+*   k: 5,
+*   run: async (scenario) => {
+*     const trace: AgentTraceEvent[] = [];
+*     const result = await runToQuiescence(
+*       createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
+*       { input: scenario.input },
+*     );
+*     return { trace, outcome: String(result.snapshot.value) };
+*   },
+*   passed: (scenario, { outcome }) => outcome === scenario.expected,
+* });
+* report.passHat(5);
+* ```
+*/
+async function repeatRuns(options) {
+	if (!Number.isInteger(options.k) || options.k < 1) throw new AgentError("invalid-repeat-count", `repeatRuns: k must be a whole number of 1 or more, got ${options.k}.`);
+	const groups = [];
+	for (const scenario of options.scenarios) {
+		const runs = [];
+		for (let attempt = 0; attempt < options.k; attempt++) try {
+			const result = await options.run(scenario, attempt);
+			runs.push({
+				passed: options.passed(scenario, result),
+				trace: result.trace,
+				...result.outcome === void 0 ? {} : { outcome: result.outcome }
+			});
+		} catch (error) {
+			runs.push({
+				passed: false,
+				error: error instanceof Error ? error.message : String(error)
+			});
+		}
+		groups.push({
+			id: scenario.id,
+			scenario,
+			runs
+		});
+	}
+	return summarizeRepeats(groups);
+}
+function summarize$1(group) {
+	const passes = group.runs.filter((run) => run.passed).length;
+	const traces = group.runs.flatMap((run) => run.trace ? [run.trace] : []);
+	const paths = new Set(traces.map((trace) => traceSteps(trace).filter((step) => !step.ignored).map((step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`).join(" ")));
+	const reference = traces.find((trace) => trace.some((event) => event.type === "machine.transition"));
+	let earliest;
+	for (const trace of reference ? traces : []) {
+		if (trace === reference) continue;
+		const divergence = firstDivergence(trace, reference);
+		if (divergence && (!earliest || divergence.index < earliest.index)) earliest = divergence;
+	}
+	return {
+		...group,
+		passes,
+		passAny: passes > 0,
+		passAll: group.runs.length > 0 && passes === group.runs.length,
+		distinctPaths: paths.size,
+		...earliest ? { divergedAt: earliest.state } : {}
+	};
+}
+/** C(passes, j) / C(runs, j), as a product so large counts don't overflow. */
+function allPass(passes, runs, j) {
+	let chance = 1;
+	for (let i = 0; i < j; i++) chance *= Math.max(0, passes - i) / (runs - i);
+	return chance;
+}
+//#endregion
+//#region src/judge-agreement.ts
+/**
+* Measures how well a judge agrees with people on the same cases: accuracy,
+* per-label recall, the confusion matrix, and the rows that disagree. With
+* `positive`, also the TPR and TNR, the recommended view for binary judges
+* (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
+* Pure.
+*
+* Build rows from confirmed labels only; draft labels gate nothing. Group by
+* decision state or request name so each narrow judge is aligned on its own.
+* A row with no group key counts in the totals and in no group. Run it on a
+* dev split and report the held-out split separately.
+*
+* Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
+*
+* @example
+* ```ts no-check
+* const report = judgeAgreement(
+*   labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
+*   { positive: 'fail' },
+* );
+* report.binary; // { tpr: 0.9, tnr: 0.8 }
+* report.correct?.(0.3); // estimated true failure rate in production
+* ```
+*/
+function judgeAgreement(rows, options = {}) {
+	if (rows.length === 0) throw new AgentError("empty-agreement-rows", "judgeAgreement: rows must contain at least one row.");
+	const report = summarize(rows, options.positive);
+	const groupOf = options.groupBy ?? ((row) => row.group);
+	if (options.groupBy || rows.some((row) => row.group !== void 0)) {
+		const grouped = /* @__PURE__ */ new Map();
+		for (const row of rows) {
+			const key = groupOf(row);
+			if (key === void 0) continue;
+			const members = grouped.get(key);
+			if (members) members.push(row);
+			else grouped.set(key, [row]);
+		}
+		const groups = Object.create(null);
+		for (const [key, members] of grouped) groups[key] = summarize(members, options.positive);
+		report.groups = groups;
+	}
+	return report;
+}
+function summarize(rows, positive) {
+	const perLabel = Object.create(null);
+	const confusion = Object.create(null);
+	const disagreements = [];
+	let agreed = 0;
+	for (const { id, human, judge } of rows) {
+		const label = perLabel[human] ??= {
+			human: 0,
+			agreed: 0
+		};
+		const row = confusion[human] ??= Object.create(null);
+		label.human++;
+		row[judge] = (row[judge] ?? 0) + 1;
+		if (human === judge) {
+			label.agreed++;
+			agreed++;
+		} else disagreements.push({
+			id,
+			human,
+			judge
+		});
+	}
+	const summary = {
+		total: rows.length,
+		accuracy: agreed / rows.length,
+		perLabel,
+		confusion,
+		disagreements
+	};
+	if (positive === void 0) return summary;
+	let positives = 0;
+	let truePositives = 0;
+	let negatives = 0;
+	let trueNegatives = 0;
+	for (const { human, judge } of rows) if (human === positive) {
+		positives++;
+		if (judge === positive) truePositives++;
+	} else {
+		negatives++;
+		if (judge !== positive) trueNegatives++;
+	}
+	const tpr = positives === 0 ? void 0 : truePositives / positives;
+	const tnr = negatives === 0 ? void 0 : trueNegatives / negatives;
+	summary.binary = {
+		tpr,
+		tnr
+	};
+	if (tpr !== void 0 && tnr !== void 0 && tpr + tnr > 1) summary.correct = (p) => Math.min(1, Math.max(0, (p + tnr - 1) / (tpr + tnr - 1)));
+	return summary;
+}
+//#endregion
 //#region src/compare-runs.ts
 /**
 * Runs every scenario through a baseline and a candidate implementation and
@@ -585,4 +1070,4 @@ function structurallyEqual(a, b) {
 	return keys.length === Object.keys(b).length && keys.every((key) => Object.hasOwn(b, key) && structurallyEqual(a[key], b[key]));
 }
 //#endregion
-export { AgentLintError, AgentUnknownStateError, assertAgentMachine, canReach, compareRuns, explorePaths, foldTraces, getDecisionRequests, lintAgentMachine, matchesTrajectory, runSeam, simulateAgent };
+export { AgentLintError, AgentUnknownStateError, assertAgentMachine, canReach, compareRuns, explorePaths, firstDivergence, foldTraces, getDecisionRequests, judgeAgreement, lintAgentMachine, matchesTrajectory, repeatRuns, runSeam, simulateAgent, summarizeRepeats, traceCoverage };

package/dist/validate.cjs

@@ -23,7 +23,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 //#endregion
 const require_event_log = require("./event-log-CKWeC7H-.cjs");
 const require_setup_agent = require("./setup-agent-CwUJyqKB.cjs");
-const require_verify = require("./verify-D_llRCTr.cjs");
+const require_verify = require("./verify-CS_WLRXn.cjs");
 let ajv_dist_2020_js = require("ajv/dist/2020.js");
 ajv_dist_2020_js = __toESM(ajv_dist_2020_js, 1);
 //#region schemas/agent-workflow.json

package/dist/validate.mjs

@@ -1,6 +1,6 @@
 import { D as STREAM_TEXT_ACTOR, E as GENERATE_TEXT_ACTOR, T as DECIDE_ACTOR } from "./event-log-DAqsalT1.mjs";
 import { i as setupAgentFromConfig } from "./setup-agent-CmhB-U5P.mjs";
-import { a as explorePaths, o as lintAgentMachine } from "./verify-CoCvR1eM.mjs";
+import { a as explorePaths, s as lintAgentMachine } from "./verify-DgB8P-Wx.mjs";
 import Ajv2020 from "ajv/dist/2020.js";
 //#region schemas/agent-workflow.json
 var agent_workflow_default = {

package/dist/{verify-D_llRCTr.cjs → verify-CS_WLRXn.cjs}

@@ -1223,10 +1223,20 @@ async function canReach(machine, target, options = {}) {
 	};
 	return {
 		reachable: false,
-		complete: !report.hitPathCap && report.unexpandedEvents.length === 0 && report.terminals.every((terminal) => terminal.status === "done" || terminal.status === "idle" || terminal.status === "error"),
+		complete: isExplorationComplete(report),
 		...resolvedTarget ? { target: resolvedTarget } : {}
 	};
 }
+/**
+* Whether an `explorePaths` report covers every path: no cap cut it short,
+* every payload was expanded, and every branch ended on its own. `done`,
+* `idle`, and `error` end a branch the machine itself ended; any other
+* terminal stopped a branch the search did not explore.
+* @internal
+*/
+function isExplorationComplete(report) {
+	return !report.hitPathCap && report.unexpandedEvents.length === 0 && report.terminals.every((terminal) => terminal.status === "done" || terminal.status === "idle" || terminal.status === "error");
+}
 //#endregion
 Object.defineProperty(exports, "AgentLintError", {
 	enumerable: true,
@@ -1258,6 +1268,12 @@ Object.defineProperty(exports, "explorePaths", {
 		return explorePaths;
 	}
 });
+Object.defineProperty(exports, "isExplorationComplete", {
+	enumerable: true,
+	get: function() {
+		return isExplorationComplete;
+	}
+});
 Object.defineProperty(exports, "lintAgentMachine", {
 	enumerable: true,
 	get: function() {

package/dist/{verify-CoCvR1eM.mjs → verify-DgB8P-Wx.mjs}

@@ -1223,9 +1223,19 @@ async function canReach(machine, target, options = {}) {
 	};
 	return {
 		reachable: false,
-		complete: !report.hitPathCap && report.unexpandedEvents.length === 0 && report.terminals.every((terminal) => terminal.status === "done" || terminal.status === "idle" || terminal.status === "error"),
+		complete: isExplorationComplete(report),
 		...resolvedTarget ? { target: resolvedTarget } : {}
 	};
 }
+/**
+* Whether an `explorePaths` report covers every path: no cap cut it short,
+* every payload was expanded, and every branch ended on its own. `done`,
+* `idle`, and `error` end a branch the machine itself ended; any other
+* terminal stopped a branch the search did not explore.
+* @internal
+*/
+function isExplorationComplete(report) {
+	return !report.hitPathCap && report.unexpandedEvents.length === 0 && report.terminals.every((terminal) => terminal.status === "done" || terminal.status === "idle" || terminal.status === "error");
+}
 //#endregion
-export { explorePaths as a, canReach as i, AgentUnknownStateError as n, lintAgentMachine as o, assertAgentMachine as r, simulateAgent as s, AgentLintError as t };
+export { explorePaths as a, simulateAgent as c, canReach as i, AgentUnknownStateError as n, isExplorationComplete as o, assertAgentMachine as r, lintAgentMachine as s, AgentLintError as t };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@statelyai/agent",
-  "version": "2.0.0-alpha.32",
+  "version": "2.0.0-alpha.33",
   "description": "Make invalid agent actions impossible. Agent logic as state machines: deterministic, inspectable, resumable, runs anywhere.",
   "keywords": [
     "agent",

package/skills/add-evals/SKILL.md

@@ -125,10 +125,16 @@ export function objective(runs: Array<{ scenario: Scenario; outcome: string; ref
 
 Read the terminal state from the event logs `add-traces` recorded (`replay(machine, entries).snapshot`, `replay` from `@statelyai/agent/log`). Effects and violations live in the world, not the log: save them next to the thread id when the run ends, so the objective can be recomputed without running anything. Use `matchesTrajectory` only for an ordering a person requires (approval before payment), never to pin the whole path; build its input from the `machine.transition` trace events' `snapshot.value`.
 
-**Stats.** `foldTraces(traces, { failed })` (from `@statelyai/agent/testing`) returns `{ state, visits, failures, withheld }` per state. A failed run counts against **every** state it visited, so states after a bad decision rank as high as the decision itself. Use the table to see which states failing runs pass through, then look at the decision states among them, their `withheld` counts, and their decision attempts. `failed` sees only the trace: to judge a run against its label, keep a map from trace to scenario.
+**Reliability.** A live model is measured over repeated runs. `repeatRuns({ scenarios, k, run, passed })` (from `@statelyai/agent/testing`) runs each scenario `k` times and reports `passes`, `passAll` (pass^k), `distinctPaths`, and `divergedAt`, the state where its runs went different ways. Report `report.passHat(k)` next to `report.passHat(1)`. For runs recorded elsewhere, `summarizeRepeats` takes them per scenario.
+
+**Judged evals.** Where a model grades text because no ground truth exists, measure the judge first: `judgeAgreement(rows, { positive })` over confirmed labels, grouped by the state or request it judges. Report the true positive and true negative rates, and gate on the judge only after the user accepts them. Use `report.correct(p)` to turn the judge's rate on unlabeled runs into an estimate.
+
+**Stats.** `foldTraces(traces, { failed, blame })` (from `@statelyai/agent/testing`) returns `{ state, visits, failures, withheld, ignored, blamed }` per state. A failed run counts in `failures` against **every** state it visited, so states after a bad decision rank as high as the decision itself. `blame` counts each failed run once, against the state it started failing in: `blame: (trace) => firstDivergence(trace, reference)?.state`, where the reference is a passing run of the same scenario or the trajectory a person expects. Sort by `blamed`, then look at the decision states among the top rows, their `withheld` counts, and their decision attempts. `failed` sees only the trace: to judge a run against its label, keep a map from trace to scenario.
 
 ## 5. Coverage
 
+Run `traceCoverage(machine, traces, { reachable })` (from `@statelyai/agent/testing`) over the traces of every scenario, with `reachable` set to an `explorePaths` report. List its `unvisitedStates` and `unvisitedHandlers`: each one is a part of the machine no scenario exercises. Draft a scenario for each, as below. Handlers in `unobservable` (`always` transitions) do not show in traces; say so, and do not draft scenarios for them.
+
 If every decider scores the same on the confirmed scenarios, the scenarios cannot tell a better decider from a worse one. Draft scenarios on the boundaries the evals found (an amount just under and over a threshold, a partial request), label them as drafts, and ask the user to confirm them and add them to the scenario set. They become part of the train or held-out split only once confirmed.
 
 ## 6. Baseline
@@ -139,6 +145,6 @@ Run every eval once on the current machine and record the numbers: which pass, t
 
 1. The state mapping from step 2.
 2. The evals by level, each named by state.
-3. The baseline numbers, train and held-out separately.
+3. The baseline numbers, train and held-out separately, with pass^k for live-model evals and the coverage ratios.
 4. Every failing eval and the state it points at.
 5. Next step: `improve-machine`, with the objective from step 4 as its scorer.

package/skills/improve-machine/SKILL.md

@@ -197,7 +197,7 @@ const failed = verified.gates.find((gate) => !gate.ok);
 Run the candidate on the training scenarios, then on the held-out scenarios. Write every replay to the store under new thread ids that name the split and version, in the scheme `add-traces` set up (`train:v3:<scenario>:r1`, `held-out:v3:<scenario>:r1`), so the next round can list them by prefix. With no persistent store, write each thread's `result.events` to a file.
 
 - Run the current policy and the candidate on the same scenarios, the same world state, and the same model refs.
-- Model calls are nondeterministic. Run each scenario k times (k ≥ 3 when calls are not cached) and compare means.
+- Model calls are nondeterministic. Run each scenario k times (k ≥ 3 when calls are not cached) with `repeatRuns` (from `@statelyai/agent/testing`), and compare `passHat(k)` as well as the mean. A candidate that raises the mean but lowers pass^k made the agent less reliable.
 - Compute the objective, invariant violations, and cost per split, for both versions.
 
 ## 7. Recommend or discard