@statelyai/agent 2.0.0-alpha.31 → 2.0.0-alpha.33

package/dist/{verify-CS_xxvVU.cjs → verify-CS_WLRXn.cjs}

@@ -1,5 +1,5 @@
-const require_event_log = require("./event-log-Bdww2l1D.cjs");
-const require_decision = require("./decision-Iys2hzg3.cjs");
+const require_event_log = require("./event-log-CKWeC7H-.cjs");
+const require_decision = require("./decision-BPdEaxPf.cjs");
 let xstate = require("xstate");
 //#region src/steps.ts
 /**
@@ -1223,10 +1223,20 @@ async function canReach(machine, target, options = {}) {
 	};
 	return {
 		reachable: false,
-		complete: !report.hitPathCap && report.unexpandedEvents.length === 0 && report.terminals.every((terminal) => terminal.status === "done" || terminal.status === "idle" || terminal.status === "error"),
+		complete: isExplorationComplete(report),
 		...resolvedTarget ? { target: resolvedTarget } : {}
 	};
 }
+/**
+* Whether an `explorePaths` report covers every path: no cap cut it short,
+* every payload was expanded, and every branch ended on its own. `done`,
+* `idle`, and `error` end a branch the machine itself ended; any other
+* terminal stopped a branch the search did not explore.
+* @internal
+*/
+function isExplorationComplete(report) {
+	return !report.hitPathCap && report.unexpandedEvents.length === 0 && report.terminals.every((terminal) => terminal.status === "done" || terminal.status === "idle" || terminal.status === "error");
+}
 //#endregion
 Object.defineProperty(exports, "AgentLintError", {
 	enumerable: true,
@@ -1258,6 +1268,12 @@ Object.defineProperty(exports, "explorePaths", {
 		return explorePaths;
 	}
 });
+Object.defineProperty(exports, "isExplorationComplete", {
+	enumerable: true,
+	get: function() {
+		return isExplorationComplete;
+	}
+});
 Object.defineProperty(exports, "lintAgentMachine", {
 	enumerable: true,
 	get: function() {

package/dist/{verify-BH5e0XLK.mjs → verify-DgB8P-Wx.mjs}

@@ -1,5 +1,5 @@
-import { P as isTextLogic, U as getRegisteredAgentExecutionOptions, V as executorBoundLogics, X as getJsonSchemaSync, lt as isRecord, ut as AgentError } from "./event-log-DE430ZCa.mjs";
-import { d as getAcceptedEvents, f as getHumanEventTypes, l as resolveDecision, n as acceptedDecisionOptions, p as isEventPattern, s as isDecisionLogic, t as AgentDecisionExhaustedError } from "./decision-DqO018Tp.mjs";
+import { P as isTextLogic, U as getRegisteredAgentExecutionOptions, V as executorBoundLogics, X as getJsonSchemaSync, lt as isRecord, ut as AgentError } from "./event-log-DAqsalT1.mjs";
+import { d as getAcceptedEvents, f as getHumanEventTypes, l as resolveDecision, n as acceptedDecisionOptions, p as isEventPattern, s as isDecisionLogic, t as AgentDecisionExhaustedError } from "./decision-CV7WiV5C.mjs";
 import { getNextTransitions, initialTransition, machineToJSON, transition } from "xstate";
 //#region src/steps.ts
 /**
@@ -1223,9 +1223,19 @@ async function canReach(machine, target, options = {}) {
 	};
 	return {
 		reachable: false,
-		complete: !report.hitPathCap && report.unexpandedEvents.length === 0 && report.terminals.every((terminal) => terminal.status === "done" || terminal.status === "idle" || terminal.status === "error"),
+		complete: isExplorationComplete(report),
 		...resolvedTarget ? { target: resolvedTarget } : {}
 	};
 }
+/**
+* Whether an `explorePaths` report covers every path: no cap cut it short,
+* every payload was expanded, and every branch ended on its own. `done`,
+* `idle`, and `error` end a branch the machine itself ended; any other
+* terminal stopped a branch the search did not explore.
+* @internal
+*/
+function isExplorationComplete(report) {
+	return !report.hitPathCap && report.unexpandedEvents.length === 0 && report.terminals.every((terminal) => terminal.status === "done" || terminal.status === "idle" || terminal.status === "error");
+}
 //#endregion
-export { explorePaths as a, canReach as i, AgentUnknownStateError as n, lintAgentMachine as o, assertAgentMachine as r, simulateAgent as s, AgentLintError as t };
+export { explorePaths as a, simulateAgent as c, canReach as i, AgentUnknownStateError as n, isExplorationComplete as o, assertAgentMachine as r, lintAgentMachine as s, AgentLintError as t };

package/dist/{verify-CPKDPMwY.d.cts → verify-Dk3Vwnha.d.cts}

@@ -1,5 +1,5 @@
 import { u as ChosenEvent } from "./types-Dp5_Ru4R.cjs";
-import { Q as AgentError } from "./text-logic-j9hShB7d.cjs";
+import { Q as AgentError } from "./text-logic-CYP502VT.cjs";
 import { AnyMachineSnapshot, AnyStateMachine } from "xstate";
 //#region src/verify.d.ts
 /** Severity of an {@link AgentLintDiagnostic}. `error` findings fail CI/the CLI; `warning`s are advisory. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@statelyai/agent",
-  "version": "2.0.0-alpha.31",
+  "version": "2.0.0-alpha.33",
   "description": "Make invalid agent actions impossible. Agent logic as state machines: deterministic, inspectable, resumable, runs anywhere.",
   "keywords": [
     "agent",

package/skills/add-evals/SKILL.md

@@ -125,10 +125,16 @@ export function objective(runs: Array<{ scenario: Scenario; outcome: string; ref
 
 Read the terminal state from the event logs `add-traces` recorded (`replay(machine, entries).snapshot`, `replay` from `@statelyai/agent/log`). Effects and violations live in the world, not the log: save them next to the thread id when the run ends, so the objective can be recomputed without running anything. Use `matchesTrajectory` only for an ordering a person requires (approval before payment), never to pin the whole path; build its input from the `machine.transition` trace events' `snapshot.value`.
 
-**Stats.** `foldTraces(traces, { failed })` (from `@statelyai/agent/testing`) returns `{ state, visits, failures, withheld }` per state. A failed run counts against **every** state it visited, so states after a bad decision rank as high as the decision itself. Use the table to see which states failing runs pass through, then look at the decision states among them, their `withheld` counts, and their decision attempts. `failed` sees only the trace: to judge a run against its label, keep a map from trace to scenario.
+**Reliability.** A live model is measured over repeated runs. `repeatRuns({ scenarios, k, run, passed })` (from `@statelyai/agent/testing`) runs each scenario `k` times and reports `passes`, `passAll` (pass^k), `distinctPaths`, and `divergedAt`, the state where its runs went different ways. Report `report.passHat(k)` next to `report.passHat(1)`. For runs recorded elsewhere, `summarizeRepeats` takes them per scenario.
+
+**Judged evals.** Where a model grades text because no ground truth exists, measure the judge first: `judgeAgreement(rows, { positive })` over confirmed labels, grouped by the state or request it judges. Report the true positive and true negative rates, and gate on the judge only after the user accepts them. Use `report.correct(p)` to turn the judge's rate on unlabeled runs into an estimate.
+
+**Stats.** `foldTraces(traces, { failed, blame })` (from `@statelyai/agent/testing`) returns `{ state, visits, failures, withheld, ignored, blamed }` per state. A failed run counts in `failures` against **every** state it visited, so states after a bad decision rank as high as the decision itself. `blame` counts each failed run once, against the state it started failing in: `blame: (trace) => firstDivergence(trace, reference)?.state`, where the reference is a passing run of the same scenario or the trajectory a person expects. Sort by `blamed`, then look at the decision states among the top rows, their `withheld` counts, and their decision attempts. `failed` sees only the trace: to judge a run against its label, keep a map from trace to scenario.
 
 ## 5. Coverage
 
+Run `traceCoverage(machine, traces, { reachable })` (from `@statelyai/agent/testing`) over the traces of every scenario, with `reachable` set to an `explorePaths` report. List its `unvisitedStates` and `unvisitedHandlers`: each one is a part of the machine no scenario exercises. Draft a scenario for each, as below. Handlers in `unobservable` (`always` transitions) do not show in traces; say so, and do not draft scenarios for them.
+
 If every decider scores the same on the confirmed scenarios, the scenarios cannot tell a better decider from a worse one. Draft scenarios on the boundaries the evals found (an amount just under and over a threshold, a partial request), label them as drafts, and ask the user to confirm them and add them to the scenario set. They become part of the train or held-out split only once confirmed.
 
 ## 6. Baseline
@@ -139,6 +145,6 @@ Run every eval once on the current machine and record the numbers: which pass, t
 
 1. The state mapping from step 2.
 2. The evals by level, each named by state.
-3. The baseline numbers, train and held-out separately.
+3. The baseline numbers, train and held-out separately, with pass^k for live-model evals and the coverage ratios.
 4. Every failing eval and the state it points at.
 5. Next step: `improve-machine`, with the objective from step 4 as its scorer.

package/skills/improve-machine/SKILL.md

@@ -197,7 +197,7 @@ const failed = verified.gates.find((gate) => !gate.ok);
 Run the candidate on the training scenarios, then on the held-out scenarios. Write every replay to the store under new thread ids that name the split and version, in the scheme `add-traces` set up (`train:v3:<scenario>:r1`, `held-out:v3:<scenario>:r1`), so the next round can list them by prefix. With no persistent store, write each thread's `result.events` to a file.
 
 - Run the current policy and the candidate on the same scenarios, the same world state, and the same model refs.
-- Model calls are nondeterministic. Run each scenario k times (k ≥ 3 when calls are not cached) and compare means.
+- Model calls are nondeterministic. Run each scenario k times (k ≥ 3 when calls are not cached) with `repeatRuns` (from `@statelyai/agent/testing`), and compare `passHat(k)` as well as the mean. A candidate that raises the mean but lowers pass^k made the agent less reliable.
 - Compute the objective, invariant violations, and cost per split, for both versions.
 
 ## 7. Recommend or discard