@statelyai/agent 2.0.0-alpha.29 → 2.0.0-alpha.31

package/dist/{verify-CBuYIpsa.d.mts → verify-tkymWz6X.d.mts}

@@ -1,5 +1,5 @@
 import { u as ChosenEvent } from "./types-Dp5_Ru4R.mjs";
-import { Q as AgentError } from "./text-logic-BDbGjYMh.mjs";
+import { Q as AgentError } from "./text-logic-BwdrDpO9.mjs";
 import { AnyMachineSnapshot, AnyStateMachine } from "xstate";
 //#region src/verify.d.ts
 /** Severity of an {@link AgentLintDiagnostic}. `error` findings fail CI/the CLI; `warning`s are advisory. */
@@ -10,7 +10,7 @@ type AgentLintSeverity = "error" | "warning";
  * or config location, and `message` explains the problem and its remedy.
  */
 interface AgentLintDiagnostic {
-  code: "decide-without-events" | "direct-object-src" | "invoke-without-on-error";
+  code: "decide-without-events" | "decide-allows-human-event" | "direct-object-src" | "invoke-without-on-error";
   severity: AgentLintSeverity;
   /** State path (`parent.child`) or config pointer (e.g. `(root)`, `context`) the finding is about. */
   path: string;
@@ -212,6 +212,14 @@ interface SimulateAgentResult {
  */
 declare function simulateAgent(machine: AnyStateMachine, options: SimulateAgentOptions): Promise<SimulateAgentResult>;
 /** Options for {@link explorePaths}. */
+/**
+ * The snapshot type the exploration callbacks (`payloads`, `dedupeBy`, a
+ * `canReach` predicate) receive. Import it from here rather than from your
+ * own `xstate`, which can be another version than the one this package uses.
+ */
+type ExploreSnapshot = AnyMachineSnapshot;
+/** What {@link canReach} looks for: a state id or path, or a snapshot predicate. */
+type CanReachTarget = string | ((snapshot: ExploreSnapshot) => boolean);
 interface ExplorePathsOptions {
   input?: unknown;
   /** Max branch points (decisions + external-event forks) along any path. Default 8. */
@@ -274,7 +282,7 @@ interface ExplorePathsOptions {
    * were not all tried, so it is listed in `unexpandedEvents`. A type with no
    * schema at all is applied as `{ type }`.
    */
-  payloads?: (snapshot: AnyMachineSnapshot) => Partial<Record<string, Record<string, unknown>[]>>;
+  payloads?: (snapshot: ExploreSnapshot) => Partial<Record<string, Record<string, unknown>[]>>;
   /**
    * A projection of the snapshot that determines its future. Settled
    * snapshots with an equal key are explored once (again only when reached
@@ -283,7 +291,7 @@ interface ExplorePathsOptions {
    * a key that drops a field the machine reads makes the search unsound.
    * Without it, every path is explored in full.
    */
-  dedupeBy?: (snapshot: AnyMachineSnapshot) => string;
+  dedupeBy?: (snapshot: ExploreSnapshot) => string;
 }
 /** A single explored path's terminal outcome. */
 interface AgentPathTerminal {
@@ -409,6 +417,6 @@ declare class AgentUnknownStateError extends AgentError {
  * // violation.reachable → false is the safety proof
  * ```
  */
-declare function canReach(machine: AnyStateMachine, target: string | ((snapshot: AnyMachineSnapshot) => boolean), options?: ExplorePathsOptions): Promise<CanReachResult>;
+declare function canReach(machine: AnyStateMachine, target: CanReachTarget, options?: ExplorePathsOptions): Promise<CanReachResult>;
 //#endregion
-export { explorePaths as _, AgentPathTerminal as a, CanReachResult as c, SimulateAgentOptions as d, SimulateAgentResult as f, canReach as g, assertAgentMachine as h, AgentPathReport as i, ExplorePathsOptions as l, SimulationTrailEntry as m, AgentLintError as n, AgentUnknownStateError as o, SimulationScript as p, AgentLintSeverity as r, AssertAgentMachineOptions as s, AgentLintDiagnostic as t, LintAgentMachineOptions as u, lintAgentMachine as v, simulateAgent as y };
+export { assertAgentMachine as _, AgentPathTerminal as a, lintAgentMachine as b, CanReachResult as c, ExploreSnapshot as d, LintAgentMachineOptions as f, SimulationTrailEntry as g, SimulationScript as h, AgentPathReport as i, CanReachTarget as l, SimulateAgentResult as m, AgentLintError as n, AgentUnknownStateError as o, SimulateAgentOptions as p, AgentLintSeverity as r, AssertAgentMachineOptions as s, AgentLintDiagnostic as t, ExplorePathsOptions as u, canReach as v, simulateAgent as x, explorePaths as y };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@statelyai/agent",
-  "version": "2.0.0-alpha.29",
+  "version": "2.0.0-alpha.31",
   "description": "Make invalid agent actions impossible. Agent logic as state machines: deterministic, inspectable, resumable, runs anywhere.",
   "keywords": [
     "agent",
@@ -21,12 +21,41 @@
   "files": [
     "dist",
     "schemas",
-    "skills"
+    "skills",
+    "CHANGELOG.md"
   ],
   "type": "module",
   "main": "dist/index.cjs",
   "module": "dist/index.mjs",
   "types": "dist/index.d.mts",
+  "typesVersions": {
+    "*": {
+      "ai-sdk": [
+        "./dist/ai-sdk.d.mts"
+      ],
+      "log": [
+        "./dist/log.d.mts"
+      ],
+      "machines": [
+        "./dist/machines.d.mts"
+      ],
+      "openai": [
+        "./dist/openai.d.mts"
+      ],
+      "otel": [
+        "./dist/otel.d.mts"
+      ],
+      "tanstack-ai": [
+        "./dist/tanstack-ai.d.mts"
+      ],
+      "testing": [
+        "./dist/testing.d.mts"
+      ],
+      "validate": [
+        "./dist/validate.d.mts"
+      ]
+    }
+  },
   "exports": {
     ".": {
       "import": {
@@ -132,6 +161,7 @@
     "@langchain/core": "^1.2.17",
     "@langchain/openai": "^1.6.2",
     "@mastra/core": "^1.75.0",
+    "@modelcontextprotocol/sdk": "^1.29.0",
     "@opentelemetry/api": "^1.9.1",
     "@opentelemetry/context-async-hooks": "^2.12.0",
     "@opentelemetry/sdk-trace-base": "^2.12.0",
@@ -159,7 +189,7 @@
   "peerDependencies": {
     "@opentelemetry/api": "^1",
     "@tanstack/ai": ">=0.66.0 <1",
-    "ai": "^7.0.0",
+    "ai": "^7.0.128",
     "ajv": "^8.20.0",
     "openai": ">=5.0.0 <8",
     "xstate": "6.0.0-alpha.69"
@@ -182,7 +212,7 @@
     }
   },
   "engines": {
-    "node": ">=22.18.0"
+    "node": ">=22"
   },
   "scripts": {
     "build": "tsdown",

package/readme.md

@@ -14,25 +14,25 @@ Environment and policy:
 
 - The environment is code you own and can verify; the policy is something a model can write inside it.
 - The environment holds the rules and invariants as TypeScript states and guards, fixed and checked with `canReach`.
-- The policy is a JSON machine a model can author and improve from traces, gated before it runs and promoted only on a measured gain. See [Environment and policy](docs/environment-and-policy.md).
+- The policy is a JSON machine a model can author and improve from traces, gated before it runs and promoted only on a measured gain. See [Environment and policy](https://github.com/statelyai/agent/blob/next/docs/environment-and-policy.md).
 
 Stately Agent 2 is in alpha. APIs may change before the stable release.
 
-[Documentation](https://stately.ai/docs/packages/agent) · [Examples](examples/README.md) · [XState](https://github.com/statelyai/xstate)
+[Documentation](https://stately.ai/docs/packages/agent) · [Examples](https://github.com/statelyai/agent/blob/next/examples/README.md) · [XState](https://github.com/statelyai/xstate)
 
 ## Four starting points
 
-- **Author a new agent.** Build a machine from states, decisions, and typed requests; run it locally with `runAgent`, test it with no API key, then use it in any framework or runtime with zero machine changes. See the [Quickstart](docs/quickstart.md) and [Use in any stack](docs/any-stack.md).
-- **Retrofit an existing agent.** Turn a `while` loop into a machine: your SDK calls, tools, and retry code become the executors; the machine replaces only the control flow. See [Migrating from a hand-rolled loop](docs/from-a-loop.md).
-- **Copy a known pattern.** ReAct, reflection, plan-and-execute, RAG, supervisor, and more, each a single runnable file you lift in 60 seconds. See [Agent patterns](docs/patterns.md).
-- **Let an agent improve its own logic.** Keep the rules in a fixed TypeScript environment and let a model rewrite the JSON policy inside it, gated by `verifyAgentConfig` and promoted only when held-out scenarios improve. See [Environment and policy](docs/environment-and-policy.md) and the [`improve-machine` skill](skills/improve-machine/SKILL.md).
+- **Author a new agent.** Build a machine from states, decisions, and typed requests; run it locally with `runAgent`, test it with no API key, then use it in any framework or runtime with zero machine changes. See the [Quickstart](https://github.com/statelyai/agent/blob/next/docs/quickstart.md) and [Use in any stack](https://github.com/statelyai/agent/blob/next/docs/any-stack.md).
+- **Retrofit an existing agent.** Turn a `while` loop into a machine: your SDK calls, tools, and retry code become the executors; the machine replaces only the control flow. See [Migrating from a hand-rolled loop](https://github.com/statelyai/agent/blob/next/docs/from-a-loop.md).
+- **Copy a known pattern.** ReAct, reflection, plan-and-execute, RAG, supervisor, and more, each a single runnable file you lift in 60 seconds. See [Agent patterns](https://github.com/statelyai/agent/blob/next/docs/patterns.md).
+- **Let an agent improve its own logic.** Keep the rules in a fixed TypeScript environment and let a model rewrite the JSON policy inside it, gated by `verifyAgentConfig` and promoted only when held-out scenarios improve. See [Environment and policy](https://github.com/statelyai/agent/blob/next/docs/environment-and-policy.md) and the [`improve-machine` skill](https://github.com/statelyai/agent/blob/next/skills/improve-machine/SKILL.md).
 
 ## Install
 
 <!-- install command matching the package prerelease channel and package.json peers -->
 
 ```sh
-pnpm add @statelyai/agent@alpha xstate@6.0.0-alpha.64 zod
+pnpm add @statelyai/agent@alpha xstate@6.0.0-alpha.69 zod
 ```
 
 XState is pinned while its experimental durable restoration API evolves.
@@ -40,7 +40,7 @@ XState is pinned while its experimental durable restoration API evolves.
 For the optional Vercel AI SDK executor:
 
 ```sh
-pnpm add ai@^7 @ai-sdk/openai@^4
+pnpm add ai@^7.0.128 @ai-sdk/openai@^4
 ```
 
 For the optional raw OpenAI SDK executor (`@statelyai/agent/openai`):
@@ -55,13 +55,43 @@ For the optional TanStack AI executor (`@statelyai/agent/tanstack-ai`):
 pnpm add @tanstack/ai @tanstack/ai-openai
 ```
 
-Classification, grading, relevance, and yes/no checks are judgments: call the AI SDK's `experimental_decide` with `@ai-sdk/typesafe-ai` (`pnpm add ai @ai-sdk/typesafe-ai`) from an ordinary actor; see [Judgments with the AI SDK](docs/judgments.md).
+Classification, grading, relevance, and yes/no checks are judgments: call the AI SDK's `experimental_decide` with `@ai-sdk/typesafe-ai` (`pnpm add ai @ai-sdk/typesafe-ai`) from an ordinary actor; see [Judgments with the AI SDK](https://github.com/statelyai/agent/blob/next/docs/judgments.md).
+
+Upgrading from an earlier alpha: see [Upgrading between alphas](https://github.com/statelyai/agent/blob/next/docs/upgrading.md). Every release's notes are in `CHANGELOG.md`, which ships in the package.
 
 Requirements:
 
-- Node 22.18 or newer, and XState v6 alpha.46 or newer.
-- The package is ESM-first. CommonJS builds are published too, so `require()` works.
-- Provider packages must match your `ai` major: `@ai-sdk/openai@^4` pairs with `ai@^7`. A bare `@ai-sdk/openai` resolves to `@latest`, which can mismatch the `ai` peer.
+- Node 22 or newer, and XState exactly `6.0.0-alpha.69`.
+- TypeScript 5.4 or newer. Older versions accept the code, but callback parameters in `createMachine` silently become `any`.
+- A [Standard Schema](https://standardschema.dev) library: zod 4, zod 3.24 or newer, Valibot, or ArkType. `setupAgent` throws `invalid-schema` for zod 3.23 and earlier.
+- The package is ESM-first. CommonJS builds are published too, so `require()` works. Subpath imports (`@statelyai/agent/testing`) resolve under every `moduleResolution`, including `node`.
+- `@statelyai/agent/ai-sdk` needs `ai` 7.0.128 or newer. Provider packages must match your `ai` major: `@ai-sdk/openai@^4` pairs with `ai@^7`. A bare `@ai-sdk/openai` resolves to `@latest`, which can mismatch the `ai` peer. `@statelyai/agent/openai` needs `openai` 5 or newer.
+
+An app that already uses XState 4 or 5 keeps it: put the agent in its own workspace package. See [Use in an XState 4 or 5 app](https://github.com/statelyai/agent/blob/next/docs/xstate-4-and-5-apps.md). If both versions land in one `node_modules`, the package fails with errors that do not mention a version:
+
+| Error | Cause |
+| --- | --- |
+| `createAsyncLogic is not a function` | `xstate` resolved to 4 or 5 from CommonJS |
+| `Cannot find module '…/node_modules/xstate/durable'` | `xstate` resolved to 4 or 5 from ESM |
+| `unmet peer dependency "xstate@6.0.0-alpha.69"` (pnpm, npm) or `has incorrect peer dependency "xstate@6.0.0-alpha.69"` (yarn 1) | The install's only warning of the two above |
+
+## Agent skills
+
+The package ships agent skills in `node_modules/@statelyai/agent/skills/`. Copy them into your coding agent's skills directory (for Claude Code, `.claude/skills/`):
+
+```sh
+mkdir -p .claude/skills
+cp -R node_modules/@statelyai/agent/skills/* .claude/skills/
+```
+
+| Skill | What it does |
+| --- | --- |
+| [`graph-engineering`](https://github.com/statelyai/agent/blob/next/skills/graph-engineering/SKILL.md) | Finds which step a project is at and routes to the next skill |
+| [`loop-to-machine`](https://github.com/statelyai/agent/blob/next/skills/loop-to-machine/SKILL.md) | Converts existing agent code into a machine and checks parity with `compareRuns` |
+| [`generate-machine`](https://github.com/statelyai/agent/blob/next/skills/generate-machine/SKILL.md) | Authors a JSON policy machine from scratch and runs `verifyAgentConfig` |
+| [`add-traces`](https://github.com/statelyai/agent/blob/next/skills/add-traces/SKILL.md) | Stamps and persists every run so it replays |
+| [`add-evals`](https://github.com/statelyai/agent/blob/next/skills/add-evals/SKILL.md) | Writes evals per state, at the cheapest level that catches the failure |
+| [`improve-machine`](https://github.com/statelyai/agent/blob/next/skills/improve-machine/SKILL.md) | Improves a policy from traces and hands the change to a person as a pull request |
 
 ## Quick start
 
@@ -151,7 +181,7 @@ import { openai } from "@ai-sdk/openai";
 import { createAiSdkExecutors } from "@statelyai/agent/ai-sdk";
 
 // In the setupAgent({ ... }) call above:
-//   models: { fast: openai("gpt-5.4-mini") },
+//   models: { fast: openai("gpt-6-luna") },
 
 const liveResult = await runAgent(refundMachine, {
   input: { request: "I was charged twice for the same order.", amount: 75 },
@@ -159,11 +189,11 @@ const liveResult = await runAgent(refundMachine, {
 });
 ```
 
-`createAiSdkExecutors(refundMachine)` reads the `models` declared in `setupAgent({ models })`, which also type the machine's model refs. `createAiSdkExecutors({ models })` still works when the host owns the map. Executors are always explicit, and core does not import the AI SDK. See [Models and providers](docs/models-and-providers.md) and [Hosts and executors](docs/hosts.md).
+`createAiSdkExecutors(refundMachine)` reads the `models` declared in `setupAgent({ models })`, which also type the machine's model refs. `createAiSdkExecutors({ models })` still works when the host owns the map. Executors are always explicit, and core does not import the AI SDK. See [Models and providers](https://github.com/statelyai/agent/blob/next/docs/models-and-providers.md) and [Hosts and executors](https://github.com/statelyai/agent/blob/next/docs/hosts.md).
 
-For a machine with several requests, the executor routes on `request.name`, or `MockLanguageModelV3` from `ai/test` stands in for the provider behind `createAiSdkExecutors`. See [Evals](docs/evals.md#testing-without-a-provider).
+For a machine with several requests, the executor routes on `request.name`, or `MockLanguageModelV3` from `ai/test` stands in for the provider behind `createAiSdkExecutors`. See [Evals](https://github.com/statelyai/agent/blob/next/docs/evals.md#testing-without-a-provider).
 
-A host that needs the transition loop itself (a durable object, events pushed mid-run, a custom checkpoint policy) uses `createAgentRuntime` directly. See [Write your own host loop](docs/hosts.md#write-your-own-host-loop).
+A host that needs the transition loop itself (a durable object, events pushed mid-run, a custom checkpoint policy) uses `createAgentRuntime` directly. See [Write your own host loop](https://github.com/statelyai/agent/blob/next/docs/hosts.md#write-your-own-host-loop).
 
 ## Architecture
 
@@ -181,38 +211,38 @@ The machine never talks to a model directly, so swapping `createAiSdkExecutors`
 
 The example above has one model decision and two final outcomes. Real machines add approval states, retries, parallel work, child agents, and long-running waits without changing how control flow is represented.
 
-The core concepts (machines owning control flow, bounded model decisions, typed requests, host-run executors, and native XState snapshots) are in the [documentation overview](docs/index.md).
+The core concepts (machines owning control flow, bounded model decisions, typed requests, host-run executors, and native XState snapshots) are in the [documentation overview](https://github.com/statelyai/agent/blob/next/docs/index.md).
 
 ## Examples
 
 <!-- starter examples derived from examples/*/metadata.json and examples/index.ts -->
 
-- [Twenty Questions](examples/twenty-questions) shows a model choosing legal events in a loop.
-- [Chameleon](examples/chameleon) combines hidden information, sequential turns, and a human vote.
-- [Snapshot migration](examples/snapshot-migration) uses XState's native `version` and `migrate` contract.
-- [Portable XState loop](examples/portable-xstate-loop) runs the artifact through native transitions and effects.
-- [Application-owned storage](examples/file-snapshot-store) persists native XState snapshots in application code.
-- [Tool calling](examples/review-tool-calls) keeps tools and message history in the AI SDK's native format.
-- [Ticket triage](examples/triage) returns structured data from a model request.
-- [JSON agent](examples/json-agent) runs a machine defined as data.
+- [Twenty Questions](https://github.com/statelyai/agent/tree/next/examples/twenty-questions) shows a model choosing legal events in a loop.
+- [Chameleon](https://github.com/statelyai/agent/tree/next/examples/chameleon) combines hidden information, sequential turns, and a human vote.
+- [Snapshot migration](https://github.com/statelyai/agent/tree/next/examples/snapshot-migration) uses XState's native `version` and `migrate` contract.
+- [Portable XState loop](https://github.com/statelyai/agent/tree/next/examples/portable-xstate-loop) runs the artifact through native transitions and effects.
+- [Application-owned storage](https://github.com/statelyai/agent/tree/next/examples/file-snapshot-store) persists native XState snapshots in application code.
+- [Tool calling](https://github.com/statelyai/agent/tree/next/examples/review-tool-calls) keeps tools and message history in the AI SDK's native format.
+- [Ticket triage](https://github.com/statelyai/agent/tree/next/examples/triage) returns structured data from a model request.
+- [JSON agent](https://github.com/statelyai/agent/tree/next/examples/json-agent) runs a machine defined as data.
 
-See [all examples](examples/README.md).
+See [all examples](https://github.com/statelyai/agent/blob/next/examples/README.md).
 
 ## Related
 
-- [Machines](docs/machines.md)
-- [Text requests](docs/text-requests.md)
-- [Decisions](docs/decisions.md)
-- [Human in the loop](docs/human-in-the-loop.md)
-- [Testing and verification](docs/verify.md)
-- [Evals](docs/evals.md)
-- [Generating machines with an LLM](docs/generate-machines.md)
-- [Hosts and executors](docs/hosts.md)
-- [Models and providers](docs/models-and-providers.md)
-- [Use in any stack](docs/any-stack.md)
-- [Persistence](docs/persistence.md)
-- [Observability](docs/observability.md)
-- [Usage and budgets](docs/usage-and-budgets.md)
-- [Agent patterns](docs/patterns.md)
-- [Migrating from a hand-rolled loop](docs/from-a-loop.md)
-- [Coming from LangGraph](docs/langgraph-comparison.md)
+- [Machines](https://github.com/statelyai/agent/blob/next/docs/machines.md)
+- [Text requests](https://github.com/statelyai/agent/blob/next/docs/text-requests.md)
+- [Decisions](https://github.com/statelyai/agent/blob/next/docs/decisions.md)
+- [Human in the loop](https://github.com/statelyai/agent/blob/next/docs/human-in-the-loop.md)
+- [Testing and verification](https://github.com/statelyai/agent/blob/next/docs/verify.md)
+- [Evals](https://github.com/statelyai/agent/blob/next/docs/evals.md)
+- [Generating machines with an LLM](https://github.com/statelyai/agent/blob/next/docs/generate-machines.md)
+- [Hosts and executors](https://github.com/statelyai/agent/blob/next/docs/hosts.md)
+- [Models and providers](https://github.com/statelyai/agent/blob/next/docs/models-and-providers.md)
+- [Use in any stack](https://github.com/statelyai/agent/blob/next/docs/any-stack.md)
+- [Persistence](https://github.com/statelyai/agent/blob/next/docs/persistence.md)
+- [Observability](https://github.com/statelyai/agent/blob/next/docs/observability.md)
+- [Usage and budgets](https://github.com/statelyai/agent/blob/next/docs/usage-and-budgets.md)
+- [Agent patterns](https://github.com/statelyai/agent/blob/next/docs/patterns.md)
+- [Migrating from a hand-rolled loop](https://github.com/statelyai/agent/blob/next/docs/from-a-loop.md)
+- [Coming from LangGraph](https://github.com/statelyai/agent/blob/next/docs/langgraph-comparison.md)

package/skills/add-evals/SKILL.md

@@ -0,0 +1,144 @@
+---
+name: add-evals
+description: Write evals for a @statelyai/agent or XState agent machine at the right level — rule checks with no model, decision evals at one decision state, outcome evals over whole runs, and per-state stats from traces — each tied to the states it covers. Use when asked to "write evals", "add evals for the approval logic", "test the agent", "measure the agent", "build an eval suite", or "what should I eval" for an agent built as a state machine.
+---
+
+# Add evals to an agent machine
+
+The machine tells you what to eval and at which level. Every eval names the states it covers, so a failing eval points at a part of the machine.
+
+```
+read the machine → map the request to states → pick the level per state → write the evals → baseline → hand back
+```
+
+This is step 3 of 4. Before: `loop-to-machine`, `add-traces`. Next: `improve-machine`.
+
+## Hard rules
+
+- **The cheapest level that can catch the failure.** A rule is checked without a model. Only judgment needs a model in the eval.
+- **Ground truth, never a model's opinion of itself.** Outcome evals read terminal states and effects in the world. A model grades text only where no ground truth exists, and that eval says so in its name.
+- **Labels belong to people.** Draft expected outcomes and expected events; ask the user to confirm them before they gate anything. Keep drafts in their own list, marked unconfirmed, and out of every gate.
+- **Never edit the machine to make an eval pass.** A failing eval is a finding. Report it.
+- **Held-out scenarios stay held out.** Evals may run on them; nobody tunes against them.
+
+## 1. Read the machine
+
+```ts
+import { describeMachine } from "@statelyai/agent";
+console.log(describeMachine(machine)); // goal, states, transitions, events
+```
+
+List:
+
+- the decision states: an `agent.decide` invoke, a `decision` tag, or a state a host answers with `decideNext`. A machine whose model choice is the output of a text request (a tool call it then checks) has no decision states: eval that choice at the **request** level with `runSeam`;
+- the person's steps: idle states a person answers (`meta.interaction`, the runner's `onIdle`). What the person decides is ground truth, not something to eval; the routing after their answer is a rule;
+- the guards. `describeMachine` shows where a transition function can go and marks it `(conditional)`, but not the guard's name or logic. Read the source for those;
+- the sentinel states and the final states.
+
+## 2. Map the request to states
+
+Turn the user's words into states. "Evals for the approval logic" means every state and guard that touches approval: the guard that routes a refund over the threshold, the approval step and each answer to it, the sentinel check before the payment, and the final states they lead to. Write the mapping down:
+
+| Asked about | States and guards                                                                                  |
+| ----------- | -------------------------------------------------------------------------------------------------- |
+| approval    | `deciding` (REFUND: over threshold → `awaitingApproval`), `awaitingApproval` (APPROVE, REJECT), `paying` (sentinel check), `issued`, `closed` |
+
+If the request names nothing ("write evals"), cover every decision state, every sentinel, and the run as a whole.
+
+## 3. Pick the level
+
+| Level        | Catches                                              | Runs a model          | Tools                                                              |
+| ------------ | ---------------------------------------------------- | --------------------- | ------------------------------------------------------------------ |
+| **Rule**     | A guard, an ordering, an invariant that must hold    | No                    | Exported guard functions, `canReach`, `explorePaths`, `simulateAgent` |
+| **Decision** | A wrong choice at one decision state                 | The decider under test | The decider called on recorded or written decision requests        |
+| **Request**  | A bad text output from one request                   | That request          | `runSeam`, scored on `seamOutput` and the path after it. Not applicable when the machine has no text requests; say so |
+| **Outcome**  | A wrong end result for a whole scenario              | The whole agent       | Scenarios, the runner, an objective over terminal states and effects |
+| **Stats**    | Which state the failures cluster in                  | No (reads traces)     | `foldTraces`                                                       |
+
+A guard or ordering is always a rule eval. Use a decision eval when the guards leave more than one legal option. Use an outcome eval when a person can label the right end result.
+
+Draft labels gate nothing, but rule evals need no labels, so they gate CI from day one. The label-based evals start gating as the user confirms labels (`draft: false`). Say so in the hand-back, so nobody reads a green CI as "the outcome evals pass".
+
+## 4. Write them
+
+Name every eval `<state>: <what it checks>`, so the report reads as a map of the machine. An outcome eval over a whole run takes the final state it expects (`issued: partial-refund refunds 50 dollars`). In `test.each` titles, `$name` is a placeholder: write amounts as `100 dollars`, not `$100`.
+
+**Rule evals**, from `@statelyai/agent/testing`:
+
+```ts
+test("paying: no refund over 100 dollars issues without approval", async () => {
+  const result = await canReach(machine, "<sentinel>", { input, payloads, outputs, errors });
+  expect(result.complete).toBe(true); // reachable: false is a proof only when the search finished
+  expect(result.reachable, JSON.stringify(result.witness)).toBe(false);
+});
+
+test("deciding: REFUND over the order amount never reaches paying", async () => {
+  const result = await canReach(machine, (s) => s.matches("paying") && s.context.amount > s.context.order.amount, { input, payloads, outputs });
+  expect(result.complete).toBe(true);
+  expect(result.reachable).toBe(false);
+});
+```
+
+Check guards through the machine, not by importing the guard and calling it: an imported `refundIsLegal` is always the one in the source, so it passes on a machine built with a loosened guard. A unit test of the guard function is fine as well, never instead.
+
+Use `explorePaths` to assert the set of reachable final states, and `await simulateAgent(...)` for one scripted path. Cover both sides of every threshold. The two treat a closed choice differently:
+
+- `explorePaths` and `canReach` use `options.payloads` **instead of** the decision's own offered options, so they explore values the machine would never offer. That is what a boundary proof needs.
+- `simulateAgent` keeps the offered options: a scripted event the decision did not offer is refused, and the run ends in the decision's `onError`. Script only offered events, or pick an input that offers the value.
+
+**Decision evals.** A row is a decision request and the event a person says is right. Write rows by hand, or take them from recorded traces with `getDecisionRequests(trace, { name })` from `@statelyai/agent/testing`: each decision's first attempt, with the states it was asked in, its `options`, and its `metadata`. Compare events without caring about key order. If the decider reads anything outside the request (the ticket, the tool history), store that in the row too.
+
+```ts
+import { getDecisionRequests } from "@statelyai/agent/testing";
+
+// One row per recorded decision; `expected` is the label a person confirmed for that decision.
+// A scenario can decide more than once, so label each decision by its position, not the scenario alone.
+const rows = traces.flatMap(({ scenario, trace }) =>
+  getDecisionRequests(trace, { name: "chooseResolution" }).map(({ request }, i) => ({
+    id: `${scenario.id}#${i}`,
+    request,
+    expected: labels[`${scenario.id}#${i}`],
+  })),
+);
+
+test.each(rows)("deciding: $id", async ({ request, expected }) => {
+  const { event } = await decide(request); // the policy decider, or the model executor
+  expect(event).toEqual(expected);
+});
+```
+
+Which deciders gate CI: a deterministic reference decider (a scripted or rules-based one) gates. A live model is a measurement, not a unit test: run it outside CI, several times, and report the rate. A seeded random decider is measured over several seeds, never gated.
+
+**Request evals.** For a text request (a drafted question, a summary), `runSeam(machine, { input, scripts, seam: { request: "draftQuestion" }, candidate })` runs the whole machine with only that request live. Score `seamOutput`, and the states the run reached after it.
+
+**Outcome evals.** One function over a batch of scenario runs, reading ground truth. Score the effects as well as the outcome: a run that ends `issued` with the wrong amount is wrong.
+
+```ts
+export function objective(runs: Array<{ scenario: Scenario; outcome: string; refund?: Refund; violations: number; modelCalls: number }>) {
+  return {
+    accuracy: runs.filter((r) => r.outcome === r.scenario.expected && sameRefund(r.refund, r.scenario.expectedRefund)).length / runs.length,
+    violations: runs.reduce((n, r) => n + r.violations, 0), // must be 0
+    modelCalls: runs.reduce((n, r) => n + r.modelCalls, 0), // cost
+  };
+}
+```
+
+Read the terminal state from the event logs `add-traces` recorded (`replay(machine, entries).snapshot`, `replay` from `@statelyai/agent/log`). Effects and violations live in the world, not the log: save them next to the thread id when the run ends, so the objective can be recomputed without running anything. Use `matchesTrajectory` only for an ordering a person requires (approval before payment), never to pin the whole path; build its input from the `machine.transition` trace events' `snapshot.value`.
+
+**Stats.** `foldTraces(traces, { failed })` (from `@statelyai/agent/testing`) returns `{ state, visits, failures, withheld }` per state. A failed run counts against **every** state it visited, so states after a bad decision rank as high as the decision itself. Use the table to see which states failing runs pass through, then look at the decision states among them, their `withheld` counts, and their decision attempts. `failed` sees only the trace: to judge a run against its label, keep a map from trace to scenario.
+
+## 5. Coverage
+
+If every decider scores the same on the confirmed scenarios, the scenarios cannot tell a better decider from a worse one. Draft scenarios on the boundaries the evals found (an amount just under and over a threshold, a partial request), label them as drafts, and ask the user to confirm them and add them to the scenario set. They become part of the train or held-out split only once confirmed.
+
+## 6. Baseline
+
+Run every eval once on the current machine and record the numbers: which pass, the outcome scores per split, the stats table. Failing evals are findings, not bugs in the eval, unless the label was wrong; then ask.
+
+## 7. Hand back
+
+1. The state mapping from step 2.
+2. The evals by level, each named by state.
+3. The baseline numbers, train and held-out separately.
+4. Every failing eval and the state it points at.
+5. Next step: `improve-machine`, with the objective from step 4 as its scorer.

package/skills/add-traces/SKILL.md

@@ -0,0 +1,147 @@
+---
+name: add-traces
+description: Make every run of a @statelyai/agent or XState machine leave a replayable, stamped trace — an event log thread plus trace events — so evals and self-improvement can read what happened and what produced it. Use when asked to "add tracing", "add logging", "persist runs", "stamp traces", "record what the agent did", or "set up observability" for an agent machine, or before writing evals or improving a policy from traces.
+---
+
+# Add traces to an agent machine
+
+Every run should leave two records, both stamped with what produced it:
+
+| Record           | What it is                                                                     | Read by                                              |
+| ---------------- | ------------------------------------------------------------------------------ | ---------------------------------------------------- |
+| Event log thread | The external inputs the machine consumed, in order. The source of truth        | `replay`, `getDecisionAttempts`, `getUsageFromEvents` |
+| Trace events     | Observations: transitions with `from`, decision requests with offered options  | `foldTraces`, OpenTelemetry, debugging               |
+
+```
+version the machines → pick a store → name threads → stamp → collect trace events → prove replay → hand back
+```
+
+This is step 2 of 4. Before: `loop-to-machine`. Next: `add-evals`, then `improve-machine`.
+
+## Hard rules
+
+- **Traces never change behavior.** Recording is an observer. No branch reads a trace. An `onTrace` or `onTransition` handler that throws is reported with `console.error` and skipped. The store is different: it is write-ahead, so a failed write stops the run with `cause: "journal"`. Ask the user whether a run should fail when its log cannot be written (the store) or carry on unrecorded (record from `onEvent` instead).
+- **A trace must name what produced it.** A run with no machine version or policy version cannot be used to compare versions, so it is useless to the improver.
+- **Replay is the test.** A log that does not replay to the run's final state with zero model calls is not a trace yet.
+- **No secrets or personal data in metadata or thread ids.** Use ids, not names or emails. The log itself holds the machine's input and every event verbatim (a ticket's text, a customer's reply), and trace snapshots hold the context: in production, redact or encrypt them per your data rules, and ask the user which applies.
+
+## 1. Version the machines
+
+Declare `version` on every machine whose runs you record: the environment and, if there is one, the policy machine.
+
+```ts
+export const deskMachine = deskSetup.createMachine({ id: "refund-desk", version: "1", ... });
+```
+
+Also name the **decider version**: whatever produces the model's answers. It is the policy machine's `version` when there is one. Otherwise it is a label for the model, its settings, and its prompts (`gpt-6-luna.xhigh.p<hash of the prompts>`); this holds for a machine with only text requests too. When the prompts are private to another module (a route file), export a prompt version from that module rather than guessing; a release or commit id is the fallback. In tests it is the stand-in model's name (`careful`). Runs by different deciders must never share a label.
+
+- Every log entry carries `machineVersion`. Without a declared `version` it falls back to a structural hash that skips functions, and in XState v6 the guards and targets live in transition functions. Changing a guard would not change the hash.
+- Bump `version` on every behavior change. Adding recording is not one: it only observes. `improve-machine` bumps the policy's on promotion.
+- A test-only variant built from the same factory (a loosened guard) gets its own `version` (`"1-loose"`), so its runs are never mistaken for the real machine's.
+
+## 2. Pick a store
+
+- **Tests and local runs:** `createInMemoryEventLogStore()` from `@statelyai/agent/log`, or write each thread's `result.events` to a JSON file after the run.
+- **Production:** implement `AgentEventLogStore` against your database and check it with `assertEventLogStoreConformance`. Implement the optional `listThreads` so a split can be paged by prefix.
+
+Pass it with a `threadId`. The store is write-ahead: no model call starts until the entries before it are durable.
+
+```ts
+const result = await runAgent(deskMachine, { input, executors, store, threadId, metadata });
+await result.drain(); // late usage entries, before you read the store or exit
+```
+
+**A thread that already has entries is resumed, not overwritten.** Give every new run a new thread id; reuse one only to continue that run. Passing different `input` to a thread that already has entries throws `thread-input-mismatch` instead of handing back another run's result.
+
+For a production store, use a table of its own (`threadId`, `index`, the entry as JSON, a unique key on both), not a table that already means something else, such as a usage counter.
+
+## 3. Name threads
+
+A thread id is how runs are found later. Use a prefix scheme and keep it in one function:
+
+| Runs                     | Thread id                                    |
+| ------------------------ | -------------------------------------------- |
+| Scenario runs for evals  | `<split>:<deciderVersion>:<scenarioId>:r<k>`, `k` = the repeat index |
+| Production               | `prod:<domain id>` (ticket id, session id)   |
+
+`store.listThreads?.({ prefix: "train:careful:" })` then pages through one split of one decider. `listThreads` is optional on a custom store.
+
+## 4. Stamp each run
+
+`metadata` lands on the log's first (`@agent.init`) entry. Put there everything needed to tell two runs' producers apart:
+
+```ts
+const metadata = {
+  environmentVersion: deskMachine.version ?? "unversioned",
+  deciderVersion, // see step 1
+  models: { decider: "openai/gpt-6-luna" }, // the machine's model refs → what the host resolved them to
+  scenario: scenario.id, // eval runs only
+  split: scenario.split, // eval runs only
+};
+```
+
+Read it back with `entries[0].metadata`. It also holds `executionId` (and `migratedFrom` after a version bridge), which the library adds and reserves, so compare with `toMatchObject`, not `toEqual`. A resumed run keeps the stamp of the run that started the log. Type the stamp as `Record<string, JsonValue>` (`JsonValue` from `@statelyai/agent/log`); an interface without an index signature does not fit.
+
+## 5. Collect trace events
+
+`onTrace` sees every trace event. Serialize and keep them per thread:
+
+```ts
+import { serializeTraceEvent, type JsonSerializableTraceEvent } from "@statelyai/agent";
+
+const trace: JsonSerializableTraceEvent[] = [];
+const result = await runAgent(deskMachine, { ..., onTrace: (event) => trace.push(serializeTraceEvent(event)) });
+await result.drain();
+
+// For local batches: one file per thread. Encoding keeps colons and path separators out of the name.
+const file = encodeURIComponent(threadId);
+await mkdir("runs", { recursive: true });
+await writeFile(`runs/${file}.events.json`, JSON.stringify(result.events));
+await writeFile(`runs/${file}.trace.json`, JSON.stringify(trace));
+```
+
+Decide with the user whether `runs/` is committed or gitignored.
+
+- In production, send them to OpenTelemetry: `createOtelTraceHandler({ tracer })` from `@statelyai/agent/otel` is an `onTrace` handler. It needs the optional peer `@opentelemetry/api`; without OpenTelemetry in the app, write the serialized events next to the log instead.
+- **A policy run inside a decider** is a separate run. Pass it the same `onTrace` (and a `store`) if you want per-state stats for the policy too. `createPolicyDecider` gives each decision's policy run its own thread, `policy:<callKey>:<SHA-256 of its input>`; never give it one fixed `threadId`, or the second decision resumes the first.
+- **A plain XState machine** (built with `createMachine` or `setup`, decided with `decideNext`): drive it with `createAgentRuntime` and `runToQuiescence`, which accept plain machines, and pass each `decideNext` result as `{ event, attempts }` so refused choices reach the log. Push `{ type: "request.start", request: result.request }` onto the trace before sending the event, since a host-driven decision emits no trace itself.
+- **A plain `createActor` run you cannot change:** attach `recordLog(actor)` from `@statelyai/agent/log` before `actor.start()`, and pass `inspect: traceTransitions((event) => trace.push(serializeTraceEvent(event)))` to `createActor`. `recordLog` cannot record decision attempts.
+
+## 6. Prove it
+
+Write these tests, with the stand-in model from `loop-to-machine`:
+
+```ts
+import { getDecisionAttempts, replay } from "@statelyai/agent/log";
+import { foldTraces } from "@statelyai/agent/testing";
+
+test("a recorded run replays to its final state with no model calls", async () => {
+  const result = await runAgent(deskMachine, { input, executors, store, threadId: "test:careful:small-clean:r1", metadata });
+  await result.drain();
+  const entries = await store.read("test:careful:small-clean:r1");
+  expect(entries[0]?.event.type).toBe("@agent.init"); // a log was written at all; a run with no outside input has only this entry
+  const { snapshot } = replay(deskMachine, entries, { verify: "strict" }); // a pure fold: nothing runs
+  // Compare more than the state value: context and output catch a field the log does not keep.
+  expect({ status: snapshot.status, value: snapshot.value, context: snapshot.context, output: snapshot.output }).toEqual({
+    status: result.snapshot.status,
+    value: result.snapshot.value,
+    context: result.snapshot.context,
+    output: result.snapshot.output,
+  });
+  expect(entries[0]!.metadata).toMatchObject({ environmentVersion: "1" });
+});
+```
+
+`verify: "strict"` checks that every entry carries a hash and that each matches the fold. It does not show the log is complete (a dropped last entry leaves the rest intact), nor that the fold matches the live run, which is why the test compares the snapshots too. A journaled `Error` replays as an `Error` with its `name`, `message`, and a string or number `code`; any other property of the thrown value is not kept. A thrown plain object replays as a plain object.
+
+- `foldTraces(traces, { failed })` over a batch returns a row for every state a run rested in. A `choice` state the run passes through in the same step gets no row. `failed` judges one whole trace; start with "the last event is not a `run.end` with status `done`, or the run ended in a failure or sentinel state", and ask the user whether a wrong outcome counts too. A serialized `run.end` is `{ type: "run.end", status, snapshot: { value, context, … } }`: `failed: (t) => { const end = t.at(-1); return end?.type !== "run.end" || end.status !== "done" || ["failed", "invariantBroken"].includes(String(end.snapshot?.value)); }`. `withheld` is above 0 only for a closed choice (`payloads`), where the guards refused an option.
+- `getDecisionAttempts(entries)` lists the refused choices of a run; for a machine with no decisions (`agent.decide` or `decideNext`) it is always empty, so skip this test. A stand-in model that picks from the offered options can never propose an illegal one, so write a raw decide executor for this test that proposes a refused event first: `decide: async (request) => ({ event: request.attempts.length ? request.options![0]!.event : { type: "REFUND", amount: 1000 } })`.
+- Running a scenario with and without recording leaves the same world, the same model calls, and the same final state, and the recorded run's log is not empty.
+
+## 7. Hand back
+
+1. Where runs are stored and how thread ids are built.
+2. The stamp fields.
+3. Where trace events go.
+4. The replay test passing.
+5. Next step: `add-evals`.