@statelyai/agent 2.0.0-alpha.40 → 2.0.0-alpha.41

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -807,6 +807,65 @@ function scriptDryError(kind, src, id, request) {
807
807
  const MAX_ADVANCE_STEPS = 1e3;
808
808
  const FINITE_PAYLOAD_CAP = 64;
809
809
  const DEFAULT_MAX_NODES = 1e5;
810
+ function isDataProperty(target, key) {
811
+ const descriptor = Object.getOwnPropertyDescriptor(target, key);
812
+ return descriptor !== void 0 && "value" in descriptor;
813
+ }
814
+ function strictKey(value, refs = /* @__PURE__ */ new Map()) {
815
+ if (value === void 0) return "u";
816
+ if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value);
817
+ if (typeof value === "number") return Object.is(value, -0) ? "n-0" : `n${String(value)}`;
818
+ if (typeof value !== "object") throw new TypeError(`unkeyable ${typeof value}`);
819
+ const ref = refs.get(value);
820
+ if (ref !== void 0) return `@${ref}`;
821
+ refs.set(value, refs.size);
822
+ const proto = Object.getPrototypeOf(value);
823
+ const own = Reflect.ownKeys(value).length;
824
+ if (Array.isArray(value)) {
825
+ if (proto !== Array.prototype) throw new TypeError("unkeyable array");
826
+ for (let index = 0; index < value.length; index++) if (!isDataProperty(value, index)) throw new TypeError("unkeyable sparse array");
827
+ if (own !== value.length + 1) throw new TypeError("unkeyable array");
828
+ return `[${value.map((item) => strictKey(item, refs)).join(",")}]`;
829
+ }
830
+ if (proto !== Object.prototype && proto !== null) throw new TypeError("unkeyable object");
831
+ const keys = Object.keys(value);
832
+ if (own !== keys.length || !keys.every((k) => isDataProperty(value, k))) throw new TypeError("unkeyable object");
833
+ return `${proto === null ? "N" : ""}{${keys.map((k) => `${JSON.stringify(k)}:${strictKey(value[k], refs)}`).join(",")}}`;
834
+ }
835
+ function tryKey(key) {
836
+ try {
837
+ return key();
838
+ } catch {
839
+ return;
840
+ }
841
+ }
842
+ function stepKeyParts(step) {
843
+ const snapshot = step.snapshot;
844
+ const children = snapshot.children ?? {};
845
+ return [
846
+ snapshot.value,
847
+ snapshot.context,
848
+ snapshot.status,
849
+ snapshot.historyValue ?? null,
850
+ step.requests.map((request) => request.id),
851
+ pendingInvokes(step).map((invoke) => [
852
+ invoke.id,
853
+ invoke.src,
854
+ invoke.input
855
+ ]),
856
+ Object.keys(children).sort().map((id) => [id, children[id]?.options?.input]),
857
+ (snapshot.timers ?? []).map((timer) => [
858
+ timer["id"],
859
+ timer["type"],
860
+ timer["delay"],
861
+ timer["event"],
862
+ timer["target"] === "self" ? "self" : timer["target"]?.id
863
+ ])
864
+ ];
865
+ }
866
+ function defaultStepKey(step) {
867
+ return strictKey(stepKeyParts(step));
868
+ }
810
869
  const MAX_CHILD_NESTING = 4;
811
870
  const DEFAULT_BUDGET_NODES = 25e4;
812
871
  const DEFAULT_CHILD_CACHE = 64;
@@ -1052,7 +1111,7 @@ function finishedChildren(step) {
1052
1111
  return [];
1053
1112
  });
1054
1113
  }
1055
- function explore(machine, options, stopWhen, nesting = 0, deliverable, childRef, trackTransits = nesting === 0 && hasTransientStates(machine.root), budget = startBudget(options.budget), stepping = false, ordering, stateTarget = false) {
1114
+ function explore(machine, options, stopWhen, nesting = 0, deliverable, childRef, trackTransits = nesting === 0 && hasTransientStates(machine.root), budget = startBudget(options.budget), stepping = false, ordering, stateTarget = false, defaultDedupe = false) {
1056
1115
  const maxDepth = options.maxDepth ?? 8;
1057
1116
  const maxPaths = options.maxPaths ?? 200;
1058
1117
  const maxNodes = options.maxNodes ?? DEFAULT_MAX_NODES;
@@ -1810,17 +1869,8 @@ function explore(machine, options, stopWhen, nesting = 0, deliverable, childRef,
1810
1869
  return `\u0003${[...kids.values()].map((kid) => `${kid.id}\u0000${kid.src}\u0000${kid.engine.stateKey(kid)}`).sort().join("")}`;
1811
1870
  };
1812
1871
  const stateKey = (kid) => {
1813
- const snapshot = kid.step.snapshot;
1814
1872
  try {
1815
- return JSON.stringify([
1816
- snapshot.value,
1817
- snapshot.context,
1818
- snapshot.status,
1819
- snapshot.historyValue ?? null,
1820
- kid.step.requests.map((request) => request.id),
1821
- Object.keys(snapshot.children ?? {}),
1822
- pumped.has(kid.step)
1823
- ]) + kidsKey(kid.kids);
1873
+ return strictKey([...stepKeyParts(kid.step), pumped.has(kid.step)]) + kidsKey(kid.kids);
1824
1874
  } catch {
1825
1875
  return `\u0002${unkeyable++}`;
1826
1876
  }
@@ -1856,8 +1906,9 @@ function explore(machine, options, stopWhen, nesting = 0, deliverable, childRef,
1856
1906
  return;
1857
1907
  }
1858
1908
  }
1859
- if (options.dedupeBy) {
1860
- const key = options.dedupeBy(withKidSnapshots(settled.snapshot, nodeKids)) + (delivered?.size ? `\u0001${[...delivered].sort().join("")}` : "") + kidsKey(nodeKids) + (logs.get(settled)?.after ? "after" : "");
1909
+ const base = options.dedupeBy ? options.dedupeBy(withKidSnapshots(settled.snapshot, nodeKids)) : defaultDedupe ? tryKey(() => defaultStepKey(settled)) : void 0;
1910
+ if (base !== void 0) {
1911
+ const key = base + (delivered?.size ? `\u0001${[...delivered].sort().join("")}` : "") + kidsKey(nodeKids) + (logs.get(settled)?.after ? "after" : "");
1861
1912
  const seenAt = visited.get(key);
1862
1913
  if (seenAt !== void 0 && seenAt <= depth) return;
1863
1914
  visited.set(key, depth);
@@ -2396,7 +2447,7 @@ async function canReach(machine, target, options = {}) {
2396
2447
  options = withMachineInput(machine, options, "canReach");
2397
2448
  const resolvedTarget = typeof target === "string" ? resolveStateTarget(machine, target) : void 0;
2398
2449
  const ordering = typeof target === "object" ? target : void 0;
2399
- const { witness, report } = explore(machine, options, typeof target === "function" ? target : ordering ? () => false : (snapshot) => snapshot.nodes.some((node) => node.id === resolvedTarget), 0, void 0, void 0, resolvedTarget !== void 0 && hasTransientStates(machine.getStateNodeById(resolvedTarget)), void 0, false, ordering, resolvedTarget !== void 0);
2450
+ const { witness, report } = explore(machine, options, typeof target === "function" ? target : ordering ? () => false : (snapshot) => snapshot.nodes.some((node) => node.id === resolvedTarget), 0, void 0, void 0, resolvedTarget !== void 0 && hasTransientStates(machine.getStateNodeById(resolvedTarget)), void 0, false, ordering, resolvedTarget !== void 0, options.dedupeBy === void 0);
2400
2451
  if (witness !== void 0) return {
2401
2452
  reachable: true,
2402
2453
  witness,
@@ -807,6 +807,65 @@ function scriptDryError(kind, src, id, request) {
807
807
  const MAX_ADVANCE_STEPS = 1e3;
808
808
  const FINITE_PAYLOAD_CAP = 64;
809
809
  const DEFAULT_MAX_NODES = 1e5;
810
+ function isDataProperty(target, key) {
811
+ const descriptor = Object.getOwnPropertyDescriptor(target, key);
812
+ return descriptor !== void 0 && "value" in descriptor;
813
+ }
814
+ function strictKey(value, refs = /* @__PURE__ */ new Map()) {
815
+ if (value === void 0) return "u";
816
+ if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value);
817
+ if (typeof value === "number") return Object.is(value, -0) ? "n-0" : `n${String(value)}`;
818
+ if (typeof value !== "object") throw new TypeError(`unkeyable ${typeof value}`);
819
+ const ref = refs.get(value);
820
+ if (ref !== void 0) return `@${ref}`;
821
+ refs.set(value, refs.size);
822
+ const proto = Object.getPrototypeOf(value);
823
+ const own = Reflect.ownKeys(value).length;
824
+ if (Array.isArray(value)) {
825
+ if (proto !== Array.prototype) throw new TypeError("unkeyable array");
826
+ for (let index = 0; index < value.length; index++) if (!isDataProperty(value, index)) throw new TypeError("unkeyable sparse array");
827
+ if (own !== value.length + 1) throw new TypeError("unkeyable array");
828
+ return `[${value.map((item) => strictKey(item, refs)).join(",")}]`;
829
+ }
830
+ if (proto !== Object.prototype && proto !== null) throw new TypeError("unkeyable object");
831
+ const keys = Object.keys(value);
832
+ if (own !== keys.length || !keys.every((k) => isDataProperty(value, k))) throw new TypeError("unkeyable object");
833
+ return `${proto === null ? "N" : ""}{${keys.map((k) => `${JSON.stringify(k)}:${strictKey(value[k], refs)}`).join(",")}}`;
834
+ }
835
+ function tryKey(key) {
836
+ try {
837
+ return key();
838
+ } catch {
839
+ return;
840
+ }
841
+ }
842
+ function stepKeyParts(step) {
843
+ const snapshot = step.snapshot;
844
+ const children = snapshot.children ?? {};
845
+ return [
846
+ snapshot.value,
847
+ snapshot.context,
848
+ snapshot.status,
849
+ snapshot.historyValue ?? null,
850
+ step.requests.map((request) => request.id),
851
+ pendingInvokes(step).map((invoke) => [
852
+ invoke.id,
853
+ invoke.src,
854
+ invoke.input
855
+ ]),
856
+ Object.keys(children).sort().map((id) => [id, children[id]?.options?.input]),
857
+ (snapshot.timers ?? []).map((timer) => [
858
+ timer["id"],
859
+ timer["type"],
860
+ timer["delay"],
861
+ timer["event"],
862
+ timer["target"] === "self" ? "self" : timer["target"]?.id
863
+ ])
864
+ ];
865
+ }
866
+ function defaultStepKey(step) {
867
+ return strictKey(stepKeyParts(step));
868
+ }
810
869
  const MAX_CHILD_NESTING = 4;
811
870
  const DEFAULT_BUDGET_NODES = 25e4;
812
871
  const DEFAULT_CHILD_CACHE = 64;
@@ -1052,7 +1111,7 @@ function finishedChildren(step) {
1052
1111
  return [];
1053
1112
  });
1054
1113
  }
1055
- function explore(machine, options, stopWhen, nesting = 0, deliverable, childRef, trackTransits = nesting === 0 && hasTransientStates(machine.root), budget = startBudget(options.budget), stepping = false, ordering, stateTarget = false) {
1114
+ function explore(machine, options, stopWhen, nesting = 0, deliverable, childRef, trackTransits = nesting === 0 && hasTransientStates(machine.root), budget = startBudget(options.budget), stepping = false, ordering, stateTarget = false, defaultDedupe = false) {
1056
1115
  const maxDepth = options.maxDepth ?? 8;
1057
1116
  const maxPaths = options.maxPaths ?? 200;
1058
1117
  const maxNodes = options.maxNodes ?? DEFAULT_MAX_NODES;
@@ -1810,17 +1869,8 @@ function explore(machine, options, stopWhen, nesting = 0, deliverable, childRef,
1810
1869
  return `\u0003${[...kids.values()].map((kid) => `${kid.id}\u0000${kid.src}\u0000${kid.engine.stateKey(kid)}`).sort().join("")}`;
1811
1870
  };
1812
1871
  const stateKey = (kid) => {
1813
- const snapshot = kid.step.snapshot;
1814
1872
  try {
1815
- return JSON.stringify([
1816
- snapshot.value,
1817
- snapshot.context,
1818
- snapshot.status,
1819
- snapshot.historyValue ?? null,
1820
- kid.step.requests.map((request) => request.id),
1821
- Object.keys(snapshot.children ?? {}),
1822
- pumped.has(kid.step)
1823
- ]) + kidsKey(kid.kids);
1873
+ return strictKey([...stepKeyParts(kid.step), pumped.has(kid.step)]) + kidsKey(kid.kids);
1824
1874
  } catch {
1825
1875
  return `\u0002${unkeyable++}`;
1826
1876
  }
@@ -1856,8 +1906,9 @@ function explore(machine, options, stopWhen, nesting = 0, deliverable, childRef,
1856
1906
  return;
1857
1907
  }
1858
1908
  }
1859
- if (options.dedupeBy) {
1860
- const key = options.dedupeBy(withKidSnapshots(settled.snapshot, nodeKids)) + (delivered?.size ? `\u0001${[...delivered].sort().join("")}` : "") + kidsKey(nodeKids) + (logs.get(settled)?.after ? "after" : "");
1909
+ const base = options.dedupeBy ? options.dedupeBy(withKidSnapshots(settled.snapshot, nodeKids)) : defaultDedupe ? tryKey(() => defaultStepKey(settled)) : void 0;
1910
+ if (base !== void 0) {
1911
+ const key = base + (delivered?.size ? `\u0001${[...delivered].sort().join("")}` : "") + kidsKey(nodeKids) + (logs.get(settled)?.after ? "after" : "");
1861
1912
  const seenAt = visited.get(key);
1862
1913
  if (seenAt !== void 0 && seenAt <= depth) return;
1863
1914
  visited.set(key, depth);
@@ -2396,7 +2447,7 @@ async function canReach(machine, target, options = {}) {
2396
2447
  options = withMachineInput(machine, options, "canReach");
2397
2448
  const resolvedTarget = typeof target === "string" ? resolveStateTarget(machine, target) : void 0;
2398
2449
  const ordering = typeof target === "object" ? target : void 0;
2399
- const { witness, report } = explore(machine, options, typeof target === "function" ? target : ordering ? () => false : (snapshot) => snapshot.nodes.some((node) => node.id === resolvedTarget), 0, void 0, void 0, resolvedTarget !== void 0 && hasTransientStates(machine.getStateNodeById(resolvedTarget)), void 0, false, ordering, resolvedTarget !== void 0);
2450
+ const { witness, report } = explore(machine, options, typeof target === "function" ? target : ordering ? () => false : (snapshot) => snapshot.nodes.some((node) => node.id === resolvedTarget), 0, void 0, void 0, resolvedTarget !== void 0 && hasTransientStates(machine.getStateNodeById(resolvedTarget)), void 0, false, ordering, resolvedTarget !== void 0, options.dedupeBy === void 0);
2400
2451
  if (witness !== void 0) return {
2401
2452
  reachable: true,
2402
2453
  witness,
package/docs/evals.md CHANGED
@@ -337,6 +337,7 @@ replay.output(await runPort(replay.client("anthropic", stand, { methods })));
337
337
  compareRecordings(recorder.recording, replay.recording); // { equivalent: true, divergences: [] }
338
338
  ```
339
339
 
340
+ - `recorder.model` records each provider attempt as a `model` effect. A failed attempt followed by the same request, such as an AI SDK retry, has `retriedBy` set to the call that retried it. Attempts made inside a `generateText` or `streamText` call that `recordAiSdk` routed carry that call's index as `aiSdkCall`, so a retry by the SDK (same `aiSdkCall`) is told apart from one by the code (a different `aiSdkCall`).
340
341
  - Each call is a `call` effect named `<name>.<method>`, such as `anthropic.messages.create`, with its arguments and its result or error.
341
342
  - `recorder.fn(name, asyncFn, options?)` wraps one async function the same way, for a client that is a function.
342
343
  - Replay answers each call with the next unused recorded call of the same name, in order, whatever its arguments. A call past the recorded ones throws `RecordingDivergenceError`.
package/docs/index.md CHANGED
@@ -11,7 +11,7 @@ Stately Agent is XState plus typed model requests, decisions, and host executors
11
11
  ## Start here
12
12
 
13
13
  - Greenfield, a new agent: [Quickstart](quickstart.md), then [Thinking in state machines](thinking-in-state-machines.md) or a copied [agent pattern](patterns.md).
14
- - Brownfield, an existing agent loop: [Migrating from a loop](from-a-loop.md), or the [`loop-to-machine`](https://github.com/statelyai/skills/blob/main/plugins/stately-agent/skills/loop-to-machine/SKILL.md) skill from [statelyai/skills](https://github.com/statelyai/skills).
14
+ - Brownfield, an existing agent loop: [Verify in place](verify-in-place.md) to check it against a model without changing it, or [Migrating from a loop](from-a-loop.md), or the [`loop-to-machine`](https://github.com/statelyai/skills/blob/main/plugins/stately-agent/skills/loop-to-machine/SKILL.md) skill from [statelyai/skills](https://github.com/statelyai/skills).
15
15
  - Brownfield, an existing async workflow: [Strangler-fig retrofit](strangler-fig.md). Wrap it unchanged with `wrapWorkflow`, then move one phase at a time into states, checked with `checkParity`.
16
16
  - [Choosing a run mode](choosing-a-run-mode.md)
17
17
  - [Advanced: bare XState actors and hand-written executors](advanced.md)
@@ -35,7 +35,7 @@ Stately Agent is XState plus typed model requests, decisions, and host executors
35
35
  | ------------------------------ | ------------------------------------------------------------------------------------------ |
36
36
  | `@statelyai/agent` | Authoring, running, human interaction, and the executor contract |
37
37
  | `@statelyai/agent/evals` | Trace analytics: `foldTraces`, `traceCoverage`, `repeatRuns`, `judgeAgreement` |
38
- | `@statelyai/agent/testing` | `lintAgentMachine`, `simulateAgent`, `canReach`, `checkParity`, trajectories, seams |
38
+ | `@statelyai/agent/testing` | `lintAgentMachine`, `canReach`, `checkConformance`, `checkParity`, trajectories, seams |
39
39
  | `@statelyai/agent/log` | The event log: `replay`, `forkEventLog`, `getDecisionAttempts`, hand-built entries, stores |
40
40
  | `@statelyai/agent/ai-sdk` | `createAiSdkExecutors`: the Vercel AI SDK adapter, for language and decision models |
41
41
  | `@statelyai/agent/openai` | Executors over the raw `openai` package |
package/docs/meta.json CHANGED
@@ -5,6 +5,7 @@
5
5
  "index",
6
6
  "quickstart",
7
7
  "thinking-in-state-machines",
8
+ "verify-in-place",
8
9
  "from-a-loop",
9
10
  "strangler-fig",
10
11
  "scope",
@@ -0,0 +1,326 @@
1
+ ---
2
+ title: Verify in place
3
+ description: Model an existing agent workflow, prove the model's rules, and check recorded runs of the unchanged code against it. Migrating the code to a machine is optional.
4
+ ---
5
+
6
+ > **Alpha:** `@statelyai/agent` 2.0 is in alpha. APIs can change between releases; pin an exact version. Feedback: [github.com/statelyai/agent](https://github.com/statelyai/agent/issues).
7
+
8
+ There are two ways to use `@statelyai/agent`:
9
+
10
+ - **Author** a new workflow as a machine. The machine is the code, and it enforces its rules at runtime. Start with the [Quickstart](quickstart.md).
11
+ - **Verify in place.** Keep the code you have, write a model of its control flow, prove the model's rules, and check recorded runs of the code against the model. This page covers that track.
12
+
13
+ You can migrate the code to a machine later. The model and its proofs carry over. See [Migrate later](#migrate-later).
14
+
15
+ The worked example is [examples/verify-in-place](https://github.com/statelyai/agent/tree/next/examples/verify-in-place). Every snippet on this page comes from it.
16
+
17
+ ## What you get
18
+
19
+ - A model file in the repo, reviewed like other code.
20
+ - Proofs of the model's rules that run in CI with no model calls.
21
+ - A check that recorded runs of the code conform to the model, and a report of the first step where a run does not.
22
+ - Coverage: the states and transitions of the model that no recorded run reached.
23
+
24
+ The claim is: the model is proved, and these recorded runs conform to it. A run you did not record can still break a rule. After a migration, the machine enforces the rules, and the claim covers every run.
25
+
26
+ ## The existing code
27
+
28
+ The example is a refund agent. Every side effect goes through one `effects` object, and the rules live in `if` statements:
29
+
30
+ ```ts no-check
31
+ export async function handleRefundRequest(
32
+ fx: RefundEffects,
33
+ request: { orderId: string; message: string },
34
+ ): Promise<RefundOutcome> {
35
+ const order = await fx.lookupOrder(request.orderId);
36
+ for (let decisions = 0; decisions < MAX_DECISIONS; decisions++) {
37
+ const decision = await fx.decide({ request: request.message, order, decisions });
38
+ if (decision.type === "reply") return "replied";
39
+ if (decision.type === "investigate") continue;
40
+ if (decision.amount > order.total) return "over-total";
41
+ if (decision.amount > APPROVAL_THRESHOLD) {
42
+ const approved = await fx.requestApproval({
43
+ orderId: order.orderId,
44
+ amount: decision.amount,
45
+ });
46
+ if (!approved) return "declined";
47
+ }
48
+ await fx.issueRefund({ orderId: order.orderId, amount: decision.amount });
49
+ return "refunded";
50
+ }
51
+ return "escalated";
52
+ }
53
+ ```
54
+
55
+ The check does not edit this file. Code whose side effects are spread across modules needs a seam first: route them through one object, or record them at the client with `createRecorder`'s `recorder.client`, `recorder.model` for an AI SDK model, and `recorder.tools` for AI SDK tools. See [Evals](evals.md#any-model-client).
56
+
57
+ ### AI SDK code
58
+
59
+ Code that builds its model inline (`openai("gpt-…")`) and passes module-level tools to `generateText` has nothing to inject. Mock `"ai"` in the test and route its calls through the recorder with `recordAiSdk`:
60
+
61
+ ```ts no-check
62
+ // seam.ts: must not import "ai", or the mock below waits on itself.
63
+ let current: AiSdkSeam | undefined;
64
+ export const currentSeam = () => current;
65
+ export function startRecording(standIn: LanguageModelV3) {
66
+ const recorder = createRecorder();
67
+ current = { recorder, model: recorder.model(standIn) };
68
+ return recorder;
69
+ }
70
+
71
+ // support-agent.test.ts
72
+ vi.mock("ai", async (importOriginal) => {
73
+ const ai = await importOriginal<typeof import("ai")>();
74
+ const { recordAiSdk } = await import("@statelyai/agent/testing");
75
+ const { currentSeam } = await import("./seam");
76
+ return { ...ai, ...recordAiSdk(ai, currentSeam) };
77
+ });
78
+ ```
79
+
80
+ - Each `generateText` and `streamText` call runs on the real AI SDK with the seam's `model` in place of the code's, and its `tools` wrapped with `recorder.tools`. The code's own tool loop runs.
81
+ - A call made while `currentSeam()` returns `undefined` throws, so no test reaches a real provider.
82
+ - A tool that has `execute` runs inside the `generateText` call. A check the code makes after the call returns comes after the tool already ran: model the order the calls happen in.
83
+ - `recorder.model` records each provider attempt, and `effect.call` counts attempts. `recordAiSdk` stamps each attempt of a `recorder.model` wrapper from `createRecorder` with `aiSdkCall`, the `generateText` or `streamText` call it belongs to, including steps that use a model from `prepareStep`: attempts with the same `aiSdkCall` are one call the code made, its tool-loop steps and the SDK's retries.
84
+ - A failed attempt followed by the same request has `retriedBy` set to the attempt that resent it. When that attempt has the same `aiSdkCall`, the SDK retried it (`maxRetries`): skip it in the mapping. When it has a different `aiSdkCall`, the code resent it: keep it, so the model checks the code's retry path and its call limit. An attempt with no `aiSdkCall` is one whose resender is unknown: keep it. That is any attempt not routed through `recordAiSdk`, any routed attempt whose model is not a `recorder.model` wrapper passed as is (a proxy around one, or one with a replaced method), and any attempt recorded through a recorder `createRecorder` did not make.
85
+
86
+ ```ts no-check
87
+ const sdkRetried = (effect: RecordedModelCall, effects: RecordedEffect[]) =>
88
+ effect.aiSdkCall !== undefined &&
89
+ effect.retriedBy !== undefined &&
90
+ effects.some(
91
+ (e) => e.kind === "model" && e.call === effect.retriedBy && e.aiSdkCall === effect.aiSdkCall,
92
+ );
93
+ ```
94
+
95
+ ## Step 1: the model file
96
+
97
+ Write the same control flow as a machine in `model.ts`. Each unit of work is an actor declared by name, with a placeholder implementation, so the model runs none of the code. Each rule is a state or a guard:
98
+
99
+ ```ts no-check
100
+ deciding: {
101
+ invoke: {
102
+ src: "decide",
103
+ input: ({ context }) => ({ decisions: context.decisions }),
104
+ onDone: ({ context, event }) => {
105
+ const decision = event.output;
106
+ const decisions = context.decisions + 1;
107
+ if (decision.type === "reply") return { target: "replied", context: { decisions } };
108
+ if (decision.type === "investigate")
109
+ return { target: "checking", context: { decisions } };
110
+ const amount = decision.amount;
111
+ if (amount > context.order!.total)
112
+ return { target: "overTotal", context: { decisions, amount } };
113
+ return amount > APPROVAL_THRESHOLD
114
+ ? { target: "awaitingApproval", context: { decisions, amount } }
115
+ : { target: "refunding", context: { decisions, amount } };
116
+ },
117
+ },
118
+ },
119
+ awaitingApproval: {
120
+ on: {
121
+ APPROVE: { target: "refunding", context: { approved: true } },
122
+ REJECT: { target: "declined" },
123
+ },
124
+ },
125
+ ```
126
+
127
+ - Declare every actor in `setupAgent({ actors })` and invoke it by `src`. `checkConformance` answers named actors from the recording, and rejects a model whose invoke names its logic inline.
128
+ - A person's input, such as an approval, is an event the model waits for in an idle state.
129
+ - The `stately` skill in [statelyai/skills](https://github.com/statelyai/skills) can draft a model from code. Review it like code: the proofs below hold for the model, not for behavior the model leaves out.
130
+
131
+ ## Step 2: proofs
132
+
133
+ Prove the model's rules with `canReach`, as ordinary tests. Give each actor the outputs it can return:
134
+
135
+ ```ts no-check
136
+ const proofOptions = {
137
+ input: { orderId: "P-1" },
138
+ outputs: {
139
+ lookupOrder: [{ orderId: "P-1", total: 300 }],
140
+ decide: [
141
+ { type: "refund", amount: 50 },
142
+ { type: "refund", amount: 250 },
143
+ { type: "refund", amount: 400 },
144
+ { type: "reply", message: "ok" },
145
+ { type: "investigate" },
146
+ ],
147
+ issueRefund: [undefined],
148
+ },
149
+ maxDepth: 12,
150
+ };
151
+
152
+ const result = await canReach(
153
+ refundModel,
154
+ (snapshot) =>
155
+ snapshot.matches("refunding") &&
156
+ snapshot.context.amount > APPROVAL_THRESHOLD &&
157
+ !snapshot.context.approved,
158
+ proofOptions,
159
+ );
160
+ // { reachable: false, complete: true }
161
+ ```
162
+
163
+ - `reachable: false` is a proof only with `complete: true`, and only over the outputs and payloads you list. The search forks on those values and no others: a refund of 150 is not explored unless `outputs` lists it. List each rule's boundary values (just under and over a threshold, the cap reached).
164
+ - Check that the search reaches the path the rule guards, for example a large refund through approval. Otherwise a model where that path does not exist proves the rule vacuously.
165
+ - See [Testing and verification](verify.md#reachability-checks) for payloads, ordering targets, and bounds.
166
+
167
+ ## Step 3: record runs of the code
168
+
169
+ Wrap each function of the `effects` object with `recorder.fn` and run the code as it is:
170
+
171
+ ```ts no-check
172
+ const recorder = createRecorder();
173
+ const outcome = await handleRefundRequest(
174
+ {
175
+ lookupOrder: recorder.fn("lookupOrder", lookupOrder),
176
+ decide: recorder.fn("decide", decide),
177
+ requestApproval: recorder.fn("requestApproval", requestApproval),
178
+ issueRefund: recorder.fn("issueRefund", issueRefund),
179
+ },
180
+ request,
181
+ );
182
+ recorder.output(outcome); // the return value, checked in step 5
183
+ const recording = recorder.recording;
184
+ ```
185
+
186
+ The example records scripted scenarios with stand-in functions. A recording from a real run works the same way; save it as JSON and check it later.
187
+
188
+ ## Step 4: map recorded calls to observations
189
+
190
+ An observation is one thing the run did, in the model's terms. Map the recording's effects with `flatMap`:
191
+
192
+ ```ts no-check
193
+ function toObservations(recording: Recording): Observation[] {
194
+ return recording.effects.flatMap((effect): Observation[] => {
195
+ if (effect.kind === "output") return [{ kind: "returned", outcome: effect.value }];
196
+ if (effect.kind !== "call") return [];
197
+ switch (effect.name) {
198
+ // A call that threw carries its error, so its actor fails with it.
199
+ case "lookupOrder":
200
+ return [
201
+ {
202
+ kind: "lookup",
203
+ orderId: (effect.args as [string])[0],
204
+ order: effect.result as Order,
205
+ error: effect.error,
206
+ },
207
+ ];
208
+ case "decide":
209
+ return [{ kind: "decision", decision: effect.result as Decision, error: effect.error }];
210
+ case "requestApproval": {
211
+ const [{ amount }] = effect.args as [{ amount: number }];
212
+ return [{ kind: "approval", approved: effect.result as boolean, amount }];
213
+ }
214
+ case "issueRefund":
215
+ return [
216
+ {
217
+ kind: "refund",
218
+ ...(effect.args as [{ orderId: string; amount: number }])[0],
219
+ error: effect.error,
220
+ },
221
+ ];
222
+ default:
223
+ return [];
224
+ }
225
+ });
226
+ }
227
+ ```
228
+
229
+ Keep the mapping to what the call shows. Do not decide in the mapping what the code should have done next: that is the model's job, and a mapping that repeats the code's logic makes the check pass for any run.
230
+
231
+ ## Step 5: check conformance
232
+
233
+ `checkConformance` runs the model with each actor answered from the observations:
234
+
235
+ ```ts no-check
236
+ const report = await checkConformance(refundModel, {
237
+ input: { orderId: scenario.order.orderId },
238
+ observations: toObservations(recording),
239
+ actors: {
240
+ lookupOrder: {
241
+ observes: "lookup",
242
+ matches: ({ input, observation }) => input === observation.orderId,
243
+ output: ({ observation }) => observation.order,
244
+ error: ({ observation }) => observation.error,
245
+ },
246
+ decide: {
247
+ observes: "decision",
248
+ output: ({ observation }) => observation.decision,
249
+ error: ({ observation }) => observation.error,
250
+ },
251
+ issueRefund: {
252
+ observes: "refund",
253
+ matches: ({ input, observation }) =>
254
+ input.orderId === observation.orderId && input.amount === observation.amount,
255
+ error: ({ observation }) => observation.error,
256
+ },
257
+ },
258
+ events: {
259
+ approval: {
260
+ event: (observation) => ({ type: observation.approved ? "APPROVE" : "REJECT" }),
261
+ matches: ({ observation, context }) => observation.amount === context.amount,
262
+ },
263
+ },
264
+ output: {
265
+ observes: "returned",
266
+ matches: ({ output, observation }) => output.outcome === observation.outcome,
267
+ },
268
+ });
269
+ ```
270
+
271
+ - An actor with `observes` consumes the next observation, which must be of that kind. `output` builds the actor's output from it.
272
+ - `matches` compares the actor's input with the observation. Here, the refund the code issued must be for the order and amount the model would refund.
273
+ - `events` sends an observation to the model as an event while the model waits, here the person's approval. Its `matches` compares the observation with the model's context: the person approved the amount the model is waiting to refund.
274
+ - `output` checks the code's return value. The model must finish where the code returned, and `matches` compares the model's output with the recorded one. Pass it for runs where the code threw too: record the throw as the outcome (`recorder.output({ threw: error instanceof Error ? error.message : String(error) })` in a `catch`, since code can throw a value that is not an `Error`) and model each failure the code throws as a final state whose output `matches` compares. Give that final state an output in the same shape as a return, such as `{ outcome: { threw: message } }`, and compare the fields, such as `output.outcome.threw === observation.outcome.threw`: the recorded outcome and the model's output are different objects, so `===` on the objects never matches. A model that errors instead of finishing leaves that outcome unconsumed, which deviates. Without `output`, a run that otherwise conforms reports a recorded throw as `extra-observations`, like a return value. Leaving the throw out of the observations leaves the outcome unchecked.
275
+ - An actor the recording cannot see takes no `observes`. It runs unchecked and is listed in `report.unobserved`.
276
+ - When an actor's output carries state the recording does not show, build it in `output` from the model's own `input` plus the observation.
277
+
278
+ All runs of the unchanged code conform. The full option and deviation reference is in [Testing and verification](verify.md#recorded-runs-against-a-model).
279
+
280
+ ## Step 6: coverage
281
+
282
+ Pass each report's `trace` to `traceCoverage`:
283
+
284
+ ```ts no-check
285
+ const coverage = traceCoverage(refundModel, reports.map((report) => report.trace));
286
+ coverage.unvisitedStates; // []
287
+ coverage.unvisitedHandlers; // transitions no recorded run took
288
+ ```
289
+
290
+ An unvisited state or handler is a part of the model no recorded run exercised. Add a scenario that reaches it, or confirm that it cannot happen in production.
291
+
292
+ ## A regression, caught
293
+
294
+ `regression.ts` is the code after a plausible change: repeat customers skip approval on refunds up to twice the threshold. The happy-path tests still pass. The conformance check reports:
295
+
296
+ ```ts no-check
297
+ {
298
+ reason: "extra-observations",
299
+ expected: "idle, waiting for an event",
300
+ observed: { kind: "refund", amount: 150 },
301
+ state: "awaitingApproval",
302
+ }
303
+ ```
304
+
305
+ The model was waiting for a person in `awaitingApproval`, and the code issued the refund instead. Only the scenario with a repeat customer and a mid-size refund shows it. The check finds a bug only on a run that exercises it. Coverage shows which parts of the model the recorded runs exercised.
306
+
307
+ ## What the check covers
308
+
309
+ - It checks the order and the content of what the recording shows against what the model allows, and the code's return value when you pass `output`.
310
+ - It does not see effects the recording does not capture. An actor without `observes` is unchecked, and the report lists it.
311
+ - It does not prove anything about runs you did not record. The proofs cover every path through the outputs and payloads listed in [Step 2](#step-2-proofs), not every value an actor could return. Conformance ties those paths to the recorded runs.
312
+ - A wrong model can conform and prove its rules. Review the model, and treat a deviation as a question about the code or the model.
313
+
314
+ ## Migrate later
315
+
316
+ When you want the rules enforced at runtime instead of checked after the fact, migrate the code to a machine. The model becomes the machine, its placeholder actors become the code's functions, and the same proofs run against it.
317
+
318
+ - [Migrating from a hand-rolled loop](from-a-loop.md) walks through a migration.
319
+ - The [`loop-to-machine`](https://github.com/statelyai/skills/blob/main/plugins/stately-agent/skills/loop-to-machine/SKILL.md) skill runs it with a coding agent.
320
+ - `checkParity` checks the migrated machine against the original code. See [Strangler-fig retrofit](strangler-fig.md#check-parity).
321
+
322
+ ## Related
323
+
324
+ - [Testing and verification](verify.md): `canReach`, `explorePaths`, and the `checkConformance` reference.
325
+ - [Evals](evals.md): `traceCoverage`, recorders, and parity.
326
+ - [examples/verify-in-place](https://github.com/statelyai/agent/tree/next/examples/verify-in-place): the code on this page, with tests.