@statelyai/agent 2.0.0-alpha.40 → 2.0.0-alpha.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/evals.cjs +1 -1
- package/dist/evals.d.cts +1 -1
- package/dist/evals.d.mts +1 -1
- package/dist/evals.mjs +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/testing.cjs +376 -17
- package/dist/testing.d.cts +181 -5
- package/dist/testing.d.mts +181 -5
- package/dist/testing.mjs +377 -20
- package/dist/{utils-DAclF-kw.d.cts → utils-BMxUcjay.d.cts} +57 -1
- package/dist/{utils-B8k-YPYI.d.mts → utils-BV7pxhD-.d.mts} +57 -1
- package/dist/validate.cjs +1 -1
- package/dist/validate.d.cts +1 -1
- package/dist/validate.d.mts +1 -1
- package/dist/validate.mjs +1 -1
- package/dist/{verify-CIu4mqIZ.d.mts → verify-C1UU84eq.d.mts} +7 -1
- package/dist/{verify-CTHlmBbh.d.cts → verify-DlN6tK8_.d.cts} +7 -1
- package/dist/{verify-D8UDqU4l.cjs → verify-IVYsI026.cjs} +65 -14
- package/dist/{verify-Cacz6XSk.mjs → verify-lKyu3buS.mjs} +65 -14
- package/docs/evals.md +1 -0
- package/docs/index.md +2 -2
- package/docs/meta.json +1 -0
- package/docs/verify-in-place.md +326 -0
- package/docs/verify.md +67 -2
- package/llms.txt +1 -0
- package/package.json +1 -1
- package/readme.md +1 -0
|
@@ -807,6 +807,65 @@ function scriptDryError(kind, src, id, request) {
|
|
|
807
807
|
const MAX_ADVANCE_STEPS = 1e3;
|
|
808
808
|
const FINITE_PAYLOAD_CAP = 64;
|
|
809
809
|
const DEFAULT_MAX_NODES = 1e5;
|
|
810
|
+
function isDataProperty(target, key) {
|
|
811
|
+
const descriptor = Object.getOwnPropertyDescriptor(target, key);
|
|
812
|
+
return descriptor !== void 0 && "value" in descriptor;
|
|
813
|
+
}
|
|
814
|
+
function strictKey(value, refs = /* @__PURE__ */ new Map()) {
|
|
815
|
+
if (value === void 0) return "u";
|
|
816
|
+
if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value);
|
|
817
|
+
if (typeof value === "number") return Object.is(value, -0) ? "n-0" : `n${String(value)}`;
|
|
818
|
+
if (typeof value !== "object") throw new TypeError(`unkeyable ${typeof value}`);
|
|
819
|
+
const ref = refs.get(value);
|
|
820
|
+
if (ref !== void 0) return `@${ref}`;
|
|
821
|
+
refs.set(value, refs.size);
|
|
822
|
+
const proto = Object.getPrototypeOf(value);
|
|
823
|
+
const own = Reflect.ownKeys(value).length;
|
|
824
|
+
if (Array.isArray(value)) {
|
|
825
|
+
if (proto !== Array.prototype) throw new TypeError("unkeyable array");
|
|
826
|
+
for (let index = 0; index < value.length; index++) if (!isDataProperty(value, index)) throw new TypeError("unkeyable sparse array");
|
|
827
|
+
if (own !== value.length + 1) throw new TypeError("unkeyable array");
|
|
828
|
+
return `[${value.map((item) => strictKey(item, refs)).join(",")}]`;
|
|
829
|
+
}
|
|
830
|
+
if (proto !== Object.prototype && proto !== null) throw new TypeError("unkeyable object");
|
|
831
|
+
const keys = Object.keys(value);
|
|
832
|
+
if (own !== keys.length || !keys.every((k) => isDataProperty(value, k))) throw new TypeError("unkeyable object");
|
|
833
|
+
return `${proto === null ? "N" : ""}{${keys.map((k) => `${JSON.stringify(k)}:${strictKey(value[k], refs)}`).join(",")}}`;
|
|
834
|
+
}
|
|
835
|
+
function tryKey(key) {
|
|
836
|
+
try {
|
|
837
|
+
return key();
|
|
838
|
+
} catch {
|
|
839
|
+
return;
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
function stepKeyParts(step) {
|
|
843
|
+
const snapshot = step.snapshot;
|
|
844
|
+
const children = snapshot.children ?? {};
|
|
845
|
+
return [
|
|
846
|
+
snapshot.value,
|
|
847
|
+
snapshot.context,
|
|
848
|
+
snapshot.status,
|
|
849
|
+
snapshot.historyValue ?? null,
|
|
850
|
+
step.requests.map((request) => request.id),
|
|
851
|
+
pendingInvokes(step).map((invoke) => [
|
|
852
|
+
invoke.id,
|
|
853
|
+
invoke.src,
|
|
854
|
+
invoke.input
|
|
855
|
+
]),
|
|
856
|
+
Object.keys(children).sort().map((id) => [id, children[id]?.options?.input]),
|
|
857
|
+
(snapshot.timers ?? []).map((timer) => [
|
|
858
|
+
timer["id"],
|
|
859
|
+
timer["type"],
|
|
860
|
+
timer["delay"],
|
|
861
|
+
timer["event"],
|
|
862
|
+
timer["target"] === "self" ? "self" : timer["target"]?.id
|
|
863
|
+
])
|
|
864
|
+
];
|
|
865
|
+
}
|
|
866
|
+
function defaultStepKey(step) {
|
|
867
|
+
return strictKey(stepKeyParts(step));
|
|
868
|
+
}
|
|
810
869
|
const MAX_CHILD_NESTING = 4;
|
|
811
870
|
const DEFAULT_BUDGET_NODES = 25e4;
|
|
812
871
|
const DEFAULT_CHILD_CACHE = 64;
|
|
@@ -1052,7 +1111,7 @@ function finishedChildren(step) {
|
|
|
1052
1111
|
return [];
|
|
1053
1112
|
});
|
|
1054
1113
|
}
|
|
1055
|
-
function explore(machine, options, stopWhen, nesting = 0, deliverable, childRef, trackTransits = nesting === 0 && hasTransientStates(machine.root), budget = startBudget(options.budget), stepping = false, ordering, stateTarget = false) {
|
|
1114
|
+
function explore(machine, options, stopWhen, nesting = 0, deliverable, childRef, trackTransits = nesting === 0 && hasTransientStates(machine.root), budget = startBudget(options.budget), stepping = false, ordering, stateTarget = false, defaultDedupe = false) {
|
|
1056
1115
|
const maxDepth = options.maxDepth ?? 8;
|
|
1057
1116
|
const maxPaths = options.maxPaths ?? 200;
|
|
1058
1117
|
const maxNodes = options.maxNodes ?? DEFAULT_MAX_NODES;
|
|
@@ -1810,17 +1869,8 @@ function explore(machine, options, stopWhen, nesting = 0, deliverable, childRef,
|
|
|
1810
1869
|
return `\u0003${[...kids.values()].map((kid) => `${kid.id}\u0000${kid.src}\u0000${kid.engine.stateKey(kid)}`).sort().join("")}`;
|
|
1811
1870
|
};
|
|
1812
1871
|
const stateKey = (kid) => {
|
|
1813
|
-
const snapshot = kid.step.snapshot;
|
|
1814
1872
|
try {
|
|
1815
|
-
return
|
|
1816
|
-
snapshot.value,
|
|
1817
|
-
snapshot.context,
|
|
1818
|
-
snapshot.status,
|
|
1819
|
-
snapshot.historyValue ?? null,
|
|
1820
|
-
kid.step.requests.map((request) => request.id),
|
|
1821
|
-
Object.keys(snapshot.children ?? {}),
|
|
1822
|
-
pumped.has(kid.step)
|
|
1823
|
-
]) + kidsKey(kid.kids);
|
|
1873
|
+
return strictKey([...stepKeyParts(kid.step), pumped.has(kid.step)]) + kidsKey(kid.kids);
|
|
1824
1874
|
} catch {
|
|
1825
1875
|
return `\u0002${unkeyable++}`;
|
|
1826
1876
|
}
|
|
@@ -1856,8 +1906,9 @@ function explore(machine, options, stopWhen, nesting = 0, deliverable, childRef,
|
|
|
1856
1906
|
return;
|
|
1857
1907
|
}
|
|
1858
1908
|
}
|
|
1859
|
-
|
|
1860
|
-
|
|
1909
|
+
const base = options.dedupeBy ? options.dedupeBy(withKidSnapshots(settled.snapshot, nodeKids)) : defaultDedupe ? tryKey(() => defaultStepKey(settled)) : void 0;
|
|
1910
|
+
if (base !== void 0) {
|
|
1911
|
+
const key = base + (delivered?.size ? `\u0001${[...delivered].sort().join("")}` : "") + kidsKey(nodeKids) + (logs.get(settled)?.after ? "after" : "");
|
|
1861
1912
|
const seenAt = visited.get(key);
|
|
1862
1913
|
if (seenAt !== void 0 && seenAt <= depth) return;
|
|
1863
1914
|
visited.set(key, depth);
|
|
@@ -2396,7 +2447,7 @@ async function canReach(machine, target, options = {}) {
|
|
|
2396
2447
|
options = withMachineInput(machine, options, "canReach");
|
|
2397
2448
|
const resolvedTarget = typeof target === "string" ? resolveStateTarget(machine, target) : void 0;
|
|
2398
2449
|
const ordering = typeof target === "object" ? target : void 0;
|
|
2399
|
-
const { witness, report } = explore(machine, options, typeof target === "function" ? target : ordering ? () => false : (snapshot) => snapshot.nodes.some((node) => node.id === resolvedTarget), 0, void 0, void 0, resolvedTarget !== void 0 && hasTransientStates(machine.getStateNodeById(resolvedTarget)), void 0, false, ordering, resolvedTarget !== void 0);
|
|
2450
|
+
const { witness, report } = explore(machine, options, typeof target === "function" ? target : ordering ? () => false : (snapshot) => snapshot.nodes.some((node) => node.id === resolvedTarget), 0, void 0, void 0, resolvedTarget !== void 0 && hasTransientStates(machine.getStateNodeById(resolvedTarget)), void 0, false, ordering, resolvedTarget !== void 0, options.dedupeBy === void 0);
|
|
2400
2451
|
if (witness !== void 0) return {
|
|
2401
2452
|
reachable: true,
|
|
2402
2453
|
witness,
|
|
@@ -807,6 +807,65 @@ function scriptDryError(kind, src, id, request) {
|
|
|
807
807
|
const MAX_ADVANCE_STEPS = 1e3;
|
|
808
808
|
const FINITE_PAYLOAD_CAP = 64;
|
|
809
809
|
const DEFAULT_MAX_NODES = 1e5;
|
|
810
|
+
function isDataProperty(target, key) {
|
|
811
|
+
const descriptor = Object.getOwnPropertyDescriptor(target, key);
|
|
812
|
+
return descriptor !== void 0 && "value" in descriptor;
|
|
813
|
+
}
|
|
814
|
+
function strictKey(value, refs = /* @__PURE__ */ new Map()) {
|
|
815
|
+
if (value === void 0) return "u";
|
|
816
|
+
if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value);
|
|
817
|
+
if (typeof value === "number") return Object.is(value, -0) ? "n-0" : `n${String(value)}`;
|
|
818
|
+
if (typeof value !== "object") throw new TypeError(`unkeyable ${typeof value}`);
|
|
819
|
+
const ref = refs.get(value);
|
|
820
|
+
if (ref !== void 0) return `@${ref}`;
|
|
821
|
+
refs.set(value, refs.size);
|
|
822
|
+
const proto = Object.getPrototypeOf(value);
|
|
823
|
+
const own = Reflect.ownKeys(value).length;
|
|
824
|
+
if (Array.isArray(value)) {
|
|
825
|
+
if (proto !== Array.prototype) throw new TypeError("unkeyable array");
|
|
826
|
+
for (let index = 0; index < value.length; index++) if (!isDataProperty(value, index)) throw new TypeError("unkeyable sparse array");
|
|
827
|
+
if (own !== value.length + 1) throw new TypeError("unkeyable array");
|
|
828
|
+
return `[${value.map((item) => strictKey(item, refs)).join(",")}]`;
|
|
829
|
+
}
|
|
830
|
+
if (proto !== Object.prototype && proto !== null) throw new TypeError("unkeyable object");
|
|
831
|
+
const keys = Object.keys(value);
|
|
832
|
+
if (own !== keys.length || !keys.every((k) => isDataProperty(value, k))) throw new TypeError("unkeyable object");
|
|
833
|
+
return `${proto === null ? "N" : ""}{${keys.map((k) => `${JSON.stringify(k)}:${strictKey(value[k], refs)}`).join(",")}}`;
|
|
834
|
+
}
|
|
835
|
+
function tryKey(key) {
|
|
836
|
+
try {
|
|
837
|
+
return key();
|
|
838
|
+
} catch {
|
|
839
|
+
return;
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
function stepKeyParts(step) {
|
|
843
|
+
const snapshot = step.snapshot;
|
|
844
|
+
const children = snapshot.children ?? {};
|
|
845
|
+
return [
|
|
846
|
+
snapshot.value,
|
|
847
|
+
snapshot.context,
|
|
848
|
+
snapshot.status,
|
|
849
|
+
snapshot.historyValue ?? null,
|
|
850
|
+
step.requests.map((request) => request.id),
|
|
851
|
+
pendingInvokes(step).map((invoke) => [
|
|
852
|
+
invoke.id,
|
|
853
|
+
invoke.src,
|
|
854
|
+
invoke.input
|
|
855
|
+
]),
|
|
856
|
+
Object.keys(children).sort().map((id) => [id, children[id]?.options?.input]),
|
|
857
|
+
(snapshot.timers ?? []).map((timer) => [
|
|
858
|
+
timer["id"],
|
|
859
|
+
timer["type"],
|
|
860
|
+
timer["delay"],
|
|
861
|
+
timer["event"],
|
|
862
|
+
timer["target"] === "self" ? "self" : timer["target"]?.id
|
|
863
|
+
])
|
|
864
|
+
];
|
|
865
|
+
}
|
|
866
|
+
function defaultStepKey(step) {
|
|
867
|
+
return strictKey(stepKeyParts(step));
|
|
868
|
+
}
|
|
810
869
|
const MAX_CHILD_NESTING = 4;
|
|
811
870
|
const DEFAULT_BUDGET_NODES = 25e4;
|
|
812
871
|
const DEFAULT_CHILD_CACHE = 64;
|
|
@@ -1052,7 +1111,7 @@ function finishedChildren(step) {
|
|
|
1052
1111
|
return [];
|
|
1053
1112
|
});
|
|
1054
1113
|
}
|
|
1055
|
-
function explore(machine, options, stopWhen, nesting = 0, deliverable, childRef, trackTransits = nesting === 0 && hasTransientStates(machine.root), budget = startBudget(options.budget), stepping = false, ordering, stateTarget = false) {
|
|
1114
|
+
function explore(machine, options, stopWhen, nesting = 0, deliverable, childRef, trackTransits = nesting === 0 && hasTransientStates(machine.root), budget = startBudget(options.budget), stepping = false, ordering, stateTarget = false, defaultDedupe = false) {
|
|
1056
1115
|
const maxDepth = options.maxDepth ?? 8;
|
|
1057
1116
|
const maxPaths = options.maxPaths ?? 200;
|
|
1058
1117
|
const maxNodes = options.maxNodes ?? DEFAULT_MAX_NODES;
|
|
@@ -1810,17 +1869,8 @@ function explore(machine, options, stopWhen, nesting = 0, deliverable, childRef,
|
|
|
1810
1869
|
return `\u0003${[...kids.values()].map((kid) => `${kid.id}\u0000${kid.src}\u0000${kid.engine.stateKey(kid)}`).sort().join("")}`;
|
|
1811
1870
|
};
|
|
1812
1871
|
const stateKey = (kid) => {
|
|
1813
|
-
const snapshot = kid.step.snapshot;
|
|
1814
1872
|
try {
|
|
1815
|
-
return
|
|
1816
|
-
snapshot.value,
|
|
1817
|
-
snapshot.context,
|
|
1818
|
-
snapshot.status,
|
|
1819
|
-
snapshot.historyValue ?? null,
|
|
1820
|
-
kid.step.requests.map((request) => request.id),
|
|
1821
|
-
Object.keys(snapshot.children ?? {}),
|
|
1822
|
-
pumped.has(kid.step)
|
|
1823
|
-
]) + kidsKey(kid.kids);
|
|
1873
|
+
return strictKey([...stepKeyParts(kid.step), pumped.has(kid.step)]) + kidsKey(kid.kids);
|
|
1824
1874
|
} catch {
|
|
1825
1875
|
return `\u0002${unkeyable++}`;
|
|
1826
1876
|
}
|
|
@@ -1856,8 +1906,9 @@ function explore(machine, options, stopWhen, nesting = 0, deliverable, childRef,
|
|
|
1856
1906
|
return;
|
|
1857
1907
|
}
|
|
1858
1908
|
}
|
|
1859
|
-
|
|
1860
|
-
|
|
1909
|
+
const base = options.dedupeBy ? options.dedupeBy(withKidSnapshots(settled.snapshot, nodeKids)) : defaultDedupe ? tryKey(() => defaultStepKey(settled)) : void 0;
|
|
1910
|
+
if (base !== void 0) {
|
|
1911
|
+
const key = base + (delivered?.size ? `\u0001${[...delivered].sort().join("")}` : "") + kidsKey(nodeKids) + (logs.get(settled)?.after ? "after" : "");
|
|
1861
1912
|
const seenAt = visited.get(key);
|
|
1862
1913
|
if (seenAt !== void 0 && seenAt <= depth) return;
|
|
1863
1914
|
visited.set(key, depth);
|
|
@@ -2396,7 +2447,7 @@ async function canReach(machine, target, options = {}) {
|
|
|
2396
2447
|
options = withMachineInput(machine, options, "canReach");
|
|
2397
2448
|
const resolvedTarget = typeof target === "string" ? resolveStateTarget(machine, target) : void 0;
|
|
2398
2449
|
const ordering = typeof target === "object" ? target : void 0;
|
|
2399
|
-
const { witness, report } = explore(machine, options, typeof target === "function" ? target : ordering ? () => false : (snapshot) => snapshot.nodes.some((node) => node.id === resolvedTarget), 0, void 0, void 0, resolvedTarget !== void 0 && hasTransientStates(machine.getStateNodeById(resolvedTarget)), void 0, false, ordering, resolvedTarget !== void 0);
|
|
2450
|
+
const { witness, report } = explore(machine, options, typeof target === "function" ? target : ordering ? () => false : (snapshot) => snapshot.nodes.some((node) => node.id === resolvedTarget), 0, void 0, void 0, resolvedTarget !== void 0 && hasTransientStates(machine.getStateNodeById(resolvedTarget)), void 0, false, ordering, resolvedTarget !== void 0, options.dedupeBy === void 0);
|
|
2400
2451
|
if (witness !== void 0) return {
|
|
2401
2452
|
reachable: true,
|
|
2402
2453
|
witness,
|
package/docs/evals.md
CHANGED
|
@@ -337,6 +337,7 @@ replay.output(await runPort(replay.client("anthropic", stand, { methods })));
|
|
|
337
337
|
compareRecordings(recorder.recording, replay.recording); // { equivalent: true, divergences: [] }
|
|
338
338
|
```
|
|
339
339
|
|
|
340
|
+
- `recorder.model` records each provider attempt as a `model` effect. A failed attempt followed by the same request, such as an AI SDK retry, has `retriedBy` set to the call that retried it. Attempts made inside a `generateText` or `streamText` call that `recordAiSdk` routed carry that call's index as `aiSdkCall`, so a retry by the SDK (same `aiSdkCall`) is told apart from one by the code (a different `aiSdkCall`).
|
|
340
341
|
- Each call is a `call` effect named `<name>.<method>`, such as `anthropic.messages.create`, with its arguments and its result or error.
|
|
341
342
|
- `recorder.fn(name, asyncFn, options?)` wraps one async function the same way, for a client that is a function.
|
|
342
343
|
- Replay answers each call with the next unused recorded call of the same name, in order, whatever its arguments. A call past the recorded ones throws `RecordingDivergenceError`.
|
package/docs/index.md
CHANGED
|
@@ -11,7 +11,7 @@ Stately Agent is XState plus typed model requests, decisions, and host executors
|
|
|
11
11
|
## Start here
|
|
12
12
|
|
|
13
13
|
- Greenfield, a new agent: [Quickstart](quickstart.md), then [Thinking in state machines](thinking-in-state-machines.md) or a copied [agent pattern](patterns.md).
|
|
14
|
-
- Brownfield, an existing agent loop: [Migrating from a loop](from-a-loop.md), or the [`loop-to-machine`](https://github.com/statelyai/skills/blob/main/plugins/stately-agent/skills/loop-to-machine/SKILL.md) skill from [statelyai/skills](https://github.com/statelyai/skills).
|
|
14
|
+
- Brownfield, an existing agent loop: [Verify in place](verify-in-place.md) to check it against a model without changing it, or [Migrating from a loop](from-a-loop.md), or the [`loop-to-machine`](https://github.com/statelyai/skills/blob/main/plugins/stately-agent/skills/loop-to-machine/SKILL.md) skill from [statelyai/skills](https://github.com/statelyai/skills).
|
|
15
15
|
- Brownfield, an existing async workflow: [Strangler-fig retrofit](strangler-fig.md). Wrap it unchanged with `wrapWorkflow`, then move one phase at a time into states, checked with `checkParity`.
|
|
16
16
|
- [Choosing a run mode](choosing-a-run-mode.md)
|
|
17
17
|
- [Advanced: bare XState actors and hand-written executors](advanced.md)
|
|
@@ -35,7 +35,7 @@ Stately Agent is XState plus typed model requests, decisions, and host executors
|
|
|
35
35
|
| ------------------------------ | ------------------------------------------------------------------------------------------ |
|
|
36
36
|
| `@statelyai/agent` | Authoring, running, human interaction, and the executor contract |
|
|
37
37
|
| `@statelyai/agent/evals` | Trace analytics: `foldTraces`, `traceCoverage`, `repeatRuns`, `judgeAgreement` |
|
|
38
|
-
| `@statelyai/agent/testing` | `lintAgentMachine`, `
|
|
38
|
+
| `@statelyai/agent/testing` | `lintAgentMachine`, `canReach`, `checkConformance`, `checkParity`, trajectories, seams |
|
|
39
39
|
| `@statelyai/agent/log` | The event log: `replay`, `forkEventLog`, `getDecisionAttempts`, hand-built entries, stores |
|
|
40
40
|
| `@statelyai/agent/ai-sdk` | `createAiSdkExecutors`: the Vercel AI SDK adapter, for language and decision models |
|
|
41
41
|
| `@statelyai/agent/openai` | Executors over the raw `openai` package |
|
package/docs/meta.json
CHANGED
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Verify in place
|
|
3
|
+
description: Model an existing agent workflow, prove the model's rules, and check recorded runs of the unchanged code against it. Migrating the code to a machine is optional.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
> **Alpha:** `@statelyai/agent` 2.0 is in alpha. APIs can change between releases; pin an exact version. Feedback: [github.com/statelyai/agent](https://github.com/statelyai/agent/issues).
|
|
7
|
+
|
|
8
|
+
There are two ways to use `@statelyai/agent`:
|
|
9
|
+
|
|
10
|
+
- **Author** a new workflow as a machine. The machine is the code, and it enforces its rules at runtime. Start with the [Quickstart](quickstart.md).
|
|
11
|
+
- **Verify in place.** Keep the code you have, write a model of its control flow, prove the model's rules, and check recorded runs of the code against the model. This page covers that track.
|
|
12
|
+
|
|
13
|
+
You can migrate the code to a machine later. The model and its proofs carry over. See [Migrate later](#migrate-later).
|
|
14
|
+
|
|
15
|
+
The worked example is [examples/verify-in-place](https://github.com/statelyai/agent/tree/next/examples/verify-in-place). Every snippet on this page comes from it.
|
|
16
|
+
|
|
17
|
+
## What you get
|
|
18
|
+
|
|
19
|
+
- A model file in the repo, reviewed like other code.
|
|
20
|
+
- Proofs of the model's rules that run in CI with no model calls.
|
|
21
|
+
- A check that recorded runs of the code conform to the model, and a report of the first step where a run does not.
|
|
22
|
+
- Coverage: the states and transitions of the model that no recorded run reached.
|
|
23
|
+
|
|
24
|
+
The claim is: the model is proved, and these recorded runs conform to it. A run you did not record can still break a rule. After a migration, the machine enforces the rules, and the claim covers every run.
|
|
25
|
+
|
|
26
|
+
## The existing code
|
|
27
|
+
|
|
28
|
+
The example is a refund agent. Every side effect goes through one `effects` object, and the rules live in `if` statements:
|
|
29
|
+
|
|
30
|
+
```ts no-check
|
|
31
|
+
export async function handleRefundRequest(
|
|
32
|
+
fx: RefundEffects,
|
|
33
|
+
request: { orderId: string; message: string },
|
|
34
|
+
): Promise<RefundOutcome> {
|
|
35
|
+
const order = await fx.lookupOrder(request.orderId);
|
|
36
|
+
for (let decisions = 0; decisions < MAX_DECISIONS; decisions++) {
|
|
37
|
+
const decision = await fx.decide({ request: request.message, order, decisions });
|
|
38
|
+
if (decision.type === "reply") return "replied";
|
|
39
|
+
if (decision.type === "investigate") continue;
|
|
40
|
+
if (decision.amount > order.total) return "over-total";
|
|
41
|
+
if (decision.amount > APPROVAL_THRESHOLD) {
|
|
42
|
+
const approved = await fx.requestApproval({
|
|
43
|
+
orderId: order.orderId,
|
|
44
|
+
amount: decision.amount,
|
|
45
|
+
});
|
|
46
|
+
if (!approved) return "declined";
|
|
47
|
+
}
|
|
48
|
+
await fx.issueRefund({ orderId: order.orderId, amount: decision.amount });
|
|
49
|
+
return "refunded";
|
|
50
|
+
}
|
|
51
|
+
return "escalated";
|
|
52
|
+
}
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
The check does not edit this file. Code whose side effects are spread across modules needs a seam first: route them through one object, or record them at the client with `createRecorder`'s `recorder.client`, `recorder.model` for an AI SDK model, and `recorder.tools` for AI SDK tools. See [Evals](evals.md#any-model-client).
|
|
56
|
+
|
|
57
|
+
### AI SDK code
|
|
58
|
+
|
|
59
|
+
Code that builds its model inline (`openai("gpt-…")`) and passes module-level tools to `generateText` has nothing to inject. Mock `"ai"` in the test and route its calls through the recorder with `recordAiSdk`:
|
|
60
|
+
|
|
61
|
+
```ts no-check
|
|
62
|
+
// seam.ts: must not import "ai", or the mock below waits on itself.
|
|
63
|
+
let current: AiSdkSeam | undefined;
|
|
64
|
+
export const currentSeam = () => current;
|
|
65
|
+
export function startRecording(standIn: LanguageModelV3) {
|
|
66
|
+
const recorder = createRecorder();
|
|
67
|
+
current = { recorder, model: recorder.model(standIn) };
|
|
68
|
+
return recorder;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// support-agent.test.ts
|
|
72
|
+
vi.mock("ai", async (importOriginal) => {
|
|
73
|
+
const ai = await importOriginal<typeof import("ai")>();
|
|
74
|
+
const { recordAiSdk } = await import("@statelyai/agent/testing");
|
|
75
|
+
const { currentSeam } = await import("./seam");
|
|
76
|
+
return { ...ai, ...recordAiSdk(ai, currentSeam) };
|
|
77
|
+
});
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
- Each `generateText` and `streamText` call runs on the real AI SDK with the seam's `model` in place of the code's, and its `tools` wrapped with `recorder.tools`. The code's own tool loop runs.
|
|
81
|
+
- A call made while `currentSeam()` returns `undefined` throws, so no test reaches a real provider.
|
|
82
|
+
- A tool that has `execute` runs inside the `generateText` call. A check the code makes after the call returns comes after the tool already ran: model the order the calls happen in.
|
|
83
|
+
- `recorder.model` records each provider attempt, and `effect.call` counts attempts. `recordAiSdk` stamps each attempt of a `recorder.model` wrapper from `createRecorder` with `aiSdkCall`, the `generateText` or `streamText` call it belongs to, including steps that use a model from `prepareStep`: attempts with the same `aiSdkCall` are one call the code made, its tool-loop steps and the SDK's retries.
|
|
84
|
+
- A failed attempt followed by the same request has `retriedBy` set to the attempt that resent it. When that attempt has the same `aiSdkCall`, the SDK retried it (`maxRetries`): skip it in the mapping. When it has a different `aiSdkCall`, the code resent it: keep it, so the model checks the code's retry path and its call limit. An attempt with no `aiSdkCall` is one whose resender is unknown: keep it. That is any attempt not routed through `recordAiSdk`, any routed attempt whose model is not a `recorder.model` wrapper passed as is (a proxy around one, or one with a replaced method), and any attempt recorded through a recorder `createRecorder` did not make.
|
|
85
|
+
|
|
86
|
+
```ts no-check
|
|
87
|
+
const sdkRetried = (effect: RecordedModelCall, effects: RecordedEffect[]) =>
|
|
88
|
+
effect.aiSdkCall !== undefined &&
|
|
89
|
+
effect.retriedBy !== undefined &&
|
|
90
|
+
effects.some(
|
|
91
|
+
(e) => e.kind === "model" && e.call === effect.retriedBy && e.aiSdkCall === effect.aiSdkCall,
|
|
92
|
+
);
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Step 1: the model file
|
|
96
|
+
|
|
97
|
+
Write the same control flow as a machine in `model.ts`. Each unit of work is an actor declared by name, with a placeholder implementation, so the model runs none of the code. Each rule is a state or a guard:
|
|
98
|
+
|
|
99
|
+
```ts no-check
|
|
100
|
+
deciding: {
|
|
101
|
+
invoke: {
|
|
102
|
+
src: "decide",
|
|
103
|
+
input: ({ context }) => ({ decisions: context.decisions }),
|
|
104
|
+
onDone: ({ context, event }) => {
|
|
105
|
+
const decision = event.output;
|
|
106
|
+
const decisions = context.decisions + 1;
|
|
107
|
+
if (decision.type === "reply") return { target: "replied", context: { decisions } };
|
|
108
|
+
if (decision.type === "investigate")
|
|
109
|
+
return { target: "checking", context: { decisions } };
|
|
110
|
+
const amount = decision.amount;
|
|
111
|
+
if (amount > context.order!.total)
|
|
112
|
+
return { target: "overTotal", context: { decisions, amount } };
|
|
113
|
+
return amount > APPROVAL_THRESHOLD
|
|
114
|
+
? { target: "awaitingApproval", context: { decisions, amount } }
|
|
115
|
+
: { target: "refunding", context: { decisions, amount } };
|
|
116
|
+
},
|
|
117
|
+
},
|
|
118
|
+
},
|
|
119
|
+
awaitingApproval: {
|
|
120
|
+
on: {
|
|
121
|
+
APPROVE: { target: "refunding", context: { approved: true } },
|
|
122
|
+
REJECT: { target: "declined" },
|
|
123
|
+
},
|
|
124
|
+
},
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
- Declare every actor in `setupAgent({ actors })` and invoke it by `src`. `checkConformance` answers named actors from the recording, and rejects a model whose invoke names its logic inline.
|
|
128
|
+
- A person's input, such as an approval, is an event the model waits for in an idle state.
|
|
129
|
+
- The `stately` skill in [statelyai/skills](https://github.com/statelyai/skills) can draft a model from code. Review it like code: the proofs below hold for the model, not for behavior the model leaves out.
|
|
130
|
+
|
|
131
|
+
## Step 2: proofs
|
|
132
|
+
|
|
133
|
+
Prove the model's rules with `canReach`, as ordinary tests. Give each actor the outputs it can return:
|
|
134
|
+
|
|
135
|
+
```ts no-check
|
|
136
|
+
const proofOptions = {
|
|
137
|
+
input: { orderId: "P-1" },
|
|
138
|
+
outputs: {
|
|
139
|
+
lookupOrder: [{ orderId: "P-1", total: 300 }],
|
|
140
|
+
decide: [
|
|
141
|
+
{ type: "refund", amount: 50 },
|
|
142
|
+
{ type: "refund", amount: 250 },
|
|
143
|
+
{ type: "refund", amount: 400 },
|
|
144
|
+
{ type: "reply", message: "ok" },
|
|
145
|
+
{ type: "investigate" },
|
|
146
|
+
],
|
|
147
|
+
issueRefund: [undefined],
|
|
148
|
+
},
|
|
149
|
+
maxDepth: 12,
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
const result = await canReach(
|
|
153
|
+
refundModel,
|
|
154
|
+
(snapshot) =>
|
|
155
|
+
snapshot.matches("refunding") &&
|
|
156
|
+
snapshot.context.amount > APPROVAL_THRESHOLD &&
|
|
157
|
+
!snapshot.context.approved,
|
|
158
|
+
proofOptions,
|
|
159
|
+
);
|
|
160
|
+
// { reachable: false, complete: true }
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
- `reachable: false` is a proof only with `complete: true`, and only over the outputs and payloads you list. The search forks on those values and no others: a refund of 150 is not explored unless `outputs` lists it. List each rule's boundary values (just under and over a threshold, the cap reached).
|
|
164
|
+
- Check that the search reaches the path the rule guards, for example a large refund through approval. Otherwise a model where that path does not exist proves the rule vacuously.
|
|
165
|
+
- See [Testing and verification](verify.md#reachability-checks) for payloads, ordering targets, and bounds.
|
|
166
|
+
|
|
167
|
+
## Step 3: record runs of the code
|
|
168
|
+
|
|
169
|
+
Wrap each function of the `effects` object with `recorder.fn` and run the code as it is:
|
|
170
|
+
|
|
171
|
+
```ts no-check
|
|
172
|
+
const recorder = createRecorder();
|
|
173
|
+
const outcome = await handleRefundRequest(
|
|
174
|
+
{
|
|
175
|
+
lookupOrder: recorder.fn("lookupOrder", lookupOrder),
|
|
176
|
+
decide: recorder.fn("decide", decide),
|
|
177
|
+
requestApproval: recorder.fn("requestApproval", requestApproval),
|
|
178
|
+
issueRefund: recorder.fn("issueRefund", issueRefund),
|
|
179
|
+
},
|
|
180
|
+
request,
|
|
181
|
+
);
|
|
182
|
+
recorder.output(outcome); // the return value, checked in step 5
|
|
183
|
+
const recording = recorder.recording;
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
The example records scripted scenarios with stand-in functions. A recording from a real run works the same way; save it as JSON and check it later.
|
|
187
|
+
|
|
188
|
+
## Step 4: map recorded calls to observations
|
|
189
|
+
|
|
190
|
+
An observation is one thing the run did, in the model's terms. Map the recording's effects with `flatMap`:
|
|
191
|
+
|
|
192
|
+
```ts no-check
|
|
193
|
+
function toObservations(recording: Recording): Observation[] {
|
|
194
|
+
return recording.effects.flatMap((effect): Observation[] => {
|
|
195
|
+
if (effect.kind === "output") return [{ kind: "returned", outcome: effect.value }];
|
|
196
|
+
if (effect.kind !== "call") return [];
|
|
197
|
+
switch (effect.name) {
|
|
198
|
+
// A call that threw carries its error, so its actor fails with it.
|
|
199
|
+
case "lookupOrder":
|
|
200
|
+
return [
|
|
201
|
+
{
|
|
202
|
+
kind: "lookup",
|
|
203
|
+
orderId: (effect.args as [string])[0],
|
|
204
|
+
order: effect.result as Order,
|
|
205
|
+
error: effect.error,
|
|
206
|
+
},
|
|
207
|
+
];
|
|
208
|
+
case "decide":
|
|
209
|
+
return [{ kind: "decision", decision: effect.result as Decision, error: effect.error }];
|
|
210
|
+
case "requestApproval": {
|
|
211
|
+
const [{ amount }] = effect.args as [{ amount: number }];
|
|
212
|
+
return [{ kind: "approval", approved: effect.result as boolean, amount }];
|
|
213
|
+
}
|
|
214
|
+
case "issueRefund":
|
|
215
|
+
return [
|
|
216
|
+
{
|
|
217
|
+
kind: "refund",
|
|
218
|
+
...(effect.args as [{ orderId: string; amount: number }])[0],
|
|
219
|
+
error: effect.error,
|
|
220
|
+
},
|
|
221
|
+
];
|
|
222
|
+
default:
|
|
223
|
+
return [];
|
|
224
|
+
}
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
Keep the mapping to what the call shows. Do not decide in the mapping what the code should have done next: that is the model's job, and a mapping that repeats the code's logic makes the check pass for any run.
|
|
230
|
+
|
|
231
|
+
## Step 5: check conformance
|
|
232
|
+
|
|
233
|
+
`checkConformance` runs the model with each actor answered from the observations:
|
|
234
|
+
|
|
235
|
+
```ts no-check
|
|
236
|
+
const report = await checkConformance(refundModel, {
|
|
237
|
+
input: { orderId: scenario.order.orderId },
|
|
238
|
+
observations: toObservations(recording),
|
|
239
|
+
actors: {
|
|
240
|
+
lookupOrder: {
|
|
241
|
+
observes: "lookup",
|
|
242
|
+
matches: ({ input, observation }) => input === observation.orderId,
|
|
243
|
+
output: ({ observation }) => observation.order,
|
|
244
|
+
error: ({ observation }) => observation.error,
|
|
245
|
+
},
|
|
246
|
+
decide: {
|
|
247
|
+
observes: "decision",
|
|
248
|
+
output: ({ observation }) => observation.decision,
|
|
249
|
+
error: ({ observation }) => observation.error,
|
|
250
|
+
},
|
|
251
|
+
issueRefund: {
|
|
252
|
+
observes: "refund",
|
|
253
|
+
matches: ({ input, observation }) =>
|
|
254
|
+
input.orderId === observation.orderId && input.amount === observation.amount,
|
|
255
|
+
error: ({ observation }) => observation.error,
|
|
256
|
+
},
|
|
257
|
+
},
|
|
258
|
+
events: {
|
|
259
|
+
approval: {
|
|
260
|
+
event: (observation) => ({ type: observation.approved ? "APPROVE" : "REJECT" }),
|
|
261
|
+
matches: ({ observation, context }) => observation.amount === context.amount,
|
|
262
|
+
},
|
|
263
|
+
},
|
|
264
|
+
output: {
|
|
265
|
+
observes: "returned",
|
|
266
|
+
matches: ({ output, observation }) => output.outcome === observation.outcome,
|
|
267
|
+
},
|
|
268
|
+
});
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
- An actor with `observes` consumes the next observation, which must be of that kind. `output` builds the actor's output from it.
|
|
272
|
+
- `matches` compares the actor's input with the observation. Here, the refund the code issued must be for the order and amount the model would refund.
|
|
273
|
+
- `events` sends an observation to the model as an event while the model waits, here the person's approval. Its `matches` compares the observation with the model's context: the person approved the amount the model is waiting to refund.
|
|
274
|
+
- `output` checks the code's return value. The model must finish where the code returned, and `matches` compares the model's output with the recorded one. Pass it for runs where the code threw too: record the throw as the outcome (`recorder.output({ threw: error instanceof Error ? error.message : String(error) })` in a `catch`, since code can throw a value that is not an `Error`) and model each failure the code throws as a final state whose output `matches` compares. Give that final state an output in the same shape as a return, such as `{ outcome: { threw: message } }`, and compare the fields, such as `output.outcome.threw === observation.outcome.threw`: the recorded outcome and the model's output are different objects, so `===` on the objects never matches. A model that errors instead of finishing leaves that outcome unconsumed, which deviates. Without `output`, a run that otherwise conforms reports a recorded throw as `extra-observations`, like a return value. Leaving the throw out of the observations leaves the outcome unchecked.
|
|
275
|
+
- An actor the recording cannot see takes no `observes`. It runs unchecked and is listed in `report.unobserved`.
|
|
276
|
+
- When an actor's output carries state the recording does not show, build it in `output` from the model's own `input` plus the observation.
|
|
277
|
+
|
|
278
|
+
All runs of the unchanged code conform. The full option and deviation reference is in [Testing and verification](verify.md#recorded-runs-against-a-model).
|
|
279
|
+
|
|
280
|
+
## Step 6: coverage
|
|
281
|
+
|
|
282
|
+
Pass each report's `trace` to `traceCoverage`:
|
|
283
|
+
|
|
284
|
+
```ts no-check
|
|
285
|
+
const coverage = traceCoverage(refundModel, reports.map((report) => report.trace));
|
|
286
|
+
coverage.unvisitedStates; // []
|
|
287
|
+
coverage.unvisitedHandlers; // transitions no recorded run took
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
An unvisited state or handler is a part of the model no recorded run exercised. Add a scenario that reaches it, or confirm that it cannot happen in production.
|
|
291
|
+
|
|
292
|
+
## A regression, caught
|
|
293
|
+
|
|
294
|
+
`regression.ts` is the code after a plausible change: repeat customers skip approval on refunds up to twice the threshold. The happy-path tests still pass. The conformance check reports:
|
|
295
|
+
|
|
296
|
+
```ts no-check
|
|
297
|
+
{
|
|
298
|
+
reason: "extra-observations",
|
|
299
|
+
expected: "idle, waiting for an event",
|
|
300
|
+
observed: { kind: "refund", amount: 150 },
|
|
301
|
+
state: "awaitingApproval",
|
|
302
|
+
}
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
The model was waiting for a person in `awaitingApproval`, and the code issued the refund instead. Only the scenario with a repeat customer and a mid-size refund shows it. The check finds a bug only on a run that exercises it. Coverage shows which parts of the model the recorded runs exercised.
|
|
306
|
+
|
|
307
|
+
## What the check covers
|
|
308
|
+
|
|
309
|
+
- It checks the order and the content of what the recording shows against what the model allows, and the code's return value when you pass `output`.
|
|
310
|
+
- It does not see effects the recording does not capture. An actor without `observes` is unchecked, and the report lists it.
|
|
311
|
+
- It does not prove anything about runs you did not record. The proofs cover every path through the outputs and payloads listed in [Step 2](#step-2-proofs), not every value an actor could return. Conformance ties those paths to the recorded runs.
|
|
312
|
+
- A wrong model can conform and prove its rules. Review the model, and treat a deviation as a question about the code or the model.
|
|
313
|
+
|
|
314
|
+
## Migrate later
|
|
315
|
+
|
|
316
|
+
When you want the rules enforced at runtime instead of checked after the fact, migrate the code to a machine. The model becomes the machine, its placeholder actors become the code's functions, and the same proofs run against it.
|
|
317
|
+
|
|
318
|
+
- [Migrating from a hand-rolled loop](from-a-loop.md) walks through a migration.
|
|
319
|
+
- The [`loop-to-machine`](https://github.com/statelyai/skills/blob/main/plugins/stately-agent/skills/loop-to-machine/SKILL.md) skill runs it with a coding agent.
|
|
320
|
+
- `checkParity` checks the migrated machine against the original code. See [Strangler-fig retrofit](strangler-fig.md#check-parity).
|
|
321
|
+
|
|
322
|
+
## Related
|
|
323
|
+
|
|
324
|
+
- [Testing and verification](verify.md): `canReach`, `explorePaths`, and the `checkConformance` reference.
|
|
325
|
+
- [Evals](evals.md): `traceCoverage`, recorders, and parity.
|
|
326
|
+
- [examples/verify-in-place](https://github.com/statelyai/agent/tree/next/examples/verify-in-place): the code on this page, with tests.
|