@statelyai/agent 2.0.0-alpha.35 → 2.0.0-alpha.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +138 -0
- package/dist/act-B9hLQls6.mjs +19 -0
- package/dist/act-CrX0JwWY.cjs +42 -0
- package/dist/ai-sdk.cjs +63 -7
- package/dist/ai-sdk.d.cts +13 -3
- package/dist/ai-sdk.d.mts +13 -3
- package/dist/ai-sdk.mjs +63 -7
- package/dist/{decision-BEsATcr1.cjs → decision-CMeuNZ1g.cjs} +262 -4
- package/dist/{decision-COkJNjOW.mjs → decision-m5hGmHpF.mjs} +239 -5
- package/dist/decision-object-B5zGSPFp.d.mts +15 -0
- package/dist/decision-object-DBcLWVvv.d.cts +15 -0
- package/dist/{run-agent-Ct_GZw_f.mjs → effects-BT_cSY0j.mjs} +1003 -109
- package/dist/{run-agent-s5D32w99.cjs → effects-DDseYtJk.cjs} +1037 -107
- package/dist/{event-log-C7jeH0y8.mjs → event-log-DkVlk3Rh.mjs} +30 -15
- package/dist/{event-log-BWSSLUff.cjs → event-log-Dkc-yIYu.cjs} +41 -14
- package/dist/{event-log-store-BzXehP7R.d.cts → event-log-store-BBS8so8a.d.mts} +2 -2
- package/dist/{event-log-store-W49TCJXi.mjs → event-log-store-Brx9tOHt.mjs} +1 -1
- package/dist/{event-log-store-CleWpsp5.cjs → event-log-store-Cl-YWDnq.cjs} +1 -1
- package/dist/{event-log-store-CWjN6lmM.d.mts → event-log-store-bc4ICGcS.d.cts} +2 -2
- package/dist/index.cjs +660 -286
- package/dist/index.d.cts +168 -9
- package/dist/index.d.mts +168 -9
- package/dist/index.mjs +639 -271
- package/dist/internal-C3Lqu538.d.cts +43 -0
- package/dist/internal-DUcdhk-i.d.mts +43 -0
- package/dist/log.cjs +2 -2
- package/dist/log.d.cts +1 -1
- package/dist/log.d.mts +1 -1
- package/dist/log.mjs +2 -2
- package/dist/machines.cjs +457 -41
- package/dist/machines.d.cts +114 -43
- package/dist/machines.d.mts +114 -43
- package/dist/machines.mjs +457 -42
- package/dist/{model-source-DmhTMK39.cjs → model-source-B4WQ8Idp.cjs} +1 -1
- package/dist/{model-source-CK3F4YuU.mjs → model-source-kjGwqAV8.mjs} +1 -1
- package/dist/openai.cjs +149 -36
- package/dist/openai.d.cts +28 -10
- package/dist/openai.d.mts +28 -10
- package/dist/openai.mjs +149 -36
- package/dist/otel.d.cts +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/{run-agent-BGdx5913.d.mts → run-agent-30mH2moE.d.cts} +91 -9
- package/dist/{run-agent-B1-Eqviq.d.cts → run-agent-DlPzBWj-.d.mts} +91 -9
- package/dist/{setup-agent-05uw4dPZ.mjs → setup-agent-BDmNr6PC.mjs} +9 -3
- package/dist/{setup-agent-BtfeFfF6.d.cts → setup-agent-BaNyz6Q3.d.cts} +4 -4
- package/dist/{setup-agent-C6nqzv_X.d.mts → setup-agent-C1CG_nfX.d.mts} +4 -4
- package/dist/{setup-agent-CXEBURdo.cjs → setup-agent-CmEToB8y.cjs} +9 -3
- package/dist/tanstack-ai.cjs +62 -4
- package/dist/tanstack-ai.d.cts +18 -6
- package/dist/tanstack-ai.d.mts +18 -6
- package/dist/tanstack-ai.mjs +62 -4
- package/dist/testing.cjs +2493 -1502
- package/dist/testing.d.cts +360 -256
- package/dist/testing.d.mts +360 -256
- package/dist/testing.mjs +2489 -1503
- package/dist/{text-logic-CV_4KGr6.d.cts → text-logic-BqxalgMr.d.mts} +87 -4
- package/dist/{text-logic-Ckh-rJFV.d.mts → text-logic-CSf9bqpP.d.cts} +87 -4
- package/dist/{types-Dp5_Ru4R.d.cts → types-DQUOOaZO.d.cts} +1 -1
- package/dist/{types-Dp5_Ru4R.d.mts → types-DQUOOaZO.d.mts} +1 -1
- package/dist/{usage-BDiJKyv9.d.cts → usage-Bm5nvpow.d.cts} +6 -1
- package/dist/{usage-DOoVUGVq.d.mts → usage-BnikdOca.d.mts} +6 -1
- package/dist/utils-7lSa4H-i.d.cts +403 -0
- package/dist/utils-CBbKw2fM.d.mts +403 -0
- package/dist/validate.cjs +44 -12
- package/dist/validate.d.cts +6 -6
- package/dist/validate.d.mts +6 -6
- package/dist/validate.mjs +44 -12
- package/dist/verify-ChyX6jq0.mjs +2306 -0
- package/dist/verify-CwlombbC.cjs +2359 -0
- package/dist/{verify-BQpE8hUz.d.cts → verify-DJHfEfm6.d.cts} +213 -14
- package/dist/{verify-CqeuEmIk.d.mts → verify-i9Hiwc5g.d.mts} +213 -14
- package/dist/with-model-calls-B_WLmOSl.cjs +248 -0
- package/dist/with-model-calls-BkOwdroS.mjs +213 -0
- package/docs/act.md +102 -0
- package/docs/advanced.md +105 -0
- package/docs/any-stack.md +62 -0
- package/docs/choosing-a-run-mode.md +55 -0
- package/docs/debugging.md +29 -0
- package/docs/decisions.md +473 -0
- package/docs/environment-and-policy.md +188 -0
- package/docs/evals.md +509 -0
- package/docs/event-log.md +312 -0
- package/docs/from-a-loop.md +528 -0
- package/docs/generate-machines.md +110 -0
- package/docs/hosts.md +414 -0
- package/docs/human-in-the-loop.md +177 -0
- package/docs/index.md +48 -0
- package/docs/judgments.md +281 -0
- package/docs/langgraph-comparison.md +198 -0
- package/docs/machines-as-data.md +454 -0
- package/docs/machines-presets.md +198 -0
- package/docs/machines.md +497 -0
- package/docs/messages.md +67 -0
- package/docs/meta.json +52 -0
- package/docs/models-and-providers.md +268 -0
- package/docs/multi-agent.md +68 -0
- package/docs/observability.md +109 -0
- package/docs/patterns.md +31 -0
- package/docs/persistence.md +207 -0
- package/docs/quickstart.md +121 -0
- package/docs/roadmap.md +22 -0
- package/docs/scope.md +75 -0
- package/docs/statechart-policy-examples.md +69 -0
- package/docs/steps.md +56 -0
- package/docs/strangler-fig.md +323 -0
- package/docs/text-requests.md +250 -0
- package/docs/thinking-in-state-machines.md +324 -0
- package/docs/tools.md +249 -0
- package/docs/upgrading.md +37 -0
- package/docs/usage-and-budgets.md +518 -0
- package/docs/validate-and-repair.md +253 -0
- package/docs/verify.md +492 -0
- package/docs/xstate-4-and-5-apps.md +161 -0
- package/llms.txt +53 -0
- package/package.json +10 -3
- package/readme.md +30 -15
- package/skills/add-evals/SKILL.md +3 -1
- package/skills/add-traces/SKILL.md +6 -1
- package/skills/build-machine/SKILL.md +84 -0
- package/skills/generate-machine/SKILL.md +1 -1
- package/skills/graph-engineering/SKILL.md +13 -3
- package/skills/improve-machine/SKILL.md +21 -2
- package/skills/loop-to-machine/SKILL.md +147 -192
- package/skills/loop-to-machine/reference.md +260 -0
- package/dist/utils-BnXdWih3.d.mts +0 -108
- package/dist/utils-ebVpCYoF.d.cts +0 -108
- package/dist/verify-Bpahmr34.mjs +0 -1241
- package/dist/verify-DCLZa9R0.cjs +0 -1288
- package/dist/with-model-calls-BXYdBdcu.cjs +0 -45
- package/dist/with-model-calls-Bv4V4EHC.mjs +0 -34
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
# Convert agent code to a machine: reference
|
|
2
|
+
|
|
3
|
+
Code and semantics behind [SKILL.md](SKILL.md). The rules are in the skill; this page is how.
|
|
4
|
+
|
|
5
|
+
## A gated action
|
|
6
|
+
|
|
7
|
+
An `agent.decide` with a closed choice, and the rule as a guard in the transition:
|
|
8
|
+
|
|
9
|
+
```ts
|
|
10
|
+
deciding: {
|
|
11
|
+
tags: ["decision"],
|
|
12
|
+
invoke: {
|
|
13
|
+
src: "agent.decide",
|
|
14
|
+
input: ({ context }) => ({
|
|
15
|
+
model: "decider", // a key the host's executors resolve
|
|
16
|
+
name: "chooseResolution", // tests, traces, and a later policy find it by this name
|
|
17
|
+
prompt: `Ticket: ${context.ticket}\nOrder: ${JSON.stringify(context.order)}`,
|
|
18
|
+
allowedEvents: ["REFUND", "ESCALATE", "CLOSE"],
|
|
19
|
+
// A closed choice: the decider picks one of these. The runtime drops the ones the guards refuse.
|
|
20
|
+
payloads: { REFUND: refundAmounts(context).map((amount) => ({ amount })) },
|
|
21
|
+
// The facts a decider reads. Arrives as `request.metadata.facts`.
|
|
22
|
+
metadata: { facts: { order: context.order, requestedAmount: context.requestedAmount } },
|
|
23
|
+
}),
|
|
24
|
+
onError: { target: "escalating" }, // retries exhausted, or no option legal
|
|
25
|
+
},
|
|
26
|
+
on: {
|
|
27
|
+
REFUND: ({ context, event }) =>
|
|
28
|
+
refundIsLegal(context, event.amount)
|
|
29
|
+
? { target: event.amount > APPROVAL_THRESHOLD ? "awaitingApproval" : "paying", context: { amount: event.amount } }
|
|
30
|
+
: undefined, // undefined rejects: the decision retries with feedback
|
|
31
|
+
ESCALATE: { target: "escalating" },
|
|
32
|
+
CLOSE: { target: "closing" },
|
|
33
|
+
},
|
|
34
|
+
},
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Proof options
|
|
38
|
+
|
|
39
|
+
```ts
|
|
40
|
+
const proofOptions: ExplorePathsOptions = {
|
|
41
|
+
payloads: () => ({
|
|
42
|
+
REFUND: [0, 40, 41, 100, 101, 240, 241, 800, 801].map((amount) => ({ amount })),
|
|
43
|
+
CUSTOMER_REPLIED: [{ reply: "It's ORD-1001" }, { reply: "no idea" }],
|
|
44
|
+
}),
|
|
45
|
+
outputs: { lookupOrder: [ownedOrder, otherCustomersOrder, refundedOrder, { found: false }] },
|
|
46
|
+
errors: { lookupOrder: new Error("down"), "agent.decide": new Error("exhausted") },
|
|
47
|
+
};
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
`errors[src]` takes one failure or an array of them (one failing branch per element; wrap an array-valued failure as `[[...]]`), and `"agent.decide"` is a valid key.
|
|
51
|
+
|
|
52
|
+
## A phase in the middle of a method
|
|
53
|
+
|
|
54
|
+
When the phase has no seam at its edges, copy only its lines into the carved actor and mark the copy:
|
|
55
|
+
|
|
56
|
+
```ts
|
|
57
|
+
// Copied verbatim from src/runtime.ts:412-470 (prepareToolCall) and :88-97 (normalizeArgs).
|
|
58
|
+
// checkParity call-args fail if this copy drifts from the original.
|
|
59
|
+
const prepare = createAsyncLogic<Prepared, ToolCall>({
|
|
60
|
+
run: async ({ input }) => {
|
|
61
|
+
/* copied lines */
|
|
62
|
+
},
|
|
63
|
+
});
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
When a public per-turn hook exists (an `onTurnEnd` or `beforeToolCall` method), replace it on the instance instead. Each call hands that boundary's facts to the machine and awaits its answer, which an actor resolves; the rest of the method stays legacy. Race the legacy `run()` against the machine so exits that never reach the hook (stop, provider errors) still settle it.
|
|
67
|
+
|
|
68
|
+
## Old-suite swap
|
|
69
|
+
|
|
70
|
+
The port imports the module the old suite mocks, so the factory must not import the port. Hold it in `vi.hoisted` and load it in `beforeAll`:
|
|
71
|
+
|
|
72
|
+
```ts
|
|
73
|
+
import { beforeAll, vi } from "vitest";
|
|
74
|
+
|
|
75
|
+
const ported = vi.hoisted(() => ({
|
|
76
|
+
module: undefined as undefined | typeof import("../src/port"),
|
|
77
|
+
}));
|
|
78
|
+
vi.mock("../../src/runner", async (importOriginal) => ({
|
|
79
|
+
...(await importOriginal<typeof import("../../src/runner")>()),
|
|
80
|
+
createRunner: (...args: Parameters<typeof import("../src/port").createRunner>) =>
|
|
81
|
+
ported.module!.createRunner(...args),
|
|
82
|
+
}));
|
|
83
|
+
beforeAll(async () => {
|
|
84
|
+
ported.module = await import("../src/port");
|
|
85
|
+
});
|
|
86
|
+
await import("../../src/runner.test"); // the old suite, unchanged
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
When the mock misses the code's own import of the module, patch the prototype from a setup file. Both imports are dynamic, so the module loads after the test file's mocks:
|
|
90
|
+
|
|
91
|
+
```ts
|
|
92
|
+
// old-suite.setup.ts, listed in the config's setupFiles
|
|
93
|
+
import { beforeAll } from "vitest";
|
|
94
|
+
|
|
95
|
+
beforeAll(async () => {
|
|
96
|
+
const { Executor } = await import("../../src/executor");
|
|
97
|
+
const { carvedRun } = await import("../src/port");
|
|
98
|
+
Executor.prototype.run = function (this: InstanceType<typeof Executor>, ...args) {
|
|
99
|
+
return carvedRun(this, ...args);
|
|
100
|
+
};
|
|
101
|
+
});
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## Recording golden runs
|
|
105
|
+
|
|
106
|
+
Wrap the model and tools, run the old code, save the recording. One recording per scenario and per stand-in. Record with the live model to capture real answers, or with a stand-in; either way the recording is the script for replay.
|
|
107
|
+
|
|
108
|
+
```ts
|
|
109
|
+
import { createRecorder } from "@statelyai/agent/testing";
|
|
110
|
+
|
|
111
|
+
const recorder = createRecorder();
|
|
112
|
+
const model = recorder.model(openai("gpt-5")); // or recorder.openai(new OpenAI())
|
|
113
|
+
const tools = recorder.tools(appTools); // AI SDK tools; recorder.tool(name, fn) for plain functions
|
|
114
|
+
recorder.output(await runLoop({ ...scenario, model, tools }));
|
|
115
|
+
writeFileSync(`fixtures/${scenario.id}.json`, JSON.stringify(recorder.recording));
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
### Effects-object workflows: `checkParity`
|
|
119
|
+
|
|
120
|
+
When every side effect goes through one `effects` object of async functions, `checkParity` (`@statelyai/agent/testing`) does the record, replay, and compare above in one call: it wraps each effect with `recorder.fn`, records `before`, replays `after` with no effect running, and returns one row per scenario (`equivalent`, `divergences`, `before`/`after` as `{ output }` or `{ error }`, `recording`).
|
|
121
|
+
|
|
122
|
+
```ts
|
|
123
|
+
const report = await checkParity({
|
|
124
|
+
scenarios, // { id, input }[]
|
|
125
|
+
effects: (scenario) => createFakeEffects(scenario),
|
|
126
|
+
before: (input, effects) => oldWorkflow(input, effects),
|
|
127
|
+
after: (input, effects) => step1.run(input, { effects }), // or a runAgent runner
|
|
128
|
+
});
|
|
129
|
+
expect(report.scenarios.filter((row) => !row.equivalent)).toEqual([]);
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
`wrapWorkflow(...).run(input, { recorder })` records a golden run from a normal run.
|
|
133
|
+
|
|
134
|
+
### Any other model client
|
|
135
|
+
|
|
136
|
+
`recorder.client(name, sdk, { methods })` wraps the named async methods of an SDK object and keeps its type; `recorder.fn(name, asyncFn)` wraps a client that is a function. Record the hand-rolled loop with the wrapped client, then put the same wrapper inside the machine's executor for replay:
|
|
137
|
+
|
|
138
|
+
```ts
|
|
139
|
+
import Anthropic from "@anthropic-ai/sdk";
|
|
140
|
+
import { runAgent } from "@statelyai/agent";
|
|
141
|
+
import { compareRecordings, createRecorder } from "@statelyai/agent/testing";
|
|
142
|
+
|
|
143
|
+
const methods = ["messages.create"];
|
|
144
|
+
|
|
145
|
+
// Record: the hand-rolled loop gets the wrapped client.
|
|
146
|
+
const recorder = createRecorder();
|
|
147
|
+
recorder.output(await runLoop(recorder.client("anthropic", new Anthropic(), { methods }), scenario));
|
|
148
|
+
|
|
149
|
+
// Replay: the same wrapper inside the machine's executor; the client is never called.
|
|
150
|
+
const replay = createRecorder({ replay: recorder.recording });
|
|
151
|
+
const client = replay.client("anthropic", {} as Anthropic, { methods });
|
|
152
|
+
const run = await runAgent(machine, {
|
|
153
|
+
input: scenario,
|
|
154
|
+
executors: {
|
|
155
|
+
// askModel: the old loop's call, moved unchanged.
|
|
156
|
+
generateText: async (request) => ({ result: await askModel(client, request.messages as Anthropic.MessageParam[]) }),
|
|
157
|
+
},
|
|
158
|
+
});
|
|
159
|
+
if (run.status === "done") replay.output(run.output);
|
|
160
|
+
const { divergences } = compareRecordings(recorder.recording, replay.recording);
|
|
161
|
+
expect(divergences).toEqual(INTENDED[scenario.id] ?? []); // the replay test fails on any unreviewed difference
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
- Each call is a `call` effect named `<name>.<method>` (`anthropic.messages.create`) with its arguments and its result or error.
|
|
165
|
+
- Replay answers each call with the next unused recorded call of the same name, in order, whatever its arguments; `compareRecordings` reports differing arguments as `call-args`.
|
|
166
|
+
- A result that is an async iterable (a stream) records its chunks; replay returns an async iterable of them.
|
|
167
|
+
- `redact(args)` records only what must match (model, offered tool names) and keeps secrets out. `serialize` maps a non-JSON result or chunk; `revive` rebuilds the replayed value, such as a stream object with helper methods.
|
|
168
|
+
|
|
169
|
+
## The parity harness
|
|
170
|
+
|
|
171
|
+
Each run becomes a `ParityRun`: the outcome, every write the world records, and the rules broken. The harness check runs the old code on both sides:
|
|
172
|
+
|
|
173
|
+
```ts
|
|
174
|
+
import { compareRuns, type ParityResult, type ParityRun } from "@statelyai/agent/testing";
|
|
175
|
+
|
|
176
|
+
function toParityRun(run: { outcome: string; world: World }): ParityRun {
|
|
177
|
+
return {
|
|
178
|
+
outcome: run.outcome,
|
|
179
|
+
// Every write the world records; model-written text as a count.
|
|
180
|
+
effects: { refunds: run.world.refunds, approvals: run.world.approvals, verified: [...run.world.verified], escalations: run.world.escalations.length },
|
|
181
|
+
violations: run.world.violations, // rules broken, from ground truth
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// A model factory, called once per scenario and per side. Construct a stateful or seeded
|
|
186
|
+
// model inside it (seeded from s.id); returning a shared stateless object is fine.
|
|
187
|
+
const runWith = (runner: typeof runLoop, model: (s: Scenario) => Model) => async (s: Scenario) =>
|
|
188
|
+
toParityRun(await runner({ ...s, model: model(s), world: createWorld(s) }));
|
|
189
|
+
|
|
190
|
+
const MODELS: Array<[string, (s: Scenario) => Model]> = [
|
|
191
|
+
["careful", () => carefulModel],
|
|
192
|
+
["sloppy", () => sloppyModel],
|
|
193
|
+
];
|
|
194
|
+
|
|
195
|
+
test.each(MODELS)("harness: the loop matches itself (%s)", async (_, model) => {
|
|
196
|
+
const report = await compareRuns({ scenarios: SCENARIOS, baseline: runWith(runLoop, model), candidate: runWith(runLoop, model) });
|
|
197
|
+
for (const row of report.scenarios) expect(row.candidate, row.id).toEqual(row.baseline);
|
|
198
|
+
});
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
The parity test switches the candidate to `runMachine`:
|
|
202
|
+
|
|
203
|
+
```ts
|
|
204
|
+
test.each(MODELS)("parity with the loop: %s", async (_, model) => {
|
|
205
|
+
const report = await compareRuns({ scenarios: SCENARIOS, baseline: runWith(runLoop, model), candidate: runWith(runMachine, model) });
|
|
206
|
+
const unreviewed = report.scenarios.filter((row) => row.label === "regression" && !REVIEWED[row.id]);
|
|
207
|
+
expect(unreviewed).toEqual([]);
|
|
208
|
+
// A reviewed difference still has to be exactly the difference a person accepted.
|
|
209
|
+
for (const row of report.scenarios.filter((row) => REVIEWED[row.id])) REVIEWED[row.id]!(row);
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
// Effect-only differences a person accepted: who, and the exact difference.
|
|
213
|
+
const REVIEWED: Record<string, (row: ParityResult<Scenario>) => void> = {
|
|
214
|
+
// "no-order-id-no-reply": (row) => { /* accepted by <name>: asks once, not three times */ expect(row.candidate.outcome).toBe(row.baseline.outcome); ... },
|
|
215
|
+
};
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
With no model in the loop, a spec baseline replaces the old code's run, and the candidate runs in a scripted world (the proposed calls, the order approvals arrive in, which tool fails):
|
|
219
|
+
|
|
220
|
+
```ts
|
|
221
|
+
// Confirmed by a person from the old code and its docs, before the machine existed.
|
|
222
|
+
const SPEC: Record<string, ParityRun> = {
|
|
223
|
+
/* "two-calls-one-rejected": { outcome: "completed", effects: { executed: ["read"], rejected: ["write"] }, violations: [] }, */
|
|
224
|
+
};
|
|
225
|
+
|
|
226
|
+
const report = await compareRuns({
|
|
227
|
+
scenarios: SCENARIOS,
|
|
228
|
+
baseline: async (s) => SPEC[s.id]!,
|
|
229
|
+
candidate: async (s) => toParityRun(await runMachine({ ...s, world: createScriptedWorld(s) })),
|
|
230
|
+
});
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
## Replay equivalence
|
|
234
|
+
|
|
235
|
+
Replay answers each model and tool call from the recording, so the machine needs no key and causes no side effects:
|
|
236
|
+
|
|
237
|
+
```ts
|
|
238
|
+
import { compareRecordings, createRecorder, type EffectDivergence, type Recording } from "@statelyai/agent/testing";
|
|
239
|
+
|
|
240
|
+
test.each(SCENARIOS)("equivalent to the recorded loop: $id", async (scenario) => {
|
|
241
|
+
const recording: Recording = JSON.parse(readFileSync(`fixtures/${scenario.id}.json`, "utf8"));
|
|
242
|
+
const replay = createRecorder({ replay: recording });
|
|
243
|
+
const model = replay.model(openai("gpt-5")); // never called in replay
|
|
244
|
+
replay.output(await runMachine({ ...scenario, model, tools: replay.tools(appTools) }));
|
|
245
|
+
const report = compareRecordings(recording, replay.recording);
|
|
246
|
+
expect(report.divergences.map(({ reason, message }) => ({ reason, message }))).toEqual(INTENDED[scenario.id] ?? []);
|
|
247
|
+
});
|
|
248
|
+
|
|
249
|
+
// Differences a person accepted: who, and the exact divergences.
|
|
250
|
+
const INTENDED: Record<string, Array<Pick<EffectDivergence, "reason" | "message">>> = {};
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
- Equivalent means the same model calls (count and shape: offered tool names, response format), the same tool calls with the same arguments, and the same output. Prompts and states may differ. Tool calls that ran at the same time compare in any order; sequential ones keep theirs.
|
|
254
|
+
- Values that differ on every run (generated ids, branch names, timestamps) fail `tool-input`. Pass the same `normalize` to `createRecorder({ replay, normalize })` and `compareRecordings(a, b, { normalize })` to redact them on both sides; never edit the fixtures. `normalize(effect, index, effects)` returns `undefined` to leave an effect out of `compareRecordings`, such as a repeat of an idempotent write; replay still answers from it. Drop an effect only when repeating it is harmless: a dropped `send` or `charge` hides a duplicate.
|
|
255
|
+
- `divergences` lists every difference, lined up so one extra call does not shift the rest; `divergence` is the first.
|
|
256
|
+
- Replay answers each model call with the earliest unused recorded call of the same method and shape, and each tool call with the earliest unused recorded call of the same name and input, not by position. A call the machine added or dropped does not shift the answers after it, as long as no other call has the same method and shape (or tool name and input). Calls that do are answered in recorded order, so an added or dropped one takes or leaves the next one's answer.
|
|
257
|
+
- A call no recorded answer fits throws `RecordingDivergenceError`, and `compareRecordings` names it. To run the machine's own path instead (its retry, its added check), pass `fallback: { model: standIn, tools: true }`: a stand-in model answers the call and the wrapped tool runs. Those effects are marked `unrecorded` and still count as extra.
|
|
258
|
+
- `model-call-shape`: the machine changed a call's tools or output format, for example a model-picked action that became `agent.decide`. Port the call unchanged first, prove equivalence, then change it in a separate step whose divergences are `INTENDED`. To keep replaying past a call you changed on purpose, pass `translate(recorded, actual)` to `createRecorder`: it returns the answer that fits the new call (`{ response }`, `{ stream }`, or `{ error }`), or `undefined` to leave that recorded call alone, and `compareRecordings` counts the pair as the same call. When the recorded answer already fits, a `modelCalls` function passed to both is enough: `(expected, actual) => expected.call === 2 || sameModelCallShape(expected, actual)`.
|
|
259
|
+
- When the machine asks the model a different way altogether (another protocol, one structured draft instead of a tool loop), compare what it did, not how it asked: `compareRecordings(a, b, { kinds: ["tool", "output"] })`, with the machine recorded on a stand-in model.
|
|
260
|
+
- A guard that blocks a rule break diverges on purpose. List it in `INTENDED` with the exact divergence.
|
|
@@ -1,108 +0,0 @@
|
|
|
1
|
-
import { C as UserMessage, S as ToolResultPart, _ as StandardSchemaV1, b as ToolCallPart, f as FilePart, l as AssistantMessage, n as AgentMessage, p as ImagePart, v as SystemMessage, x as ToolMessage, y as TextPart } from "./types-Dp5_Ru4R.mjs";
|
|
2
|
-
import { AnyMachineSnapshot, AnyStateMachine, StateValue } from "xstate";
|
|
3
|
-
//#region src/utils.d.ts
|
|
4
|
-
/** Returns the human-readable text carried by a message, ignoring non-text parts. */
|
|
5
|
-
declare function getMessageText(message: AgentMessage): string;
|
|
6
|
-
/**
|
|
7
|
-
* A stable, dependency-free structural fingerprint of a machine — a short hex
|
|
8
|
-
* `djb2` hash over its **structural** config only: state ids/nesting, transition
|
|
9
|
-
* event types and targets, invoke `src`s, `initial`, and any other serializable
|
|
10
|
-
* config fields. Function values (context/output builders, prompts, inline
|
|
11
|
-
* guards/actions) are excluded entirely, so two machines that differ only in
|
|
12
|
-
* their prompts or executors hash identically; adding/removing/retargeting a
|
|
13
|
-
* state or transition changes the hash.
|
|
14
|
-
*
|
|
15
|
-
* Used by {@link createAgentRuntime} to stamp settled snapshots with a `version` and to
|
|
16
|
-
* detect a structurally-edited machine on resume. It is a change detector, not
|
|
17
|
-
* a cryptographic digest — collisions are possible but unlikely for real
|
|
18
|
-
* configs. Declare `createMachine({ version })` to pin an explicit version instead.
|
|
19
|
-
*/
|
|
20
|
-
declare function getMachineStructuralHash(machine: AnyStateMachine): string;
|
|
21
|
-
/** Builds a {@link UserMessage} from a string or multimodal content parts. */
|
|
22
|
-
declare function userMessage(content: string | Array<TextPart | ImagePart | FilePart>): UserMessage;
|
|
23
|
-
/** Builds an {@link AssistantMessage} from a string or content parts (text, files, tool calls/results). */
|
|
24
|
-
declare function assistantMessage(content: string | Array<TextPart | FilePart | ToolCallPart | ToolResultPart>): AssistantMessage;
|
|
25
|
-
/** Builds a {@link SystemMessage}. */
|
|
26
|
-
declare function systemMessage(content: string): SystemMessage;
|
|
27
|
-
/** Builds a {@link ToolMessage} from one or more tool-result parts. */
|
|
28
|
-
declare function toolMessage(content: Array<ToolResultPart>): ToolMessage;
|
|
29
|
-
type MetaOfSnapshot<TSnapshot extends {
|
|
30
|
-
getMeta(): Record<string, unknown>;
|
|
31
|
-
}> = NonNullable<ReturnType<TSnapshot["getMeta"]>[keyof ReturnType<TSnapshot["getMeta"]>]>;
|
|
32
|
-
/**
|
|
33
|
-
* Returns the merged `meta` of a snapshot's active state(s) — the typed
|
|
34
|
-
* replacement for the `Object.values(snapshot.getMeta())[0]` dance.
|
|
35
|
-
*
|
|
36
|
-
* `snapshot.getMeta()` is keyed by state id; a leaf machine has one active
|
|
37
|
-
* state, but parallel/nested machines can have several. This shallow-merges
|
|
38
|
-
* every active state's meta into one object and returns `{}` when no active
|
|
39
|
-
* state declares meta.
|
|
40
|
-
*
|
|
41
|
-
* Merge order is fixed and does not depend on XState's internal node order:
|
|
42
|
-
* entries are sorted by structural depth (the state node's distance from the
|
|
43
|
-
* root, regardless of custom `id` strings), then by state id lexicographically,
|
|
44
|
-
* and merged in that order. So a deeper state's key wins over an ancestor's,
|
|
45
|
-
* and between equal-depth parallel siblings the later state id alphabetically
|
|
46
|
-
* wins.
|
|
47
|
-
*
|
|
48
|
-
* The return type is recovered from the snapshot's own `getMeta()` type, so a
|
|
49
|
-
* schema-typed machine (`setupAgent({ meta })`) yields the meta schema's
|
|
50
|
-
* output type. Pass an explicit `TMeta` to override when the snapshot is
|
|
51
|
-
* untyped (e.g. `AnyMachineSnapshot`).
|
|
52
|
-
*
|
|
53
|
-
* @example HITL: read the current state's interaction protocol off an idle
|
|
54
|
-
* snapshot to render for a human.
|
|
55
|
-
* ```ts
|
|
56
|
-
* const { interaction } = getStateMeta(result.snapshot);
|
|
57
|
-
* ```
|
|
58
|
-
*/
|
|
59
|
-
declare function getStateMeta<TSnapshot extends {
|
|
60
|
-
getMeta(): Record<string, unknown>;
|
|
61
|
-
} = AnyMachineSnapshot, TMeta = MetaOfSnapshot<TSnapshot>>(snapshot: TSnapshot): Partial<TMeta>;
|
|
62
|
-
/**
|
|
63
|
-
* Renders a snapshot's state value as one deterministic string, so a log line,
|
|
64
|
-
* a progress field, or a test assertion reads the same on every run.
|
|
65
|
-
*
|
|
66
|
-
* - An atomic state is its own name: `'writing'`.
|
|
67
|
-
* - A nested state joins with dots: `{ review: { editing: 'draft' } }` →
|
|
68
|
-
* `'review.editing.draft'`.
|
|
69
|
-
* - A parallel state brackets its regions, sorted by region name, so key order
|
|
70
|
-
* never leaks into the string: `{ p: { left: 'x', right: { a: 'b' } } }` →
|
|
71
|
-
* `'p:{left.x,right.a.b}'`.
|
|
72
|
-
*
|
|
73
|
-
* Prefer this over `String(snapshot.value)`, which renders every non-atomic
|
|
74
|
-
* value as `[object Object]`.
|
|
75
|
-
*
|
|
76
|
-
* @example
|
|
77
|
-
* ```ts
|
|
78
|
-
* onTransition: (snapshot) => console.log(getStatePath(snapshot));
|
|
79
|
-
* ```
|
|
80
|
-
*/
|
|
81
|
-
declare function getStatePath(snapshot: {
|
|
82
|
-
value: StateValue;
|
|
83
|
-
status?: string;
|
|
84
|
-
} | StateValue): string;
|
|
85
|
-
/**
|
|
86
|
-
* Structural guard for a {@link StandardSchemaV1}: `true` when `value` carries
|
|
87
|
-
* the `~standard` marker. Used to tell an already-schema'd tool `inputSchema`
|
|
88
|
-
* (a Zod/Valibot/… schema) apart from an SDK-specific schema wrapper that core
|
|
89
|
-
* can't read directly — see the `ai-sdk` tool pass-through.
|
|
90
|
-
*/
|
|
91
|
-
declare function isStandardSchema(value: unknown): value is StandardSchemaV1;
|
|
92
|
-
/**
|
|
93
|
-
* Pulls the JSON Schema off a {@link StandardSchemaV1} via its optional
|
|
94
|
-
* `~standard.jsonSchema.input()` extension (implemented by e.g. Zod v4's
|
|
95
|
-
* `z.toJSONSchema`), awaiting it when the producer is async. Returns
|
|
96
|
-
* `undefined` when the schema doesn't expose the extension. Use this to build
|
|
97
|
-
* a provider request's `response_format`/tool `parameters` from a schema.
|
|
98
|
-
*/
|
|
99
|
-
declare function getJsonSchema(schema?: StandardSchemaV1): Promise<Record<string, unknown> | undefined>;
|
|
100
|
-
/**
|
|
101
|
-
* Synchronous variant of {@link getJsonSchema}, for call sites that can't
|
|
102
|
-
* await (building tool/event descriptors inline). An async JSON Schema
|
|
103
|
-
* producer is treated as absent (returns `undefined`) — in practice Zod's
|
|
104
|
-
* `z.toJSONSchema` resolves synchronously.
|
|
105
|
-
*/
|
|
106
|
-
declare function getJsonSchemaSync(schema?: StandardSchemaV1): Record<string, unknown> | undefined;
|
|
107
|
-
//#endregion
|
|
108
|
-
export { getMachineStructuralHash as a, getStatePath as c, toolMessage as d, userMessage as f, getJsonSchemaSync as i, isStandardSchema as l, assistantMessage as n, getMessageText as o, getJsonSchema as r, getStateMeta as s, MetaOfSnapshot as t, systemMessage as u };
|
|
@@ -1,108 +0,0 @@
|
|
|
1
|
-
import { C as UserMessage, S as ToolResultPart, _ as StandardSchemaV1, b as ToolCallPart, f as FilePart, l as AssistantMessage, n as AgentMessage, p as ImagePart, v as SystemMessage, x as ToolMessage, y as TextPart } from "./types-Dp5_Ru4R.cjs";
|
|
2
|
-
import { AnyMachineSnapshot, AnyStateMachine, StateValue } from "xstate";
|
|
3
|
-
//#region src/utils.d.ts
|
|
4
|
-
/** Returns the human-readable text carried by a message, ignoring non-text parts. */
|
|
5
|
-
declare function getMessageText(message: AgentMessage): string;
|
|
6
|
-
/**
|
|
7
|
-
* A stable, dependency-free structural fingerprint of a machine — a short hex
|
|
8
|
-
* `djb2` hash over its **structural** config only: state ids/nesting, transition
|
|
9
|
-
* event types and targets, invoke `src`s, `initial`, and any other serializable
|
|
10
|
-
* config fields. Function values (context/output builders, prompts, inline
|
|
11
|
-
* guards/actions) are excluded entirely, so two machines that differ only in
|
|
12
|
-
* their prompts or executors hash identically; adding/removing/retargeting a
|
|
13
|
-
* state or transition changes the hash.
|
|
14
|
-
*
|
|
15
|
-
* Used by {@link createAgentRuntime} to stamp settled snapshots with a `version` and to
|
|
16
|
-
* detect a structurally-edited machine on resume. It is a change detector, not
|
|
17
|
-
* a cryptographic digest — collisions are possible but unlikely for real
|
|
18
|
-
* configs. Declare `createMachine({ version })` to pin an explicit version instead.
|
|
19
|
-
*/
|
|
20
|
-
declare function getMachineStructuralHash(machine: AnyStateMachine): string;
|
|
21
|
-
/** Builds a {@link UserMessage} from a string or multimodal content parts. */
|
|
22
|
-
declare function userMessage(content: string | Array<TextPart | ImagePart | FilePart>): UserMessage;
|
|
23
|
-
/** Builds an {@link AssistantMessage} from a string or content parts (text, files, tool calls/results). */
|
|
24
|
-
declare function assistantMessage(content: string | Array<TextPart | FilePart | ToolCallPart | ToolResultPart>): AssistantMessage;
|
|
25
|
-
/** Builds a {@link SystemMessage}. */
|
|
26
|
-
declare function systemMessage(content: string): SystemMessage;
|
|
27
|
-
/** Builds a {@link ToolMessage} from one or more tool-result parts. */
|
|
28
|
-
declare function toolMessage(content: Array<ToolResultPart>): ToolMessage;
|
|
29
|
-
type MetaOfSnapshot<TSnapshot extends {
|
|
30
|
-
getMeta(): Record<string, unknown>;
|
|
31
|
-
}> = NonNullable<ReturnType<TSnapshot["getMeta"]>[keyof ReturnType<TSnapshot["getMeta"]>]>;
|
|
32
|
-
/**
|
|
33
|
-
* Returns the merged `meta` of a snapshot's active state(s) — the typed
|
|
34
|
-
* replacement for the `Object.values(snapshot.getMeta())[0]` dance.
|
|
35
|
-
*
|
|
36
|
-
* `snapshot.getMeta()` is keyed by state id; a leaf machine has one active
|
|
37
|
-
* state, but parallel/nested machines can have several. This shallow-merges
|
|
38
|
-
* every active state's meta into one object and returns `{}` when no active
|
|
39
|
-
* state declares meta.
|
|
40
|
-
*
|
|
41
|
-
* Merge order is fixed and does not depend on XState's internal node order:
|
|
42
|
-
* entries are sorted by structural depth (the state node's distance from the
|
|
43
|
-
* root, regardless of custom `id` strings), then by state id lexicographically,
|
|
44
|
-
* and merged in that order. So a deeper state's key wins over an ancestor's,
|
|
45
|
-
* and between equal-depth parallel siblings the later state id alphabetically
|
|
46
|
-
* wins.
|
|
47
|
-
*
|
|
48
|
-
* The return type is recovered from the snapshot's own `getMeta()` type, so a
|
|
49
|
-
* schema-typed machine (`setupAgent({ meta })`) yields the meta schema's
|
|
50
|
-
* output type. Pass an explicit `TMeta` to override when the snapshot is
|
|
51
|
-
* untyped (e.g. `AnyMachineSnapshot`).
|
|
52
|
-
*
|
|
53
|
-
* @example HITL: read the current state's interaction protocol off an idle
|
|
54
|
-
* snapshot to render for a human.
|
|
55
|
-
* ```ts
|
|
56
|
-
* const { interaction } = getStateMeta(result.snapshot);
|
|
57
|
-
* ```
|
|
58
|
-
*/
|
|
59
|
-
declare function getStateMeta<TSnapshot extends {
|
|
60
|
-
getMeta(): Record<string, unknown>;
|
|
61
|
-
} = AnyMachineSnapshot, TMeta = MetaOfSnapshot<TSnapshot>>(snapshot: TSnapshot): Partial<TMeta>;
|
|
62
|
-
/**
|
|
63
|
-
* Renders a snapshot's state value as one deterministic string, so a log line,
|
|
64
|
-
* a progress field, or a test assertion reads the same on every run.
|
|
65
|
-
*
|
|
66
|
-
* - An atomic state is its own name: `'writing'`.
|
|
67
|
-
* - A nested state joins with dots: `{ review: { editing: 'draft' } }` →
|
|
68
|
-
* `'review.editing.draft'`.
|
|
69
|
-
* - A parallel state brackets its regions, sorted by region name, so key order
|
|
70
|
-
* never leaks into the string: `{ p: { left: 'x', right: { a: 'b' } } }` →
|
|
71
|
-
* `'p:{left.x,right.a.b}'`.
|
|
72
|
-
*
|
|
73
|
-
* Prefer this over `String(snapshot.value)`, which renders every non-atomic
|
|
74
|
-
* value as `[object Object]`.
|
|
75
|
-
*
|
|
76
|
-
* @example
|
|
77
|
-
* ```ts
|
|
78
|
-
* onTransition: (snapshot) => console.log(getStatePath(snapshot));
|
|
79
|
-
* ```
|
|
80
|
-
*/
|
|
81
|
-
declare function getStatePath(snapshot: {
|
|
82
|
-
value: StateValue;
|
|
83
|
-
status?: string;
|
|
84
|
-
} | StateValue): string;
|
|
85
|
-
/**
|
|
86
|
-
* Structural guard for a {@link StandardSchemaV1}: `true` when `value` carries
|
|
87
|
-
* the `~standard` marker. Used to tell an already-schema'd tool `inputSchema`
|
|
88
|
-
* (a Zod/Valibot/… schema) apart from an SDK-specific schema wrapper that core
|
|
89
|
-
* can't read directly — see the `ai-sdk` tool pass-through.
|
|
90
|
-
*/
|
|
91
|
-
declare function isStandardSchema(value: unknown): value is StandardSchemaV1;
|
|
92
|
-
/**
|
|
93
|
-
* Pulls the JSON Schema off a {@link StandardSchemaV1} via its optional
|
|
94
|
-
* `~standard.jsonSchema.input()` extension (implemented by e.g. Zod v4's
|
|
95
|
-
* `z.toJSONSchema`), awaiting it when the producer is async. Returns
|
|
96
|
-
* `undefined` when the schema doesn't expose the extension. Use this to build
|
|
97
|
-
* a provider request's `response_format`/tool `parameters` from a schema.
|
|
98
|
-
*/
|
|
99
|
-
declare function getJsonSchema(schema?: StandardSchemaV1): Promise<Record<string, unknown> | undefined>;
|
|
100
|
-
/**
|
|
101
|
-
* Synchronous variant of {@link getJsonSchema}, for call sites that can't
|
|
102
|
-
* await (building tool/event descriptors inline). An async JSON Schema
|
|
103
|
-
* producer is treated as absent (returns `undefined`) — in practice Zod's
|
|
104
|
-
* `z.toJSONSchema` resolves synchronously.
|
|
105
|
-
*/
|
|
106
|
-
declare function getJsonSchemaSync(schema?: StandardSchemaV1): Record<string, unknown> | undefined;
|
|
107
|
-
//#endregion
|
|
108
|
-
export { getMachineStructuralHash as a, getStatePath as c, toolMessage as d, userMessage as f, getJsonSchemaSync as i, isStandardSchema as l, assistantMessage as n, getMessageText as o, getJsonSchema as r, getStateMeta as s, MetaOfSnapshot as t, systemMessage as u };
|