@statelyai/agent 2.0.0-alpha.35 → 2.0.0-alpha.37

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/CHANGELOG.md +138 -0
  2. package/dist/act-B9hLQls6.mjs +19 -0
  3. package/dist/act-CrX0JwWY.cjs +42 -0
  4. package/dist/ai-sdk.cjs +63 -7
  5. package/dist/ai-sdk.d.cts +13 -3
  6. package/dist/ai-sdk.d.mts +13 -3
  7. package/dist/ai-sdk.mjs +63 -7
  8. package/dist/{decision-BEsATcr1.cjs → decision-CMeuNZ1g.cjs} +262 -4
  9. package/dist/{decision-COkJNjOW.mjs → decision-m5hGmHpF.mjs} +239 -5
  10. package/dist/decision-object-B5zGSPFp.d.mts +15 -0
  11. package/dist/decision-object-DBcLWVvv.d.cts +15 -0
  12. package/dist/{run-agent-Ct_GZw_f.mjs → effects-BT_cSY0j.mjs} +1003 -109
  13. package/dist/{run-agent-s5D32w99.cjs → effects-DDseYtJk.cjs} +1037 -107
  14. package/dist/{event-log-C7jeH0y8.mjs → event-log-DkVlk3Rh.mjs} +30 -15
  15. package/dist/{event-log-BWSSLUff.cjs → event-log-Dkc-yIYu.cjs} +41 -14
  16. package/dist/{event-log-store-BzXehP7R.d.cts → event-log-store-BBS8so8a.d.mts} +2 -2
  17. package/dist/{event-log-store-W49TCJXi.mjs → event-log-store-Brx9tOHt.mjs} +1 -1
  18. package/dist/{event-log-store-CleWpsp5.cjs → event-log-store-Cl-YWDnq.cjs} +1 -1
  19. package/dist/{event-log-store-CWjN6lmM.d.mts → event-log-store-bc4ICGcS.d.cts} +2 -2
  20. package/dist/index.cjs +660 -286
  21. package/dist/index.d.cts +168 -9
  22. package/dist/index.d.mts +168 -9
  23. package/dist/index.mjs +639 -271
  24. package/dist/internal-C3Lqu538.d.cts +43 -0
  25. package/dist/internal-DUcdhk-i.d.mts +43 -0
  26. package/dist/log.cjs +2 -2
  27. package/dist/log.d.cts +1 -1
  28. package/dist/log.d.mts +1 -1
  29. package/dist/log.mjs +2 -2
  30. package/dist/machines.cjs +457 -41
  31. package/dist/machines.d.cts +114 -43
  32. package/dist/machines.d.mts +114 -43
  33. package/dist/machines.mjs +457 -42
  34. package/dist/{model-source-DmhTMK39.cjs → model-source-B4WQ8Idp.cjs} +1 -1
  35. package/dist/{model-source-CK3F4YuU.mjs → model-source-kjGwqAV8.mjs} +1 -1
  36. package/dist/openai.cjs +149 -36
  37. package/dist/openai.d.cts +28 -10
  38. package/dist/openai.d.mts +28 -10
  39. package/dist/openai.mjs +149 -36
  40. package/dist/otel.d.cts +1 -1
  41. package/dist/otel.d.mts +1 -1
  42. package/dist/{run-agent-BGdx5913.d.mts → run-agent-30mH2moE.d.cts} +91 -9
  43. package/dist/{run-agent-B1-Eqviq.d.cts → run-agent-DlPzBWj-.d.mts} +91 -9
  44. package/dist/{setup-agent-05uw4dPZ.mjs → setup-agent-BDmNr6PC.mjs} +9 -3
  45. package/dist/{setup-agent-BtfeFfF6.d.cts → setup-agent-BaNyz6Q3.d.cts} +4 -4
  46. package/dist/{setup-agent-C6nqzv_X.d.mts → setup-agent-C1CG_nfX.d.mts} +4 -4
  47. package/dist/{setup-agent-CXEBURdo.cjs → setup-agent-CmEToB8y.cjs} +9 -3
  48. package/dist/tanstack-ai.cjs +62 -4
  49. package/dist/tanstack-ai.d.cts +18 -6
  50. package/dist/tanstack-ai.d.mts +18 -6
  51. package/dist/tanstack-ai.mjs +62 -4
  52. package/dist/testing.cjs +2493 -1502
  53. package/dist/testing.d.cts +360 -256
  54. package/dist/testing.d.mts +360 -256
  55. package/dist/testing.mjs +2489 -1503
  56. package/dist/{text-logic-CV_4KGr6.d.cts → text-logic-BqxalgMr.d.mts} +87 -4
  57. package/dist/{text-logic-Ckh-rJFV.d.mts → text-logic-CSf9bqpP.d.cts} +87 -4
  58. package/dist/{types-Dp5_Ru4R.d.cts → types-DQUOOaZO.d.cts} +1 -1
  59. package/dist/{types-Dp5_Ru4R.d.mts → types-DQUOOaZO.d.mts} +1 -1
  60. package/dist/{usage-BDiJKyv9.d.cts → usage-Bm5nvpow.d.cts} +6 -1
  61. package/dist/{usage-DOoVUGVq.d.mts → usage-BnikdOca.d.mts} +6 -1
  62. package/dist/utils-7lSa4H-i.d.cts +403 -0
  63. package/dist/utils-CBbKw2fM.d.mts +403 -0
  64. package/dist/validate.cjs +44 -12
  65. package/dist/validate.d.cts +6 -6
  66. package/dist/validate.d.mts +6 -6
  67. package/dist/validate.mjs +44 -12
  68. package/dist/verify-ChyX6jq0.mjs +2306 -0
  69. package/dist/verify-CwlombbC.cjs +2359 -0
  70. package/dist/{verify-BQpE8hUz.d.cts → verify-DJHfEfm6.d.cts} +213 -14
  71. package/dist/{verify-CqeuEmIk.d.mts → verify-i9Hiwc5g.d.mts} +213 -14
  72. package/dist/with-model-calls-B_WLmOSl.cjs +248 -0
  73. package/dist/with-model-calls-BkOwdroS.mjs +213 -0
  74. package/docs/act.md +102 -0
  75. package/docs/advanced.md +105 -0
  76. package/docs/any-stack.md +62 -0
  77. package/docs/choosing-a-run-mode.md +55 -0
  78. package/docs/debugging.md +29 -0
  79. package/docs/decisions.md +473 -0
  80. package/docs/environment-and-policy.md +188 -0
  81. package/docs/evals.md +509 -0
  82. package/docs/event-log.md +312 -0
  83. package/docs/from-a-loop.md +528 -0
  84. package/docs/generate-machines.md +110 -0
  85. package/docs/hosts.md +414 -0
  86. package/docs/human-in-the-loop.md +177 -0
  87. package/docs/index.md +48 -0
  88. package/docs/judgments.md +281 -0
  89. package/docs/langgraph-comparison.md +198 -0
  90. package/docs/machines-as-data.md +454 -0
  91. package/docs/machines-presets.md +198 -0
  92. package/docs/machines.md +497 -0
  93. package/docs/messages.md +67 -0
  94. package/docs/meta.json +52 -0
  95. package/docs/models-and-providers.md +268 -0
  96. package/docs/multi-agent.md +68 -0
  97. package/docs/observability.md +109 -0
  98. package/docs/patterns.md +31 -0
  99. package/docs/persistence.md +207 -0
  100. package/docs/quickstart.md +121 -0
  101. package/docs/roadmap.md +22 -0
  102. package/docs/scope.md +75 -0
  103. package/docs/statechart-policy-examples.md +69 -0
  104. package/docs/steps.md +56 -0
  105. package/docs/strangler-fig.md +323 -0
  106. package/docs/text-requests.md +250 -0
  107. package/docs/thinking-in-state-machines.md +324 -0
  108. package/docs/tools.md +249 -0
  109. package/docs/upgrading.md +37 -0
  110. package/docs/usage-and-budgets.md +518 -0
  111. package/docs/validate-and-repair.md +253 -0
  112. package/docs/verify.md +492 -0
  113. package/docs/xstate-4-and-5-apps.md +161 -0
  114. package/llms.txt +53 -0
  115. package/package.json +10 -3
  116. package/readme.md +30 -15
  117. package/skills/add-evals/SKILL.md +3 -1
  118. package/skills/add-traces/SKILL.md +6 -1
  119. package/skills/build-machine/SKILL.md +84 -0
  120. package/skills/generate-machine/SKILL.md +1 -1
  121. package/skills/graph-engineering/SKILL.md +13 -3
  122. package/skills/improve-machine/SKILL.md +21 -2
  123. package/skills/loop-to-machine/SKILL.md +147 -192
  124. package/skills/loop-to-machine/reference.md +260 -0
  125. package/dist/utils-BnXdWih3.d.mts +0 -108
  126. package/dist/utils-ebVpCYoF.d.cts +0 -108
  127. package/dist/verify-Bpahmr34.mjs +0 -1241
  128. package/dist/verify-DCLZa9R0.cjs +0 -1288
  129. package/dist/with-model-calls-BXYdBdcu.cjs +0 -45
  130. package/dist/with-model-calls-Bv4V4EHC.mjs +0 -34
@@ -0,0 +1,253 @@
1
+ ---
2
+ title: Validate and repair model output with a loop
3
+ description: Check model output with your own host code, feed the rejection back as a repair prompt, and cap the loop in the machine.
4
+ ---
5
+
6
+ > **Alpha:** `@statelyai/agent` 2.0 is in alpha. APIs can change between releases; pin an exact version. Feedback: [github.com/statelyai/agent](https://github.com/statelyai/agent/issues).
7
+
8
+ This page builds a machine that asks a model for output, validates it with the app's own code, and sends the rejection back to the model as a repair prompt, up to a fixed number of rounds. The worked version is [examples/generate-and-repair](https://github.com/statelyai/agent/blob/next/examples/generate-and-repair/index.ts).
9
+
10
+ ## When to use this
11
+
12
+ Use a repair loop when validity is a question your code answers, not the schema. A [structured request](text-requests.md#structured-output-vs-plain-text) already guarantees the output parses and matches its schema; core rejects anything else before the machine sees it. What a schema cannot check is whether the output means anything: whether the SQL runs against your tables, whether the machine config's transitions name states that exist, whether the cited document ID is in your corpus. Answering that needs the database, the parser, or the corpus, so it happens in a host actor.
13
+
14
+ Two other mechanisms handle narrower cases:
15
+
16
+ - **SDK-level JSON repair**, such as the AI SDK's `experimental_repairText`, patches malformed JSON before parsing. It is syntactic and single-shot: one pass, no model call, no knowledge of your domain. It fixes a truncated brace. It does not know that `"open"` is not a state.
17
+ - **Decision retries** re-ask the model when it picks an event the machine will not accept. That is [decisions](decisions.md), and the legal set is already in the prompt.
18
+
19
+ A repair loop is the case left over: a semantic rejection, described in words, sent back as a new model call.
20
+
21
+ ## The machine
22
+
23
+ Six states. Model calls are invokes; the branching is the machine's.
24
+
25
+ ```ts no-check
26
+ initial: "generating",
27
+ states: {
28
+ generating: {
29
+ // Three invokes of the same request: the fan-out is in the machine.
30
+ invoke: ["generateConfig1", "generateConfig2", "generateConfig3"].map((id) => ({
31
+ id,
32
+ src: "generateConfig",
33
+ input: ({ context }) => ({ prompt: context.prompt }),
34
+ // Targetless: the slot records its draft and the other two keep running.
35
+ onDone: ({ context, output }) => ({
36
+ context: { pending: [...context.pending, output.result], settled: context.settled + 1 },
37
+ }),
38
+ onError: ({ context, event }) => ({
39
+ context: { settled: context.settled + 1, failureReason: reasonFor(event.error) },
40
+ }),
41
+ })),
42
+ // Leave only once every slot has settled, either way.
43
+ always: ({ context }) => {
44
+ if (context.settled < 3) return;
45
+ const [candidate, ...rest] = context.pending;
46
+ return candidate === undefined
47
+ ? { target: "failed" }
48
+ : { target: "parsing", context: { candidate, pending: rest } };
49
+ },
50
+ },
51
+ parsing: {
52
+ invoke: {
53
+ src: "parseConfig", // the host's validator
54
+ input: ({ context }) => ({ text: context.candidate }),
55
+ onDone: ({ output }) => ({ target: "done", context: { config: output } }),
56
+ onError: ({ context, event }) => {
57
+ const error = errorMessage(event.error);
58
+ const [next, ...rest] = context.pending;
59
+ // Try the next candidate before spending a repair.
60
+ return next === undefined
61
+ ? { target: "checkingRepairBudget", context: { lastError: error } }
62
+ : {
63
+ target: "parsing",
64
+ reenter: true,
65
+ context: { candidate: next, pending: rest, lastError: error },
66
+ };
67
+ },
68
+ },
69
+ },
70
+ checkingRepairBudget: {
71
+ type: "choice",
72
+ choice: ({ context }) =>
73
+ context.repairs < context.maxRepairs
74
+ ? { target: "repairing" }
75
+ : { target: "failed", context: { failureReason: `No valid config after ${context.maxRepairs} rounds.` } },
76
+ },
77
+ repairing: {
78
+ invoke: {
79
+ src: "repairConfig",
80
+ input: ({ context }) => ({
81
+ prompt: context.prompt,
82
+ code: context.candidate,
83
+ error: context.lastError ?? "unknown error",
84
+ }),
85
+ onDone: ({ context, output }) => ({
86
+ target: "parsing",
87
+ context: { candidate: output.result, pending: [], repairs: context.repairs + 1 },
88
+ }),
89
+ onError: { target: "failed" },
90
+ },
91
+ },
92
+ done: { type: "final", output: ({ context }) => ({ summary: summarize(context), config: context.config, repairs: context.repairs }) },
93
+ failed: { type: "final", output: ({ context }) => ({ summary: context.failureReason ?? "", config: null, repairs: context.repairs }) },
94
+ }
95
+ ```
96
+
97
+ Four things are worth reading off that shape.
98
+
99
+ `generating` fans out by invoking the same request three times. The concurrency is a fact about the state, not a hidden option on the request, and the `always` transition is the join: nothing leaves `generating` until all three slots have settled.
100
+
101
+ `parsing` re-enters itself while candidates remain. The transition sets `reenter: true`, so leaving and re-entering the state restarts the invoke with the next candidate. Only when `pending` is empty does the machine leave for `checkingRepairBudget`.
102
+
103
+ `checkingRepairBudget` is a [choice state](machines.md#choice-states). It routes on `repairs < maxRepairs` and never waits, so the budget is a fact about the machine rather than a counter buried in a prompt. A choice state is transient, so it never appears in the transition log; only the states around it do.
104
+
105
+ `failed` is a real outcome with its own output, not a thrown error. A run that gives up still returns a `summary` explaining why.
106
+
107
+ ## The validator as a host actor
108
+
109
+ The machine declares `parseConfig` as an ordinary actor and leaves it empty. The host supplies the real one:
110
+
111
+ ```ts no-check
112
+ import { createAsyncLogic } from "xstate";
113
+
114
+ export const parseConfigActor = createAsyncLogic<GeneratedMachineConfig, { text: string }>({
115
+ run: async ({ input }) => parseGeneratedConfig(input.text),
116
+ });
117
+
118
+ export const machine = agentMachine.provide({ actors: { parseConfig: parseConfigActor } });
119
+ ```
120
+
121
+ The validator throws, with a message written for the model to read:
122
+
123
+ ```ts no-check
124
+ throw new Error(
125
+ `State "${stateName}" transitions on "${eventType}" to "${target}", which is not a declared state. ` +
126
+ `Declared states: ${stateNames.join(", ")}.`,
127
+ );
128
+ ```
129
+
130
+ That message is the repair request's whole input. A validator that throws `Invalid config` produces a repair round that guesses. Name the offending value and the legal set.
131
+
132
+ Keeping the validator behind `provide` means the machine artifact carries no database client, no file system, and no corpus. It stays serializable, testable, and portable between hosts.
133
+
134
+ ## Fan-out and truncation
135
+
136
+ `generating` samples three drafts by invoking `generateConfig` three times. There is no `candidates` option on a request: fan-out is spelled out in the machine, so the concurrency, the join, and what a failed call costs are all visible in the state.
137
+
138
+ Fan-out and repairs solve different halves of the problem. The three calls run concurrently, so three drafts cost roughly one call's latency rather than three, and trying all three before spending a repair means an unlucky sample never costs a repair round.
139
+
140
+ A `finishReason` of `'length'` makes the adapter throw an [`AgentTruncatedError`](text-requests.md#finish-reason-and-truncation) for a structured request. With separate invokes, that failure lands on one slot: its `onError` marks the slot settled with no draft, and the other two carry on. Only when every slot failed is `pending` empty, and then there is no partial output to repair from, so the machine stops:
141
+
142
+ ```ts no-check
143
+ // `event.error` is `unknown`: any value can reach an onError. `errorCode`
144
+ // reads `code` only when the failure is an object that has one, so a thrown
145
+ // string or a plain Error falls through to the generic message.
146
+ onError: ({ context, event }) => ({
147
+ context: {
148
+ settled: context.settled + 1,
149
+ failureReason:
150
+ errorCode(event.error) === "truncated"
151
+ ? "The model ran out of output tokens before finishing a config. Ask for a smaller machine, or raise maxOutputTokens."
152
+ : `Generating a config failed: ${errorMessage(event.error)}`,
153
+ },
154
+ }),
155
+ ```
156
+
157
+ Branch on the code, not on `instanceof AgentTruncatedError`: the error crosses a package boundary, and two copies of the library in one tree make the class check fail where the code still matches.
158
+
159
+ ## Capping the loop
160
+
161
+ The machine's own cap is `checkingRepairBudget`. It is the one the app reasons about: `repairs` is in context, it is in the output, and a test asserts on it.
162
+
163
+ `maxModelCalls` is the second cap, and a different kind. It is the host's runaway backstop, it counts every model and decision call in the run, and the machine cannot read it. See [Usage and budgets](usage-and-budgets.md#the-global-backstop-maxmodelcalls).
164
+
165
+ ```ts no-check
166
+ const result = await runToQuiescence(
167
+ createAgentRuntime(machine, { executors, maxModelCalls: 10 }),
168
+ { input },
169
+ );
170
+ ```
171
+
172
+ With three author calls and `maxRepairs: 2`, a worst-case run makes five calls, so `result.usage.modelCalls` is `5`.
173
+
174
+ ## Testing the loop without a key
175
+
176
+ The loop is worth testing precisely because the interesting paths are the ones a live run rarely takes. A [plain-function executor](evals.md#plain-function-executors) supplies canned model output while the real validator runs, so every branch is a plain unit test.
177
+
178
+ ```ts no-check
179
+ const answers: Record<string, string[]> = {
180
+ generateConfig: [BAD_JSON, NO_BLOCK, BAD_TARGET],
181
+ repairConfig: [VALID],
182
+ };
183
+ const calls: string[] = [];
184
+
185
+ const result = await runToQuiescence(
186
+ createAgentRuntime(machine, {
187
+ executors: {
188
+ generateText: async (request) => {
189
+ calls.push(request.name);
190
+ return { result: answers[request.name].shift() };
191
+ },
192
+ },
193
+ }),
194
+ { input: { prompt: "a turnstile" } },
195
+ );
196
+
197
+ expect(result.output.repairs).toBe(1);
198
+ expect(calls).toEqual(["generateConfig", "generateConfig", "generateConfig", "repairConfig"]);
199
+ ```
200
+
201
+ One queued answer per invoke: the fan-out makes three `generateConfig` calls, so the queue holds three drafts. Three cases cover the loop:
202
+
203
+ - **One candidate is valid.** The run finishes with `repairs === 0`, and no repair request was ever built.
204
+ - **All candidates are rejected, and the repair fixes it.** `repairs === 1`, and `matchesTrajectory` pins the path through `repairing` and back into `parsing`.
205
+ - **No repair ever parses.** The run ends in `failed` with `repairs === maxRepairs`, `config` is `null`, and `result.usage.modelCalls` is `3 + maxRepairs`, which is the assertion that the cap held. `repairs` counts rounds that produced a candidate to re-parse: a repair _call_ that fails goes straight to `failed` through `repairing`'s `onError` without spending a round, so that path ends with `repairs` below `maxRepairs`.
206
+
207
+ `assertAgentMachine(machine)` covers the structure statically: a clean lint is the proof that `failed` is a target of some transition. [`canReach`](verify.md#reachability-checks) covers it dynamically. Exploration tracks every invoke a state is still waiting on, so it walks straight through the three-way fan-out, and a canned `parseConfig` failure drives the loop to the end of its repair budget.
208
+
209
+ ```ts no-check
210
+ const canned = {
211
+ input: { prompt: "a turnstile" },
212
+ outputs: { generateConfig: [VALID], repairConfig: [VALID], parseConfig: [PARSED] },
213
+ };
214
+
215
+ // The parser rejects every candidate and every repair: the budget runs out.
216
+ expect(
217
+ (await canReach(machine, "failed", { ...canned, errors: { parseConfig: new Error("bad") } }))
218
+ .reachable,
219
+ ).toBe(true);
220
+ expect((await canReach(machine, "done", canned)).reachable).toBe(true);
221
+ ```
222
+
223
+ ## Evaluating the generate request live
224
+
225
+ Once the loop is pinned, the question that remains is whether a prompt change to the author request makes the parse succeed more often. That is one call in a chain, so score it with [`runSeam`](evals.md#seam-evals): every other request stays scripted, and the seam gets the real model.
226
+
227
+ ```ts no-check
228
+ const { generateText } = createAiSdkExecutors({ models });
229
+
230
+ const run = await runSeam(machine, {
231
+ input: { prompt: "a turnstile that locks and unlocks" },
232
+ seam: { request: "generateConfig" },
233
+ candidate: generateText,
234
+ // The script is the whole call plan, seam slot included: three
235
+ // `generateConfig` calls fan out, and the live one still consumes its slot.
236
+ scripts: { generateConfig: [VALID, VALID, VALID], repairConfig: [VALID] },
237
+ });
238
+
239
+ // The branch the live candidate caused: straight to `done`, or through `repairing`.
240
+ matchesTrajectory(run.after.statePath, ["parsing", "done"]).matched;
241
+ run.seamUsage?.totalTokens;
242
+ ```
243
+
244
+ The score is the branch the seam caused, not a rubric over its text. A candidate prompt that reaches `done` without entering `repairing` is better by the only measure the app has, and `run.seamUsage` says what it cost.
245
+
246
+ ## Related
247
+
248
+ - [examples/generate-and-repair](https://github.com/statelyai/agent/blob/next/examples/generate-and-repair/index.ts): the full machine, the validator, and the three scripted tests.
249
+ - [Text requests](text-requests.md): finish reasons and `AgentTruncatedError`.
250
+ - [Hosts and executors](hosts.md): the executor contract, and what a host owes a truncated call.
251
+ - [Testing and verification](verify.md): linting, scripted playthroughs, and reachability.
252
+ - [Evals](evals.md): seam runs and trajectory scoring.
253
+ - [Usage and budgets](usage-and-budgets.md): `maxModelCalls` and per-run token accounting.