@parcha/agentrun-dsl 0.1.0-beta.11 → 0.1.0-beta.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/dist/author.d.ts +9 -2
  2. package/dist/author.js +6 -3
  3. package/dist/decision-receipts.d.ts +98 -0
  4. package/dist/decision-receipts.js +51 -0
  5. package/dist/index.d.ts +5 -3
  6. package/dist/index.js +2 -1
  7. package/dist/recovery/binding.d.ts +10 -0
  8. package/dist/recovery/binding.js +43 -0
  9. package/dist/recovery/call-key.d.ts +13 -0
  10. package/dist/recovery/call-key.js +24 -0
  11. package/dist/recovery/canonical-hash.d.ts +7 -0
  12. package/dist/recovery/canonical-hash.js +22 -0
  13. package/dist/recovery/driver.d.ts +186 -0
  14. package/dist/recovery/driver.js +777 -0
  15. package/dist/recovery/effect-failure.d.ts +34 -0
  16. package/dist/recovery/effect-failure.js +62 -0
  17. package/dist/recovery/errors.d.ts +31 -0
  18. package/dist/recovery/errors.js +51 -0
  19. package/dist/recovery/execution-path.d.ts +45 -0
  20. package/dist/recovery/execution-path.js +153 -0
  21. package/dist/recovery/file-store.d.ts +7 -0
  22. package/dist/recovery/file-store.js +131 -0
  23. package/dist/recovery/frozen-snapshot.d.ts +149 -0
  24. package/dist/recovery/frozen-snapshot.js +136 -0
  25. package/dist/recovery/handoff-block.d.ts +36 -0
  26. package/dist/recovery/handoff-block.js +148 -0
  27. package/dist/recovery/handoff-digest.d.ts +58 -0
  28. package/dist/recovery/handoff-digest.js +195 -0
  29. package/dist/recovery/handoff-files.d.ts +54 -0
  30. package/dist/recovery/handoff-files.js +162 -0
  31. package/dist/recovery/handoff-receipts.d.ts +42 -0
  32. package/dist/recovery/handoff-receipts.js +53 -0
  33. package/dist/recovery/handoff-redact.d.ts +12 -0
  34. package/dist/recovery/handoff-redact.js +44 -0
  35. package/dist/recovery/handoff.d.ts +44 -0
  36. package/dist/recovery/handoff.js +46 -0
  37. package/dist/recovery/index.d.ts +18 -0
  38. package/dist/recovery/index.js +16 -0
  39. package/dist/recovery/journal.d.ts +25 -0
  40. package/dist/recovery/journal.js +127 -0
  41. package/dist/recovery/memory-store.d.ts +4 -0
  42. package/dist/recovery/memory-store.js +20 -0
  43. package/dist/recovery/store.d.ts +81 -0
  44. package/dist/recovery/store.js +6 -0
  45. package/dist/recovery/testing/conformance.d.ts +4 -0
  46. package/dist/recovery/testing/conformance.js +213 -0
  47. package/dist/recovery/testing/crash-points.d.ts +18 -0
  48. package/dist/recovery/testing/crash-points.js +43 -0
  49. package/dist/recovery/testing/crash-sweep.d.ts +65 -0
  50. package/dist/recovery/testing/crash-sweep.js +146 -0
  51. package/dist/recovery/testing/index.d.ts +3 -0
  52. package/dist/recovery/testing/index.js +4 -0
  53. package/dist/recovery/testing/preload.d.ts +1 -0
  54. package/dist/recovery/testing/preload.js +3 -0
  55. package/dist/recovery/transport-facts.d.ts +23 -0
  56. package/dist/recovery/transport-facts.js +46 -0
  57. package/dist/recovery/with-recovery.d.ts +44 -0
  58. package/dist/recovery/with-recovery.js +84 -0
  59. package/dist/workflow.d.ts +28 -0
  60. package/dist/workflow.js +247 -87
  61. package/package.json +9 -1
@@ -0,0 +1,777 @@
1
+ // The recovery driver: the durable state of one workflow run, addressed by the interpreter's execution paths. Execution
2
+ // stays in the interpreter. The driver implements its `recovery` hooks over a RecoveryStore, admits each effect before
3
+ // it is dispatched, and holds each LLM step's record, each route's decision and the escalation that ends a run.
4
+ import fs from "node:fs";
5
+ import path from "node:path";
6
+ import { createHash } from "node:crypto";
7
+ import { desugarWorkflow } from "../workflow.js";
8
+ import { chainStep, childrenOf, completedIteration, enclosingChains, enclosingIterations, isWithin, nodeAt, topLevelCompletion } from "./execution-path.js";
9
+ import { bindingDigests, LEDGER_FORMAT } from "./binding.js";
10
+ import { canonicalHash as hash } from "./canonical-hash.js";
11
+ import { RecoveryError } from "./errors.js";
12
+ import { gatewayIntentOf } from "./handoff-receipts.js";
13
+ import { validateFrozenSnapshot, openedFrozenSnapshot, serializeFrozenSnapshot, frozenStepSessionId, frozenStepId, frozenEffectId, frozenEffectPath, admitFrozenStep, closeFrozenStepAttempt, frozenStop } from "./frozen-snapshot.js";
14
+ const failure = (code, message) => new RecoveryError(message, code);
15
+ /** The branch a route runs for its answer, as the interpreter takes it: the node's `unsure` branch when
16
+ * the answer's confidence is below `unsure.gte`, else the choice the answer names. */
17
+ export function routeTaken(node, answers) {
18
+ const answer = answers?.branch;
19
+ const choice = String(answer?.choice ?? "");
20
+ const unsure = Boolean(node.unsure && typeof answer?.confidence === "number" && answer.confidence < node.unsure.gte);
21
+ return { branch: unsure ? node.unsure.branch : choice, choice, unsure };
22
+ }
23
+ /** A path the driver cannot place: the path is in the message, so the fault names where it happened. */
24
+ const pathFault = (at, message) => failure("FROZEN_PATH_INVALID", `${message}: ${typeof at === "string" ? at : JSON.stringify(at)}`);
25
+ /** Completes the route decisions of a checkpoint that holds a decision without `choice` and `unsure`:
26
+ * each takes the branch, the choice and the flag `routeTaken` derives from its stored answer, so a
27
+ * decision whose answer was unsure names the unsure branch it ran, never the answer's choice. */
28
+ function completeRouteDecisions(frame, doc) {
29
+ for (const [at, decision] of Object.entries(frame.routes)) {
30
+ if (typeof decision.choice === "string" && typeof decision.unsure === "boolean")
31
+ continue;
32
+ const node = nodeAt(doc, at);
33
+ if (node?.node !== "route")
34
+ throw pathFault(at, "Only a route's answer is a decision");
35
+ Object.assign(decision, routeTaken(node, decision.result.answers));
36
+ }
37
+ }
38
+ const LLM_KINDS = new Set(["agent", "decide", "extract", "report"]);
39
+ /** What a run's binding covers: the workflow as authored and what the host binds beside it, never the build. A run
40
+ * resumes across a deploy, and is refused when one of these changed: changing any of them is a new run. */
41
+ const bindingParts = (workflow, bind) => ({ engine: "frozen.v2", ledger: LEDGER_FORMAT, workflow, ...bind });
42
+ /** The binding of a run of `workflow` under what the host binds beside it. */
43
+ export function recoveryBinding(workflow, bind) {
44
+ return hash(bindingParts(workflow, bind));
45
+ }
46
+ /** What a store is opened under for a run of `workflow`: its binding, and the digest of each input the binding
47
+ * covers. The driver opens its store under it, and so does an operator who reconciles an effect by hand. */
48
+ export function recoveryBound(workflow, bind) {
49
+ return { binding: recoveryBinding(workflow, bind), inputs: bindingDigests(bindingParts(workflow, bind)) };
50
+ }
51
+ /** The files of a workspace directory: a name is relative to it, and a file that resolves outside it is refused. */
52
+ export function workspaceFiles(cwd) {
53
+ /** The file a name reaches, followed link by link as the system follows it, whether or not anything is at the end
54
+ * yet: a link to a reserved file that does not exist still reaches that file. What does not exist is kept as
55
+ * written. */
56
+ const reached = (name) => {
57
+ const parts = (full) => path.relative(path.parse(full).root, full).split(path.sep).filter(Boolean);
58
+ const start = path.resolve(cwd, name);
59
+ let at = path.parse(start).root;
60
+ let pending = parts(start);
61
+ for (let links = 0; pending.length > 0;) {
62
+ const next = path.join(at, pending[0]);
63
+ let link;
64
+ try {
65
+ if (fs.lstatSync(next).isSymbolicLink())
66
+ link = fs.readlinkSync(next);
67
+ }
68
+ catch {
69
+ return path.join(at, ...pending);
70
+ }
71
+ pending = pending.slice(1);
72
+ if (link === undefined) {
73
+ at = next;
74
+ continue;
75
+ }
76
+ // A chain of links that never ends reaches no file: it is its own name.
77
+ if ((links += 1) > 40)
78
+ return path.join(next, ...pending);
79
+ const target = path.resolve(at, link);
80
+ at = path.parse(target).root;
81
+ pending = [...parts(target), ...pending];
82
+ }
83
+ return at;
84
+ };
85
+ /** The file itself, whatever names it has: two hard links are one file. */
86
+ const identity = (file) => {
87
+ try {
88
+ const stat = fs.statSync(file, { bigint: true });
89
+ return `${stat.dev}:${stat.ino}`;
90
+ }
91
+ catch {
92
+ return undefined;
93
+ }
94
+ };
95
+ return {
96
+ same: (a, b) => {
97
+ const [first, second] = [reached(a), reached(b)];
98
+ if (first === second)
99
+ return true;
100
+ const file = identity(first);
101
+ return file !== undefined && file === identity(second);
102
+ },
103
+ hashes: (names) => Object.fromEntries(names.map(name => {
104
+ let filename;
105
+ try {
106
+ filename = fs.realpathSync(path.resolve(cwd, name));
107
+ }
108
+ catch (error) {
109
+ // A promised file the workspace no longer holds (a store that travelled without it) is an invalid artifact, never a raw errno.
110
+ if (["ENOENT", "ENOTDIR"].includes(error.code ?? ""))
111
+ throw failure("FROZEN_ARTIFACT_INVALID", `Recovery artifact is missing from the workspace: ${name}`);
112
+ throw error;
113
+ }
114
+ const relative = path.relative(fs.realpathSync(cwd), filename);
115
+ if (!relative || relative === ".." || relative.startsWith(".." + path.sep) || path.isAbsolute(relative)) {
116
+ throw failure("FROZEN_ARTIFACT_INVALID", "Recovery artifact escaped workspace");
117
+ }
118
+ return [name, createHash("sha256").update(fs.readFileSync(filename)).digest("hex")];
119
+ })),
120
+ };
121
+ }
122
+ /** Every file a node, or anything inside it, declares it writes: an artifact's path, a call's outputs. */
123
+ function declaredFiles(node, out = []) {
124
+ if (node.node === "artifact" && node.path)
125
+ out.push(node.path);
126
+ if (node.node === "call")
127
+ out.push(...(node.produces ?? []));
128
+ for (const child of childrenOf(node))
129
+ declaredFiles(child, out);
130
+ return out;
131
+ }
132
+ const labelOf = (node) => "label" in node && typeof node.label === "string" ? node.label : undefined;
133
+ /** Every label in the document, a child workflow's steps named `<invocation>/<step>` as the interpreter
134
+ * names them, so a label that appears twice is found. */
135
+ function labelsOf(node, prefix = "", out = []) {
136
+ const label = labelOf(node);
137
+ if (label !== undefined)
138
+ out.push(`${prefix}${label}`);
139
+ for (const child of childrenOf(node))
140
+ labelsOf(child, node.node === "workflow" ? `${prefix}${label}/` : prefix, out);
141
+ return out;
142
+ }
143
+ /** Opens the durable state of one run of `workflow` in `store` and takes ownership of it. Execution remains in the
144
+ * interpreter: the workflow runs with the returned driver's hooks, and `close` lets go of the run. */
145
+ export async function openRecovery(store, workflow, options) {
146
+ const reader = options.files ?? workspaceFiles(process.cwd());
147
+ const reservedOutputs = options.reservedOutputs ?? [];
148
+ const reserved = (name) => reservedOutputs.some(file => reader.same(name, file));
149
+ /** A refusal found before any step runs: a workflow with a label on two nodes, or one that writes a file the host
150
+ * owns. The driver still opens (a run is durable whatever its shape), and the host ends the run through an
151
+ * escalation commit naming the refusal. */
152
+ const hostOwned = declaredFiles(workflow.root).find(reserved);
153
+ // Execution paths address every node; labels still name the node an escalation, a step record and a
154
+ // decision are about, so two nodes may not share one.
155
+ const labels = labelsOf(workflow.root);
156
+ const duplicate = labels.find((label, index) => labels.indexOf(label) !== index);
157
+ const preStepRefusal = duplicate !== undefined ? {
158
+ code: "RUN_CONTROL_UNSUPPORTED", kind: "duplicate_label", label: duplicate,
159
+ message: `Frozen workflow recovery requires unique labels; "${duplicate}" names two nodes`,
160
+ } : hostOwned !== undefined ? {
161
+ code: "RUN_CONTROL_UNSUPPORTED", kind: "host_owned_output", label: "workflow",
162
+ message: `Recovered artifacts must not overwrite ${reservedOutputs.length > 1 ? `${reservedOutputs.slice(0, -1).join(", ")} or ${reservedOutputs.at(-1)}` : reservedOutputs[0]}; the host owns those files, and the pin writes ${hostOwned}`,
163
+ } : null;
164
+ // The document the interpreter runs, and so the one its paths resolve in.
165
+ const view = options.view ?? desugarWorkflow(workflow);
166
+ const bound = recoveryBound(workflow, options.bind);
167
+ const binding = bound.binding;
168
+ const driver = options.key;
169
+ const controller = new AbortController();
170
+ let fatal;
171
+ /** A pause or cancel was applied: its commit still lands, and nothing after it. */
172
+ let stopping = false;
173
+ const journal = await store.open(bound);
174
+ /** The host's stop as it was handed over (a cancel outranks a pause), and who issued it. */
175
+ let requested;
176
+ /** The one error the driver's stop is: its signal's reason and every refusal after it, naming the committed stop. */
177
+ let stopError;
178
+ let snapshot;
179
+ /** This process's ownership generation of the run: one per process that holds it. */
180
+ const generation = journal.generation;
181
+ const serial = () => serializeFrozenSnapshot(snapshot);
182
+ /** The journal's commits, in the order the driver made them: a commit a synchronous method starts lands before any
183
+ * later one, and an effect's admission waits for every earlier commit. A failed commit stops the run. */
184
+ let chain = Promise.resolve();
185
+ const persist = (write) => {
186
+ const next = chain.then(() => { if (fatal && !stopping)
187
+ throw fatal; return write(); });
188
+ chain = next.catch((error) => { if (!fatal) {
189
+ fatal = error;
190
+ controller.abort(error);
191
+ } });
192
+ return next;
193
+ };
194
+ /** Every commit the driver started has landed; a failed one is thrown. */
195
+ const flush = async () => { await chain; if (fatal && !stopping)
196
+ throw fatal; };
197
+ try {
198
+ // A driver an earlier process bound, with no checkpoint, is a run killed between its binding and its first
199
+ // checkpoint when the journal holds nothing else of it: it never started, and it starts now. One with effects and
200
+ // no checkpoint needs reconciliation.
201
+ const saved = journal.state;
202
+ if (journal.existing && !saved && journal.effects().length > 0)
203
+ throw failure("FROZEN_CHECKPOINT_INVALID", "Existing frozen run has no checkpoint; retained workspace requires reconciliation");
204
+ // Only the paths of the run's own document are resolved.
205
+ if (saved)
206
+ validateFrozenSnapshot(saved, { resolves: (at) => Boolean(nodeAt(view, at)) });
207
+ snapshot = openedFrozenSnapshot(saved);
208
+ // A route decision a checkpoint holds without the choice and the unsure flag holds the
209
+ // answer whole: the branch taken, the choice and the flag are derived from it by the
210
+ // interpreter's rule, and the commit below stores them.
211
+ completeRouteDecisions(snapshot.pin, view);
212
+ const left = journal.effects().filter((effect) => effect.status === "unknown");
213
+ if (options.reconcile && left.length > 0) {
214
+ let reconciling = true;
215
+ try {
216
+ await options.reconcile(structuredClone(left), async (id, value, receipt) => {
217
+ if (!reconciling)
218
+ throw failure("RECOVERY_RECONCILE_FAILED", `Reconcile ${id} after reconcile returned`);
219
+ if (!left.some((effect) => effect.id === id))
220
+ throw failure("RECOVERY_RECONCILE_FAILED", `Reconcile ${id}: not an unknown effect of this run`);
221
+ // The files the effect declares are hashed now, as a normal completion records them: a missing output refuses the
222
+ // settlement, and a later resume refuses an output that changed.
223
+ const at = frozenEffectPath(id)?.path;
224
+ const declared = at === undefined ? undefined : nodeAt(view, at);
225
+ const files = reader.hashes(declared?.node === "call" ? declared.produces ?? [] : []);
226
+ Object.assign(snapshot.files, files);
227
+ // The calls an earlier process kept stay: the host's re-send is added after them, so a call that applied before the kill is still on record.
228
+ if (receipt?.calls?.length)
229
+ await journal.called(id, [...(journal.effect(id)?.calls ?? []), ...receipt.calls]);
230
+ await journal.complete(id, { value, files, ...(receipt?.intent ? { intent: receipt.intent } : {}) });
231
+ });
232
+ }
233
+ catch (error) {
234
+ throw error instanceof RecoveryError && error.code === "RECOVERY_RECONCILE_FAILED" ? error
235
+ : new RecoveryError(`Reconcile failed: ${error instanceof Error ? error.message : String(error)}`, "RECOVERY_RECONCILE_FAILED", { cause: error });
236
+ }
237
+ finally {
238
+ reconciling = false;
239
+ }
240
+ }
241
+ await journal.save(serial());
242
+ }
243
+ catch (error) {
244
+ await journal.close();
245
+ throw error;
246
+ }
247
+ const ordinals = new Map();
248
+ /** One commit of the driver's whole state. */
249
+ const save = () => {
250
+ const state = serial();
251
+ // A synchronous method does not wait for it; the failure is the run's, and the next awaited commit throws it.
252
+ return persist(() => journal.save(state)).catch(() => undefined);
253
+ };
254
+ /** Effects dispatched and not yet completed or failed. A pause is not taken while one is in flight. */
255
+ let inFlight = 0;
256
+ /** New work waiting at a seam for a pause to be taken, woken when the last effect in flight ends or a stop is taken. */
257
+ let drained;
258
+ const wake = () => { const waiting = drained; drained = undefined; waiting?.wake(); };
259
+ const pausePending = () => !fatal && requested?.action === "pause" && snapshot.status !== "paused" && inFlight > 0;
260
+ const guard = () => {
261
+ if (fatal)
262
+ throw fatal;
263
+ try {
264
+ // The driver owns the run-wide pause and cancel; its steps stop through its signal. A pause waits for every
265
+ // effect in flight to complete and commit; a cancel does not.
266
+ const stop = frozenStop(snapshot.status, requested?.action);
267
+ if (stop === "paused" && inFlight > 0)
268
+ return;
269
+ if (stop) {
270
+ snapshot.status = stop;
271
+ // The stopped state is still committed, after which nothing else is.
272
+ stopping = true;
273
+ const error = stopError = new RecoveryError(`Frozen run ${snapshot.status}`, stop === "cancelled" ? "FROZEN_CANCELLED" : "FROZEN_PAUSED", { source: requested?.source });
274
+ fatal = error;
275
+ controller.abort(error);
276
+ void save();
277
+ throw error;
278
+ }
279
+ }
280
+ catch (error) {
281
+ if (!fatal) {
282
+ fatal = error;
283
+ controller.abort(error);
284
+ }
285
+ throw error;
286
+ }
287
+ };
288
+ /** A seam: where new work starts. While a pause waits for an effect in flight, new work waits here; then the pause
289
+ * is taken and the work is refused with it. */
290
+ const seam = async () => {
291
+ while (pausePending()) {
292
+ if (!drained) {
293
+ let wake;
294
+ drained = { promise: new Promise((resolve) => { wake = resolve; }), wake };
295
+ }
296
+ await drained.promise;
297
+ }
298
+ guard();
299
+ };
300
+ /** An effect in flight has ended. The last one lets a waiting pause be taken. */
301
+ const settled = () => {
302
+ inFlight -= 1;
303
+ if (inFlight > 0)
304
+ return;
305
+ if (requested?.action === "pause") {
306
+ try {
307
+ guard();
308
+ }
309
+ catch { /* The pause is taken: its commit is queued. */ }
310
+ }
311
+ wake();
312
+ };
313
+ const fileHashes = (names) => reader.hashes(names);
314
+ const verifyFiles = (files) => {
315
+ if (hash(fileHashes(Object.keys(files))) !== hash(files))
316
+ throw failure("FROZEN_ARTIFACT_INVALID", "Committed artifact bytes changed; effect will not be repeated");
317
+ };
318
+ /** An effect-level clock is set once and kept across resume: a poll window or an attempt deadline
319
+ * belongs to the outside world, so downtime counts against it, never against the run. */
320
+ const clock = (key, duration) => {
321
+ if (snapshot.clocks[key] === undefined) {
322
+ snapshot.clocks[key] = Date.now() + duration;
323
+ save();
324
+ }
325
+ if (!Number.isFinite(snapshot.clocks[key]))
326
+ throw failure("FROZEN_CHECKPOINT_INVALID", "Invalid durable clock");
327
+ return snapshot.clocks[key];
328
+ };
329
+ // A row that names a commit is checked against the journal: the escalation row's revision is an
330
+ // escalation commit, or the checkpoint is refused. A placeholder revision never survives a resume.
331
+ const escalated = snapshot.escalation;
332
+ if (escalated && !(escalated.revision >= 1 && journal.notes().some((note) => note.revision === escalated.revision && note.kind === "escalation"))) {
333
+ await journal.close();
334
+ throw failure("FROZEN_CHECKPOINT_INVALID", `The escalation row names revision ${escalated.revision}, which is not an escalation commit`);
335
+ }
336
+ /** The frame the interpreter is running: the run's document. Its tag names its sessions and effects. */
337
+ const active = () => ({ frame: snapshot.pin, tag: "step", doc: view });
338
+ const doneSets = new WeakMap();
339
+ const doneOf = (frame) => doneSets.get(frame) ?? doneSets.set(frame, new Set(frame.done)).get(frame);
340
+ /** Frames whose document input this process has checked: the first resume of a process starts the
341
+ * document, so it is where a changed input shows. */
342
+ const inputChecked = new WeakSet();
343
+ /** The path the interpreter named, when it resolves in the running document. */
344
+ const placed = (at, doc) => {
345
+ if (typeof at !== "string" || !nodeAt(doc, at))
346
+ throw pathFault(at, "The interpreter named a path this run's document does not hold");
347
+ return at;
348
+ };
349
+ /** The committed state a resume at `at` is answered from: its own, or the frontier of a chain it sits
350
+ * in when the frontier lies past it. A chain threads one state, so every step before its frontier,
351
+ * and everything inside those steps, is answered with the frontier's. */
352
+ const answer = (frame, at) => {
353
+ if (frame.states[at])
354
+ return frame.states[at];
355
+ for (const { chain, index } of enclosingChains(at)) {
356
+ const head = frame.chains[chain];
357
+ if (head && head.index > index)
358
+ return frame.states[head.path];
359
+ }
360
+ return undefined;
361
+ };
362
+ /** Drops what no resume can ask for again: the states, chain frontiers, loop counts, route decisions
363
+ * and start bindings of the paths `gone` names. Step records stay: a host reads its sessions from them. */
364
+ const prune = (frame, gone) => {
365
+ for (const table of [frame.states, frame.chains, frame.loops, frame.routes, frame.inputs]) {
366
+ for (const key of Object.keys(table))
367
+ if (gone(key))
368
+ delete table[key];
369
+ }
370
+ };
371
+ const recovery = {
372
+ // Every store key below is the interpreter's execution path, so a composed graph (a loop, a route, a
373
+ // map whose body is structure, a nested child) resumes where it stopped.
374
+ supportsExecutionPaths: true,
375
+ async resume(_node, state, _item, executionPath) {
376
+ await seam();
377
+ const { frame, doc } = active();
378
+ const at = placed(executionPath, doc);
379
+ if (frame.input === null) {
380
+ frame.input = hash(state);
381
+ inputChecked.add(frame);
382
+ await save();
383
+ await flush();
384
+ }
385
+ else if (!inputChecked.has(frame)) {
386
+ if (frame.input !== hash(state))
387
+ throw failure("FROZEN_INPUT_CHANGED", `Materialized workflow input changed: ${at}`);
388
+ inputChecked.add(frame);
389
+ }
390
+ verifyFiles(snapshot.files);
391
+ const committed = answer(frame, at);
392
+ if (committed)
393
+ return committed;
394
+ if (doneOf(frame).has(at))
395
+ throw pathFault(at, "A committed path was asked for again after its state was released");
396
+ // A node started in an earlier process starts again from the same state, or not at all. The
397
+ // binding is durable with the next commit, which is the first durable thing done under it.
398
+ const digest = hash(state);
399
+ if (frame.inputs[at] === undefined)
400
+ frame.inputs[at] = digest;
401
+ else if (frame.inputs[at] !== digest)
402
+ throw failure("FROZEN_INPUT_CHANGED", `The input of ${at} changed since it started`);
403
+ return undefined;
404
+ },
405
+ async commit(node, state, _item, executionPath) {
406
+ // Finished work commits while a pause waits for an effect in flight; it starts nothing.
407
+ guard();
408
+ const { frame, doc } = active();
409
+ const at = placed(executionPath, doc);
410
+ const done = doneOf(frame);
411
+ if (done.has(at))
412
+ throw pathFault(at, "A path commits once");
413
+ // Every loop the path runs in is on the iteration the path belongs to: iteration k+1 commits
414
+ // nothing before iteration k is complete.
415
+ for (const { loop, index } of enclosingIterations(at)) {
416
+ if (index !== (frame.loops[loop] ?? 0))
417
+ throw pathFault(at, `Iteration ${index} committed while its loop is on iteration ${frame.loops[loop] ?? 0}`);
418
+ }
419
+ const step = chainStep(at);
420
+ if (step && (frame.chains[step.chain]?.index ?? -1) >= step.index)
421
+ throw pathFault(at, "A chain step committed behind its chain's frontier");
422
+ if (node.node === "artifact" && node.path)
423
+ Object.assign(snapshot.files, fileHashes([node.path]));
424
+ frame.done.push(at);
425
+ done.add(at);
426
+ // A map that committed, alone or inside what committed, has its results in the state: what it had finished
427
+ // when it failed earlier is no longer the run's.
428
+ if (frame.partial && isWithin(frame.partial.path, at))
429
+ delete frame.partial;
430
+ // Everything inside the committed node is answered by it now, and so is everything before it
431
+ // in its chain.
432
+ prune(frame, key => isWithin(key, at));
433
+ if (step) {
434
+ prune(frame, key => enclosingChains(key).some(c => c.chain === step.chain && c.index < step.index));
435
+ frame.chains[step.chain] = { index: step.index, path: at };
436
+ }
437
+ frame.states[at] = state;
438
+ const completed = completedIteration(doc, at);
439
+ if (completed)
440
+ frame.loops[completed.loop] = completed.index + 1;
441
+ await save();
442
+ await flush();
443
+ },
444
+ async fail(node, results, _error, executionPath) {
445
+ // Finished work commits while a pause waits for an effect in flight; it starts nothing.
446
+ guard();
447
+ const { frame, doc } = active();
448
+ // A map that ended on an item's failure keeps its finished items committed: the handoff's state
449
+ // carries them under the map's key, and a later open answers them from their paths.
450
+ frame.partial = { path: placed(executionPath, doc), as: String(node.as ?? node.node), results: results.map(result => result === undefined ? null : result) };
451
+ await save();
452
+ await flush();
453
+ },
454
+ pollStartedAt(node, _item, executionPath) {
455
+ guard();
456
+ const { tag, doc } = active();
457
+ const at = placed(executionPath, doc);
458
+ return clock(`poll:${tag}:${at}`, node.poll.deadline_s * 1000) - node.poll.deadline_s * 1000;
459
+ },
460
+ async wait(_node, ms, _item, executionPath) {
461
+ await seam();
462
+ const { tag, doc } = active();
463
+ const key = `${tag}:${placed(executionPath, doc)}`;
464
+ const until = Math.min(snapshot.clocks[`poll:${key}`] ?? Infinity, clock(`wait:${key}:${ordinals.get(key) ?? 0}`, ms));
465
+ // The wait's clock is durable before the wait starts, so a process that dies inside it resumes the same wait.
466
+ await flush();
467
+ await new Promise((resolve, reject) => {
468
+ const onAbort = () => { clearTimeout(wait); reject(controller.signal.reason); };
469
+ const wait = setTimeout(() => { controller.signal.removeEventListener("abort", onAbort); resolve(); }, Math.max(0, until - Date.now()));
470
+ controller.signal.addEventListener("abort", onAbort, { once: true });
471
+ });
472
+ guard();
473
+ },
474
+ };
475
+ /** An LLM step's record in the running frame, by its execution path. */
476
+ const llmStep = (executionPath) => {
477
+ const { frame, tag, doc } = active();
478
+ const at = placed(executionPath, doc);
479
+ if (!LLM_KINDS.has(nodeAt(doc, at).node))
480
+ throw pathFault(at, "Only an LLM step has a session");
481
+ return { frame, tag, at };
482
+ };
483
+ const sessionOf = (tag, at, step, attempt = step.attempt) => frozenStepSessionId(driver, step.label, tag, at, attempt);
484
+ /** The highest top-level step of `doc` the frame completed, and the committed path its state is
485
+ * read at (`topLevelCompletion`). */
486
+ const completedHead = (frame, doc) => {
487
+ let head;
488
+ for (const at of frame.done) {
489
+ const index = topLevelCompletion(doc, at);
490
+ if (index !== undefined && (!head || index > head.index))
491
+ head = { index, at };
492
+ }
493
+ return head;
494
+ };
495
+ return {
496
+ recovery, signal: controller.signal, resumed: journal.existing,
497
+ /** This process's generation of the run: a request this process makes is named with it, so a
498
+ * process that asks again never reuses an earlier process's invocation. */
499
+ get generation() { return generation; },
500
+ /** A refusal found before any step, or null: the host escalates it before any step runs. */
501
+ preStepRefusal,
502
+ binding,
503
+ /** The run's key: what its step sessions and effects are named under. */
504
+ driverId: driver,
505
+ /** The host hands the driver its stop here. A cancel stops the driver at once, and its steps and effects through
506
+ * its signal. A pause refuses new work at once and is taken when no effect is in flight: an effect already
507
+ * dispatched completes and commits its receipt first, so a paused run resumes without a reconciliation. */
508
+ stop: (stop) => {
509
+ if (requested?.action !== "cancel")
510
+ requested = stop;
511
+ // A cancel after the driver paused replaces the pause it committed and the stop it reports: a cancelled run never resumes.
512
+ if (stopError && snapshot.status === "paused" && stop.action === "cancel") {
513
+ snapshot.status = "cancelled";
514
+ Object.assign(stopError, { code: "FROZEN_CANCELLED", message: "Frozen run cancelled", source: stop.source });
515
+ void save();
516
+ }
517
+ try {
518
+ guard();
519
+ }
520
+ catch { /* Sticky: the stop blocks every admission. */ }
521
+ if (fatal)
522
+ wake();
523
+ },
524
+ /** The driver's stop check, for a branch that hands off without entering the interpreter: a pause or cancel the
525
+ * driver took stops the run here, before anything resumes. */
526
+ checkStop() { guard(); },
527
+ /** A seam for new work: it waits while a pause waits for an effect in flight, then applies any stop taken. */
528
+ seam,
529
+ /** An LLM step's execution record, admitted before it dispatches. The first call fixes the label
530
+ * the interpreter names it by (a child's step under its invocation) and the retry limit; a resume
531
+ * at this path gets the same record, so its session is found again. A closed attempt advances to a new
532
+ * session identity here, and only here; a failed step never gets another. */
533
+ stepSession(executionPath, label, admission) {
534
+ guard();
535
+ const { frame, tag, at } = llmStep(executionPath);
536
+ const { step, changed } = admitFrozenStep(frame.steps, at, label, admission);
537
+ if (changed)
538
+ save();
539
+ return { ...step, stepId: frozenStepId(tag, at), sessionId: sessionOf(tag, at, step), earlierSessionIds: Array.from({ length: step.attempt }, (_, earlier) => sessionOf(tag, at, step, earlier)) };
540
+ },
541
+ /** Whether the step already has an execution record: a resumed step was admitted in an earlier
542
+ * process and is continued, never refused again. */
543
+ stepAdmitted(executionPath) { const { frame, at } = llmStep(executionPath); return frame.steps[at] !== undefined; },
544
+ /** The current attempt did not submit: its session closes with its spend kept.
545
+ * The step is `closed` until a retry is admitted, or `failed` when no attempt remains. */
546
+ stepAttemptFailed(executionPath, final = false) {
547
+ guard();
548
+ const { frame, tag, at } = llmStep(executionPath);
549
+ const step = frame.steps[at];
550
+ if (!step)
551
+ throw pathFault(at, "Step attempt failed before admission");
552
+ const id = sessionOf(tag, at, step);
553
+ if (options.closeStepSession)
554
+ void persist(() => options.closeStepSession(id)).catch(() => undefined);
555
+ closeFrozenStepAttempt(step, final);
556
+ save();
557
+ },
558
+ /** The step delivered its record; the row and the step record agree before the step commit. */
559
+ stepSubmitted(executionPath) {
560
+ guard();
561
+ const { frame, at } = llmStep(executionPath);
562
+ const step = frame.steps[at];
563
+ if (step) {
564
+ step.status = "submitted";
565
+ save();
566
+ }
567
+ },
568
+ /** The short identity of an admitted LLM step (a digest of its frame and path), for the escalation
569
+ * row when it fails; null for a path that ran no LLM step. */
570
+ stepId(executionPath) {
571
+ const { frame, tag, doc } = active();
572
+ return typeof executionPath === "string" && nodeAt(doc, executionPath) && frame.steps[executionPath] ? frozenStepId(tag, executionPath) : null;
573
+ },
574
+ /** Spend outside any question or session, committed with the driver's state the moment it is priced, so the run's
575
+ * recorded spend holds it. It may be called inside a tool, so it never applies a stop; a failed commit is the run's
576
+ * persistence failure. */
577
+ async charge(usd) {
578
+ if (fatal)
579
+ throw fatal;
580
+ if (!Number.isFinite(usd) || usd <= 0)
581
+ return;
582
+ snapshot.questionSpendUsd = +((snapshot.questionSpendUsd ?? 0) + usd).toFixed(8);
583
+ await save();
584
+ await flush();
585
+ },
586
+ /** A question's metered spend, committed with the driver's state so a resume that skips the
587
+ * finished question still reports it; an unpriced answer (`null`) adds nothing. A route's answer is its decision: it commits in the same save, once per route path, and a resume
588
+ * inside the branch follows it instead of asking again. */
589
+ recordQuestionSpend(usd, route) {
590
+ guard();
591
+ let changed = false;
592
+ if (usd !== null && Number.isFinite(usd) && usd >= 0) {
593
+ snapshot.questionSpendUsd = +((snapshot.questionSpendUsd ?? 0) + usd).toFixed(8);
594
+ changed = true;
595
+ }
596
+ if (route) {
597
+ const { frame, doc } = active();
598
+ const at = placed(route.executionPath, doc);
599
+ const node = nodeAt(doc, at);
600
+ if (node?.node !== "route")
601
+ throw pathFault(at, "Only a route's answer is a decision");
602
+ if (frame.routes[at])
603
+ throw pathFault(at, "A route is decided once");
604
+ frame.routes[at] = { label: route.label, ...routeTaken(node, route.result.answers), request_sha256: typeof route.result.request_sha256 === "string" ? route.result.request_sha256 : null, result: route.result, receipts: route.receipts };
605
+ changed = true;
606
+ }
607
+ if (changed)
608
+ save();
609
+ },
610
+ /** The host has written what it keeps of the route at `executionPath`: its decision no longer carries the
611
+ * receipts a resume would write. */
612
+ routeReceipted(executionPath) {
613
+ guard();
614
+ const { frame, doc } = active();
615
+ const decision = frame.routes[placed(executionPath, doc)];
616
+ if (!decision?.receipts)
617
+ return;
618
+ decision.receipts = null;
619
+ save();
620
+ },
621
+ /** The committed decision of the route at `executionPath`, if it was answered in an earlier process. */
622
+ routeDecision(executionPath) {
623
+ const { frame, doc } = active();
624
+ return frame.routes[placed(executionPath, doc)];
625
+ },
626
+ /** The escalation that ended this run (a gate that fired, or an LLM step that failed), written once
627
+ * as an escalation commit before anything acts on it. A later open gets the same row and the same
628
+ * continuation identity, whatever fires again. */
629
+ async escalate(row) {
630
+ guard();
631
+ if (snapshot.escalation)
632
+ return snapshot.escalation;
633
+ const stepId = row.step_exec_id ?? null;
634
+ const tailId = stepId ?? Math.random().toString(36).slice(2, 10);
635
+ // The row is placed before the commit so the commit's state carries it, and the state is
636
+ // serialized inside the commit's transaction with that commit's revision: no boundary
637
+ // exists where the row is durable and its revision is not.
638
+ snapshot.escalation = { ...row, step_exec_id: stepId, tail: `${driver}:escalated:${tailId}`, revision: 0 };
639
+ // The note names the handoff, not the state: the state rides the snapshot.
640
+ const detail = { kind: row.kind, stage: row.stage, summary: String(row.summary ?? "").slice(0, 512), step_label: row.step_label, step_exec_id: stepId, workflow_sha_used: row.workflow_sha_used,
641
+ cost_usd: row.cost_usd, tail: snapshot.escalation.tail, state_keys: Object.keys(row.state ?? {}) };
642
+ await persist(() => journal.save((revision) => { snapshot.escalation.revision = revision; return serial(); }, { kind: "escalation", detail }));
643
+ return snapshot.escalation;
644
+ },
645
+ escalation() { return snapshot.escalation; },
646
+ /** How many effects the driver admitted. */
647
+ effectCalls() { return journal.effects().length; },
648
+ /** What the continuation inherited, against the escalation that admitted it: the level, the sizes of what it
649
+ * read and its identity. */
650
+ async annotateHandoff(detail) {
651
+ await persist(() => journal.note("inherited", detail));
652
+ },
653
+ /** The run's spend at the moment its continuation is admitted. Spend may land under this run after
654
+ * the escalation commit, so the row the continuation reads the spend from is brought current here,
655
+ * by one `handoff` commit that names the escalation. The escalation's revision, what the
656
+ * continuation binds on, does not change. */
657
+ async handoff(spend) {
658
+ guard();
659
+ const row = snapshot.escalation;
660
+ if (!row)
661
+ throw failure("FROZEN_CHECKPOINT_INVALID", "A handoff follows an escalation; none is recorded");
662
+ if (row.cost_usd === spend.cost_usd && row.turns === spend.turns && row.effect_calls === spend.effect_calls)
663
+ return row;
664
+ const detail = { escalation_revision: row.revision, previous_cost_usd: row.cost_usd,
665
+ cost_usd: spend.cost_usd, turns: spend.turns, effect_calls: spend.effect_calls };
666
+ Object.assign(row, spend);
667
+ await persist(() => journal.save(serial(), { kind: "handoff", detail }));
668
+ return row;
669
+ },
670
+ /** The state the document's committed top-level prefix left behind, plus what a failed map's finished
671
+ * items produced under the map's key: what an escalation on a failure carries. */
672
+ lastState() {
673
+ const { frame, doc } = active();
674
+ const head = completedHead(frame, doc);
675
+ const state = head ? frame.states[head.at] ?? {} : {};
676
+ const partial = frame.partial;
677
+ return partial ? { ...state, [partial.as]: partial.results } : state;
678
+ },
679
+ get started() {
680
+ if (snapshot.pin.input === null && journal.effects().length > 0) {
681
+ throw failure("FROZEN_CHECKPOINT_INVALID", "Effects exist before the frozen input checkpoint");
682
+ }
683
+ return snapshot.pin.input !== null;
684
+ },
685
+ wrapEffect(run) {
686
+ return async (params) => {
687
+ await seam();
688
+ const { tag, doc } = active();
689
+ const at = placed(params.executionPath, doc);
690
+ const key = `${tag}:${at}`;
691
+ const ordinal = ordinals.get(key) ?? 0;
692
+ ordinals.set(key, ordinal + 1);
693
+ const id = frozenEffectId(tag, at, ordinal);
694
+ const argsHash = hash({ input: params.input, node: params.node, idempotencyKey: params.idempotencyKey });
695
+ const existing = journal.effect(id);
696
+ if (existing) {
697
+ if (existing.argsHash !== argsHash)
698
+ throw failure("FROZEN_INPUT_CHANGED", "Recovered effect arguments changed");
699
+ if (existing.status !== "completed")
700
+ throw failure("FROZEN_EFFECT_UNKNOWN", `Reconcile ${id} before resuming; it will not be repeated`);
701
+ const receipt = existing.result;
702
+ verifyFiles(receipt.files);
703
+ Object.assign(snapshot.files, receipt.files);
704
+ await save();
705
+ await flush();
706
+ return receipt.value;
707
+ }
708
+ if (params.produces.some(reserved))
709
+ throw failure("RUN_CONTROL_UNSUPPORTED", "Effect declares a host-owned output path");
710
+ // The effect's own clock: the node's deadline, and the poll window when it is a poll.
711
+ const deadline = Math.min(snapshot.clocks[`poll:${key}`] ?? Infinity, clock(`attempt:${id}`, params.node.deadline_s * 1000));
712
+ if (Date.now() >= deadline)
713
+ throw failure("FROZEN_EFFECT_DEADLINE", "Effect exceeded its own deadline");
714
+ // A tool effect is admitted with the external call it makes, keyed as a continuation keys its own calls
715
+ // (`gatewayIntentOf`: a fetch wrapper's call is the gateway tool it names and its arguments), so a continuation
716
+ // that inherits it unknown refuses the same call.
717
+ const external = params.node.via === "tool" ? gatewayIntentOf(String(params.node.tool), params.input ?? {}) : undefined;
718
+ // From its admission the effect is in flight: a pause waits for it to complete and commit, so the admission is
719
+ // never left without its dispatch.
720
+ inFlight += 1;
721
+ try {
722
+ // The driver's state and the admission are one transaction.
723
+ const admitted = await persist(() => journal.admit(id, String(params.node.label), argsHash, serial(), driver, external, params.input ?? null));
724
+ if (admitted !== "new")
725
+ throw failure("FROZEN_EFFECT_UNKNOWN", `Reconcile ${id} before dispatch`);
726
+ // What landed while the admission was being committed still counts: after a cancel, the caller's own
727
+ // cancellation or the effect's deadline, the effect stays admitted and unknown and the adapter is never called.
728
+ // A pause lets it dispatch and complete.
729
+ guard();
730
+ if (params.signal.aborted)
731
+ throw params.signal.reason;
732
+ if (Date.now() >= deadline)
733
+ throw failure("FROZEN_EFFECT_DEADLINE", "Effect exceeded its own deadline");
734
+ const signal = AbortSignal.any([params.signal, controller.signal, AbortSignal.timeout(Math.max(1, deadline - Date.now()))]);
735
+ // A thrown/aborted call remains UNKNOWN, including transient failures: no blind retry. Each outside call the effect
736
+ // makes is kept on its record as it ends, on the driver's commit chain, so a process killed mid-effect keeps the
737
+ // calls it made and a commit that fails is the run's failure, which aborts the effect at once.
738
+ const calls = [];
739
+ const keep = (record) => { calls.push(record); const held = [...calls]; persist(() => journal.called(id, held)).catch(() => undefined); };
740
+ let value;
741
+ try {
742
+ value = await run({ ...params, signal, call: keep, effect: { id, argsHash } });
743
+ }
744
+ finally {
745
+ if (calls.length)
746
+ await flush();
747
+ }
748
+ if (signal.aborted)
749
+ throw signal.reason;
750
+ const returned = value;
751
+ if (returned?.outcome === "unknown" || returned?.effect_status === "unknown"
752
+ || returned?.details?.outcome === "unknown" || returned?.details?.effect_status === "unknown") {
753
+ (snapshot.unknownResponses ??= {})[id] = value;
754
+ await save(); // Preserve the observation without certifying completion.
755
+ await flush();
756
+ throw failure("FROZEN_EFFECT_UNKNOWN", `Reconcile ${id}: tool returned an unknown outcome`);
757
+ }
758
+ const files = fileHashes(params.produces);
759
+ // A tool effect's receipt names the external call it paid for (the tool and its arguments),
760
+ // so a continuation that inherits the receipt can answer the same call from it. Any other
761
+ // effect keeps its arguments beside its result.
762
+ const intent = params.node.via === "tool" ? { tool: String(params.node.tool), args: params.input } : undefined;
763
+ Object.assign(snapshot.files, files);
764
+ await persist(() => journal.complete(id, { value, files, ...(intent ? { intent } : { args: params.input ?? null }) }, serial()));
765
+ return value;
766
+ }
767
+ finally {
768
+ settled();
769
+ }
770
+ };
771
+ },
772
+ /** Every commit the driver started has landed; a failed one is thrown. */
773
+ flush,
774
+ /** Let go of the run once every commit the driver started has landed. */
775
+ async close() { await chain; await journal.close(); },
776
+ };
777
+ }