@parcha/agentrun-dsl 0.1.0-beta.14 → 0.1.0-beta.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/author.js +1 -1
- package/dist/dry-run.d.ts +6 -0
- package/dist/dry-run.js +30 -5
- package/dist/graph.d.ts +67 -0
- package/dist/graph.js +158 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +1 -0
- package/dist/job.d.ts +14 -0
- package/dist/job.js +5 -0
- package/dist/recovery/canonical-hash.d.ts +1 -1
- package/dist/recovery/canonical-hash.js +3 -3
- package/dist/recovery/driver.d.ts +21 -1
- package/dist/recovery/driver.js +394 -16
- package/dist/recovery/execution-path.d.ts +8 -0
- package/dist/recovery/execution-path.js +14 -0
- package/dist/recovery/frozen-snapshot.d.ts +76 -1
- package/dist/recovery/frozen-snapshot.js +21 -0
- package/dist/recovery/index.d.ts +3 -3
- package/dist/recovery/index.js +2 -2
- package/dist/recovery/item-state.d.ts +48 -0
- package/dist/recovery/item-state.js +89 -0
- package/dist/recovery/journal.d.ts +6 -3
- package/dist/recovery/journal.js +20 -8
- package/dist/recovery/store.d.ts +33 -9
- package/dist/recovery/testing/conformance.js +55 -0
- package/dist/system-one.d.ts +17 -1
- package/dist/system-one.js +17 -1
- package/dist/workflow.d.ts +3 -0
- package/dist/workflow.js +42 -1
- package/package.json +8 -1
- package/skills/author/references/language.md +1 -1
- package/skills/author/references/sdk-authoring.md +2 -0
package/dist/recovery/driver.js
CHANGED
|
@@ -5,12 +5,12 @@ import fs from "node:fs";
|
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import { createHash } from "node:crypto";
|
|
7
7
|
import { desugarWorkflow } from "../workflow.js";
|
|
8
|
-
import { chainStep, childrenOf, completedIteration, enclosingChains, enclosingIterations, isWithin, nodeAt, topLevelCompletion } from "./execution-path.js";
|
|
8
|
+
import { chainStep, childrenOf, completedIteration, enclosingChains, enclosingIterations, isWithin, mapItemOf, nodeAt, topLevelCompletion } from "./execution-path.js";
|
|
9
9
|
import { bindingDigests, LEDGER_FORMAT } from "./binding.js";
|
|
10
10
|
import { canonicalHash as hash } from "./canonical-hash.js";
|
|
11
11
|
import { RecoveryError } from "./errors.js";
|
|
12
12
|
import { gatewayIntentOf } from "./handoff-receipts.js";
|
|
13
|
-
import { validateFrozenSnapshot, openedFrozenSnapshot, serializeFrozenSnapshot, frozenStepSessionId, frozenStepId, frozenEffectId, frozenEffectPath, admitFrozenStep, closeFrozenStepAttempt, frozenStop } from "./frozen-snapshot.js";
|
|
13
|
+
import { validateFrozenSnapshot, openedFrozenSnapshot, serializeFrozenSnapshot, frozenStepSessionId, frozenStepId, frozenEffectId, frozenEffectPath, admitFrozenStep, closeFrozenStepAttempt, frozenStop, FROZEN_SNAPSHOT_SCHEMA, FROZEN_SNAPSHOT_SCHEMA_V4, stateDelta, applyStateDelta } from "./frozen-snapshot.js";
|
|
14
14
|
const failure = (code, message) => new RecoveryError(message, code);
|
|
15
15
|
/** The branch a route runs for its answer, as the interpreter takes it: the node's `unsure` branch when
|
|
16
16
|
* the answer's confidence is below `unsure.gte`, else the choice the answer names. */
|
|
@@ -171,6 +171,147 @@ export async function openRecovery(store, workflow, options) {
|
|
|
171
171
|
/** A pause or cancel was applied: its commit still lands, and nothing after it. */
|
|
172
172
|
let stopping = false;
|
|
173
173
|
const journal = await store.open(bound);
|
|
174
|
+
/** Whether the store keeps parts beside the state: the keyed layout needs both reads. */
|
|
175
|
+
const keyed = typeof journal.items === "function" && typeof journal.stateOf === "function";
|
|
176
|
+
/** The keyed layout is in effect for this open, and why not when the host asked for it. `moving` is a run stored whole
|
|
177
|
+
* that takes the keyed layout at its first write that knows the base of every map still running in it: a map's base
|
|
178
|
+
* is the state it starts from, which the run's whole record does not hold, and the map names it when it resumes. */
|
|
179
|
+
let layoutV4 = false;
|
|
180
|
+
let moving = false;
|
|
181
|
+
let layoutReason;
|
|
182
|
+
// The keyed layout's bookkeeping: the maps the run started (with the base each item is kept against), the states
|
|
183
|
+
// already stored by digest, each item's record as last written (its revision aside), and the path that produced each
|
|
184
|
+
// file a map item owns.
|
|
185
|
+
const maps = new Map();
|
|
186
|
+
const writtenStates = new Set();
|
|
187
|
+
const lastItems = new Map();
|
|
188
|
+
const fileOwners = new Map();
|
|
189
|
+
/** At a restore: the files the driver's own record keeps (the latest write of each, whoever made it), and, for a file
|
|
190
|
+
* only item records keep, the revision of the record it was taken from (a later record's write is the latest). */
|
|
191
|
+
const driverFiles = new Set();
|
|
192
|
+
const fileRevisions = new Map();
|
|
193
|
+
/** The effects this process admitted, landed or still queued on the commit chain: an item's record lists each one it
|
|
194
|
+
* owns, even one whose admission has not landed when the record is built. */
|
|
195
|
+
const queued = new Set();
|
|
196
|
+
const deltas = new WeakMap();
|
|
197
|
+
const digests = new WeakMap();
|
|
198
|
+
const digestOf = (state) => { let digest = digests.get(state); if (digest === undefined) {
|
|
199
|
+
digest = hash(state);
|
|
200
|
+
digests.set(state, digest);
|
|
201
|
+
} return digest; };
|
|
202
|
+
const invalid = (message) => failure("FROZEN_CHECKPOINT_INVALID", message);
|
|
203
|
+
/** A state the run references by digest, read back and checked against it. */
|
|
204
|
+
const stateBy = async (sha256) => {
|
|
205
|
+
const state = await journal.stateOf(sha256);
|
|
206
|
+
if (!state || typeof state !== "object" || Array.isArray(state) || hash(state) !== sha256)
|
|
207
|
+
throw invalid(`The state ${sha256} the run references is missing or changed`);
|
|
208
|
+
writtenStates.add(sha256);
|
|
209
|
+
digests.set(state, sha256);
|
|
210
|
+
return state;
|
|
211
|
+
};
|
|
212
|
+
/** A value the run references by digest (a failed map's result, which may be any JSON value), read back and checked. */
|
|
213
|
+
const valueBy = async (sha256) => {
|
|
214
|
+
const value = await journal.stateOf(sha256);
|
|
215
|
+
if (value === undefined || hash(value) !== sha256)
|
|
216
|
+
throw invalid(`The value ${sha256} the run references is missing or changed`);
|
|
217
|
+
writtenStates.add(sha256);
|
|
218
|
+
return value;
|
|
219
|
+
};
|
|
220
|
+
const SHA = /^[a-f0-9]{64}$/;
|
|
221
|
+
const isRecord = (value) => Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
222
|
+
const validEntry = (entry) => isRecord(entry) && Number.isSafeInteger(entry.length) && entry.length >= 0
|
|
223
|
+
&& Number.isSafeInteger(entry.items) && entry.items >= 0 && typeof entry.base === "string" && SHA.test(entry.base);
|
|
224
|
+
/** The path of map item `index` of the map at `mapPath`: the prefix the interpreter gives every path inside it. */
|
|
225
|
+
const itemPathOf = (mapPath, index) => `${mapPath}/items/${index}`;
|
|
226
|
+
/** Reads back the records of every item the maps in `table` wrote, merging into `snapshot` what lies in them: all of
|
|
227
|
+
* it for a map still running, and what a committed map's items keep in the whole state (their committed paths, step
|
|
228
|
+
* records, clocks, files and effects) for one that committed. Nested maps are read through their item. */
|
|
229
|
+
const restoreMaps = async (table, snapshot) => {
|
|
230
|
+
if (!isRecord(table))
|
|
231
|
+
throw invalid("The run's maps are not a record");
|
|
232
|
+
const frame = snapshot.pin;
|
|
233
|
+
for (const [mapPath, entry] of Object.entries(table)) {
|
|
234
|
+
if (!validEntry(entry) || nodeAt(view, mapPath)?.node !== "map")
|
|
235
|
+
throw invalid(`The map at ${mapPath} is not one this run can hold`);
|
|
236
|
+
const committed = frame.done.includes(mapPath);
|
|
237
|
+
const baseState = committed ? undefined : await stateBy(entry.base);
|
|
238
|
+
maps.set(mapPath, { length: entry.length, items: entry.items, base: entry.base, ...(baseState ? { baseState } : {}) });
|
|
239
|
+
const paths = Array.from({ length: entry.items }, (_, index) => itemPathOf(mapPath, index));
|
|
240
|
+
const records = await journal.items(paths);
|
|
241
|
+
for (const [index, item] of records.entries()) {
|
|
242
|
+
if (item === undefined)
|
|
243
|
+
continue;
|
|
244
|
+
const path = paths[index];
|
|
245
|
+
const record = item;
|
|
246
|
+
const valid = record.driver === driver && record.map === mapPath && record.index === index && record.base === entry.base && Number.isSafeInteger(record.revision)
|
|
247
|
+
&& ["running", "committed", "failed"].includes(record.status) && isRecord(record.frame) && Array.isArray(record.frame.done)
|
|
248
|
+
&& [record.frame.states, record.frame.chains, record.frame.loops, record.frame.routes, record.frame.inputs, record.frame.steps, record.clocks, record.files, record.unknownResponses, record.maps].every(isRecord)
|
|
249
|
+
&& Array.isArray(record.effects) && record.effects.every((id) => typeof id === "string");
|
|
250
|
+
if (!valid)
|
|
251
|
+
throw invalid(`The record of map item ${path} is not one this run wrote`);
|
|
252
|
+
lastItems.set(path, JSON.stringify({ ...record, revision: 0 }));
|
|
253
|
+
frame.done.push(...record.frame.done);
|
|
254
|
+
Object.assign(frame.steps, record.frame.steps);
|
|
255
|
+
Object.assign(snapshot.clocks, record.clocks);
|
|
256
|
+
for (const [name, sha256] of Object.entries(record.files)) {
|
|
257
|
+
// A committed item keeps the record it last wrote, so a file a later step wrote again holds an older hash there.
|
|
258
|
+
if (driverFiles.has(name) || (fileRevisions.get(name) ?? -1) > record.revision)
|
|
259
|
+
continue;
|
|
260
|
+
snapshot.files[name] = sha256;
|
|
261
|
+
fileOwners.set(name, `${path}/body`);
|
|
262
|
+
fileRevisions.set(name, record.revision);
|
|
263
|
+
}
|
|
264
|
+
if (Object.keys(record.unknownResponses).length)
|
|
265
|
+
Object.assign(snapshot.unknownResponses ??= {}, record.unknownResponses);
|
|
266
|
+
if (!committed) {
|
|
267
|
+
for (const [at, delta] of Object.entries(record.frame.states)) {
|
|
268
|
+
if (!isRecord(delta) || !isRecord(delta.set) || !Array.isArray(delta.unset) || typeof delta.sha256 !== "string")
|
|
269
|
+
throw invalid(`The state at ${at} is not a delta`);
|
|
270
|
+
const state = applyStateDelta(baseState, delta);
|
|
271
|
+
if (hash(state) !== delta.sha256)
|
|
272
|
+
throw invalid(`The state at ${at} does not rebuild to the state that was committed`);
|
|
273
|
+
frame.states[at] = state;
|
|
274
|
+
deltas.set(state, delta);
|
|
275
|
+
digests.set(state, delta.sha256);
|
|
276
|
+
}
|
|
277
|
+
Object.assign(frame.chains, record.frame.chains);
|
|
278
|
+
Object.assign(frame.loops, record.frame.loops);
|
|
279
|
+
Object.assign(frame.routes, record.frame.routes);
|
|
280
|
+
Object.assign(frame.inputs, record.frame.inputs);
|
|
281
|
+
}
|
|
282
|
+
await restoreMaps(record.maps, snapshot);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
};
|
|
286
|
+
/** The whole state of a run stored in the keyed layout: its own record with every state read back by digest, and what
|
|
287
|
+
* its map items keep. */
|
|
288
|
+
const restoreV4 = async (stored) => {
|
|
289
|
+
// A copy: what the items hold is merged into the restored frame, and the store's own state (which it writes back for
|
|
290
|
+
// a completion that names none) must stay the driver's record alone.
|
|
291
|
+
const { maps: table, pin, ...rest } = structuredClone(stored);
|
|
292
|
+
if (!isRecord(pin) || !isRecord(pin.states) || !Object.values(pin.states).every((sha256) => typeof sha256 === "string" && SHA.test(sha256)) || !Array.isArray(pin.done))
|
|
293
|
+
throw invalid("Invalid frozen recovery checkpoint");
|
|
294
|
+
const states = {};
|
|
295
|
+
for (const [at, sha256] of Object.entries(pin.states))
|
|
296
|
+
states[at] = await stateBy(sha256);
|
|
297
|
+
const restored = { ...rest, schema: FROZEN_SNAPSHOT_SCHEMA, pin: { ...pin, done: [...pin.done], states }, clocks: { ...rest.clocks }, files: { ...rest.files } };
|
|
298
|
+
for (const name of Object.keys(restored.files))
|
|
299
|
+
driverFiles.add(name);
|
|
300
|
+
// A failed map's finished results: the list of their slots is one value kept by its digest, each slot the digest of
|
|
301
|
+
// its result or null for an item that never finished.
|
|
302
|
+
if (pin.partial !== undefined) {
|
|
303
|
+
const partial = pin.partial;
|
|
304
|
+
if (!isRecord(partial) || typeof partial.path !== "string" || typeof partial.as !== "string" || typeof partial.results !== "string" || !SHA.test(partial.results)) {
|
|
305
|
+
throw invalid("A failed map's results are not kept by digest");
|
|
306
|
+
}
|
|
307
|
+
const slots = await valueBy(partial.results);
|
|
308
|
+
if (!Array.isArray(slots) || !slots.every((slot) => slot === null || (typeof slot === "string" && SHA.test(slot))))
|
|
309
|
+
throw invalid("A failed map's results are not a list of digests");
|
|
310
|
+
restored.pin.partial = { path: partial.path, as: partial.as, results: await Promise.all(slots.map((slot) => (slot === null ? null : valueBy(slot)))) };
|
|
311
|
+
}
|
|
312
|
+
await restoreMaps(table, restored);
|
|
313
|
+
return restored;
|
|
314
|
+
};
|
|
174
315
|
/** The host's stop as it was handed over (a cancel outranks a pause), and who issued it. */
|
|
175
316
|
let requested;
|
|
176
317
|
/** The one error the driver's stop is: its signal's reason and every refusal after it, naming the committed stop. */
|
|
@@ -179,6 +320,205 @@ export async function openRecovery(store, workflow, options) {
|
|
|
179
320
|
/** This process's ownership generation of the run: one per process that holds it. */
|
|
180
321
|
const generation = journal.generation;
|
|
181
322
|
const serial = () => serializeFrozenSnapshot(snapshot);
|
|
323
|
+
/** Files the run holds, with the path that produced them: a map item's file is kept in the item's record. */
|
|
324
|
+
const keepFiles = (files, at) => {
|
|
325
|
+
Object.assign(snapshot.files, files);
|
|
326
|
+
for (const name of Object.keys(files)) {
|
|
327
|
+
if (at === undefined)
|
|
328
|
+
fileOwners.delete(name);
|
|
329
|
+
else
|
|
330
|
+
fileOwners.set(name, at);
|
|
331
|
+
}
|
|
332
|
+
};
|
|
333
|
+
/** The path an effect clock belongs to: an attempt's effect, a poll's or a wait's node. */
|
|
334
|
+
const clockPath = (key) => {
|
|
335
|
+
if (key.startsWith("attempt:"))
|
|
336
|
+
return frozenEffectPath(key.slice("attempt:".length))?.path;
|
|
337
|
+
// poll:<tag>:<path>, and wait:<tag>:<path>:<ordinal>.
|
|
338
|
+
const poll = /^poll:[^:]+:(\/root.*)$/s.exec(key);
|
|
339
|
+
if (poll)
|
|
340
|
+
return poll[1];
|
|
341
|
+
const wait = /^wait:[^:]+:(\/root.*):\d+$/s.exec(key);
|
|
342
|
+
return wait ? wait[1] : undefined;
|
|
343
|
+
};
|
|
344
|
+
/** One write in the keyed layout: the driver's record without what lies in map items, the records of the items that
|
|
345
|
+
* changed, and the states first referenced now. `admitting` is an effect admitted in this write; `owner` the item that
|
|
346
|
+
* lists it. */
|
|
347
|
+
const keyedWrite = (admitting) => {
|
|
348
|
+
const frame = snapshot.pin;
|
|
349
|
+
const done = new Set(frame.done);
|
|
350
|
+
const builds = new Map();
|
|
351
|
+
/** The record a path lies in: undefined for the driver's own, null for an item of a map that committed (it keeps
|
|
352
|
+
* the record it last wrote). */
|
|
353
|
+
const own = (path) => {
|
|
354
|
+
const at = path === undefined ? undefined : mapItemOf(path);
|
|
355
|
+
if (!at)
|
|
356
|
+
return undefined;
|
|
357
|
+
// A committed map's items keep the records they last wrote; one that committed before the run took the keyed
|
|
358
|
+
// layout has none, and what lies in its items stays in the driver's record, as it was.
|
|
359
|
+
// A map that commits in the very write that moves the run has items with no record yet: they are written once, here.
|
|
360
|
+
if (done.has(at.map) && (!maps.has(at.map) || lastItems.has(at.item)))
|
|
361
|
+
return maps.has(at.map) ? null : undefined;
|
|
362
|
+
let build = builds.get(at.item);
|
|
363
|
+
if (!build) {
|
|
364
|
+
const map = maps.get(at.map);
|
|
365
|
+
if (!map)
|
|
366
|
+
throw invalid(`The map item ${at.item} belongs to no map this run started`);
|
|
367
|
+
build = { path: at.item, map: at.map, index: at.index, record: {
|
|
368
|
+
driver, map: at.map, index: at.index, base: map.base, revision: 0, status: "running",
|
|
369
|
+
frame: { done: [], states: {}, chains: {}, loops: {}, routes: {}, inputs: {}, steps: {} },
|
|
370
|
+
effects: [], clocks: {}, files: {}, unknownResponses: {}, maps: {},
|
|
371
|
+
} };
|
|
372
|
+
builds.set(at.item, build);
|
|
373
|
+
}
|
|
374
|
+
return build;
|
|
375
|
+
};
|
|
376
|
+
const states = {};
|
|
377
|
+
const kept = (state) => {
|
|
378
|
+
const digest = digestOf(state);
|
|
379
|
+
if (!writtenStates.has(digest)) {
|
|
380
|
+
writtenStates.add(digest);
|
|
381
|
+
states[digest] = state;
|
|
382
|
+
}
|
|
383
|
+
return digest;
|
|
384
|
+
};
|
|
385
|
+
/** A failed map's result, which may be any JSON value, kept once by its digest. */
|
|
386
|
+
const keptValue = (value) => {
|
|
387
|
+
if (isRecord(value))
|
|
388
|
+
return kept(value);
|
|
389
|
+
const digest = hash(value);
|
|
390
|
+
if (!writtenStates.has(digest)) {
|
|
391
|
+
writtenStates.add(digest);
|
|
392
|
+
states[digest] = value;
|
|
393
|
+
}
|
|
394
|
+
return digest;
|
|
395
|
+
};
|
|
396
|
+
const deltaOf = (map, state) => {
|
|
397
|
+
const known = deltas.get(state);
|
|
398
|
+
if (known)
|
|
399
|
+
return known;
|
|
400
|
+
const base = maps.get(map)?.baseState;
|
|
401
|
+
if (!base)
|
|
402
|
+
throw invalid(`The base of the map at ${map} is not known`);
|
|
403
|
+
const delta = stateDelta(base, state, digestOf(state));
|
|
404
|
+
deltas.set(state, delta);
|
|
405
|
+
return delta;
|
|
406
|
+
};
|
|
407
|
+
const pin = { input: frame.input, done: [], states: {}, chains: {}, loops: {}, routes: {}, inputs: {}, steps: {},
|
|
408
|
+
// A failed map's results: one digest, of the list of each result's digest (null for an item that never finished).
|
|
409
|
+
...(frame.partial ? { partial: { path: frame.partial.path, as: frame.partial.as, results: keptValue(frame.partial.results.map((result) => (result === null || result === undefined ? null : keptValue(result)))) } } : {}) };
|
|
410
|
+
for (const at of frame.done) {
|
|
411
|
+
const build = own(at);
|
|
412
|
+
if (build === undefined)
|
|
413
|
+
pin.done.push(at);
|
|
414
|
+
else if (build)
|
|
415
|
+
build.record.frame.done.push(at);
|
|
416
|
+
}
|
|
417
|
+
for (const [at, state] of Object.entries(frame.states)) {
|
|
418
|
+
const build = own(at);
|
|
419
|
+
if (build === undefined)
|
|
420
|
+
pin.states[at] = kept(state);
|
|
421
|
+
else if (build)
|
|
422
|
+
build.record.frame.states[at] = deltaOf(build.map, state);
|
|
423
|
+
}
|
|
424
|
+
for (const table of ["chains", "loops", "routes", "inputs", "steps"]) {
|
|
425
|
+
for (const [at, value] of Object.entries(frame[table])) {
|
|
426
|
+
const build = own(at);
|
|
427
|
+
if (build === undefined)
|
|
428
|
+
pin[table][at] = value;
|
|
429
|
+
else if (build)
|
|
430
|
+
build.record.frame[table][at] = value;
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
const clocks = {};
|
|
434
|
+
for (const [key, value] of Object.entries(snapshot.clocks)) {
|
|
435
|
+
const build = own(clockPath(key));
|
|
436
|
+
if (build === undefined)
|
|
437
|
+
clocks[key] = value;
|
|
438
|
+
else if (build)
|
|
439
|
+
build.record.clocks[key] = value;
|
|
440
|
+
}
|
|
441
|
+
const files = {};
|
|
442
|
+
for (const [name, sha256] of Object.entries(snapshot.files)) {
|
|
443
|
+
const build = own(fileOwners.get(name));
|
|
444
|
+
if (build === undefined)
|
|
445
|
+
files[name] = sha256;
|
|
446
|
+
else if (build)
|
|
447
|
+
build.record.files[name] = sha256;
|
|
448
|
+
}
|
|
449
|
+
const unknownResponses = {};
|
|
450
|
+
for (const [id, value] of Object.entries(snapshot.unknownResponses ?? {})) {
|
|
451
|
+
const build = own(frozenEffectPath(id)?.path);
|
|
452
|
+
if (build === undefined)
|
|
453
|
+
unknownResponses[id] = value;
|
|
454
|
+
else if (build)
|
|
455
|
+
build.record.unknownResponses[id] = value;
|
|
456
|
+
}
|
|
457
|
+
for (const id of new Set([...journal.effects().map((effect) => effect.id), ...queued, ...(admitting === undefined ? [] : [admitting])])) {
|
|
458
|
+
const build = own(frozenEffectPath(id)?.path);
|
|
459
|
+
if (build)
|
|
460
|
+
build.record.effects.push(id);
|
|
461
|
+
}
|
|
462
|
+
// A map's item records are the first ones: count them before the maps are written.
|
|
463
|
+
for (const build of builds.values()) {
|
|
464
|
+
const map = maps.get(build.map);
|
|
465
|
+
map.items = Math.max(map.items, build.index + 1);
|
|
466
|
+
}
|
|
467
|
+
const table = {};
|
|
468
|
+
for (const [at, map] of maps) {
|
|
469
|
+
const build = own(at);
|
|
470
|
+
const entry = { length: map.length, items: map.items, base: map.base };
|
|
471
|
+
if (build === undefined)
|
|
472
|
+
table[at] = entry;
|
|
473
|
+
else if (build)
|
|
474
|
+
build.record.maps[at] = entry;
|
|
475
|
+
if (!done.has(at) && map.baseState)
|
|
476
|
+
kept(map.baseState);
|
|
477
|
+
}
|
|
478
|
+
/** An item is done when its body committed: a chain never commits itself, so the last step a chain body runs
|
|
479
|
+
* through, descending through nested chains, is where the body ends. */
|
|
480
|
+
const finished = (item) => {
|
|
481
|
+
let at = `${item}/body`;
|
|
482
|
+
let node = nodeAt(view, at);
|
|
483
|
+
while (node?.node === "chain" && node.steps.length > 0) {
|
|
484
|
+
at = `${at}/steps/${node.steps.length - 1}`;
|
|
485
|
+
node = node.steps[node.steps.length - 1];
|
|
486
|
+
}
|
|
487
|
+
return done.has(at);
|
|
488
|
+
};
|
|
489
|
+
const items = {};
|
|
490
|
+
for (const build of builds.values()) {
|
|
491
|
+
build.record.status = finished(build.path) ? "committed" : "running";
|
|
492
|
+
const text = JSON.stringify(build.record);
|
|
493
|
+
if (lastItems.get(build.path) !== text) {
|
|
494
|
+
lastItems.set(build.path, text);
|
|
495
|
+
items[build.path] = build.record;
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
const { pin: _pin, schema: _schema, clocks: _clocks, files: _files, unknownResponses: _unknown, ...rest } = snapshot;
|
|
499
|
+
const state = serializeFrozenSnapshot({ ...rest, schema: FROZEN_SNAPSHOT_SCHEMA_V4, pin, clocks, files,
|
|
500
|
+
...(Object.keys(unknownResponses).length ? { unknownResponses } : {}), maps: table });
|
|
501
|
+
const owner = admitting === undefined ? undefined : (own(frozenEffectPath(admitting)?.path) ?? undefined)?.path;
|
|
502
|
+
return { state, parts: (revision) => ({ items: Object.fromEntries(Object.entries(items).map(([path, item]) => [path, { ...item, revision }])), states }), ...(owner ? { owner } : {}) };
|
|
503
|
+
};
|
|
504
|
+
/** Whether every map still running with something in its items has its base: then a run stored whole can move. */
|
|
505
|
+
const basesKnown = () => {
|
|
506
|
+
const frame = snapshot.pin;
|
|
507
|
+
const done = new Set(frame.done);
|
|
508
|
+
const paths = [...frame.done, ...Object.keys(frame.states), ...Object.keys(frame.chains), ...Object.keys(frame.inputs), ...Object.keys(frame.steps),
|
|
509
|
+
...Object.keys(frame.loops), ...Object.keys(frame.routes)];
|
|
510
|
+
return paths.every((path) => { const at = mapItemOf(path); return !at || done.has(at.map) || Boolean(maps.get(at.map)?.baseState); });
|
|
511
|
+
};
|
|
512
|
+
/** One write of the driver's state, in the layout this open keeps. */
|
|
513
|
+
const writeOf = (admitting) => {
|
|
514
|
+
if (admitting !== undefined)
|
|
515
|
+
queued.add(admitting);
|
|
516
|
+
if (moving && basesKnown()) {
|
|
517
|
+
moving = false;
|
|
518
|
+
layoutV4 = true;
|
|
519
|
+
}
|
|
520
|
+
return layoutV4 ? keyedWrite(admitting) : { state: serial() };
|
|
521
|
+
};
|
|
182
522
|
/** The journal's commits, in the order the driver made them: a commit a synchronous method starts lands before any
|
|
183
523
|
* later one, and an effect's admission waits for every earlier commit. A failed commit stops the run. */
|
|
184
524
|
let chain = Promise.resolve();
|
|
@@ -198,7 +538,28 @@ export async function openRecovery(store, workflow, options) {
|
|
|
198
538
|
// A driver an earlier process bound, with no checkpoint, is a run killed between its binding and its first
|
|
199
539
|
// checkpoint when the journal holds nothing else of it: it never started, and it starts now. One with effects and
|
|
200
540
|
// no checkpoint needs reconciliation.
|
|
201
|
-
const
|
|
541
|
+
const stored = journal.state;
|
|
542
|
+
// A run a later build stored in a schema this build does not know: refused with its own code, so a host leaves the
|
|
543
|
+
// run for a build that knows it rather than ending it.
|
|
544
|
+
const known = [FROZEN_SNAPSHOT_SCHEMA, FROZEN_SNAPSHOT_SCHEMA_V4];
|
|
545
|
+
const found = stored?.schema;
|
|
546
|
+
const version = typeof found === "string" ? /^agentrun\.frozen_run\.v(\d+)$/.exec(found)?.[1] : undefined;
|
|
547
|
+
if (version !== undefined && !known.includes(found) && Number(version) > 4) {
|
|
548
|
+
throw Object.assign(new RecoveryError(`This run is stored as ${found}, newer than this build knows (${known.join(", ")}); open it with a build that knows it`, "RECOVERY_SCHEMA_NEWER"), { found, known });
|
|
549
|
+
}
|
|
550
|
+
// The keyed layout is sticky: a run stored in it stays in it. A run stored whole, or a new one, takes it when the
|
|
551
|
+
// host asks for it and the store keeps parts.
|
|
552
|
+
if (stored?.schema === FROZEN_SNAPSHOT_SCHEMA_V4) {
|
|
553
|
+
if (!keyed)
|
|
554
|
+
throw invalid("This run is stored in the keyed layout (agentrun.frozen_run.v4), and this store keeps no parts");
|
|
555
|
+
layoutV4 = true;
|
|
556
|
+
}
|
|
557
|
+
else {
|
|
558
|
+
moving = options.layout === "v4" && keyed;
|
|
559
|
+
if (options.layout === "v4" && !keyed)
|
|
560
|
+
layoutReason = "the store keeps no parts (its journal has no items and stateOf)";
|
|
561
|
+
}
|
|
562
|
+
const saved = stored?.schema === FROZEN_SNAPSHOT_SCHEMA_V4 ? await restoreV4(stored) : stored;
|
|
202
563
|
if (journal.existing && !saved && journal.effects().length > 0)
|
|
203
564
|
throw failure("FROZEN_CHECKPOINT_INVALID", "Existing frozen run has no checkpoint; retained workspace requires reconciliation");
|
|
204
565
|
// Only the paths of the run's own document are resolved.
|
|
@@ -223,7 +584,7 @@ export async function openRecovery(store, workflow, options) {
|
|
|
223
584
|
const at = frozenEffectPath(id)?.path;
|
|
224
585
|
const declared = at === undefined ? undefined : nodeAt(view, at);
|
|
225
586
|
const files = reader.hashes(declared?.node === "call" ? declared.produces ?? [] : []);
|
|
226
|
-
|
|
587
|
+
keepFiles(files, at);
|
|
227
588
|
// The calls an earlier process kept stay: the host's re-send is added after them, so a call that applied before the kill is still on record.
|
|
228
589
|
if (receipt?.calls?.length)
|
|
229
590
|
await journal.called(id, [...(journal.effect(id)?.calls ?? []), ...receipt.calls]);
|
|
@@ -238,18 +599,21 @@ export async function openRecovery(store, workflow, options) {
|
|
|
238
599
|
reconciling = false;
|
|
239
600
|
}
|
|
240
601
|
}
|
|
241
|
-
|
|
602
|
+
// The open's commit: a run stored whole that takes the keyed layout moves to it here, whole or not at all.
|
|
603
|
+
const opening = writeOf();
|
|
604
|
+
const layoutNote = layoutReason ? { kind: "layout", detail: { schema: FROZEN_SNAPSHOT_SCHEMA, reason: layoutReason } } : undefined;
|
|
605
|
+
await journal.save(opening.state, layoutNote, opening.parts);
|
|
242
606
|
}
|
|
243
607
|
catch (error) {
|
|
244
608
|
await journal.close();
|
|
245
609
|
throw error;
|
|
246
610
|
}
|
|
247
611
|
const ordinals = new Map();
|
|
248
|
-
/** One commit of the driver's
|
|
612
|
+
/** One commit of the driver's state. */
|
|
249
613
|
const save = () => {
|
|
250
|
-
const
|
|
614
|
+
const write = writeOf();
|
|
251
615
|
// A synchronous method does not wait for it; the failure is the run's, and the next awaited commit throws it.
|
|
252
|
-
return persist(() => journal.save(state)).catch(() => undefined);
|
|
616
|
+
return persist(() => journal.save(write.state, undefined, write.parts)).catch(() => undefined);
|
|
253
617
|
};
|
|
254
618
|
/** Effects dispatched and not yet completed or failed. A pause is not taken while one is in flight. */
|
|
255
619
|
let inFlight = 0;
|
|
@@ -372,7 +736,7 @@ export async function openRecovery(store, workflow, options) {
|
|
|
372
736
|
// Every store key below is the interpreter's execution path, so a composed graph (a loop, a route, a
|
|
373
737
|
// map whose body is structure, a nested child) resumes where it stopped.
|
|
374
738
|
supportsExecutionPaths: true,
|
|
375
|
-
async resume(
|
|
739
|
+
async resume(node, state, _item, executionPath) {
|
|
376
740
|
await seam();
|
|
377
741
|
const { frame, doc } = active();
|
|
378
742
|
const at = placed(executionPath, doc);
|
|
@@ -393,6 +757,11 @@ export async function openRecovery(store, workflow, options) {
|
|
|
393
757
|
return committed;
|
|
394
758
|
if (doneOf(frame).has(at))
|
|
395
759
|
throw pathFault(at, "A committed path was asked for again after its state was released");
|
|
760
|
+
// A map that starts, or starts again, names the base its items are kept against: the state it starts from.
|
|
761
|
+
if ((layoutV4 || moving) && node.node === "map") {
|
|
762
|
+
const list = String(node.itemsPath).split(".").reduce((value, key) => value?.[key], state);
|
|
763
|
+
maps.set(at, { length: Array.isArray(list) ? list.length : 0, items: maps.get(at)?.items ?? 0, base: digestOf(state), baseState: state });
|
|
764
|
+
}
|
|
396
765
|
// A node started in an earlier process starts again from the same state, or not at all. The
|
|
397
766
|
// binding is durable with the next commit, which is the first durable thing done under it.
|
|
398
767
|
const digest = hash(state);
|
|
@@ -420,7 +789,7 @@ export async function openRecovery(store, workflow, options) {
|
|
|
420
789
|
if (step && (frame.chains[step.chain]?.index ?? -1) >= step.index)
|
|
421
790
|
throw pathFault(at, "A chain step committed behind its chain's frontier");
|
|
422
791
|
if (node.node === "artifact" && node.path)
|
|
423
|
-
|
|
792
|
+
keepFiles(fileHashes([node.path]), at);
|
|
424
793
|
frame.done.push(at);
|
|
425
794
|
done.add(at);
|
|
426
795
|
// A map that committed, alone or inside what committed, has its results in the state: what it had finished
|
|
@@ -499,6 +868,10 @@ export async function openRecovery(store, workflow, options) {
|
|
|
499
868
|
get generation() { return generation; },
|
|
500
869
|
/** A refusal found before any step, or null: the host escalates it before any step runs. */
|
|
501
870
|
preStepRefusal,
|
|
871
|
+
/** How this open keeps the run: the layout's schema, and why it is not the keyed one the host asked for. */
|
|
872
|
+
get layout() {
|
|
873
|
+
return { schema: layoutV4 ? FROZEN_SNAPSHOT_SCHEMA_V4 : FROZEN_SNAPSHOT_SCHEMA, ...(layoutReason ? { reason: layoutReason } : {}) };
|
|
874
|
+
},
|
|
502
875
|
binding,
|
|
503
876
|
/** The run's key: what its step sessions and effects are named under. */
|
|
504
877
|
driverId: driver,
|
|
@@ -639,7 +1012,9 @@ export async function openRecovery(store, workflow, options) {
|
|
|
639
1012
|
// The note names the handoff, not the state: the state rides the snapshot.
|
|
640
1013
|
const detail = { kind: row.kind, stage: row.stage, summary: String(row.summary ?? "").slice(0, 512), step_label: row.step_label, step_exec_id: stepId, workflow_sha_used: row.workflow_sha_used,
|
|
641
1014
|
cost_usd: row.cost_usd, tail: snapshot.escalation.tail, state_keys: Object.keys(row.state ?? {}) };
|
|
642
|
-
|
|
1015
|
+
// The parts are computed with the state, after the row has its revision.
|
|
1016
|
+
let parts;
|
|
1017
|
+
await persist(() => journal.save((revision) => { snapshot.escalation.revision = revision; const write = writeOf(); parts = write.parts; return write.state; }, { kind: "escalation", detail }, (revision) => parts?.(revision) ?? {}));
|
|
643
1018
|
return snapshot.escalation;
|
|
644
1019
|
},
|
|
645
1020
|
escalation() { return snapshot.escalation; },
|
|
@@ -664,7 +1039,8 @@ export async function openRecovery(store, workflow, options) {
|
|
|
664
1039
|
const detail = { escalation_revision: row.revision, previous_cost_usd: row.cost_usd,
|
|
665
1040
|
cost_usd: spend.cost_usd, turns: spend.turns, effect_calls: spend.effect_calls };
|
|
666
1041
|
Object.assign(row, spend);
|
|
667
|
-
|
|
1042
|
+
const write = writeOf();
|
|
1043
|
+
await persist(() => journal.save(write.state, { kind: "handoff", detail }, write.parts));
|
|
668
1044
|
return row;
|
|
669
1045
|
},
|
|
670
1046
|
/** The state the document's committed top-level prefix left behind, plus what a failed map's finished
|
|
@@ -700,7 +1076,7 @@ export async function openRecovery(store, workflow, options) {
|
|
|
700
1076
|
throw failure("FROZEN_EFFECT_UNKNOWN", `Reconcile ${id} before resuming; it will not be repeated`);
|
|
701
1077
|
const receipt = existing.result;
|
|
702
1078
|
verifyFiles(receipt.files);
|
|
703
|
-
|
|
1079
|
+
keepFiles(receipt.files, at);
|
|
704
1080
|
await save();
|
|
705
1081
|
await flush();
|
|
706
1082
|
return receipt.value;
|
|
@@ -720,7 +1096,8 @@ export async function openRecovery(store, workflow, options) {
|
|
|
720
1096
|
inFlight += 1;
|
|
721
1097
|
try {
|
|
722
1098
|
// The driver's state and the admission are one transaction.
|
|
723
|
-
const
|
|
1099
|
+
const admission = writeOf(id);
|
|
1100
|
+
const admitted = await persist(() => journal.admit(id, String(params.node.label), argsHash, admission.state, driver, external, params.input ?? null, admission.parts, admission.owner));
|
|
724
1101
|
if (admitted !== "new")
|
|
725
1102
|
throw failure("FROZEN_EFFECT_UNKNOWN", `Reconcile ${id} before dispatch`);
|
|
726
1103
|
// What landed while the admission was being committed still counts: after a cancel, the caller's own
|
|
@@ -760,8 +1137,9 @@ export async function openRecovery(store, workflow, options) {
|
|
|
760
1137
|
// so a continuation that inherits the receipt can answer the same call from it. Any other
|
|
761
1138
|
// effect keeps its arguments beside its result.
|
|
762
1139
|
const intent = params.node.via === "tool" ? { tool: String(params.node.tool), args: params.input } : undefined;
|
|
763
|
-
|
|
764
|
-
|
|
1140
|
+
keepFiles(files, at);
|
|
1141
|
+
const completion = writeOf();
|
|
1142
|
+
await persist(() => journal.complete(id, { value, files, ...(intent ? { intent } : { args: params.input ?? null }) }, completion.state, completion.parts));
|
|
765
1143
|
return value;
|
|
766
1144
|
}
|
|
767
1145
|
finally {
|
|
@@ -32,6 +32,14 @@ export declare function completedIteration(workflow: Workflow, path: string): {
|
|
|
32
32
|
loop: string;
|
|
33
33
|
index: number;
|
|
34
34
|
} | undefined;
|
|
35
|
+
/** The innermost map item `path` lies in, when it lies in one: the item's own path (`<map>/items/<i>`, a prefix of
|
|
36
|
+
* `path` without the item's `/body`), its map's path and its index. A map's own path lies in the item that holds the
|
|
37
|
+
* map, not in its own items. */
|
|
38
|
+
export declare function mapItemOf(path: string): {
|
|
39
|
+
item: string;
|
|
40
|
+
map: string;
|
|
41
|
+
index: number;
|
|
42
|
+
} | undefined;
|
|
35
43
|
/** Whether `inner` is `outer` or lies inside it, compared segment by segment. */
|
|
36
44
|
export declare function isWithin(inner: string, outer: string): boolean;
|
|
37
45
|
/** The top-level step whose completion `path`'s commit is, or undefined. A top-level step commits at
|
|
@@ -111,6 +111,20 @@ export function completedIteration(workflow, path) {
|
|
|
111
111
|
}
|
|
112
112
|
return node && node.node !== "chain" ? innermost : undefined;
|
|
113
113
|
}
|
|
114
|
+
/** The innermost map item `path` lies in, when it lies in one: the item's own path (`<map>/items/<i>`, a prefix of
|
|
115
|
+
* `path` without the item's `/body`), its map's path and its index. A map's own path lies in the item that holds the
|
|
116
|
+
* map, not in its own items. */
|
|
117
|
+
export function mapItemOf(path) {
|
|
118
|
+
const segments = pathSegments(path);
|
|
119
|
+
if (!segments)
|
|
120
|
+
return undefined;
|
|
121
|
+
for (let i = segments.length - 3; i >= 1; i -= 1) {
|
|
122
|
+
if (segments[i].segment === "items" && INDEX.test(segments[i + 1].segment) && segments[i + 2].segment === "body") {
|
|
123
|
+
return { item: prefix(path, segments, i + 2), map: prefix(path, segments, i), index: Number(segments[i + 1].segment) };
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return undefined;
|
|
127
|
+
}
|
|
114
128
|
/** Whether `inner` is `outer` or lies inside it, compared segment by segment. */
|
|
115
129
|
export function isWithin(inner, outer) {
|
|
116
130
|
const a = pathSegments(inner);
|
|
@@ -1,5 +1,81 @@
|
|
|
1
|
+
import { type PartialSlot, type Sha256, type StateDelta } from "./item-state.js";
|
|
1
2
|
export declare const FROZEN_SNAPSHOT_SCHEMA = "agentrun.frozen_run.v3";
|
|
3
|
+
/** The keyed layout: what lies in a map item is kept in that item's record beside the state, and a state the driver
|
|
4
|
+
* references is kept once by its digest, so no record rewritten at a commit grows with a map's items. */
|
|
5
|
+
export declare const FROZEN_SNAPSHOT_SCHEMA_V4 = "agentrun.frozen_run.v4";
|
|
2
6
|
type State = Record<string, unknown>;
|
|
7
|
+
export { stateDelta, applyStateDelta, canonicalJson, type StateDelta, type Sha256, type PartialSlot } from "./item-state.js";
|
|
8
|
+
/** A map the run started: how many items it has, how many item records were written (always the first ones: items
|
|
9
|
+
* start in order, and an item has a record from its start), and the digest of its base. */
|
|
10
|
+
export type MapEntry = {
|
|
11
|
+
length: number;
|
|
12
|
+
items: number;
|
|
13
|
+
base: string;
|
|
14
|
+
};
|
|
15
|
+
/** One map item's share of the driver's state (`agentrun.items`), keyed by the item's path. Every path in it lies in
|
|
16
|
+
* this item and in no item nested in it. */
|
|
17
|
+
export type ItemRecord = {
|
|
18
|
+
driver: string;
|
|
19
|
+
map: string;
|
|
20
|
+
index: number;
|
|
21
|
+
base: string;
|
|
22
|
+
/** The journal revision of the commit that wrote this value. */
|
|
23
|
+
revision: number;
|
|
24
|
+
status: "running" | "committed" | "failed";
|
|
25
|
+
frame: {
|
|
26
|
+
done: string[];
|
|
27
|
+
states: Record<string, StateDelta>;
|
|
28
|
+
chains: PathFrame["chains"];
|
|
29
|
+
loops: PathFrame["loops"];
|
|
30
|
+
routes: PathFrame["routes"];
|
|
31
|
+
inputs: PathFrame["inputs"];
|
|
32
|
+
steps: PathFrame["steps"];
|
|
33
|
+
};
|
|
34
|
+
effects: string[];
|
|
35
|
+
clocks: Record<string, number>;
|
|
36
|
+
files: Record<string, string>;
|
|
37
|
+
unknownResponses: Record<string, unknown>;
|
|
38
|
+
/** Maps started inside this item. */
|
|
39
|
+
maps: Record<string, MapEntry>;
|
|
40
|
+
};
|
|
41
|
+
/** The driver's own record in the keyed layout: the v3 snapshot without what lies in map items, its frame's states by
|
|
42
|
+
* digest, and the maps the run started. */
|
|
43
|
+
export type FrozenSnapshotV4 = Omit<FrozenSnapshot, "schema" | "pin"> & {
|
|
44
|
+
schema: typeof FROZEN_SNAPSHOT_SCHEMA_V4;
|
|
45
|
+
pin: Omit<PathFrame, "states" | "partial"> & {
|
|
46
|
+
states: Record<string, string>;
|
|
47
|
+
partial?: {
|
|
48
|
+
path: string;
|
|
49
|
+
as: string;
|
|
50
|
+
results: string;
|
|
51
|
+
};
|
|
52
|
+
};
|
|
53
|
+
maps: Record<string, MapEntry>;
|
|
54
|
+
};
|
|
55
|
+
/** Two values a state holds under one key are the same value. A key an item copied from its base holds the base's own
|
|
56
|
+
* object, or one with the same JSON. */
|
|
57
|
+
/** A map item's state at `path`, rebuilt as the driver rebuilds it (`./item-state.ts`, which imports nothing): the
|
|
58
|
+
* `agentrun.states` value its `base` names, with the item's `set` and `unset` for that path applied, checked against
|
|
59
|
+
* that path's `sha256`. `item` is an `agentrun.items` value; `states` holds `agentrun.states` values by SHA-256. When
|
|
60
|
+
* the base is missing, or a digest differs, it says why and never guesses. `sha256` defaults to Node's. */
|
|
61
|
+
export declare function itemState(item: unknown, states: Readonly<Record<string, unknown>> | ReadonlyMap<string, unknown>, path: string, options?: {
|
|
62
|
+
sha256?: Sha256;
|
|
63
|
+
}): {
|
|
64
|
+
state: State;
|
|
65
|
+
} | {
|
|
66
|
+
unreadable: string;
|
|
67
|
+
};
|
|
68
|
+
/** A failed map's finished results from the driver state's `pin.partial` (`./item-state.ts`, which imports nothing): in
|
|
69
|
+
* the keyed layout, the list of slots and each result read by digest and checked; a whole-state run's list as it is.
|
|
70
|
+
* Each slot is null (the item never finished), `{ result }`, or `{ unreadable }`, never a guess. `sha256` defaults to
|
|
71
|
+
* Node's. */
|
|
72
|
+
export declare function partialResults(partial: unknown, states: Readonly<Record<string, unknown>> | ReadonlyMap<string, unknown>, options?: {
|
|
73
|
+
sha256?: Sha256;
|
|
74
|
+
}): {
|
|
75
|
+
slots: PartialSlot[];
|
|
76
|
+
} | {
|
|
77
|
+
unreadable: string;
|
|
78
|
+
};
|
|
3
79
|
/** What a question's receipts need beyond its answer: the request's producer invocation, its size and its
|
|
4
80
|
* latency. */
|
|
5
81
|
export type QuestionReceipts = {
|
|
@@ -146,4 +222,3 @@ export declare function closeFrozenStepAttempt(step: StepRecord, final: boolean)
|
|
|
146
222
|
/** The run-wide stop the driver applies: the host's pause or cancel it was handed, or null when it keeps
|
|
147
223
|
* running. A cancelled snapshot stays cancelled. */
|
|
148
224
|
export declare function frozenStop(status: FrozenSnapshot["status"], requested: "pause" | "cancel" | undefined): "paused" | "cancelled" | null;
|
|
149
|
-
export {};
|