@parcha/agentrun-pi 0.1.0-beta.2 → 0.1.0-beta.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +104 -27
- package/dist/cli.js +15 -4
- package/dist/demo-cli.d.ts +2 -0
- package/dist/demo-cli.js +25 -0
- package/dist/demo.js +1 -0
- package/dist/durable/decision-recovery.d.ts +23 -0
- package/dist/durable/decision-recovery.js +57 -0
- package/dist/durable/decisions.d.ts +113 -0
- package/dist/durable/decisions.js +71 -0
- package/dist/durable/index.d.ts +20 -0
- package/dist/durable/index.js +14 -0
- package/dist/durable/node-runner.d.ts +106 -0
- package/dist/durable/node-runner.js +253 -0
- package/dist/durable/node.d.ts +88 -0
- package/dist/durable/node.js +98 -0
- package/dist/durable/pi-versions.d.ts +34 -0
- package/dist/durable/pi-versions.js +66 -0
- package/dist/durable/readers.d.ts +28 -0
- package/dist/durable/readers.js +103 -0
- package/dist/durable/receipts.d.ts +54 -0
- package/dist/durable/receipts.js +92 -0
- package/dist/durable/record-file.d.ts +10 -0
- package/dist/durable/record-file.js +56 -0
- package/dist/durable/record-tool.d.ts +76 -0
- package/dist/durable/record-tool.js +123 -0
- package/dist/durable/record.d.ts +157 -0
- package/dist/durable/record.js +402 -0
- package/dist/durable/recording.d.ts +90 -0
- package/dist/durable/recording.js +292 -0
- package/dist/durable/store.d.ts +23 -0
- package/dist/durable/store.js +240 -0
- package/dist/durable/task.d.ts +133 -0
- package/dist/durable/task.js +411 -0
- package/dist/durable/workflow-tool.d.ts +32 -0
- package/dist/durable/workflow-tool.js +188 -0
- package/dist/extension-service.js +63 -22
- package/dist/extension.d.ts +2 -0
- package/dist/extension.js +469 -80
- package/dist/host-addendum.d.ts +4 -0
- package/dist/host-addendum.js +20 -0
- package/dist/host-session.js +5 -2
- package/dist/index.d.ts +1 -2
- package/dist/index.js +1 -2
- package/dist/pi-host.d.ts +7 -0
- package/dist/pi-host.js +22 -0
- package/dist/presentation.js +2 -2
- package/dist/run-error.d.ts +6 -0
- package/dist/run-error.js +13 -0
- package/dist/runner.d.ts +1 -6
- package/dist/runner.js +7 -13
- package/dist/signin/index.d.ts +9 -0
- package/dist/signin/index.js +7 -0
- package/dist/signin/limit.d.ts +10 -0
- package/dist/signin/limit.js +23 -0
- package/dist/signin/resolve.d.ts +47 -0
- package/dist/signin/resolve.js +163 -0
- package/dist/signin/signin.d.ts +44 -0
- package/dist/signin/signin.js +126 -0
- package/dist/signin/spec.d.ts +116 -0
- package/dist/signin/spec.js +181 -0
- package/dist/signin/store.d.ts +40 -0
- package/dist/signin/store.js +206 -0
- package/dist/skills/author/SKILL.md +138 -0
- package/{skills → dist/skills}/author/references/jev-decisions.md +27 -14
- package/dist/skills/author/references/language.md +134 -0
- package/dist/skills/author/references/sdk-authoring.md +11 -0
- package/{skills → dist/skills}/author/references/workflow-format.md +7 -15
- package/dist/sop.d.ts +12 -0
- package/dist/sop.js +29 -0
- package/dist/thinking.d.ts +3 -0
- package/dist/thinking.js +7 -0
- package/dist/tool-context.d.ts +21 -0
- package/dist/tool-context.js +13 -0
- package/dist/triage-demo.d.ts +62 -0
- package/dist/triage-demo.js +153 -0
- package/dist/types.d.ts +7 -0
- package/dist/workflow-inspector.d.ts +47 -0
- package/dist/workflow-inspector.js +235 -0
- package/dist/workflow-session.d.ts +23 -0
- package/dist/workflow-session.js +168 -0
- package/dist/workflow-store.d.ts +36 -0
- package/dist/workflow-store.js +255 -0
- package/dist/workflow-tool.d.ts +81 -0
- package/dist/workflow-tool.js +94 -0
- package/dist/workflow-view.d.ts +64 -0
- package/dist/workflow-view.js +273 -0
- package/dist/workflows.d.ts +1 -0
- package/dist/workflows.js +3 -0
- package/package.json +57 -11
- package/dist/admission.d.ts +0 -6
- package/dist/admission.js +0 -51
- package/dist/author.d.ts +0 -21
- package/dist/author.js +0 -72
- package/dist/benchmark-contract.d.ts +0 -88
- package/dist/benchmark-contract.js +0 -50
- package/dist/benchmark-freeze.d.ts +0 -9
- package/dist/benchmark-freeze.js +0 -9
- package/dist/benchmark.d.ts +0 -127
- package/dist/benchmark.js +0 -260
- package/dist/comparison.d.ts +0 -69
- package/dist/comparison.js +0 -229
- package/dist/skill-bundle.d.ts +0 -5
- package/dist/skill-bundle.js +0 -15
- package/skills/author/SKILL.md +0 -102
- package/skills/author/references/sdk-authoring.md +0 -16
- /package/{skills → dist/skills}/author/examples/evidence-gate.input.json +0 -0
- /package/{skills → dist/skills}/author/examples/evidence-gate.json +0 -0
- /package/{skills → dist/skills}/author/examples/read-file.input.json +0 -0
- /package/{skills → dist/skills}/author/examples/read-file.json +0 -0
- /package/{skills → dist/skills}/author/examples/read-source-decision.input.json +0 -0
- /package/{skills → dist/skills}/author/examples/read-source-decision.json +0 -0
- /package/{skills → dist/skills}/author/examples/release-note.txt +0 -0
- /package/{skills → dist/skills}/author/examples/review-release-notes.input.json +0 -0
- /package/{skills → dist/skills}/author/examples/review-release-notes.json +0 -0
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
// A run's store as durable recording lines (agentrun.recording/1, section 3.1). Three readers share one mapper, so a
|
|
2
|
+
// commit gives the same lines whichever reads it:
|
|
3
|
+
// - recordCommits, inside the run's process, maps each commit pi publishes as it is adopted (the full history,
|
|
4
|
+
// tasks at their real commit);
|
|
5
|
+
// - snapshotLines writes every document at one commit, for a process to record after it opens the run;
|
|
6
|
+
// - recordingLines reads a store file no process holds: the history pi kept (it deletes a current-only document's
|
|
7
|
+
// older revisions when it writes a new base, and the driver document checkpoints every 32 deltas).
|
|
8
|
+
import { apply } from "@earendil-works/chord/delta";
|
|
9
|
+
import { frozenEffectPath } from "@parcha/agentrun-dsl/recovery";
|
|
10
|
+
import { withDurableFile } from "./readers.js";
|
|
11
|
+
const PREFIXES = [["agentrun.", "workflow"], ["browser.", "browser"], ["vision.", "vision"], ["transport.", "transport"], ["pi.", "pi"]];
|
|
12
|
+
const sourceOf = (kind) => PREFIXES.find(([prefix]) => kind.startsWith(prefix))?.[1] ?? "host";
|
|
13
|
+
/** The path a node attempt's session key names: `<driver>:<label>:<frame>:<path>:a<n>` (`frozenStepSessionId`). A label
|
|
14
|
+
* or a path may hold the frame marker too, and then the key names no single path: only a key with one marker has one. */
|
|
15
|
+
function nodePath(key) {
|
|
16
|
+
const frames = [...key.matchAll(/:(?:step|adapt):\//g)];
|
|
17
|
+
const attempt = /:a\d+$/.exec(key);
|
|
18
|
+
if (frames.length !== 1 || !attempt)
|
|
19
|
+
return undefined;
|
|
20
|
+
return key.slice(frames[0].index + frames[0][0].length - 1, attempt.index);
|
|
21
|
+
}
|
|
22
|
+
/** The execution path a document names: an effect's id is `<frame>:<path>#call:<n>` after the journal key and NUL
|
|
23
|
+
* (`frozenEffectId`, read by the DSL's own `frozenEffectPath`), a node attempt's key is `nodePath`'s, and any
|
|
24
|
+
* other value may carry `executionPath` or `execution_path`. */
|
|
25
|
+
function pathOf(kind, key, value) {
|
|
26
|
+
if (kind === "agentrun.effects" && key !== undefined)
|
|
27
|
+
return frozenEffectPath(key.slice(key.indexOf("\u0000") + 1))?.path;
|
|
28
|
+
if (kind === "agentrun.nodes" && key !== undefined)
|
|
29
|
+
return nodePath(key);
|
|
30
|
+
const named = value?.executionPath ?? value?.execution_path;
|
|
31
|
+
return typeof named === "string" ? named : undefined;
|
|
32
|
+
}
|
|
33
|
+
/** A node attempt's conversation and path, from an `agentrun.nodes` document: its value names the conversation the
|
|
34
|
+
* attempt runs in, its key the step's path. */
|
|
35
|
+
function nodeConversation(key, value) {
|
|
36
|
+
const path = key === undefined ? undefined : nodePath(key);
|
|
37
|
+
return typeof value?.conversation === "number" && path !== undefined ? [value.conversation, path] : undefined;
|
|
38
|
+
}
|
|
39
|
+
/** A line in a node's conversation (an entry, a conversation's document, a task) takes the node's path when it names
|
|
40
|
+
* none of its own, so a branch's view holds its agent's turns. */
|
|
41
|
+
function stampPaths(placed, conversations) {
|
|
42
|
+
for (const at of placed) {
|
|
43
|
+
const path = at.line.conv === undefined || at.line.path !== undefined ? undefined : conversations.get(at.line.conv);
|
|
44
|
+
if (path !== undefined)
|
|
45
|
+
at.line = { ...at.line, path };
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
/** A document revision: `key` is the family key (absent for a singleton), `conv` the conversation the document
|
|
49
|
+
* belongs to (its scope's, or its task's). */
|
|
50
|
+
function documentLine(id, kind, key, value, version, conv, commit) {
|
|
51
|
+
const path = pathOf(kind, key, value);
|
|
52
|
+
return { commit, rank: 0, id, line: {
|
|
53
|
+
channel: "durable", source: sourceOf(kind), kind, commit, ...(path !== undefined ? { path } : {}), ...(conv !== undefined ? { conv } : {}),
|
|
54
|
+
data: { doc: kind, ...(key !== undefined ? { key } : {}), value, version },
|
|
55
|
+
} };
|
|
56
|
+
}
|
|
57
|
+
const entryLine = (entry, commit) => ({ commit, rank: 1, id: Number(entry.id), line: {
|
|
58
|
+
channel: "durable", source: "pi", kind: String(entry?.kind ?? "pi.entry"), commit, conv: Number(entry.conversationId), data: entry,
|
|
59
|
+
} });
|
|
60
|
+
const taskLine = (task, commit) => {
|
|
61
|
+
const kind = String(task.kind);
|
|
62
|
+
return { commit, rank: 2, id: Number(task.id), line: { channel: "durable", source: sourceOf(kind), kind, commit, conv: Number(task.conversationId), data: task } };
|
|
63
|
+
};
|
|
64
|
+
/** Lines in `(commit, order)` order, each with the generation `genAt` gives its commit. */
|
|
65
|
+
function numbered(placed, genAt, extra = {}) {
|
|
66
|
+
placed.sort((a, b) => a.commit - b.commit || a.rank - b.rank || a.id - b.id);
|
|
67
|
+
const lines = [];
|
|
68
|
+
for (let i = 0; i < placed.length; i += 1) {
|
|
69
|
+
const { commit, line } = placed[i];
|
|
70
|
+
const order = i > 0 && placed[i - 1].commit === commit ? lines[i - 1].order + 1 : 0;
|
|
71
|
+
const gen = genAt(commit);
|
|
72
|
+
const { data, ...rest } = line;
|
|
73
|
+
lines.push({ ...rest, ...(gen !== undefined ? { gen } : {}), order, ...extra, data });
|
|
74
|
+
}
|
|
75
|
+
return lines;
|
|
76
|
+
}
|
|
77
|
+
/** Each node attempt's conversation and path, from every `agentrun.nodes` document's latest value. */
|
|
78
|
+
function nodeConversations(db) {
|
|
79
|
+
const out = new Map();
|
|
80
|
+
for (const doc of db.prepare("SELECT id, key_value FROM documents WHERE kind = ? AND family = 1").all(JSON.stringify("agentrun.nodes"))) {
|
|
81
|
+
let value;
|
|
82
|
+
for (const rev of db.prepare("SELECT kind, content FROM document_revisions WHERE document_id = ? ORDER BY seq").all(doc.id)) {
|
|
83
|
+
value = rev.kind === "base" ? JSON.parse(rev.content) : apply(structuredClone(value), JSON.parse(rev.content));
|
|
84
|
+
}
|
|
85
|
+
const found = nodeConversation(JSON.parse(doc.key_value), value);
|
|
86
|
+
if (found)
|
|
87
|
+
out.set(found[0], found[1]);
|
|
88
|
+
}
|
|
89
|
+
return out;
|
|
90
|
+
}
|
|
91
|
+
/** Every surviving `agentrun.driver` revision as `[commit, generation]`, in commit order. */
|
|
92
|
+
function driverGenerations(db) {
|
|
93
|
+
const out = [];
|
|
94
|
+
for (const { id } of db.prepare("SELECT id FROM documents WHERE kind = ?").all(JSON.stringify("agentrun.driver"))) {
|
|
95
|
+
let value;
|
|
96
|
+
for (const rev of db.prepare("SELECT seq, kind, content FROM document_revisions WHERE document_id = ? ORDER BY seq").all(id)) {
|
|
97
|
+
value = rev.kind === "base" ? JSON.parse(rev.content) : apply(structuredClone(value), JSON.parse(rev.content));
|
|
98
|
+
if (typeof value?.generation === "number")
|
|
99
|
+
out.push([rev.seq, value.generation]);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
// Two journals' revisions in one commit: the higher generation.
|
|
103
|
+
return out.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Every change the store `file` committed after `after` (default: all), as durable recording lines in `(commit, order)`
|
|
107
|
+
* order, and `last`, the highest commit returned (`after` when nothing is). For a store no process holds (a disk run's
|
|
108
|
+
* store admits one connection): read-only, in one read transaction.
|
|
109
|
+
*
|
|
110
|
+
* - A document revision is a line of the document's kind, with the whole value after it (each delta applied over its
|
|
111
|
+
* base); `key` is the family key as stored, absent for a singleton.
|
|
112
|
+
* - A conversation entry is a line of the entry's own kind, source `pi`, `conv` its conversation, the entry as stored
|
|
113
|
+
* (image blocks included: the recorder moves them to blobs).
|
|
114
|
+
* - Every task's current record is a line of the task's kind, at the store's last commit, after that commit's other
|
|
115
|
+
* lines, by task id: the state at export, since the store keeps no commit for a task's change. Its memos ride in the
|
|
116
|
+
* record; there are no memo lines.
|
|
117
|
+
* - `gen` is the generation of the latest surviving `agentrun.driver` revision at or before the line's commit (it
|
|
118
|
+
* counts journal opens, not processes), and absent before the earliest surviving one. No line has `t`: the store
|
|
119
|
+
* keeps no commit time.
|
|
120
|
+
*
|
|
121
|
+
* This is the store's surviving history: pi deletes a current-only document's older revisions when it writes a new
|
|
122
|
+
* base. The full history is `recordCommits`', written inside the run's process as each commit lands.
|
|
123
|
+
*/
|
|
124
|
+
export function recordingLines(file, options = {}) {
|
|
125
|
+
const after = options.after ?? 0;
|
|
126
|
+
return withDurableFile(file, (db) => {
|
|
127
|
+
const placed = [];
|
|
128
|
+
const taskConversation = new Map(db.prepare("SELECT id, conversation_id FROM tasks").all().map((task) => [task.id, task.conversation_id]));
|
|
129
|
+
const docs = db.prepare("SELECT DISTINCT d.id, d.kind, d.family, d.key_value, d.scope_kind, d.owner_id FROM documents d JOIN document_revisions r ON r.document_id = d.id WHERE r.seq > ?")
|
|
130
|
+
.all(after);
|
|
131
|
+
for (const doc of docs) {
|
|
132
|
+
const kind = JSON.parse(doc.kind);
|
|
133
|
+
const key = doc.family === 1 ? JSON.parse(doc.key_value) : undefined;
|
|
134
|
+
const conv = doc.scope_kind === "conversation" ? doc.owner_id : doc.scope_kind === "task" ? taskConversation.get(doc.owner_id) : undefined;
|
|
135
|
+
let value;
|
|
136
|
+
for (const rev of db.prepare("SELECT seq, kind, version, content FROM document_revisions WHERE document_id = ? ORDER BY seq").all(doc.id)) {
|
|
137
|
+
value = rev.kind === "base" ? JSON.parse(rev.content) : apply(structuredClone(value), JSON.parse(rev.content));
|
|
138
|
+
if (rev.seq > after)
|
|
139
|
+
placed.push(documentLine(doc.id, kind, key, structuredClone(value), rev.version, conv, rev.seq));
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
for (const entry of db.prepare("SELECT commit_seq, record FROM entries WHERE commit_seq > ?").all(after)) {
|
|
143
|
+
placed.push(entryLine(JSON.parse(entry.record), entry.commit_seq));
|
|
144
|
+
}
|
|
145
|
+
const end = db.prepare("SELECT next_seq FROM durable_metadata WHERE singleton = 1").get().next_seq - 1;
|
|
146
|
+
if (end > after)
|
|
147
|
+
for (const task of db.prepare("SELECT record FROM tasks").all())
|
|
148
|
+
placed.push(taskLine(JSON.parse(task.record), end));
|
|
149
|
+
stampPaths(placed, nodeConversations(db));
|
|
150
|
+
const generations = driverGenerations(db);
|
|
151
|
+
let at = 0;
|
|
152
|
+
const lines = numbered(placed, (commit) => {
|
|
153
|
+
while (at < generations.length && generations[at][0] <= commit)
|
|
154
|
+
at += 1;
|
|
155
|
+
return at > 0 ? generations[at - 1][1] : undefined;
|
|
156
|
+
});
|
|
157
|
+
return { lines, last: lines.length ? lines[lines.length - 1].commit : after };
|
|
158
|
+
}) ?? { lines: [], last: after };
|
|
159
|
+
}
|
|
160
|
+
/** The lines of one commit pi published, as `recordingLines` would give them. A commit that writes driver revisions
|
|
161
|
+
* sets the generation to the highest among them, as the file read does; any other commit keeps it. */
|
|
162
|
+
function publicationLines(publication, state) {
|
|
163
|
+
const commit = Number(publication.seq);
|
|
164
|
+
const placed = [];
|
|
165
|
+
let written;
|
|
166
|
+
for (const change of publication.changes) {
|
|
167
|
+
if (change.type === "document") {
|
|
168
|
+
// A retirement adopts no value and the store keeps no revision for it.
|
|
169
|
+
if (change.value === null || change.version === undefined)
|
|
170
|
+
continue;
|
|
171
|
+
const { record } = change;
|
|
172
|
+
placed.push(documentLine(Number(record.id), record.kind, record.key, structuredClone(change.value), change.version, change.conversationId === undefined ? undefined : Number(change.conversationId), commit));
|
|
173
|
+
const generation = change.value.generation;
|
|
174
|
+
if (record.kind === "agentrun.driver" && typeof generation === "number")
|
|
175
|
+
written = Math.max(written ?? generation, generation);
|
|
176
|
+
const node = record.kind === "agentrun.nodes" ? nodeConversation(record.key, change.value) : undefined;
|
|
177
|
+
if (node)
|
|
178
|
+
state.conversations.set(node[0], node[1]);
|
|
179
|
+
}
|
|
180
|
+
else if (change.type === "entry")
|
|
181
|
+
placed.push(entryLine(structuredClone(change.value), commit));
|
|
182
|
+
else if (change.type === "task")
|
|
183
|
+
placed.push(taskLine(structuredClone(change.value), commit));
|
|
184
|
+
// A conversation's or a submission's record, and a document copy (it carries no value), make no line.
|
|
185
|
+
}
|
|
186
|
+
if (written !== undefined)
|
|
187
|
+
state.gen = written;
|
|
188
|
+
stampPaths(placed, state.conversations);
|
|
189
|
+
return numbered(placed, () => state.gen);
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Records the run's commits as durable lines inside its process, on pi's commit publications: each commit adopted
|
|
193
|
+
* after this call becomes the lines `recordingLines` would give it, with tasks at their real commit, `t` the adoption
|
|
194
|
+
* time (`clock()`, default `Date.now()`), and `gen` the generation of the latest `agentrun.driver` revision at or
|
|
195
|
+
* before the commit. `from` is the snapshot this recording continues (`snapshotLines`, written before the run
|
|
196
|
+
* resumed): its driver generation and its nodes' conversations carry on until this process writes its own. `gen` is
|
|
197
|
+
* only for a run with no driver revision yet; it never overrides a driver generation.
|
|
198
|
+
*
|
|
199
|
+
* The listener only maps and queues: it never throws, blocks or calls the Session. `write` receives one commit's lines
|
|
200
|
+
* at a time, in commit order, after the listener has returned. A `write` that completes synchronously (an append to a
|
|
201
|
+
* file) runs before pi adopts the next commit, so a process that dies loses at most the commit in flight; one that
|
|
202
|
+
* completes later leaves the commits after it queued in memory, and a death loses those too. The next process's
|
|
203
|
+
* `snapshotLines` restores the documents' state; a lost commit's entries stay missing, a gap the viewer shows. The
|
|
204
|
+
* returned stop ends the subscription, waits for every queued write, and then throws the first write that failed.
|
|
205
|
+
*/
|
|
206
|
+
export function recordCommits(session, options) {
|
|
207
|
+
const clock = options.clock ?? Date.now;
|
|
208
|
+
const fromDriver = (options.from ?? []).reduce((max, line) => {
|
|
209
|
+
const generation = line.kind === "agentrun.driver" ? line.data?.value?.generation : undefined;
|
|
210
|
+
return typeof generation === "number" ? Math.max(max ?? generation, generation) : max;
|
|
211
|
+
}, undefined);
|
|
212
|
+
const state = { gen: fromDriver ?? options.from?.at(-1)?.gen ?? options.gen, conversations: new Map() };
|
|
213
|
+
for (const line of options.from ?? []) {
|
|
214
|
+
const node = line.kind === "agentrun.nodes" ? nodeConversation(line.data?.key, line.data?.value) : undefined;
|
|
215
|
+
if (node)
|
|
216
|
+
state.conversations.set(node[0], node[1]);
|
|
217
|
+
}
|
|
218
|
+
let chain = Promise.resolve();
|
|
219
|
+
let failure;
|
|
220
|
+
const unsubscribe = session.subscribeCommits((publication) => {
|
|
221
|
+
try {
|
|
222
|
+
const t = clock();
|
|
223
|
+
const mapped = publicationLines(publication, state);
|
|
224
|
+
if (mapped.length === 0)
|
|
225
|
+
return;
|
|
226
|
+
const lines = mapped.map((line) => ({ ...line, t }));
|
|
227
|
+
chain = chain.then(() => options.write(lines)).catch((error) => { failure ??= { error }; });
|
|
228
|
+
}
|
|
229
|
+
catch (error) {
|
|
230
|
+
failure ??= { error };
|
|
231
|
+
}
|
|
232
|
+
});
|
|
233
|
+
return async () => {
|
|
234
|
+
unsubscribe();
|
|
235
|
+
await chain;
|
|
236
|
+
if (failure)
|
|
237
|
+
throw failure.error;
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
/** Every page of a storage scan. */
|
|
241
|
+
async function scanAll(scan) {
|
|
242
|
+
const out = [];
|
|
243
|
+
let cursor;
|
|
244
|
+
do {
|
|
245
|
+
const page = await scan(cursor);
|
|
246
|
+
out.push(...page.items);
|
|
247
|
+
cursor = page.next;
|
|
248
|
+
} while (cursor !== undefined);
|
|
249
|
+
return out;
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* Every document of the run as it stands, as whole values marked `snapshot: true` at commit `at`, in document id order:
|
|
253
|
+
* the state a process records after it opens the run and before it resumes, so the recording is exact after any kill.
|
|
254
|
+
* `storage` is the store the run's Harness was opened over; `at` is the store's latest commit (pi's `Storage` has no
|
|
255
|
+
* head query: a disk run reads it with pi-durable-disk's `storeHead`). The reads are of the current state (pi keeps no
|
|
256
|
+
* past revision of a session or task document), so the snapshot is exact only while nothing commits: take it before
|
|
257
|
+
* the run resumes. Every line's `gen` is the highest `agentrun.driver` generation in it; `gen` is only for a run with no
|
|
258
|
+
* driver revision yet, and never overrides one.
|
|
259
|
+
*/
|
|
260
|
+
export async function snapshotLines(storage, options) {
|
|
261
|
+
const { at, context } = options;
|
|
262
|
+
const page = 256;
|
|
263
|
+
const conversations = await scanAll((cursor) => storage.scanConversations({}, page, cursor, context));
|
|
264
|
+
const tasks = await scanAll((cursor) => storage.scanTasks({}, page, cursor, context));
|
|
265
|
+
const scopes = [
|
|
266
|
+
{ scope: { kind: "session" }, conv: undefined },
|
|
267
|
+
...conversations.map((conversation) => ({ scope: { kind: "conversation", conversationId: conversation.id }, conv: Number(conversation.id) })),
|
|
268
|
+
...tasks.map((task) => ({ scope: { kind: "task", taskId: task.id }, conv: Number(task.conversationId) })),
|
|
269
|
+
];
|
|
270
|
+
const placed = [];
|
|
271
|
+
let driverGen;
|
|
272
|
+
for (const { scope, conv } of scopes) {
|
|
273
|
+
for (const record of await scanAll((cursor) => storage.scanDocuments({ scope, at: "current" }, page, cursor, context))) {
|
|
274
|
+
const stored = await storage.document(record.id, "current", context);
|
|
275
|
+
if (!stored)
|
|
276
|
+
continue;
|
|
277
|
+
placed.push(documentLine(Number(record.id), record.kind, record.key, structuredClone(stored.value), stored.version, conv, at));
|
|
278
|
+
const generation = stored.value.generation;
|
|
279
|
+
if (record.kind === "agentrun.driver" && typeof generation === "number")
|
|
280
|
+
driverGen = Math.max(driverGen ?? generation, generation);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
const nodes = new Map();
|
|
284
|
+
for (const { line } of placed) {
|
|
285
|
+
const node = line.kind === "agentrun.nodes" ? nodeConversation(line.data.key, line.data.value) : undefined;
|
|
286
|
+
if (node)
|
|
287
|
+
nodes.set(node[0], node[1]);
|
|
288
|
+
}
|
|
289
|
+
stampPaths(placed, nodes);
|
|
290
|
+
const gen = driverGen ?? options.gen;
|
|
291
|
+
return numbered(placed, () => gen, { snapshot: true });
|
|
292
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { Harness } from "@earendil-works/pi-durable";
|
|
2
|
+
import type { Context } from "@earendil-works/chord";
|
|
3
|
+
import { type RecoveryStore } from "@parcha/agentrun-dsl/recovery";
|
|
4
|
+
/** An effect's key in `agentrun.effects`: its journal key, then its id. The two are joined by NUL, so neither may hold
|
|
5
|
+
* one: two pairs could otherwise name the same document. */
|
|
6
|
+
export declare const effectKey: (key: string, id: string) => string;
|
|
7
|
+
/** What a store writes through: a Harness, or the `commit` a running task is handed. */
|
|
8
|
+
export type DocumentStoreHost = Pick<Harness, "commit">;
|
|
9
|
+
export type DocumentStoreOptions = {
|
|
10
|
+
/** The one owner of the journal across the invocations of a task (its id). An open by this owner takes over a journal
|
|
11
|
+
* an earlier invocation left open, whose later writes then fail; an open by another owner is refused while the
|
|
12
|
+
* journal is open. The guard is per process, by journal key: an owner is unique to its run, and two Harness files in
|
|
13
|
+
* one process that hold one run key under one owner count as one. Without an owner the guard is per host object,
|
|
14
|
+
* and a second open of a key is refused until the first is closed. */
|
|
15
|
+
owner?: string;
|
|
16
|
+
/** The context every commit runs under; default `BACKGROUND_CONTEXT`. A task passes its invocation's, so the
|
|
17
|
+
* invocation's cancellation ends its writes. */
|
|
18
|
+
context?: Context;
|
|
19
|
+
};
|
|
20
|
+
/** The recovery store for the journal `key` that `host` writes. Opening it takes ownership: a journal already open is
|
|
21
|
+
* refused (unless the same `owner` opens it again), a binding that differs from the one the journal was first opened
|
|
22
|
+
* under is refused naming the inputs that moved, and the generation advances. */
|
|
23
|
+
export declare function documentStore(host: DocumentStoreHost, key: string, options?: DocumentStoreOptions): RecoveryStore;
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
import { defineDocFamily } from "@earendil-works/pi-durable";
|
|
2
|
+
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
|
|
3
|
+
import { RecoveryError, asRecoveryError, bindingMismatch } from "@parcha/agentrun-dsl/recovery";
|
|
4
|
+
import { storageError } from "./pi-versions.js";
|
|
5
|
+
/** A failed commit as the driver reads it: its own message and code, kept as a RecoveryError. A Session that failed at a
|
|
6
|
+
* storage error answers with `SessionFailed` (pi-durable 1.2); the error is then the storage's, as 1.1 gave it, and the
|
|
7
|
+
* `SessionFailed` is its cause. */
|
|
8
|
+
function storeFailure(error) {
|
|
9
|
+
const own = storageError(error);
|
|
10
|
+
if (own === error || own instanceof RecoveryError)
|
|
11
|
+
return asRecoveryError(own);
|
|
12
|
+
const code = own?.code;
|
|
13
|
+
return new RecoveryError(own instanceof Error ? own.message : String(own), typeof code === "string" ? code : undefined, { cause: error });
|
|
14
|
+
}
|
|
15
|
+
/** An effect's key in `agentrun.effects`: its journal key, then its id. The two are joined by NUL, so neither may hold
|
|
16
|
+
* one: two pairs could otherwise name the same document. */
|
|
17
|
+
export const effectKey = (key, id) => {
|
|
18
|
+
if (key.includes("\u0000") || id.includes("\u0000"))
|
|
19
|
+
throw new RecoveryError("A journal key and an effect id hold no NUL character");
|
|
20
|
+
return `${key}\u0000${id}`;
|
|
21
|
+
};
|
|
22
|
+
/** A base every 32 deltas keeps a read of the journal short, however long the run. */
|
|
23
|
+
const checkpointWhen = (_value, _ops, info) => info.deltasSinceBase >= 32;
|
|
24
|
+
const DriverDoc = defineDocFamily({ kind: "agentrun.driver", version: 1, scope: "session", family: true, checkpointWhen,
|
|
25
|
+
initial: () => ({ binding: null, inputs: null, generation: 0, revision: 0, state: null, notes: [], effects: [] }) });
|
|
26
|
+
const EffectDoc = defineDocFamily({ kind: "agentrun.effects", version: 1, scope: "session", family: true,
|
|
27
|
+
initial: () => ({ driver: "", name: "", argsHash: "", status: "unknown", session: null, result: null }) });
|
|
28
|
+
const ItemDoc = defineDocFamily({ kind: "agentrun.items", version: 1, scope: "session", family: true,
|
|
29
|
+
initial: () => ({ driver: "", effects: [] }) });
|
|
30
|
+
/** A state by its digest; `sha256` is empty only for one never written. */
|
|
31
|
+
const StateDoc = defineDocFamily({ kind: "agentrun.states", version: 1, scope: "session", family: true,
|
|
32
|
+
initial: () => ({ driver: "", sha256: "", state: null }) });
|
|
33
|
+
/** A draft made to hold `value` whole: every field set, every other field removed. */
|
|
34
|
+
function replaceWhole(draft, value) {
|
|
35
|
+
for (const field of Object.keys(draft))
|
|
36
|
+
if (!Object.hasOwn(value, field))
|
|
37
|
+
delete draft[field];
|
|
38
|
+
for (const [field, item] of Object.entries(value))
|
|
39
|
+
draft[field] = item;
|
|
40
|
+
}
|
|
41
|
+
/** A value as the journal stores it: plain JSON, every number finite. */
|
|
42
|
+
function json(value) {
|
|
43
|
+
return value === undefined ? null : JSON.parse(JSON.stringify(value, (_key, item) => {
|
|
44
|
+
if (typeof item === "bigint" || typeof item === "function" || typeof item === "symbol" || (typeof item === "number" && !Number.isFinite(item))) {
|
|
45
|
+
throw new RecoveryError("A journal holds finite JSON values only");
|
|
46
|
+
}
|
|
47
|
+
return item;
|
|
48
|
+
}));
|
|
49
|
+
}
|
|
50
|
+
/** The journals open in this process. The run's lock keeps every other process off the run, and the journal's generation
|
|
51
|
+
* fences a stale writer; this keeps a second open of one key off it here. Without an owner a host's own keys are
|
|
52
|
+
* tracked; with one, the keys by owner. */
|
|
53
|
+
const hostKeys = new WeakMap();
|
|
54
|
+
const ownedKeys = new Map();
|
|
55
|
+
/** Take the in-process hold on `key`: what gives it back, and what undoes the take when the open fails. */
|
|
56
|
+
function hold(host, key, owner) {
|
|
57
|
+
const refused = () => new RecoveryError("Run already has a live owner");
|
|
58
|
+
if (owner === undefined) {
|
|
59
|
+
const keys = hostKeys.get(host) ?? hostKeys.set(host, new Set()).get(host);
|
|
60
|
+
if (keys.has(key))
|
|
61
|
+
throw refused();
|
|
62
|
+
keys.add(key);
|
|
63
|
+
const release = () => { keys.delete(key); };
|
|
64
|
+
return { release, undo: release };
|
|
65
|
+
}
|
|
66
|
+
const previous = ownedKeys.get(key);
|
|
67
|
+
if (previous && previous.owner !== owner)
|
|
68
|
+
throw refused();
|
|
69
|
+
const token = Symbol(key);
|
|
70
|
+
ownedKeys.set(key, { token, owner });
|
|
71
|
+
return {
|
|
72
|
+
release: () => { if (ownedKeys.get(key)?.token === token)
|
|
73
|
+
ownedKeys.delete(key); },
|
|
74
|
+
undo: () => { if (ownedKeys.get(key)?.token !== token)
|
|
75
|
+
return; if (previous)
|
|
76
|
+
ownedKeys.set(key, previous);
|
|
77
|
+
else
|
|
78
|
+
ownedKeys.delete(key); },
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
/** The recovery store for the journal `key` that `host` writes. Opening it takes ownership: a journal already open is
|
|
82
|
+
* refused (unless the same `owner` opens it again), a binding that differs from the one the journal was first opened
|
|
83
|
+
* under is refused naming the inputs that moved, and the generation advances. */
|
|
84
|
+
export function documentStore(host, key, options = {}) {
|
|
85
|
+
return { open: (bound) => openDocumentJournal(host, key, bound, options) };
|
|
86
|
+
}
|
|
87
|
+
async function openDocumentJournal(host, key, bound, options) {
|
|
88
|
+
effectKey(key, "");
|
|
89
|
+
const context = options.context ?? BACKGROUND_CONTEXT;
|
|
90
|
+
const held = hold(host, key, options.owner);
|
|
91
|
+
const opened = await host.commit(async (tx) => {
|
|
92
|
+
const doc = await tx.doc(DriverDoc, key, null);
|
|
93
|
+
const existing = doc.binding !== null;
|
|
94
|
+
if (existing && doc.binding !== bound.binding)
|
|
95
|
+
throw bindingMismatch(doc.inputs ? JSON.stringify(doc.inputs) : null, bound.inputs);
|
|
96
|
+
if (!existing) {
|
|
97
|
+
doc.binding = bound.binding;
|
|
98
|
+
doc.inputs = bound.inputs ? { ...bound.inputs } : null;
|
|
99
|
+
}
|
|
100
|
+
doc.generation += 1;
|
|
101
|
+
const effects = [];
|
|
102
|
+
for (const id of doc.effects) {
|
|
103
|
+
const e = await tx.doc(EffectDoc, effectKey(key, id), null);
|
|
104
|
+
effects.push({ id, name: e.name, argsHash: e.argsHash, status: e.status, session: e.session, result: json(e.result), ...(e.intent ? { intent: json(e.intent) } : {}), ...(e.args !== undefined ? { args: json(e.args) } : {}), ...(Array.isArray(e.calls) ? { calls: json(e.calls) } : {}) });
|
|
105
|
+
}
|
|
106
|
+
return { existing, generation: doc.generation, revision: doc.revision, state: json(doc.state), notes: json(doc.notes), effects };
|
|
107
|
+
}, context).catch((error) => { held.undo(); throw storeFailure(error); });
|
|
108
|
+
const effects = new Map(opened.effects.map((effect) => [effect.id, effect]));
|
|
109
|
+
const notes = [...opened.notes];
|
|
110
|
+
let revision = opened.revision;
|
|
111
|
+
let closed = false;
|
|
112
|
+
/** Writes run one at a time, in the order they were asked for, each with its checks. A failed write does not stop
|
|
113
|
+
* the next, and one asked for before `close` still lands. */
|
|
114
|
+
let queue = Promise.resolve();
|
|
115
|
+
const inOrder = (write) => {
|
|
116
|
+
if (closed)
|
|
117
|
+
return Promise.reject(new RecoveryError("The journal is closed"));
|
|
118
|
+
const done = queue.then(write).catch((error) => { throw storeFailure(error); });
|
|
119
|
+
queue = done.catch(() => undefined);
|
|
120
|
+
return done;
|
|
121
|
+
};
|
|
122
|
+
/** One Session commit on the journal's record: it checks this open's generation is the record's, advances the
|
|
123
|
+
* revision, and lets `write` change the record (its state computed with that revision) and the effect documents,
|
|
124
|
+
* then writes the parts computed with that revision. */
|
|
125
|
+
const commit = async (write, parts) => host.commit(async (tx) => {
|
|
126
|
+
const doc = await tx.doc(DriverDoc, key, null);
|
|
127
|
+
if (doc.generation !== opened.generation)
|
|
128
|
+
throw new RecoveryError("Run owner generation is not acquired");
|
|
129
|
+
const next = doc.revision + 1;
|
|
130
|
+
const value = await write(doc, next, tx);
|
|
131
|
+
const laid = parts?.(next);
|
|
132
|
+
for (const [path, item] of Object.entries(laid?.items ?? {})) {
|
|
133
|
+
replaceWhole(await tx.doc(ItemDoc, effectKey(key, path), null), json(item));
|
|
134
|
+
}
|
|
135
|
+
for (const [sha256, state] of Object.entries(laid?.states ?? {})) {
|
|
136
|
+
const held = await tx.doc(StateDoc, effectKey(key, sha256), null);
|
|
137
|
+
if (held.sha256 === "")
|
|
138
|
+
Object.assign(held, { driver: key, sha256, state: json(state) });
|
|
139
|
+
}
|
|
140
|
+
doc.revision = next;
|
|
141
|
+
return { revision: next, value };
|
|
142
|
+
}, context).then((done) => { revision = done.revision; return done; });
|
|
143
|
+
/** A read in a commit of its own: it writes nothing the caller did not ask for. */
|
|
144
|
+
const read = (body) => inOrder(() => host.commit(body, context));
|
|
145
|
+
const storedEffect = (id, e) => ({ id, name: e.name, argsHash: e.argsHash, status: e.status, session: e.session, result: json(e.result), ...(e.intent ? { intent: json(e.intent) } : {}), ...(e.args !== undefined ? { args: json(e.args) } : {}), ...(Array.isArray(e.calls) ? { calls: json(e.calls) } : {}) });
|
|
146
|
+
return {
|
|
147
|
+
existing: opened.existing, generation: opened.generation, state: opened.state,
|
|
148
|
+
get revision() { return revision; },
|
|
149
|
+
effects: () => [...effects.values()],
|
|
150
|
+
effect: (id) => effects.get(id),
|
|
151
|
+
notes: () => [...notes],
|
|
152
|
+
save: (state, note, parts) => inOrder(async () => {
|
|
153
|
+
let entry;
|
|
154
|
+
const done = await commit((doc, next) => {
|
|
155
|
+
doc.state = json(typeof state === "function" ? state(next) : state);
|
|
156
|
+
if (note) {
|
|
157
|
+
entry = { revision: next, kind: note.kind, detail: json(note.detail), at: new Date().toISOString() };
|
|
158
|
+
doc.notes.push(entry);
|
|
159
|
+
}
|
|
160
|
+
}, parts);
|
|
161
|
+
if (entry)
|
|
162
|
+
notes.push(entry);
|
|
163
|
+
return done.revision;
|
|
164
|
+
}),
|
|
165
|
+
note: (kind, detail) => inOrder(async () => {
|
|
166
|
+
const entry = { kind, detail: json(detail), at: new Date().toISOString() };
|
|
167
|
+
const done = await commit((doc, next) => { doc.notes.push({ revision: next, ...entry }); });
|
|
168
|
+
notes.push({ revision: done.revision, ...entry });
|
|
169
|
+
return done.revision;
|
|
170
|
+
}),
|
|
171
|
+
admit: (id, name, argsHash, state, session = null, intent, args, parts, owner) => inOrder(async () => {
|
|
172
|
+
effectKey(key, id);
|
|
173
|
+
const known = effects.get(id);
|
|
174
|
+
if (known)
|
|
175
|
+
return known;
|
|
176
|
+
const stored = json(state);
|
|
177
|
+
const kept = intent === undefined ? undefined : json(intent);
|
|
178
|
+
const keptArgs = args === undefined ? undefined : json(args);
|
|
179
|
+
await commit(async (doc, _next, tx) => {
|
|
180
|
+
Object.assign(await tx.doc(EffectDoc, effectKey(key, id), null), { driver: key, name, argsHash, status: "unknown", session, result: null, ...(kept ? { intent: kept } : {}), ...(keptArgs !== undefined ? { args: keptArgs } : {}) });
|
|
181
|
+
// An effect a map item owns is listed by that item's record, written in the same commit.
|
|
182
|
+
if (owner === undefined || !parts)
|
|
183
|
+
doc.effects.push(id);
|
|
184
|
+
doc.state = stored;
|
|
185
|
+
}, parts);
|
|
186
|
+
effects.set(id, { id, name, argsHash, status: "unknown", session, result: null, ...(kept ? { intent: kept } : {}), ...(keptArgs !== undefined ? { args: keptArgs } : {}) });
|
|
187
|
+
return "new";
|
|
188
|
+
}),
|
|
189
|
+
complete: (id, result, state = opened.state, parts) => inOrder(async () => {
|
|
190
|
+
const known = effects.get(id);
|
|
191
|
+
if (!known)
|
|
192
|
+
throw new RecoveryError(`Effect ${id} completes without an admission`);
|
|
193
|
+
if (known.status === "completed")
|
|
194
|
+
throw new RecoveryError(`Effect ${id} completes once`);
|
|
195
|
+
const value = json(result);
|
|
196
|
+
const next = json(state);
|
|
197
|
+
await commit(async (doc, _next, tx) => {
|
|
198
|
+
const effect = await tx.doc(EffectDoc, effectKey(key, id), null);
|
|
199
|
+
effect.status = "completed";
|
|
200
|
+
effect.result = value;
|
|
201
|
+
doc.state = next;
|
|
202
|
+
}, parts);
|
|
203
|
+
effects.set(id, { ...known, status: "completed", result: value });
|
|
204
|
+
}),
|
|
205
|
+
called: (id, calls) => inOrder(async () => {
|
|
206
|
+
if (!effects.has(id))
|
|
207
|
+
throw new RecoveryError(`Effect ${id} records calls without an admission`);
|
|
208
|
+
const value = json(calls);
|
|
209
|
+
await commit(async (_doc, _next, tx) => { (await tx.doc(EffectDoc, effectKey(key, id), null)).calls = value; });
|
|
210
|
+
effects.set(id, { ...effects.get(id), ...(Array.isArray(value) ? { calls: value } : {}) });
|
|
211
|
+
}),
|
|
212
|
+
async close() {
|
|
213
|
+
if (closed)
|
|
214
|
+
return;
|
|
215
|
+
closed = true;
|
|
216
|
+
await queue;
|
|
217
|
+
held.release();
|
|
218
|
+
},
|
|
219
|
+
// Only paths and digests the driver wrote are asked for: a read of a document never written would create it.
|
|
220
|
+
items: (paths) => read(async (tx) => {
|
|
221
|
+
const found = [];
|
|
222
|
+
for (const path of paths) {
|
|
223
|
+
const item = await tx.doc(ItemDoc, effectKey(key, path), null);
|
|
224
|
+
if (item.driver === "") {
|
|
225
|
+
found.push(undefined);
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
228
|
+
found.push(json(item));
|
|
229
|
+
for (const id of (item.effects ?? []))
|
|
230
|
+
if (!effects.has(id))
|
|
231
|
+
effects.set(id, storedEffect(id, await tx.doc(EffectDoc, effectKey(key, id), null)));
|
|
232
|
+
}
|
|
233
|
+
return found;
|
|
234
|
+
}),
|
|
235
|
+
stateOf: (sha256) => read(async (tx) => {
|
|
236
|
+
const held = await tx.doc(StateDoc, effectKey(key, sha256), null);
|
|
237
|
+
return held.sha256 === "" ? undefined : json(held.state);
|
|
238
|
+
}),
|
|
239
|
+
};
|
|
240
|
+
}
|