@awebai/oats 0.61.0 → 0.63.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +1 -1
  2. package/bin/oats.mjs +27 -12
  3. package/docs/capabilities.md +1 -1
  4. package/docs/desktop-cli-api.md +30 -2
  5. package/docs/execution-targets.md +89 -4
  6. package/docs/implementation.md +2 -1
  7. package/docs/named-test-runs.md +382 -0
  8. package/docs/official-catalog.md +1 -1
  9. package/docs/packages.md +5 -5
  10. package/docs/release-lane.md +13 -0
  11. package/docs/release-notes/v0.62.0.md +223 -0
  12. package/docs/release-notes/v0.63.0.md +135 -0
  13. package/docs/souls-and-instances.md +6 -1
  14. package/docs/upgrading.md +135 -11
  15. package/docs/workspaces.md +1 -1
  16. package/lib/core.mjs +73 -23
  17. package/lib/doctor-capture.mjs +35 -0
  18. package/lib/session-viewer.mjs +86 -7
  19. package/lib/tree-copy.mjs +188 -7
  20. package/package-catalog.json +1 -1
  21. package/package.json +1 -1
  22. package/packages/client/cli-locator.mjs +2 -2
  23. package/packages/record/README.md +87 -13
  24. package/packages/record/bin/capture-hook.mjs +8 -0
  25. package/packages/record/bin/capture.mjs +200 -139
  26. package/packages/record/bin/recall.mjs +8 -3
  27. package/packages/record/bin/setup.mjs +116 -225
  28. package/packages/record/bin/turn-record.mjs +2 -2
  29. package/packages/record/docs/capture-service.md +1527 -28
  30. package/packages/record/docs/capture-status.schema.json +429 -12
  31. package/packages/record/lib/capture-admission.mjs +82 -0
  32. package/packages/record/lib/capture-aw-unit.mjs +137 -0
  33. package/packages/record/lib/capture-aw.mjs +31 -12
  34. package/packages/record/lib/capture-budget.mjs +31 -0
  35. package/packages/record/lib/capture-cc.mjs +39 -6
  36. package/packages/record/lib/capture-cleanup-queue.mjs +135 -0
  37. package/packages/record/lib/capture-cleanup-work.mjs +77 -0
  38. package/packages/record/lib/capture-command-operation.mjs +37 -0
  39. package/packages/record/lib/capture-command-output.mjs +62 -0
  40. package/packages/record/lib/capture-command-worker.mjs +6 -0
  41. package/packages/record/lib/capture-command.mjs +65 -0
  42. package/packages/record/lib/capture-completion.mjs +152 -0
  43. package/packages/record/lib/capture-context.mjs +50 -0
  44. package/packages/record/lib/capture-default-roots.mjs +30 -0
  45. package/packages/record/lib/capture-discovery.mjs +114 -0
  46. package/packages/record/lib/capture-failures.mjs +45 -0
  47. package/packages/record/lib/capture-health-doctor.mjs +13 -0
  48. package/packages/record/lib/capture-health-installation.mjs +117 -0
  49. package/packages/record/lib/capture-health.mjs +116 -0
  50. package/packages/record/lib/capture-hook-ownership.mjs +58 -0
  51. package/packages/record/lib/capture-hook-runner.mjs +17 -0
  52. package/packages/record/lib/capture-hook.mjs +26 -0
  53. package/packages/record/lib/capture-index-inventory.mjs +205 -0
  54. package/packages/record/lib/capture-index-outcomes.mjs +113 -0
  55. package/packages/record/lib/capture-index-reset.mjs +110 -0
  56. package/packages/record/lib/capture-index-work.mjs +77 -0
  57. package/packages/record/lib/capture-issue-preparation.mjs +73 -0
  58. package/packages/record/lib/capture-issues.mjs +15 -2
  59. package/packages/record/lib/capture-lock.mjs +137 -17
  60. package/packages/record/lib/capture-native-unit.mjs +322 -0
  61. package/packages/record/lib/capture-pass.mjs +116 -0
  62. package/packages/record/lib/capture-recovery-snapshot.mjs +182 -0
  63. package/packages/record/lib/capture-retained-scratch.mjs +213 -0
  64. package/packages/record/lib/capture-retry.mjs +102 -0
  65. package/packages/record/lib/capture-run.mjs +125 -0
  66. package/packages/record/lib/capture-scratch.mjs +12 -3
  67. package/packages/record/lib/capture-service-definition.mjs +104 -0
  68. package/packages/record/lib/capture-service-install.mjs +362 -0
  69. package/packages/record/lib/capture-service-ownership.mjs +137 -0
  70. package/packages/record/lib/capture-service-runner.mjs +18 -0
  71. package/packages/record/lib/capture-small-snapshot.mjs +85 -0
  72. package/packages/record/lib/capture-source-discovery.mjs +319 -0
  73. package/packages/record/lib/capture-source-queue.mjs +199 -0
  74. package/packages/record/lib/capture-source-work.mjs +85 -0
  75. package/packages/record/lib/capture-state.mjs +16 -71
  76. package/packages/record/lib/capture-status.mjs +171 -0
  77. package/packages/record/lib/capture-supervisor.mjs +136 -0
  78. package/packages/record/lib/capture-watch.mjs +32 -0
  79. package/packages/record/lib/capture-work-session.mjs +241 -0
  80. package/packages/record/lib/capture-worker.mjs +74 -0
  81. package/packages/record/lib/derived-state.mjs +58 -17
  82. package/packages/record/lib/file-lines.mjs +22 -6
  83. package/packages/record/lib/formats.mjs +4 -3
  84. package/packages/record/lib/index-capture.mjs +97 -0
  85. package/packages/record/lib/index-db.mjs +121 -66
  86. package/packages/record/lib/index-reset.mjs +19 -0
  87. package/packages/record/lib/store.mjs +46 -14
  88. package/packages/record/lib/transcript-prefix.mjs +125 -0
  89. package/skills/oats-getting-started/SKILL.md +1 -1
  90. package/docs/macos-test-runs.md +0 -246
@@ -0,0 +1,58 @@
1
+ // Shared setup/health recognition. This is evidence only, never permission to
2
+ // execute a shell command or widen setup ownership.
3
+ const SHELL_PLAIN = /^[A-Za-z0-9_./:@%+,-]+$/;
4
+
5
+ // The words of a command as /bin/sh reads them, for what setup writes: plain
6
+ // characters, single-quoted runs and \' for a quote, adjacent pieces joined
7
+ // into one word, words apart by spaces or tabs. null when the command holds
8
+ // anything else outside quotes (an expansion, an operator, a redirection, a
9
+ // double quote, a newline, which ends a command) or a quote left open: the
10
+ // shell would do more with it than run those words, so it is not read.
11
+ //
12
+ // Any character above U+007F is part of a word too, as an earlier setup
13
+ // wrote it bare in a path (shellWord now quotes it). bash, dash and zsh end
14
+ // a word only at a space, a tab, a newline or an operator, all ASCII, so a
15
+ // command of such words carries nothing a rewrite would lose. POSIX lets a
16
+ // shell end a word at any blank of its locale (U+3000, say): such a shell
17
+ // would see one more argument of the same command, which a rewrite does not
18
+ // lose either, so no character above U+007F is refused.
19
+ export function shellWords(command) {
20
+ const words = [];
21
+ let word = null;
22
+ for (let i = 0; i < command.length; ) {
23
+ const c = command[i];
24
+ if (c === " " || c === "\t") {
25
+ if (word !== null) words.push(word);
26
+ word = null;
27
+ i++;
28
+ } else if (c === "'") {
29
+ const end = command.indexOf("'", i + 1);
30
+ if (end < 0) return null;
31
+ word = (word ?? "") + command.slice(i + 1, end);
32
+ i = end + 1;
33
+ } else if (c === "\\" && command[i + 1] === "'") {
34
+ word = (word ?? "") + "'";
35
+ i += 2;
36
+ } else if (SHELL_PLAIN.test(c) || c > "\x7F") {
37
+ word = (word ?? "") + c;
38
+ i++;
39
+ } else return null;
40
+ }
41
+ if (word !== null) words.push(word);
42
+ return words;
43
+ }
44
+
45
+ // A capture hook setup wrote, of any release: its script is the second word,
46
+ // capture.mjs (earlier releases) or the hook entry capture-hook.mjs.
47
+ export function isCaptureHook(hook) {
48
+ if (hook?.type !== "command") return false;
49
+ const tokens = shellWords(String(hook.command));
50
+ return tokens !== null && tokens.length >= 2 && (tokens[1].endsWith("/capture.mjs") || tokens[1].endsWith("/capture-hook.mjs")) &&
51
+ tokens.includes("--owner") && tokens.includes("--quiet");
52
+ }
53
+
54
+ // The form this release's setup writes: the hook entry, which an older
55
+ // release cannot load. Any other capture hook needs setup.
56
+ export function isCurrentCaptureHook(hook) {
57
+ return isCaptureHook(hook) && shellWords(String(hook.command))[1].endsWith("/capture-hook.mjs");
58
+ }
@@ -0,0 +1,17 @@
1
+ // Fixed detached supervisor entry. The short-lived hook launcher owns no lock
2
+ // or worker; this process retains both until the ordinary pass transport ends.
3
+ import { captureCommandOptions, runCaptureCommand } from "./capture-command.mjs";
4
+
5
+ try {
6
+ if (process.argv.length !== 3) throw new Error("invalid capture hook request");
7
+ const request = JSON.parse(process.argv[2]);
8
+ if (!request || Object.keys(request).some(key => !["root", "options"].includes(key))) throw new Error("invalid capture hook request");
9
+ const options = captureCommandOptions(request.options);
10
+ if (options.trigger !== "hook" || options.retrySetAside) throw new Error("invalid capture hook intent");
11
+ const result = await runCaptureCommand(request.root, options, { entry: "hook-runner" });
12
+ process.exitCode = result.exitCode;
13
+ } catch {
14
+ // stdout/stderr are deliberately detached from the harness. Only admitted
15
+ // workers write status; startup failure must not invent a receipt here.
16
+ process.exitCode = 1;
17
+ }
@@ -0,0 +1,26 @@
1
+ // Explicit hook launch, not capture completion. No inherited harness pipe,
2
+ // shell, preload or caller-selected worker survives into the detached owner.
3
+ import { spawn } from "node:child_process";
4
+ import { resolve } from "node:path";
5
+ import { fileURLToPath } from "node:url";
6
+ import { captureCommandOptions } from "./capture-command.mjs";
7
+ import { captureWorkerEnvironment } from "./capture-supervisor.mjs";
8
+
9
+ export async function launchCaptureHook(root, options, { spawnChild = spawn, env = process.env } = {}) {
10
+ if (typeof root !== "string" || !root || root.includes("\0")) throw new Error("invalid capture hook root");
11
+ const selected = captureCommandOptions(options);
12
+ if (selected.trigger !== "hook" || selected.retrySetAside) throw new Error("invalid capture hook intent");
13
+ const request = { root: resolve(selected.plan.cwd, root), options: selected };
14
+ const child = spawnChild(process.execPath, ["--no-warnings",
15
+ ...(Number(process.versions.node.split(".")[0]) === 22 && Number(process.versions.node.split(".")[1]) < 13 ? ["--experimental-sqlite"] : []),
16
+ fileURLToPath(new URL("./capture-hook-runner.mjs", import.meta.url)), JSON.stringify(request)],
17
+ { cwd: selected.plan.cwd, env: captureWorkerEnvironment(env), detached: true, stdio: "ignore", shell: false });
18
+ await new Promise((done, fail) => {
19
+ child.once("error", fail);
20
+ child.once("spawn", done);
21
+ });
22
+ child.unref();
23
+ // The OS launched this fixed supervisor. Its module startup, lock admission,
24
+ // worker completion and later failure are deliberately NOT acknowledged here.
25
+ return { launched: true };
26
+ }
@@ -0,0 +1,205 @@
1
+ // Private root-lock-owned index inventory. Discovery includes old index rows
2
+ // so a deleted journal can still invalidate cross-stream tombstone authority.
3
+ // Inventory membership/positions confer no journal or index-cursor authority.
4
+ import { readdirSync, statSync } from "node:fs";
5
+ import { validStreamId } from "./store.mjs";
6
+
7
+ const pageSize = 128;
8
+ const identity = stat => ({ dev: String(stat.dev), ino: String(stat.ino) });
9
+ const sameIdentity = (a, b) => a?.dev === b?.dev && a?.ino === b?.ino;
10
+ const validIdentity = value => value === null || value &&
11
+ typeof value.dev === "string" && typeof value.ino === "string";
12
+
13
+ export class CaptureIndexInventory {
14
+ // The scheduler persists the sweep identity and replaces it after a finished
15
+ // sweep or union reset. Reopening with the same identity resumes discovery.
16
+ constructor(state, { id }) {
17
+ if (state.depth || typeof id !== "string" || !id) throw new Error("invalid capture index inventory identity");
18
+ this.state = state; this.id = id; this.directory = null;
19
+ // Streams whose index evidence this worker verified complete in its pass,
20
+ // and whether its last full walk found every row verified. Writes drop them.
21
+ this.verified = new Set(); this.clean = false;
22
+ const present = state.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name='capture_index_inventory'").get()?.found;
23
+ if (!present) state.transaction(() => {
24
+ state.db.exec(`CREATE TABLE IF NOT EXISTS capture_index_inventory (
25
+ scope TEXT NOT NULL, stream TEXT NOT NULL, phase TEXT NOT NULL DEFAULT 'ready',
26
+ proof TEXT, PRIMARY KEY(scope,stream)
27
+ )`);
28
+ // A missing derived inventory cannot inherit an enumeration checkpoint.
29
+ state.db.exec("DELETE FROM state WHERE kind IN ('index-inventory','index-inventory-garbage')");
30
+ });
31
+ const columns = new Set(state.db.prepare("PRAGMA table_info(capture_index_inventory)").all().map(column => column.name));
32
+ const indexed = state.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='index' AND name='capture_index_inventory_phase'").get()?.found;
33
+ if (!columns.has("phase") || !columns.has("proof") || !indexed) state.transaction(() => {
34
+ if (!columns.has("phase")) state.db.exec("ALTER TABLE capture_index_inventory ADD COLUMN phase TEXT NOT NULL DEFAULT 'ready'");
35
+ if (!columns.has("proof")) state.db.exec("ALTER TABLE capture_index_inventory ADD COLUMN proof TEXT");
36
+ state.db.exec("CREATE INDEX IF NOT EXISTS capture_index_inventory_phase ON capture_index_inventory(scope,phase,stream)");
37
+ });
38
+ const prior = state.get("index-inventory", "current");
39
+ if (prior && (typeof prior.id !== "string" || !prior.id || !["index", "live", "complete"].includes(prior.phase) ||
40
+ typeof prior.indexAfter !== "string" || typeof prior.liveAfter !== "string" || !validIdentity(prior.directory)))
41
+ throw new Error("invalid capture index inventory checkpoint");
42
+ if (prior?.id !== id) state.transaction(() => {
43
+ if (state.get("index-inventory-garbage", id) !== null) throw new Error("capture index inventory identity was retired");
44
+ if (prior) state.set("index-inventory-garbage", prior.id, true);
45
+ state.set("index-inventory", "current", { id, phase: "index", indexAfter: "", liveAfter: "", directory: null });
46
+ });
47
+ }
48
+
49
+ current() {
50
+ const row = this.state.get("index-inventory", "current");
51
+ if (row?.id !== this.id) throw new Error("capture index inventory changed");
52
+ return row;
53
+ }
54
+
55
+ // Sorted directory names retain the accepted metadata-array qualification.
56
+ // Only a worker-local array is cached; restart resumes at a binary name key,
57
+ // without depending on a removed anchor still existing in the directory.
58
+ liveDirectory(store, checkpoint, work) {
59
+ const path = store.streamsDir();
60
+ let before;
61
+ try { before = statSync(path, { bigint: true }); }
62
+ catch (error) {
63
+ if (error.code === "ENOENT" && checkpoint.directory === null) return [];
64
+ throw error;
65
+ }
66
+ if (!before.isDirectory()) throw new Error("capture streams root is not a directory");
67
+ const found = identity(before);
68
+ if (checkpoint.directory && !sameIdentity(checkpoint.directory, found))
69
+ throw new Error("capture index inventory directory changed");
70
+ if (!this.directory) {
71
+ const names = readdirSync(path).sort();
72
+ if (!sameIdentity(found, identity(statSync(path, { bigint: true }))))
73
+ throw new Error("capture index inventory directory changed during enumeration");
74
+ this.directory = { path, identity: found, names };
75
+ if (work) { work.directoriesRead = (work.directoriesRead ?? 0) + 1;
76
+ work.directoryEntriesRead = (work.directoryEntriesRead ?? 0) + names.length; }
77
+ }
78
+ if (this.directory.path !== path || !sameIdentity(this.directory.identity, found))
79
+ throw new Error("capture index inventory directory changed");
80
+ checkpoint.directory = found;
81
+ return this.directory.names;
82
+ }
83
+
84
+ // Exactly one bounded page per call. The index phase must precede live
85
+ // discovery; a reset/deletion must not make an already-enumerated old stream
86
+ // disappear from this sweep's inventory. Caller restarts the sweep on reset.
87
+ discover(index, { limit = pageSize, stopped = () => false, work } = {}) {
88
+ if (this.state.depth || !Number.isSafeInteger(limit) || limit < 1 || limit > pageSize)
89
+ throw new Error("invalid capture index discovery unit");
90
+ const before = this.current(), next = structuredClone(before), streams = [];
91
+ let scanned = 0;
92
+ if (before.phase === "complete" || stopped()) return { scanned, complete: before.phase === "complete" };
93
+ if (before.phase === "index") {
94
+ const rows = index.db.prepare("SELECT stream FROM index_state WHERE stream>? ORDER BY stream LIMIT ?")
95
+ .all(before.indexAfter, limit);
96
+ for (const row of rows) {
97
+ if (stopped()) break;
98
+ if (!validStreamId(row.stream)) throw new Error("invalid indexed stream in capture inventory");
99
+ streams.push(row.stream); next.indexAfter = row.stream; scanned++;
100
+ }
101
+ if (scanned === rows.length && rows.length < limit) next.phase = "live";
102
+ } else {
103
+ const names = this.liveDirectory(index.store, next, work);
104
+ // Binary search avoids rescanning all earlier names in a retained array.
105
+ let low = 0, high = names.length;
106
+ while (low < high) { const middle = Math.floor((low + high) / 2);
107
+ if (names[middle] <= before.liveAfter) low = middle + 1; else high = middle; }
108
+ let position = low;
109
+ while (position < names.length && scanned < limit && !stopped()) {
110
+ const name = names[position++]; scanned++; next.liveAfter = name;
111
+ if (!validStreamId(name)) continue;
112
+ try {
113
+ if (!statSync(index.store.journalPath(name)).isFile()) throw new Error("capture inventory journal is not a regular file");
114
+ } catch (error) { if (error.code === "ENOENT") continue; throw error; }
115
+ streams.push(name);
116
+ }
117
+ if (position === names.length) next.phase = "complete";
118
+ if (next.directory && !sameIdentity(next.directory, identity(statSync(index.store.streamsDir(), { bigint: true }))))
119
+ throw new Error("capture index inventory directory changed during enumeration");
120
+ }
121
+ if (work) work.entriesVisited = (work.entriesVisited ?? 0) + scanned;
122
+ this.state.transaction(() => {
123
+ if (JSON.stringify(this.current()) !== JSON.stringify(before)) throw new Error("capture index inventory checkpoint changed");
124
+ this.include(streams);
125
+ this.state.set("index-inventory", "current", next);
126
+ });
127
+ if (next.phase === "complete") this.directory = null;
128
+ return { scanned, complete: next.phase === "complete" };
129
+ }
130
+
131
+ // Source work joins new stream membership to its checkpoint/accounting
132
+ // commit, even after directory enumeration ended. Existing evidence is not
133
+ // promoted or cleared: outcome readers requalify it against current state.
134
+ include(streams) {
135
+ if (!this.state.depth || !Array.isArray(streams) || streams.length > pageSize || streams.some(stream => !validStreamId(stream)))
136
+ throw new Error("capture index membership requires a bounded processing transaction");
137
+ this.current();
138
+ const insert = this.state.db.prepare("INSERT OR IGNORE INTO capture_index_inventory(scope,stream) VALUES (?,?)");
139
+ // A new member has not been walked: the stage is no longer clean.
140
+ for (const stream of streams) if (Number(insert.run(this.id, stream).changes)) this.clean = false;
141
+ }
142
+
143
+ // This pass wrote these streams (journal, checkpoint or index cursor): their
144
+ // verification no longer holds, and the next walk must look again.
145
+ changed(streams) {
146
+ if (!streams.length) return;
147
+ for (const stream of streams) this.verified.delete(stream);
148
+ this.clean = false;
149
+ }
150
+
151
+ // One finite rotation of discovered names; admission owns the durable next
152
+ // position. No row is omitted because a previous unit appeared complete:
153
+ // this inventory alone cannot detect every external index reset or write.
154
+ // A caller may `skip` rows it verified itself earlier in the same pass.
155
+ *ordered({ stopped = () => false, work, skip = null } = {}) {
156
+ this.current();
157
+ const after = this.state.get("capture-schedule", "position:index")?.after ?? "";
158
+ if (typeof after !== "string") throw new Error("invalid capture index inventory position");
159
+ for (const later of [true, false]) {
160
+ let last = later ? after : "";
161
+ while (!stopped()) {
162
+ this.current();
163
+ const rows = this.state.db.prepare(`SELECT stream FROM capture_index_inventory WHERE scope=? AND stream>?
164
+ ${later ? "" : "AND stream<=?"} ORDER BY stream LIMIT ${pageSize}`)
165
+ .all(...(later ? [this.id, last] : [this.id, last, after]));
166
+ if (work) work.inventoryRows = (work.inventoryRows ?? 0) + rows.length;
167
+ if (!rows.length) break;
168
+ for (const row of rows) {
169
+ if (skip?.(row.stream)) { last = row.stream; continue; }
170
+ if (stopped()) return;
171
+ last = row.stream; yield row.stream;
172
+ }
173
+ }
174
+ }
175
+ }
176
+
177
+ // Publish a fresh pending sweep in the reset's capture-state commit. Do not
178
+ // change this object's id: rollback must leave its in-memory identity valid,
179
+ // and success requires callers to reopen against the newly published scope.
180
+ restart(id) {
181
+ if (!this.state.depth || typeof id !== "string" || !id || id === this.id)
182
+ throw new Error("capture index restart requires a new transactional identity");
183
+ this.current();
184
+ if (this.state.get("index-inventory-garbage", id) !== null ||
185
+ this.state.db.prepare("SELECT 1 AS found FROM capture_index_inventory WHERE scope=? LIMIT 1").get(id)?.found)
186
+ throw new Error("capture index restart identity was already used");
187
+ this.state.set("index-inventory-garbage", this.id, true);
188
+ this.state.set("index-inventory", "current", { id, phase: "index", indexAfter: "", liveAfter: "", directory: null });
189
+ }
190
+
191
+ cleanup() {
192
+ if (!this.state.depth) throw new Error("capture index inventory cleanup requires a transaction");
193
+ this.current();
194
+ const obsolete = this.state.db.prepare("SELECT key FROM state WHERE kind='index-inventory-garbage' ORDER BY key LIMIT 1").get()?.key;
195
+ if (obsolete == null) return 0;
196
+ if (obsolete === this.id) throw new Error("active capture index inventory cannot be deleted");
197
+ const count = Number(this.state.db.prepare(`DELETE FROM capture_index_inventory WHERE rowid IN (
198
+ SELECT rowid FROM capture_index_inventory WHERE scope=? ORDER BY stream LIMIT ${pageSize})`).run(obsolete).changes);
199
+ if (!this.state.db.prepare("SELECT 1 AS found FROM capture_index_inventory WHERE scope=? LIMIT 1").get(obsolete)?.found)
200
+ this.state.delete("index-inventory-garbage", obsolete);
201
+ return count;
202
+ }
203
+
204
+ close() { this.directory = null; }
205
+ }
@@ -0,0 +1,113 @@
1
+ // Processing outcomes are distinct from inventory membership. A stored complete
2
+ // mark can suppress work only while journal validation and index cursors still
3
+ // match. No outcome or audit page by itself is a final command receipt.
4
+ import { captureContext } from "./capture-context.mjs";
5
+ import { captureIndexWork } from "./capture-index-work.mjs";
6
+ import { readCaptureRetry } from "./capture-admission.mjs";
7
+ import { fileVersion, sameVersion } from "./derived-state.mjs";
8
+
9
+ function evidence(index, state, stream) {
10
+ const checkpoint = state.get("journal", stream), path = index.store.journalPath(stream);
11
+ const generation = fileVersion(path);
12
+ if (!checkpoint || checkpoint.complete !== true || checkpoint.resetting ||
13
+ !sameVersion(checkpoint.version, generation) || typeof checkpoint.epoch !== "string" ||
14
+ !Number.isSafeInteger(checkpoint.bytes) || checkpoint.bytes < 0 || checkpoint.bytes > (generation?.size ?? 0) ||
15
+ !Number.isSafeInteger(checkpoint.count) || checkpoint.count < 0 || checkpoint.count > checkpoint.bytes) return null;
16
+ const row = index.db.prepare(`SELECT s.indexed_count,b.bytes,b.indexed_count AS byte_count,b.epoch
17
+ FROM index_state s LEFT JOIN index_bytes b ON b.stream=s.stream WHERE s.stream=?`).get(stream);
18
+ const cursor = row?.indexed_count != null ? { count: row.indexed_count, bytes: row.bytes,
19
+ byteCount: row.byte_count, epoch: row.epoch } : null;
20
+ if (cursor ? cursor.count !== checkpoint.count || cursor.byteCount !== checkpoint.count ||
21
+ cursor.bytes !== checkpoint.bytes || cursor.epoch !== checkpoint.epoch : checkpoint.bytes !== 0 || checkpoint.count !== 0)
22
+ return null;
23
+ if (!sameVersion(generation, fileVersion(path))) return null;
24
+ return { generation, epoch: checkpoint.epoch, bytes: checkpoint.bytes, count: checkpoint.count, cursor };
25
+ }
26
+
27
+ export class CaptureIndexOutcomes {
28
+ constructor(index, inventory) {
29
+ this.index = index; this.inventory = inventory; this.state = inventory.state;
30
+ inventory.current();
31
+ }
32
+
33
+ complete(row) {
34
+ if (row.phase !== "complete" || row.proof === null || readCaptureRetry(this.state, row.stream)) return false;
35
+ const current = evidence(this.index, this.state, row.stream);
36
+ return current !== null && JSON.stringify(JSON.parse(row.proof)) === JSON.stringify(current);
37
+ }
38
+
39
+ // With `memo`, a stream verified complete is not verified again in this
40
+ // worker's pass until the pass writes it, and a walk that ends with every row
41
+ // verified leaves the stage clean until then. Every pass verifies each row.
42
+ *ready({ stopped = () => false, work, memo = false } = {}) {
43
+ const verified = memo ? this.inventory.verified : null;
44
+ let pending = false;
45
+ for (const stream of this.inventory.ordered({ stopped, work, skip: verified && (name => verified.has(name)) })) {
46
+ const row = this.state.db.prepare("SELECT stream,phase,proof FROM capture_index_inventory WHERE scope=? AND stream=?")
47
+ .get(this.inventory.id, stream);
48
+ if (this.complete(row)) verified?.add(stream);
49
+ else { pending = true; yield stream; }
50
+ }
51
+ if (memo && !pending && !stopped()) this.inventory.clean = true;
52
+ }
53
+
54
+ resetRequired() {
55
+ this.inventory.current();
56
+ return !!this.state.db.prepare("SELECT 1 AS found FROM capture_index_inventory WHERE scope=? AND phase='reset' LIMIT 1")
57
+ .get(this.inventory.id)?.found;
58
+ }
59
+
60
+ // Called by captureIndexWork after index commit and accounting, still inside
61
+ // its stream locks and capture.db processing transaction. Any throw rolls
62
+ // these marks/counters back together and leaves durable admission intact.
63
+ record(result) {
64
+ const context = captureContext(this.index.store), names = result.streams.map(row => row.stream);
65
+ if (context?.state !== this.state || !this.state.depth || context.admission?.stage !== "index" ||
66
+ names.length !== context.admission.members.length || new Set(names).size !== names.length ||
67
+ names.some(stream => !context.held.has(stream))) throw new Error("index outcomes require their admitted processing transaction");
68
+ this.inventory.include(names);
69
+ const update = this.state.db.prepare(`UPDATE capture_index_inventory SET phase=?,proof=? WHERE scope=? AND stream=?
70
+ AND (phase!=? OR proof IS NOT ?)`);
71
+ for (const item of result.streams) {
72
+ if (!["complete", "validation", "index", "reset"].includes(item.phase)) throw new Error("invalid index processing outcome");
73
+ const proof = item.phase === "complete" ? evidence(this.index, this.state, item.stream) : null;
74
+ if (item.phase === "complete" && proof === null) throw new Error("index completion evidence changed before publication");
75
+ const encoded = proof === null ? null : JSON.stringify(proof);
76
+ update.run(item.phase, encoded, this.inventory.id, item.stream, item.phase, encoded);
77
+ // A complete outcome was just verified against current evidence; any
78
+ // other leaves work for a later walk.
79
+ if (item.phase === "complete") this.inventory.verified.add(item.stream);
80
+ else this.inventory.changed([item.stream]);
81
+ }
82
+ }
83
+
84
+ advance(run, start, { memo = false, ...options } = {}) {
85
+ if (this.state.depth) throw new Error("index outcome scheduling must precede processing");
86
+ if (start.budget.sample() !== null) return { selected: 0, stopped: true };
87
+ // Reset is union-wide. A completed/failed detecting admission does not
88
+ // authorize retrying reset in this worker or quietly dropping the request.
89
+ if (this.resetRequired()) return { selected: 0, resetRequired: true };
90
+ // A clean stage offers no candidates; due retries are still selected.
91
+ const streams = memo && this.inventory.clean ? [] : this.ready({ stopped: () => start.budget.sample() !== null, memo });
92
+ return captureIndexWork(this.index, this.state, streams, run, start, { ...options, outcomes: this });
93
+ }
94
+
95
+ // Page completion is only the end of this audit's enumeration. The owning
96
+ // scheduler must include unfinished discovery, retries, newly captured
97
+ // streams, reset and other stages in its final decision. Recheck after writes.
98
+ audit({ after = "", limit = 128, stopped = () => false } = {}) {
99
+ if (typeof after !== "string" || !Number.isSafeInteger(limit) || limit < 1 || limit > 128 || typeof stopped !== "function")
100
+ throw new Error("invalid capture index outcome audit page");
101
+ this.inventory.current();
102
+ const rows = this.state.db.prepare(`SELECT stream,phase,proof FROM capture_index_inventory
103
+ WHERE scope=? AND stream>? ORDER BY stream LIMIT ?`).all(this.inventory.id, after, limit);
104
+ let pending = 0, checked = 0;
105
+ for (const row of rows) {
106
+ if (stopped()) return { checked, pending, after, complete: false, stopped: true };
107
+ if (!this.complete(row)) pending++;
108
+ checked++; after = row.stream;
109
+ }
110
+ return { checked: rows.length, pending, after: rows.at(-1)?.stream ?? after,
111
+ complete: rows.length < limit, discovered: this.inventory.current().phase === "complete", resetRequired: this.resetRequired() };
112
+ }
113
+ }
@@ -0,0 +1,110 @@
1
+ // Union reset has its own admission: a reset failure does not prove that any
2
+ // journal stream/frame failed. Caller owns the root lock, worker reconciliation
3
+ // and final receipts. This helper never deletes a stream's failure/retry row.
4
+ import { randomUUID } from "node:crypto";
5
+ import { readCaptureAdmission } from "./capture-admission.mjs";
6
+ import { FAILURE_REASONS, retryAfterAttempt } from "./capture-failures.mjs";
7
+ import { resetIndexTables } from "./index-reset.mjs";
8
+
9
+ const kind = "capture-index-reset", receiptKey = "capture-index-reset";
10
+ const validText = value => typeof value === "string" && value.length > 0 && value.length <= 128;
11
+ const validDate = value => typeof value === "string" && Number.isFinite(Date.parse(value)) && new Date(value).toISOString() === value;
12
+
13
+ export class CaptureIndexReset {
14
+ constructor(index, inventory) { this.index = index; this.inventory = inventory; this.state = inventory.state; }
15
+
16
+ current() {
17
+ this.inventory.current();
18
+ const row = this.state.get(kind, "current");
19
+ if (row !== null && (!row || row.scope !== this.inventory.id ||
20
+ ![row.id, row.nextScope, row.requestedBy].every(validText) || row.nextScope === row.scope ||
21
+ !Number.isSafeInteger(row.attempts) || row.attempts < 0 ||
22
+ row.attemptedRun !== null && !validText(row.attemptedRun) ||
23
+ row.nextRetryAt !== null && !validDate(row.nextRetryAt) ||
24
+ row.attempt !== null && (!validText(row.attempt?.workerNonce) || !validText(row.attempt?.runId) ||
25
+ row.attempt.runId !== row.attemptedRun))) throw new Error("invalid capture index reset request");
26
+ return row;
27
+ }
28
+
29
+ worker(start) {
30
+ if (this.state.depth || !validText(start.workerNonce) || !validText(start.pass?.runId))
31
+ throw new Error("capture index reset requires a supervised worker outside processing");
32
+ start.budget.check();
33
+ if (readCaptureAdmission(this.state)) throw new Error("capture index reset requires stream admission reconciliation");
34
+ }
35
+
36
+ request(start) {
37
+ this.worker(start);
38
+ const prior = this.current();
39
+ if (prior) return prior;
40
+ if (!this.state.db.prepare("SELECT 1 AS found FROM capture_index_inventory WHERE scope=? AND phase='reset' LIMIT 1")
41
+ .get(this.inventory.id)?.found) return null;
42
+ const row = { id: randomUUID(), scope: this.inventory.id, nextScope: randomUUID(), requestedBy: start.workerNonce,
43
+ attempts: 0, attemptedRun: null, nextRetryAt: null, attempt: null };
44
+ this.state.transaction(() => this.state.set(kind, "current", row));
45
+ return row;
46
+ }
47
+
48
+ committed(row) {
49
+ return this.index.db.prepare("SELECT value FROM meta WHERE key=?").get(receiptKey)?.value === row.id;
50
+ }
51
+
52
+ // Only after the predecessor has ended, like stream admission reconciliation.
53
+ // The registered replacement's nonce must differ. Return pass-bound failure
54
+ // for the run writer; never invent a failed member or exact inner-frame offset.
55
+ reconcile(start, { reason = "interrupted", now = Date.now() } = {}) {
56
+ this.worker(start);
57
+ if (!FAILURE_REASONS.includes(reason)) throw new Error("invalid capture reset interruption reason");
58
+ const row = this.current();
59
+ if (!row?.attempt) return null;
60
+ if (row.attempt.workerNonce === start.workerNonce) throw new Error("capture reset worker has not been replaced");
61
+ const failure = { id: row.id, runId: row.attempt.runId, reason }, attempts = row.attempts + 1;
62
+ const nextRetryAt = retryAfterAttempt(attempts, now);
63
+ this.state.transaction(() => {
64
+ this.state.set(kind, "current", { ...row, attempt: null, attempts, nextRetryAt });
65
+ this.state.set("capture-index-reset-failure", "last", failure);
66
+ });
67
+ return { ...failure, committed: this.committed(row) };
68
+ }
69
+
70
+ advance(run, start, { now = Date.now() } = {}) {
71
+ this.worker(start);
72
+ if (!Number.isSafeInteger(now) || now < 0 || run.state !== this.state || run.id !== start.pass.runId)
73
+ throw new Error("invalid capture reset run admission");
74
+ const row = this.current();
75
+ if (!row) return { reset: false };
76
+ if (row.attempt) throw new Error("unreconciled capture index reset admission");
77
+ const recovered = this.committed(row);
78
+ if (!recovered) {
79
+ if (row.attemptedRun === run.id || row.nextRetryAt !== null && now < Date.parse(row.nextRetryAt))
80
+ return { reset: false, deferred: true };
81
+ if (start.pass.reset !== true || start.pass.finalize !== false || row.requestedBy === start.workerNonce)
82
+ return { reset: false, needsFreshWorker: true };
83
+ this.state.transaction(() => {
84
+ run.current();
85
+ this.state.set(kind, "current", { ...row, attemptedRun: run.id,
86
+ attempt: { workerNonce: start.workerNonce, runId: run.id } });
87
+ });
88
+ start.budget.check();
89
+ // No capture.db transaction spans this atomic index operation. The token
90
+ // commits with the reset, so recovery cannot erase index work performed
91
+ // after that commit just because sweep publication subsequently failed.
92
+ this.index.statements.clear(); // prepared against the tables the reset recreates
93
+ this.index.db.exec("BEGIN");
94
+ try {
95
+ resetIndexTables(this.index.db);
96
+ this.index.db.prepare("INSERT INTO meta(key,value) VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value")
97
+ .run(receiptKey, row.id);
98
+ this.index.db.exec("COMMIT");
99
+ } catch (error) { this.index.db.exec("ROLLBACK"); throw error; }
100
+ }
101
+ start.budget.check();
102
+ this.state.transaction(() => {
103
+ if (this.current()?.id !== row.id || !this.committed(row)) throw new Error("capture index reset evidence changed");
104
+ this.inventory.restart(row.nextScope);
105
+ run.record({}, { progress: true, peakRssBytes: start.budget.peakRssBytes });
106
+ this.state.delete(kind, "current");
107
+ });
108
+ return { reset: true, recovered, scope: row.nextScope };
109
+ }
110
+ }
@@ -0,0 +1,77 @@
1
+ // One admitted index group, with confirmed run accounting. The caller owns a
2
+ // finite fair union of live/indexed streams, its sweep checkpoints, atomic reset
3
+ // admission and final completion. Returning no selection is not root completion.
4
+ import { withCaptureGroup, captureContext } from "./capture-context.mjs";
5
+ import { indexCaptureGroup } from "./index-capture.mjs";
6
+ import { selectCaptureWork, resolveCaptureRetry } from "./capture-retry.mjs";
7
+ import { readCaptureRetry } from "./capture-admission.mjs";
8
+ import { fileVersion, sameVersion } from "./derived-state.mjs";
9
+
10
+ function unitStart(index, state, stream, generation) {
11
+ const checkpoint = state.get("journal", stream);
12
+ // Journal recovery precedes indexing. An invalidated generation starts at
13
+ // zero; an interrupted validation starts at its saved byte boundary, even
14
+ // when an older index cursor points farther ahead.
15
+ if (!checkpoint || !sameVersion(checkpoint.version, generation) ||
16
+ !Number.isSafeInteger(checkpoint.bytes) || checkpoint.bytes < 0 || checkpoint.bytes > (generation?.size ?? 0) ||
17
+ !Number.isSafeInteger(checkpoint.count) || checkpoint.count < 0 || checkpoint.count > checkpoint.bytes ||
18
+ typeof checkpoint.epoch !== "string" || typeof checkpoint.complete !== "boolean" ||
19
+ checkpoint.resetting !== undefined && typeof checkpoint.resetting !== "boolean" || checkpoint.resetting) return 0;
20
+ if (!checkpoint.complete) return checkpoint.bytes;
21
+ const row = index.db.prepare(`SELECT b.bytes,b.epoch,b.indexed_count,s.indexed_count AS count
22
+ FROM index_bytes b JOIN index_state s ON s.stream=b.stream WHERE b.stream=?`).get(stream);
23
+ return row?.epoch === checkpoint.epoch && Number.isSafeInteger(row.bytes) && row.bytes >= 0 &&
24
+ row.bytes <= checkpoint.bytes && Number.isSafeInteger(row.count) && row.count >= 0 &&
25
+ row.count === row.indexed_count && row.count <= checkpoint.count ? row.bytes : 0;
26
+ }
27
+
28
+ export function captureIndexWork(index, state, streams, run, start, {
29
+ limit = 128, maxRecords = 2000, maxBytes = 8388608, now = Date.now(), outcomes = null, allowRetry = true, retryFilter,
30
+ } = {}) {
31
+ if (!Number.isSafeInteger(limit) || limit < 1 || limit > 128 ||
32
+ !Number.isSafeInteger(maxRecords) || maxRecords < limit || maxRecords > 2000 ||
33
+ !Number.isSafeInteger(maxBytes) || maxBytes < limit || maxBytes > 8388608)
34
+ throw new Error("invalid capture index work allowance");
35
+ const store = index.store, budget = start.budget, stopped = () => budget.sample() !== null;
36
+ if (outcomes !== null && (outcomes.index !== index || outcomes.state !== state || typeof outcomes.record !== "function"))
37
+ throw new Error("capture index outcomes must belong to the current index and state");
38
+ if (stopped()) return { stopped: true, selected: 0 };
39
+ function* candidates() {
40
+ for (const stream of streams) {
41
+ if (stopped()) return;
42
+ if (readCaptureRetry(state, stream)) continue;
43
+ const generation = fileVersion(store.journalPath(stream));
44
+ // This is an attribution boundary, not permission to consume an index
45
+ // cursor. The group validates count/epoch/boundary before using that row.
46
+ const offset = unitStart(index, state, stream, generation);
47
+ yield { stream, stage: "index", offset, generation, next: { stage: "index", after: stream } };
48
+ }
49
+ }
50
+ const selection = selectCaptureWork(store, state, candidates(), { runId: run.id, now, limit, allowRetry, retryFilter, stopped });
51
+ if (!selection) return { selected: 0 };
52
+ // The common retry queue can return a different stage when this healthy
53
+ // sweep is exhausted. Never erase that failure through index processing.
54
+ if (selection.stage !== "index") return { selected: 0, deferred: selection };
55
+ return withCaptureGroup(store, selection.members.map(member => member.stream), () => {
56
+ const before = state.db.prepare("SELECT total_changes() AS n").get().n;
57
+ const result = indexCaptureGroup(index, selection.members.map(member => member.stream), {
58
+ maxRecords, maxBytes, stopped,
59
+ // A detecting unit, including a due retry, cannot reserve the fresh full
60
+ // worker allowance required for an atomic union reset.
61
+ allowReset: false,
62
+ });
63
+ const progressed = result.bytesIndexed > 0 || state.db.prepare("SELECT total_changes() AS n").get().n !== before;
64
+ if (selection.retry && result.streams[0].phase === "complete") {
65
+ // Complete means the current validated journal is fully indexed. A
66
+ // partial validation/index cursor or reset request cannot resolve retry.
67
+ resolveCaptureRetry(state, { id: captureContext(store).admission.id,
68
+ stream: selection.members[0].stream, reconciled: true });
69
+ }
70
+ // turns.db has committed before these facts enter capture.db's commit.
71
+ // Failure between commits leaves admission for reconciliation; a later
72
+ // no-op must not reconstruct the missing count from existing index rows.
73
+ run.record({ turnsIndexed: result.turnsIndexed }, { progress: progressed, peakRssBytes: budget.peakRssBytes });
74
+ if (outcomes) outcomes.record(result);
75
+ return { selected: selection.members.length, ...result };
76
+ }, { state, admission: selection, admissionNow: now, batchBytes: maxBytes, check: () => budget.check() });
77
+ }