@awebai/oats 0.61.0 → 0.63.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +1 -1
  2. package/bin/oats.mjs +27 -12
  3. package/docs/capabilities.md +1 -1
  4. package/docs/desktop-cli-api.md +30 -2
  5. package/docs/execution-targets.md +89 -4
  6. package/docs/implementation.md +2 -1
  7. package/docs/named-test-runs.md +382 -0
  8. package/docs/official-catalog.md +1 -1
  9. package/docs/packages.md +5 -5
  10. package/docs/release-lane.md +13 -0
  11. package/docs/release-notes/v0.62.0.md +223 -0
  12. package/docs/release-notes/v0.63.0.md +135 -0
  13. package/docs/souls-and-instances.md +6 -1
  14. package/docs/upgrading.md +135 -11
  15. package/docs/workspaces.md +1 -1
  16. package/lib/core.mjs +73 -23
  17. package/lib/doctor-capture.mjs +35 -0
  18. package/lib/session-viewer.mjs +86 -7
  19. package/lib/tree-copy.mjs +188 -7
  20. package/package-catalog.json +1 -1
  21. package/package.json +1 -1
  22. package/packages/client/cli-locator.mjs +2 -2
  23. package/packages/record/README.md +87 -13
  24. package/packages/record/bin/capture-hook.mjs +8 -0
  25. package/packages/record/bin/capture.mjs +200 -139
  26. package/packages/record/bin/recall.mjs +8 -3
  27. package/packages/record/bin/setup.mjs +116 -225
  28. package/packages/record/bin/turn-record.mjs +2 -2
  29. package/packages/record/docs/capture-service.md +1527 -28
  30. package/packages/record/docs/capture-status.schema.json +429 -12
  31. package/packages/record/lib/capture-admission.mjs +82 -0
  32. package/packages/record/lib/capture-aw-unit.mjs +137 -0
  33. package/packages/record/lib/capture-aw.mjs +31 -12
  34. package/packages/record/lib/capture-budget.mjs +31 -0
  35. package/packages/record/lib/capture-cc.mjs +39 -6
  36. package/packages/record/lib/capture-cleanup-queue.mjs +135 -0
  37. package/packages/record/lib/capture-cleanup-work.mjs +77 -0
  38. package/packages/record/lib/capture-command-operation.mjs +37 -0
  39. package/packages/record/lib/capture-command-output.mjs +62 -0
  40. package/packages/record/lib/capture-command-worker.mjs +6 -0
  41. package/packages/record/lib/capture-command.mjs +65 -0
  42. package/packages/record/lib/capture-completion.mjs +152 -0
  43. package/packages/record/lib/capture-context.mjs +50 -0
  44. package/packages/record/lib/capture-default-roots.mjs +30 -0
  45. package/packages/record/lib/capture-discovery.mjs +114 -0
  46. package/packages/record/lib/capture-failures.mjs +45 -0
  47. package/packages/record/lib/capture-health-doctor.mjs +13 -0
  48. package/packages/record/lib/capture-health-installation.mjs +117 -0
  49. package/packages/record/lib/capture-health.mjs +116 -0
  50. package/packages/record/lib/capture-hook-ownership.mjs +58 -0
  51. package/packages/record/lib/capture-hook-runner.mjs +17 -0
  52. package/packages/record/lib/capture-hook.mjs +26 -0
  53. package/packages/record/lib/capture-index-inventory.mjs +205 -0
  54. package/packages/record/lib/capture-index-outcomes.mjs +113 -0
  55. package/packages/record/lib/capture-index-reset.mjs +110 -0
  56. package/packages/record/lib/capture-index-work.mjs +77 -0
  57. package/packages/record/lib/capture-issue-preparation.mjs +73 -0
  58. package/packages/record/lib/capture-issues.mjs +15 -2
  59. package/packages/record/lib/capture-lock.mjs +137 -17
  60. package/packages/record/lib/capture-native-unit.mjs +322 -0
  61. package/packages/record/lib/capture-pass.mjs +116 -0
  62. package/packages/record/lib/capture-recovery-snapshot.mjs +182 -0
  63. package/packages/record/lib/capture-retained-scratch.mjs +213 -0
  64. package/packages/record/lib/capture-retry.mjs +102 -0
  65. package/packages/record/lib/capture-run.mjs +125 -0
  66. package/packages/record/lib/capture-scratch.mjs +12 -3
  67. package/packages/record/lib/capture-service-definition.mjs +104 -0
  68. package/packages/record/lib/capture-service-install.mjs +362 -0
  69. package/packages/record/lib/capture-service-ownership.mjs +137 -0
  70. package/packages/record/lib/capture-service-runner.mjs +18 -0
  71. package/packages/record/lib/capture-small-snapshot.mjs +85 -0
  72. package/packages/record/lib/capture-source-discovery.mjs +319 -0
  73. package/packages/record/lib/capture-source-queue.mjs +199 -0
  74. package/packages/record/lib/capture-source-work.mjs +85 -0
  75. package/packages/record/lib/capture-state.mjs +16 -71
  76. package/packages/record/lib/capture-status.mjs +171 -0
  77. package/packages/record/lib/capture-supervisor.mjs +136 -0
  78. package/packages/record/lib/capture-watch.mjs +32 -0
  79. package/packages/record/lib/capture-work-session.mjs +241 -0
  80. package/packages/record/lib/capture-worker.mjs +74 -0
  81. package/packages/record/lib/derived-state.mjs +58 -17
  82. package/packages/record/lib/file-lines.mjs +22 -6
  83. package/packages/record/lib/formats.mjs +4 -3
  84. package/packages/record/lib/index-capture.mjs +97 -0
  85. package/packages/record/lib/index-db.mjs +121 -66
  86. package/packages/record/lib/index-reset.mjs +19 -0
  87. package/packages/record/lib/store.mjs +46 -14
  88. package/packages/record/lib/transcript-prefix.mjs +125 -0
  89. package/skills/oats-getting-started/SKILL.md +1 -1
  90. package/docs/macos-test-runs.md +0 -246
@@ -0,0 +1,319 @@
1
+ // Private source-plan bridge. The caller supplies explicit roots or a stable
2
+ // default-root selection. Resolution/enumeration never opens source content.
3
+ // Pages and their positions commit together; enumeration never grants success.
4
+ import { readdirSync, statSync } from "node:fs";
5
+ import { createHash, randomUUID } from "node:crypto";
6
+ import { isAbsolute, join } from "node:path";
7
+ import { NativeCaptureDiscovery } from "./capture-discovery.mjs";
8
+ import { SESSION_FORMATS } from "./formats.mjs";
9
+ import { awStream } from "./capture-aw.mjs";
10
+ import { guardCapturedPath } from "./native-history.mjs";
11
+ import { FAILURE_REASONS, retryAfterAttempt } from "./capture-failures.mjs";
12
+ import { CaptureStop } from "./capture-budget.mjs";
13
+ import { captureDefaultRootPlan, resolveCaptureDefaultRoots } from "./capture-default-roots.mjs";
14
+
15
+ const sources = ["cc", "pi", "codex", "aw"];
16
+ const attemptsKind = "capture-discovery-attempt";
17
+ const text = value => typeof value === "string" && value.length > 0;
18
+ const identity = stat => ({ dev: String(stat.dev), ino: String(stat.ino) });
19
+ const sameIdentity = (a, b) => a?.dev === b?.dev && a?.ino === b?.ino;
20
+ const absolute = path => typeof path === "string" && isAbsolute(path) && !path.includes("\0");
21
+ const validIdentity = value => value && typeof value.dev === "string" && typeof value.ino === "string";
22
+ const validName = name => typeof name === "string" && name && name !== "." && name !== ".." && !/[\/\\\0]/.test(name);
23
+ const validFailure = row => row && typeof row.runId === "string" && row.runId &&
24
+ typeof row.attemptedRun === "string" && row.attemptedRun && FAILURE_REASONS.includes(row.reason) &&
25
+ Number.isSafeInteger(row.attempts) && row.attempts > 0 && typeof row.nextRetryAt === "string" &&
26
+ Number.isFinite(Date.parse(row.nextRetryAt)) && new Date(row.nextRetryAt).toISOString() === row.nextRetryAt;
27
+
28
+ // AW uses sorted code-unit basenames, not native localeCompare ordering. A
29
+ // cached name array is metadata proportional to the directory, not content.
30
+ class AwDiscovery {
31
+ constructor(path, checkpoint) {
32
+ this.path = path;
33
+ this.cursor = structuredClone(checkpoint ?? { after: null, identity: null, complete: false });
34
+ const row = this.cursor;
35
+ if (!row || row.after !== null && !validName(row.after) || typeof row.complete !== "boolean" ||
36
+ row.identity !== null && (!row.identity || typeof row.identity.dev !== "string" || typeof row.identity.ino !== "string"))
37
+ throw new Error("invalid AW discovery checkpoint");
38
+ }
39
+
40
+ advance({ limit, stopped, work }) {
41
+ const row = this.cursor, files = []; let visited = 0;
42
+ if (row.complete) return { files, visited, complete: true, checkpoint: structuredClone(row) };
43
+ guardCapturedPath(this.path);
44
+ let before;
45
+ try { before = statSync(this.path, { bigint: true }); }
46
+ catch (error) {
47
+ if (error.code !== "ENOENT" || row.identity !== null || row.after !== null) throw error;
48
+ row.complete = true;
49
+ return { files, visited, complete: true, checkpoint: structuredClone(row) };
50
+ }
51
+ if (!before.isDirectory() || row.identity && !sameIdentity(row.identity, identity(before)))
52
+ throw new Error("AW discovery directory identity changed");
53
+ if (!this.names) {
54
+ this.names = readdirSync(this.path).sort();
55
+ if (!sameIdentity(identity(before), identity(statSync(this.path, { bigint: true }))))
56
+ throw new Error("AW discovery directory changed during enumeration");
57
+ if (work) { work.directoriesRead = (work.directoriesRead ?? 0) + 1;
58
+ work.directoryEntriesRead = (work.directoryEntriesRead ?? 0) + this.names.length; }
59
+ let low = 0, high = this.names.length;
60
+ while (low < high) {
61
+ const mid = Math.floor((low + high) / 2);
62
+ if (row.after !== null && this.names[mid] <= row.after) low = mid + 1; else high = mid;
63
+ }
64
+ this.position = low;
65
+ }
66
+ row.identity = identity(before);
67
+ while (this.position < this.names.length && visited < limit && !stopped()) {
68
+ const name = this.names[this.position++]; row.after = name; visited++;
69
+ if (name.endsWith(".jsonl")) {
70
+ const path = join(this.path, name); guardCapturedPath(path); files.push(path);
71
+ }
72
+ }
73
+ if (!sameIdentity(row.identity, identity(statSync(this.path, { bigint: true }))))
74
+ throw new Error("AW discovery directory changed during enumeration");
75
+ row.complete = this.position === this.names.length;
76
+ if (work) work.entriesVisited = (work.entriesVisited ?? 0) + visited;
77
+ return { files, visited, complete: row.complete, checkpoint: structuredClone(row) };
78
+ }
79
+
80
+ close() { this.names = null; }
81
+ }
82
+
83
+ export class CaptureSourceDiscovery {
84
+ constructor(queue, roots) {
85
+ if (roots?.kind === "default-roots") {
86
+ if (Object.keys(roots).some(key => !["kind", "home", "cwd", "env", "sources"].includes(key)))
87
+ throw new Error("invalid capture discovery root plan");
88
+ this.defaults = captureDefaultRootPlan(roots);
89
+ } else {
90
+ if (!Array.isArray(roots) || roots.some(root => !root || !sources.includes(root.source) ||
91
+ !absolute(root.path) || Object.keys(root).some(key => !["source", "path"].includes(key))))
92
+ throw new Error("invalid capture discovery root plan");
93
+ this.roots = sources.map(source => roots.filter(root => root.source === source).map(root => root.path));
94
+ }
95
+ this.queue = queue;
96
+ this.plan = createHash("sha256").update(JSON.stringify(this.defaults ?? this.roots)).digest("hex");
97
+ this.readers = new Map();
98
+ this.current();
99
+ }
100
+
101
+ current() {
102
+ const sweep = this.queue.current();
103
+ let row = sweep.discovery;
104
+ if (row === null && !sweep.discovered) row = { v: this.defaults ? 2 : 1, plan: this.plan, next: 0,
105
+ groups: sources.map((source, index) => this.defaults ?
106
+ { root: 0, checkpoint: null, complete: !this.defaults.sources.includes(source),
107
+ roots: this.defaults.sources.includes(source) ? null : [] } :
108
+ { root: 0, checkpoint: null, complete: this.roots[index].length === 0 }) };
109
+ if (!row || row.v !== (this.defaults ? 2 : 1) || row.plan !== this.plan ||
110
+ !Number.isSafeInteger(row.next) || row.next < 0 || row.next >= sources.length ||
111
+ !Array.isArray(row.groups) || row.groups.length !== sources.length || row.groups.some((group, index) => !this.validGroup(group, index)) ||
112
+ sweep.discovered && row.groups.some(group => !group.complete))
113
+ throw new Error("capture source discovery plan or checkpoint changed");
114
+ return structuredClone(row);
115
+ }
116
+
117
+ healthyPending() { return this.current().groups.some(group => !group.complete && !group.failure); }
118
+
119
+ validGroup(group, index) {
120
+ if (!group || !Number.isSafeInteger(group.root) || group.root < 0 ||
121
+ group.failure != null && (group.complete || !validFailure(group.failure)) ||
122
+ group.checkpoint !== null && (group.complete || typeof group.checkpoint !== "object")) return false;
123
+ if (this.defaults) {
124
+ if (group.roots === null) return this.defaults.sources.includes(sources[index]) &&
125
+ group.root === 0 && group.checkpoint === null && group.complete === false;
126
+ if (!Array.isArray(group.roots) || group.roots.some(root => !root || !absolute(root.path) ||
127
+ !(validIdentity(root.identity) || sources[index] === "aw" && root.identity === null)) ||
128
+ !this.defaults.sources.includes(sources[index]) && group.roots.length !== 0) return false;
129
+ }
130
+ const count = this.defaults ? group.roots.length : this.roots[index].length;
131
+ return group.root <= count && group.complete === (group.root === count);
132
+ }
133
+
134
+ pathFor(group, index) {
135
+ return this.defaults ? group.roots === null ? this.defaults.home : group.roots[group.root]?.path : this.roots[index][group.root];
136
+ }
137
+
138
+ // Resolving one format can enumerate/sort home-directory metadata. Keep it
139
+ // outside the admission transaction but inside the worker's bounded life.
140
+ resolveRoots(source) {
141
+ return resolveCaptureDefaultRoots(this.defaults, source).map(path => {
142
+ guardCapturedPath(path);
143
+ try {
144
+ const stat = statSync(path, { bigint: true });
145
+ if (!stat.isDirectory()) throw new Error("capture default root is not a directory");
146
+ return { path, identity: identity(stat) };
147
+ } catch (error) {
148
+ if (source === "aw" && error.code === "ENOENT") return { path, identity: null };
149
+ throw error;
150
+ }
151
+ });
152
+ }
153
+
154
+ active() {
155
+ const attempt = this.queue.state.get(attemptsKind, this.queue.owner);
156
+ if (attempt !== null && (![attempt?.id, attempt?.runId, attempt?.workerNonce].every(text) ||
157
+ attempt.scope !== this.queue.current().id || attempt.plan !== this.plan ||
158
+ !Number.isSafeInteger(attempt.index) || attempt.index < 0 || attempt.index >= sources.length ||
159
+ attempt.source !== sources[attempt.index] || !text(attempt.path) || !this.validGroup(attempt.group, attempt.index) ||
160
+ attempt.group.complete !== false ||
161
+ attempt.phase !== (this.defaults && attempt.group.roots === null ? "roots" : undefined) ||
162
+ attempt.path !== this.pathFor(attempt.group, attempt.index)))
163
+ throw new Error("invalid capture discovery admission");
164
+ return attempt;
165
+ }
166
+
167
+ admitted(attempt) {
168
+ if (JSON.stringify(this.active()) !== JSON.stringify(attempt) ||
169
+ JSON.stringify(this.current().groups[attempt.index]) !== JSON.stringify(attempt.group))
170
+ throw new Error("capture discovery admission or checkpoint changed");
171
+ }
172
+
173
+ failed(attempt, reason, now) {
174
+ if (!FAILURE_REASONS.includes(reason)) throw new Error("invalid discovery interruption reason");
175
+ this.queue.state.transaction(() => {
176
+ this.admitted(attempt);
177
+ const row = this.current(), group = row.groups[attempt.index];
178
+ const attempts = (group.failure?.attempts ?? 0) + 1;
179
+ group.failure = { runId: attempt.runId, attemptedRun: attempt.runId, reason,
180
+ attempts, nextRetryAt: retryAfterAttempt(attempts, now) };
181
+ row.next = (attempt.index + 1) % sources.length;
182
+ this.queue.add([], row);
183
+ this.queue.state.set("capture-discovery-failure", "last", { runId: attempt.runId, reason,
184
+ scope: attempt.scope, source: attempt.source, path: attempt.path });
185
+ this.queue.state.delete(attemptsKind, this.queue.owner);
186
+ });
187
+ }
188
+
189
+ // Owning bootstrap has registered the new worker and ended its predecessor.
190
+ // Never infer stream/frame attribution from this metadata-only operation.
191
+ reconcile(start, { reason = "interrupted", now = Date.now() } = {}) {
192
+ if (this.queue.state.depth || !text(start?.workerNonce) || !text(start.pass?.runId) ||
193
+ !Number.isSafeInteger(now) || now < 0) throw new Error("invalid discovery replacement");
194
+ const attempt = this.active();
195
+ if (!attempt) return null;
196
+ if (attempt.workerNonce === start.workerNonce) throw new Error("capture discovery worker has not been replaced");
197
+ this.failed(attempt, reason, now);
198
+ this.clear();
199
+ return attempt;
200
+ }
201
+
202
+ publish(inputs, row, complete, attempt) {
203
+ if (!attempt) return this.queue.add(inputs, row, { complete });
204
+ this.queue.state.transaction(() => {
205
+ this.admitted(attempt);
206
+ this.queue.add(inputs, row, { complete });
207
+ this.queue.state.delete(attemptsKind, this.queue.owner);
208
+ });
209
+ }
210
+
211
+ // Only the owning session supplies run identity. Direct readers keep their
212
+ // refusal/rollback contract; discovery errors never become stream failures.
213
+ advance({ limit = 128, stopped = () => false, work, runId = null, workerNonce = null, now = Date.now(), retry = false } = {}) {
214
+ if (this.closed || this.queue.state.depth || !Number.isSafeInteger(limit) || limit < 1 || limit > 128 || typeof stopped !== "function")
215
+ throw new Error("invalid capture source discovery unit");
216
+ if (typeof retry !== "boolean" || (runId === null ? workerNonce !== null : !text(runId) || !text(workerNonce)) ||
217
+ retry && runId === null || !Number.isSafeInteger(now) || now < 0)
218
+ throw new Error("invalid capture discovery failure context");
219
+ if (this.active()) throw new Error("unreconciled capture discovery admission");
220
+ if (stopped()) return { visited: 0, added: 0, stopped: true, complete: false };
221
+ const row = this.current();
222
+ if (this.queue.current().discovered) return { visited: 0, added: 0, complete: true };
223
+ let index = -1;
224
+ for (let offset = 0; offset < sources.length; offset++) {
225
+ const candidate = (row.next + offset) % sources.length;
226
+ const group = row.groups[candidate], failure = group.failure;
227
+ if (!group.complete && (retry ? failure && failure.attemptedRun !== runId && now >= Date.parse(failure.nextRetryAt) : !failure)) {
228
+ index = candidate; break;
229
+ }
230
+ }
231
+ if (index < 0) {
232
+ if (row.groups.some(group => !group.complete)) return { visited: 0, added: 0, complete: false, deferred: true };
233
+ this.queue.add([], row, { complete: true });
234
+ return { visited: 0, added: 0, complete: true };
235
+ }
236
+ const group = row.groups[index], source = sources[index], path = this.pathFor(group, index);
237
+ const resolving = this.defaults && group.roots === null;
238
+ if (resolving && runId === null) throw new Error("capture default-root resolution requires owned discovery admission");
239
+ const attempt = runId === null ? null : { id: randomUUID(), runId, workerNonce,
240
+ scope: this.queue.current().id, plan: this.plan, index, source, path, group: structuredClone(group),
241
+ ...(resolving ? { phase: "roots" } : {}) };
242
+ if (attempt) this.queue.state.transaction(() => {
243
+ if (this.active() || JSON.stringify(this.current()) !== JSON.stringify(row))
244
+ throw new Error("capture discovery changed before admission");
245
+ this.queue.state.set(attemptsKind, this.queue.owner, attempt);
246
+ });
247
+ try {
248
+ if (resolving) {
249
+ let resolved;
250
+ try { resolved = this.resolveRoots(source); }
251
+ catch (error) {
252
+ if (attempt && !(error instanceof CaptureStop)) this.failed(attempt, "io", now);
253
+ throw error;
254
+ }
255
+ group.roots = resolved; group.complete = resolved.length === 0;
256
+ delete group.failure;
257
+ row.next = (index + 1) % sources.length;
258
+ const complete = row.groups.every(group => group.complete);
259
+ this.publish([], row, complete, attempt);
260
+ return { source, resolvedRoots: true, visited: 0, added: 0, complete };
261
+ }
262
+ let cached = this.readers.get(source);
263
+ const checkpoint = JSON.stringify(group.checkpoint);
264
+ const selectedIdentity = this.defaults ? group.roots[group.root].identity : null;
265
+ if (!cached || cached.root !== group.root || cached.checkpoint !== checkpoint) {
266
+ cached?.reader.close();
267
+ // Carry selection's witness into the first reader, not only a stat
268
+ // before construction: the reader also checks it around enumeration.
269
+ const initial = group.checkpoint ?? (selectedIdentity ? source === "aw" ?
270
+ { after: null, identity: selectedIdentity, complete: false } :
271
+ { format: source, root: path, complete: false, stack: [{ name: null, after: null, identity: selectedIdentity }] } : null);
272
+ const reader = source === "aw" ? new AwDiscovery(path, initial) : new NativeCaptureDiscovery(source, path, initial);
273
+ cached = { root: group.root, checkpoint, reader }; this.readers.set(source, cached);
274
+ }
275
+ let result;
276
+ try {
277
+ // A previously refused root is not a fresh optional default, even when
278
+ // its first page failed before establishing a directory identity.
279
+ if (retry || selectedIdentity) {
280
+ guardCapturedPath(path);
281
+ const stat = statSync(path, { bigint: true });
282
+ if (!stat.isDirectory()) throw new Error("capture discovery failed root is not a directory");
283
+ if (selectedIdentity && !sameIdentity(selectedIdentity, identity(stat)))
284
+ throw new Error("capture default-root directory identity changed");
285
+ }
286
+ result = cached.reader.advance({ limit, stopped, work });
287
+ }
288
+ catch (error) {
289
+ if (attempt && !(error instanceof CaptureStop)) {
290
+ // Keep the exact rejected checkpoint/root. Publishing the failure and
291
+ // rotation together lets a replacement reach safe independent work.
292
+ // A storage failure here still propagates: it is not a root refusal.
293
+ this.failed(attempt, "io", now);
294
+ }
295
+ throw error;
296
+ }
297
+ const inputs = result.files.map(path => ({ source, path,
298
+ stream: source === "aw" ? awStream(this.queue.owner) : `${this.queue.owner}~${source}.${SESSION_FORMATS[source].sessionId(path)}` }));
299
+ if (result.complete) {
300
+ group.root++; group.checkpoint = null;
301
+ group.complete = group.root === (this.defaults ? group.roots.length : this.roots[index].length);
302
+ delete group.failure;
303
+ } else group.checkpoint = result.checkpoint;
304
+ // Partial retry remains pending and receives at most one unit per pass.
305
+ if (group.failure) group.failure.attemptedRun = runId;
306
+ row.next = (index + 1) % sources.length;
307
+ const complete = row.groups.every(group => group.complete);
308
+ // A transaction failure must discard the advanced volatile walker. The
309
+ // next attempt reloads the last durable page, with idempotent insertion.
310
+ this.publish(inputs, row, complete, attempt);
311
+ cached.checkpoint = JSON.stringify(group.checkpoint);
312
+ if (result.complete) { cached.reader.close(); this.readers.delete(source); }
313
+ return { source, visited: result.visited, added: inputs.length, complete };
314
+ } catch (error) { this.clear(); throw error; }
315
+ }
316
+
317
+ clear() { for (const { reader } of this.readers.values()) reader.close(); this.readers.clear(); }
318
+ close() { this.clear(); this.closed = true; }
319
+ }
@@ -0,0 +1,199 @@
1
+ // Private command inventory. The caller owns the root capture lock; public
2
+ // synchronous capture never opens this queue. Pages and discovery checkpoints
3
+ // commit together. Source content/cursors remain the owning capture helpers'
4
+ // responsibility, and losing this derived queue only restarts enumeration.
5
+ import { randomUUID } from "node:crypto";
6
+ import { isAbsolute } from "node:path";
7
+
8
+ const sources = new Set(["aw", "cc", "pi", "codex"]);
9
+ const outcomes = new Set(["ready", "complete", "pending", "set-aside", "ignored", "failed"]);
10
+ const pageSize = 128;
11
+ const visitFlags = Object.freeze({ filesVisited: 1, inputsPending: 2, inputsSetAside: 4, inputsFailed: 8, inputsIgnored: 16 });
12
+ const validInput = input => input && sources.has(input.source) && typeof input.path === "string" &&
13
+ isAbsolute(input.path) && !input.path.includes("\0") && typeof input.stream === "string" && input.stream &&
14
+ (input.details === undefined || input.details && typeof input.details === "object" && !Array.isArray(input.details));
15
+
16
+ export class CaptureSourceQueue {
17
+ constructor(state, { owner, selection }) {
18
+ if (state.depth || typeof owner !== "string" || !owner || typeof selection !== "string" || !selection)
19
+ throw new Error("invalid capture source queue selection");
20
+ this.state = state; this.owner = owner; this.selection = selection;
21
+ const present = state.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name='capture_sources'").get()?.found;
22
+ if (!present) state.transaction(() => { state.db.exec(`
23
+ CREATE TABLE IF NOT EXISTS capture_sources (
24
+ scope TEXT NOT NULL, position INTEGER NOT NULL, source TEXT NOT NULL,
25
+ path TEXT NOT NULL, stream TEXT NOT NULL, details TEXT NOT NULL,
26
+ outcome TEXT NOT NULL DEFAULT 'ready', PRIMARY KEY(scope,position), UNIQUE(scope,source,path)
27
+ );
28
+ CREATE INDEX IF NOT EXISTS capture_sources_ready ON capture_sources(scope,outcome,position);
29
+ CREATE INDEX IF NOT EXISTS capture_sources_stream ON capture_sources(scope,stream,outcome,position);
30
+ `);
31
+ // A missing inventory table cannot inherit a completed discovery cursor.
32
+ // Re-enumeration is safe: journal IDs/source proofs still own dedupe.
33
+ state.db.exec("DELETE FROM state WHERE kind IN ('source-sweep','source-queue-garbage')");
34
+ });
35
+ // Private inventory metadata, never an outgoing-release source cursor.
36
+ // Additive columns also accept queues created by an earlier service build.
37
+ const columns = new Set(state.db.prepare("PRAGMA table_info(capture_sources)").all().map(column => column.name));
38
+ if (!columns.has("visited_run") || !columns.has("visited_flags")) state.transaction(() => {
39
+ if (!columns.has("visited_run")) state.db.exec("ALTER TABLE capture_sources ADD COLUMN visited_run TEXT");
40
+ if (!columns.has("visited_flags")) state.db.exec("ALTER TABLE capture_sources ADD COLUMN visited_flags INTEGER NOT NULL DEFAULT 0");
41
+ });
42
+ const prior = state.get("source-sweep", owner);
43
+ if (prior && (!prior.id || prior.owner !== owner || typeof prior.selection !== "string" ||
44
+ !Number.isSafeInteger(prior.next) || prior.next < 1 || typeof prior.discovered !== "boolean" ||
45
+ typeof prior.finished !== "boolean")) throw new Error("invalid capture source sweep");
46
+ if (prior && prior.selection === selection && !prior.finished) this.sweep = prior;
47
+ else {
48
+ this.sweep = { id: randomUUID(), owner, selection, next: 1, discovered: false, finished: false, discovery: null };
49
+ state.transaction(() => {
50
+ if (prior) state.set("source-queue-garbage", prior.id, true);
51
+ state.set("source-sweep", owner, this.sweep);
52
+ });
53
+ }
54
+ }
55
+
56
+ current() {
57
+ const row = this.state.get("source-sweep", this.owner);
58
+ if (row?.id !== this.sweep.id) throw new Error("capture source sweep changed");
59
+ return row;
60
+ }
61
+
62
+ // The walker may conservatively replay a basename after a removed anchor.
63
+ // Duplicate discovery never renumbers an input or erases completed work.
64
+ add(inputs, checkpoint, { complete = false } = {}) {
65
+ if (!Array.isArray(inputs) || inputs.length > pageSize || inputs.some(input => !validInput(input)) || typeof complete !== "boolean")
66
+ throw new Error("invalid capture discovery page");
67
+ this.state.transaction(() => {
68
+ const row = this.current();
69
+ if (row.discovered || row.finished) throw new Error("capture source discovery already completed");
70
+ const insert = this.state.db.prepare(`INSERT INTO capture_sources(scope,position,source,path,stream,details)
71
+ VALUES (?,?,?,?,?,?) ON CONFLICT(scope,source,path) DO NOTHING`);
72
+ for (const input of inputs) {
73
+ const existing = this.state.db.prepare("SELECT stream,details FROM capture_sources WHERE scope=? AND source=? AND path=?")
74
+ .get(row.id, input.source, input.path);
75
+ const details = JSON.stringify(input.details ?? {});
76
+ // Early node:sqlite can return an object of nulls for no match.
77
+ // stream is NOT NULL for every real inventory row.
78
+ if (existing?.stream != null && (existing.stream !== input.stream || existing.details !== details)) throw new Error("capture input changed during discovery");
79
+ if (existing?.stream == null) {
80
+ if (!Number.isSafeInteger(row.next + 1)) throw new Error("capture source inventory position overflow");
81
+ insert.run(row.id, row.next++, input.source, input.path, input.stream, details);
82
+ }
83
+ }
84
+ row.discovery = checkpoint; row.discovered = complete;
85
+ this.state.set("source-sweep", this.owner, row);
86
+ this.sweep = row;
87
+ });
88
+ }
89
+
90
+ // Admission writes this cursor BEFORE processing; a rolled-back append must
91
+ // not force every new worker back to the first growing input. Each stage has
92
+ // its own position, so index work cannot erase source-stage fairness.
93
+ cursor(position) { return `${this.sweep.id}:${position}`; }
94
+ position(cursor) {
95
+ if (typeof cursor !== "string" || !cursor.startsWith(`${this.sweep.id}:`)) return 0;
96
+ const n = Number(cursor.slice(this.sweep.id.length + 1));
97
+ if (!Number.isSafeInteger(n) || n < 0) throw new Error("invalid capture source queue cursor");
98
+ return n;
99
+ }
100
+
101
+ *ready({ stopped = () => false, work } = {}) {
102
+ const row = this.current();
103
+ const after = this.position(this.state.get("capture-schedule", "position:capture")?.after);
104
+ // Native collision precedence is unchanged: an earlier source sharing its
105
+ // stream must finish its fixed snapshot (or reach pending/refusal) before
106
+ // a later one can run. AW alone may interleave across its shared stream.
107
+ for (const later of [true, false]) {
108
+ let last = later ? after : 0;
109
+ while (!stopped()) {
110
+ const page = this.state.db.prepare(`SELECT q.position,q.source,q.path,q.stream,q.details,q.outcome
111
+ FROM capture_sources q WHERE q.scope=? AND q.outcome='ready' AND q.position>?
112
+ ${later ? "" : "AND q.position<=?"}
113
+ AND (q.source='aw' OR NOT EXISTS (
114
+ SELECT 1 FROM capture_sources prior WHERE prior.scope=q.scope AND prior.stream=q.stream
115
+ AND prior.outcome='ready' AND prior.position<q.position))
116
+ ORDER BY q.position LIMIT ${pageSize}`).all(...(later ? [row.id, last] : [row.id, last, after]));
117
+ if (work) work.inventoryRows = (work.inventoryRows ?? 0) + page.length;
118
+ if (!page.length) break;
119
+ for (const input of page) {
120
+ if (stopped()) return;
121
+ last = input.position;
122
+ yield { ...input, details: JSON.parse(input.details), next: { stage: "capture", after: this.cursor(input.position) } };
123
+ }
124
+ }
125
+ }
126
+ }
127
+
128
+ // Use inside the SAME processing transaction as the input's committed
129
+ // source checkpoint. This is scheduling state, never append/proof authority.
130
+ mark(position, outcome) {
131
+ if (!this.state.depth || !Number.isSafeInteger(position) || position < 1 || !outcomes.has(outcome))
132
+ throw new Error("capture source result requires its processing transaction");
133
+ this.current();
134
+ const changed = this.state.db.prepare("UPDATE capture_sources SET outcome=? WHERE scope=? AND position=? AND outcome!=?")
135
+ .run(outcome, this.sweep.id, position, outcome);
136
+ if (!changed.changes && !this.state.db.prepare("SELECT 1 AS found FROM capture_sources WHERE scope=? AND position=?").get(this.sweep.id, position)?.found)
137
+ throw new Error("capture source queue input is missing");
138
+ }
139
+
140
+ // Count each input/category at most once in a run even when its recovery or
141
+ // append spans many workers. These facts commit with the source checkpoint.
142
+ visit(position, runId, categories = {}) {
143
+ if (!this.state.depth || typeof runId !== "string" || !runId ||
144
+ !Number.isSafeInteger(position) || position < 1 ||
145
+ Object.keys(categories).some(key => !Object.hasOwn(visitFlags, key) || typeof categories[key] !== "boolean"))
146
+ throw new Error("capture input visits require valid committed accounting");
147
+ this.current();
148
+ const row = this.state.db.prepare("SELECT visited_run,visited_flags FROM capture_sources WHERE scope=? AND position=?")
149
+ .get(this.sweep.id, position);
150
+ if (!row || !Number.isSafeInteger(row.visited_flags) || row.visited_flags < 0 || row.visited_flags > 31)
151
+ throw new Error("capture input visit state is missing or invalid");
152
+ const before = row.visited_run === runId ? row.visited_flags : 0;
153
+ let flags = before; const delta = {};
154
+ for (const [name, flag] of Object.entries(visitFlags)) if (name === "filesVisited" || categories[name]) {
155
+ if (!(flags & flag)) { flags |= flag; delta[name] = 1; }
156
+ }
157
+ if (row.visited_run !== runId || flags !== before) this.state.db
158
+ .prepare("UPDATE capture_sources SET visited_run=?,visited_flags=? WHERE scope=? AND position=?")
159
+ .run(runId, flags, this.sweep.id, position);
160
+ return delta;
161
+ }
162
+
163
+ // Used only for a due single-stream retry. Native precedence remains the
164
+ // earliest unfinished source; AW resumes after the durable account position.
165
+ forStream(stream) {
166
+ const after = this.position(this.state.get("capture-schedule", "position:capture")?.after);
167
+ const row = this.state.db.prepare(`SELECT position,source,path,stream,details,outcome FROM capture_sources
168
+ WHERE scope=? AND stream=? AND outcome='ready'
169
+ ORDER BY CASE WHEN source='aw' AND position<=? THEN 1 ELSE 0 END,position LIMIT 1`)
170
+ .get(this.sweep.id, stream, after);
171
+ return row?.position != null ? { ...row, details: JSON.parse(row.details), next: { stage: "capture", after: this.cursor(row.position) } } : null;
172
+ }
173
+
174
+ finish() {
175
+ if (!this.state.depth) throw new Error("capture source sweep finish requires a transaction");
176
+ const row = this.current();
177
+ if (!row.discovered || this.state.db.prepare("SELECT 1 AS found FROM capture_sources WHERE scope=? AND outcome='ready' LIMIT 1").get(row.id)?.found)
178
+ return false;
179
+ row.finished = true; this.state.set("source-sweep", this.owner, row); this.sweep = row;
180
+ // Finished enumeration/visits do NOT imply a successful pass: pending,
181
+ // set-aside and failed outcomes still need the worker's truthful receipt.
182
+ return true;
183
+ }
184
+
185
+ // No table-wide reset or scan of every current row. Replacing a sweep
186
+ // atomically publishes its exact obsolete scope; cleanup seeks that scope
187
+ // and removes at most one page. Other owners' active scopes are untouched.
188
+ cleanup() {
189
+ if (!this.state.depth) throw new Error("capture source cleanup requires a transaction");
190
+ const obsolete = this.state.db.prepare("SELECT key FROM state WHERE kind='source-queue-garbage' ORDER BY key LIMIT 1").get()?.key;
191
+ if (obsolete == null) return 0;
192
+ if (obsolete === this.current().id) throw new Error("active capture source inventory cannot be deleted");
193
+ const count = Number(this.state.db.prepare(`DELETE FROM capture_sources WHERE rowid IN (
194
+ SELECT rowid FROM capture_sources WHERE scope=? ORDER BY position LIMIT ${pageSize})`).run(obsolete).changes);
195
+ if (!this.state.db.prepare("SELECT 1 AS found FROM capture_sources WHERE scope=? LIMIT 1").get(obsolete)?.found)
196
+ this.state.delete("source-queue-garbage", obsolete);
197
+ return count;
198
+ }
199
+ }
@@ -0,0 +1,85 @@
1
+ // One admitted source group in the command worker. This joins queue progress
2
+ // and run counters to durable capture checkpoints; discovery/index completion
3
+ // and final receipts remain the owning scheduler's responsibility.
4
+ import { captureAwUnit } from "./capture-aw-unit.mjs";
5
+ import { captureNativeUnit } from "./capture-native-unit.mjs";
6
+ import { withCaptureGroup, captureContext } from "./capture-context.mjs";
7
+ import { selectCaptureWork, resolveCaptureRetry } from "./capture-retry.mjs";
8
+ import { readCaptureRetry } from "./capture-admission.mjs";
9
+ import { fileVersion } from "./derived-state.mjs";
10
+ import { loadIgnore } from "./ignore.mjs";
11
+
12
+ export function captureSourceWork(store, state, queue, run, start, {
13
+ limit = 128, maxRecords = 2000, maxBytes = 8388608, ignore = loadIgnore(store.root),
14
+ now = Date.now(), work, indexInventory = null, allowRetry = true, retryFilter, inputReady = () => true,
15
+ } = {}) {
16
+ if (!Number.isSafeInteger(limit) || limit < 1 || limit > 128 ||
17
+ !Number.isSafeInteger(maxRecords) || maxRecords < limit || maxRecords > 2000 ||
18
+ !Number.isSafeInteger(maxBytes) || maxBytes < limit || maxBytes > 8388608 || typeof inputReady !== "function")
19
+ throw new Error("invalid capture source work allowance");
20
+ if (indexInventory !== null && (indexInventory.state !== state || typeof indexInventory.include !== "function"))
21
+ throw new Error("capture source index inventory must belong to the current state");
22
+ const budget = start.budget, stopped = () => budget.sample() !== null;
23
+ if (stopped()) return { stopped: true, selected: 0, cleanup: [] };
24
+ const inputs = new Map();
25
+ function* candidates() {
26
+ for (const input of queue.ready({ stopped, work })) {
27
+ if (readCaptureRetry(state, input.stream) || inputs.has(input.stream) || !inputReady(input)) continue;
28
+ // At most one file from a shared stream belongs to an admission. The
29
+ // durable queue position rotates AW accounts between such admissions.
30
+ inputs.set(input.stream, input);
31
+ const generation = fileVersion(store.journalPath(input.stream));
32
+ yield { stream: input.stream, stage: "capture", offset: generation?.size ?? 0, generation, next: input.next };
33
+ }
34
+ }
35
+ const selection = selectCaptureWork(store, state, candidates(), { runId: run.id, now, limit, allowRetry, retryFilter, stopped });
36
+ if (!selection) return { selected: 0, cleanup: [] };
37
+ // Retry selection spans stages. Return other-stage work to the scheduler;
38
+ // never execute it as capture or clear its failure through the wrong path.
39
+ if (selection.stage !== "capture") return { selected: 0, deferred: selection, cleanup: [] };
40
+ if (selection.retry) {
41
+ const input = queue.forStream(selection.members[0].stream);
42
+ if (!input || !inputReady(input)) return { selected: 0, deferred: selection, cleanup: [] };
43
+ inputs.set(input.stream, input);
44
+ selection.next = input.next;
45
+ }
46
+ const recordsEach = Math.floor(maxRecords / selection.members.length);
47
+ const bytesEach = Math.floor(maxBytes / selection.members.length);
48
+ return withCaptureGroup(store, selection.members.map(member => member.stream), () => {
49
+ const results = [], cleanup = [];
50
+ for (const member of selection.members) {
51
+ if (stopped()) break;
52
+ const input = inputs.get(member.stream), before = state.db.prepare("SELECT total_changes() AS n").get().n;
53
+ const options = { owner: queue.owner, path: input.path, ignore, maxRecords: recordsEach,
54
+ maxBytes: bytesEach, maxRecordBytes: budget.policy.maxRecordBytes, stopped, work: work?.content };
55
+ const result = input.source === "aw"
56
+ ? captureAwUnit(store, { ...options, kind: input.details.kind,
57
+ selfName: input.details.selfName, workspace: input.details.workspace })
58
+ : captureNativeUnit(store, { ...options, format: input.source, capturedPi: input.details.capturedPi, start });
59
+ // Inspect mutation before visits/counters: repeatedly visiting an
60
+ // unchanged or deferred input is not processing progress.
61
+ const progressed = state.db.prepare("SELECT total_changes() AS n").get().n !== before;
62
+ const outcome = result.ignored ? "ignored" : result.setAside ? "set-aside" :
63
+ result.pending ? "pending" : result.complete ? (result.failed ? "failed" : "complete") : "ready";
64
+ queue.mark(input.position, outcome);
65
+ const delta = queue.visit(input.position, run.id, { inputsIgnored: !!result.ignored,
66
+ inputsSetAside: !!result.setAside, inputsPending: !!result.pending, inputsFailed: !!result.failed });
67
+ run.record({ ...delta, bytesCommitted: result.bytesCommitted, turnsAppended: result.appended },
68
+ { progress: progressed, peakRssBytes: budget.peakRssBytes });
69
+ if (selection.retry && result.complete) {
70
+ // Completing the fixed input validates that this source admission no
71
+ // longer needs work. A partial prefix proof cannot clear the failure.
72
+ resolveCaptureRetry(state, { id: captureContext(store).admission.id, stream: member.stream, reconciled: true });
73
+ }
74
+ if (result.cleanup) cleanup.push({ stream: member.stream, key: result.key });
75
+ results.push({ position: input.position, ...result });
76
+ }
77
+ // Include every committed input, including replay with no new append.
78
+ // Journal bytes may survive an earlier rolled-back source transaction.
79
+ if (indexInventory) indexInventory.include(results.map(result => result.stream));
80
+ // An input that took the unchanged path touched neither its journal nor its
81
+ // checkpoint; every other one may have.
82
+ if (indexInventory) indexInventory.changed(results.filter(result => !result.unchanged).map(result => result.stream));
83
+ return { selected: selection.members.length, results, cleanup };
84
+ }, { state, admission: selection, admissionNow: now, batchBytes: maxBytes, check: () => budget.check() });
85
+ }