@awebai/oats 0.60.0 → 0.62.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/README.md +15 -19
  2. package/bin/oats.mjs +68 -21
  3. package/docs/capabilities.md +1 -1
  4. package/docs/configuration.md +9 -0
  5. package/docs/desktop-cli-api.md +28 -5
  6. package/docs/desktop.md +7 -0
  7. package/docs/execution-targets.md +49 -3
  8. package/docs/first-team.md +8 -0
  9. package/docs/implementation.md +2 -1
  10. package/docs/named-test-runs.md +382 -0
  11. package/docs/official-catalog.md +1 -1
  12. package/docs/packages.md +5 -5
  13. package/docs/release-notes/v0.61.0.md +155 -0
  14. package/docs/release-notes/v0.62.0.md +223 -0
  15. package/docs/tui.md +10 -4
  16. package/docs/upgrading.md +133 -11
  17. package/docs/workspaces.md +1 -1
  18. package/lib/core.mjs +2 -2
  19. package/lib/doctor-capture.mjs +35 -0
  20. package/lib/session-viewer.mjs +86 -7
  21. package/package-catalog.json +1 -1
  22. package/package.json +1 -1
  23. package/packages/client/cli-locator.mjs +2 -2
  24. package/packages/client/instance-presentation.mjs +10 -0
  25. package/packages/client/instance-tree.mjs +48 -6
  26. package/packages/record/README.md +87 -13
  27. package/packages/record/bin/capture-hook.mjs +8 -0
  28. package/packages/record/bin/capture.mjs +143 -124
  29. package/packages/record/bin/recall.mjs +8 -3
  30. package/packages/record/bin/setup.mjs +115 -225
  31. package/packages/record/bin/turn-record.mjs +2 -2
  32. package/packages/record/docs/capture-service.md +1443 -28
  33. package/packages/record/docs/capture-status.schema.json +429 -12
  34. package/packages/record/lib/capture-admission.mjs +82 -0
  35. package/packages/record/lib/capture-aw-unit.mjs +137 -0
  36. package/packages/record/lib/capture-aw.mjs +31 -12
  37. package/packages/record/lib/capture-budget.mjs +31 -0
  38. package/packages/record/lib/capture-cc.mjs +39 -6
  39. package/packages/record/lib/capture-cleanup-queue.mjs +135 -0
  40. package/packages/record/lib/capture-cleanup-work.mjs +77 -0
  41. package/packages/record/lib/capture-command-operation.mjs +37 -0
  42. package/packages/record/lib/capture-command-output.mjs +61 -0
  43. package/packages/record/lib/capture-command-worker.mjs +6 -0
  44. package/packages/record/lib/capture-command.mjs +48 -0
  45. package/packages/record/lib/capture-completion.mjs +152 -0
  46. package/packages/record/lib/capture-context.mjs +50 -0
  47. package/packages/record/lib/capture-default-roots.mjs +30 -0
  48. package/packages/record/lib/capture-discovery.mjs +114 -0
  49. package/packages/record/lib/capture-failures.mjs +45 -0
  50. package/packages/record/lib/capture-health-doctor.mjs +13 -0
  51. package/packages/record/lib/capture-health-installation.mjs +117 -0
  52. package/packages/record/lib/capture-health.mjs +116 -0
  53. package/packages/record/lib/capture-hook-ownership.mjs +58 -0
  54. package/packages/record/lib/capture-hook-runner.mjs +17 -0
  55. package/packages/record/lib/capture-hook.mjs +26 -0
  56. package/packages/record/lib/capture-index-inventory.mjs +205 -0
  57. package/packages/record/lib/capture-index-outcomes.mjs +113 -0
  58. package/packages/record/lib/capture-index-reset.mjs +109 -0
  59. package/packages/record/lib/capture-index-work.mjs +77 -0
  60. package/packages/record/lib/capture-issue-preparation.mjs +73 -0
  61. package/packages/record/lib/capture-issues.mjs +15 -2
  62. package/packages/record/lib/capture-lock.mjs +102 -16
  63. package/packages/record/lib/capture-native-unit.mjs +322 -0
  64. package/packages/record/lib/capture-pass.mjs +102 -0
  65. package/packages/record/lib/capture-recovery-snapshot.mjs +182 -0
  66. package/packages/record/lib/capture-retained-scratch.mjs +213 -0
  67. package/packages/record/lib/capture-retry.mjs +102 -0
  68. package/packages/record/lib/capture-run.mjs +125 -0
  69. package/packages/record/lib/capture-scratch.mjs +12 -3
  70. package/packages/record/lib/capture-service-definition.mjs +104 -0
  71. package/packages/record/lib/capture-service-install.mjs +362 -0
  72. package/packages/record/lib/capture-service-ownership.mjs +137 -0
  73. package/packages/record/lib/capture-service-runner.mjs +18 -0
  74. package/packages/record/lib/capture-small-snapshot.mjs +85 -0
  75. package/packages/record/lib/capture-source-discovery.mjs +319 -0
  76. package/packages/record/lib/capture-source-queue.mjs +199 -0
  77. package/packages/record/lib/capture-source-work.mjs +85 -0
  78. package/packages/record/lib/capture-state.mjs +16 -71
  79. package/packages/record/lib/capture-status.mjs +171 -0
  80. package/packages/record/lib/capture-supervisor.mjs +117 -0
  81. package/packages/record/lib/capture-watch.mjs +32 -0
  82. package/packages/record/lib/capture-work-session.mjs +238 -0
  83. package/packages/record/lib/capture-worker.mjs +58 -0
  84. package/packages/record/lib/derived-state.mjs +28 -15
  85. package/packages/record/lib/file-lines.mjs +22 -6
  86. package/packages/record/lib/formats.mjs +4 -3
  87. package/packages/record/lib/index-capture.mjs +97 -0
  88. package/packages/record/lib/index-db.mjs +3 -17
  89. package/packages/record/lib/index-reset.mjs +19 -0
  90. package/packages/record/lib/store.mjs +36 -11
  91. package/packages/record/lib/transcript-prefix.mjs +125 -0
  92. package/packages/tui/lib/client.mjs +2 -2
  93. package/packages/tui/lib/model.mjs +15 -5
  94. package/packages/tui/lib/views/palette.mjs +7 -3
  95. package/skills/oats-getting-started/SKILL.md +1 -1
  96. package/docs/macos-test-runs.md +0 -246
@@ -20,6 +20,7 @@ import { basename, join } from "node:path";
20
20
  import { homedir } from "node:os";
21
21
  import { createHash } from "node:crypto";
22
22
  import { DerivedState, statVersion, sameVersion } from "./derived-state.mjs";
23
+ import { captureContext } from "./capture-context.mjs";
23
24
 
24
25
  import {
25
26
  projectCommLogEntry,
@@ -50,32 +51,40 @@ export function listCommLogs(dir = defaultCommLogDir()) {
50
51
  // does not, it is a torn tail the client is still writing, never counted. An
51
52
  // interior bad line is skipped and counted (`onSkipped`), never fatal.
52
53
  function* logEntries(fd, start, end, onSkipped, onConsumed) {
54
+ for (const record of awLogLines(fileChunks(fd, { start, end, size: 65536 }), { start })) {
55
+ if (!record.final) onConsumed(record.end);
56
+ if (record.skipped) onSkipped();
57
+ if (record.parsed) yield record;
58
+ }
59
+ }
60
+
61
+ // Command units must account for every scanned line, including empty/malformed
62
+ // ones. Public capture's wrapper above retains its existing entry counters.
63
+ export function* awLogLines(chunks, { start = 0, maxBytes = Infinity, frontier = 0 } = {}) {
53
64
  let offset = start;
54
- for (const line of bufferLines(fileChunks(fd, { start, end, size: 65536 }))) {
65
+ for (const line of bufferLines(chunks, { maxBytes, limitFrom: Math.max(0, frontier - start) })) {
55
66
  const lineStart = offset;
56
67
  offset += line.length;
57
68
  const final = line[line.length - 1] !== 10;
58
69
  // Even ignored whitespace is scanned progress when newline-terminated.
59
70
  // The caller persists it only after its pending append batch is durable.
60
- if (!final) onConsumed(offset);
61
- let entry;
71
+ const record = { start: lineStart, end: final ? lineStart : offset, bytes: line.length, final };
62
72
  try {
63
73
  // Inside the try: a line beyond V8's string limit is a bad line like any other.
64
74
  const text = line.toString("utf8");
65
- if (text.trim() === "") continue;
66
- entry = JSON.parse(text);
75
+ if (text.trim() === "") { yield record; continue; }
76
+ const entry = JSON.parse(text);
77
+ yield { ...record, entry, parsed: true };
67
78
  } catch {
68
- if (!final) onSkipped();
69
- continue;
79
+ yield { ...record, skipped: !final };
70
80
  }
71
- yield { entry, start: lineStart, end: final ? lineStart : offset, bytes: line.length };
72
81
  }
73
82
  }
74
83
 
75
84
  const APPEND_BATCH = 2000;
76
85
 
77
86
  // Check only a bounded checkpoint boundary under the append-only source contract.
78
- function boundaryWitness(fd, offset) {
87
+ export function awBoundaryWitness(fd, offset) {
79
88
  const size = Math.min(offset, 4096), bytes = Buffer.alloc(size);
80
89
  let done = 0;
81
90
  while (done < size) {
@@ -86,6 +95,8 @@ function boundaryWitness(fd, offset) {
86
95
  return createHash("sha256").update(bytes).digest("hex");
87
96
  }
88
97
 
98
+ const boundaryWitness = awBoundaryWitness;
99
+
89
100
  function captureLogFile(store, { streamId, path, project, knownIds, incremental = false }) {
90
101
  const fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK);
91
102
  let state;
@@ -93,7 +104,7 @@ function captureLogFile(store, { streamId, path, project, knownIds, incremental
93
104
  const stat = fstatSync(fd, { bigint: true });
94
105
  if (!stat.isFile()) throw new Error(`log source is not a regular file: ${path}`);
95
106
  const version = statVersion(stat), end = version.size;
96
- state = new DerivedState(store.root);
107
+ state = captureContext(store)?.state ?? new DerivedState(store.root);
97
108
  const generation = state.get("reset", streamId.split("~")[0] + ":aw") ?? "";
98
109
  const key = `${streamId}:${path}`;
99
110
  let previous = incremental ? state.get("aw", key) : null;
@@ -120,7 +131,15 @@ function captureLogFile(store, { streamId, path, project, knownIds, incremental
120
131
  }
121
132
  checkpoint();
122
133
  };
134
+ const batchLimit = captureContext(store)?.batchBytes ?? 8 * 1024 * 1024;
123
135
  for (const record of logEntries(fd, start, end, () => skipped++, (offset) => { consumed = offset; })) {
136
+ if (batch.length && batchBytes + record.bytes > batchLimit) {
137
+ // The parser has reported this complete line as scanned, but it has
138
+ // not yet been appended. An earlier batch may checkpoint only up to
139
+ // this line's START, including intervening consumed blank lines.
140
+ const scanned = consumed; consumed = record.start;
141
+ flush(); consumed = scanned;
142
+ }
124
143
  entries++;
125
144
  let turn;
126
145
  // Projection refusal consumes this complete entry just like a skipped
@@ -131,14 +150,14 @@ function captureLogFile(store, { streamId, path, project, knownIds, incremental
131
150
  catch { failed++; continue; }
132
151
  if (knownIds) { if (knownIds.has(turn.id)) continue; knownIds.add(turn.id); }
133
152
  batch.push(turn); batchBytes += record.bytes;
134
- if (batch.length >= APPEND_BATCH || batchBytes >= 8 * 1024 * 1024) flush();
153
+ if (batch.length >= APPEND_BATCH || batchBytes >= batchLimit) flush();
135
154
  }
136
155
  // Complete refused/malformed/blank lines are consumed; an unfinished final
137
156
  // fragment keeps its starting boundary so later growth can finish it.
138
157
  complete = true;
139
158
  flush();
140
159
  return { entries, appended, skipped, failed };
141
- } finally { try { state?.close(); } finally { closeSync(fd); } }
160
+ } finally { try { if (!captureContext(store)) state?.close(); } finally { closeSync(fd); } }
142
161
  }
143
162
 
144
163
  // Capture one comm-log file into `<owner>~aw`. The account name is the
@@ -0,0 +1,31 @@
1
+ // Fixed command policy. Alternate values are an internal test seam, never CLI
2
+ // tuning flags or evidence that unbounded public receipt operations are bounded.
3
+ import { performance } from "node:perf_hooks";
4
+
5
+ export const CAPTURE_POLICY = Object.freeze({ intervalSeconds: 900, heapMiB: 256,
6
+ rssBytes: 2147483648, deadlineSeconds: 300, maxRecordBytes: 8388608 });
7
+
8
+ export class CaptureStop extends Error {
9
+ constructor(reason) { super(`capture stopped: ${reason}`); this.reason = reason; }
10
+ }
11
+
12
+ export class CaptureBudget {
13
+ constructor({ policy = CAPTURE_POLICY, clock = () => performance.now(),
14
+ rss = () => process.memoryUsage().rss, supervisorAlive = () => true, drainSeconds = 5, drainAt } = {}) {
15
+ this.policy = policy; this.clock = clock; this.rss = rss;
16
+ this.supervisorAlive = supervisorAlive; this.started = clock();
17
+ this.drainAt = drainAt ?? this.started + Math.max(0, policy.deadlineSeconds - drainSeconds) * 1000;
18
+ this.peakRssBytes = null; this.reason = null;
19
+ }
20
+ sample() {
21
+ const rss = this.rss();
22
+ this.peakRssBytes = Math.max(this.peakRssBytes ?? 0, rss);
23
+ if (!this.reason) {
24
+ if (!this.supervisorAlive()) this.reason = "interrupted";
25
+ else if (rss > this.policy.rssBytes) this.reason = "rss";
26
+ else if (this.clock() >= this.drainAt) this.reason = "deadline";
27
+ }
28
+ return this.reason;
29
+ }
30
+ check() { const reason = this.sample(); if (reason) throw new CaptureStop(reason); }
31
+ }
@@ -21,6 +21,7 @@
21
21
  // un-ignoring a file later makes the next pass capture it normally.
22
22
 
23
23
  import { isUtf8 } from "node:buffer";
24
+ import { createHash } from "node:crypto";
24
25
  import { closeSync, constants, fstatSync, openSync } from "node:fs";
25
26
  import { basename } from "node:path";
26
27
 
@@ -33,6 +34,7 @@ import { bufferLines } from "./file-lines.mjs";
33
34
  import { DerivedState, fileVersion, sameVersion, statVersion } from "./derived-state.mjs";
34
35
  import { compareTranscriptPrefix } from "./transcript-prefix.mjs";
35
36
  import { isDeepStrictEqual } from "node:util";
37
+ import { captureContext } from "./capture-context.mjs";
36
38
 
37
39
  export const SESSION_STREAM_SOURCE = "cc";
38
40
 
@@ -91,10 +93,10 @@ const FLUSH_BYTES = 8 * 1024 * 1024;
91
93
  // replacement characters would change both the verbatim line and its byte
92
94
  // offset, possibly treating a later fragment as a record) or a line beyond
93
95
  // V8's string limit.
94
- function* completeLines(chunks, walk) {
96
+ export function* completeNativeLines(chunks, walk, { maxBytes = Infinity } = {}) {
95
97
  let scanned = 0;
96
98
  walk.scanned = 0;
97
- for (const line of bufferLines(chunks)) {
99
+ for (const line of bufferLines(chunks, { maxBytes })) {
98
100
  if (line[line.length - 1] !== 10) { walk.reason = "torn-tail"; return; }
99
101
  const bytes = line.subarray(0, -1);
100
102
  if (!isUtf8(bytes)) { walk.reason = "invalid-utf8"; return; }
@@ -121,6 +123,33 @@ function* completeLines(chunks, walk) {
121
123
  }
122
124
  }
123
125
 
126
+ // The digests of completeNativeLines' lines, for proving an already captured
127
+ // prefix: SHA-256 of each line without its newline, streamed over the chunks
128
+ // with no join, no decode to a string and no size limit, so a captured record
129
+ // of any size is compared without being held whole. Stops exactly where
130
+ // completeNativeLines stops, with the same walk.reason.
131
+ export function* completeNativeLineDigests(chunks, walk) {
132
+ let hash = createHash("sha256"), utf8 = new TextDecoder("utf-8", { fatal: true }), bytes = 0, scanned = 0, valid = true;
133
+ const check = (piece, stream) => { if (valid) try { utf8.decode(piece, { stream }); } catch { valid = false; } };
134
+ walk.scanned = 0;
135
+ for (const chunk of chunks) {
136
+ for (let from = 0; from < chunk.length;) {
137
+ const newline = chunk.indexOf(10, from), end = newline < 0 ? chunk.length : newline;
138
+ const piece = chunk.subarray(from, end);
139
+ hash.update(piece); check(piece, true); bytes += piece.length;
140
+ if (newline < 0) break;
141
+ check(undefined, false);
142
+ if (!valid) { walk.reason = "invalid-utf8"; return; }
143
+ scanned += bytes + 1; walk.scanned = scanned;
144
+ yield { hash: hash.digest("hex"), bytes: bytes + 1 };
145
+ hash = createHash("sha256"); utf8 = new TextDecoder("utf-8", { fatal: true }); bytes = 0; from = newline + 1;
146
+ }
147
+ }
148
+ if (bytes > 0) walk.reason = "torn-tail";
149
+ }
150
+
151
+ const completeLines = completeNativeLines;
152
+
124
153
  // One reconciliation pass for one format: one turn per NEW complete line
125
154
  // of every session file under `roots`. Unstamped leading lines are held
126
155
  // until the file shows its first timestamp (then they carry it backward),
@@ -140,9 +169,9 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
140
169
  const fmt = SESSION_FORMATS[format];
141
170
  if (!fmt) throw new Error(`unknown session format ${format}`);
142
171
  const ign = ignore ?? loadIgnore(store.root);
143
- const offsets = new DerivedState(store.root);
172
+ const context = captureContext(store), offsets = context?.state ?? new DerivedState(store.root);
144
173
  try {
145
- offsets.db.exec("CREATE TEMP TABLE captured_streams (stream TEXT PRIMARY KEY)");
174
+ offsets.db.exec("CREATE TEMP TABLE IF NOT EXISTS captured_streams (stream TEXT PRIMARY KEY); DELETE FROM captured_streams");
146
175
  const markStream = offsets.db.prepare("INSERT OR IGNORE INTO captured_streams VALUES (?)");
147
176
  let sessions = 0;
148
177
  let appended = 0;
@@ -299,7 +328,11 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
299
328
  let lineNo = state.line;
300
329
  let consumed = 0;
301
330
  const walk = {};
331
+ const batchBytes = context?.batchBytes ?? FLUSH_BYTES;
302
332
  for (const l of completeLines(chunks(state.bytes), walk)) {
333
+ // Commit the earlier batch BEFORE its cursor/timestamp can include the
334
+ // next record. Public callers still accept a larger individual record.
335
+ if (fresh.length && freshBytes + l.bytes > batchBytes) flush();
303
336
  if (l.ts) lastTs = l.ts;
304
337
  lineNo++;
305
338
  consumed += l.bytes;
@@ -311,7 +344,7 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
311
344
  ),
312
345
  );
313
346
  freshBytes += l.bytes;
314
- if (fresh.length >= 2000 || freshBytes >= FLUSH_BYTES) flush();
347
+ if (fresh.length >= 2000 || freshBytes >= batchBytes) flush();
315
348
  }
316
349
  stopped(walk);
317
350
  flush();
@@ -331,7 +364,7 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
331
364
  streams,
332
365
  stream: `${owner}~${fmt.source}.*`,
333
366
  };
334
- } finally { offsets.close(); }
367
+ } finally { if (!context) offsets.close(); }
335
368
  }
336
369
 
337
370
  // One pass over every known format at its default roots. The ignore list
@@ -0,0 +1,135 @@
1
+ // Allocation-bound housekeeping admission, independent of journal-unit retries.
2
+ // Bootstrap owns the root lock and has ended the preceding worker. Neither a
3
+ // scan endpoint nor successful deletion proves source/index/pass completion.
4
+ import { randomUUID } from "node:crypto";
5
+ import { captureCleanupPage, cleanupCaptureRecovery } from "./capture-cleanup-work.mjs";
6
+ import { readCaptureAdmission } from "./capture-admission.mjs";
7
+ import { FAILURE_REASONS, retryAfterAttempt } from "./capture-failures.mjs";
8
+ import { CaptureStop } from "./capture-budget.mjs";
9
+
10
+ const kind = "capture-cleanup", retries = "capture-cleanup-retry";
11
+ const text = value => typeof value === "string" && value.length > 0;
12
+ const validJob = job => text(job?.key) && text(job.stream) && /^[a-f0-9]{32}$/.test(job.allocation);
13
+ const validDate = value => typeof value === "string" && Number.isFinite(Date.parse(value)) && new Date(value).toISOString() === value;
14
+
15
+ export class CaptureCleanupQueue {
16
+ constructor(store, state) { this.store = store; this.state = state; }
17
+
18
+ active() {
19
+ const row = this.state.get(kind, "active");
20
+ if (row !== null && (!validJob(row?.job) || ![row.id, row.runId, row.workerNonce].every(text)))
21
+ throw new Error("invalid capture cleanup admission");
22
+ return row;
23
+ }
24
+
25
+ retry(job) {
26
+ const row = this.state.get(retries, job.allocation);
27
+ if (row !== null && (!row || row.key !== job.key || row.stream !== job.stream || row.allocation !== job.allocation ||
28
+ !text(row.attemptedRun) || !Number.isSafeInteger(row.attempts) || row.attempts < 1 ||
29
+ !validDate(row.nextRetryAt))) throw new Error("invalid capture cleanup retry");
30
+ return row;
31
+ }
32
+
33
+ worker(start, now) {
34
+ if (this.state.depth || !text(start.workerNonce) || !text(start.pass?.runId) ||
35
+ typeof start.pass.finalize !== "boolean" || !Number.isSafeInteger(now) || now < 0)
36
+ throw new Error("capture cleanup requires a supervised worker outside processing");
37
+ if (readCaptureAdmission(this.state)) throw new Error("capture cleanup requires stream admission reconciliation");
38
+ }
39
+
40
+ // A failed allocation operation is not a failed journal frame. Keep its
41
+ // backoff private and its failure pass-bound, without changing stream rows.
42
+ reconcile(start, { reason = "interrupted", now = Date.now() } = {}) {
43
+ this.worker(start, now);
44
+ start.budget.check();
45
+ if (!FAILURE_REASONS.includes(reason)) throw new Error("invalid cleanup interruption reason");
46
+ const active = this.active();
47
+ if (!active) return null;
48
+ if (active.workerNonce === start.workerNonce) throw new Error("capture cleanup worker has not been replaced");
49
+ const attempts = (this.retry(active.job)?.attempts ?? 0) + 1;
50
+ const failure = { id: active.id, runId: active.runId, reason };
51
+ this.state.transaction(() => {
52
+ this.state.set(retries, active.job.allocation, { ...active.job, attempts,
53
+ attemptedRun: active.runId, nextRetryAt: retryAfterAttempt(attempts, now) });
54
+ this.state.set("capture-cleanup-failure", "last", failure);
55
+ this.state.delete(kind, "active");
56
+ });
57
+ return failure;
58
+ }
59
+
60
+ advance(run, start, { retry = false, now = Date.now(), limit = 128, maxRecords = 2000 } = {}) {
61
+ this.worker(start, now);
62
+ if (run.state !== this.state || run.id !== start.pass.runId || typeof retry !== "boolean" ||
63
+ !Number.isSafeInteger(maxRecords) || maxRecords < 2 || maxRecords > 2000)
64
+ throw new Error("invalid cleanup dispatch allowance");
65
+ if (this.active()) throw new Error("unreconciled cleanup admission");
66
+ if (start.pass.finalize) return { selected: 0, finalize: true, complete: false };
67
+ const stopped = () => start.budget.sample() !== null;
68
+ if (stopped()) return { selected: 0, stopped: true, complete: false };
69
+ const position = retry ? "retry-after" : "after", after = this.state.get(kind, position) ?? "";
70
+ // A unit that committed no progress is not selected again in its pass, so
71
+ // an allocation that cannot shrink cannot spin until the deadline.
72
+ const idle = this.state.get(kind, "no-progress"), skipped = new Set(idle?.runId === run.id ? idle.allocations : []);
73
+ const select = from => {
74
+ const page = captureCleanupPage(this.state, { after: from, limit });
75
+ for (const candidate of page.jobs) {
76
+ if (stopped()) return { page, stopped: true };
77
+ if (skipped.has(candidate.allocation)) continue;
78
+ const row = this.retry(candidate);
79
+ if (retry ? row && row.attemptedRun !== run.id && now >= Date.parse(row.nextRetryAt) : !row) return { page, job: candidate, prior: row };
80
+ }
81
+ return { page };
82
+ };
83
+ let selection = select(after), rescanned = 0;
84
+ // A complete page past the cursor has not seen the allocations before it,
85
+ // among them an unfinished one the cursor has just passed: one page from
86
+ // the start before answering that no cleanup remains.
87
+ if (!selection.stopped && !selection.job && after !== "" && selection.page.complete) {
88
+ const again = select("");
89
+ if (again.stopped || again.job) selection = again;
90
+ else rescanned = again.page.scanned;
91
+ }
92
+ if (selection.stopped) return { selected: 0, stopped: true, complete: false };
93
+ const { page, job, prior } = selection;
94
+ if (!job) {
95
+ const next = page.complete ? "" : page.after;
96
+ if (next !== after) this.state.transaction(() => this.state.set(kind, position, next));
97
+ return { selected: 0, scanned: page.scanned + rescanned, complete: page.complete };
98
+ }
99
+ const admission = { id: randomUUID(), runId: run.id, workerNonce: start.workerNonce, job };
100
+ this.state.transaction(() => {
101
+ const claim = this.state.get("recovery-allocation", job.key);
102
+ if (this.active() || claim?.id !== job.allocation || claim.stream !== job.stream || claim.phase !== "deleting")
103
+ throw new Error("capture cleanup selection changed before admission");
104
+ this.state.set(kind, "active", admission);
105
+ this.state.set(kind, position, job.key);
106
+ if (prior) this.state.set(retries, job.allocation, { ...prior, attemptedRun: run.id });
107
+ });
108
+ const retire = () => {
109
+ if (this.active()?.id !== admission.id) throw new Error("capture cleanup admission changed before completion");
110
+ this.state.delete(kind, "active");
111
+ };
112
+ let result;
113
+ try {
114
+ result = cleanupCaptureRecovery(this.store, this.state, start, job, { maxRecords, stopped,
115
+ onCommit: (result, { progress }) => {
116
+ retire();
117
+ if (!result.complete && !progress)
118
+ this.state.set(kind, "no-progress", { runId: run.id, allocations: [...skipped, job.allocation] });
119
+ if (result.complete) this.state.delete(retries, job.allocation);
120
+ run.record({}, { progress, peakRssBytes: start.budget.peakRssBytes });
121
+ } });
122
+ } catch (error) {
123
+ // Lock acquisition can observe cooperative exhaustion after selection.
124
+ // Rollback has preserved the claim; orphan/operational failures still
125
+ // leave admission for the registered replacement to diagnose.
126
+ if (!(error instanceof CaptureStop) || !["deadline", "rss"].includes(error.reason)) throw error;
127
+ result = { complete: false, stopped: true };
128
+ }
129
+ // A cooperative stop is not an operational failure. The deleting claim
130
+ // remains, even when removal happened before the metadata unit could start.
131
+ if (result.stopped && start.budget.reason === "interrupted") throw new CaptureStop("interrupted");
132
+ if (result.stopped) this.state.transaction(retire);
133
+ return { selected: 1, job, ...result };
134
+ }
135
+ }
@@ -0,0 +1,77 @@
1
+ // Private retained-recovery housekeeping. A completed source can still own a
2
+ // deleting claim: enumerate those claims independently of the source queue.
3
+ // The caller owns scheduling/fairness, backoff and the final pass receipt.
4
+ import { withCaptureGroup } from "./capture-context.mjs";
5
+ import { prepareRetainedCaptureCleanup, removeRetainedCaptureScratch } from "./capture-retained-scratch.mjs";
6
+ import { discardRecoveryChunks } from "./capture-recovery-snapshot.mjs";
7
+ import { discardTranscriptPrefix } from "./transcript-prefix.mjs";
8
+
9
+ // One keyset page, not a scan-until-match: a large inventory of live recovery
10
+ // jobs must not consume a worker's entire allowance hiding a later cleanup.
11
+ // The scheduler checkpoints `after` and resumes it in subsequent work units.
12
+ // `complete` means this enumeration reached its end, not a successful pass.
13
+ export function captureCleanupPage(state, { after = "", limit = 128 } = {}) {
14
+ if (typeof after !== "string" || !Number.isSafeInteger(limit) || limit < 1 || limit > 128)
15
+ throw new Error("invalid capture cleanup page");
16
+ const rows = state.db.prepare("SELECT key,value FROM state WHERE kind='recovery-allocation' AND key>? ORDER BY key LIMIT ?")
17
+ .all(after, limit);
18
+ const jobs = [];
19
+ for (const row of rows) {
20
+ const claim = JSON.parse(row.value);
21
+ if (!claim || !["active", "deleting"].includes(claim.phase) ||
22
+ typeof claim.stream !== "string" || !claim.stream || !/^[a-f0-9]{32}$/.test(claim.id))
23
+ throw new Error("invalid retained capture cleanup inventory");
24
+ if (claim.phase === "deleting") jobs.push({ key: row.key, stream: claim.stream, allocation: claim.id });
25
+ }
26
+ return { jobs, scanned: rows.length, after: rows.at(-1)?.key ?? after, complete: rows.length < limit };
27
+ }
28
+
29
+ // No pathname from a queue item is authority. The allocation nonce binds this
30
+ // item to the current durable reference; retained-scratch then verifies every
31
+ // directory, owner and payload identity and the registered root-lock worker.
32
+ export function cleanupCaptureRecovery(store, state, start, job, {
33
+ maxRecords = 2000, stopped = () => start.budget.sample() !== null, onCommit = () => {},
34
+ } = {}) {
35
+ if (state.depth || typeof job?.key !== "string" || !job.key ||
36
+ typeof job.stream !== "string" || !job.stream || !/^[a-f0-9]{32}$/.test(job.allocation) ||
37
+ !Number.isSafeInteger(maxRecords) || maxRecords < 2 || maxRecords > 2000 || typeof stopped !== "function" ||
38
+ typeof onCommit !== "function")
39
+ throw new Error("invalid capture recovery cleanup unit");
40
+ const current = () => {
41
+ const claim = state.get("recovery-allocation", job.key);
42
+ if (claim?.id !== job.allocation || claim.stream !== job.stream || claim.phase !== "deleting")
43
+ throw new Error("capture recovery cleanup reference changed");
44
+ return claim;
45
+ };
46
+ current();
47
+ if (stopped()) return { complete: false, stopped: true };
48
+ withCaptureGroup(store, [job.stream], () => {
49
+ current();
50
+ // Adoption must commit before removal. An already-absent directory is
51
+ // accepted only for this deleting claim under its unchanged scratch base.
52
+ prepareRetainedCaptureCleanup(store, job.stream, job.key, start);
53
+ }, { state, check: () => start.budget.check() });
54
+ if (stopped()) return { complete: false, stopped: true };
55
+ current();
56
+ removeRetainedCaptureScratch(store.root, state, job.key, start);
57
+ if (stopped()) return { complete: false, stopped: true };
58
+ return withCaptureGroup(store, [job.stream], () => {
59
+ current();
60
+ const before = state.db.prepare("SELECT total_changes() AS n").get().n;
61
+ // Divide one allowance between the two tables, including on restarts.
62
+ // Neither table's full cleanup is a prerequisite to checkpointing the other.
63
+ const chunks = discardRecoveryChunks(state, job.allocation, { maxRecords: Math.floor(maxRecords / 2) });
64
+ const prefix = discardTranscriptPrefix(store, job.stream, job.key, { maxRecords: Math.ceil(maxRecords / 2) });
65
+ if (chunks && prefix) {
66
+ state.delete("recovery-allocation", job.key);
67
+ state.delete("recovery-stage", job.key);
68
+ }
69
+ // Keep the deleting reference across partial cleanup and transaction loss.
70
+ // Source/journal cursors and run counts are never reconstructed here.
71
+ const result = { complete: chunks && prefix, stopped: false };
72
+ // Admission retirement and confirmed progress share this final commit.
73
+ // A failed callback leaves both the deleting claim and admission resumable.
74
+ onCommit(result, { progress: state.db.prepare("SELECT total_changes() AS n").get().n !== before });
75
+ return result;
76
+ }, { state, check: () => start.budget.check() });
77
+ }
@@ -0,0 +1,37 @@
1
+ // Production owning operation, shared by the fixed entry and isolated fixture
2
+ // workers. Bootstrap supplies registration/root-lock/ended-predecessor proof.
3
+ import { CaptureWorkSession } from "./capture-work-session.mjs";
4
+ import { CaptureStop } from "./capture-budget.mjs";
5
+ import { captureCommandOptions, captureCommandSelection } from "./capture-command.mjs";
6
+ import { readCaptureStatus } from "./capture-status.mjs";
7
+
8
+ export function captureCommandOperation(start) {
9
+ const options = captureCommandOptions(start.options);
10
+ let session;
11
+ const receipt = result => {
12
+ if (result.continuation) return result;
13
+ const status = readCaptureStatus(start.root);
14
+ if (status?.lastRun?.id !== start.pass.runId || status.lastRun.outcome === "running")
15
+ throw new Error("capture command has no finalized matching receipt");
16
+ // This bounded private IPC projection is read while the supervisor still
17
+ // holds the root lock. The parent does not write or reconstruct counters.
18
+ return { ...result, receipt: { appended: status.lastRun.counts.turnsAppended, pending: status.backlog.pending } };
19
+ };
20
+ try {
21
+ session = new CaptureWorkSession(start, { owner: options.owner, selection: captureCommandSelection(options),
22
+ roots: options.plan, trigger: options.trigger, indexEnabled: options.indexEnabled,
23
+ issues: { sources: options.plan.sources, retry: options.retrySetAside } });
24
+ while (true) {
25
+ const result = session.step();
26
+ if (result.resetRequired) return session.yieldWorker({ resetRequired: true });
27
+ if (result.finalize) return receipt(session.finish({ reason: "deadline" }));
28
+ if (result.stopped) return receipt(session.yieldWorker());
29
+ if (result.idle) return receipt(session.complete());
30
+ }
31
+ } catch (error) {
32
+ if (error instanceof CaptureStop && session) return receipt(session.yieldWorker());
33
+ // An operational exception leaves its admission and running receipt for a
34
+ // registered replacement, which can continue independent healthy work.
35
+ throw error;
36
+ } finally { session?.close(); }
37
+ }
@@ -0,0 +1,61 @@
1
+ // Private foreground command presentation shared by manual/watch, setup and service.
2
+ // Only the still-held-lock worker projection supplies counts, never a later
3
+ // status read or a guessed zero when transport failed before publication.
4
+ import { runCaptureCommand } from "./capture-command.mjs";
5
+
6
+ export function reportLockInitializationFailure(err, error = console.error) {
7
+ if (!err.lockCleanup) return;
8
+ const c = err.lockCleanup;
9
+ const outcome = c.removed ? "the initializing lock was removed"
10
+ : c.reason === "gone" ? "the initializing lock was already gone (removed by another party)"
11
+ : c.reason === "replaced" ? (c.owner
12
+ ? `the lock now belongs to pid ${c.owner.pid} (started ${c.owner.startedAt || "?"}) and was left alone`
13
+ : "the lock directory changed or its identity could not be verified, no owner record was readable, and it was left alone")
14
+ : `the initializing lock could NOT be removed${c.error ? ` (${c.error})` : ""}; ${c.recovery}`;
15
+ error(`capture: could not write the owner record of ${c.path}: ${err.message}; ${outcome}`);
16
+ }
17
+
18
+ export function reportReclaimedLock(r, error = console.error) {
19
+ if (r) error(`capture: reclaimed ${r.path} from pid ${r.pid}, which died (started ${r.startedAt || "?"})`);
20
+ }
21
+
22
+ // A status this pass moved aside, said by the pass that moved it. A pass that
23
+ // then failed has not written the new status.
24
+ function reportInvalidStatus(moved, error = console.error, { failed = false } = {}) {
25
+ if (moved) error(`capture: the capture status was unreadable (${moved.reason}); moved it aside to ${moved.path}${failed ? "" : " and started a new status"}` +
26
+ (moved.stale ? `; older moved-aside copies could not be removed: ${moved.stale.join(", ")}` : ""));
27
+ }
28
+
29
+ export function releaseDetail(r) {
30
+ return r.reason === "gone" ? "it was already removed by another party (an operator recovery?); nothing to release"
31
+ : r.reason === "not-owner" ? `it now belongs to pid ${r.owner.pid} (started ${r.owner.startedAt || "?"}); left alone`
32
+ : r.reason === "unknown-owner" ? `its owner record is missing or unreadable, so it may be an operator removal in progress or a newer pass initializing; left alone; ${r.recovery}`
33
+ : `${r.error ? `${r.error}; ` : ""}${r.recovery}`;
34
+ }
35
+
36
+ export async function captureCommandPass(root, options, { log = console.log, error = console.error, run = runCaptureCommand } = {}) {
37
+ let result;
38
+ try { result = await run(root, options); }
39
+ catch (failure) {
40
+ reportLockInitializationFailure(failure, error);
41
+ reportReclaimedLock(failure.reclaimedLock, error);
42
+ reportInvalidStatus(failure.invalidStatus, error, { failed: true });
43
+ if (failure.release) error(`capture: did not release lock at ${root} (${failure.release.reason}): ${releaseDetail(failure.release)}`);
44
+ throw failure;
45
+ }
46
+ reportReclaimedLock(result.reclaimedLock, error);
47
+ reportInvalidStatus(result.invalidStatus, error, { failed: result.exitCode !== 0 || result.outcome === "failed" });
48
+ if (result.outcome === "overlap") {
49
+ const line = `capture: another pass holds ${root}: ${result.held.recovery}; skipping this pass`;
50
+ if (result.held.liveness === "alive") log(line); else error(line);
51
+ return { skipped: true };
52
+ }
53
+ if (result.exitCode !== 0 || result.outcome === "failed")
54
+ throw new Error(`capture worker failed (${result.reason ?? "io"})${result.message ? `: ${result.message}` : ""}; ${result.receipt ? "see capture health for confirmed progress" : "matching worker receipt unavailable; appended count is unknown"}`);
55
+ const receipt = result.receipt;
56
+ if (!receipt || !Number.isSafeInteger(receipt.appended) || receipt.appended < 0 || typeof receipt.pending !== "boolean")
57
+ throw new Error("matching worker receipt unavailable; appended count is unknown");
58
+ log(`capture: ${receipt.appended} new turns; ${result.outcome}${result.reason ? ` (${result.reason})` : ""}`);
59
+ if (options.indexEnabled) log(result.outcome === "complete" ? "index: updated" : "index: pending; see capture health");
60
+ return { appended: receipt.appended, pending: receipt.pending };
61
+ }
@@ -0,0 +1,6 @@
1
+ // Fixed private entry. It accepts only the supervisor's authenticated IPC
2
+ // start; invoking this file directly cannot perform a capture operation.
3
+ import { runCaptureWorker } from "./capture-worker.mjs";
4
+ import { captureCommandOperation } from "./capture-command-operation.mjs";
5
+
6
+ await runCaptureWorker(captureCommandOperation);
@@ -0,0 +1,48 @@
1
+ // Private command caller. Root discovery runs only in the admitted worker;
2
+ // callers cannot choose its executable, entry path, heap or pass allowances.
3
+ import { createHash } from "node:crypto";
4
+ import { isAbsolute, resolve } from "node:path";
5
+ import { fileURLToPath } from "node:url";
6
+ import { captureDefaultRootPlan } from "./capture-default-roots.mjs";
7
+ import { superviseCapturePass } from "./capture-pass.mjs";
8
+ import { acquireCaptureLock } from "./capture-lock.mjs";
9
+ import { validStreamId } from "./store.mjs";
10
+
11
+ const triggers = new Set(["manual", "watch", "hook", "setup", "service"]);
12
+ export function captureCommandOptions(value) {
13
+ if (!value || Object.keys(value).some(key => !["owner", "plan", "indexEnabled", "trigger", "retrySetAside"].includes(key)) ||
14
+ typeof value.owner !== "string" || !value.owner || !validStreamId(`${value.owner}~aw`) ||
15
+ typeof value.indexEnabled !== "boolean" || !triggers.has(value.trigger) || typeof value.retrySetAside !== "boolean" ||
16
+ value.plan?.kind !== "default-roots" ||
17
+ Object.keys(value.plan).some(key => !["kind", "home", "cwd", "env", "sources"].includes(key)))
18
+ throw new Error("invalid private capture command selection");
19
+ const plan = captureDefaultRootPlan(value.plan);
20
+ if (value.retrySetAside && plan.sources.length !== 4) throw new Error("explicit retry requires all capture sources");
21
+ return { owner: value.owner, plan, indexEnabled: value.indexEnabled, trigger: value.trigger, retrySetAside: value.retrySetAside };
22
+ }
23
+
24
+ export function captureCommandSelection(options) {
25
+ return createHash("sha256").update(JSON.stringify(options.plan)).digest("hex");
26
+ }
27
+
28
+ export async function runCaptureCommand(root, options) {
29
+ if (typeof root !== "string" || !root || root.includes("\0")) throw new Error("invalid capture command root");
30
+ const selected = captureCommandOptions(options);
31
+ // Resolve relative root spelling against the caller, never a child's cwd.
32
+ const absolute = isAbsolute(root) ? root : resolve(selected.plan.cwd, root);
33
+ let reclaimedLock = null;
34
+ let result;
35
+ try { result = await superviseCapturePass(absolute, selected, {
36
+ workerPath: fileURLToPath(new URL("./capture-command-worker.mjs", import.meta.url)),
37
+ acquire: path => {
38
+ const lock = acquireCaptureLock(path);
39
+ if (lock.reclaimed) reclaimedLock = { path: lock.path, ...lock.reclaimed };
40
+ return lock;
41
+ },
42
+ }); } catch (error) {
43
+ if (reclaimedLock) error.reclaimedLock = reclaimedLock;
44
+ throw error;
45
+ }
46
+ // This is observed lock recovery, not worker accounting or completion.
47
+ return { ...result, reclaimedLock };
48
+ }