@awebai/oats 0.61.0 → 0.62.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/README.md +1 -1
  2. package/bin/oats.mjs +26 -11
  3. package/docs/capabilities.md +1 -1
  4. package/docs/execution-targets.md +49 -3
  5. package/docs/implementation.md +2 -1
  6. package/docs/named-test-runs.md +382 -0
  7. package/docs/official-catalog.md +1 -1
  8. package/docs/packages.md +5 -5
  9. package/docs/release-notes/v0.62.0.md +223 -0
  10. package/docs/upgrading.md +133 -11
  11. package/docs/workspaces.md +1 -1
  12. package/lib/core.mjs +2 -2
  13. package/lib/doctor-capture.mjs +35 -0
  14. package/lib/session-viewer.mjs +86 -7
  15. package/package-catalog.json +1 -1
  16. package/package.json +1 -1
  17. package/packages/client/cli-locator.mjs +2 -2
  18. package/packages/record/README.md +87 -13
  19. package/packages/record/bin/capture-hook.mjs +8 -0
  20. package/packages/record/bin/capture.mjs +143 -124
  21. package/packages/record/bin/recall.mjs +8 -3
  22. package/packages/record/bin/setup.mjs +115 -225
  23. package/packages/record/bin/turn-record.mjs +2 -2
  24. package/packages/record/docs/capture-service.md +1443 -28
  25. package/packages/record/docs/capture-status.schema.json +429 -12
  26. package/packages/record/lib/capture-admission.mjs +82 -0
  27. package/packages/record/lib/capture-aw-unit.mjs +137 -0
  28. package/packages/record/lib/capture-aw.mjs +31 -12
  29. package/packages/record/lib/capture-budget.mjs +31 -0
  30. package/packages/record/lib/capture-cc.mjs +39 -6
  31. package/packages/record/lib/capture-cleanup-queue.mjs +135 -0
  32. package/packages/record/lib/capture-cleanup-work.mjs +77 -0
  33. package/packages/record/lib/capture-command-operation.mjs +37 -0
  34. package/packages/record/lib/capture-command-output.mjs +61 -0
  35. package/packages/record/lib/capture-command-worker.mjs +6 -0
  36. package/packages/record/lib/capture-command.mjs +48 -0
  37. package/packages/record/lib/capture-completion.mjs +152 -0
  38. package/packages/record/lib/capture-context.mjs +50 -0
  39. package/packages/record/lib/capture-default-roots.mjs +30 -0
  40. package/packages/record/lib/capture-discovery.mjs +114 -0
  41. package/packages/record/lib/capture-failures.mjs +45 -0
  42. package/packages/record/lib/capture-health-doctor.mjs +13 -0
  43. package/packages/record/lib/capture-health-installation.mjs +117 -0
  44. package/packages/record/lib/capture-health.mjs +116 -0
  45. package/packages/record/lib/capture-hook-ownership.mjs +58 -0
  46. package/packages/record/lib/capture-hook-runner.mjs +17 -0
  47. package/packages/record/lib/capture-hook.mjs +26 -0
  48. package/packages/record/lib/capture-index-inventory.mjs +205 -0
  49. package/packages/record/lib/capture-index-outcomes.mjs +113 -0
  50. package/packages/record/lib/capture-index-reset.mjs +109 -0
  51. package/packages/record/lib/capture-index-work.mjs +77 -0
  52. package/packages/record/lib/capture-issue-preparation.mjs +73 -0
  53. package/packages/record/lib/capture-issues.mjs +15 -2
  54. package/packages/record/lib/capture-lock.mjs +102 -16
  55. package/packages/record/lib/capture-native-unit.mjs +322 -0
  56. package/packages/record/lib/capture-pass.mjs +102 -0
  57. package/packages/record/lib/capture-recovery-snapshot.mjs +182 -0
  58. package/packages/record/lib/capture-retained-scratch.mjs +213 -0
  59. package/packages/record/lib/capture-retry.mjs +102 -0
  60. package/packages/record/lib/capture-run.mjs +125 -0
  61. package/packages/record/lib/capture-scratch.mjs +12 -3
  62. package/packages/record/lib/capture-service-definition.mjs +104 -0
  63. package/packages/record/lib/capture-service-install.mjs +362 -0
  64. package/packages/record/lib/capture-service-ownership.mjs +137 -0
  65. package/packages/record/lib/capture-service-runner.mjs +18 -0
  66. package/packages/record/lib/capture-small-snapshot.mjs +85 -0
  67. package/packages/record/lib/capture-source-discovery.mjs +319 -0
  68. package/packages/record/lib/capture-source-queue.mjs +199 -0
  69. package/packages/record/lib/capture-source-work.mjs +85 -0
  70. package/packages/record/lib/capture-state.mjs +16 -71
  71. package/packages/record/lib/capture-status.mjs +171 -0
  72. package/packages/record/lib/capture-supervisor.mjs +117 -0
  73. package/packages/record/lib/capture-watch.mjs +32 -0
  74. package/packages/record/lib/capture-work-session.mjs +238 -0
  75. package/packages/record/lib/capture-worker.mjs +58 -0
  76. package/packages/record/lib/derived-state.mjs +28 -15
  77. package/packages/record/lib/file-lines.mjs +22 -6
  78. package/packages/record/lib/formats.mjs +4 -3
  79. package/packages/record/lib/index-capture.mjs +97 -0
  80. package/packages/record/lib/index-db.mjs +3 -17
  81. package/packages/record/lib/index-reset.mjs +19 -0
  82. package/packages/record/lib/store.mjs +36 -11
  83. package/packages/record/lib/transcript-prefix.mjs +125 -0
  84. package/skills/oats-getting-started/SKILL.md +1 -1
  85. package/docs/macos-test-runs.md +0 -246
@@ -0,0 +1,102 @@
1
+ // Private continuation transport. Reconciliation, durable fairness, failure
2
+ // attribution and status remain worker responsibilities. The parent never
3
+ // promotes a transient result into journal/checkpoint evidence; it only moves
4
+ // an unreadable status aside, under the lock and before any worker.
5
+ import { randomUUID } from "node:crypto";
6
+ import { CAPTURE_POLICY } from "./capture-budget.mjs";
7
+ import { acquireCaptureLock } from "./capture-lock.mjs";
8
+ import { cleanupCaptureScratch } from "./capture-scratch.mjs";
9
+ import { setAsideInvalidCaptureStatus } from "./capture-status.mjs";
10
+ import { captureMonotonicMs, superviseCapture } from "./capture-supervisor.mjs";
11
+
12
+ // Internal scheduling choices, not public tuning or portable timing promises.
13
+ const ALLOWANCES = Object.freeze({ workMs: 60_000, finalizeMs: 70_000, hardMs: 75_000,
14
+ finalReserveMs: 15_000, finalDrainMs: 10_000, maxStarts: 8 });
15
+
16
+ export async function superviseCapturePass(root, options, {
17
+ policy = CAPTURE_POLICY, clock = captureMonotonicMs, acquire = acquireCaptureLock,
18
+ allowances = ALLOWANCES, drainSeconds = 5, ...transport
19
+ } = {}) {
20
+ const { workMs, finalizeMs, hardMs, finalReserveMs, finalDrainMs, maxStarts } = allowances;
21
+ if (![workMs, finalizeMs, hardMs, finalReserveMs, finalDrainMs].every(n => Number.isFinite(n) && n > 0) ||
22
+ workMs > finalizeMs || finalizeMs > hardMs || finalDrainMs > finalReserveMs ||
23
+ !Number.isSafeInteger(maxStarts) || maxStarts < 2 || maxStarts > 8) throw new Error("invalid internal capture scheduling allowances");
24
+ const started = clock(), deadline = started + policy.deadlineSeconds * 1000;
25
+ const drainAt = Math.max(started, deadline - drainSeconds * 1000), runId = randomUUID();
26
+ const lock = acquire(root);
27
+ if (!lock.release) return { outcome: "overlap", reason: null, exitCode: 0, starts: 0, held: lock.held };
28
+ let starts = 0, registered, publicationFailed = false, interrupted = false, firstFailure = null, failureMessage;
29
+ let previous = null, result = { outcome: "partial", reason: "deadline", exitCode: 0 };
30
+ const onSignal = () => { interrupted = true; };
31
+ const attempt = async finalize => {
32
+ const launch = clock();
33
+ starts++;
34
+ const window = finalize
35
+ ? { drainAt: Math.min(drainAt, launch + finalDrainMs), finalizeAt: Math.min(deadline, launch + finalDrainMs), hardAt: Math.min(deadline, launch + finalReserveMs) }
36
+ : { drainAt: Math.min(drainAt, launch + workMs), finalizeAt: Math.min(deadline, launch + finalizeMs), hardAt: Math.min(deadline, launch + hardMs) };
37
+ const response = await superviseCapture(root, options, { ...transport, policy, clock, window,
38
+ pass: { runId, attempt: starts, finalize, reset: !finalize && previous?.resetRequired === true, started, deadline, drainAt,
39
+ failure: firstFailure,
40
+ // Only observed process outcome, never an asserted failed member/frame.
41
+ previous: previous && { outcome: previous.outcome, reason: previous.reason } },
42
+ acquire: () => ({ ...lock,
43
+ registerWorker: worker => {
44
+ try { lock.registerWorker(worker, { previous: registered }); registered = worker; }
45
+ catch (error) { publicationFailed = true; throw error; }
46
+ },
47
+ release: () => ({ released: true }), // outer owner releases once, after all children are reaped
48
+ }),
49
+ });
50
+ previous = response;
51
+ if (response.outcome === "failed" && firstFailure === null) { firstFailure = response.reason; failureMessage = response.message; }
52
+ return response;
53
+ };
54
+ // A worker's private diagnostic accompanies only the first failure it explains.
55
+ // A status moved aside by this pass is reported with whatever it returns.
56
+ let invalidStatus = null;
57
+ const failed = () => ({ outcome: "failed", reason: firstFailure, exitCode: 1, starts,
58
+ ...(failureMessage ? { message: failureMessage } : {}), ...(invalidStatus ? { invalidStatus } : {}) });
59
+ const ended = value => invalidStatus ? { ...value, invalidStatus } : value;
60
+ try {
61
+ // Under the root lock and before this pass starts any child: reclaim
62
+ // per-call scratch whose allocating process is proven dead (killed workers).
63
+ cleanupCaptureScratch(root, lock);
64
+ invalidStatus = setAsideInvalidCaptureStatus(root);
65
+ process.on("SIGINT", onSignal); process.on("SIGTERM", onSignal);
66
+ // Keep one start and time for a worker to reconcile/write final facts.
67
+ // A normal child is admitted only with its full allowance; in particular,
68
+ // atomic reset must not be admitted as a shortened last-minute operation.
69
+ while (!interrupted && starts < maxStarts - 1 && clock() + hardMs + finalReserveMs <= deadline) {
70
+ result = await attempt(false);
71
+ if (publicationFailed || interrupted) break;
72
+ // A worker may finish an honestly failed pass after committing healthy
73
+ // independent work and exhausting ready work. That explicit terminal
74
+ // result needs no idle retries; an unexplained crash still continues.
75
+ if (result.outcome === "complete" || result.outcome === "partial" && !result.continuation ||
76
+ result.outcome === "failed" && result.continuation === false) {
77
+ return firstFailure ? failed() : ended({ ...result, starts });
78
+ }
79
+ // On a crash the next child reconciles durable admission, then selects
80
+ // independent ready work. There is NO same-pass retry/diagnostic class.
81
+ }
82
+ if (!interrupted && !publicationFailed && starts < maxStarts && clock() < deadline) {
83
+ result = await attempt(true);
84
+ // Finalization cannot certify a sweep that this continuation transport
85
+ // stopped early. The worker likewise must not manufacture lastSuccess.
86
+ if (result.outcome === "complete") {
87
+ result = { outcome: "failed", reason: "io", exitCode: 1 }; firstFailure ??= "io";
88
+ }
89
+ }
90
+ if (interrupted) return ended({ outcome: "failed", reason: "interrupted", exitCode: 1, starts });
91
+ if (firstFailure) return failed();
92
+ return ended({ ...result, starts });
93
+ } catch (error) {
94
+ if (invalidStatus) error.invalidStatus ??= invalidStatus;
95
+ throw error;
96
+ } finally {
97
+ process.off("SIGINT", onSignal); process.off("SIGTERM", onSignal);
98
+ const released = lock.release();
99
+ if (!released.released) throw Object.assign(new Error("capture pass could not release its lock"), { release: released,
100
+ ...(invalidStatus ? { invalidStatus } : {}) });
101
+ }
102
+ }
@@ -0,0 +1,182 @@
1
+ // Command-only staged source copying. The fixed source end and committed chunk
2
+ // hashes survive workers; ownership is handled by retained-scratch, never by
3
+ // reopening an allocation by name alone. Explicit receipt snapshots do not use
4
+ // this append-only/bounded-witness path.
5
+ import { closeSync, constants, fstatSync, fsyncSync, ftruncateSync, openSync, readSync, writeSync } from "node:fs";
6
+ import { randomUUID } from "node:crypto";
7
+ import { captureContext } from "./capture-context.mjs";
8
+ import { createRetainedCaptureScratch, openRetainedCaptureScratch } from "./capture-retained-scratch.mjs";
9
+ import { statVersion, fileVersion, sameVersion } from "./derived-state.mjs";
10
+ import { assertProtectedDescriptor, digest } from "./session-snapshot.mjs";
11
+ import { guardCapturedPath } from "./native-history.mjs";
12
+
13
+ const CHUNK = 65536;
14
+ function readExact(fd, position, size) {
15
+ const bytes = Buffer.allocUnsafe(size); let read = 0;
16
+ while (read < size) {
17
+ const n = readSync(fd, bytes, read, size - read, position + read);
18
+ if (!n) throw new Error("short read of recovery snapshot");
19
+ read += n;
20
+ }
21
+ return bytes;
22
+ }
23
+
24
+ export function advanceRecoverySnapshot(store, stream, key, path, start, {
25
+ capturedPi, generation = "", fixedEnd, maxBytes = 8 * 1024 * 1024, stopped = () => false, work,
26
+ } = {}) {
27
+ const context = captureContext(store), state = context?.state;
28
+ if (!state?.depth || !context.held.has(stream)) throw new Error("recovery snapshot requires its stream work group");
29
+ if (typeof generation !== "string" || !Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > 8 * 1024 * 1024) throw new Error("invalid recovery snapshot budget");
30
+ // The scheduling caller applies ignore before entering this helper. Custody
31
+ // checks still precede every protected source open/read and final verification.
32
+ guardCapturedPath(path, capturedPi);
33
+ const sourceFd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK | (capturedPi ? constants.O_NOFOLLOW : 0));
34
+ let allocation, retained = false;
35
+ try {
36
+ assertProtectedDescriptor(sourceFd, path, capturedPi);
37
+ if (!fstatSync(sourceFd).isFile()) throw new Error("recovery source is not a regular file");
38
+ const observe = () => {
39
+ assertProtectedDescriptor(sourceFd, path, capturedPi);
40
+ const version = statVersion(fstatSync(sourceFd, { bigint: true }));
41
+ if (!sameVersion(version, fileVersion(path))) throw new Error("recovery source descriptor differs from its named file");
42
+ return version;
43
+ };
44
+ const version = observe(), journalVersion = fileVersion(store.journalPath(stream)), db = state.db;
45
+ if (fixedEnd !== undefined && (!Number.isSafeInteger(fixedEnd) || fixedEnd < 0 || fixedEnd > version.size)) throw new Error("invalid fixed recovery end");
46
+ if (db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name='recovery_chunks'").get()?.found !== 1) {
47
+ db.exec("CREATE TABLE recovery_chunks (allocation TEXT, offset INTEGER, length INTEGER NOT NULL, hash TEXT NOT NULL, PRIMARY KEY(allocation,offset))");
48
+ state.set("recovery-meta", "epoch", randomUUID());
49
+ }
50
+ let epoch = state.get("recovery-meta", "epoch");
51
+ if (epoch === null) { epoch = randomUUID(); state.set("recovery-meta", "epoch", epoch); }
52
+ if (typeof epoch !== "string" || !epoch) throw new Error("invalid recovery chunk epoch");
53
+ let stage = state.get("recovery-stage", key);
54
+ const existing = state.get("recovery-allocation", key);
55
+ allocation = existing ? openRetainedCaptureScratch(store, stream, key, start) : createRetainedCaptureScratch(store, stream, key, start);
56
+ const invalidate = reason => {
57
+ allocation.close();
58
+ allocation = openRetainedCaptureScratch(store, stream, key, start, { cleanup: true });
59
+ state.set("recovery-stage", key, { allocation: allocation.claim.id, phase: "invalid", reason });
60
+ return { complete: false, invalidated: true, cleanup: true, reason };
61
+ };
62
+ if (allocation.claim.phase === "deleting") return { complete: false, cleanup: true };
63
+ if (!existing) {
64
+ if (typeof stage?.allocation === "string") state.set("recovery-garbage", stage.allocation, true);
65
+ stage = { epoch, generation, journalVersion, allocation: allocation.claim.id, path, sourceVersion: version, observedVersion: version,
66
+ size: fixedEnd ?? version.size, copied: 0, checked: 0,
67
+ witness: digest(Buffer.alloc(0)), payloadVersion: statVersion(fstatSync(allocation.fd, { bigint: true })), phase: "copy" };
68
+ } else if (!stage || stage.epoch !== epoch || stage.allocation !== allocation.claim.id || stage.path !== path) {
69
+ return invalidate("recovery-state-lost");
70
+ }
71
+ if (stage.generation !== generation) return invalidate("recovery-generation-changed");
72
+ // A caller narrowing the end never reuses a copy fixed beyond it. A larger requested end is ordinary later growth: the
73
+ // first observation stays this sweep's fixed end.
74
+ if (fixedEnd !== undefined && fixedEnd < stage.size) return invalidate("recovery-end-changed");
75
+ // A public/outgoing writer may have captured a later source suffix between
76
+ // workers. Comparing that longer journal with this old fixed end would
77
+ // manufacture a missing-prefix conflict. Rebuild the source snapshot first;
78
+ // the append-only source witness never qualifies a changed journal version.
79
+ if (!sameVersion(stage.journalVersion, journalVersion)) return invalidate("recovery-journal-changed");
80
+ if (!["copy", "validate", "ready"].includes(stage.phase) ||
81
+ !["size", "copied", "checked"].every(k => Number.isSafeInteger(stage[k]) && stage[k] >= 0) ||
82
+ stage.copied > stage.size || stage.checked > stage.copied) {
83
+ return invalidate("recovery-state-invalid");
84
+ }
85
+ const witness = end => {
86
+ assertProtectedDescriptor(sourceFd, path, capturedPi);
87
+ const size = Math.min(4096, end), bytes = readExact(sourceFd, end - size, size);
88
+ if (work) work.sourceProbeBytes = (work.sourceProbeBytes ?? 0) + size;
89
+ return digest(bytes);
90
+ };
91
+ const sourceMatches = observed => sameVersion(stage.observedVersion, observed) ||
92
+ observed.dev === stage.sourceVersion.dev && observed.ino === stage.sourceVersion.ino &&
93
+ observed.size > stage.observedVersion.size && observed.size >= stage.size && witness(stage.copied) === stage.witness;
94
+ if (!sourceMatches(version)) return invalidate("recovery-source-changed");
95
+ stage.observedVersion = version;
96
+ allocation.verify();
97
+ let payloadVersion = statVersion(fstatSync(allocation.fd, { bigint: true }));
98
+ if (!sameVersion(payloadVersion, stage.payloadVersion)) {
99
+ if (payloadVersion.size < stage.copied) return invalidate("recovery-payload-shrank");
100
+ // The extra suffix was never committed. Remove it only from this adopted
101
+ // allocation, then verify every committed chunk before extending/using it.
102
+ if (payloadVersion.size > stage.copied) { ftruncateSync(allocation.fd, stage.copied); fsyncSync(allocation.fd); }
103
+ payloadVersion = statVersion(fstatSync(allocation.fd, { bigint: true }));
104
+ stage.phase = "validate"; stage.checked = 0; stage.payloadVersion = payloadVersion;
105
+ }
106
+ let used = 0;
107
+ const stop = () => used >= maxBytes || stopped();
108
+ if (stage.phase === "validate") {
109
+ const get = db.prepare("SELECT length,hash FROM recovery_chunks WHERE allocation=? AND offset=?");
110
+ while (stage.checked < stage.copied && !stop()) {
111
+ const row = get.get(stage.allocation, stage.checked);
112
+ if (!row || !Number.isSafeInteger(row.length) || row.length < 1 || row.length > CHUNK || stage.checked + row.length > stage.copied) return invalidate("recovery-chunk-missing");
113
+ const bytes = readExact(allocation.fd, stage.checked, row.length);
114
+ if (digest(bytes) !== row.hash) return invalidate("recovery-payload-changed");
115
+ stage.checked += row.length; used += row.length;
116
+ if (work) work.stagedReadBytes = (work.stagedReadBytes ?? 0) + row.length;
117
+ }
118
+ if (!sameVersion(stage.payloadVersion, statVersion(fstatSync(allocation.fd, { bigint: true })))) return invalidate("recovery-payload-changed");
119
+ if (stage.checked === stage.copied) stage.phase = stage.copied === stage.size ? "ready" : "copy";
120
+ }
121
+ if (stage.phase === "copy") {
122
+ const put = db.prepare("INSERT INTO recovery_chunks VALUES (?,?,?,?)");
123
+ while (stage.copied < stage.size && !stop()) {
124
+ assertProtectedDescriptor(sourceFd, path, capturedPi); allocation.verify();
125
+ const bytes = readExact(sourceFd, stage.copied, Math.min(CHUNK, maxBytes - used, stage.size - stage.copied));
126
+ if (work) work.sourceBytes = (work.sourceBytes ?? 0) + bytes.length;
127
+ // Copying never joins a line, so a record of any size stages in chunks;
128
+ // the raw-record limit applies to capture beyond the proven prefix.
129
+ for (let written = 0; written < bytes.length;) {
130
+ const n = writeSync(allocation.fd, bytes, written, bytes.length - written, stage.copied + written);
131
+ if (!n) throw new Error("short write of recovery snapshot"); written += n;
132
+ }
133
+ put.run(stage.allocation, stage.copied, bytes.length, digest(bytes));
134
+ stage.copied += bytes.length; used += bytes.length;
135
+ if (work) work.stagedWriteBytes = (work.stagedWriteBytes ?? 0) + bytes.length;
136
+ }
137
+ // File bytes precede the surrounding DB commit; a crash in between leaves
138
+ // a suffix that the next worker truncates/revalidates, never progress.
139
+ fsyncSync(allocation.fd);
140
+ stage.payloadVersion = statVersion(fstatSync(allocation.fd, { bigint: true }));
141
+ const probeBytes = Math.min(4096, stage.copied);
142
+ stage.witness = digest(readExact(allocation.fd, stage.copied - probeBytes, probeBytes));
143
+ if (work) work.stagedProbeBytes = (work.stagedProbeBytes ?? 0) + probeBytes;
144
+ if (stage.copied === stage.size) stage.phase = "ready";
145
+ }
146
+ const after = observe();
147
+ if (!sourceMatches(after)) return invalidate("recovery-source-changed");
148
+ stage.observedVersion = after;
149
+ allocation.verify();
150
+ state.set("recovery-stage", key, stage);
151
+ if (stage.phase !== "ready") return { complete: false, phase: stage.phase, copied: stage.copied, checked: stage.checked };
152
+ const verify = () => {
153
+ allocation.verify();
154
+ if (!sameVersion(stage.payloadVersion, statVersion(fstatSync(allocation.fd, { bigint: true }))) || !sourceMatches(observe())) {
155
+ throw new Error("recovery snapshot changed before use");
156
+ }
157
+ };
158
+ verify(); retained = true; let closed = false;
159
+ return { complete: true, snapshot: { id: stage.allocation, size: stage.size,
160
+ *chunks(from = 0, end = stage.size) {
161
+ if (!Number.isSafeInteger(from) || from < 0 || !Number.isSafeInteger(end) || end < from || end > stage.size) throw new Error("invalid recovery snapshot range");
162
+ for (let position = from; position < end;) {
163
+ verify(); const bytes = readExact(allocation.fd, position, Math.min(CHUNK, end - position));
164
+ position += bytes.length; yield bytes;
165
+ }
166
+ verify();
167
+ } }, verify, close() { if (!closed) { closed = true; try { allocation.close(); } finally { closeSync(sourceFd); } } } };
168
+ } finally { if (!retained) { try { allocation?.close(); } finally { closeSync(sourceFd); } } }
169
+ }
170
+
171
+ // Retired hash rows are derived metadata, not cleanup authority for any path.
172
+ // The worker queues an allocation id after retiring its job and reclaims only a
173
+ // bounded number per unit; payload ownership remains the separate scratch rule.
174
+ export function discardRecoveryChunks(state, allocation, { maxRecords = 2000 } = {}) {
175
+ if (!state.depth || typeof allocation !== "string" || !/^[a-f0-9]{32}$/.test(allocation) ||
176
+ !Number.isSafeInteger(maxRecords) || maxRecords < 1 || maxRecords > 2000) throw new Error("invalid recovery chunk cleanup");
177
+ const exists = state.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name='recovery_chunks'").get()?.found === 1;
178
+ const done = !exists || Number(state.db.prepare(`DELETE FROM recovery_chunks WHERE rowid IN
179
+ (SELECT rowid FROM recovery_chunks WHERE allocation=? LIMIT ?)`).run(allocation, maxRecords).changes) < maxRecords;
180
+ if (done) state.delete("recovery-garbage", allocation);
181
+ return done;
182
+ }
@@ -0,0 +1,213 @@
1
+ // Private command allocations survive worker lifetimes deliberately. A durable
2
+ // job reference plus allocation/file identities is required; names alone never
3
+ // authorize adoption, proof or deletion. Public per-call scratch stays separate.
4
+ import { closeSync, constants, fstatSync, fsyncSync, lstatSync, mkdtempSync, openSync,
5
+ renameSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
6
+ import { randomBytes, createHash } from "node:crypto";
7
+ import { hostname } from "node:os";
8
+ import { join } from "node:path";
9
+ import { captureContext } from "./capture-context.mjs";
10
+ import { holderLiveness, readCaptureLockOwner } from "./capture-lock.mjs";
11
+ import { captureScratchRoot, captureScratchDirectory, readCaptureScratchMarker,
12
+ sameCaptureScratchMarker } from "./capture-scratch.mjs";
13
+
14
+ const hash = value => createHash("sha256").update(value).digest("hex");
15
+ const identity = stat => ({ dev: String(stat.dev), ino: String(stat.ino) });
16
+ const equal = (a, b) => JSON.stringify(a) === JSON.stringify(b);
17
+ const stamp = marker => ({ identity: identity(marker.stat), hash: hash(marker.raw) });
18
+ const validIdentity = value => value && ["dev", "ino"].every(k => typeof value[k] === "string" && /^\d+$/.test(value[k]));
19
+ const validStamp = value => value && validIdentity(value.identity) && /^[a-f0-9]{64}$/.test(value.hash);
20
+
21
+ function worker(root, start) {
22
+ const owner = readCaptureLockOwner(root);
23
+ if (!process.connected || start?.root !== root || start.parent?.pid !== process.ppid ||
24
+ owner?.pid !== process.ppid || owner.host !== hostname() || owner.nonce !== start.parent.nonce ||
25
+ owner.worker?.pid !== process.pid || owner.worker.host !== hostname() || owner.worker.nonce !== start.workerNonce) {
26
+ throw new Error("retained capture scratch requires the registered root-lock worker");
27
+ }
28
+ }
29
+ function context(store, stream, start) {
30
+ worker(store.root, start);
31
+ const current = captureContext(store);
32
+ if (!current?.state.depth || !current.held.has(stream)) throw new Error("retained capture scratch requires its stream work group");
33
+ return current.state;
34
+ }
35
+ function claimFor(state, key) {
36
+ const claim = state.get("recovery-allocation", key);
37
+ if (claim === null) return null;
38
+ if (claim.v !== 1 || typeof claim.stream !== "string" || !claim.stream ||
39
+ !/^recovery-[A-Za-z0-9]{6}$/.test(claim.name) || !/^[a-f0-9]{32}$/.test(claim.id) ||
40
+ ![claim.base, claim.directory, claim.payload].every(validIdentity) || !validStamp(claim.owner) ||
41
+ !["active", "deleting"].includes(claim.phase)) throw new Error("invalid retained capture job reference");
42
+ return claim;
43
+ }
44
+ function location(root, claim) {
45
+ const base = captureScratchRoot(root, false), path = join(base.path, claim.name);
46
+ if (!equal(identity(base.stat), claim.base) || !equal(identity(captureScratchDirectory(path)), claim.directory)) {
47
+ throw new Error("retained capture allocation identity changed");
48
+ }
49
+ return { base, path };
50
+ }
51
+ function ownerAt(path, key, claim) {
52
+ const marker = readCaptureScratchMarker(path), value = marker?.value;
53
+ if (!marker || value.v !== 2 || value.kind !== "recovery" || value.allocation !== claim.id || value.job !== hash(key) ||
54
+ value.host !== hostname() || !Number.isInteger(value.pid) || value.pid <= 0 || value.pid > 2147483647 ||
55
+ !/^[a-f0-9]{32}$/.test(value.nonce) || !validIdentity(value.publication) ||
56
+ !equal(value.publication, identity(marker.stat)) ||
57
+ !(equal(stamp(marker), claim.owner) || validStamp(value.previous) && equal(value.previous, claim.owner))) {
58
+ throw new Error("retained capture owner differs from its durable reference");
59
+ }
60
+ return marker;
61
+ }
62
+ function payloadAt(path, claim, { allowMissing = false } = {}) {
63
+ try {
64
+ const stat = lstatSync(join(path, "payload"), { bigint: true });
65
+ if (!stat.isFile() || stat.isSymbolicLink() || !equal(identity(stat), claim.payload) ||
66
+ (stat.mode & 0o077n) !== 0n || (process.getuid && stat.uid !== BigInt(process.getuid()))) {
67
+ throw new Error("retained capture payload identity or privacy changed");
68
+ }
69
+ return stat;
70
+ } catch (error) { if (allowMissing && error.code === "ENOENT") return null; throw error; }
71
+ }
72
+ function assertReference(root, state, key, claim, path, marker, start) {
73
+ worker(root, start);
74
+ if (!equal(claimFor(state, key), claim) || location(root, claim).path !== path ||
75
+ !sameCaptureScratchMarker(marker, readCaptureScratchMarker(path))) throw new Error("retained capture reference changed");
76
+ }
77
+ function publish(path, data, check) {
78
+ const temporary = join(path, `.owner-${randomBytes(16).toString("hex")}.tmp`);
79
+ let fd, dir, made, renamed = false;
80
+ try {
81
+ dir = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW);
82
+ fd = openSync(temporary, "wx", 0o600); made = identity(fstatSync(fd));
83
+ // The self identity distinguishes this publication from copied/replaced
84
+ // bytes, even in the crash window before its stamp reaches the DB.
85
+ writeFileSync(fd, JSON.stringify({ ...data, publication: made })); fsyncSync(fd);
86
+ check();
87
+ if (!equal(identity(lstatSync(temporary)), made)) throw new Error("retained capture owner publication changed");
88
+ renameSync(temporary, join(path, "owner.json")); renamed = true; fsyncSync(dir);
89
+ const marker = readCaptureScratchMarker(path);
90
+ if (!marker || !equal(identity(marker.stat), made)) throw new Error("retained capture owner publication failed");
91
+ return marker;
92
+ } finally {
93
+ if (!renamed && made) {
94
+ try { if (equal(identity(lstatSync(temporary)), made)) unlinkSync(temporary); } catch { /* uncertain files remain */ }
95
+ }
96
+ if (fd !== undefined) closeSync(fd); if (dir !== undefined) closeSync(dir);
97
+ }
98
+ }
99
+ function ownerValue(key, id, start, previous) {
100
+ return { v: 2, kind: "recovery", allocation: id, job: hash(key), host: hostname(), pid: process.pid,
101
+ nonce: start.workerNonce, ...(previous ? { previous } : {}) };
102
+ }
103
+ function handle(root, state, key, claim, path, marker, start, fd) {
104
+ let closed = false;
105
+ return { path, claim, fd,
106
+ verify() {
107
+ if (closed) throw new Error("retained capture allocation is closed");
108
+ assertReference(root, state, key, claim, path, marker, start);
109
+ const named = payloadAt(path, claim, { allowMissing: claim.phase === "deleting" });
110
+ if (fd !== null && (!named || !equal(identity(fstatSync(fd)), claim.payload))) throw new Error("retained capture descriptor changed");
111
+ },
112
+ close() { if (!closed && fd !== null) closeSync(fd); closed = true; },
113
+ };
114
+ }
115
+
116
+ export function createRetainedCaptureScratch(store, stream, key, start) {
117
+ const state = context(store, stream, start);
118
+ if (typeof key !== "string" || !key || claimFor(state, key)) throw new Error("retained capture job already exists or is invalid");
119
+ const base = captureScratchRoot(store.root, true), path = mkdtempSync(join(base.path, "recovery-"));
120
+ const directory = identity(captureScratchDirectory(path)), id = randomBytes(16).toString("hex");
121
+ let fd, parent, rootFd;
122
+ try {
123
+ fd = openSync(join(path, "payload"), "wx+", 0o600); fsyncSync(fd);
124
+ const marker = publish(path, ownerValue(key, id, start), () => {
125
+ worker(store.root, start);
126
+ if (!equal(identity(captureScratchDirectory(path)), directory) ||
127
+ !equal(identity(captureScratchRoot(store.root, false).stat), identity(base.stat))) throw new Error("retained capture allocation changed before publication");
128
+ });
129
+ parent = openSync(base.path, constants.O_RDONLY | constants.O_NOFOLLOW); fsyncSync(parent);
130
+ rootFd = openSync(store.root, "r"); fsyncSync(rootFd);
131
+ const claim = { v: 1, stream, id, name: path.slice(base.path.length + 1), base: identity(base.stat),
132
+ directory, payload: identity(fstatSync(fd)), owner: stamp(marker), phase: "active" };
133
+ payloadAt(path, claim);
134
+ state.set("recovery-allocation", key, claim);
135
+ return handle(store.root, state, key, claim, path, marker, start, fd);
136
+ } catch (error) {
137
+ if (fd !== undefined) closeSync(fd);
138
+ // A killed/uncommitted creation has no durable reference. It is not safe
139
+ // to infer cleanup authority later merely from its pathname or age.
140
+ throw error;
141
+ } finally { if (parent !== undefined) closeSync(parent); if (rootFd !== undefined) closeSync(rootFd); }
142
+ }
143
+
144
+ export function openRetainedCaptureScratch(store, stream, key, start, { cleanup = false, liveness = holderLiveness } = {}) {
145
+ const state = context(store, stream, start); let claim = claimFor(state, key);
146
+ if (!claim || claim.stream !== stream) throw new Error("retained capture job reference is missing or belongs to another stream");
147
+ const { path } = location(store.root, claim); let marker = ownerAt(path, key, claim), fd = null;
148
+ const payload = payloadAt(path, claim, { allowMissing: cleanup || claim.phase === "deleting" });
149
+ if (payload) {
150
+ fd = openSync(join(path, "payload"), constants.O_RDWR | constants.O_NOFOLLOW | constants.O_NONBLOCK);
151
+ if (!equal(identity(fstatSync(fd)), claim.payload)) { closeSync(fd); throw new Error("retained capture payload changed during open"); }
152
+ }
153
+ try {
154
+ const mine = marker.value.pid === process.pid && marker.value.nonce === start.workerNonce;
155
+ if (!mine) {
156
+ if (liveness(marker.value.pid) !== "dead") throw new Error("retained capture owner is not proven dead");
157
+ marker = publish(path, ownerValue(key, claim.id, start, claim.owner), () => {
158
+ assertReference(store.root, state, key, claim, path, marker, start);
159
+ payloadAt(path, claim, { allowMissing: cleanup || claim.phase === "deleting" });
160
+ if (liveness(marker.value.pid) !== "dead") throw new Error("retained capture owner death became uncertain");
161
+ });
162
+ }
163
+ // Publication may precede this transaction's commit. A successor accepts
164
+ // the self-identified publication's previous stamp, never an arbitrary
165
+ // replacement marker, then publishes its own lease after death proof.
166
+ claim = { ...claim, owner: stamp(marker), ...(cleanup ? { phase: "deleting" } : {}) };
167
+ state.set("recovery-allocation", key, claim);
168
+ return handle(store.root, state, key, claim, path, marker, start, fd);
169
+ } catch (error) { if (fd !== null) closeSync(fd); throw error; }
170
+ }
171
+
172
+ // Resume deletion after either side of its filesystem/DB boundary. Once the
173
+ // exact allocation is gone, there is no owner marker to adopt. Only an already
174
+ // committed deleting claim in the unchanged scratch base qualifies absence;
175
+ // a missing owner/payload inside an extant directory is still uncertain.
176
+ export function prepareRetainedCaptureCleanup(store, stream, key, start) {
177
+ const state = context(store, stream, start), claim = claimFor(state, key);
178
+ if (!claim || claim.stream !== stream) throw new Error("retained cleanup reference is missing or differs");
179
+ try { location(store.root, claim); }
180
+ catch (error) {
181
+ if (error.code !== "ENOENT" || claim.phase !== "deleting" ||
182
+ !equal(identity(captureScratchRoot(store.root, false).stat), claim.base)) throw error;
183
+ return;
184
+ }
185
+ openRetainedCaptureScratch(store, stream, key, start, { cleanup: true }).close();
186
+ }
187
+
188
+ // Mark deletion through open(..., {cleanup:true}), COMMIT that group, then
189
+ // remove outside a DB transaction. The caller can finalize many removed job
190
+ // references in one later transaction. A crash at either boundary is repeatable.
191
+ export function removeRetainedCaptureScratch(root, state, key, start) {
192
+ worker(root, start);
193
+ if (state.depth) throw new Error("retained capture cleanup needs a committed deletion mark");
194
+ const claim = claimFor(state, key);
195
+ if (!claim || claim.phase !== "deleting") throw new Error("retained capture cleanup lacks a durable deletion mark");
196
+ let where;
197
+ try { where = location(root, claim); }
198
+ catch (error) {
199
+ if (error.code === "ENOENT") {
200
+ // Absence is completion only for the exact referenced allocation. A
201
+ // replaced scratch base must still refuse even if this child is absent.
202
+ if (!equal(identity(captureScratchRoot(root, false).stat), claim.base)) throw error;
203
+ return true;
204
+ }
205
+ throw error;
206
+ }
207
+ const { path } = where, marker = ownerAt(path, key, claim);
208
+ if (marker.value.pid !== process.pid || marker.value.nonce !== start.workerNonce) throw new Error("retained capture cleanup requires adoption by this worker");
209
+ payloadAt(path, claim, { allowMissing: true });
210
+ assertReference(root, state, key, claim, path, marker, start);
211
+ rmSync(path, { recursive: true });
212
+ return true;
213
+ }
@@ -0,0 +1,102 @@
1
+ // Private retry queue for the root-lock-owning worker. A predecessor must have
2
+ // exited before reconciliation: a durable group start proves no member failed.
3
+ // There is no diagnostic execution mode; due members use normal single-stream
4
+ // admissions after healthy work. Public synchronous writers do not enter here.
5
+ import { readCaptureAdmission, readCaptureRetry } from "./capture-admission.mjs";
6
+ import { failureAfterAttempt, retryAfterAttempt, FAILURE_REASONS } from "./capture-failures.mjs";
7
+ import { fileVersion, sameVersion } from "./derived-state.mjs";
8
+ import { keysetRows } from "./sqlite-rows.mjs";
9
+
10
+ export function reconcileCaptureInterruption(state, { reason = "interrupted", now = Date.now() } = {}) {
11
+ if (state.depth) throw new Error("interruption reconciliation must precede processing");
12
+ if (!FAILURE_REASONS.includes(reason)) throw new Error("invalid capture interruption reason");
13
+ const active = readCaptureAdmission(state);
14
+ if (!active) return null;
15
+ return state.transaction(() => {
16
+ if (readCaptureAdmission(state)?.id !== active.id) throw new Error("capture admission changed before reconciliation");
17
+ const single = active.members.length === 1;
18
+ const failure = { id: active.id, runId: active.runId, reason: single ? reason : "interrupted",
19
+ memberCount: active.members.length };
20
+ for (const member of active.members) {
21
+ let nextRetryAt = retryAfterAttempt(1, now);
22
+ if (single) {
23
+ const row = failureAfterAttempt(state.get("capture-failed", member.stream), {
24
+ stream: member.stream, offset: member.offset, reason,
25
+ }, now);
26
+ nextRetryAt = row.nextRetryAt;
27
+ state.set("capture-failed", member.stream, { ...row, unitId: active.id,
28
+ stage: active.stage, generation: member.generation });
29
+ }
30
+ state.set("capture-retry", member.stream, { ...member, stage: active.stage,
31
+ unitId: active.id, attemptedRun: active.runId, nextRetryAt });
32
+ }
33
+ // This fact is consumed by the run/status writer. Unknown group membership
34
+ // is pending work, never a collection of invented public failed rows.
35
+ state.set("capture-run-failure", "last", failure);
36
+ state.delete("capture-admission", "active");
37
+ return failure;
38
+ });
39
+ }
40
+
41
+ // candidates is the scheduler's finite, fairly ordered sweep, not a history
42
+ // inventory retained here. Scanning it entails metadata work; journal content
43
+ // is never read. The caller supplies the next sweep position with each item.
44
+ export function selectCaptureWork(store, state, candidates, { runId, now = Date.now(), limit = 128,
45
+ allowRetry = true, retryFilter = () => true, stopped = () => false } = {}) {
46
+ if (typeof runId !== "string" || !runId || !Number.isSafeInteger(now) || now < 0 ||
47
+ !Number.isSafeInteger(limit) || limit < 1 || limit > 128 || typeof allowRetry !== "boolean" ||
48
+ typeof retryFilter !== "function" || typeof stopped !== "function") throw new Error("invalid capture queue selection");
49
+ const members = [], seen = new Set(); let stage, next;
50
+ for (const item of candidates) {
51
+ if (stopped()) break;
52
+ if (readCaptureRetry(state, item.stream) || seen.has(item.stream)) continue;
53
+ if (stage !== undefined && stage !== item.stage) break;
54
+ stage = item.stage; next = item.next; seen.add(item.stream);
55
+ members.push({ stream: item.stream, offset: item.offset, generation: item.generation });
56
+ if (members.length === limit) break;
57
+ }
58
+ if (stopped()) return null;
59
+ if (members.length) return { runId, stage, next, members };
60
+ if (!allowRetry) return null;
61
+
62
+ // Keyset iteration retains only a bounded page. Dates are not used to erase
63
+ // rows, and an incomplete member cannot be admitted repeatedly in one pass.
64
+ const after = state.get("capture-schedule", "retry-after") ?? "";
65
+ if (typeof after !== "string") throw new Error("invalid capture retry position");
66
+ function* deferred() {
67
+ for (const comparison of [">", "<="]) {
68
+ yield* keysetRows(state.db, `SELECT key FROM state WHERE kind = 'capture-retry' AND key ${comparison} ?`, "key", [after]);
69
+ }
70
+ }
71
+ for (const { key } of deferred()) {
72
+ if (stopped()) return null;
73
+ const retry = readCaptureRetry(state, key);
74
+ if (retry.attemptedRun === runId || now < Date.parse(retry.nextRetryAt) || !retryFilter(retry)) continue;
75
+ const generation = fileVersion(store.journalPath(retry.stream));
76
+ return { runId, stage: retry.stage, retry: true,
77
+ next: { stage: retry.stage, after: retry.stream },
78
+ members: [{ stream: retry.stream, generation,
79
+ offset: sameVersion(generation, retry.generation) ? retry.offset : 0 }] };
80
+ }
81
+ return null;
82
+ }
83
+
84
+ // Call inside the processing transaction only AFTER committing the stage's
85
+ // work through the failed snapshot, or validating that it no longer needs work.
86
+ // A checkpoint inside that snapshot cannot erase a multi-frame unit's failure.
87
+ // Index commits precede this capture-state transaction's commit; a crash between
88
+ // them conservatively leaves the retry visible for reconciliation.
89
+ export function resolveCaptureRetry(state, { id, stream, through, reconciled = false }) {
90
+ const active = readCaptureAdmission(state), retry = readCaptureRetry(state, stream);
91
+ if (!state.depth || active?.id !== id || !active.retry || active.members.length !== 1 ||
92
+ active.members[0].stream !== stream || !retry || retry.attemptedRun !== active.runId) {
93
+ throw new Error("capture retry resolution requires its active processing transaction");
94
+ }
95
+ if (!reconciled && (!sameVersion(active.members[0].generation, retry.generation) ||
96
+ !Number.isSafeInteger(through) || through < (retry.generation?.size ?? 1))) return false;
97
+ const failed = state.get("capture-failed", stream);
98
+ if (failed && failed.unitId !== retry.unitId) throw new Error("capture failure changed before resolution");
99
+ state.delete("capture-retry", stream);
100
+ if (failed) state.delete("capture-failed", stream);
101
+ return true;
102
+ }