@awebai/oats 0.60.0 → 0.62.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/README.md +15 -19
  2. package/bin/oats.mjs +68 -21
  3. package/docs/capabilities.md +1 -1
  4. package/docs/configuration.md +9 -0
  5. package/docs/desktop-cli-api.md +28 -5
  6. package/docs/desktop.md +7 -0
  7. package/docs/execution-targets.md +49 -3
  8. package/docs/first-team.md +8 -0
  9. package/docs/implementation.md +2 -1
  10. package/docs/named-test-runs.md +382 -0
  11. package/docs/official-catalog.md +1 -1
  12. package/docs/packages.md +5 -5
  13. package/docs/release-notes/v0.61.0.md +155 -0
  14. package/docs/release-notes/v0.62.0.md +223 -0
  15. package/docs/tui.md +10 -4
  16. package/docs/upgrading.md +133 -11
  17. package/docs/workspaces.md +1 -1
  18. package/lib/core.mjs +2 -2
  19. package/lib/doctor-capture.mjs +35 -0
  20. package/lib/session-viewer.mjs +86 -7
  21. package/package-catalog.json +1 -1
  22. package/package.json +1 -1
  23. package/packages/client/cli-locator.mjs +2 -2
  24. package/packages/client/instance-presentation.mjs +10 -0
  25. package/packages/client/instance-tree.mjs +48 -6
  26. package/packages/record/README.md +87 -13
  27. package/packages/record/bin/capture-hook.mjs +8 -0
  28. package/packages/record/bin/capture.mjs +143 -124
  29. package/packages/record/bin/recall.mjs +8 -3
  30. package/packages/record/bin/setup.mjs +115 -225
  31. package/packages/record/bin/turn-record.mjs +2 -2
  32. package/packages/record/docs/capture-service.md +1443 -28
  33. package/packages/record/docs/capture-status.schema.json +429 -12
  34. package/packages/record/lib/capture-admission.mjs +82 -0
  35. package/packages/record/lib/capture-aw-unit.mjs +137 -0
  36. package/packages/record/lib/capture-aw.mjs +31 -12
  37. package/packages/record/lib/capture-budget.mjs +31 -0
  38. package/packages/record/lib/capture-cc.mjs +39 -6
  39. package/packages/record/lib/capture-cleanup-queue.mjs +135 -0
  40. package/packages/record/lib/capture-cleanup-work.mjs +77 -0
  41. package/packages/record/lib/capture-command-operation.mjs +37 -0
  42. package/packages/record/lib/capture-command-output.mjs +61 -0
  43. package/packages/record/lib/capture-command-worker.mjs +6 -0
  44. package/packages/record/lib/capture-command.mjs +48 -0
  45. package/packages/record/lib/capture-completion.mjs +152 -0
  46. package/packages/record/lib/capture-context.mjs +50 -0
  47. package/packages/record/lib/capture-default-roots.mjs +30 -0
  48. package/packages/record/lib/capture-discovery.mjs +114 -0
  49. package/packages/record/lib/capture-failures.mjs +45 -0
  50. package/packages/record/lib/capture-health-doctor.mjs +13 -0
  51. package/packages/record/lib/capture-health-installation.mjs +117 -0
  52. package/packages/record/lib/capture-health.mjs +116 -0
  53. package/packages/record/lib/capture-hook-ownership.mjs +58 -0
  54. package/packages/record/lib/capture-hook-runner.mjs +17 -0
  55. package/packages/record/lib/capture-hook.mjs +26 -0
  56. package/packages/record/lib/capture-index-inventory.mjs +205 -0
  57. package/packages/record/lib/capture-index-outcomes.mjs +113 -0
  58. package/packages/record/lib/capture-index-reset.mjs +109 -0
  59. package/packages/record/lib/capture-index-work.mjs +77 -0
  60. package/packages/record/lib/capture-issue-preparation.mjs +73 -0
  61. package/packages/record/lib/capture-issues.mjs +15 -2
  62. package/packages/record/lib/capture-lock.mjs +102 -16
  63. package/packages/record/lib/capture-native-unit.mjs +322 -0
  64. package/packages/record/lib/capture-pass.mjs +102 -0
  65. package/packages/record/lib/capture-recovery-snapshot.mjs +182 -0
  66. package/packages/record/lib/capture-retained-scratch.mjs +213 -0
  67. package/packages/record/lib/capture-retry.mjs +102 -0
  68. package/packages/record/lib/capture-run.mjs +125 -0
  69. package/packages/record/lib/capture-scratch.mjs +12 -3
  70. package/packages/record/lib/capture-service-definition.mjs +104 -0
  71. package/packages/record/lib/capture-service-install.mjs +362 -0
  72. package/packages/record/lib/capture-service-ownership.mjs +137 -0
  73. package/packages/record/lib/capture-service-runner.mjs +18 -0
  74. package/packages/record/lib/capture-small-snapshot.mjs +85 -0
  75. package/packages/record/lib/capture-source-discovery.mjs +319 -0
  76. package/packages/record/lib/capture-source-queue.mjs +199 -0
  77. package/packages/record/lib/capture-source-work.mjs +85 -0
  78. package/packages/record/lib/capture-state.mjs +16 -71
  79. package/packages/record/lib/capture-status.mjs +171 -0
  80. package/packages/record/lib/capture-supervisor.mjs +117 -0
  81. package/packages/record/lib/capture-watch.mjs +32 -0
  82. package/packages/record/lib/capture-work-session.mjs +238 -0
  83. package/packages/record/lib/capture-worker.mjs +58 -0
  84. package/packages/record/lib/derived-state.mjs +28 -15
  85. package/packages/record/lib/file-lines.mjs +22 -6
  86. package/packages/record/lib/formats.mjs +4 -3
  87. package/packages/record/lib/index-capture.mjs +97 -0
  88. package/packages/record/lib/index-db.mjs +3 -17
  89. package/packages/record/lib/index-reset.mjs +19 -0
  90. package/packages/record/lib/store.mjs +36 -11
  91. package/packages/record/lib/transcript-prefix.mjs +125 -0
  92. package/packages/tui/lib/client.mjs +2 -2
  93. package/packages/tui/lib/model.mjs +15 -5
  94. package/packages/tui/lib/views/palette.mjs +7 -3
  95. package/skills/oats-getting-started/SKILL.md +1 -1
  96. package/docs/macos-test-runs.md +0 -246
@@ -0,0 +1,117 @@
1
+ // Private process transport, separate from reconciliation and status writing.
2
+ // The caller supplies the package worker entry; no shell, inherited execArgv,
3
+ // or caller-controlled NODE_OPTIONS can override its fixed heap policy.
4
+ import { spawn } from "node:child_process";
5
+ import { CAPTURE_POLICY } from "./capture-budget.mjs";
6
+ import { acquireCaptureLock } from "./capture-lock.mjs";
7
+
8
+ // hrtime uses the same monotonic origin in both local processes. Startup and
9
+ // lock publication consume this budget; the child never starts a new 300s clock.
10
+ export const captureMonotonicMs = () => Number(process.hrtime.bigint() / 1000000n);
11
+ export function captureWorkerEnvironment(env = process.env) {
12
+ const out = {};
13
+ for (const key of ["HOME", "PATH", "TMPDIR", "TMP", "TEMP", "XDG_CACHE_HOME", "XDG_CONFIG_HOME", "LANG", "LC_ALL", "TZ", "USERPROFILE", "SYSTEMROOT"])
14
+ if (env[key] !== undefined) out[key] = env[key];
15
+ return out;
16
+ }
17
+
18
+ const outcomes = new Set(["complete", "partial", "failed"]);
19
+ const reasons = new Set([null, "deadline", "rss", "heap", "interrupted", "input", "io"]);
20
+ const MESSAGE_LENGTH = 512, CONTROL = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/;
21
+ // A failed worker's private one-line diagnostic for the supervising command's
22
+ // stderr, never published status. Parser messages can quote input bytes, so a
23
+ // SyntaxError is reduced to its class name.
24
+ export function captureWorkerMessage(error) {
25
+ const text = error instanceof SyntaxError ? "SyntaxError" : error instanceof Error ? String(error.message) : "non-Error exception";
26
+ const message = text.replace(new RegExp(CONTROL.source + "+", "g"), " ").trim().slice(0, MESSAGE_LENGTH).replace(/[\ud800-\udbff]$/, "").trim();
27
+ return message || "unexplained operational failure";
28
+ }
29
+ export function validWorkerResult(value) {
30
+ return value && outcomes.has(value.outcome) && reasons.has(value.reason) &&
31
+ (value.outcome !== "complete" || value.reason === null) &&
32
+ (value.outcome !== "failed" || value.reason !== null) &&
33
+ (value.message === undefined || value.outcome === "failed" && typeof value.message === "string" &&
34
+ value.message.length > 0 && value.message.length <= MESSAGE_LENGTH && !CONTROL.test(value.message)) &&
35
+ (value.continuation === undefined || typeof value.continuation === "boolean") &&
36
+ (value.resetRequired === undefined || value.resetRequired === true && value.outcome === "partial" && value.continuation === true) &&
37
+ !(value.outcome === "complete" && value.continuation);
38
+ }
39
+
40
+ /** Internal seams are for isolated tests, not public tuning. Only the worker
41
+ * writes run/status facts. The supervisor owns admission, process lifetime and
42
+ * its command result; a forced death leaves an unfinished worker receipt. */
43
+ export async function superviseCapture(root, options, {
44
+ workerPath, policy = CAPTURE_POLICY, drainSeconds = 5,
45
+ executable = process.execPath, env = process.env, clock = captureMonotonicMs,
46
+ acquire = acquireCaptureLock, spawnChild = spawn,
47
+ window, pass,
48
+ } = {}) {
49
+ if (typeof workerPath !== "string" || !workerPath) throw new Error("capture worker entry is required");
50
+ if (window && (![window.drainAt, window.finalizeAt, window.hardAt].every(Number.isFinite) ||
51
+ window.drainAt > window.finalizeAt || window.finalizeAt > window.hardAt)) throw new Error("invalid capture child time window");
52
+ const started = clock(), deadline = Math.min(started + policy.deadlineSeconds * 1000, window?.hardAt ?? Infinity);
53
+ const drainAt = Math.min(deadline, window?.drainAt ?? Math.max(started, deadline - drainSeconds * 1000));
54
+ const finalizeAt = Math.min(deadline, window?.finalizeAt ?? deadline);
55
+ const lock = acquire(root);
56
+ if (!lock.release) return { outcome: "overlap", reason: null, exitCode: 0, held: lock.held };
57
+ let child, timer, finish, forced, result, ready = false, diagnostics = "", outputBytes = 0;
58
+ const stop = reason => {
59
+ forced ??= reason;
60
+ // ChildProcess owns this exact child until exit/reaping. Never derive a
61
+ // process group, or signal a PID recovered from a persisted lock record.
62
+ if (child?.pid > 0 && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
63
+ };
64
+ const interrupted = () => stop("interrupted");
65
+ try {
66
+ process.on("SIGTERM", interrupted); process.on("SIGINT", interrupted);
67
+ child = spawnChild(executable, [`--max-old-space-size=${policy.heapMiB}`, "--no-warnings",
68
+ // Node 22.5–22.12 need this flag. It does not import SQLite or alter the
69
+ // heap, and unlike execArgv it cannot forward arbitrary preload code.
70
+ ...(Number(process.versions.node.split(".")[0]) === 22 && Number(process.versions.node.split(".")[1]) < 13 ? ["--experimental-sqlite"] : []),
71
+ workerPath], { env: captureWorkerEnvironment(env), stdio: ["ignore", "ignore", "pipe", "ipc"], serialization: "json" });
72
+ finish = new Promise(resolve => {
73
+ let spawnError;
74
+ child.once("error", () => { spawnError = true; stop("io"); });
75
+ child.once("close", (code, signal) => resolve({ code, signal, spawnError }));
76
+ });
77
+ child.stderr?.on("data", chunk => {
78
+ outputBytes += chunk.length;
79
+ if (outputBytes > 64 * 1024) { stop("io"); return; }
80
+ diagnostics += chunk.toString("utf8");
81
+ });
82
+ child.on("message", message => {
83
+ if (forced) return;
84
+ if (!ready && message?.type === "ready" && message.pid === child.pid &&
85
+ typeof message.nonce === "string" && /^[a-f0-9]{32}$/.test(message.nonce) &&
86
+ typeof message.startedAt === "string" && Number.isFinite(Date.parse(message.startedAt))) {
87
+ ready = true;
88
+ try {
89
+ lock.registerWorker(message);
90
+ if (clock() >= deadline) { stop("deadline"); return; }
91
+ child.send({ type: "start", root, options, policy, parent: { pid: process.pid, nonce: lock.nonce },
92
+ workerNonce: message.nonce, drainAt, finalizeAt, hardAt: deadline, pass,
93
+ reclaimedLock: lock.reclaimed ?? null }, error => { if (error) stop("io"); });
94
+ } catch { stop("io"); }
95
+ } else if (ready && !result && message?.type === "result" && validWorkerResult(message.result)) {
96
+ result = message.result;
97
+ } else stop("io");
98
+ });
99
+ timer = setTimeout(() => stop("deadline"), Math.max(0, deadline - clock()));
100
+ const ended = await finish;
101
+ if (forced || ended.spawnError || ended.signal || !result || ended.code !== (result.outcome === "failed" ? 1 : 0)) {
102
+ const heap = /heap out of memory|allocation failed.*heap/i.test(diagnostics);
103
+ return { outcome: "failed", reason: heap ? "heap" : forced ?? (ended.signal ? "interrupted" : "io"), exitCode: 1 };
104
+ }
105
+ return { ...result, exitCode: result.outcome === "failed" ? 1 : 0 };
106
+ } catch {
107
+ stop("io");
108
+ if (finish) await finish;
109
+ return { outcome: "failed", reason: "io", exitCode: 1 };
110
+ } finally {
111
+ clearTimeout(timer);
112
+ process.off("SIGTERM", interrupted); process.off("SIGINT", interrupted);
113
+ const released = lock.release();
114
+ // Cleanup failure is an operational error even after a successful worker.
115
+ if (!released.released) throw Object.assign(new Error("capture supervisor could not release its lock"), { release: released });
116
+ }
117
+ }
@@ -0,0 +1,32 @@
1
+ // Foreground notification scheduling only. A queued event never authorizes
2
+ // capture, selects roots or certifies progress; each pass owns those decisions.
3
+ export function captureWatchQueue(run, { onError = error => console.error(error), delay = 2000, setTimer = setTimeout, clearTimer = clearTimeout } = {}) {
4
+ let timer = null, active = null, pending = false, closed = false;
5
+ const drain = async () => {
6
+ while (pending && !closed) {
7
+ pending = false;
8
+ try { await run(); } catch (error) {
9
+ // A broken diagnostic sink must not strand the queue or turn a timer
10
+ // callback into an unhandled rejection. It supplies no run evidence.
11
+ try { onError(error); } catch { /* best-effort diagnostics */ }
12
+ }
13
+ }
14
+ };
15
+ const ready = () => {
16
+ timer = null;
17
+ if (closed) return;
18
+ pending = true;
19
+ if (!active) active = drain().finally(() => { active = null; });
20
+ };
21
+ return {
22
+ schedule() {
23
+ if (closed) return;
24
+ if (timer !== null) clearTimer(timer);
25
+ timer = setTimer(ready, delay);
26
+ },
27
+ // Do not abandon an admitted child. Its supervisor handles signals and
28
+ // reaps it; closing only cancels future notification-driven passes.
29
+ close() { closed = true; pending = false; if (timer !== null) clearTimer(timer); timer = null; },
30
+ async settled() { await active; },
31
+ };
32
+ }
@@ -0,0 +1,238 @@
1
+ // Private owning worker session. Bootstrap has already registered this worker
2
+ // under the root lock and reaped its predecessor. Root selection and ordinary
3
+ // command routing remain separate; completion requires a fresh all-stage audit.
4
+ import { randomUUID } from "node:crypto";
5
+ import { RecordStore } from "./store.mjs";
6
+ import { RecordIndex } from "./index-db.mjs";
7
+ import { DerivedState } from "./derived-state.mjs";
8
+ import { CaptureSourceQueue } from "./capture-source-queue.mjs";
9
+ import { captureSourceWork } from "./capture-source-work.mjs";
10
+ import { CaptureIndexInventory } from "./capture-index-inventory.mjs";
11
+ import { CaptureIndexOutcomes } from "./capture-index-outcomes.mjs";
12
+ import { CaptureIndexReset } from "./capture-index-reset.mjs";
13
+ import { CaptureRun } from "./capture-run.mjs";
14
+ import { readCaptureStatus } from "./capture-status.mjs";
15
+ import { reconcileCaptureInterruption, selectCaptureWork } from "./capture-retry.mjs";
16
+ import { readCaptureAdmission } from "./capture-admission.mjs";
17
+ import { FAILURE_REASONS } from "./capture-failures.mjs";
18
+ import { CaptureCleanupQueue } from "./capture-cleanup-queue.mjs";
19
+ import { CaptureSourceDiscovery } from "./capture-source-discovery.mjs";
20
+ import { auditCaptureCompletion, auditCaptureSourceRetirement } from "./capture-completion.mjs";
21
+ import { CaptureIssuePreparation } from "./capture-issue-preparation.mjs";
22
+
23
+ export class CaptureWorkSession {
24
+ constructor(start, { owner, selection, trigger = "manual", now = Date.now, unit = {}, roots = null, indexEnabled = true, issues = null } = {}) {
25
+ if (typeof start?.root !== "string" || typeof start.workerNonce !== "string" || !start.workerNonce ||
26
+ typeof start.pass?.runId !== "string" || !start.pass.runId ||
27
+ typeof start.pass.finalize !== "boolean" || typeof start.pass.reset !== "boolean" || typeof now !== "function" ||
28
+ typeof indexEnabled !== "boolean")
29
+ throw new Error("capture work session requires a supervised pass");
30
+ this.start = start; this.now = now; this.unit = unit; this.indexEnabled = indexEnabled;
31
+ try {
32
+ start.budget.check();
33
+ this.store = new RecordStore(start.root, { owner });
34
+ this.state = new DerivedState(start.root);
35
+ // Append-only workers must not even open an existing index: construction
36
+ // can migrate/rebuild it before any scheduling decision is made.
37
+ this.index = indexEnabled ? new RecordIndex(this.store) : null;
38
+ const prior = readCaptureStatus(start.root);
39
+ if (prior?.lastRun?.id === start.pass.runId && prior.lastRun.outcome !== "running")
40
+ throw new Error("capture run identity already finished");
41
+ const previous = start.pass.previous?.outcome === "failed" ? start.pass.previous.reason : null;
42
+ // An observed child failure belongs to this pass. It cannot diagnose an
43
+ // older abandoned admission merely because this child reconciles it.
44
+ const reasonFor = id => id === start.pass.runId && FAILURE_REASONS.includes(previous) ? previous : "interrupted";
45
+ reconcileCaptureInterruption(this.state, { reason: reasonFor(readCaptureAdmission(this.state)?.runId), now: now() });
46
+ this.cleanup = new CaptureCleanupQueue(this.store, this.state);
47
+ this.cleanup.reconcile(start, { reason: reasonFor(this.cleanup.active()?.runId), now: now() });
48
+ this.queue = new CaptureSourceQueue(this.state, { owner, selection });
49
+ const retirement = this.state.get("capture-sweep-retirement", owner);
50
+ if (!start.pass.finalize && retirement?.sourceScope === this.queue.current().id &&
51
+ retirement.runId === prior?.lastRun?.id && prior.lastRun.outcome !== "running") {
52
+ this.state.transaction(() => {
53
+ this.queue.finish();
54
+ this.state.delete("capture-sweep-retirement", owner);
55
+ });
56
+ this.queue = new CaptureSourceQueue(this.state, { owner, selection });
57
+ }
58
+ if (roots !== null) {
59
+ this.sourceDiscovery = new CaptureSourceDiscovery(this.queue, roots);
60
+ this.sourceDiscovery.reconcile(start, { reason: reasonFor(this.sourceDiscovery.active()?.runId), now: now() });
61
+ }
62
+ if (issues !== null) this.issues = new CaptureIssuePreparation(this.state, this.queue,
63
+ { ...issues, runId: start.pass.runId });
64
+ // Retain a reset's scope even if derived inventory tables were lost.
65
+ // Recreating membership may restart enumeration; it cannot erase reset
66
+ // admission or the index token needed to recover a committed operation.
67
+ if (indexEnabled) {
68
+ const previousSource = this.state.get("capture-index-source", owner);
69
+ const scope = this.state.get("capture-index-reset", "current")?.scope ??
70
+ (previousSource && previousSource.sourceScope !== this.queue.current().id ? randomUUID() :
71
+ this.state.get("index-inventory", "current")?.id ?? randomUUID());
72
+ this.openIndexScope(scope);
73
+ this.reset.reconcile(start, { reason: reasonFor(this.reset.current()?.attempt?.runId), now: now() });
74
+ }
75
+ const failures = [this.state.get("capture-run-failure", "last"),
76
+ this.state.get("capture-index-reset-failure", "last"), this.state.get("capture-cleanup-failure", "last"),
77
+ this.state.get("capture-discovery-failure", "last")].filter(Boolean);
78
+ const currentReason = start.pass.failure ?? failures.find(row => row.runId === start.pass.runId)?.reason ?? null;
79
+ const previousReason = failures.find(row => row.runId === prior?.lastRun?.id)?.reason ??
80
+ (prior?.lastRun?.id === start.pass.runId ? previous : null);
81
+ this.run = new CaptureRun(start.root, this.state, { id: start.pass.runId, trigger,
82
+ reclaimedLock: start.reclaimedLock ?? null, previousFailure: previousReason,
83
+ currentFailure: currentReason ? { id: start.pass.runId, reason: currentReason } : null });
84
+ } catch (error) { this.close(); throw error; }
85
+ }
86
+
87
+ openIndexScope(id) {
88
+ if (!this.indexEnabled) throw new Error("append-only capture cannot open index inventory");
89
+ this.inventory?.close();
90
+ this.inventory = new CaptureIndexInventory(this.state, { id });
91
+ this.outcomes = new CaptureIndexOutcomes(this.index, this.inventory);
92
+ this.reset = new CaptureIndexReset(this.index, this.inventory);
93
+ const binding = { sourceScope: this.queue.current().id, indexScope: id };
94
+ if (JSON.stringify(this.state.get("capture-index-source", this.queue.owner)) !== JSON.stringify(binding))
95
+ this.state.transaction(() => this.state.set("capture-index-source", this.queue.owner, binding));
96
+ }
97
+
98
+ sourceReady(input) {
99
+ if (!input) return false;
100
+ if (input.source === "aw") return true;
101
+ // Native recovery keys bind the exact input, not every source sharing a
102
+ // stream. A deleting allocation is a dependency, even if its directory is
103
+ // already gone: admitting cleanup-only capture would starve its due retry.
104
+ const claim = this.state.get("recovery-allocation", `${input.stream}:${input.path}`);
105
+ if (claim?.phase !== "deleting") return true;
106
+ if (claim.stream !== input.stream || !/^[a-f0-9]{32}$/.test(claim.id))
107
+ throw new Error("invalid native cleanup dependency");
108
+ return false;
109
+ }
110
+
111
+ process(stage, allowRetry) {
112
+ if (stage !== "capture" && (stage !== "index" || !this.indexEnabled))
113
+ throw new Error("capture stage is not enabled");
114
+ const options = { ...this.unit, now: this.now(), allowRetry,
115
+ inputReady: input => this.sourceReady(input),
116
+ retryFilter: retry => retry.stage === stage && (stage !== "capture" || this.sourceReady(this.queue.forStream(retry.stream))) };
117
+ const result = stage === "capture"
118
+ ? captureSourceWork(this.store, this.state, this.queue, this.run, this.start, { ...options, indexInventory: this.inventory ?? null })
119
+ : this.outcomes.advance(this.run, this.start, { ...options, memo: true });
120
+ if (result.resetRequired) this.reset.request(this.start);
121
+ // One independent bounded cleanup turn cannot change stream-stage rotation.
122
+ // A throw retains the already-committed source/index counts and its own
123
+ // admission, so a replacement can back off deletion and keep healthy work.
124
+ const cleanupWork = result.selected ? this.clean(false) : null;
125
+ return { stage, ...result, cleanupWork };
126
+ }
127
+
128
+ clean(retry) {
129
+ return this.cleanup.advance(this.run, this.start, { now: this.now(), retry,
130
+ maxRecords: Math.max(2, this.unit.maxRecords ?? 2000) });
131
+ }
132
+
133
+ collect() {
134
+ if (this.state.db.prepare("SELECT 1 AS found FROM state WHERE kind='source-queue-garbage' OR (? AND kind='index-inventory-garbage') LIMIT 1")
135
+ .get(Number(this.indexEnabled))?.found !== 1)
136
+ return null;
137
+ return this.state.transaction(() => {
138
+ const sources = this.queue.cleanup(), streams = this.inventory?.cleanup() ?? 0;
139
+ // Retiring obsolete enumeration is bookkeeping, not new capture/index
140
+ // progress. In particular it must not refresh every no-op pass's history.
141
+ return { stage: "inventory-cleanup", sources, streams };
142
+ });
143
+ }
144
+
145
+ step() {
146
+ const stopped = () => this.start.budget.sample() !== null;
147
+ if (stopped()) return { stopped: true };
148
+ if (this.reset?.current()) {
149
+ const result = this.reset.advance(this.run, this.start, { now: this.now() });
150
+ if (result.reset) { this.openIndexScope(result.scope); return { stage: "reset", ...result }; }
151
+ if (!this.start.pass.finalize && result.needsFreshWorker) return { resetRequired: true };
152
+ }
153
+ // A committed reset token may finish bounded publication above. Neither
154
+ // source/index work nor new atomic reset may use finalization allowance.
155
+ if (this.start.pass.finalize) return { finalize: true };
156
+
157
+ if (this.issues && !this.issues.complete())
158
+ return { stage: "issue-preparation", ...this.issues.advance(this.start.root, { stopped }) };
159
+
160
+ const sourceDiscovery = this.sourceDiscovery && !this.queue.current().discovered
161
+ ? this.sourceDiscovery.advance({ stopped, limit: this.unit.discoveryLimit ?? 128,
162
+ runId: this.run.id, workerNonce: this.start.workerNonce, now: this.now() }) : null;
163
+ let discovery = null;
164
+ if (this.inventory && this.inventory.current().phase !== "complete") discovery = this.inventory.discover(this.index, { stopped });
165
+ if (stopped()) return { stopped: true, discovery, sourceDiscovery };
166
+ const blockedIndex = !this.indexEnabled || this.reset.current() !== null;
167
+ // Reuse the last durable admission's stage. Failed groups rotate too, and
168
+ // routing adds no separate per-record or per-unit scheduling transaction.
169
+ const last = this.state.get("capture-schedule", "position")?.stage;
170
+ for (const stage of last === "capture" ? ["index", "capture"] : ["capture", "index"]) {
171
+ if (stage === "index" && blockedIndex) continue;
172
+ const result = this.process(stage, false);
173
+ if (result.selected || result.resetRequired || result.stopped) return { ...result, discovery, sourceDiscovery };
174
+ }
175
+ if (stopped()) return { stopped: true, discovery, sourceDiscovery };
176
+ const cleanup = this.clean(false);
177
+ if (cleanup.selected || !cleanup.complete) return { stage: "cleanup", ...cleanup, discovery, sourceDiscovery };
178
+ if (stopped()) return { stopped: true, discovery, sourceDiscovery };
179
+ const collected = this.collect();
180
+ if (collected) return { ...collected, discovery, sourceDiscovery };
181
+ // Unknown eligible discovery can still contain healthy work. A refused
182
+ // root remains pending but cannot withhold unrelated due work indefinitely.
183
+ if ((this.sourceDiscovery ? this.sourceDiscovery.healthyPending() : !this.queue.current().discovered) ||
184
+ this.inventory && this.inventory.current().phase !== "complete")
185
+ return { waitingForDiscovery: true, discovery, sourceDiscovery };
186
+ const selection = selectCaptureWork(this.store, this.state, [], { runId: this.run.id, now: this.now(), stopped,
187
+ retryFilter: retry => retry.stage === "index" ? !blockedIndex :
188
+ retry.stage === "capture" && this.sourceReady(this.queue.forStream(retry.stream)) });
189
+ if (selection) return { ...this.process(selection.stage, true), discovery, sourceDiscovery };
190
+ const cleanupRetry = this.clean(true);
191
+ if (cleanupRetry.selected || !cleanupRetry.complete) return { stage: "cleanup", ...cleanupRetry, discovery, sourceDiscovery };
192
+ if (this.sourceDiscovery && !this.queue.current().discovered) {
193
+ if (sourceDiscovery?.source) return { waitingForDiscovery: true, discovery, sourceDiscovery };
194
+ const retried = this.sourceDiscovery.advance({ stopped, limit: this.unit.discoveryLimit ?? 128,
195
+ runId: this.run.id, workerNonce: this.start.workerNonce, now: this.now(), retry: true });
196
+ if (retried.source) return { waitingForDiscovery: true, discovery, sourceDiscovery: retried };
197
+ }
198
+ return { idle: true, discovery, sourceDiscovery };
199
+ }
200
+
201
+ // Yield/stop never infer completion. The owning worker calls complete() only
202
+ // when it wants a final all-stage decision; idle is merely a scheduling hint.
203
+ yieldWorker({ resetRequired = false } = {}) {
204
+ if (this.start.pass.finalize) return this.finish({ reason: "deadline" });
205
+ if (this.start.budget.reason === "interrupted") return this.finish({ reason: "interrupted" });
206
+ this.run.publish({ pending: true, peakRssBytes: this.start.budget.peakRssBytes });
207
+ return { outcome: "partial", reason: this.start.budget.reason ?? "deadline", continuation: true,
208
+ ...(resetRequired ? { resetRequired: true } : {}) };
209
+ }
210
+
211
+ finish({ reason = "input" } = {}) {
212
+ const snapshot = this.run.finish({ outcome: reason === "interrupted" ? "failed" : "partial", reason,
213
+ pending: true, peakRssBytes: this.start.budget.peakRssBytes });
214
+ return { outcome: snapshot.lastRun.outcome, reason: snapshot.lastRun.reason, continuation: false };
215
+ }
216
+
217
+ complete(options) {
218
+ if (this.start.pass.finalize || this.start.budget.sample() !== null)
219
+ return this.finish({ reason: this.start.budget.reason ?? "deadline" });
220
+ const audit = this.indexEnabled ? auditCaptureCompletion(this, options) : auditCaptureSourceRetirement(this, options);
221
+ if (audit.stopped || audit.needsWork) return this.yieldWorker();
222
+ // Normally these facts entered the run during replacement construction.
223
+ // Also retain a caught failure if an owning caller continues in this worker.
224
+ const failure = ["capture-run-failure", "capture-index-reset-failure", "capture-cleanup-failure", "capture-discovery-failure"]
225
+ .map(kind => this.state.get(kind, "last")).find(row => row?.runId === this.run.id);
226
+ const snapshot = this.run.finish({ outcome: failure ? "failed" : audit.complete ? "complete" : "partial",
227
+ reason: failure?.reason ?? (audit.complete ? null : "input"),
228
+ pending: audit.pending, peakRssBytes: this.start.budget.peakRssBytes });
229
+ return { outcome: snapshot.lastRun.outcome, reason: snapshot.lastRun.reason, continuation: false };
230
+ }
231
+
232
+ close() {
233
+ this.sourceDiscovery?.close();
234
+ this.inventory?.close();
235
+ try { this.index?.close(); } finally { this.state?.close(); }
236
+ this.index = null; this.state = null;
237
+ }
238
+ }
@@ -0,0 +1,58 @@
1
+ // Private worker bootstrap. Reconciliation is supplied by the owning entry;
2
+ // this handshake never captures or writes status on behalf of the operation.
3
+ import { randomBytes } from "node:crypto";
4
+ import { CaptureBudget, CaptureStop } from "./capture-budget.mjs";
5
+ import { readCaptureLockOwner } from "./capture-lock.mjs";
6
+ import { captureMonotonicMs, captureWorkerMessage, validWorkerResult } from "./capture-supervisor.mjs";
7
+
8
+ export async function runCaptureWorker(operation) {
9
+ if (!process.send || !process.connected) throw new Error("capture worker requires its supervisor");
10
+ const nonce = randomBytes(16).toString("hex"), parentPid = process.ppid;
11
+ const start = await new Promise((resolve, reject) => {
12
+ const disconnected = () => { cleanup(); reject(new Error("capture supervisor disconnected before start")); };
13
+ const receive = message => {
14
+ cleanup();
15
+ if (message?.type !== "start" || message.workerNonce !== nonce || message.parent?.pid !== parentPid ||
16
+ typeof message.root !== "string" || !Number.isFinite(message.drainAt) ||
17
+ !Number.isFinite(message.finalizeAt) || !Number.isFinite(message.hardAt) ||
18
+ message.drainAt > message.finalizeAt || message.finalizeAt > message.hardAt) {
19
+ reject(new Error("invalid capture worker start")); return;
20
+ }
21
+ resolve(message);
22
+ };
23
+ const cleanup = () => { process.off("message", receive); process.off("disconnect", disconnected); };
24
+ process.once("message", receive); process.once("disconnect", disconnected);
25
+ process.send({ type: "ready", pid: process.pid, nonce, startedAt: new Date().toISOString() }, error => {
26
+ if (error) { cleanup(); reject(new Error("capture worker could not announce readiness")); }
27
+ });
28
+ });
29
+ const alive = () => {
30
+ if (!process.connected || process.ppid !== parentPid) return false;
31
+ const owner = readCaptureLockOwner(start.root);
32
+ return owner?.pid === parentPid && owner.nonce === start.parent.nonce &&
33
+ owner.worker?.pid === process.pid && owner.worker.nonce === nonce;
34
+ };
35
+ // Publication must already be durable before the first operation, including
36
+ // its first status write. A callback must check again between bounded units.
37
+ if (!alive()) throw new Error("capture worker lock publication is missing");
38
+ const budget = new CaptureBudget({ policy: start.policy, clock: captureMonotonicMs,
39
+ drainAt: start.drainAt, supervisorAlive: alive });
40
+ let result;
41
+ try {
42
+ result = await operation({ ...start, budget });
43
+ if (!validWorkerResult(result)) throw new Error("invalid capture worker result");
44
+ } catch (error) {
45
+ // The operation owns failure receipts. Bootstrap never invents completed
46
+ // progress or rewrites a receipt after an exception.
47
+ const reason = error instanceof CaptureStop ? error.reason : budget.reason === "interrupted" ? "interrupted" : "io";
48
+ // An operational failure (status persistence among them) names its cause
49
+ // to the supervising command only; the run receipt keeps its fixed message.
50
+ result = { outcome: "failed", reason, ...(reason === "io" ? { message: captureWorkerMessage(error) } : {}) };
51
+ }
52
+ if (process.connected) {
53
+ await new Promise(resolve => process.send({ type: "result", result }, () => resolve()));
54
+ process.disconnect();
55
+ }
56
+ process.exitCode = result.outcome === "failed" ? 1 : 0;
57
+ return result;
58
+ }
@@ -29,19 +29,27 @@ export class DerivedState {
29
29
  const { DatabaseSync } = require("node:sqlite");
30
30
  mkdirSync(join(root, "index"), { recursive: true });
31
31
  this.db = new DatabaseSync(join(root, "index", "capture.db"));
32
- this.db.exec("PRAGMA busy_timeout = 60000; PRAGMA cache_size = -2048; PRAGMA temp_store = FILE");
33
- this.db.exec("BEGIN IMMEDIATE");
34
32
  try {
35
- const hadIds = this.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name='journal_ids'").get()?.found === 1;
36
- this.db.exec(`CREATE TABLE IF NOT EXISTS state (kind TEXT, key TEXT, value TEXT NOT NULL, PRIMARY KEY(kind, key));
37
- CREATE TABLE IF NOT EXISTS journal_ids (stream TEXT, id TEXT, PRIMARY KEY(stream, id));`);
38
- // Recreating a lost lookup atomically revokes the old validation witness.
39
- // A crash cannot qualify an empty replacement table with that witness.
40
- if (!hadIds) this.db.exec("DELETE FROM state WHERE kind = 'journal'");
41
- this.db.exec("COMMIT");
42
- } catch (err) { this.db.exec("ROLLBACK"); this.db.close(); throw err; }
43
- this.read = this.db.prepare("SELECT value FROM state WHERE kind = ? AND key = ?");
44
- this.write = this.db.prepare("INSERT INTO state VALUES (?, ?, ?) ON CONFLICT(kind,key) DO UPDATE SET value=excluded.value");
33
+ this.db.exec("PRAGMA busy_timeout = 60000; PRAGMA cache_size = -2048; PRAGMA temp_store = FILE");
34
+ // A healthy accelerator needs no schema write transaction on every open.
35
+ // Repair rechecks under the write lock, including a lost ID lookup.
36
+ const present = name => this.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name=?").get(name)?.found === 1;
37
+ if (!present("state") || !present("journal_ids")) {
38
+ this.db.exec("BEGIN IMMEDIATE");
39
+ try {
40
+ const hadIds = this.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name='journal_ids'").get()?.found === 1;
41
+ this.db.exec(`CREATE TABLE IF NOT EXISTS state (kind TEXT, key TEXT, value TEXT NOT NULL, PRIMARY KEY(kind, key));
42
+ CREATE TABLE IF NOT EXISTS journal_ids (stream TEXT, id TEXT, PRIMARY KEY(stream, id));`);
43
+ // Recreating a lost lookup atomically revokes the old validation witness.
44
+ // A crash cannot qualify an empty replacement table with that witness.
45
+ if (!hadIds) this.db.exec("DELETE FROM state WHERE kind = 'journal'");
46
+ this.db.exec("COMMIT");
47
+ } catch (err) { this.db.exec("ROLLBACK"); throw err; }
48
+ }
49
+ this.depth = 0;
50
+ this.read = this.db.prepare("SELECT value FROM state WHERE kind = ? AND key = ?");
51
+ this.write = this.db.prepare("INSERT INTO state VALUES (?, ?, ?) ON CONFLICT(kind,key) DO UPDATE SET value=excluded.value WHERE state.value != excluded.value");
52
+ } catch (err) { this.db.close(); throw err; }
45
53
  }
46
54
  get(kind, key) { const row = this.read.get(kind, key); return row ? JSON.parse(row.value) : null; }
47
55
  set(kind, key, value) { this.write.run(kind, key, JSON.stringify(value)); }
@@ -52,9 +60,14 @@ export class DerivedState {
52
60
  }
53
61
  }
54
62
  transaction(fn) {
55
- this.db.exec("BEGIN IMMEDIATE");
56
- try { const result = fn(); this.db.exec("COMMIT"); return result; }
57
- catch (err) { this.db.exec("ROLLBACK"); throw err; }
63
+ const nested = this.depth > 0, savepoint = `capture_${this.depth}`;
64
+ this.db.exec(nested ? `SAVEPOINT ${savepoint}` : "BEGIN IMMEDIATE");
65
+ this.depth++;
66
+ try { const result = fn(); this.db.exec(nested ? `RELEASE ${savepoint}` : "COMMIT"); return result; }
67
+ catch (err) {
68
+ this.db.exec(nested ? `ROLLBACK TO ${savepoint}; RELEASE ${savepoint}` : "ROLLBACK");
69
+ throw err;
70
+ } finally { this.depth--; }
58
71
  }
59
72
  close() { this.db.close(); }
60
73
  }
@@ -8,8 +8,9 @@ import { readSync } from "node:fs";
8
8
  // The bytes of `fd` from `start` up to `end` (default: the end of the file), in
9
9
  // chunks of at most `size` bytes. Each chunk is a fresh buffer, so a caller may
10
10
  // keep one past the next read.
11
- export function* fileChunks(fd, { start = 0, end = Infinity, size = 1 << 20 } = {}) {
11
+ export function* fileChunks(fd, { start = 0, end = Infinity, size = 1 << 20, check = () => {} } = {}) {
12
12
  for (let position = start; position < end; ) {
13
+ check();
13
14
  const chunk = Buffer.allocUnsafe(Math.min(size, end - position));
14
15
  const n = readSync(fd, chunk, 0, chunk.length, position);
15
16
  if (n === 0) return;
@@ -22,17 +23,32 @@ export function* fileChunks(fd, { start = 0, end = Infinity, size = 1 << 20 } =
22
23
  // then a final fragment without one, if the bytes end mid-line. A caller tells
23
24
  // them apart by the last byte. Fragments are joined only at a newline, so a
24
25
  // long line is copied once, not as each chunk arrives.
25
- export function* bufferLines(chunks) {
26
- let pieces = [], size = 0;
26
+ export class RecordSizeError extends Error {
27
+ constructor(limit, offset) {
28
+ super(`raw record exceeds ${limit} bytes at offset ${offset}`);
29
+ this.limit = limit; this.offset = offset;
30
+ }
31
+ }
32
+
33
+ // A line that starts before `limitFrom` (an offset into these chunks) was
34
+ // already captured and is not size-limited; the limit applies from there on.
35
+ export function* bufferLines(chunks, { maxBytes = Infinity, limitFrom = 0 } = {}) {
36
+ let pieces = [], size = 0, offset = 0;
37
+ const append = piece => {
38
+ // Enforce a bounded command's policy before joining or retaining the whole
39
+ // offending record. Public callers retain their unlimited default.
40
+ if (offset >= limitFrom && size + piece.length > maxBytes) throw new RecordSizeError(maxBytes, offset);
41
+ pieces.push(piece); size += piece.length;
42
+ };
27
43
  for (const chunk of chunks) {
28
44
  let from = 0;
29
45
  for (let nl = chunk.indexOf(10, from); nl >= 0; nl = chunk.indexOf(10, from)) {
30
- pieces.push(chunk.subarray(from, nl + 1));
31
- size += nl + 1 - from;
46
+ append(chunk.subarray(from, nl + 1));
32
47
  yield pieces.length === 1 ? pieces[0] : Buffer.concat(pieces, size);
48
+ offset += size;
33
49
  pieces = []; size = 0; from = nl + 1;
34
50
  }
35
- if (from < chunk.length) { pieces.push(chunk.subarray(from)); size += chunk.length - from; }
51
+ if (from < chunk.length) append(chunk.subarray(from));
36
52
  }
37
53
  if (size > 0) yield pieces.length === 1 ? pieces[0] : Buffer.concat(pieces, size);
38
54
  }
@@ -58,12 +58,13 @@ function* parsedLines(bytes) {
58
58
  // Preserve the public inventory's per-directory localeCompare order and stable
59
59
  // ties for ordinary capture too. Path metadata grows with directory size;
60
60
  // source contents are streamed independently of this discovery inventory.
61
- function* directoryEntries(path, optional = false) {
61
+ export function sortedSessionEntries(path, optional = false) {
62
62
  let entries;
63
63
  try { entries = readdirSync(path, { withFileTypes: true }); }
64
- catch (err) { if (optional && err.code === "ENOENT") return; throw err; }
65
- yield* entries.sort((a, b) => a.name.localeCompare(b.name));
64
+ catch (err) { if (optional && err.code === "ENOENT") return []; throw err; }
65
+ return entries.sort((a, b) => a.name.localeCompare(b.name));
66
66
  }
67
+ function* directoryEntries(path, optional = false) { yield* sortedSessionEntries(path, optional); }
67
68
  function* iterateJsonlFiles(root, maxDepth, { strict = false } = {}, allowCaptured = false) {
68
69
  const capturedPi = typeof root === "object" && root !== null ? root.capturedPi : undefined;
69
70
  if (typeof root !== "string") {