@awebai/oats 0.61.0 → 0.63.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +1 -1
  2. package/bin/oats.mjs +27 -12
  3. package/docs/capabilities.md +1 -1
  4. package/docs/desktop-cli-api.md +30 -2
  5. package/docs/execution-targets.md +89 -4
  6. package/docs/implementation.md +2 -1
  7. package/docs/named-test-runs.md +382 -0
  8. package/docs/official-catalog.md +1 -1
  9. package/docs/packages.md +5 -5
  10. package/docs/release-lane.md +13 -0
  11. package/docs/release-notes/v0.62.0.md +223 -0
  12. package/docs/release-notes/v0.63.0.md +135 -0
  13. package/docs/souls-and-instances.md +6 -1
  14. package/docs/upgrading.md +135 -11
  15. package/docs/workspaces.md +1 -1
  16. package/lib/core.mjs +73 -23
  17. package/lib/doctor-capture.mjs +35 -0
  18. package/lib/session-viewer.mjs +86 -7
  19. package/lib/tree-copy.mjs +188 -7
  20. package/package-catalog.json +1 -1
  21. package/package.json +1 -1
  22. package/packages/client/cli-locator.mjs +2 -2
  23. package/packages/record/README.md +87 -13
  24. package/packages/record/bin/capture-hook.mjs +8 -0
  25. package/packages/record/bin/capture.mjs +200 -139
  26. package/packages/record/bin/recall.mjs +8 -3
  27. package/packages/record/bin/setup.mjs +116 -225
  28. package/packages/record/bin/turn-record.mjs +2 -2
  29. package/packages/record/docs/capture-service.md +1527 -28
  30. package/packages/record/docs/capture-status.schema.json +429 -12
  31. package/packages/record/lib/capture-admission.mjs +82 -0
  32. package/packages/record/lib/capture-aw-unit.mjs +137 -0
  33. package/packages/record/lib/capture-aw.mjs +31 -12
  34. package/packages/record/lib/capture-budget.mjs +31 -0
  35. package/packages/record/lib/capture-cc.mjs +39 -6
  36. package/packages/record/lib/capture-cleanup-queue.mjs +135 -0
  37. package/packages/record/lib/capture-cleanup-work.mjs +77 -0
  38. package/packages/record/lib/capture-command-operation.mjs +37 -0
  39. package/packages/record/lib/capture-command-output.mjs +62 -0
  40. package/packages/record/lib/capture-command-worker.mjs +6 -0
  41. package/packages/record/lib/capture-command.mjs +65 -0
  42. package/packages/record/lib/capture-completion.mjs +152 -0
  43. package/packages/record/lib/capture-context.mjs +50 -0
  44. package/packages/record/lib/capture-default-roots.mjs +30 -0
  45. package/packages/record/lib/capture-discovery.mjs +114 -0
  46. package/packages/record/lib/capture-failures.mjs +45 -0
  47. package/packages/record/lib/capture-health-doctor.mjs +13 -0
  48. package/packages/record/lib/capture-health-installation.mjs +117 -0
  49. package/packages/record/lib/capture-health.mjs +116 -0
  50. package/packages/record/lib/capture-hook-ownership.mjs +58 -0
  51. package/packages/record/lib/capture-hook-runner.mjs +17 -0
  52. package/packages/record/lib/capture-hook.mjs +26 -0
  53. package/packages/record/lib/capture-index-inventory.mjs +205 -0
  54. package/packages/record/lib/capture-index-outcomes.mjs +113 -0
  55. package/packages/record/lib/capture-index-reset.mjs +110 -0
  56. package/packages/record/lib/capture-index-work.mjs +77 -0
  57. package/packages/record/lib/capture-issue-preparation.mjs +73 -0
  58. package/packages/record/lib/capture-issues.mjs +15 -2
  59. package/packages/record/lib/capture-lock.mjs +137 -17
  60. package/packages/record/lib/capture-native-unit.mjs +322 -0
  61. package/packages/record/lib/capture-pass.mjs +116 -0
  62. package/packages/record/lib/capture-recovery-snapshot.mjs +182 -0
  63. package/packages/record/lib/capture-retained-scratch.mjs +213 -0
  64. package/packages/record/lib/capture-retry.mjs +102 -0
  65. package/packages/record/lib/capture-run.mjs +125 -0
  66. package/packages/record/lib/capture-scratch.mjs +12 -3
  67. package/packages/record/lib/capture-service-definition.mjs +104 -0
  68. package/packages/record/lib/capture-service-install.mjs +362 -0
  69. package/packages/record/lib/capture-service-ownership.mjs +137 -0
  70. package/packages/record/lib/capture-service-runner.mjs +18 -0
  71. package/packages/record/lib/capture-small-snapshot.mjs +85 -0
  72. package/packages/record/lib/capture-source-discovery.mjs +319 -0
  73. package/packages/record/lib/capture-source-queue.mjs +199 -0
  74. package/packages/record/lib/capture-source-work.mjs +85 -0
  75. package/packages/record/lib/capture-state.mjs +16 -71
  76. package/packages/record/lib/capture-status.mjs +171 -0
  77. package/packages/record/lib/capture-supervisor.mjs +136 -0
  78. package/packages/record/lib/capture-watch.mjs +32 -0
  79. package/packages/record/lib/capture-work-session.mjs +241 -0
  80. package/packages/record/lib/capture-worker.mjs +74 -0
  81. package/packages/record/lib/derived-state.mjs +58 -17
  82. package/packages/record/lib/file-lines.mjs +22 -6
  83. package/packages/record/lib/formats.mjs +4 -3
  84. package/packages/record/lib/index-capture.mjs +97 -0
  85. package/packages/record/lib/index-db.mjs +121 -66
  86. package/packages/record/lib/index-reset.mjs +19 -0
  87. package/packages/record/lib/store.mjs +46 -14
  88. package/packages/record/lib/transcript-prefix.mjs +125 -0
  89. package/skills/oats-getting-started/SKILL.md +1 -1
  90. package/docs/macos-test-runs.md +0 -246
@@ -0,0 +1,241 @@
1
+ // Private owning worker session. Bootstrap has already registered this worker
2
+ // under the root lock and reaped its predecessor. Root selection and ordinary
3
+ // command routing remain separate; completion requires a fresh all-stage audit.
4
+ import { randomUUID } from "node:crypto";
5
+ import { RecordStore } from "./store.mjs";
6
+ import { RecordIndex } from "./index-db.mjs";
7
+ import { DerivedState } from "./derived-state.mjs";
8
+ import { CaptureSourceQueue } from "./capture-source-queue.mjs";
9
+ import { captureSourceWork } from "./capture-source-work.mjs";
10
+ import { CaptureIndexInventory } from "./capture-index-inventory.mjs";
11
+ import { CaptureIndexOutcomes } from "./capture-index-outcomes.mjs";
12
+ import { CaptureIndexReset } from "./capture-index-reset.mjs";
13
+ import { CaptureRun } from "./capture-run.mjs";
14
+ import { readCaptureStatus } from "./capture-status.mjs";
15
+ import { reconcileCaptureInterruption, selectCaptureWork } from "./capture-retry.mjs";
16
+ import { readCaptureAdmission } from "./capture-admission.mjs";
17
+ import { FAILURE_REASONS } from "./capture-failures.mjs";
18
+ import { CaptureCleanupQueue } from "./capture-cleanup-queue.mjs";
19
+ import { CaptureSourceDiscovery } from "./capture-source-discovery.mjs";
20
+ import { auditCaptureCompletion, auditCaptureSourceRetirement } from "./capture-completion.mjs";
21
+ import { CaptureIssuePreparation } from "./capture-issue-preparation.mjs";
22
+
23
+ export class CaptureWorkSession {
24
+ constructor(start, { owner, selection, trigger = "manual", now = Date.now, unit = {}, roots = null, indexEnabled = true, issues = null } = {}) {
25
+ if (typeof start?.root !== "string" || typeof start.workerNonce !== "string" || !start.workerNonce ||
26
+ typeof start.pass?.runId !== "string" || !start.pass.runId ||
27
+ typeof start.pass.finalize !== "boolean" || typeof start.pass.reset !== "boolean" || typeof now !== "function" ||
28
+ typeof indexEnabled !== "boolean")
29
+ throw new Error("capture work session requires a supervised pass");
30
+ this.start = start; this.now = now; this.unit = unit; this.indexEnabled = indexEnabled;
31
+ try {
32
+ start.budget.check();
33
+ this.store = new RecordStore(start.root, { owner });
34
+ this.state = new DerivedState(start.root);
35
+ // Append-only workers must not even open an existing index: construction
36
+ // can migrate/rebuild it before any scheduling decision is made.
37
+ this.index = indexEnabled ? new RecordIndex(this.store) : null;
38
+ const prior = readCaptureStatus(start.root);
39
+ if (prior?.lastRun?.id === start.pass.runId && prior.lastRun.outcome !== "running")
40
+ throw new Error("capture run identity already finished");
41
+ const previous = start.pass.previous?.outcome === "failed" ? start.pass.previous.reason : null;
42
+ // An observed child failure belongs to this pass. It cannot diagnose an
43
+ // older abandoned admission merely because this child reconciles it.
44
+ const reasonFor = id => id === start.pass.runId && FAILURE_REASONS.includes(previous) ? previous : "interrupted";
45
+ reconcileCaptureInterruption(this.state, { reason: reasonFor(readCaptureAdmission(this.state)?.runId), now: now() });
46
+ this.cleanup = new CaptureCleanupQueue(this.store, this.state);
47
+ this.cleanup.reconcile(start, { reason: reasonFor(this.cleanup.active()?.runId), now: now() });
48
+ this.queue = new CaptureSourceQueue(this.state, { owner, selection });
49
+ const retirement = this.state.get("capture-sweep-retirement", owner);
50
+ if (!start.pass.finalize && retirement?.sourceScope === this.queue.current().id &&
51
+ retirement.runId === prior?.lastRun?.id && prior.lastRun.outcome !== "running") {
52
+ this.state.transaction(() => {
53
+ this.queue.finish();
54
+ this.state.delete("capture-sweep-retirement", owner);
55
+ });
56
+ this.queue = new CaptureSourceQueue(this.state, { owner, selection });
57
+ }
58
+ if (roots !== null) {
59
+ this.sourceDiscovery = new CaptureSourceDiscovery(this.queue, roots);
60
+ this.sourceDiscovery.reconcile(start, { reason: reasonFor(this.sourceDiscovery.active()?.runId), now: now() });
61
+ }
62
+ if (issues !== null) this.issues = new CaptureIssuePreparation(this.state, this.queue,
63
+ { ...issues, runId: start.pass.runId });
64
+ // Retain a reset's scope even if derived inventory tables were lost.
65
+ // Recreating membership may restart enumeration; it cannot erase reset
66
+ // admission or the index token needed to recover a committed operation.
67
+ if (indexEnabled) {
68
+ const previousSource = this.state.get("capture-index-source", owner);
69
+ const scope = this.state.get("capture-index-reset", "current")?.scope ??
70
+ (previousSource && previousSource.sourceScope !== this.queue.current().id ? randomUUID() :
71
+ this.state.get("index-inventory", "current")?.id ?? randomUUID());
72
+ this.openIndexScope(scope);
73
+ this.reset.reconcile(start, { reason: reasonFor(this.reset.current()?.attempt?.runId), now: now() });
74
+ }
75
+ const failures = [this.state.get("capture-run-failure", "last"),
76
+ this.state.get("capture-index-reset-failure", "last"), this.state.get("capture-cleanup-failure", "last"),
77
+ this.state.get("capture-discovery-failure", "last")].filter(Boolean);
78
+ const currentReason = start.pass.failure ?? failures.find(row => row.runId === start.pass.runId)?.reason ?? null;
79
+ const previousReason = failures.find(row => row.runId === prior?.lastRun?.id)?.reason ??
80
+ (prior?.lastRun?.id === start.pass.runId ? previous : null);
81
+ this.run = new CaptureRun(start.root, this.state, { id: start.pass.runId, trigger,
82
+ reclaimedLock: start.reclaimedLock ?? null, previousFailure: previousReason,
83
+ currentFailure: currentReason ? { id: start.pass.runId, reason: currentReason } : null });
84
+ } catch (error) { this.close(); throw error; }
85
+ }
86
+
87
+ openIndexScope(id) {
88
+ if (!this.indexEnabled) throw new Error("append-only capture cannot open index inventory");
89
+ this.inventory?.close();
90
+ this.inventory = new CaptureIndexInventory(this.state, { id });
91
+ this.outcomes = new CaptureIndexOutcomes(this.index, this.inventory);
92
+ this.reset = new CaptureIndexReset(this.index, this.inventory);
93
+ const binding = { sourceScope: this.queue.current().id, indexScope: id };
94
+ if (JSON.stringify(this.state.get("capture-index-source", this.queue.owner)) !== JSON.stringify(binding))
95
+ this.state.transaction(() => this.state.set("capture-index-source", this.queue.owner, binding));
96
+ }
97
+
98
+ sourceReady(input) {
99
+ if (!input) return false;
100
+ if (input.source === "aw") return true;
101
+ // Native recovery keys bind the exact input, not every source sharing a
102
+ // stream. A deleting allocation is a dependency, even if its directory is
103
+ // already gone: admitting cleanup-only capture would starve its due retry.
104
+ const claim = this.state.get("recovery-allocation", `${input.stream}:${input.path}`);
105
+ if (claim?.phase !== "deleting") return true;
106
+ if (claim.stream !== input.stream || !/^[a-f0-9]{32}$/.test(claim.id))
107
+ throw new Error("invalid native cleanup dependency");
108
+ return false;
109
+ }
110
+
111
+ process(stage, allowRetry) {
112
+ if (stage !== "capture" && (stage !== "index" || !this.indexEnabled))
113
+ throw new Error("capture stage is not enabled");
114
+ const options = { ...this.unit, now: this.now(), allowRetry,
115
+ inputReady: input => this.sourceReady(input),
116
+ retryFilter: retry => retry.stage === stage && (stage !== "capture" || this.sourceReady(this.queue.forStream(retry.stream))) };
117
+ const result = stage === "capture"
118
+ ? captureSourceWork(this.store, this.state, this.queue, this.run, this.start, { ...options, indexInventory: this.inventory ?? null })
119
+ : this.outcomes.advance(this.run, this.start, { ...options, memo: true });
120
+ if (result.resetRequired) this.reset.request(this.start);
121
+ // One independent bounded cleanup turn cannot change stream-stage rotation.
122
+ // A throw retains the already-committed source/index counts and its own
123
+ // admission, so a replacement can back off deletion and keep healthy work.
124
+ const cleanupWork = result.selected ? this.clean(false) : null;
125
+ return { stage, ...result, cleanupWork };
126
+ }
127
+
128
+ clean(retry) {
129
+ return this.cleanup.advance(this.run, this.start, { now: this.now(), retry,
130
+ maxRecords: Math.max(2, this.unit.maxRecords ?? 2000) });
131
+ }
132
+
133
+ collect() {
134
+ if (this.state.db.prepare("SELECT 1 AS found FROM state WHERE kind='source-queue-garbage' OR (? AND kind='index-inventory-garbage') LIMIT 1")
135
+ .get(Number(this.indexEnabled))?.found !== 1)
136
+ return null;
137
+ return this.state.transaction(() => {
138
+ const sources = this.queue.cleanup(), streams = this.inventory?.cleanup() ?? 0;
139
+ // Retiring obsolete enumeration is bookkeeping, not new capture/index
140
+ // progress. In particular it must not refresh every no-op pass's history.
141
+ return { stage: "inventory-cleanup", sources, streams };
142
+ });
143
+ }
144
+
145
+ step() {
146
+ const stopped = () => this.start.budget.sample() !== null;
147
+ if (stopped()) return { stopped: true };
148
+ if (this.reset?.current()) {
149
+ const result = this.reset.advance(this.run, this.start, { now: this.now() });
150
+ if (result.reset) { this.openIndexScope(result.scope); return { stage: "reset", ...result }; }
151
+ if (!this.start.pass.finalize && result.needsFreshWorker) return { resetRequired: true };
152
+ }
153
+ // A committed reset token may finish bounded publication above. Neither
154
+ // source/index work nor new atomic reset may use finalization allowance.
155
+ if (this.start.pass.finalize) return { finalize: true };
156
+
157
+ if (this.issues && !this.issues.complete())
158
+ return { stage: "issue-preparation", ...this.issues.advance(this.start.root, { stopped }) };
159
+
160
+ const sourceDiscovery = this.sourceDiscovery && !this.queue.current().discovered
161
+ ? this.sourceDiscovery.advance({ stopped, limit: this.unit.discoveryLimit ?? 128,
162
+ runId: this.run.id, workerNonce: this.start.workerNonce, now: this.now() }) : null;
163
+ let discovery = null;
164
+ if (this.inventory && this.inventory.current().phase !== "complete") discovery = this.inventory.discover(this.index, { stopped });
165
+ if (stopped()) return { stopped: true, discovery, sourceDiscovery };
166
+ const blockedIndex = !this.indexEnabled || this.reset.current() !== null;
167
+ // Reuse the last durable admission's stage. Failed groups rotate too, and
168
+ // routing adds no separate per-record or per-unit scheduling transaction.
169
+ const last = this.state.get("capture-schedule", "position")?.stage;
170
+ for (const stage of last === "capture" ? ["index", "capture"] : ["capture", "index"]) {
171
+ if (stage === "index" && blockedIndex) continue;
172
+ const result = this.process(stage, false);
173
+ if (result.selected || result.resetRequired || result.stopped) return { ...result, discovery, sourceDiscovery };
174
+ }
175
+ if (stopped()) return { stopped: true, discovery, sourceDiscovery };
176
+ const cleanup = this.clean(false);
177
+ if (cleanup.selected || !cleanup.complete) return { stage: "cleanup", ...cleanup, discovery, sourceDiscovery };
178
+ if (stopped()) return { stopped: true, discovery, sourceDiscovery };
179
+ const collected = this.collect();
180
+ if (collected) return { ...collected, discovery, sourceDiscovery };
181
+ // Unknown eligible discovery can still contain healthy work. A refused
182
+ // root remains pending but cannot withhold unrelated due work indefinitely.
183
+ if ((this.sourceDiscovery ? this.sourceDiscovery.healthyPending() : !this.queue.current().discovered) ||
184
+ this.inventory && this.inventory.current().phase !== "complete")
185
+ return { waitingForDiscovery: true, discovery, sourceDiscovery };
186
+ const selection = selectCaptureWork(this.store, this.state, [], { runId: this.run.id, now: this.now(), stopped,
187
+ retryFilter: retry => retry.stage === "index" ? !blockedIndex :
188
+ retry.stage === "capture" && this.sourceReady(this.queue.forStream(retry.stream)) });
189
+ if (selection) return { ...this.process(selection.stage, true), discovery, sourceDiscovery };
190
+ const cleanupRetry = this.clean(true);
191
+ if (cleanupRetry.selected || !cleanupRetry.complete) return { stage: "cleanup", ...cleanupRetry, discovery, sourceDiscovery };
192
+ if (this.sourceDiscovery && !this.queue.current().discovered) {
193
+ if (sourceDiscovery?.source) return { waitingForDiscovery: true, discovery, sourceDiscovery };
194
+ const retried = this.sourceDiscovery.advance({ stopped, limit: this.unit.discoveryLimit ?? 128,
195
+ runId: this.run.id, workerNonce: this.start.workerNonce, now: this.now(), retry: true });
196
+ if (retried.source) return { waitingForDiscovery: true, discovery, sourceDiscovery: retried };
197
+ }
198
+ return { idle: true, discovery, sourceDiscovery };
199
+ }
200
+
201
+ // Yield/stop never infer completion. The owning worker calls complete() only
202
+ // when it wants a final all-stage decision; idle is merely a scheduling hint.
203
+ // An observed interruption ends the run as interrupted, finalization included.
204
+ yieldWorker({ resetRequired = false } = {}) {
205
+ if (this.start.pass.finalize || this.start.budget.reason === "interrupted")
206
+ return this.finish({ reason: this.start.budget.reason ?? "deadline" });
207
+ this.run.publish({ pending: true, peakRssBytes: this.start.budget.peakRssBytes });
208
+ return { outcome: "partial", reason: this.start.budget.reason ?? "deadline", continuation: true,
209
+ ...(resetRequired ? { resetRequired: true } : {}) };
210
+ }
211
+
212
+ finish({ reason = "input" } = {}) {
213
+ const snapshot = this.run.finish({ outcome: reason === "interrupted" ? "failed" : "partial", reason,
214
+ pending: true, peakRssBytes: this.start.budget.peakRssBytes });
215
+ return { outcome: snapshot.lastRun.outcome, reason: snapshot.lastRun.reason, continuation: false };
216
+ }
217
+
218
+ complete(options) {
219
+ // The sampled reason (interrupted, rss, deadline) is kept; finalization
220
+ // without one reports its allowance.
221
+ const stopped = this.start.budget.sample();
222
+ if (this.start.pass.finalize || stopped !== null) return this.finish({ reason: stopped ?? "deadline" });
223
+ const audit = this.indexEnabled ? auditCaptureCompletion(this, options) : auditCaptureSourceRetirement(this, options);
224
+ if (audit.stopped || audit.needsWork) return this.yieldWorker();
225
+ // Normally these facts entered the run during replacement construction.
226
+ // Also retain a caught failure if an owning caller continues in this worker.
227
+ const failure = ["capture-run-failure", "capture-index-reset-failure", "capture-cleanup-failure", "capture-discovery-failure"]
228
+ .map(kind => this.state.get(kind, "last")).find(row => row?.runId === this.run.id);
229
+ const snapshot = this.run.finish({ outcome: failure ? "failed" : audit.complete ? "complete" : "partial",
230
+ reason: failure?.reason ?? (audit.complete ? null : "input"),
231
+ pending: audit.pending, peakRssBytes: this.start.budget.peakRssBytes });
232
+ return { outcome: snapshot.lastRun.outcome, reason: snapshot.lastRun.reason, continuation: false };
233
+ }
234
+
235
+ close() {
236
+ this.sourceDiscovery?.close();
237
+ this.inventory?.close();
238
+ try { this.index?.close(); } finally { this.state?.close(); }
239
+ this.index = null; this.state = null;
240
+ }
241
+ }
@@ -0,0 +1,74 @@
1
+ // Private worker bootstrap. Reconciliation is supplied by the owning entry;
2
+ // this handshake never captures or writes status on behalf of the operation.
3
+ import { randomBytes } from "node:crypto";
4
+ import { CaptureBudget, CaptureStop } from "./capture-budget.mjs";
5
+ import { readCaptureLockOwner } from "./capture-lock.mjs";
6
+ import { captureMonotonicMs, captureWorkerMessage, validWorkerResult } from "./capture-supervisor.mjs";
7
+
8
+ export async function runCaptureWorker(operation) {
9
+ if (!process.send || !process.connected) throw new Error("capture worker requires its supervisor");
10
+ // TERM/INT sent to the caller's whole process group (a terminal's Ctrl-C,
11
+ // timeout(1), a service manager's control group) reaches this worker too.
12
+ // Caught, it cannot end the worker mid-unit; the supervisor, which received
13
+ // the same signal, owns the stop request and the grace. A signal this worker
14
+ // observes itself also ends it as interrupted at its next check.
15
+ let signalled = false;
16
+ const onSignal = () => { signalled = true; };
17
+ process.on("SIGTERM", onSignal); process.on("SIGINT", onSignal);
18
+ try { return await superviseOperation(operation, () => signalled); }
19
+ finally { process.off("SIGTERM", onSignal); process.off("SIGINT", onSignal); }
20
+ }
21
+
22
+ async function superviseOperation(operation, signalled) {
23
+ const nonce = randomBytes(16).toString("hex"), parentPid = process.ppid;
24
+ const start = await new Promise((resolve, reject) => {
25
+ const disconnected = () => { cleanup(); reject(new Error("capture supervisor disconnected before start")); };
26
+ const receive = message => {
27
+ cleanup();
28
+ if (message?.type !== "start" || message.workerNonce !== nonce || message.parent?.pid !== parentPid ||
29
+ typeof message.root !== "string" || !Number.isFinite(message.drainAt) ||
30
+ !Number.isFinite(message.finalizeAt) || !Number.isFinite(message.hardAt) ||
31
+ message.drainAt > message.finalizeAt || message.finalizeAt > message.hardAt) {
32
+ reject(new Error("invalid capture worker start")); return;
33
+ }
34
+ resolve(message);
35
+ };
36
+ const cleanup = () => { process.off("message", receive); process.off("disconnect", disconnected); };
37
+ process.once("message", receive); process.once("disconnect", disconnected);
38
+ process.send({ type: "ready", pid: process.pid, nonce, startedAt: new Date().toISOString() }, error => {
39
+ if (error) { cleanup(); reject(new Error("capture worker could not announce readiness")); }
40
+ });
41
+ });
42
+ // The same owner read between bounded units also carries the supervisor's
43
+ // cooperative stop request; either way this worker stops as interrupted.
44
+ const owned = () => {
45
+ if (!process.connected || process.ppid !== parentPid) return null;
46
+ const owner = readCaptureLockOwner(start.root);
47
+ return owner?.pid === parentPid && owner.nonce === start.parent.nonce &&
48
+ owner.worker?.pid === process.pid && owner.worker.nonce === nonce ? owner : null;
49
+ };
50
+ const alive = () => { if (signalled()) return false; const owner = owned(); return owner !== null && owner.stop !== true; };
51
+ // Publication must already be durable before the first operation, including
52
+ // its first status write. A callback must check again between bounded units.
53
+ if (!owned()) throw new Error("capture worker lock publication is missing");
54
+ const budget = new CaptureBudget({ policy: start.policy, clock: captureMonotonicMs,
55
+ drainAt: start.drainAt, supervisorAlive: alive });
56
+ let result;
57
+ try {
58
+ result = await operation({ ...start, budget });
59
+ if (!validWorkerResult(result)) throw new Error("invalid capture worker result");
60
+ } catch (error) {
61
+ // The operation owns failure receipts. Bootstrap never invents completed
62
+ // progress or rewrites a receipt after an exception.
63
+ const reason = error instanceof CaptureStop ? error.reason : budget.reason === "interrupted" ? "interrupted" : "io";
64
+ // An operational failure (status persistence among them) names its cause
65
+ // to the supervising command only; the run receipt keeps its fixed message.
66
+ result = { outcome: "failed", reason, ...(reason === "io" ? { message: captureWorkerMessage(error) } : {}) };
67
+ }
68
+ if (process.connected) {
69
+ await new Promise(resolve => process.send({ type: "result", result }, () => resolve()));
70
+ process.disconnect();
71
+ }
72
+ process.exitCode = result.outcome === "failed" ? 1 : 0;
73
+ return result;
74
+ }
@@ -23,38 +23,79 @@ export function fileVersion(path) {
23
23
  } catch (err) { if (err.code === "ENOENT") return null; throw err; }
24
24
  }
25
25
  export const sameVersion = (a, b) => JSON.stringify(a) === JSON.stringify(b);
26
+ function fileIdentity(path) {
27
+ try { const st = statSync(path, { bigint: true }); return { dev: String(st.dev), ino: String(st.ino) }; }
28
+ catch (err) { if (err.code === "ENOENT") return null; throw err; }
29
+ }
26
30
 
27
31
  export class DerivedState {
28
32
  constructor(root) {
29
33
  const { DatabaseSync } = require("node:sqlite");
30
34
  mkdirSync(join(root, "index"), { recursive: true });
31
- this.db = new DatabaseSync(join(root, "index", "capture.db"));
32
- this.db.exec("PRAGMA busy_timeout = 60000; PRAGMA cache_size = -2048; PRAGMA temp_store = FILE");
33
- this.db.exec("BEGIN IMMEDIATE");
35
+ this.path = join(root, "index", "capture.db");
36
+ const before = fileIdentity(this.path);
37
+ this.db = new DatabaseSync(this.path);
34
38
  try {
35
- const hadIds = this.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name='journal_ids'").get()?.found === 1;
36
- this.db.exec(`CREATE TABLE IF NOT EXISTS state (kind TEXT, key TEXT, value TEXT NOT NULL, PRIMARY KEY(kind, key));
37
- CREATE TABLE IF NOT EXISTS journal_ids (stream TEXT, id TEXT, PRIMARY KEY(stream, id));`);
38
- // Recreating a lost lookup atomically revokes the old validation witness.
39
- // A crash cannot qualify an empty replacement table with that witness.
40
- if (!hadIds) this.db.exec("DELETE FROM state WHERE kind = 'journal'");
41
- this.db.exec("COMMIT");
42
- } catch (err) { this.db.exec("ROLLBACK"); this.db.close(); throw err; }
43
- this.read = this.db.prepare("SELECT value FROM state WHERE kind = ? AND key = ?");
44
- this.write = this.db.prepare("INSERT INTO state VALUES (?, ?, ?) ON CONFLICT(kind,key) DO UPDATE SET value=excluded.value");
39
+ this.db.exec("PRAGMA busy_timeout = 60000; PRAGMA cache_size = -2048; PRAGMA temp_store = FILE");
40
+ // A healthy accelerator needs no schema write transaction on every open.
41
+ // Repair rechecks under the write lock, including a lost ID lookup.
42
+ const present = name => this.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name=?").get(name)?.found === 1;
43
+ if (!present("state") || !present("journal_ids")) {
44
+ this.db.exec("BEGIN IMMEDIATE");
45
+ try {
46
+ const hadIds = this.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name='journal_ids'").get()?.found === 1;
47
+ this.db.exec(`CREATE TABLE IF NOT EXISTS state (kind TEXT, key TEXT, value TEXT NOT NULL, PRIMARY KEY(kind, key));
48
+ CREATE TABLE IF NOT EXISTS journal_ids (stream TEXT, id TEXT, PRIMARY KEY(stream, id));`);
49
+ // Recreating a lost lookup atomically revokes the old validation witness.
50
+ // A crash cannot qualify an empty replacement table with that witness.
51
+ if (!hadIds) this.db.exec("DELETE FROM state WHERE kind = 'journal'");
52
+ this.db.exec("COMMIT");
53
+ } catch (err) { this.db.exec("ROLLBACK"); throw err; }
54
+ }
55
+ // The file this connection writes, for callers that hold it across
56
+ // streams. SQLite does not expose the opened descriptor, so the path's
57
+ // identity binds the connection only when it is the same before the
58
+ // open and once the schema is on disk; otherwise it stays unbound and
59
+ // assertFile refuses.
60
+ const after = fileIdentity(this.path);
61
+ this.file = after && (!before || before.dev === after.dev && before.ino === after.ino) ? after : null;
62
+ this.depth = 0;
63
+ this.statements = new Map();
64
+ this.read = this.db.prepare("SELECT value FROM state WHERE kind = ? AND key = ?");
65
+ this.write = this.db.prepare("INSERT INTO state VALUES (?, ?, ?) ON CONFLICT(kind,key) DO UPDATE SET value=excluded.value WHERE state.value != excluded.value");
66
+ } catch (err) { this.db.close(); throw err; }
45
67
  }
46
68
  get(kind, key) { const row = this.read.get(kind, key); return row ? JSON.parse(row.value) : null; }
47
69
  set(kind, key, value) { this.write.run(kind, key, JSON.stringify(value)); }
48
- delete(kind, key) { this.db.prepare("DELETE FROM state WHERE kind = ? AND key = ?").run(kind, key); }
70
+ delete(kind, key) { this.statement("DELETE FROM state WHERE kind = ? AND key = ?").run(kind, key); }
71
+ // One prepared statement per SQL text for this connection's lifetime. Only
72
+ // for statements run to completion (run/get/all), never left iterating.
73
+ statement(sql) {
74
+ let prepared = this.statements.get(sql);
75
+ if (!prepared) { prepared = this.db.prepare(sql); this.statements.set(sql, prepared); }
76
+ return prepared;
77
+ }
78
+ // A connection held across many streams must not publish checkpoint proof
79
+ // into a capture.db that was replaced or removed after it opened.
80
+ assertFile() {
81
+ const now = fileIdentity(this.path);
82
+ if (!now || !this.file || now.dev !== this.file.dev || now.ino !== this.file.ino)
83
+ throw new Error(`capture.db changed during the pass; retry: ${this.path}`);
84
+ }
49
85
  *entries(kind) {
50
86
  for (const row of keysetRows(this.db, "SELECT key, value FROM state WHERE kind = ?", "key", [kind])) {
51
87
  yield { key: row.key, value: JSON.parse(row.value) };
52
88
  }
53
89
  }
54
90
  transaction(fn) {
55
- this.db.exec("BEGIN IMMEDIATE");
56
- try { const result = fn(); this.db.exec("COMMIT"); return result; }
57
- catch (err) { this.db.exec("ROLLBACK"); throw err; }
91
+ const nested = this.depth > 0, savepoint = `capture_${this.depth}`;
92
+ this.db.exec(nested ? `SAVEPOINT ${savepoint}` : "BEGIN IMMEDIATE");
93
+ this.depth++;
94
+ try { const result = fn(); this.db.exec(nested ? `RELEASE ${savepoint}` : "COMMIT"); return result; }
95
+ catch (err) {
96
+ this.db.exec(nested ? `ROLLBACK TO ${savepoint}; RELEASE ${savepoint}` : "ROLLBACK");
97
+ throw err;
98
+ } finally { this.depth--; }
58
99
  }
59
100
  close() { this.db.close(); }
60
101
  }
@@ -8,8 +8,9 @@ import { readSync } from "node:fs";
8
8
  // The bytes of `fd` from `start` up to `end` (default: the end of the file), in
9
9
  // chunks of at most `size` bytes. Each chunk is a fresh buffer, so a caller may
10
10
  // keep one past the next read.
11
- export function* fileChunks(fd, { start = 0, end = Infinity, size = 1 << 20 } = {}) {
11
+ export function* fileChunks(fd, { start = 0, end = Infinity, size = 1 << 20, check = () => {} } = {}) {
12
12
  for (let position = start; position < end; ) {
13
+ check();
13
14
  const chunk = Buffer.allocUnsafe(Math.min(size, end - position));
14
15
  const n = readSync(fd, chunk, 0, chunk.length, position);
15
16
  if (n === 0) return;
@@ -22,17 +23,32 @@ export function* fileChunks(fd, { start = 0, end = Infinity, size = 1 << 20 } =
22
23
  // then a final fragment without one, if the bytes end mid-line. A caller tells
23
24
  // them apart by the last byte. Fragments are joined only at a newline, so a
24
25
  // long line is copied once, not as each chunk arrives.
25
- export function* bufferLines(chunks) {
26
- let pieces = [], size = 0;
26
+ export class RecordSizeError extends Error {
27
+ constructor(limit, offset) {
28
+ super(`raw record exceeds ${limit} bytes at offset ${offset}`);
29
+ this.limit = limit; this.offset = offset;
30
+ }
31
+ }
32
+
33
+ // A line that starts before `limitFrom` (an offset into these chunks) was
34
+ // already captured and is not size-limited; the limit applies from there on.
35
+ export function* bufferLines(chunks, { maxBytes = Infinity, limitFrom = 0 } = {}) {
36
+ let pieces = [], size = 0, offset = 0;
37
+ const append = piece => {
38
+ // Enforce a bounded command's policy before joining or retaining the whole
39
+ // offending record. Public callers retain their unlimited default.
40
+ if (offset >= limitFrom && size + piece.length > maxBytes) throw new RecordSizeError(maxBytes, offset);
41
+ pieces.push(piece); size += piece.length;
42
+ };
27
43
  for (const chunk of chunks) {
28
44
  let from = 0;
29
45
  for (let nl = chunk.indexOf(10, from); nl >= 0; nl = chunk.indexOf(10, from)) {
30
- pieces.push(chunk.subarray(from, nl + 1));
31
- size += nl + 1 - from;
46
+ append(chunk.subarray(from, nl + 1));
32
47
  yield pieces.length === 1 ? pieces[0] : Buffer.concat(pieces, size);
48
+ offset += size;
33
49
  pieces = []; size = 0; from = nl + 1;
34
50
  }
35
- if (from < chunk.length) { pieces.push(chunk.subarray(from)); size += chunk.length - from; }
51
+ if (from < chunk.length) append(chunk.subarray(from));
36
52
  }
37
53
  if (size > 0) yield pieces.length === 1 ? pieces[0] : Buffer.concat(pieces, size);
38
54
  }
@@ -58,12 +58,13 @@ function* parsedLines(bytes) {
58
58
  // Preserve the public inventory's per-directory localeCompare order and stable
59
59
  // ties for ordinary capture too. Path metadata grows with directory size;
60
60
  // source contents are streamed independently of this discovery inventory.
61
- function* directoryEntries(path, optional = false) {
61
+ export function sortedSessionEntries(path, optional = false) {
62
62
  let entries;
63
63
  try { entries = readdirSync(path, { withFileTypes: true }); }
64
- catch (err) { if (optional && err.code === "ENOENT") return; throw err; }
65
- yield* entries.sort((a, b) => a.name.localeCompare(b.name));
64
+ catch (err) { if (optional && err.code === "ENOENT") return []; throw err; }
65
+ return entries.sort((a, b) => a.name.localeCompare(b.name));
66
66
  }
67
+ function* directoryEntries(path, optional = false) { yield* sortedSessionEntries(path, optional); }
67
68
  function* iterateJsonlFiles(root, maxDepth, { strict = false } = {}, allowCaptured = false) {
68
69
  const capturedPi = typeof root === "object" && root !== null ? root.capturedPi : undefined;
69
70
  if (typeof root !== "string") {
@@ -0,0 +1,97 @@
1
+ // Internal bounded indexing units. The scheduler must visit the union of live
2
+ // streams and index_state streams (including deleted journals), and restart
3
+ // its index sweep when reset is returned. This helper alone cannot declare a
4
+ // root complete. Public synchronous update/rebuild keep their existing API.
5
+ import { closeSync, openSync, readSync } from "node:fs";
6
+ import { captureContext } from "./capture-context.mjs";
7
+
8
+ function cursorValid(store, stream, row, checkpoint) {
9
+ if (!row) return true;
10
+ if (!Number.isSafeInteger(row.bytes) || row.bytes < 0 ||
11
+ !Number.isSafeInteger(row.indexed_count) || row.indexed_count < 0 ||
12
+ row.indexed_count !== row.byte_count || row.epoch !== checkpoint.epoch ||
13
+ row.bytes > checkpoint.bytes || row.indexed_count > checkpoint.count) return false;
14
+ if (row.bytes > 0 && row.bytes < checkpoint.bytes) {
15
+ const fd = openSync(store.journalPath(stream), "r"), byte = Buffer.alloc(1);
16
+ try { return readSync(fd, byte, 0, 1, row.bytes - 1) === 1 && byte[0] === 10; }
17
+ finally { closeSync(fd); }
18
+ }
19
+ return true;
20
+ }
21
+
22
+ export function indexCaptureGroup(index, streams, {
23
+ maxRecords = 2000, maxBytes = 8 * 1024 * 1024, stopped = () => false, allowReset = false,
24
+ } = {}) {
25
+ if (typeof allowReset !== "boolean") throw new Error("invalid index reset admission");
26
+ const store = index.store, context = captureContext(store);
27
+ if (!context?.state.depth) throw new Error("bounded indexing requires an active capture work group");
28
+ const names = [...new Set(streams)].sort();
29
+ if (!names.length || names.length > 128 || names.some(name => !context.held.has(name)))
30
+ throw new Error("bounded indexing streams must belong to the active capture work group");
31
+ if (!Number.isSafeInteger(maxRecords) || maxRecords < names.length || !Number.isSafeInteger(maxBytes) || maxBytes < names.length)
32
+ throw new Error("bounded indexing must allow at least one work unit per stream");
33
+ const recordsEach = Math.floor(maxRecords / names.length), bytesEach = Math.floor(maxBytes / names.length);
34
+ const get = index.db.prepare(`SELECT s.indexed_count,b.bytes,b.indexed_count AS byte_count,b.epoch
35
+ FROM index_state s LEFT JOIN index_bytes b ON s.stream=b.stream WHERE s.stream=?`);
36
+ const prepared = [], work = { records: 0, bytes: 0, deletedIds: 0 };
37
+ let reset = false;
38
+ // Every group's locks were taken before capture.db's transaction. Validate
39
+ // journals before opening turns.db's transaction; never acquire another
40
+ // stream from inside either database transaction.
41
+ for (const stream of names) {
42
+ if (stopped()) { prepared.push({ stream, phase: "validation" }); continue; }
43
+ const checkpoint = store.validateJournal(stream, context.state, {
44
+ maxRecords: recordsEach, maxBytes: bytesEach, stopped, work,
45
+ });
46
+ const row = get.get(stream);
47
+ if (!cursorValid(store, stream, row?.indexed_count != null ? row : null, checkpoint)) reset = true;
48
+ prepared.push({ stream, checkpoint });
49
+ }
50
+ // Cross-stream tombstones and first-stream attribution make a per-stream
51
+ // row deletion unsound. Keep the reviewed atomic union reset. Publication
52
+ // of its cleared cursor tables is part of that same SQLite transaction.
53
+ if (reset && !allowReset) {
54
+ // The scheduler must end this work unit and reserve a fresh full worker
55
+ // allowance for atomic reset. Detection is not permission to start it in
56
+ // a nearly exhausted capture/index unit or the finalization reserve.
57
+ return { reset: false, resetRequired: true, turnsIndexed: 0, bytesIndexed: 0, validation: work,
58
+ streams: names.map(stream => ({ stream, phase: "reset" })) };
59
+ }
60
+ if (reset) index.reset();
61
+ const setCount = index.db.prepare("INSERT INTO index_state VALUES (?, ?) ON CONFLICT(stream) DO UPDATE SET indexed_count=excluded.indexed_count");
62
+ const setBytes = index.db.prepare("INSERT INTO index_bytes VALUES (?, ?, ?, ?) ON CONFLICT(stream) DO UPDATE SET bytes=excluded.bytes,indexed_count=excluded.indexed_count,epoch=excluded.epoch");
63
+ let transaction = false, turnsIndexed = 0, bytesIndexed = 0;
64
+ const begin = () => { if (!transaction) { index.db.exec("BEGIN IMMEDIATE"); transaction = true; } };
65
+ try {
66
+ const recordedOwner = index.db.prepare("SELECT value FROM meta WHERE key='owner'").get()?.value ?? null;
67
+ const owner = store.owner ?? recordedOwner;
68
+ if (owner && recordedOwner !== owner) {
69
+ begin(); index.db.prepare("INSERT INTO meta VALUES ('owner',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value WHERE meta.value!=excluded.value").run(owner);
70
+ }
71
+ for (const item of prepared) {
72
+ const { stream, checkpoint } = item;
73
+ if (!checkpoint?.complete) { item.phase = "validation"; continue; }
74
+ const row = reset ? undefined : get.get(stream);
75
+ let position = row?.bytes ?? 0, count = row?.indexed_count ?? 0;
76
+ if (position === checkpoint.bytes) { item.phase = "complete"; continue; }
77
+ if (stopped()) { item.phase = "index"; continue; }
78
+ // Fix the validated end for this unit. Later growth is subsequent work.
79
+ const iterator = store.iterateJournal(stream, { start: position, end: checkpoint.bytes });
80
+ let records = 0, bytes = 0;
81
+ try {
82
+ while (records < recordsEach && bytes < bytesEach && !stopped()) {
83
+ const next = iterator.next(); if (next.done) break;
84
+ begin();
85
+ const entry = next.value;
86
+ if (entry.turn !== null) { index.addTurn(entry.turn, stream, owner); count++; turnsIndexed++; }
87
+ position = entry.end; records++; bytes += entry.bytes; bytesIndexed += entry.bytes;
88
+ }
89
+ } finally { iterator.return(); }
90
+ if (records) { setCount.run(stream, count); setBytes.run(stream, position, count, checkpoint.epoch); }
91
+ item.phase = position === checkpoint.bytes ? "complete" : "index";
92
+ }
93
+ if (transaction) index.db.exec("COMMIT");
94
+ } catch (err) { if (transaction) index.db.exec("ROLLBACK"); throw err; }
95
+ return { reset, turnsIndexed, bytesIndexed, validation: work,
96
+ streams: prepared.map(({ stream, phase }) => ({ stream, phase })) };
97
+ }