@awebai/oats 0.61.0 → 0.62.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/README.md +1 -1
  2. package/bin/oats.mjs +26 -11
  3. package/docs/capabilities.md +1 -1
  4. package/docs/execution-targets.md +49 -3
  5. package/docs/implementation.md +2 -1
  6. package/docs/named-test-runs.md +382 -0
  7. package/docs/official-catalog.md +1 -1
  8. package/docs/packages.md +5 -5
  9. package/docs/release-notes/v0.62.0.md +223 -0
  10. package/docs/upgrading.md +133 -11
  11. package/docs/workspaces.md +1 -1
  12. package/lib/core.mjs +2 -2
  13. package/lib/doctor-capture.mjs +35 -0
  14. package/lib/session-viewer.mjs +86 -7
  15. package/package-catalog.json +1 -1
  16. package/package.json +1 -1
  17. package/packages/client/cli-locator.mjs +2 -2
  18. package/packages/record/README.md +87 -13
  19. package/packages/record/bin/capture-hook.mjs +8 -0
  20. package/packages/record/bin/capture.mjs +143 -124
  21. package/packages/record/bin/recall.mjs +8 -3
  22. package/packages/record/bin/setup.mjs +115 -225
  23. package/packages/record/bin/turn-record.mjs +2 -2
  24. package/packages/record/docs/capture-service.md +1443 -28
  25. package/packages/record/docs/capture-status.schema.json +429 -12
  26. package/packages/record/lib/capture-admission.mjs +82 -0
  27. package/packages/record/lib/capture-aw-unit.mjs +137 -0
  28. package/packages/record/lib/capture-aw.mjs +31 -12
  29. package/packages/record/lib/capture-budget.mjs +31 -0
  30. package/packages/record/lib/capture-cc.mjs +39 -6
  31. package/packages/record/lib/capture-cleanup-queue.mjs +135 -0
  32. package/packages/record/lib/capture-cleanup-work.mjs +77 -0
  33. package/packages/record/lib/capture-command-operation.mjs +37 -0
  34. package/packages/record/lib/capture-command-output.mjs +61 -0
  35. package/packages/record/lib/capture-command-worker.mjs +6 -0
  36. package/packages/record/lib/capture-command.mjs +48 -0
  37. package/packages/record/lib/capture-completion.mjs +152 -0
  38. package/packages/record/lib/capture-context.mjs +50 -0
  39. package/packages/record/lib/capture-default-roots.mjs +30 -0
  40. package/packages/record/lib/capture-discovery.mjs +114 -0
  41. package/packages/record/lib/capture-failures.mjs +45 -0
  42. package/packages/record/lib/capture-health-doctor.mjs +13 -0
  43. package/packages/record/lib/capture-health-installation.mjs +117 -0
  44. package/packages/record/lib/capture-health.mjs +116 -0
  45. package/packages/record/lib/capture-hook-ownership.mjs +58 -0
  46. package/packages/record/lib/capture-hook-runner.mjs +17 -0
  47. package/packages/record/lib/capture-hook.mjs +26 -0
  48. package/packages/record/lib/capture-index-inventory.mjs +205 -0
  49. package/packages/record/lib/capture-index-outcomes.mjs +113 -0
  50. package/packages/record/lib/capture-index-reset.mjs +109 -0
  51. package/packages/record/lib/capture-index-work.mjs +77 -0
  52. package/packages/record/lib/capture-issue-preparation.mjs +73 -0
  53. package/packages/record/lib/capture-issues.mjs +15 -2
  54. package/packages/record/lib/capture-lock.mjs +102 -16
  55. package/packages/record/lib/capture-native-unit.mjs +322 -0
  56. package/packages/record/lib/capture-pass.mjs +102 -0
  57. package/packages/record/lib/capture-recovery-snapshot.mjs +182 -0
  58. package/packages/record/lib/capture-retained-scratch.mjs +213 -0
  59. package/packages/record/lib/capture-retry.mjs +102 -0
  60. package/packages/record/lib/capture-run.mjs +125 -0
  61. package/packages/record/lib/capture-scratch.mjs +12 -3
  62. package/packages/record/lib/capture-service-definition.mjs +104 -0
  63. package/packages/record/lib/capture-service-install.mjs +362 -0
  64. package/packages/record/lib/capture-service-ownership.mjs +137 -0
  65. package/packages/record/lib/capture-service-runner.mjs +18 -0
  66. package/packages/record/lib/capture-small-snapshot.mjs +85 -0
  67. package/packages/record/lib/capture-source-discovery.mjs +319 -0
  68. package/packages/record/lib/capture-source-queue.mjs +199 -0
  69. package/packages/record/lib/capture-source-work.mjs +85 -0
  70. package/packages/record/lib/capture-state.mjs +16 -71
  71. package/packages/record/lib/capture-status.mjs +171 -0
  72. package/packages/record/lib/capture-supervisor.mjs +117 -0
  73. package/packages/record/lib/capture-watch.mjs +32 -0
  74. package/packages/record/lib/capture-work-session.mjs +238 -0
  75. package/packages/record/lib/capture-worker.mjs +58 -0
  76. package/packages/record/lib/derived-state.mjs +28 -15
  77. package/packages/record/lib/file-lines.mjs +22 -6
  78. package/packages/record/lib/formats.mjs +4 -3
  79. package/packages/record/lib/index-capture.mjs +97 -0
  80. package/packages/record/lib/index-db.mjs +3 -17
  81. package/packages/record/lib/index-reset.mjs +19 -0
  82. package/packages/record/lib/store.mjs +36 -11
  83. package/packages/record/lib/transcript-prefix.mjs +125 -0
  84. package/skills/oats-getting-started/SKILL.md +1 -1
  85. package/docs/macos-test-runs.md +0 -246
@@ -0,0 +1,73 @@
1
+ // Private command preparation. Page mutations and their resume key commit
2
+ // together; this is retry intent, never source/custody or completion proof.
3
+ import { basename } from "node:path";
4
+ import { SESSION_FORMATS } from "./formats.mjs";
5
+ import { loadIgnore } from "./ignore.mjs";
6
+
7
+ const formats = ["cc", "pi", "codex", "aw"];
8
+ export class CaptureIssuePreparation {
9
+ constructor(state, queue, { runId, sources, retry = false }) {
10
+ if (typeof runId !== "string" || !runId || typeof retry !== "boolean" ||
11
+ !Array.isArray(sources) || new Set(sources).size !== sources.length || sources.some(source => !formats.includes(source)) ||
12
+ retry && sources.length !== formats.length)
13
+ throw new Error("invalid capture issue preparation");
14
+ this.state = state; this.queue = queue; this.runId = runId; this.retry = retry;
15
+ this.sources = formats.filter(source => sources.includes(source));
16
+ this.scope = queue.current().id;
17
+ }
18
+
19
+ current() {
20
+ if (this.queue.current().id !== this.scope) throw new Error("capture issue preparation scope changed");
21
+ const row = this.state.get("capture-issue-preparation", this.queue.owner);
22
+ if (!row || row.runId !== this.runId) return null;
23
+ if (row.scope !== this.scope || row.retry !== this.retry || JSON.stringify(row.sources) !== JSON.stringify(this.sources) ||
24
+ row.after !== null && typeof row.after !== "string" || typeof row.complete !== "boolean")
25
+ throw new Error("capture issue preparation selection changed");
26
+ return row;
27
+ }
28
+
29
+ complete() { return this.current()?.complete === true; }
30
+
31
+ advance(root, { stopped = () => false, finalize = false, limit = 128 } = {}) {
32
+ if (this.state.depth || !Number.isSafeInteger(limit) || limit < 1 || limit > 128)
33
+ throw new Error("capture issue preparation requires a bounded standalone transaction");
34
+ if (finalize || stopped()) return { complete: false, stopped: true, checked: 0 };
35
+ if (this.complete()) return { complete: true, checked: 0 };
36
+ // Privacy failure must propagate; it cannot authorize opening inputs.
37
+ const ignore = loadIgnore(root);
38
+ return this.state.transaction(() => {
39
+ const row = this.current() ?? { runId: this.runId, scope: this.scope, retry: this.retry,
40
+ sources: this.sources, after: null, complete: false };
41
+ const page = this.state.db.prepare(`SELECT key,value FROM state WHERE kind='set-aside'
42
+ ${row.after === null ? "" : "AND key>?"} ORDER BY key LIMIT ?`)
43
+ .all(...(row.after === null ? [limit] : [row.after, limit]));
44
+ let checked = 0;
45
+ for (const { key, value } of page) {
46
+ if (stopped()) break;
47
+ const issue = JSON.parse(value), fmt = SESSION_FORMATS[issue.source];
48
+ if (!formats.includes(issue.source) || typeof issue.path !== "string")
49
+ throw new Error("invalid capture issue during preparation");
50
+ // Preserve native public reconciliation for missing/ignored inputs,
51
+ // but do not clear another owner or an excluded format's evidence.
52
+ const ignored = fmt && issue.owner === this.queue.owner && this.sources.includes(issue.source) &&
53
+ ignore.ignores(issue.path, [basename(issue.path), fmt.sessionId(issue.path), ...(fmt.ignoreKeys?.(issue.path) ?? [])]);
54
+ if (this.retry) {
55
+ // The existing explicit all-input retry marks the root's issues,
56
+ // including other owners, without changing their source selection.
57
+ // Run identity stays stable across replacement and partial pages.
58
+ this.state.set("source-retry", key, this.runId);
59
+ }
60
+ if (ignored) this.state.delete("set-aside", key);
61
+ else if (this.retry) this.state.set("set-aside", key, { ...issue, retry: true });
62
+ if ((ignored || this.retry) && issue.owner === this.queue.owner && this.sources.includes(issue.source)) {
63
+ this.state.db.prepare("UPDATE capture_sources SET outcome='ready' WHERE scope=? AND source=? AND path=? AND outcome='set-aside'")
64
+ .run(this.scope, issue.source, issue.path);
65
+ }
66
+ row.after = key; checked++;
67
+ }
68
+ row.complete = checked === page.length && page.length < limit;
69
+ this.state.set("capture-issue-preparation", this.queue.owner, row);
70
+ return { complete: row.complete, stopped: stopped(), checked };
71
+ });
72
+ }
73
+ }
@@ -2,6 +2,7 @@ import { statSync } from "node:fs";
2
2
  import { spawnSync } from "node:child_process";
3
3
  import { fileURLToPath } from "node:url";
4
4
  import { join } from "node:path";
5
+ import { validFailure, validFailureBacklog } from "./capture-failures.mjs";
5
6
  // Feature landed separately on the two release lines; 23.0/23.1 lack it.
6
7
  export function supportsReadOnlySqlite(version) {
7
8
  const match = /^(\d+)\.(\d+)\.(\d+)$/.exec(version);
@@ -33,8 +34,20 @@ export function readIssueBacklog(db, { pending = false, validate = false } = {})
33
34
  }
34
35
  return { source, path, offset, reason, ...(line === undefined ? {} : { line }) };
35
36
  });
37
+ // Both inventories share this read snapshot, but have independent samples.
38
+ // Private generation/admission fields never enter the public projection.
39
+ const failedCount = Number(db.prepare("SELECT count(*) AS n FROM state WHERE kind = 'capture-failed'").get().n);
40
+ const failed = db.prepare("SELECT value FROM state WHERE kind = 'capture-failed' ORDER BY key LIMIT 100").all()
41
+ .map(({ value }) => {
42
+ const { stream, offset, reason, attempts, nextRetryAt } = JSON.parse(value);
43
+ const row = { stream, offset, reason, attempts, nextRetryAt };
44
+ if (!validFailure(row)) throw new Error("malformed capture failure row");
45
+ return row;
46
+ });
47
+ const deferred = db.prepare("SELECT 1 AS found FROM state WHERE kind = 'capture-retry' LIMIT 1").get()?.found === 1;
36
48
  db.exec("COMMIT");
37
- return { pending: pending || count > 0, setAsideCount: count, setAside, truncated: count > sample.length };
49
+ return { pending: pending || count > 0 || failedCount > 0 || deferred, setAsideCount: count, setAside, truncated: count > sample.length,
50
+ failedCount, failed, failedTruncated: failedCount > failed.length };
38
51
  } catch (err) { db.exec("ROLLBACK"); throw err; }
39
52
  }
40
53
 
@@ -59,6 +72,6 @@ export function readCurrentIssueBacklog(root) {
59
72
  const b = response.backlog;
60
73
  if (!b || typeof b.pending !== "boolean" || !Number.isSafeInteger(b.setAsideCount) || b.setAsideCount < 0 ||
61
74
  !Array.isArray(b.setAside) || b.setAside.length !== Math.min(b.setAsideCount, 100) ||
62
- b.truncated !== (b.setAsideCount > 100)) throw new Error("malformed capture database reader output");
75
+ b.truncated !== (b.setAsideCount > 100) || !validFailureBacklog(b)) throw new Error("malformed capture database reader output");
63
76
  return b;
64
77
  }
@@ -9,7 +9,7 @@
9
9
  // initializing (owner-less) lock refuses the pass and names the holder and
10
10
  // the operator recovery; it is never stolen.
11
11
  //
12
- // A lock whose recorded owner is DEAD on this host is reclaimed (a capture
12
+ // A lock whose recorded owner AND registered worker are DEAD on this host is reclaimed (a capture
13
13
  // killed mid-pass by a hook or caller timeout runs no finally). Records
14
14
  // without host predate host recording and live under this user's home, so
15
15
  // they count as this host's (RECLAIM_HOSTLESS_RECORDS). Reclaimers
@@ -19,10 +19,12 @@
19
19
  // meanwhile. A guard whose holder died is never removed (that would race
20
20
  // exactly as removing the lock does); it is named with its recovery.
21
21
  // Acquire never waits: it reclaims once or skips, and the next pass catches up.
22
- // There is no signal handler: a pass is synchronous, so a JS handler would
23
- // only run after the whole pass (turning a caller's timeout kill into a full
24
- // pass), and SIGKILL cannot be handled; the reclaim is the recovery.
25
- import { closeSync, existsSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
22
+ // A supervisor publishes its idle child's identity before permitting work.
23
+ // The worker is part of the same lock, not a second independently reclaimed
24
+ // lock. A live/reused/unknown child PID prevents release and reclamation even
25
+ // when the supervisor died. Callers own child termination/reaping; this module
26
+ // never sends a terminating signal or infers a process group.
27
+ import { closeSync, constants, existsSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readSync, renameSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
26
28
  import { randomBytes } from "node:crypto";
27
29
  import { hostname } from "node:os";
28
30
  import { join } from "node:path";
@@ -35,18 +37,46 @@ export function captureLockPath(root) { return join(root, ".capture.lock"); }
35
37
 
36
38
  /** "alive" | "dead" | "unknown" for an owner pid ("unknown" = exists but not signalable). */
37
39
  export function holderLiveness(pid) {
38
- if (!Number.isInteger(pid) || pid <= 0) return "unknown";
39
- try { process.kill(pid, 0); return "alive"; } catch (e) { return e.code === "EPERM" ? "unknown" : "dead"; }
40
+ if (!Number.isInteger(pid) || pid <= 0 || pid > 2147483647) return "unknown";
41
+ try { process.kill(pid, 0); return "alive"; } catch (e) { return e.code === "ESRCH" ? "dead" : "unknown"; }
40
42
  }
41
43
 
42
- const readJson = (path) => { try { return JSON.parse(readFileSync(path, "utf8")); } catch { return undefined; } };
43
- const readOwner = (dir) => readJson(join(dir, "owner.json"));
44
+ const sameFile = (a, b) => a.dev === b.dev && a.ino === b.ino && a.size === b.size && a.mtimeNs === b.mtimeNs && a.ctimeNs === b.ctimeNs;
45
+ const readJson = (path) => {
46
+ let fd;
47
+ try {
48
+ const before = lstatSync(path, { bigint: true });
49
+ if (!before.isFile() || before.isSymbolicLink() || before.size > 4096n) return undefined;
50
+ fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW);
51
+ const opened = fstatSync(fd, { bigint: true });
52
+ if (!sameFile(before, opened)) return undefined;
53
+ const bytes = Buffer.alloc(Number(opened.size)); let offset = 0;
54
+ while (offset < bytes.length) { const n = readSync(fd, bytes, offset, bytes.length - offset, offset); if (!n) return undefined; offset += n; }
55
+ if (!sameFile(opened, fstatSync(fd, { bigint: true })) || !sameFile(opened, lstatSync(path, { bigint: true }))) return undefined;
56
+ return JSON.parse(bytes.toString("utf8"));
57
+ } catch { return undefined; }
58
+ finally { if (fd !== undefined) closeSync(fd); }
59
+ };
60
+ const readOwner = (dir) => {
61
+ try { const st = lstatSync(dir); return st.isDirectory() && !st.isSymbolicLink() ? readJson(join(dir, "owner.json")) : undefined; }
62
+ catch { return undefined; }
63
+ };
64
+ export const readCaptureLockOwner = root => readOwner(captureLockPath(root));
65
+
66
+ function validPid(pid) { return Number.isInteger(pid) && pid > 0 && pid <= 2147483647; }
67
+ function workerDead(owner, { host, liveness }) {
68
+ if (!Object.hasOwn(owner, "worker")) return true;
69
+ const worker = owner.worker;
70
+ return validPid(worker?.pid) && worker.pid !== owner.pid && worker.host === host && typeof worker.nonce === "string" &&
71
+ worker.nonce.length > 0 && worker.nonce.length <= 128 && typeof worker.startedAt === "string" &&
72
+ Number.isFinite(Date.parse(worker.startedAt)) && liveness(worker.pid) === "dead";
73
+ }
44
74
 
45
75
  /** Whether `owner` is a record of this host whose process is dead: the only lock this module reclaims. */
46
76
  function deadHere(owner, { host, liveness, reclaimHostless }) {
47
- if (!owner || !Number.isInteger(owner.pid)) return false;
77
+ if (!owner || !validPid(owner.pid)) return false;
48
78
  const here = owner.host === undefined ? reclaimHostless : owner.host === host;
49
- return here && liveness(owner.pid) === "dead";
79
+ return here && liveness(owner.pid) === "dead" && workerDead(owner, { host, liveness });
50
80
  }
51
81
 
52
82
  /** Remove the lock `dir` held by the dead `owner`, serialized by the guard `<dir>.reclaim`. → { removed }
@@ -60,7 +90,7 @@ function reclaimDeadLock(dir, owner, me, opts) {
60
90
  const g = readJson(guard);
61
91
  return g && deadHere(g, opts) ? { abandoned: { guard, pid: g.pid } } : { removed: false };
62
92
  }
63
- const same = (o) => o && o.pid === owner.pid && o.nonce === owner.nonce && o.startedAt === owner.startedAt;
93
+ const same = (o) => o && o.pid === owner.pid && o.nonce === owner.nonce && o.startedAt === owner.startedAt && JSON.stringify(o.worker) === JSON.stringify(owner.worker);
64
94
  let removed = false;
65
95
  try {
66
96
  const now = readOwner(dir);
@@ -79,6 +109,7 @@ export function shellQuote(s) { return "'" + String(s).replace(/'/g, "'\\''") +
79
109
  /** The operator's recovery line for a lock that is not ours. */
80
110
  export function recoveryInstruction(dir, owner, liveness) {
81
111
  const remove = `rm -r -- ${shellQuote(dir)}`;
112
+ if (owner && Object.hasOwn(owner, "worker")) return `${dir} records supervisor pid ${owner.pid} (${liveness}) and worker pid ${owner.worker?.pid ?? "unknown"}; wait for both to finish. If interrupted, stop capture triggers and verify BOTH processes are gone before removing the lock with: ${remove} and rerun`;
82
113
  if (!owner?.pid) return `${dir} is held by a pass that has not written its owner record yet (initializing, or killed before it could); stop capture triggers (hooks, launchd), verify no capture process is running (pgrep -f capture.mjs), then remove the lock with: ${remove} and rerun`;
83
114
  const who = `pid ${owner.pid} (started ${owner.startedAt || "?"}, now ${liveness})`;
84
115
  if (liveness === "alive") return `${dir} is held by ${who}; let it finish, the next pass catches up`;
@@ -103,7 +134,7 @@ export function recoveryInstruction(dir, owner, liveness) {
103
134
  * saying what happened. And `release()` never throws: it answers
104
135
  * `{ released: true }` only when the lock is verifiably gone, otherwise
105
136
  * `{ released: false, reason: "gone" | "unknown-owner" | "not-owner" |
106
- * "remove-failed", ... }` with the actual observation, never a guess.
137
+ * "remove-failed" | "worker-active", ... }` with the actual observation, never a guess.
107
138
  *
108
139
  * The owner record carries a per-acquisition nonce, so a release kept from
109
140
  * an earlier acquisition cannot erase a later one by the same pid (an
@@ -111,13 +142,14 @@ export function recoveryInstruction(dir, owner, liveness) {
111
142
  *
112
143
  * `io` exists for fault injection in tests only. */
113
144
  export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, liveness = holderLiveness, host = hostname(), reclaimHostless = RECLAIM_HOSTLESS_RECORDS, io = {} } = {}) {
114
- const fs = { writeFileSync, rmSync, openSync, closeSync, lstatSync, ...io };
145
+ if (!validPid(pid)) throw new Error("capture lock owner must have a positive process ID");
146
+ const fs = { writeFileSync, rmSync, openSync, closeSync, lstatSync, fsyncSync, renameSync, ...io };
115
147
  const dir = captureLockPath(root);
116
148
  const nonce = randomBytes(8).toString("hex");
117
149
  mkdirSync(root, { recursive: true }); // the store creates the root lazily; the lock may come first
118
150
  let reclaimed;
119
151
  for (let attempt = 0; ; attempt++) {
120
- try { mkdirSync(dir); break; }
152
+ try { mkdirSync(dir, { mode: 0o700 }); break; }
121
153
  catch (e) {
122
154
  if (e.code !== "EEXIST") throw e;
123
155
  const owner = readOwner(dir);
@@ -148,7 +180,7 @@ export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, li
148
180
  // a record-less replacement look like the directory we created.
149
181
  directoryFd = fs.openSync(dir, "r");
150
182
  identity = fstatSync(directoryFd);
151
- fs.writeFileSync(join(dir, "owner.json"), JSON.stringify({ pid, nonce, startedAt: new Date(now()).toISOString(), host }));
183
+ fs.writeFileSync(join(dir, "owner.json"), JSON.stringify({ pid, nonce, startedAt: new Date(now()).toISOString(), host }), { mode: 0o600 });
152
184
  } catch (err) {
153
185
  // Ownership was proven by the mkdir, not by the moment of cleanup: the
154
186
  // directory is removed only if it is still ours (same inode) and holds
@@ -175,12 +207,66 @@ export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, li
175
207
  }
176
208
  return {
177
209
  path: dir,
210
+ nonce,
178
211
  ...(reclaimed ? { reclaimed } : {}),
212
+ // The child must remain idle until this durable publication succeeds.
213
+ // A dead supervisor's lock cannot then be reclaimed while its worker lives.
214
+ registerWorker(worker, { previous } = {}) {
215
+ if (!validPid(worker?.pid) || worker.pid === pid || typeof worker.nonce !== "string" || !worker.nonce || worker.nonce.length > 128 || typeof worker.startedAt !== "string" || !Number.isFinite(Date.parse(worker.startedAt))) throw new Error("invalid capture worker identity");
216
+ const current = readOwner(dir), st = fs.lstatSync(dir);
217
+ if (st.dev !== identity.dev || st.ino !== identity.ino || current?.pid !== pid || current?.nonce !== nonce) throw new Error("capture lock changed before worker registration");
218
+ // Continuation never drops the root lock between children. Replace only
219
+ // the caller's exact previous publication, after positive death proof;
220
+ // reused PIDs and uncertain liveness preserve the old lock unchanged.
221
+ if (Object.hasOwn(current, "worker")) {
222
+ if (!previous || current.worker?.pid !== previous.pid || current.worker?.nonce !== previous.nonce ||
223
+ current.worker?.startedAt !== previous.startedAt || !workerDead(current, { host, liveness })) {
224
+ throw new Error("previous capture worker is not proven dead and unchanged");
225
+ }
226
+ } else if (previous) throw new Error("capture lock changed before worker continuation");
227
+ const temp = join(dir, `worker-${nonce}.tmp`); let fd, dirFd, rootFd, temporary, published = false;
228
+ try {
229
+ dirFd = fs.openSync(dir, constants.O_RDONLY | constants.O_NOFOLLOW);
230
+ const pinned = fstatSync(dirFd);
231
+ if (pinned.dev !== identity.dev || pinned.ino !== identity.ino) throw new Error("capture lock changed before worker registration");
232
+ fd = fs.openSync(temp, "wx", 0o600);
233
+ fs.writeFileSync(fd, JSON.stringify({ ...current, worker: { pid: worker.pid, nonce: worker.nonce, startedAt: worker.startedAt, host } }));
234
+ fs.fsyncSync(fd); temporary = fstatSync(fd, { bigint: true });
235
+ const again = readOwner(dir), named = fs.lstatSync(dir);
236
+ if (named.dev !== identity.dev || named.ino !== identity.ino || JSON.stringify(again) !== JSON.stringify(current) || !sameFile(temporary, fs.lstatSync(temp, { bigint: true }))) throw new Error("capture lock changed during worker registration");
237
+ fs.renameSync(temp, join(dir, "owner.json"));
238
+ published = true;
239
+ // Publish file bytes, their rename, and the lock directory's parent
240
+ // entry before GO. A failure is not permission to start the worker.
241
+ fs.fsyncSync(dirFd);
242
+ rootFd = fs.openSync(root, "r"); fs.fsyncSync(rootFd);
243
+ } finally {
244
+ // Never remove a path another caller replaced, even after a failed
245
+ // registration. An unpublished/unverifiable file is conservatively kept.
246
+ if (!published && temporary) {
247
+ try {
248
+ const named = fs.lstatSync(dir);
249
+ if (named.dev === identity.dev && named.ino === identity.ino && sameFile(temporary, fs.lstatSync(temp, { bigint: true }))) unlinkSync(temp);
250
+ } catch { /* absent or unknown: preserve */ }
251
+ }
252
+ if (fd !== undefined) fs.closeSync(fd);
253
+ if (rootFd !== undefined) fs.closeSync(rootFd);
254
+ if (dirFd !== undefined) fs.closeSync(dirFd);
255
+ }
256
+ },
179
257
  release: () => {
180
258
  if (!existsSync(dir)) return { released: false, reason: "gone" };
181
259
  const cur = readOwner(dir);
182
260
  if (!cur) return { released: false, reason: "unknown-owner", recovery: recoveryInstruction(dir, undefined, "unknown") };
183
261
  if (cur.pid !== pid || cur.nonce !== nonce) return { released: false, reason: "not-owner", owner: cur };
262
+ try {
263
+ const named = fs.lstatSync(dir);
264
+ if (!named.isDirectory() || named.dev !== identity.dev || named.ino !== identity.ino) return { released: false, reason: "not-owner", owner: cur };
265
+ } catch (err) {
266
+ return { released: false, reason: "remove-failed", error: err.message,
267
+ recovery: "could not verify capture lock identity before removal; inspect the filesystem error and rerun capture" };
268
+ }
269
+ if (!workerDead(cur, { host, liveness })) return { released: false, reason: "worker-active", recovery: recoveryInstruction(dir, cur, "alive") };
184
270
  let error;
185
271
  try { fs.rmSync(dir, { recursive: true, force: true }); } catch (e) { error = e; }
186
272
  let remains;
@@ -0,0 +1,322 @@
1
+ // Private bounded native command work. Public captureSessions and exact
2
+ // receipts retain their synchronous complete-pass contracts. The scheduler
3
+ // supplies a stream lock/transaction and advances to another input after this
4
+ // unit; a partial result is never a completed sweep.
5
+ import { closeSync, constants, fstatSync, openSync } from "node:fs";
6
+ import { basename } from "node:path";
7
+ import { randomUUID } from "node:crypto";
8
+ import { captureContext } from "./capture-context.mjs";
9
+ import { CAPTURE_POLICY } from "./capture-budget.mjs";
10
+ import { completeNativeLineDigests, completeNativeLines, eventTurnCore } from "./capture-cc.mjs";
11
+ import { finishTurn } from "./canonical.mjs";
12
+ import { SESSION_FORMATS } from "./formats.mjs";
13
+ import { loadIgnore } from "./ignore.mjs";
14
+ import { fileVersion, sameVersion, statVersion } from "./derived-state.mjs";
15
+ import { assertProtectedDescriptor, digest, sourceChunks } from "./session-snapshot.mjs";
16
+ import { guardCapturedPath } from "./native-history.mjs";
17
+ import { RecordSizeError } from "./file-lines.mjs";
18
+ import { advanceRecoverySnapshot } from "./capture-recovery-snapshot.mjs";
19
+ import { advanceTranscriptPrefix } from "./transcript-prefix.mjs";
20
+ import { prepareRetainedCaptureCleanup } from "./capture-retained-scratch.mjs";
21
+ import { smallCommandSnapshot, SMALL_SNAPSHOT_BYTES } from "./capture-small-snapshot.mjs";
22
+
23
+ const nonnegative = n => Number.isSafeInteger(n) && n >= 0;
24
+ const validCursor = (value, size) => value && nonnegative(value.bytes) && value.bytes <= size &&
25
+ nonnegative(value.line) && typeof value.lastTs === "string";
26
+
27
+ export function captureNativeUnit(store, { owner, path, format = "cc", capturedPi, ignore,
28
+ start, maxRecords = 2000, maxBytes = 8388608, maxRecordBytes = CAPTURE_POLICY.maxRecordBytes,
29
+ stopped = () => false, work } = {}) {
30
+ const fmt = SESSION_FORMATS[format];
31
+ if (!fmt) throw new Error("unknown native command format");
32
+ const sessionId = fmt.sessionId(path), stream = `${owner}~${fmt.source}.${sessionId}`, key = `${stream}:${path}`;
33
+ const context = captureContext(store), state = context?.state;
34
+ if (!state?.depth || !context.held.has(stream)) throw new Error("native command unit requires its stream work group");
35
+ if (!Number.isSafeInteger(maxRecords) || maxRecords < 1 || maxRecords > 2000 ||
36
+ !Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > context.batchBytes ||
37
+ !Number.isSafeInteger(maxRecordBytes) || maxRecordBytes < 1 || maxRecordBytes > CAPTURE_POLICY.maxRecordBytes) {
38
+ throw new Error("invalid native command unit budget");
39
+ }
40
+ const result = { stream, key, appended: 0, bytesCommitted: 0, complete: false, pending: false };
41
+ if ((ignore ?? loadIgnore(store.root)).ignores(path, [basename(path), sessionId, ...(fmt.ignoreKeys?.(path) ?? [])])) {
42
+ state.delete("set-aside", key);
43
+ // Ignore is evaluated before custody/source access. An extant retained
44
+ // allocation is housekeeping, not authority to reopen an excluded source.
45
+ return { ...result, complete: true, ignored: true };
46
+ }
47
+ const blocked = state.get("set-aside", key);
48
+ if (blocked && !blocked.retry) return { ...result, setAside: blocked };
49
+ if (capturedPi && fmt.source !== "pi") throw new Error("protected capture proof requires pi format");
50
+ guardCapturedPath(path, capturedPi);
51
+ const fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK | (capturedPi ? constants.O_NOFOLLOW : 0));
52
+ let staged;
53
+ const stagedChunks = function* (from, end) {
54
+ for (const bytes of staged.snapshot.chunks(from, end)) {
55
+ if (work) work.stagedReadBytes = (work.stagedReadBytes ?? 0) + bytes.length;
56
+ yield bytes;
57
+ }
58
+ };
59
+ const stagedWitness = offset => {
60
+ if (work) work.stagedProbeBytes = (work.stagedProbeBytes ?? 0) + Math.min(4096, offset);
61
+ return digest(Buffer.concat([...staged.snapshot.chunks(Math.max(0, offset - 4096), offset)]));
62
+ };
63
+ try {
64
+ const observe = () => {
65
+ assertProtectedDescriptor(fd, path, capturedPi);
66
+ if (!fstatSync(fd).isFile()) throw new Error("native source is not a regular file");
67
+ const value = statVersion(fstatSync(fd, { bigint: true }));
68
+ if (!sameVersion(value, fileVersion(path))) throw new Error("native source descriptor differs from its named file");
69
+ return value;
70
+ };
71
+ const version = observe();
72
+ const chunks = function* (from, end) {
73
+ for (const bytes of sourceChunks(fd, from, end, path, capturedPi)) {
74
+ // Sampling between reads does not turn an incomplete line into a
75
+ // checkpoint. The outer unit stops at a complete-record boundary.
76
+ stopped();
77
+ if (work) work.sourceBytes = (work.sourceBytes ?? 0) + bytes.length;
78
+ yield bytes;
79
+ }
80
+ };
81
+ const witness = (from, read = chunks) => digest(Buffer.concat([...read(Math.max(0, from - 4096), from)]));
82
+ const probe = offset => {
83
+ const read = function* (from, end) {
84
+ for (const bytes of sourceChunks(fd, from, end, path, capturedPi)) {
85
+ if (work) work.sourceProbeBytes = (work.sourceProbeBytes ?? 0) + bytes.length;
86
+ yield bytes;
87
+ }
88
+ };
89
+ return witness(offset, read);
90
+ };
91
+ const generation = JSON.stringify([state.get("reset", `${owner}:${fmt.source}`) ?? "", state.get("source-retry", key) ?? ""]);
92
+ // Ordinary public cursors use the reset value directly. Command cursors
93
+ // retain it too; the private job additionally binds an explicit retry.
94
+ const reset = state.get("reset", `${owner}:${fmt.source}`) ?? "";
95
+ const journalVersion = fileVersion(store.journalPath(stream));
96
+ let cursor = state.get(fmt.source, key);
97
+ const matchesSource = value => value?.version && (sameVersion(value.version, version) ||
98
+ value.version.dev === version.dev && value.version.ino === version.ino && version.size > value.version.size &&
99
+ value.bytes <= version.size && probe(value.bytes) === value.witness);
100
+ let job = state.get("native-unit", key);
101
+ const jobValid = job && job.generation === generation && sameVersion(job.journalVersion, journalVersion) &&
102
+ validCursor(job, version.size) && nonnegative(job.end) && job.bytes <= job.end && job.end <= version.size &&
103
+ matchesSource(job);
104
+ if (!jobValid) { state.delete("native-unit", key); job = null; }
105
+ const cursorValid = validCursor(cursor, version.size) && cursor.generation === reset &&
106
+ sameVersion(cursor.journalVersion, journalVersion) && matchesSource(cursor) && !blocked;
107
+ const recover = !job && !cursorValid && (journalVersion?.size ?? 0) > 0;
108
+ const allocation = state.get("recovery-allocation", key);
109
+ if (allocation?.phase === "deleting") {
110
+ prepareRetainedCaptureCleanup(store, stream, key, start);
111
+ return { ...result, cleanup: true, phase: "cleanup" };
112
+ }
113
+ if (job?.waiting && job.end === version.size && sameVersion(job.version, version)) {
114
+ return { ...result, pending: true, phase: job.scan ? "timestamp" : "capture", reason: job.waiting };
115
+ }
116
+ if (!job && !allocation && cursorValid && cursor.bytes === version.size) {
117
+ return { ...result, complete: true, unchanged: true, phase: "captured" };
118
+ }
119
+ // Journal validation can itself need several units. Never let appendBatch
120
+ // fall back to an unbounded validation inside the first source append.
121
+ const validated = store.validateJournal(stream, state, { maxRecords, maxBytes, stopped, work: work?.journal });
122
+ if (!validated.complete) return { ...result, phase: "journal" };
123
+ const unitWitness = probe(version.size);
124
+ let read = chunks, verify = () => {
125
+ const after = observe();
126
+ if (!sameVersion(version, after) && !(version.dev === after.dev && version.ino === after.ino &&
127
+ after.size > version.size && probe(version.size) === unitWitness)) throw new Error("native source changed during unit");
128
+ };
129
+ if (recover || allocation || job?.snapshot) {
130
+ let small = state.get("small-stage", key);
131
+ if (!small || small.generation !== generation || !sameVersion(small.journalVersion, journalVersion) ||
132
+ !nonnegative(small.size) || small.size > SMALL_SNAPSHOT_BYTES || small.size > version.size ||
133
+ typeof small.id !== "string" || !matchesSource({ ...small, bytes: small.size })) small = null;
134
+ const end = job?.end ?? small?.size ?? version.size;
135
+ // A killed unit has durable admission/retry evidence but cannot have
136
+ // published an in-memory small-copy outcome. Its next due single-stream
137
+ // retry uses retained copying rather than repeating that overhead forever.
138
+ const retain = allocation || state.get("native-retain", key) || context.admission?.retry || end > SMALL_SNAPSHOT_BYTES;
139
+ if (retain) {
140
+ staged = advanceRecoverySnapshot(store, stream, key, path, start, { capturedPi, generation, fixedEnd: end, maxBytes, stopped, work });
141
+ } else {
142
+ // This observation fixes the sweep end even if copying cannot finish.
143
+ // It is NOT a digest/proof checkpoint: reuse below always requires a
144
+ // complete fresh copy/hash before advanceTranscriptPrefix sees its id.
145
+ small ??= { id: randomUUID(), size: end };
146
+ small = { ...small, generation, journalVersion, version, witness: probe(end) };
147
+ state.set("small-stage", key, small);
148
+ staged = smallCommandSnapshot(store.root, fd, path, end, { capturedPi, stopped, work });
149
+ if (staged.complete) {
150
+ // The nonce binds source observation/reset history; digest+size bind
151
+ // bytes. Equal content under a changed identity still restarts proof.
152
+ staged.snapshot.id = `${small.id}:${staged.snapshot.id}`;
153
+ }
154
+ if (!staged.complete || stopped()) {
155
+ state.set("native-retain", key, true);
156
+ state.delete("native-unit", key);
157
+ return { ...result, phase: "snapshot", retainedFallback: true };
158
+ }
159
+ }
160
+ if (!staged.complete) {
161
+ if (staged.cleanup) state.delete("native-unit", key);
162
+ return { ...result, ...staged, phase: staged.phase ?? "snapshot" };
163
+ }
164
+ if (!job || job.snapshot !== staged.snapshot.id) {
165
+ const priorProof = state.get("prefix-proof", key);
166
+ // The captured prefix is proved by streamed per-line digests, so a
167
+ // record captured before the raw-record limit existed is compared, not
168
+ // refused. The limit applies to capture beyond the proven prefix.
169
+ const proof = advanceTranscriptPrefix(store, stream, { key, snapshot: staged.snapshot, maxRecords, maxBytes, stopped,
170
+ work: work?.prefix, lines: from => completeNativeLineDigests(stagedChunks(from), {}) });
171
+ staged.verify();
172
+ if (proof.conflict) {
173
+ // Journals preserve session/line, not the path that first supplied
174
+ // those bytes. A mismatch can also be a rewrite of this same file;
175
+ // neither a cursor nor today's discovery proves a second source.
176
+ const reason = `${proof.conflict.reason}: refused input ${JSON.stringify(path)} conflicts with stream ${JSON.stringify(stream)} at native line ${proof.conflict.line}, source byte ${proof.conflict.offset}; establishing source path unavailable`;
177
+ const row = { owner, source: fmt.source, path, ...proof.conflict, reason, retry: false };
178
+ state.set("set-aside", key, row);
179
+ const cleanup = !staged.small;
180
+ staged.close(); staged = null;
181
+ state.delete("native-unit", key); state.delete("small-stage", key);
182
+ if (cleanup) prepareRetainedCaptureCleanup(store, stream, key, start);
183
+ return { ...result, setAside: row, cleanup };
184
+ }
185
+ if (!proof.complete) {
186
+ // A full restage that leaves no allowance for proof progress must not
187
+ // recur indefinitely. One committed switch selects resumable copying.
188
+ if (staged.small && stopped() && priorProof &&
189
+ ["phase", "journalBytes", "bytes", "line"].every(k => proof[k] === priorProof[k])) {
190
+ state.set("native-retain", key, true);
191
+ }
192
+ return { ...result, phase: "prefix" };
193
+ }
194
+ job = { bytes: proof.bytes, line: proof.line, lastTs: proof.lastTs, end: staged.snapshot.size,
195
+ snapshot: staged.snapshot.id, small: Boolean(staged.small), generation, journalVersion, version, witness: stagedWitness(proof.bytes) };
196
+ state.delete("set-aside", key);
197
+ }
198
+ } else if (!job) {
199
+ cursor = cursorValid ? cursor : { bytes: 0, line: 0, lastTs: "" };
200
+ job = { bytes: cursor.bytes, line: cursor.line, lastTs: cursor.lastTs, end: version.size,
201
+ generation, journalVersion, version, witness: probe(cursor.bytes) };
202
+ }
203
+ if (staged) { read = stagedChunks; verify = staged.verify; }
204
+ // A raw record over the limit is set aside at its exact start, after the
205
+ // complete records before it were committed. Later records wait for an
206
+ // edited source or an ignore of the whole input. A staged job's copy and
207
+ // fixed end retire with it; a retry reproves and returns here.
208
+ const setAsideOversized = offset => {
209
+ state.delete("native-unit", key); state.delete("native-retain", key); state.delete("small-stage", key);
210
+ if (job.snapshot && !job.small) {
211
+ staged.close(); staged = null;
212
+ prepareRetainedCaptureCleanup(store, stream, key, start);
213
+ result.cleanup = true;
214
+ }
215
+ const row = { owner, source: fmt.source, path, offset, reason: "record-too-large", retry: false };
216
+ state.set("set-aside", key, row);
217
+ return { ...result, setAside: row };
218
+ };
219
+ if (job.waiting) {
220
+ if (job.end === version.size) return { ...result, pending: true, phase: job.scan ? "timestamp" : "capture", reason: job.waiting };
221
+ if (job.snapshot) {
222
+ staged.close(); staged = null;
223
+ if (!job.small) prepareRetainedCaptureCleanup(store, stream, key, start);
224
+ state.delete("native-unit", key); state.delete("small-stage", key);
225
+ return { ...result, cleanup: !job.small, phase: "cleanup" };
226
+ }
227
+ // An exhausted fixed end was pending, so the next source growth is a
228
+ // new bounded sweep. Keep its timestamp frontier, not its old EOF.
229
+ job.end = version.size; delete job.waiting;
230
+ }
231
+ const checkpoint = () => {
232
+ verify();
233
+ job.version = staged && !staged.small ? state.get("recovery-stage", key).observedVersion : version;
234
+ job.witness = staged ? stagedWitness(job.bytes) : probe(job.bytes);
235
+ job.journalVersion = fileVersion(store.journalPath(stream));
236
+ state.set("native-unit", key, job);
237
+ state.set(fmt.source, key, { bytes: job.bytes, line: job.line, lastTs: job.lastTs,
238
+ version: job.version, witness: job.witness, generation: reset, journalVersion: job.journalVersion });
239
+ if (job.small) {
240
+ state.set("small-stage", key, { ...state.get("small-stage", key), journalVersion: job.journalVersion });
241
+ } else if (job.snapshot) {
242
+ // Only our successful fsynced append under this stream lock can carry
243
+ // the proven fixed snapshot into the next exact journal generation.
244
+ const stage = state.get("recovery-stage", key);
245
+ if (stage?.allocation !== job.snapshot) throw new Error("native unit lost its staged proof");
246
+ state.set("recovery-stage", key, { ...stage, journalVersion: job.journalVersion });
247
+ }
248
+ };
249
+ let used = 0, records = 0;
250
+ const stop = () => records >= maxRecords || used >= maxBytes || stopped();
251
+ const walk = {};
252
+ const lines = from => completeNativeLines(read(from, job.end), walk, { maxBytes: maxRecordBytes });
253
+ let scanStart = job.bytes;
254
+ try {
255
+ if (!job.lastTs && job.bytes < job.end) {
256
+ const scan = job.scan ?? { bytes: job.bytes, line: job.line };
257
+ if (!validCursor({ ...scan, lastTs: "" }, job.end) || scan.bytes < job.bytes) throw new Error("invalid native timestamp scan");
258
+ scanStart = scan.bytes;
259
+ const iterator = lines(scan.bytes);
260
+ try {
261
+ while (!stop()) {
262
+ const next = iterator.next(); if (next.done) break;
263
+ const line = next.value; scan.bytes += line.bytes; scan.line++; used += line.bytes; records++;
264
+ if (line.ts) { job.lastTs = line.ts; delete job.scan; break; }
265
+ }
266
+ } finally { iterator.return(); }
267
+ if (!job.lastTs) {
268
+ job.scan = scan;
269
+ if (scan.bytes === job.end || walk.reason) job.waiting = walk.reason ?? "unstamped";
270
+ checkpoint();
271
+ // No stamp yet is pending, not a successful empty capture. Preserve
272
+ // the scan frontier so later growth does not rescan this prefix.
273
+ return { ...result, phase: "timestamp", pending: scan.bytes === job.end || !!walk.reason, reason: walk.reason ?? "unstamped" };
274
+ }
275
+ }
276
+ // Timestamp search is work too; it cannot be followed by another full
277
+ // batch after exhausting this unit's allowance.
278
+ const turns = [], from = job.bytes, iterator = lines(from);
279
+ scanStart = from;
280
+ let oversized = null;
281
+ try {
282
+ while (!stop()) {
283
+ const next = iterator.next(); if (next.done) break;
284
+ const line = next.value;
285
+ if (turns.length && used + line.bytes > maxBytes) break;
286
+ if (line.ts) job.lastTs = line.ts;
287
+ job.bytes += line.bytes; job.line++; used += line.bytes; records++;
288
+ turns.push(finishTurn(eventTurnCore({ owner, source: fmt.source, sessionId, line: line.text, lineNo: job.line, ts: job.lastTs })));
289
+ }
290
+ } catch (error) {
291
+ if (!(error instanceof RecordSizeError)) throw error;
292
+ // The raw record was rejected before allocation/projection. Every
293
+ // complete stamped line before it is still processable: append and
294
+ // checkpoint them through the ordinary commit below, then set aside.
295
+ oversized = from + error.offset;
296
+ if (oversized !== job.bytes) throw new Error("native oversized record does not follow the consumed lines");
297
+ } finally { iterator.return(); }
298
+ verify();
299
+ if (turns.length) result.appended = store.appendUniqueBatch(stream, turns);
300
+ result.bytesCommitted = job.bytes - from;
301
+ if (walk.reason) job.waiting = walk.reason;
302
+ checkpoint();
303
+ if (oversized !== null) return setAsideOversized(oversized);
304
+ if (job.small && job.bytes < job.end && !records && stopped()) state.set("native-retain", key, true);
305
+ if (job.bytes === job.end) {
306
+ state.delete("native-unit", key); state.delete("native-retain", key); state.delete("small-stage", key);
307
+ if (job.snapshot && !job.small) {
308
+ staged.close(); staged = null;
309
+ prepareRetainedCaptureCleanup(store, stream, key, start);
310
+ result.cleanup = true;
311
+ }
312
+ return { ...result, complete: true, phase: "captured" };
313
+ }
314
+ return { ...result, phase: "capture", pending: !!walk.reason, reason: walk.reason ?? null };
315
+ } catch (error) {
316
+ if (!(error instanceof RecordSizeError)) throw error;
317
+ // The offending raw record was rejected before allocation/projection
318
+ // while searching for the first timestamp: no turn precedes it here.
319
+ return setAsideOversized(scanStart + error.offset);
320
+ }
321
+ } finally { try { staged?.close?.(); } finally { closeSync(fd); } }
322
+ }