@awebai/oats 0.53.0 → 0.55.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/bin/oats.mjs +17 -7
  2. package/docs/design/2026-09-23-workspace-module-contracts.md +3 -0
  3. package/docs/desktop-cli-api.md +44 -12
  4. package/docs/implementation.md +12 -4
  5. package/docs/integrations.md +2 -2
  6. package/docs/knowledge.md +14 -2
  7. package/docs/official-catalog.md +5 -5
  8. package/docs/packages.md +14 -14
  9. package/docs/release-notes/v0.53.0.md +14 -0
  10. package/docs/release-notes/v0.54.0.md +142 -0
  11. package/docs/release-notes/v0.55.0.md +158 -0
  12. package/docs/servers.md +13 -3
  13. package/docs/souls-and-instances.md +2 -1
  14. package/docs/upgrading.md +14 -2
  15. package/docs/workspaces.md +122 -2
  16. package/lib/packages.mjs +5 -1
  17. package/lib/remote.mjs +255 -76
  18. package/lib/servers.mjs +82 -34
  19. package/lib/workspace.mjs +27 -9
  20. package/package-catalog.json +4 -4
  21. package/package.json +1 -1
  22. package/packages/client/cli-locator.mjs +2 -2
  23. package/packages/client/workspace-views.mjs +4 -3
  24. package/packages/record/README.md +54 -13
  25. package/packages/record/bin/capture.mjs +107 -53
  26. package/packages/record/docs/capture-service.md +193 -0
  27. package/packages/record/docs/capture-status.schema.json +100 -0
  28. package/packages/record/lib/capture-aw.mjs +96 -94
  29. package/packages/record/lib/capture-cc.mjs +109 -187
  30. package/packages/record/lib/capture-db-reader.mjs +39 -0
  31. package/packages/record/lib/capture-issues.mjs +64 -0
  32. package/packages/record/lib/capture-scratch.mjs +111 -0
  33. package/packages/record/lib/capture-state.mjs +105 -0
  34. package/packages/record/lib/derived-state.mjs +60 -0
  35. package/packages/record/lib/formats.mjs +31 -31
  36. package/packages/record/lib/index-db.mjs +74 -34
  37. package/packages/record/lib/receipt-summary.mjs +62 -0
  38. package/packages/record/lib/session-snapshot.mjs +56 -1
  39. package/packages/record/lib/sqlite-rows.mjs +14 -0
  40. package/packages/record/lib/store.mjs +122 -43
  41. package/packages/record/lib/transcript-prefix.mjs +40 -0
  42. package/skills/oats-getting-started/SKILL.md +8 -2
@@ -0,0 +1,111 @@
1
+ // Private same-volume scratch for synchronous capture/receipt work. Allocations
2
+ // are never transferred to subprocesses. Root-lock cleanup reclaims only proven
3
+ // dead local owners; an incomplete marker or reused PID remains conservative.
4
+ import { closeSync, constants, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, opendirSync, readSync, rmSync, rmdirSync, statSync, writeFileSync } from "node:fs";
5
+ import { randomBytes } from "node:crypto";
6
+ import { hostname } from "node:os";
7
+ import { join } from "node:path";
8
+
9
+ const sameObject = (a, b) => a.dev === b.dev && a.ino === b.ino;
10
+ const sameFile = (a, b) => sameObject(a, b) && a.size === b.size && a.mtimeNs === b.mtimeNs && a.ctimeNs === b.ctimeNs;
11
+ function directory(path) {
12
+ const stat = lstatSync(path, { bigint: true });
13
+ if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error(`capture scratch is not a real directory: ${path}`);
14
+ return stat;
15
+ }
16
+ function privateRoot(root, create) {
17
+ const path = join(root, ".capture-tmp");
18
+ if (create) { mkdirSync(root, { recursive: true }); try { mkdirSync(path, { mode: 0o700 }); } catch (e) { if (e.code !== "EEXIST") throw e; } }
19
+ const stat = directory(path);
20
+ if (stat.dev !== statSync(root, { bigint: true }).dev) throw new Error(`capture scratch must be on the record root volume: ${path}`);
21
+ if ((stat.mode & 0o077n) !== 0n || (process.getuid && stat.uid !== BigInt(process.getuid()))) throw new Error(`capture scratch directory must be private and owned by this user: ${path}`);
22
+ return { path, stat };
23
+ }
24
+ // Bound the read, refuse links/special files before opening, and bind bytes to
25
+ // the descriptor and unchanged named file. Missing/unreadable means unknown.
26
+ function marker(dir) {
27
+ const path = join(dir, "owner.json"); let fd;
28
+ try {
29
+ const before = lstatSync(path, { bigint: true });
30
+ if (!before.isFile() || before.isSymbolicLink() || before.size > 4096n) return null;
31
+ fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
32
+ const stat = fstatSync(fd, { bigint: true });
33
+ if (!stat.isFile() || !sameFile(before, stat)) return null;
34
+ const bytes = Buffer.alloc(Number(stat.size)); let offset = 0;
35
+ while (offset < bytes.length) { const n = readSync(fd, bytes, offset, bytes.length - offset, offset); if (!n) return null; offset += n; }
36
+ if (!sameFile(stat, fstatSync(fd, { bigint: true })) || !sameFile(stat, lstatSync(path, { bigint: true }))) return null;
37
+ const raw = bytes.toString("utf8"); return { stat, raw, value: JSON.parse(raw) };
38
+ } catch { return null; }
39
+ finally { if (fd !== undefined) closeSync(fd); }
40
+ }
41
+ function sameMarker(a, b) { return a && b && sameFile(a.stat, b.stat) && a.raw === b.raw; }
42
+ function unchanged(dir, stat, owner) {
43
+ try { return sameObject(stat, directory(dir)) && sameMarker(owner, marker(dir)); }
44
+ catch { return false; }
45
+ }
46
+ function localLiveness(pid) {
47
+ try { process.kill(pid, 0); return "alive"; }
48
+ catch (err) { return err.code === "ESRCH" ? "dead" : "unknown"; }
49
+ }
50
+
51
+ export function createCaptureScratch(root, kind) {
52
+ if (!root || !["snapshot", "receipt"].includes(kind)) throw new Error("capture scratch requires a record root and recognized purpose");
53
+ const base = privateRoot(root, true);
54
+ const path = mkdtempSync(join(base.path, `${kind}-`)), stat = directory(path);
55
+ const owner = { v: 1, kind, pid: process.pid, host: hostname(), nonce: randomBytes(16).toString("hex") };
56
+ try {
57
+ writeFileSync(join(path, "owner.json"), JSON.stringify(owner), { flag: "wx", mode: 0o600 });
58
+ const published = marker(path);
59
+ if (!published || !sameObject(base.stat, directory(base.path))) throw new Error("capture scratch ownership publication failed");
60
+ return { path, close() {
61
+ // Ordinary callers remove only their own unchanged allocation. An
62
+ // external replacement is preserved, even if its pathname is familiar.
63
+ if (sameObject(base.stat, directory(base.path)) && unchanged(path, stat, published)) rmSync(path, { recursive: true });
64
+ } };
65
+ } catch (err) {
66
+ // No payload has been opened yet. A partial/unreadable owner marker is
67
+ // deliberately preserved; only an empty or positively owned allocation
68
+ // can be removed. An empty-dir removal cannot erase substituted contents.
69
+ try {
70
+ if (sameObject(base.stat, directory(base.path)) && sameObject(stat, directory(path))) {
71
+ const current = marker(path);
72
+ if (current?.value?.nonce === owner.nonce) rmSync(path, { recursive: true });
73
+ else rmdirSync(path);
74
+ }
75
+ } catch { /* uncertain ownership remains for conservative operator recovery */ }
76
+ throw err;
77
+ }
78
+ }
79
+
80
+ export function cleanupCaptureScratch(root, lock, { liveness = localLiveness } = {}) {
81
+ const lockPath = join(root, ".capture.lock"), lockStat = directory(lockPath), owner = marker(lockPath);
82
+ if (lock?.path !== lockPath || lock.held || typeof lock.release !== "function" ||
83
+ owner?.value?.pid !== process.pid || owner.value.host !== hostname() || typeof owner.value.nonce !== "string") {
84
+ throw new Error("capture scratch cleanup requires the current process's capture lock");
85
+ }
86
+ let base;
87
+ try { base = privateRoot(root, false); } catch (err) { if (err.code === "ENOENT") return 0; throw err; }
88
+ const entries = opendirSync(base.path); let removed = 0;
89
+ try {
90
+ for (let entry; (entry = entries.readSync()) !== null;) {
91
+ const match = /^(snapshot|receipt)-[A-Za-z0-9]{6}$/.exec(entry.name);
92
+ if (!match || !entry.isDirectory() || entry.isSymbolicLink()) continue;
93
+ const path = join(base.path, entry.name); let stat;
94
+ try { stat = directory(path); } catch { continue; }
95
+ const candidate = marker(path), value = candidate?.value;
96
+ if (value?.v !== 1 || value.kind !== match[1] || value.host !== hostname() ||
97
+ !Number.isInteger(value.pid) || value.pid <= 0 || value.pid > 2147483647 ||
98
+ typeof value.nonce !== "string" || !/^[a-f0-9]{32}$/.test(value.nonce)) continue;
99
+ let live;
100
+ try { live = liveness(value.pid); } catch { continue; }
101
+ if (live !== "dead") continue;
102
+ if (!unchanged(lockPath, lockStat, owner)) throw new Error("capture lock changed during scratch cleanup");
103
+ if (!sameObject(base.stat, directory(base.path))) throw new Error("capture scratch root changed during cleanup");
104
+ if (!unchanged(path, stat, candidate)) continue;
105
+ // No supported user of this helper transfers scratch/FD ownership to a
106
+ // subprocess. Dead allocator + unchanged identity is therefore required.
107
+ rmSync(path, { recursive: true }); removed++;
108
+ }
109
+ } finally { entries.closeSync(); }
110
+ return removed;
111
+ }
@@ -0,0 +1,105 @@
1
+ // Shared capture status and operator recovery. Service supervision later fills
2
+ // policy/run evidence; null means no such evidence has been supplied.
3
+ import { randomUUID } from "node:crypto";
4
+ import { closeSync, constants, fstatSync, fsyncSync, openSync, readSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
5
+ import { join } from "node:path";
6
+ import { DerivedState, invalidateLegacyMaps } from "./derived-state.mjs";
7
+ import { readCurrentIssueBacklog, readIssueBacklog } from "./capture-issues.mjs";
8
+
9
+ export const CAPTURE_SOURCES = ["aw", "cc", "pi", "codex"];
10
+ export const SET_ASIDE_REMEDY = "Restore the captured prefix and run capture --retry-set-aside; or add the source to <root>/ignore (later turns from that session will not be captured). Retry revalidates; it never authorizes a changed prefix.";
11
+
12
+ // Caller owns the root capture lock. One transaction persists every selected
13
+ // generation; a missing source cursor cannot erase this rescan request.
14
+ export function resetSourceCursors(root, owner, source) {
15
+ if (source !== undefined && !CAPTURE_SOURCES.includes(source)) throw new Error(`unknown capture source: ${source}`);
16
+ invalidateLegacyMaps(root);
17
+ const state = new DerivedState(root);
18
+ try {
19
+ state.transaction(() => {
20
+ for (const name of source ? [source] : CAPTURE_SOURCES) state.set("reset", `${owner}:${name}`, randomUUID());
21
+ });
22
+ } finally { state.close(); }
23
+ }
24
+
25
+ export function retrySetAside(root) {
26
+ const state = new DerivedState(root);
27
+ try {
28
+ state.transaction(() => {
29
+ for (const { key, value } of state.entries("set-aside")) state.set("set-aside", key, { ...value, retry: true });
30
+ });
31
+ } finally { state.close(); }
32
+ }
33
+
34
+ export function writeCaptureStatus(root, { pending = false } = {}) {
35
+ const state = new DerivedState(root);
36
+ let backlog;
37
+ try {
38
+ backlog = readIssueBacklog(state.db, { pending });
39
+ } finally { state.close(); }
40
+ const status = { schemaVersion: 1, policy: null, lastRun: null, lastSuccess: null, lastFailure: null, lastProgressAt: null, backlog };
41
+ const path = join(root, "capture-status.json"), temp = `${path}.${randomUUID()}.tmp`;
42
+ let fd;
43
+ try {
44
+ fd = openSync(temp, "wx", 0o600);
45
+ writeFileSync(fd, JSON.stringify(status) + "\n"); fsyncSync(fd); closeSync(fd); fd = undefined;
46
+ renameSync(temp, path);
47
+ } finally {
48
+ if (fd !== undefined) closeSync(fd);
49
+ try { unlinkSync(temp); } catch (err) { if (err.code !== "ENOENT") throw err; }
50
+ }
51
+ return status;
52
+ }
53
+
54
+ function validStatus(status) {
55
+ const b = status?.backlog;
56
+ return status?.schemaVersion === 1 && status.policy === null && status.lastRun === null && status.lastSuccess === null &&
57
+ status.lastFailure === null && status.lastProgressAt === null && typeof b?.pending === "boolean" &&
58
+ Number.isSafeInteger(b.setAsideCount) && b.setAsideCount >= 0 && Array.isArray(b.setAside) && b.setAside.length <= 100 &&
59
+ b.setAside.length === Math.min(b.setAsideCount, 100) && b.truncated === (b.setAsideCount > 100) &&
60
+ b.setAside.every(e => CAPTURE_SOURCES.includes(e.source) && typeof e.path === "string" && typeof e.reason === "string" &&
61
+ Number.isSafeInteger(e.offset) && e.offset >= 0 && (e.line === undefined || Number.isSafeInteger(e.line) && e.line > 0));
62
+ }
63
+
64
+ // Read-only, including a never-installed/absent root: no writable DB construction,
65
+ // directory creation, manager query, index rebuild or capture.
66
+ export function captureHealth(root) {
67
+ const path = join(root, "capture-status.json");
68
+ let status = { lastRun: null, lastSuccess: null, lastFailure: null,
69
+ backlog: { pending: false, setAsideCount: 0, setAside: [], truncated: false } };
70
+ const rows = [];
71
+ try {
72
+ const fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK);
73
+ let text;
74
+ try {
75
+ const stat = fstatSync(fd);
76
+ if (!stat.isFile()) throw new Error("status is not a regular file");
77
+ const limit = 1024 * 1024;
78
+ if (stat.size > limit) throw new Error("status exceeds 1 MiB");
79
+ const bytes = Buffer.alloc(limit + 1);
80
+ let size = 0;
81
+ while (size < bytes.length) {
82
+ const count = readSync(fd, bytes, size, bytes.length - size, size);
83
+ if (!count) break;
84
+ size += count;
85
+ }
86
+ if (size > limit) throw new Error("status exceeds 1 MiB");
87
+ text = bytes.toString("utf8", 0, size);
88
+ } finally { closeSync(fd); }
89
+ const parsed = JSON.parse(text);
90
+ if (!validStatus(parsed)) throw new Error("unsupported or malformed status");
91
+ status = parsed;
92
+ } catch (err) {
93
+ if (err.code !== "ENOENT") rows.push({ code: "capture-status-invalid", message: `Cannot read capture status at ${path}: ${err.message}` });
94
+ }
95
+ let backlog;
96
+ try {
97
+ backlog = readCurrentIssueBacklog(root);
98
+ if (!backlog && status.backlog.setAsideCount) throw new Error("capture.db is missing but the last command reported active set-aside inputs");
99
+ } catch (err) { throw new Error(`Cannot read current capture issues at ${join(root, "index", "capture.db")}: ${err.message}`, { cause: err }); }
100
+ backlog ??= { pending: false, setAsideCount: 0, setAside: [], truncated: false };
101
+ if (backlog.setAsideCount) rows.push({ code: "capture-input-set-aside", message:
102
+ `${backlog.setAsideCount} capture inputs set aside. ${SET_ASIDE_REMEDY.replace("<root>", root)}` });
103
+ return { schemaVersion: 1, status: rows.length ? "warning" : "ok", rows,
104
+ lastRun: status.lastRun, lastSuccess: status.lastSuccess, lastFailure: status.lastFailure, backlog };
105
+ }
@@ -0,0 +1,60 @@
1
+ // Local accelerators only. Journal bytes remain authority; removing this
2
+ // database costs bounded rebuild work, never changes a turn or its ID.
3
+ import { mkdirSync, statSync, unlinkSync } from "node:fs";
4
+ import { join } from "node:path";
5
+ import { keysetRows } from "./sqlite-rows.mjs";
6
+ import { createRequire } from "node:module";
7
+ const require = createRequire(import.meta.url);
8
+
9
+ export function invalidateLegacyMaps(root) {
10
+ for (const name of ["capture-offsets.json", "capture-aw-seen.json"]) {
11
+ try { unlinkSync(join(root, "index", name)); }
12
+ catch (err) { if (err.code !== "ENOENT") throw err; }
13
+ }
14
+ }
15
+ export function statVersion(st) {
16
+ return { dev: String(st.dev), ino: String(st.ino), size: Number(st.size), mtime: String(st.mtimeNs ?? st.mtimeMs), ctime: String(st.ctimeNs ?? st.ctimeMs) };
17
+ }
18
+ export function fileVersion(path) {
19
+ try {
20
+ const st = statSync(path, { bigint: true });
21
+ if (!st.isFile()) throw new Error(`not a regular file: ${path}`);
22
+ return statVersion(st);
23
+ } catch (err) { if (err.code === "ENOENT") return null; throw err; }
24
+ }
25
+ export const sameVersion = (a, b) => JSON.stringify(a) === JSON.stringify(b);
26
+
27
+ export class DerivedState {
28
+ constructor(root) {
29
+ const { DatabaseSync } = require("node:sqlite");
30
+ mkdirSync(join(root, "index"), { recursive: true });
31
+ this.db = new DatabaseSync(join(root, "index", "capture.db"));
32
+ this.db.exec("PRAGMA busy_timeout = 60000; PRAGMA cache_size = -2048; PRAGMA temp_store = FILE");
33
+ this.db.exec("BEGIN IMMEDIATE");
34
+ try {
35
+ const hadIds = this.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name='journal_ids'").get()?.found === 1;
36
+ this.db.exec(`CREATE TABLE IF NOT EXISTS state (kind TEXT, key TEXT, value TEXT NOT NULL, PRIMARY KEY(kind, key));
37
+ CREATE TABLE IF NOT EXISTS journal_ids (stream TEXT, id TEXT, PRIMARY KEY(stream, id));`);
38
+ // Recreating a lost lookup atomically revokes the old validation witness.
39
+ // A crash cannot qualify an empty replacement table with that witness.
40
+ if (!hadIds) this.db.exec("DELETE FROM state WHERE kind = 'journal'");
41
+ this.db.exec("COMMIT");
42
+ } catch (err) { this.db.exec("ROLLBACK"); this.db.close(); throw err; }
43
+ this.read = this.db.prepare("SELECT value FROM state WHERE kind = ? AND key = ?");
44
+ this.write = this.db.prepare("INSERT INTO state VALUES (?, ?, ?) ON CONFLICT(kind,key) DO UPDATE SET value=excluded.value");
45
+ }
46
+ get(kind, key) { const row = this.read.get(kind, key); return row ? JSON.parse(row.value) : null; }
47
+ set(kind, key, value) { this.write.run(kind, key, JSON.stringify(value)); }
48
+ delete(kind, key) { this.db.prepare("DELETE FROM state WHERE kind = ? AND key = ?").run(kind, key); }
49
+ *entries(kind) {
50
+ for (const row of keysetRows(this.db, "SELECT key, value FROM state WHERE kind = ?", "key", [kind])) {
51
+ yield { key: row.key, value: JSON.parse(row.value) };
52
+ }
53
+ }
54
+ transaction(fn) {
55
+ this.db.exec("BEGIN IMMEDIATE");
56
+ try { const result = fn(); this.db.exec("COMMIT"); return result; }
57
+ catch (err) { this.db.exec("ROLLBACK"); throw err; }
58
+ }
59
+ close() { this.db.close(); }
60
+ }
@@ -55,38 +55,37 @@ function* parsedLines(bytes) {
55
55
  }
56
56
  }
57
57
 
58
- function listJsonlFiles(root, maxDepth, { strict = false } = {}, allowCaptured = false) {
58
+ // Preserve the public inventory's per-directory localeCompare order and stable
59
+ // ties for ordinary capture too. Path metadata grows with directory size;
60
+ // source contents are streamed independently of this discovery inventory.
61
+ function* directoryEntries(path, optional = false) {
62
+ let entries;
63
+ try { entries = readdirSync(path, { withFileTypes: true }); }
64
+ catch (err) { if (optional && err.code === "ENOENT") return; throw err; }
65
+ yield* entries.sort((a, b) => a.name.localeCompare(b.name));
66
+ }
67
+ function* iterateJsonlFiles(root, maxDepth, { strict = false } = {}, allowCaptured = false) {
59
68
  const capturedPi = typeof root === "object" && root !== null ? root.capturedPi : undefined;
60
69
  if (typeof root !== "string") {
61
70
  if (!allowCaptured || !capturedPi || Object.keys(root).some(k => !["path", "capturedPi"].includes(k))) throw new Error("unsupported protected session inventory");
62
71
  root = root.path;
63
72
  }
64
- guardCapturedPath(root, capturedPi);
65
- const out = [];
66
- const walk = (dir, depth) => {
73
+ function* walk(dir, depth) {
67
74
  guardCapturedPath(dir, capturedPi);
68
- let names;
69
- try {
70
- names = readdirSync(dir, { withFileTypes: true });
71
- } catch (err) {
72
- // An optional, absent root is normal; an unreadable directory or a
73
- // subtree that disappeared during discovery is not an empty scan.
74
- if (!strict && depth === 0 && err.code === "ENOENT") return;
75
- throw err;
76
- }
77
- for (const entry of names.sort((a, b) => a.name.localeCompare(b.name))) {
75
+ for (const entry of directoryEntries(dir, !strict && depth === 0)) {
78
76
  const path = join(dir, entry.name);
79
- guardCapturedPath(path, capturedPi); // protected links refuse before traversal or native reads
80
- if (entry.name.endsWith(".jsonl") && !entry.isDirectory()) {
81
- out.push(capturedPi ? { path, capturedPi } : path); // privacy rules precede opening/statting source links
82
- } else if (entry.isDirectory() || (entry.isSymbolicLink() && statSync(path).isDirectory())) {
83
- if (depth < maxDepth) walk(path, depth + 1);
77
+ guardCapturedPath(path, capturedPi);
78
+ if (entry.name.endsWith(".jsonl") && !entry.isDirectory()) yield capturedPi ? { path, capturedPi } : path;
79
+ else if (entry.isDirectory() || (entry.isSymbolicLink() && statSync(path).isDirectory())) {
80
+ if (depth < maxDepth) yield* walk(path, depth + 1);
84
81
  }
85
82
  }
86
- };
87
- walk(root, 0);
83
+ }
84
+ yield* walk(root, 0);
88
85
  guardCapturedPath(root, capturedPi);
89
- return out;
86
+ }
87
+ function listJsonlFiles(root, depth, options, allowCaptured) {
88
+ return [...iterateJsonlFiles(root, depth, options, allowCaptured)];
90
89
  }
91
90
 
92
91
  // Only absence makes a default root optional. existsSync also hides access
@@ -138,24 +137,22 @@ function ccRoots(home = homedir(), env = process.env, options = {}) {
138
137
  // Native Claude child transcripts live at
139
138
  // projects/<project>/<sessionId>/subagents/agent-*.jsonl, not beside the
140
139
  // parent's file. Enumerate only this layout, never arbitrary project files.
141
- function listCcFiles(root, { strict = false } = {}) {
142
- const out = [];
140
+ function* iterateCcFiles(root, { strict = false } = {}) {
143
141
  const entries = (dir, optional = false) => {
144
142
  guardCapturedPath(dir);
145
- try { return readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)); }
146
- catch (err) { if (optional && err.code === "ENOENT") return []; throw err; }
143
+ return directoryEntries(dir, optional);
147
144
  };
148
145
  const directory = (entry, path) => entry.isDirectory() || (entry.isSymbolicLink() && !entry.name.endsWith(".jsonl") && statSync(path).isDirectory());
149
146
  for (const project of entries(root, !strict)) {
150
147
  const projectPath = join(root, project.name);
151
148
  if (!directory(project, projectPath)) {
152
- if (project.name.endsWith(".jsonl")) out.push(projectPath);
149
+ if (project.name.endsWith(".jsonl")) yield projectPath;
153
150
  continue;
154
151
  }
155
152
  for (const entry of entries(projectPath)) {
156
153
  const path = join(projectPath, entry.name);
157
154
  if (!directory(entry, path)) {
158
- if (entry.name.endsWith(".jsonl")) out.push(path);
155
+ if (entry.name.endsWith(".jsonl")) yield path;
159
156
  continue;
160
157
  }
161
158
  // Listing the session directory (rather than treating ENOENT at an
@@ -165,13 +162,13 @@ function listCcFiles(root, { strict = false } = {}) {
165
162
  const children = join(path, childDir.name);
166
163
  if (!directory(childDir, children)) throw new Error(`session subagents root is not a directory: ${children}`);
167
164
  for (const child of entries(children)) {
168
- if (child.name.endsWith(".jsonl")) out.push(join(children, child.name));
165
+ if (child.name.endsWith(".jsonl")) yield join(children, child.name);
169
166
  }
170
167
  }
171
168
  }
172
169
  }
173
- return out;
174
- }
170
+ }
171
+ function listCcFiles(root, options) { return [...iterateCcFiles(root, options)]; }
175
172
 
176
173
  function ccParentId(path) {
177
174
  return basename(dirname(path)) === "subagents" ? basename(dirname(dirname(path))) : null;
@@ -366,6 +363,7 @@ export const SESSION_FORMATS = {
366
363
  source: "cc",
367
364
  defaultRoots: ccRoots,
368
365
  listFiles: (roots, options) => roots.flatMap((r) => listCcFiles(r, options)),
366
+ *iterateFiles(roots) { for (const root of roots) yield* iterateCcFiles(root); },
369
367
  sessionId: ccSessionId,
370
368
  ignoreKeys: (path) => [basename(path, ".jsonl"), ccParentId(path)].filter(Boolean),
371
369
  extractText: extractCcText,
@@ -374,6 +372,7 @@ export const SESSION_FORMATS = {
374
372
  source: "pi",
375
373
  defaultRoots: piRoots,
376
374
  listFiles: (roots, options) => roots.flatMap((r) => listJsonlFiles(r, 1, options, true)),
375
+ *iterateFiles(roots) { for (const root of roots) yield* iterateJsonlFiles(root, 1, {}, true); },
377
376
  sessionId: piSessionId,
378
377
  extractText: extractPiText,
379
378
  },
@@ -381,6 +380,7 @@ export const SESSION_FORMATS = {
381
380
  source: "codex",
382
381
  defaultRoots: codexRoots,
383
382
  listFiles: (roots, options) => roots.flatMap((r) => listJsonlFiles(r, 3, options)),
383
+ *iterateFiles(roots) { for (const root of roots) yield* iterateJsonlFiles(root, 3); },
384
384
  sessionId: codexSessionId,
385
385
  extractText: extractCodexText,
386
386
  },
@@ -11,9 +11,11 @@
11
11
  // verbatim record); text is extracted from that line, so a hit names the
12
12
  // turn AND the event's line in the original transcript.
13
13
 
14
- import { existsSync, mkdirSync, rmSync, unlinkSync } from "node:fs";
14
+ import { closeSync, existsSync, mkdirSync, openSync, readSync, rmSync, unlinkSync } from "node:fs";
15
15
  import { join } from "node:path";
16
- import { DatabaseSync } from "node:sqlite";
16
+ import { keysetRows } from "./sqlite-rows.mjs";
17
+ import { createRequire } from "node:module";
18
+ const require = createRequire(import.meta.url);
17
19
 
18
20
  // Bump whenever the derived schema changes shape. The index is cache:
19
21
  // a version mismatch self-heals by deleting the database and rebuilding
@@ -50,6 +52,12 @@ CREATE TABLE IF NOT EXISTS index_state (
50
52
  stream TEXT PRIMARY KEY,
51
53
  indexed_count INTEGER NOT NULL
52
54
  );
55
+ CREATE TABLE IF NOT EXISTS index_bytes (
56
+ stream TEXT PRIMARY KEY,
57
+ bytes INTEGER NOT NULL,
58
+ indexed_count INTEGER NOT NULL,
59
+ epoch TEXT NOT NULL
60
+ );
53
61
  CREATE TABLE IF NOT EXISTS meta (
54
62
  key TEXT PRIMARY KEY,
55
63
  value TEXT NOT NULL
@@ -102,6 +110,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS turn_text USING fts5(
102
110
 
103
111
  export class RecordIndex {
104
112
  constructor(store) {
113
+ const { DatabaseSync } = require("node:sqlite");
105
114
  this.store = store;
106
115
  const dir = join(store.root, "index");
107
116
  mkdirSync(dir, { recursive: true });
@@ -110,7 +119,7 @@ export class RecordIndex {
110
119
  const db = new DatabaseSync(path);
111
120
  // Concurrent index users (watcher pass, manual reindex, recall)
112
121
  // wait for each other instead of dying on "database is locked".
113
- db.exec("PRAGMA busy_timeout = 60000");
122
+ db.exec("PRAGMA busy_timeout = 60000; PRAGMA cache_size = -2048; PRAGMA temp_store = FILE");
114
123
  return db;
115
124
  };
116
125
  this.db = open();
@@ -147,39 +156,70 @@ export class RecordIndex {
147
156
  return row ? row.value : null;
148
157
  }
149
158
 
150
- // Index everything appended since the last pass.
159
+ // Old releases keep writing index_state's count. A separate cursor table
160
+ // leaves that schema/writer compatible; differing counts invalidate our
161
+ // cursor rather than trusting a byte position left behind by a downgrade.
151
162
  update() {
152
- const owner = this.store.owner;
153
- this.db.exec("BEGIN");
154
- try {
155
- if (owner) {
156
- this.db
157
- .prepare("INSERT INTO meta (key, value) VALUES ('owner', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value")
158
- .run(owner);
163
+ let reset = false;
164
+ for (const row of keysetRows(this.db, `SELECT s.stream, s.indexed_count, b.bytes,
165
+ b.indexed_count AS byte_count, b.epoch FROM index_state s
166
+ LEFT JOIN index_bytes b ON s.stream = b.stream`, "stream")) {
167
+ const checkpoint = this.store.journalCheckpoint(row.stream);
168
+ let boundary = true;
169
+ if (Number.isSafeInteger(row.bytes) && row.bytes > 0 && row.bytes < checkpoint.bytes) {
170
+ const fd = openSync(this.store.journalPath(row.stream), "r"), byte = Buffer.alloc(1);
171
+ try { boundary = readSync(fd, byte, 0, 1, row.bytes - 1) === 1 && byte[0] === 10; }
172
+ finally { closeSync(fd); }
159
173
  }
160
- const effectiveOwner = this.recordOwner();
161
- const getState = this.db.prepare("SELECT indexed_count FROM index_state WHERE stream = ?");
162
- const setState = this.db.prepare(
163
- "INSERT INTO index_state (stream, indexed_count) VALUES (?, ?) ON CONFLICT(stream) DO UPDATE SET indexed_count = excluded.indexed_count",
164
- );
165
- for (const streamId of this.store.listStreams()) {
166
- const done = Number(getState.get(streamId)?.indexed_count ?? 0);
167
- let count = 0;
168
- for (const turn of this.store.iterateStream(streamId)) {
169
- if (count >= done) this.addTurn(turn, streamId, effectiveOwner);
170
- count++;
171
- }
172
- setState.run(streamId, count);
174
+ if (!Number.isSafeInteger(row.bytes) || row.bytes < 0 || !boundary || row.indexed_count !== row.byte_count ||
175
+ row.epoch !== checkpoint.epoch || row.bytes > checkpoint.bytes || row.byte_count > checkpoint.count) {
176
+ reset = true;
177
+ break;
173
178
  }
174
- this.db.exec("COMMIT");
175
- } catch (err) {
176
- this.db.exec("ROLLBACK");
177
- throw err;
179
+ }
180
+ // Deleting a stream or changing its history can change tombstone authority
181
+ // and cross-stream dedupe. Rebuild the derived union, not only its FTS rows.
182
+ if (reset) this.reset();
183
+ const owner = this.store.owner;
184
+ if (owner) this.db.prepare("INSERT INTO meta (key, value) VALUES ('owner', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(owner);
185
+ const effectiveOwner = this.recordOwner();
186
+ const getState = this.db.prepare("SELECT * FROM index_bytes WHERE stream = ?");
187
+ const setCount = this.db.prepare("INSERT INTO index_state VALUES (?, ?) ON CONFLICT(stream) DO UPDATE SET indexed_count=excluded.indexed_count");
188
+ const setBytes = this.db.prepare("INSERT INTO index_bytes VALUES (?, ?, ?, ?) ON CONFLICT(stream) DO UPDATE SET bytes=excluded.bytes, indexed_count=excluded.indexed_count, epoch=excluded.epoch");
189
+ for (const streamId of this.store.iterateStreams()) {
190
+ this.store.withStreamLock(streamId, () => this.store.withDerived(state => {
191
+ const checkpoint = this.store.validateJournal(streamId, state);
192
+ const row = getState.get(streamId);
193
+ const previous = row?.stream != null ? row : null;
194
+ let position = previous?.bytes ?? 0, count = previous?.indexed_count ?? 0;
195
+ if (previous && previous.epoch !== checkpoint.epoch) throw new Error("journal changed during index update; retry to rebuild derived state");
196
+ if (previous && position === checkpoint.bytes) return;
197
+ const iterator = this.store.iterateJournal(streamId, { start: position, end: checkpoint.bytes });
198
+ let done = false;
199
+ try {
200
+ while (!done) {
201
+ this.db.exec("BEGIN IMMEDIATE");
202
+ try {
203
+ let batch = 0, bytes = 0;
204
+ while (batch < 2000 && bytes < 8 * 1024 * 1024) {
205
+ const next = iterator.next();
206
+ if (next.done) { done = true; break; }
207
+ const entry = next.value;
208
+ if (entry.turn !== null) { this.addTurn(entry.turn, streamId, effectiveOwner); count++; }
209
+ position = entry.end; bytes += entry.bytes; batch++;
210
+ }
211
+ setCount.run(streamId, count);
212
+ setBytes.run(streamId, position, count, checkpoint.epoch);
213
+ this.db.exec("COMMIT");
214
+ } catch (err) { this.db.exec("ROLLBACK"); throw err; }
215
+ }
216
+ } finally { iterator.return(); }
217
+ }));
178
218
  }
179
219
  }
180
220
 
181
221
  // Full rebuild = reset + update from zero. Same code path as update().
182
- rebuild() {
222
+ reset() {
183
223
  this.db.exec("BEGIN");
184
224
  try {
185
225
  this.db.exec("DELETE FROM turns");
@@ -190,14 +230,16 @@ export class RecordIndex {
190
230
  this.db.exec("DELETE FROM segments");
191
231
  this.db.exec("DELETE FROM spawns");
192
232
  this.db.exec("DELETE FROM index_state");
233
+ this.db.exec("DELETE FROM index_bytes");
193
234
  this.db.exec("COMMIT");
194
235
  } catch (err) {
195
236
  this.db.exec("ROLLBACK");
196
237
  throw err;
197
238
  }
198
- this.update();
199
239
  }
200
240
 
241
+ rebuild() { this.reset(); this.update(); }
242
+
201
243
  // Insert one turn, maintaining the hidden invariant in both arrival
202
244
  // orders (tombstone before or after target). Duplicate ids across
203
245
  // streams: first seen wins.
@@ -216,10 +258,8 @@ export class RecordIndex {
216
258
 
217
259
  const fromName = String(turn.from ?? "");
218
260
  // Hidden if a valid tombstone already arrived for this id.
219
- const tombs = this.db.prepare("SELECT from_name FROM tombstones WHERE ref = ?").all(turn.id);
220
- const hidden = tombs.some((t) => t.from_name === fromName || (owner && t.from_name === owner))
221
- ? 1
222
- : 0;
261
+ const hidden = this.db.prepare("SELECT 1 AS found FROM tombstones WHERE ref = ? AND (from_name = ? OR from_name = ?) LIMIT 1")
262
+ .get(turn.id, fromName, owner ?? fromName)?.found === 1 ? 1 : 0;
223
263
 
224
264
  // Sessions are one turn per native event (no snapshots, nothing to
225
265
  // supersede); events/superseded columns remain for schema stability.
@@ -0,0 +1,62 @@
1
+ // Explicit capture receipts prove visible boundaries without retaining journal
2
+ // contents or the IDs of the record in memory. Claims use the same first-seen
3
+ // non-session union and tombstone authority as RecordStore.tombstoneClaims().
4
+ import { writeFileSync } from "node:fs";
5
+ import { createCaptureScratch } from "./capture-scratch.mjs";
6
+ import { join } from "node:path";
7
+ import { createRequire } from "node:module";
8
+ const require = createRequire(import.meta.url);
9
+ import { keysetRows } from "./sqlite-rows.mjs";
10
+ import { RecordStore } from "./store.mjs";
11
+
12
+ export function withReceiptSummaries(store, fn) {
13
+ const { DatabaseSync } = require("node:sqlite");
14
+ const scratch = createCaptureScratch(store.root, "receipt"), dir = scratch.path;
15
+ let db;
16
+ try {
17
+ writeFileSync(join(dir, "claims.db"), "", { flag: "wx", mode: 0o600 });
18
+ db = new DatabaseSync(join(dir, "claims.db"));
19
+ db.exec(`PRAGMA cache_size = -2048; PRAGMA temp_store = FILE;
20
+ CREATE TABLE streams (stream TEXT PRIMARY KEY);
21
+ CREATE TABLE seen (id TEXT PRIMARY KEY);
22
+ CREATE TABLE claims (ref TEXT, author TEXT, PRIMARY KEY(ref, author));`);
23
+ const seen = db.prepare("INSERT OR IGNORE INTO seen VALUES (?)");
24
+ const claim = db.prepare("INSERT OR IGNORE INTO claims VALUES (?, ?)");
25
+ const authorKey = (value) => JSON.stringify(value) ?? "undefined";
26
+ let batch = 0;
27
+ db.exec("BEGIN");
28
+ const addStream = db.prepare("INSERT INTO streams VALUES (?)");
29
+ for (const stream of store.iterateStreams()) {
30
+ if (!RecordStore.isSessionStream(stream)) addStream.run(stream);
31
+ if (++batch >= 2000) { db.exec("COMMIT; BEGIN"); batch = 0; }
32
+ }
33
+ for (const { stream } of keysetRows(db, "SELECT stream FROM streams", "stream")) {
34
+ if (RecordStore.isSessionStream(stream)) continue;
35
+ for (const turn of store.iterateStream(stream)) {
36
+ if (typeof turn.id !== "string" || !seen.run(turn.id).changes) continue;
37
+ if (turn.kind === "tombstone") {
38
+ for (const link of turn.links ?? []) {
39
+ if (link.rel === "tombstones" && typeof link.ref === "string") claim.run(link.ref, authorKey(turn.from));
40
+ }
41
+ }
42
+ if (++batch >= 2000) { db.exec("COMMIT; BEGIN"); batch = 0; }
43
+ }
44
+ }
45
+ db.exec("COMMIT");
46
+ const hidden = db.prepare("SELECT 1 AS found FROM claims WHERE ref = ? AND (author = ? OR author = ?) LIMIT 1");
47
+ const summary = (stream) => {
48
+ let count = 0, first = null, last = null;
49
+ for (const turn of store.iterateStream(stream)) {
50
+ if (typeof turn.id === "string" && hidden.get(turn.id, authorKey(turn.from), authorKey(store.owner ?? turn.from))?.found === 1) continue;
51
+ count++;
52
+ first ??= turn;
53
+ last = turn;
54
+ }
55
+ return { count, first, last };
56
+ };
57
+ return fn(summary);
58
+ } finally {
59
+ try { db?.close(); }
60
+ finally { scratch.close(); }
61
+ }
62
+ }