@awebai/oats 0.54.0 → 0.55.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/oats.mjs +5 -5
- package/docs/desktop-cli-api.md +15 -4
- package/docs/integrations.md +2 -2
- package/docs/knowledge.md +14 -2
- package/docs/official-catalog.md +4 -4
- package/docs/packages.md +10 -10
- package/docs/release-notes/v0.55.0.md +158 -0
- package/docs/release-notes/v0.55.1.md +45 -0
- package/docs/servers.md +13 -3
- package/docs/workspaces.md +1 -1
- package/lib/servers.mjs +82 -34
- package/package-catalog.json +3 -3
- package/package.json +1 -1
- package/packages/client/cli-locator.mjs +2 -2
- package/packages/client/workspace-views.mjs +4 -3
- package/packages/record/README.md +58 -13
- package/packages/record/bin/capture.mjs +107 -53
- package/packages/record/bin/recall.mjs +3 -2
- package/packages/record/docs/capture-service.md +193 -0
- package/packages/record/docs/capture-status.schema.json +100 -0
- package/packages/record/lib/capture-aw.mjs +96 -94
- package/packages/record/lib/capture-cc.mjs +109 -187
- package/packages/record/lib/capture-db-reader.mjs +39 -0
- package/packages/record/lib/capture-issues.mjs +64 -0
- package/packages/record/lib/capture-scratch.mjs +111 -0
- package/packages/record/lib/capture-state.mjs +105 -0
- package/packages/record/lib/derived-state.mjs +60 -0
- package/packages/record/lib/formats.mjs +31 -31
- package/packages/record/lib/index-db.mjs +74 -34
- package/packages/record/lib/receipt-summary.mjs +62 -0
- package/packages/record/lib/session-snapshot.mjs +56 -1
- package/packages/record/lib/sqlite-rows.mjs +14 -0
- package/packages/record/lib/store.mjs +122 -43
- package/packages/record/lib/transcript-prefix.mjs +40 -0
- package/skills/oats-getting-started/SKILL.md +1 -1
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
// Private same-volume scratch for synchronous capture/receipt work. Allocations
|
|
2
|
+
// are never transferred to subprocesses. Root-lock cleanup reclaims only proven
|
|
3
|
+
// dead local owners; an incomplete marker or reused PID remains conservative.
|
|
4
|
+
import { closeSync, constants, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, opendirSync, readSync, rmSync, rmdirSync, statSync, writeFileSync } from "node:fs";
|
|
5
|
+
import { randomBytes } from "node:crypto";
|
|
6
|
+
import { hostname } from "node:os";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
|
|
9
|
+
const sameObject = (a, b) => a.dev === b.dev && a.ino === b.ino;
|
|
10
|
+
const sameFile = (a, b) => sameObject(a, b) && a.size === b.size && a.mtimeNs === b.mtimeNs && a.ctimeNs === b.ctimeNs;
|
|
11
|
+
function directory(path) {
|
|
12
|
+
const stat = lstatSync(path, { bigint: true });
|
|
13
|
+
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error(`capture scratch is not a real directory: ${path}`);
|
|
14
|
+
return stat;
|
|
15
|
+
}
|
|
16
|
+
function privateRoot(root, create) {
|
|
17
|
+
const path = join(root, ".capture-tmp");
|
|
18
|
+
if (create) { mkdirSync(root, { recursive: true }); try { mkdirSync(path, { mode: 0o700 }); } catch (e) { if (e.code !== "EEXIST") throw e; } }
|
|
19
|
+
const stat = directory(path);
|
|
20
|
+
if (stat.dev !== statSync(root, { bigint: true }).dev) throw new Error(`capture scratch must be on the record root volume: ${path}`);
|
|
21
|
+
if ((stat.mode & 0o077n) !== 0n || (process.getuid && stat.uid !== BigInt(process.getuid()))) throw new Error(`capture scratch directory must be private and owned by this user: ${path}`);
|
|
22
|
+
return { path, stat };
|
|
23
|
+
}
|
|
24
|
+
// Bound the read, refuse links/special files before opening, and bind bytes to
|
|
25
|
+
// the descriptor and unchanged named file. Missing/unreadable means unknown.
|
|
26
|
+
function marker(dir) {
|
|
27
|
+
const path = join(dir, "owner.json"); let fd;
|
|
28
|
+
try {
|
|
29
|
+
const before = lstatSync(path, { bigint: true });
|
|
30
|
+
if (!before.isFile() || before.isSymbolicLink() || before.size > 4096n) return null;
|
|
31
|
+
fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
32
|
+
const stat = fstatSync(fd, { bigint: true });
|
|
33
|
+
if (!stat.isFile() || !sameFile(before, stat)) return null;
|
|
34
|
+
const bytes = Buffer.alloc(Number(stat.size)); let offset = 0;
|
|
35
|
+
while (offset < bytes.length) { const n = readSync(fd, bytes, offset, bytes.length - offset, offset); if (!n) return null; offset += n; }
|
|
36
|
+
if (!sameFile(stat, fstatSync(fd, { bigint: true })) || !sameFile(stat, lstatSync(path, { bigint: true }))) return null;
|
|
37
|
+
const raw = bytes.toString("utf8"); return { stat, raw, value: JSON.parse(raw) };
|
|
38
|
+
} catch { return null; }
|
|
39
|
+
finally { if (fd !== undefined) closeSync(fd); }
|
|
40
|
+
}
|
|
41
|
+
function sameMarker(a, b) { return a && b && sameFile(a.stat, b.stat) && a.raw === b.raw; }
|
|
42
|
+
function unchanged(dir, stat, owner) {
|
|
43
|
+
try { return sameObject(stat, directory(dir)) && sameMarker(owner, marker(dir)); }
|
|
44
|
+
catch { return false; }
|
|
45
|
+
}
|
|
46
|
+
function localLiveness(pid) {
|
|
47
|
+
try { process.kill(pid, 0); return "alive"; }
|
|
48
|
+
catch (err) { return err.code === "ESRCH" ? "dead" : "unknown"; }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function createCaptureScratch(root, kind) {
|
|
52
|
+
if (!root || !["snapshot", "receipt"].includes(kind)) throw new Error("capture scratch requires a record root and recognized purpose");
|
|
53
|
+
const base = privateRoot(root, true);
|
|
54
|
+
const path = mkdtempSync(join(base.path, `${kind}-`)), stat = directory(path);
|
|
55
|
+
const owner = { v: 1, kind, pid: process.pid, host: hostname(), nonce: randomBytes(16).toString("hex") };
|
|
56
|
+
try {
|
|
57
|
+
writeFileSync(join(path, "owner.json"), JSON.stringify(owner), { flag: "wx", mode: 0o600 });
|
|
58
|
+
const published = marker(path);
|
|
59
|
+
if (!published || !sameObject(base.stat, directory(base.path))) throw new Error("capture scratch ownership publication failed");
|
|
60
|
+
return { path, close() {
|
|
61
|
+
// Ordinary callers remove only their own unchanged allocation. An
|
|
62
|
+
// external replacement is preserved, even if its pathname is familiar.
|
|
63
|
+
if (sameObject(base.stat, directory(base.path)) && unchanged(path, stat, published)) rmSync(path, { recursive: true });
|
|
64
|
+
} };
|
|
65
|
+
} catch (err) {
|
|
66
|
+
// No payload has been opened yet. A partial/unreadable owner marker is
|
|
67
|
+
// deliberately preserved; only an empty or positively owned allocation
|
|
68
|
+
// can be removed. An empty-dir removal cannot erase substituted contents.
|
|
69
|
+
try {
|
|
70
|
+
if (sameObject(base.stat, directory(base.path)) && sameObject(stat, directory(path))) {
|
|
71
|
+
const current = marker(path);
|
|
72
|
+
if (current?.value?.nonce === owner.nonce) rmSync(path, { recursive: true });
|
|
73
|
+
else rmdirSync(path);
|
|
74
|
+
}
|
|
75
|
+
} catch { /* uncertain ownership remains for conservative operator recovery */ }
|
|
76
|
+
throw err;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function cleanupCaptureScratch(root, lock, { liveness = localLiveness } = {}) {
|
|
81
|
+
const lockPath = join(root, ".capture.lock"), lockStat = directory(lockPath), owner = marker(lockPath);
|
|
82
|
+
if (lock?.path !== lockPath || lock.held || typeof lock.release !== "function" ||
|
|
83
|
+
owner?.value?.pid !== process.pid || owner.value.host !== hostname() || typeof owner.value.nonce !== "string") {
|
|
84
|
+
throw new Error("capture scratch cleanup requires the current process's capture lock");
|
|
85
|
+
}
|
|
86
|
+
let base;
|
|
87
|
+
try { base = privateRoot(root, false); } catch (err) { if (err.code === "ENOENT") return 0; throw err; }
|
|
88
|
+
const entries = opendirSync(base.path); let removed = 0;
|
|
89
|
+
try {
|
|
90
|
+
for (let entry; (entry = entries.readSync()) !== null;) {
|
|
91
|
+
const match = /^(snapshot|receipt)-[A-Za-z0-9]{6}$/.exec(entry.name);
|
|
92
|
+
if (!match || !entry.isDirectory() || entry.isSymbolicLink()) continue;
|
|
93
|
+
const path = join(base.path, entry.name); let stat;
|
|
94
|
+
try { stat = directory(path); } catch { continue; }
|
|
95
|
+
const candidate = marker(path), value = candidate?.value;
|
|
96
|
+
if (value?.v !== 1 || value.kind !== match[1] || value.host !== hostname() ||
|
|
97
|
+
!Number.isInteger(value.pid) || value.pid <= 0 || value.pid > 2147483647 ||
|
|
98
|
+
typeof value.nonce !== "string" || !/^[a-f0-9]{32}$/.test(value.nonce)) continue;
|
|
99
|
+
let live;
|
|
100
|
+
try { live = liveness(value.pid); } catch { continue; }
|
|
101
|
+
if (live !== "dead") continue;
|
|
102
|
+
if (!unchanged(lockPath, lockStat, owner)) throw new Error("capture lock changed during scratch cleanup");
|
|
103
|
+
if (!sameObject(base.stat, directory(base.path))) throw new Error("capture scratch root changed during cleanup");
|
|
104
|
+
if (!unchanged(path, stat, candidate)) continue;
|
|
105
|
+
// No supported user of this helper transfers scratch/FD ownership to a
|
|
106
|
+
// subprocess. Dead allocator + unchanged identity is therefore required.
|
|
107
|
+
rmSync(path, { recursive: true }); removed++;
|
|
108
|
+
}
|
|
109
|
+
} finally { entries.closeSync(); }
|
|
110
|
+
return removed;
|
|
111
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
// Shared capture status and operator recovery. Service supervision later fills
|
|
2
|
+
// policy/run evidence; null means no such evidence has been supplied.
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
4
|
+
import { closeSync, constants, fstatSync, fsyncSync, openSync, readSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { DerivedState, invalidateLegacyMaps } from "./derived-state.mjs";
|
|
7
|
+
import { readCurrentIssueBacklog, readIssueBacklog } from "./capture-issues.mjs";
|
|
8
|
+
|
|
9
|
+
export const CAPTURE_SOURCES = ["aw", "cc", "pi", "codex"];
|
|
10
|
+
export const SET_ASIDE_REMEDY = "Restore the captured prefix and run capture --retry-set-aside; or add the source to <root>/ignore (later turns from that session will not be captured). Retry revalidates; it never authorizes a changed prefix.";
|
|
11
|
+
|
|
12
|
+
// Caller owns the root capture lock. One transaction persists every selected
|
|
13
|
+
// generation; a missing source cursor cannot erase this rescan request.
|
|
14
|
+
export function resetSourceCursors(root, owner, source) {
|
|
15
|
+
if (source !== undefined && !CAPTURE_SOURCES.includes(source)) throw new Error(`unknown capture source: ${source}`);
|
|
16
|
+
invalidateLegacyMaps(root);
|
|
17
|
+
const state = new DerivedState(root);
|
|
18
|
+
try {
|
|
19
|
+
state.transaction(() => {
|
|
20
|
+
for (const name of source ? [source] : CAPTURE_SOURCES) state.set("reset", `${owner}:${name}`, randomUUID());
|
|
21
|
+
});
|
|
22
|
+
} finally { state.close(); }
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function retrySetAside(root) {
|
|
26
|
+
const state = new DerivedState(root);
|
|
27
|
+
try {
|
|
28
|
+
state.transaction(() => {
|
|
29
|
+
for (const { key, value } of state.entries("set-aside")) state.set("set-aside", key, { ...value, retry: true });
|
|
30
|
+
});
|
|
31
|
+
} finally { state.close(); }
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function writeCaptureStatus(root, { pending = false } = {}) {
|
|
35
|
+
const state = new DerivedState(root);
|
|
36
|
+
let backlog;
|
|
37
|
+
try {
|
|
38
|
+
backlog = readIssueBacklog(state.db, { pending });
|
|
39
|
+
} finally { state.close(); }
|
|
40
|
+
const status = { schemaVersion: 1, policy: null, lastRun: null, lastSuccess: null, lastFailure: null, lastProgressAt: null, backlog };
|
|
41
|
+
const path = join(root, "capture-status.json"), temp = `${path}.${randomUUID()}.tmp`;
|
|
42
|
+
let fd;
|
|
43
|
+
try {
|
|
44
|
+
fd = openSync(temp, "wx", 0o600);
|
|
45
|
+
writeFileSync(fd, JSON.stringify(status) + "\n"); fsyncSync(fd); closeSync(fd); fd = undefined;
|
|
46
|
+
renameSync(temp, path);
|
|
47
|
+
} finally {
|
|
48
|
+
if (fd !== undefined) closeSync(fd);
|
|
49
|
+
try { unlinkSync(temp); } catch (err) { if (err.code !== "ENOENT") throw err; }
|
|
50
|
+
}
|
|
51
|
+
return status;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function validStatus(status) {
|
|
55
|
+
const b = status?.backlog;
|
|
56
|
+
return status?.schemaVersion === 1 && status.policy === null && status.lastRun === null && status.lastSuccess === null &&
|
|
57
|
+
status.lastFailure === null && status.lastProgressAt === null && typeof b?.pending === "boolean" &&
|
|
58
|
+
Number.isSafeInteger(b.setAsideCount) && b.setAsideCount >= 0 && Array.isArray(b.setAside) && b.setAside.length <= 100 &&
|
|
59
|
+
b.setAside.length === Math.min(b.setAsideCount, 100) && b.truncated === (b.setAsideCount > 100) &&
|
|
60
|
+
b.setAside.every(e => CAPTURE_SOURCES.includes(e.source) && typeof e.path === "string" && typeof e.reason === "string" &&
|
|
61
|
+
Number.isSafeInteger(e.offset) && e.offset >= 0 && (e.line === undefined || Number.isSafeInteger(e.line) && e.line > 0));
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Read-only, including a never-installed/absent root: no writable DB construction,
|
|
65
|
+
// directory creation, manager query, index rebuild or capture.
|
|
66
|
+
export function captureHealth(root) {
|
|
67
|
+
const path = join(root, "capture-status.json");
|
|
68
|
+
let status = { lastRun: null, lastSuccess: null, lastFailure: null,
|
|
69
|
+
backlog: { pending: false, setAsideCount: 0, setAside: [], truncated: false } };
|
|
70
|
+
const rows = [];
|
|
71
|
+
try {
|
|
72
|
+
const fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK);
|
|
73
|
+
let text;
|
|
74
|
+
try {
|
|
75
|
+
const stat = fstatSync(fd);
|
|
76
|
+
if (!stat.isFile()) throw new Error("status is not a regular file");
|
|
77
|
+
const limit = 1024 * 1024;
|
|
78
|
+
if (stat.size > limit) throw new Error("status exceeds 1 MiB");
|
|
79
|
+
const bytes = Buffer.alloc(limit + 1);
|
|
80
|
+
let size = 0;
|
|
81
|
+
while (size < bytes.length) {
|
|
82
|
+
const count = readSync(fd, bytes, size, bytes.length - size, size);
|
|
83
|
+
if (!count) break;
|
|
84
|
+
size += count;
|
|
85
|
+
}
|
|
86
|
+
if (size > limit) throw new Error("status exceeds 1 MiB");
|
|
87
|
+
text = bytes.toString("utf8", 0, size);
|
|
88
|
+
} finally { closeSync(fd); }
|
|
89
|
+
const parsed = JSON.parse(text);
|
|
90
|
+
if (!validStatus(parsed)) throw new Error("unsupported or malformed status");
|
|
91
|
+
status = parsed;
|
|
92
|
+
} catch (err) {
|
|
93
|
+
if (err.code !== "ENOENT") rows.push({ code: "capture-status-invalid", message: `Cannot read capture status at ${path}: ${err.message}` });
|
|
94
|
+
}
|
|
95
|
+
let backlog;
|
|
96
|
+
try {
|
|
97
|
+
backlog = readCurrentIssueBacklog(root);
|
|
98
|
+
if (!backlog && status.backlog.setAsideCount) throw new Error("capture.db is missing but the last command reported active set-aside inputs");
|
|
99
|
+
} catch (err) { throw new Error(`Cannot read current capture issues at ${join(root, "index", "capture.db")}: ${err.message}`, { cause: err }); }
|
|
100
|
+
backlog ??= { pending: false, setAsideCount: 0, setAside: [], truncated: false };
|
|
101
|
+
if (backlog.setAsideCount) rows.push({ code: "capture-input-set-aside", message:
|
|
102
|
+
`${backlog.setAsideCount} capture inputs set aside. ${SET_ASIDE_REMEDY.replace("<root>", root)}` });
|
|
103
|
+
return { schemaVersion: 1, status: rows.length ? "warning" : "ok", rows,
|
|
104
|
+
lastRun: status.lastRun, lastSuccess: status.lastSuccess, lastFailure: status.lastFailure, backlog };
|
|
105
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// Local accelerators only. Journal bytes remain authority; removing this
|
|
2
|
+
// database costs bounded rebuild work, never changes a turn or its ID.
|
|
3
|
+
import { mkdirSync, statSync, unlinkSync } from "node:fs";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { keysetRows } from "./sqlite-rows.mjs";
|
|
6
|
+
import { createRequire } from "node:module";
|
|
7
|
+
const require = createRequire(import.meta.url);
|
|
8
|
+
|
|
9
|
+
export function invalidateLegacyMaps(root) {
|
|
10
|
+
for (const name of ["capture-offsets.json", "capture-aw-seen.json"]) {
|
|
11
|
+
try { unlinkSync(join(root, "index", name)); }
|
|
12
|
+
catch (err) { if (err.code !== "ENOENT") throw err; }
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
export function statVersion(st) {
|
|
16
|
+
return { dev: String(st.dev), ino: String(st.ino), size: Number(st.size), mtime: String(st.mtimeNs ?? st.mtimeMs), ctime: String(st.ctimeNs ?? st.ctimeMs) };
|
|
17
|
+
}
|
|
18
|
+
export function fileVersion(path) {
|
|
19
|
+
try {
|
|
20
|
+
const st = statSync(path, { bigint: true });
|
|
21
|
+
if (!st.isFile()) throw new Error(`not a regular file: ${path}`);
|
|
22
|
+
return statVersion(st);
|
|
23
|
+
} catch (err) { if (err.code === "ENOENT") return null; throw err; }
|
|
24
|
+
}
|
|
25
|
+
export const sameVersion = (a, b) => JSON.stringify(a) === JSON.stringify(b);
|
|
26
|
+
|
|
27
|
+
export class DerivedState {
|
|
28
|
+
constructor(root) {
|
|
29
|
+
const { DatabaseSync } = require("node:sqlite");
|
|
30
|
+
mkdirSync(join(root, "index"), { recursive: true });
|
|
31
|
+
this.db = new DatabaseSync(join(root, "index", "capture.db"));
|
|
32
|
+
this.db.exec("PRAGMA busy_timeout = 60000; PRAGMA cache_size = -2048; PRAGMA temp_store = FILE");
|
|
33
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
34
|
+
try {
|
|
35
|
+
const hadIds = this.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name='journal_ids'").get()?.found === 1;
|
|
36
|
+
this.db.exec(`CREATE TABLE IF NOT EXISTS state (kind TEXT, key TEXT, value TEXT NOT NULL, PRIMARY KEY(kind, key));
|
|
37
|
+
CREATE TABLE IF NOT EXISTS journal_ids (stream TEXT, id TEXT, PRIMARY KEY(stream, id));`);
|
|
38
|
+
// Recreating a lost lookup atomically revokes the old validation witness.
|
|
39
|
+
// A crash cannot qualify an empty replacement table with that witness.
|
|
40
|
+
if (!hadIds) this.db.exec("DELETE FROM state WHERE kind = 'journal'");
|
|
41
|
+
this.db.exec("COMMIT");
|
|
42
|
+
} catch (err) { this.db.exec("ROLLBACK"); this.db.close(); throw err; }
|
|
43
|
+
this.read = this.db.prepare("SELECT value FROM state WHERE kind = ? AND key = ?");
|
|
44
|
+
this.write = this.db.prepare("INSERT INTO state VALUES (?, ?, ?) ON CONFLICT(kind,key) DO UPDATE SET value=excluded.value");
|
|
45
|
+
}
|
|
46
|
+
get(kind, key) { const row = this.read.get(kind, key); return row ? JSON.parse(row.value) : null; }
|
|
47
|
+
set(kind, key, value) { this.write.run(kind, key, JSON.stringify(value)); }
|
|
48
|
+
delete(kind, key) { this.db.prepare("DELETE FROM state WHERE kind = ? AND key = ?").run(kind, key); }
|
|
49
|
+
*entries(kind) {
|
|
50
|
+
for (const row of keysetRows(this.db, "SELECT key, value FROM state WHERE kind = ?", "key", [kind])) {
|
|
51
|
+
yield { key: row.key, value: JSON.parse(row.value) };
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
transaction(fn) {
|
|
55
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
56
|
+
try { const result = fn(); this.db.exec("COMMIT"); return result; }
|
|
57
|
+
catch (err) { this.db.exec("ROLLBACK"); throw err; }
|
|
58
|
+
}
|
|
59
|
+
close() { this.db.close(); }
|
|
60
|
+
}
|
|
@@ -55,38 +55,37 @@ function* parsedLines(bytes) {
|
|
|
55
55
|
}
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
-
|
|
58
|
+
// Preserve the public inventory's per-directory localeCompare order and stable
|
|
59
|
+
// ties for ordinary capture too. Path metadata grows with directory size;
|
|
60
|
+
// source contents are streamed independently of this discovery inventory.
|
|
61
|
+
function* directoryEntries(path, optional = false) {
|
|
62
|
+
let entries;
|
|
63
|
+
try { entries = readdirSync(path, { withFileTypes: true }); }
|
|
64
|
+
catch (err) { if (optional && err.code === "ENOENT") return; throw err; }
|
|
65
|
+
yield* entries.sort((a, b) => a.name.localeCompare(b.name));
|
|
66
|
+
}
|
|
67
|
+
function* iterateJsonlFiles(root, maxDepth, { strict = false } = {}, allowCaptured = false) {
|
|
59
68
|
const capturedPi = typeof root === "object" && root !== null ? root.capturedPi : undefined;
|
|
60
69
|
if (typeof root !== "string") {
|
|
61
70
|
if (!allowCaptured || !capturedPi || Object.keys(root).some(k => !["path", "capturedPi"].includes(k))) throw new Error("unsupported protected session inventory");
|
|
62
71
|
root = root.path;
|
|
63
72
|
}
|
|
64
|
-
|
|
65
|
-
const out = [];
|
|
66
|
-
const walk = (dir, depth) => {
|
|
73
|
+
function* walk(dir, depth) {
|
|
67
74
|
guardCapturedPath(dir, capturedPi);
|
|
68
|
-
|
|
69
|
-
try {
|
|
70
|
-
names = readdirSync(dir, { withFileTypes: true });
|
|
71
|
-
} catch (err) {
|
|
72
|
-
// An optional, absent root is normal; an unreadable directory or a
|
|
73
|
-
// subtree that disappeared during discovery is not an empty scan.
|
|
74
|
-
if (!strict && depth === 0 && err.code === "ENOENT") return;
|
|
75
|
-
throw err;
|
|
76
|
-
}
|
|
77
|
-
for (const entry of names.sort((a, b) => a.name.localeCompare(b.name))) {
|
|
75
|
+
for (const entry of directoryEntries(dir, !strict && depth === 0)) {
|
|
78
76
|
const path = join(dir, entry.name);
|
|
79
|
-
guardCapturedPath(path, capturedPi);
|
|
80
|
-
if (entry.name.endsWith(".jsonl") && !entry.isDirectory()) {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
if (depth < maxDepth) walk(path, depth + 1);
|
|
77
|
+
guardCapturedPath(path, capturedPi);
|
|
78
|
+
if (entry.name.endsWith(".jsonl") && !entry.isDirectory()) yield capturedPi ? { path, capturedPi } : path;
|
|
79
|
+
else if (entry.isDirectory() || (entry.isSymbolicLink() && statSync(path).isDirectory())) {
|
|
80
|
+
if (depth < maxDepth) yield* walk(path, depth + 1);
|
|
84
81
|
}
|
|
85
82
|
}
|
|
86
|
-
}
|
|
87
|
-
walk(root, 0);
|
|
83
|
+
}
|
|
84
|
+
yield* walk(root, 0);
|
|
88
85
|
guardCapturedPath(root, capturedPi);
|
|
89
|
-
|
|
86
|
+
}
|
|
87
|
+
function listJsonlFiles(root, depth, options, allowCaptured) {
|
|
88
|
+
return [...iterateJsonlFiles(root, depth, options, allowCaptured)];
|
|
90
89
|
}
|
|
91
90
|
|
|
92
91
|
// Only absence makes a default root optional. existsSync also hides access
|
|
@@ -138,24 +137,22 @@ function ccRoots(home = homedir(), env = process.env, options = {}) {
|
|
|
138
137
|
// Native Claude child transcripts live at
|
|
139
138
|
// projects/<project>/<sessionId>/subagents/agent-*.jsonl, not beside the
|
|
140
139
|
// parent's file. Enumerate only this layout, never arbitrary project files.
|
|
141
|
-
function
|
|
142
|
-
const out = [];
|
|
140
|
+
function* iterateCcFiles(root, { strict = false } = {}) {
|
|
143
141
|
const entries = (dir, optional = false) => {
|
|
144
142
|
guardCapturedPath(dir);
|
|
145
|
-
|
|
146
|
-
catch (err) { if (optional && err.code === "ENOENT") return []; throw err; }
|
|
143
|
+
return directoryEntries(dir, optional);
|
|
147
144
|
};
|
|
148
145
|
const directory = (entry, path) => entry.isDirectory() || (entry.isSymbolicLink() && !entry.name.endsWith(".jsonl") && statSync(path).isDirectory());
|
|
149
146
|
for (const project of entries(root, !strict)) {
|
|
150
147
|
const projectPath = join(root, project.name);
|
|
151
148
|
if (!directory(project, projectPath)) {
|
|
152
|
-
if (project.name.endsWith(".jsonl"))
|
|
149
|
+
if (project.name.endsWith(".jsonl")) yield projectPath;
|
|
153
150
|
continue;
|
|
154
151
|
}
|
|
155
152
|
for (const entry of entries(projectPath)) {
|
|
156
153
|
const path = join(projectPath, entry.name);
|
|
157
154
|
if (!directory(entry, path)) {
|
|
158
|
-
if (entry.name.endsWith(".jsonl"))
|
|
155
|
+
if (entry.name.endsWith(".jsonl")) yield path;
|
|
159
156
|
continue;
|
|
160
157
|
}
|
|
161
158
|
// Listing the session directory (rather than treating ENOENT at an
|
|
@@ -165,13 +162,13 @@ function listCcFiles(root, { strict = false } = {}) {
|
|
|
165
162
|
const children = join(path, childDir.name);
|
|
166
163
|
if (!directory(childDir, children)) throw new Error(`session subagents root is not a directory: ${children}`);
|
|
167
164
|
for (const child of entries(children)) {
|
|
168
|
-
if (child.name.endsWith(".jsonl"))
|
|
165
|
+
if (child.name.endsWith(".jsonl")) yield join(children, child.name);
|
|
169
166
|
}
|
|
170
167
|
}
|
|
171
168
|
}
|
|
172
169
|
}
|
|
173
|
-
|
|
174
|
-
}
|
|
170
|
+
}
|
|
171
|
+
function listCcFiles(root, options) { return [...iterateCcFiles(root, options)]; }
|
|
175
172
|
|
|
176
173
|
function ccParentId(path) {
|
|
177
174
|
return basename(dirname(path)) === "subagents" ? basename(dirname(dirname(path))) : null;
|
|
@@ -366,6 +363,7 @@ export const SESSION_FORMATS = {
|
|
|
366
363
|
source: "cc",
|
|
367
364
|
defaultRoots: ccRoots,
|
|
368
365
|
listFiles: (roots, options) => roots.flatMap((r) => listCcFiles(r, options)),
|
|
366
|
+
*iterateFiles(roots) { for (const root of roots) yield* iterateCcFiles(root); },
|
|
369
367
|
sessionId: ccSessionId,
|
|
370
368
|
ignoreKeys: (path) => [basename(path, ".jsonl"), ccParentId(path)].filter(Boolean),
|
|
371
369
|
extractText: extractCcText,
|
|
@@ -374,6 +372,7 @@ export const SESSION_FORMATS = {
|
|
|
374
372
|
source: "pi",
|
|
375
373
|
defaultRoots: piRoots,
|
|
376
374
|
listFiles: (roots, options) => roots.flatMap((r) => listJsonlFiles(r, 1, options, true)),
|
|
375
|
+
*iterateFiles(roots) { for (const root of roots) yield* iterateJsonlFiles(root, 1, {}, true); },
|
|
377
376
|
sessionId: piSessionId,
|
|
378
377
|
extractText: extractPiText,
|
|
379
378
|
},
|
|
@@ -381,6 +380,7 @@ export const SESSION_FORMATS = {
|
|
|
381
380
|
source: "codex",
|
|
382
381
|
defaultRoots: codexRoots,
|
|
383
382
|
listFiles: (roots, options) => roots.flatMap((r) => listJsonlFiles(r, 3, options)),
|
|
383
|
+
*iterateFiles(roots) { for (const root of roots) yield* iterateJsonlFiles(root, 3); },
|
|
384
384
|
sessionId: codexSessionId,
|
|
385
385
|
extractText: extractCodexText,
|
|
386
386
|
},
|
|
@@ -11,9 +11,11 @@
|
|
|
11
11
|
// verbatim record); text is extracted from that line, so a hit names the
|
|
12
12
|
// turn AND the event's line in the original transcript.
|
|
13
13
|
|
|
14
|
-
import { existsSync, mkdirSync, rmSync, unlinkSync } from "node:fs";
|
|
14
|
+
import { closeSync, existsSync, mkdirSync, openSync, readSync, rmSync, unlinkSync } from "node:fs";
|
|
15
15
|
import { join } from "node:path";
|
|
16
|
-
import {
|
|
16
|
+
import { keysetRows } from "./sqlite-rows.mjs";
|
|
17
|
+
import { createRequire } from "node:module";
|
|
18
|
+
const require = createRequire(import.meta.url);
|
|
17
19
|
|
|
18
20
|
// Bump whenever the derived schema changes shape. The index is cache:
|
|
19
21
|
// a version mismatch self-heals by deleting the database and rebuilding
|
|
@@ -50,6 +52,12 @@ CREATE TABLE IF NOT EXISTS index_state (
|
|
|
50
52
|
stream TEXT PRIMARY KEY,
|
|
51
53
|
indexed_count INTEGER NOT NULL
|
|
52
54
|
);
|
|
55
|
+
CREATE TABLE IF NOT EXISTS index_bytes (
|
|
56
|
+
stream TEXT PRIMARY KEY,
|
|
57
|
+
bytes INTEGER NOT NULL,
|
|
58
|
+
indexed_count INTEGER NOT NULL,
|
|
59
|
+
epoch TEXT NOT NULL
|
|
60
|
+
);
|
|
53
61
|
CREATE TABLE IF NOT EXISTS meta (
|
|
54
62
|
key TEXT PRIMARY KEY,
|
|
55
63
|
value TEXT NOT NULL
|
|
@@ -102,6 +110,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS turn_text USING fts5(
|
|
|
102
110
|
|
|
103
111
|
export class RecordIndex {
|
|
104
112
|
constructor(store) {
|
|
113
|
+
const { DatabaseSync } = require("node:sqlite");
|
|
105
114
|
this.store = store;
|
|
106
115
|
const dir = join(store.root, "index");
|
|
107
116
|
mkdirSync(dir, { recursive: true });
|
|
@@ -110,7 +119,7 @@ export class RecordIndex {
|
|
|
110
119
|
const db = new DatabaseSync(path);
|
|
111
120
|
// Concurrent index users (watcher pass, manual reindex, recall)
|
|
112
121
|
// wait for each other instead of dying on "database is locked".
|
|
113
|
-
db.exec("PRAGMA busy_timeout = 60000");
|
|
122
|
+
db.exec("PRAGMA busy_timeout = 60000; PRAGMA cache_size = -2048; PRAGMA temp_store = FILE");
|
|
114
123
|
return db;
|
|
115
124
|
};
|
|
116
125
|
this.db = open();
|
|
@@ -147,39 +156,70 @@ export class RecordIndex {
|
|
|
147
156
|
return row ? row.value : null;
|
|
148
157
|
}
|
|
149
158
|
|
|
150
|
-
//
|
|
159
|
+
// Old releases keep writing index_state's count. A separate cursor table
|
|
160
|
+
// leaves that schema/writer compatible; differing counts invalidate our
|
|
161
|
+
// cursor rather than trusting a byte position left behind by a downgrade.
|
|
151
162
|
update() {
|
|
152
|
-
|
|
153
|
-
this.db.
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
163
|
+
let reset = false;
|
|
164
|
+
for (const row of keysetRows(this.db, `SELECT s.stream, s.indexed_count, b.bytes,
|
|
165
|
+
b.indexed_count AS byte_count, b.epoch FROM index_state s
|
|
166
|
+
LEFT JOIN index_bytes b ON s.stream = b.stream`, "stream")) {
|
|
167
|
+
const checkpoint = this.store.journalCheckpoint(row.stream);
|
|
168
|
+
let boundary = true;
|
|
169
|
+
if (Number.isSafeInteger(row.bytes) && row.bytes > 0 && row.bytes < checkpoint.bytes) {
|
|
170
|
+
const fd = openSync(this.store.journalPath(row.stream), "r"), byte = Buffer.alloc(1);
|
|
171
|
+
try { boundary = readSync(fd, byte, 0, 1, row.bytes - 1) === 1 && byte[0] === 10; }
|
|
172
|
+
finally { closeSync(fd); }
|
|
159
173
|
}
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
);
|
|
165
|
-
for (const streamId of this.store.listStreams()) {
|
|
166
|
-
const done = Number(getState.get(streamId)?.indexed_count ?? 0);
|
|
167
|
-
let count = 0;
|
|
168
|
-
for (const turn of this.store.iterateStream(streamId)) {
|
|
169
|
-
if (count >= done) this.addTurn(turn, streamId, effectiveOwner);
|
|
170
|
-
count++;
|
|
171
|
-
}
|
|
172
|
-
setState.run(streamId, count);
|
|
174
|
+
if (!Number.isSafeInteger(row.bytes) || row.bytes < 0 || !boundary || row.indexed_count !== row.byte_count ||
|
|
175
|
+
row.epoch !== checkpoint.epoch || row.bytes > checkpoint.bytes || row.byte_count > checkpoint.count) {
|
|
176
|
+
reset = true;
|
|
177
|
+
break;
|
|
173
178
|
}
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
179
|
+
}
|
|
180
|
+
// Deleting a stream or changing its history can change tombstone authority
|
|
181
|
+
// and cross-stream dedupe. Rebuild the derived union, not only its FTS rows.
|
|
182
|
+
if (reset) this.reset();
|
|
183
|
+
const owner = this.store.owner;
|
|
184
|
+
if (owner) this.db.prepare("INSERT INTO meta (key, value) VALUES ('owner', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(owner);
|
|
185
|
+
const effectiveOwner = this.recordOwner();
|
|
186
|
+
const getState = this.db.prepare("SELECT * FROM index_bytes WHERE stream = ?");
|
|
187
|
+
const setCount = this.db.prepare("INSERT INTO index_state VALUES (?, ?) ON CONFLICT(stream) DO UPDATE SET indexed_count=excluded.indexed_count");
|
|
188
|
+
const setBytes = this.db.prepare("INSERT INTO index_bytes VALUES (?, ?, ?, ?) ON CONFLICT(stream) DO UPDATE SET bytes=excluded.bytes, indexed_count=excluded.indexed_count, epoch=excluded.epoch");
|
|
189
|
+
for (const streamId of this.store.iterateStreams()) {
|
|
190
|
+
this.store.withStreamLock(streamId, () => this.store.withDerived(state => {
|
|
191
|
+
const checkpoint = this.store.validateJournal(streamId, state);
|
|
192
|
+
const row = getState.get(streamId);
|
|
193
|
+
const previous = row?.stream != null ? row : null;
|
|
194
|
+
let position = previous?.bytes ?? 0, count = previous?.indexed_count ?? 0;
|
|
195
|
+
if (previous && previous.epoch !== checkpoint.epoch) throw new Error("journal changed during index update; retry to rebuild derived state");
|
|
196
|
+
if (previous && position === checkpoint.bytes) return;
|
|
197
|
+
const iterator = this.store.iterateJournal(streamId, { start: position, end: checkpoint.bytes });
|
|
198
|
+
let done = false;
|
|
199
|
+
try {
|
|
200
|
+
while (!done) {
|
|
201
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
202
|
+
try {
|
|
203
|
+
let batch = 0, bytes = 0;
|
|
204
|
+
while (batch < 2000 && bytes < 8 * 1024 * 1024) {
|
|
205
|
+
const next = iterator.next();
|
|
206
|
+
if (next.done) { done = true; break; }
|
|
207
|
+
const entry = next.value;
|
|
208
|
+
if (entry.turn !== null) { this.addTurn(entry.turn, streamId, effectiveOwner); count++; }
|
|
209
|
+
position = entry.end; bytes += entry.bytes; batch++;
|
|
210
|
+
}
|
|
211
|
+
setCount.run(streamId, count);
|
|
212
|
+
setBytes.run(streamId, position, count, checkpoint.epoch);
|
|
213
|
+
this.db.exec("COMMIT");
|
|
214
|
+
} catch (err) { this.db.exec("ROLLBACK"); throw err; }
|
|
215
|
+
}
|
|
216
|
+
} finally { iterator.return(); }
|
|
217
|
+
}));
|
|
178
218
|
}
|
|
179
219
|
}
|
|
180
220
|
|
|
181
221
|
// Full rebuild = reset + update from zero. Same code path as update().
|
|
182
|
-
|
|
222
|
+
reset() {
|
|
183
223
|
this.db.exec("BEGIN");
|
|
184
224
|
try {
|
|
185
225
|
this.db.exec("DELETE FROM turns");
|
|
@@ -190,14 +230,16 @@ export class RecordIndex {
|
|
|
190
230
|
this.db.exec("DELETE FROM segments");
|
|
191
231
|
this.db.exec("DELETE FROM spawns");
|
|
192
232
|
this.db.exec("DELETE FROM index_state");
|
|
233
|
+
this.db.exec("DELETE FROM index_bytes");
|
|
193
234
|
this.db.exec("COMMIT");
|
|
194
235
|
} catch (err) {
|
|
195
236
|
this.db.exec("ROLLBACK");
|
|
196
237
|
throw err;
|
|
197
238
|
}
|
|
198
|
-
this.update();
|
|
199
239
|
}
|
|
200
240
|
|
|
241
|
+
rebuild() { this.reset(); this.update(); }
|
|
242
|
+
|
|
201
243
|
// Insert one turn, maintaining the hidden invariant in both arrival
|
|
202
244
|
// orders (tombstone before or after target). Duplicate ids across
|
|
203
245
|
// streams: first seen wins.
|
|
@@ -216,10 +258,8 @@ export class RecordIndex {
|
|
|
216
258
|
|
|
217
259
|
const fromName = String(turn.from ?? "");
|
|
218
260
|
// Hidden if a valid tombstone already arrived for this id.
|
|
219
|
-
const
|
|
220
|
-
|
|
221
|
-
? 1
|
|
222
|
-
: 0;
|
|
261
|
+
const hidden = this.db.prepare("SELECT 1 AS found FROM tombstones WHERE ref = ? AND (from_name = ? OR from_name = ?) LIMIT 1")
|
|
262
|
+
.get(turn.id, fromName, owner ?? fromName)?.found === 1 ? 1 : 0;
|
|
223
263
|
|
|
224
264
|
// Sessions are one turn per native event (no snapshots, nothing to
|
|
225
265
|
// supersede); events/superseded columns remain for schema stability.
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
// Explicit capture receipts prove visible boundaries without retaining journal
|
|
2
|
+
// contents or the IDs of the record in memory. Claims use the same first-seen
|
|
3
|
+
// non-session union and tombstone authority as RecordStore.tombstoneClaims().
|
|
4
|
+
import { writeFileSync } from "node:fs";
|
|
5
|
+
import { createCaptureScratch } from "./capture-scratch.mjs";
|
|
6
|
+
import { join } from "node:path";
|
|
7
|
+
import { createRequire } from "node:module";
|
|
8
|
+
const require = createRequire(import.meta.url);
|
|
9
|
+
import { keysetRows } from "./sqlite-rows.mjs";
|
|
10
|
+
import { RecordStore } from "./store.mjs";
|
|
11
|
+
|
|
12
|
+
export function withReceiptSummaries(store, fn) {
|
|
13
|
+
const { DatabaseSync } = require("node:sqlite");
|
|
14
|
+
const scratch = createCaptureScratch(store.root, "receipt"), dir = scratch.path;
|
|
15
|
+
let db;
|
|
16
|
+
try {
|
|
17
|
+
writeFileSync(join(dir, "claims.db"), "", { flag: "wx", mode: 0o600 });
|
|
18
|
+
db = new DatabaseSync(join(dir, "claims.db"));
|
|
19
|
+
db.exec(`PRAGMA cache_size = -2048; PRAGMA temp_store = FILE;
|
|
20
|
+
CREATE TABLE streams (stream TEXT PRIMARY KEY);
|
|
21
|
+
CREATE TABLE seen (id TEXT PRIMARY KEY);
|
|
22
|
+
CREATE TABLE claims (ref TEXT, author TEXT, PRIMARY KEY(ref, author));`);
|
|
23
|
+
const seen = db.prepare("INSERT OR IGNORE INTO seen VALUES (?)");
|
|
24
|
+
const claim = db.prepare("INSERT OR IGNORE INTO claims VALUES (?, ?)");
|
|
25
|
+
const authorKey = (value) => JSON.stringify(value) ?? "undefined";
|
|
26
|
+
let batch = 0;
|
|
27
|
+
db.exec("BEGIN");
|
|
28
|
+
const addStream = db.prepare("INSERT INTO streams VALUES (?)");
|
|
29
|
+
for (const stream of store.iterateStreams()) {
|
|
30
|
+
if (!RecordStore.isSessionStream(stream)) addStream.run(stream);
|
|
31
|
+
if (++batch >= 2000) { db.exec("COMMIT; BEGIN"); batch = 0; }
|
|
32
|
+
}
|
|
33
|
+
for (const { stream } of keysetRows(db, "SELECT stream FROM streams", "stream")) {
|
|
34
|
+
if (RecordStore.isSessionStream(stream)) continue;
|
|
35
|
+
for (const turn of store.iterateStream(stream)) {
|
|
36
|
+
if (typeof turn.id !== "string" || !seen.run(turn.id).changes) continue;
|
|
37
|
+
if (turn.kind === "tombstone") {
|
|
38
|
+
for (const link of turn.links ?? []) {
|
|
39
|
+
if (link.rel === "tombstones" && typeof link.ref === "string") claim.run(link.ref, authorKey(turn.from));
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
if (++batch >= 2000) { db.exec("COMMIT; BEGIN"); batch = 0; }
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
db.exec("COMMIT");
|
|
46
|
+
const hidden = db.prepare("SELECT 1 AS found FROM claims WHERE ref = ? AND (author = ? OR author = ?) LIMIT 1");
|
|
47
|
+
const summary = (stream) => {
|
|
48
|
+
let count = 0, first = null, last = null;
|
|
49
|
+
for (const turn of store.iterateStream(stream)) {
|
|
50
|
+
if (typeof turn.id === "string" && hidden.get(turn.id, authorKey(turn.from), authorKey(store.owner ?? turn.from))?.found === 1) continue;
|
|
51
|
+
count++;
|
|
52
|
+
first ??= turn;
|
|
53
|
+
last = turn;
|
|
54
|
+
}
|
|
55
|
+
return { count, first, last };
|
|
56
|
+
};
|
|
57
|
+
return fn(summary);
|
|
58
|
+
} finally {
|
|
59
|
+
try { db?.close(); }
|
|
60
|
+
finally { scratch.close(); }
|
|
61
|
+
}
|
|
62
|
+
}
|