@staix/vault-mem 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/CHANGELOG.md +32 -0
  2. package/README.md +19 -6
  3. package/bin/vmem.js +2 -2
  4. package/package.json +2 -2
  5. package/src/analysis-commands.ts +24 -14
  6. package/src/analysis-input.ts +12 -0
  7. package/src/analysis-types.ts +5 -1
  8. package/src/assert-command.ts +48 -0
  9. package/src/assertion-exposure.ts +367 -0
  10. package/src/assertion-index.ts +86 -0
  11. package/src/assertion-policy.ts +63 -0
  12. package/src/assertion-read.ts +170 -0
  13. package/src/assertions-command.ts +123 -0
  14. package/src/assertions.ts +1013 -0
  15. package/src/cli-output.ts +9 -3
  16. package/src/cli.ts +44 -23
  17. package/src/commands-table.ts +16 -2
  18. package/src/evidence.ts +222 -0
  19. package/src/graph-evidence.ts +169 -0
  20. package/src/index.ts +14 -0
  21. package/src/links.ts +18 -0
  22. package/src/log-commands.ts +7 -7
  23. package/src/mcp-spec.ts +11 -11
  24. package/src/mcp-vault.ts +105 -6
  25. package/src/mcp.ts +4 -0
  26. package/src/org-commands.ts +5 -5
  27. package/src/org-promote.ts +9 -0
  28. package/src/promote-command.ts +6 -6
  29. package/src/proposal-commands.ts +100 -0
  30. package/src/proposal-provider.ts +335 -0
  31. package/src/proposal-storage.ts +334 -0
  32. package/src/proposals.ts +170 -0
  33. package/src/read-commands.ts +44 -12
  34. package/src/read-output.ts +42 -14
  35. package/src/skills-mine.ts +12 -7
  36. package/src/source-age.ts +131 -0
  37. package/src/spec-output.ts +14 -13
  38. package/src/sync.ts +14 -7
  39. package/src/text-output.ts +86 -21
  40. package/src/vault-brief.ts +8 -4
  41. package/src/vault-graph.ts +12 -3
  42. package/src/vault-history.ts +662 -0
  43. package/src/vault-index.ts +2 -0
  44. package/src/vault-lint.ts +15 -4
  45. package/src/write-commands.ts +18 -17
  46. package/src/write-output.ts +6 -6
  47. package/src/write-types.ts +1 -0
  48. package/src/write-validation.ts +3 -1
  49. package/src/writer.ts +257 -11
@@ -0,0 +1,367 @@
1
+ /**
2
+ * Read-only exposure report (issue 259; design `docs/specs/2026-11-19-temporal-assertions-design.md`, Retention and
3
+ * deletion). It answers one question about one vault: does a Git object this repository can reach still hold a purged
4
+ * assertion payload? Git plumbing only, bounded like the journal reader, and never a payload in the receipt.
5
+ *
6
+ * A history rewrite changes every later commit id, needs a force push and invalidates other clones, so vmem reports
7
+ * and writes an expressions file; the rewrite itself stays with the person.
8
+ */
9
+ import { spawnSync } from "node:child_process";
10
+ import { closeSync, constants, existsSync, fsyncSync, openSync, readFileSync, realpathSync, unlinkSync, writeFileSync } from "node:fs";
11
+ import { basename, dirname, join, resolve, sep } from "node:path";
12
+ import { ASSERTIONS_DIR, MAX_BATCH_BYTES, MAX_JOURNAL_BYTES, MAX_SEGMENT_BYTES, evaluateAssertions, payloadHash, readJournal, type AssertionItem, type JournalEntry, type SegmentReport, type VerifiedJournal } from "./assertions.ts";
13
+ import { memoryGitArgv, memoryGitEnv } from "./platform.ts";
14
+ import { loadRegistry } from "./registry.ts";
15
+ import type { VaultSpec } from "./config.ts";
16
+
17
+ /** Commits one scan may visit. The bound names the scan's own work, not a repository limit. */
18
+ export const EXPOSURE_COMMIT_BUDGET = 100_000;
19
+ /** One `assertions/*.jsonl` blob: the journal's segment bound plus one batch of framing, as `readJournal` reads it. */
20
+ export const EXPOSURE_BLOB_BYTES = MAX_SEGMENT_BYTES + MAX_BATCH_BYTES * 4;
21
+ /** One commit: the journal's own 64 MiB total across the assertion blobs that commit holds. */
22
+ export const EXPOSURE_COMMIT_BYTES = MAX_JOURNAL_BYTES;
23
+ /** Bound of a complete `vmem_assertions_exposure` reply as the encoded JSON-RPC frame. */
24
+ export const EXPOSURE_FRAME_BYTES = 48 * 1024;
25
+
26
+ const GIT_TIMEOUT_MS = 20_000;
27
+ /** `rev-list` and `ls-tree` listings: 100000 commit ids, or one line per assertion blob. */
28
+ const LIST_BUFFER_BYTES = 16 * 1024 * 1024;
29
+ /** Issue sentences kept in one receipt; a corrupt repository must not grow the reply without bound. */
30
+ const ISSUE_LIMIT = 15;
31
+ /** Paths inside an `assertions` tree object, which `ls-tree <tree oid>` lists without the directory prefix. */
32
+ const SEGMENT_NAME = /^[0-9a-f]{16}\.jsonl$/;
33
+ const OID = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/;
34
+ const NO_PURGED_LINE = "the current verified segment no longer holds this entry's purged line, so there is no byte-for-byte replacement to write; restore or re-purge the assertion, then rerun exposure";
35
+ const NOT_EXPRESSIBLE = "this line contains a newline, a carriage return, \"==>\" or a leading #, which git-filter-repo --replace-text cannot express literally; open the exposed line by hand in the fresh clone and remove its payload";
36
+ const LOCAL_ONLY = "the scan reads this local repository only (local branches, tags and remote-tracking refs); a remote or another clone this repository cannot read may still hold these payloads";
37
+
38
+ export interface ExposureRequest { vault: string; assertion?: string }
39
+ export interface ExposureCommits { count: number; first: string; last: string }
40
+ export interface ExposureRow {
41
+ assertion: string; segment: string; seq: number; hash: string;
42
+ /** `first` is the oldest and `last` the newest commit still holding the line, in the scan's newest-first order. */
43
+ commits: ExposureCommits;
44
+ refs: string[];
45
+ /** False when `--replace-file` cannot map the line; `manual` then names the step that replaces it. */
46
+ replaceable: boolean;
47
+ manual?: string;
48
+ }
49
+ export interface ReplaceFileReceipt { path: string; lines: number; unreplaceable: number; delete: string }
50
+ export interface ExposureReceipt {
51
+ vault: string;
52
+ scanned: { commits: number; blobs: number; bytes: number };
53
+ exposed: ExposureRow[];
54
+ clean: boolean | null;
55
+ truncated: string | null;
56
+ issues: string[];
57
+ /** Set when the MCP reply bound dropped `exposed[].refs`. */
58
+ refsOmitted?: true;
59
+ /** Count of exposed rows the MCP reply bound dropped. */
60
+ omitted?: number;
61
+ /** Present only when the CLI wrote a replace file. */
62
+ replaceFile?: ReplaceFileReceipt;
63
+ }
64
+ /** One literal `git-filter-repo --replace-text` expression's two sides, taken byte for byte from the two lines. */
65
+ export interface ExposureLine { exposed: string; purged: string }
66
+ export interface ExposureScan { receipt: ExposureReceipt; /** the verified vault root the purged forms are read from */ root: string; lines: ExposureLine[] }
67
+
68
+ const fail: (code: string, message: string) => never = (code, message) => { throw Object.assign(new Error(message), { code }); };
69
+ const text = (value: Buffer) => value.toString("utf8");
70
+ const real = (path: string) => { try { return realpathSync(path); } catch { return path; } };
71
+
72
+ /** One literal line of a `--replace-text` file: a single line, no separator and no comment marker. */
73
+ const expressible = (line: string) => line.length > 0 && !line.includes("\n") && !line.includes("\r") && !line.includes("==>") && !line.startsWith("#");
74
+
75
+ type GitOutcome = { status: number | null; stdout: Buffer; stderr: string };
76
+ /** Hardened Git call: no pager, hooks, fsmonitor, external diff, prompts, lazy fetch or inherited Git directories. */
77
+ function git(root: string, args: string[], env: NodeJS.ProcessEnv, maxBuffer = 1 << 16, stdin?: string): GitOutcome {
78
+ const argv = memoryGitArgv(["-c", "log.follow=false", "-c", "log.showSignature=false", "-c", "gpg.program=", "-C", root, ...args]);
79
+ const safe = memoryGitEnv({ ...env, GIT_OPTIONAL_LOCKS: "0", LC_ALL: "C" }, { repositoryIsolation: true, readOnlyDiagnostic: true });
80
+ safe.GIT_LITERAL_PATHSPECS = "1";
81
+ const result = spawnSync(argv[0]!, argv.slice(1), { env: safe, timeout: GIT_TIMEOUT_MS, maxBuffer, stdio: ["pipe", "pipe", "pipe"], ...(stdin === undefined ? {} : { input: stdin }) });
82
+ if (result.error) {
83
+ const code = (result.error as NodeJS.ErrnoException).code;
84
+ if (code === "ENOENT") return { status: null, stdout: Buffer.alloc(0), stderr: "ENOENT" };
85
+ if (code === "ENOBUFS") fail("V09", "the exposure scan's Git listing exceeded its bounded buffer; the scan is not reported clean. Narrow the repository or rerun once it is repaired");
86
+ fail("V09", `the exposure scan could not run git (${code ?? "spawn failure"}); install git or fix PATH, then retry`);
87
+ }
88
+ const stdout = result.stdout ?? Buffer.alloc(0), stderr = result.stderr === undefined ? "" : String(result.stderr);
89
+ if (result.status === null) fail("V09", `the exposure scan's Git call was terminated (signal ${result.signal ?? "unavailable"}); retry once the repository is responsive`);
90
+ return { status: result.status, stdout, stderr };
91
+ }
92
+
93
+ /** The registered vault, its verified root and the deny list, gated exactly as `vmem assertions` gates a read. */
94
+ function openVault(home: string, request: ExposureRequest): { vault: VaultSpec; root: string; denyPaths: string[] } {
95
+ const config = loadRegistry(home), vault = config.vaults.find(value => value.id === request?.vault);
96
+ if (!vault) fail("V04", "assertions: unknown vault id; list registered vaults with vmem status");
97
+ let root: string;
98
+ try { root = realpathSync(vault.path); }
99
+ catch { fail("V04", "registered vault path is missing; restore it or re-register the vault"); }
100
+ if (root !== resolve(vault.path)) fail("V04", "assertions: registered vault root changed; re-register the vault with vmem vaults add");
101
+ return { vault, root, denyPaths: config.denyPaths ?? [] };
102
+ }
103
+
104
+ interface Target { assertion: string; entry: JournalEntry; exposed: Set<string>; commits: string[]; seen: Set<string> }
105
+ const key = (entry: JournalEntry) => `${entry.seg}\u0000${entry.seq}\u0000${entry.hash}`;
106
+
107
+ /** Every purged entry the current journal holds, each with the assertion it belongs to (`parents` resolve it). */
108
+ function purgedTargets(journal: VerifiedJournal, history: readonly AssertionItem[], id: string | undefined): Target[] {
109
+ const targets: Target[] = [];
110
+ for (const item of history) {
111
+ if (item.purged !== true || item.segment === undefined || item.seq === undefined) continue;
112
+ if (id !== undefined && item.id !== id) continue;
113
+ const report = journal.segments.find(segment => segment.segment === item.segment);
114
+ const entry = report?.entries[item.seq - 1];
115
+ // `evaluateAssertions` reads verified entries only; this guard keeps the position and hash honest anyway.
116
+ if (!entry || entry.hash !== item.hash || entry.purged === undefined) continue;
117
+ targets.push({ assertion: item.id, entry, exposed: new Set(), commits: [], seen: new Set() });
118
+ }
119
+ return targets.sort((a, b) => a.assertion < b.assertion ? -1 : a.assertion > b.assertion ? 1 : a.entry.seg < b.entry.seg ? -1 : a.entry.seg > b.entry.seg ? 1 : a.entry.seq - b.entry.seq);
120
+ }
121
+
122
+ interface TreeBlob { oid: string; size: number }
123
+ /** The `assertions/<16 hex>.jsonl` blobs of one `assertions/` tree from `ls-tree -r -l -z`, with the sizes it lists. */
124
+ function treeBlobs(stdout: Buffer): TreeBlob[] {
125
+ const blobs: TreeBlob[] = [];
126
+ for (const record of text(stdout).split("\0")) {
127
+ if (!record) continue;
128
+ const tab = record.indexOf("\t");
129
+ if (tab < 0) continue;
130
+ // `-l` pads the size column with spaces, so the fields are split on any whitespace.
131
+ const [, type, oid, size] = record.slice(0, tab).split(/\s+/);
132
+ if (type !== "blob" || !oid || !SEGMENT_NAME.test(record.slice(tab + 1))) continue;
133
+ const bytes = Number(size);
134
+ if (!Number.isSafeInteger(bytes) || bytes < 0) continue;
135
+ blobs.push({ oid, size: bytes });
136
+ }
137
+ return blobs;
138
+ }
139
+
140
+ /**
141
+ * Each commit's `assertions/` tree oid, resolved for every commit in one `cat-file --batch-check` call, or null when
142
+ * that call failed. A commit that changes no journal byte resolves to the tree it already had, which lets the deep
143
+ * listing be cached per tree instead of per commit.
144
+ */
145
+ function assertionsTrees(root: string, env: NodeJS.ProcessEnv, commits: readonly string[]): Map<string, string> | null {
146
+ const out = git(root, ["cat-file", "--batch-check"], env, LIST_BUFFER_BYTES, commits.map(commit => `${commit}:${ASSERTIONS_DIR}\n`).join(""));
147
+ if (out.status !== 0) return null;
148
+ const lines = text(out.stdout).split("\n");
149
+ // One answer per query, in order: `<tree oid> tree <size>`, or `<query> missing` when the commit has no assertions/.
150
+ if (lines.length < commits.length) return null;
151
+ const found = new Map<string, string>();
152
+ for (const [index, commit] of commits.entries()) {
153
+ const [oid, type] = (lines[index] ?? "").split(" ");
154
+ if (type === "tree" && oid !== undefined && OID.test(oid)) found.set(commit, oid);
155
+ }
156
+ return found;
157
+ }
158
+
159
+ /** Ref names containing `commit`, resolved once per commit. Annotated tags are peeled by `for-each-ref` itself. */
160
+ function refsContaining(root: string, env: NodeJS.ProcessEnv, commit: string, cache: Map<string, { refs: string[]; ok: boolean }>): { refs: string[]; ok: boolean } {
161
+ const known = cache.get(commit);
162
+ if (known) return known;
163
+ const out = git(root, ["for-each-ref", `--contains=${commit}`, "--format=%(refname)", "refs/heads", "refs/tags", "refs/remotes"], env, 1 << 18);
164
+ const refs = out.status === 0 ? text(out.stdout).split("\n").map(line => line.trim()).filter(line => line.length > 0 && !line.endsWith("/HEAD")) : [];
165
+ const found = { refs, ok: out.status === 0 };
166
+ cache.set(commit, found);
167
+ return found;
168
+ }
169
+
170
+ /** The purged line as the current verified segment holds it, or null when it no longer does. */
171
+ function purgedLine(root: string, report: SegmentReport, entry: JournalEntry): string | null {
172
+ try {
173
+ const data = readFileSync(join(root, ASSERTIONS_DIR, `${entry.seg}.jsonl`));
174
+ const prefix = data.subarray(0, Math.min(report.verifiedBytes, data.length)).toString("utf8"), lines = prefix.split("\n");
175
+ const line = lines[entry.seq - 1];
176
+ if (line === undefined) return null;
177
+ const parsed = JSON.parse(line) as { hash?: unknown };
178
+ return parsed?.hash === entry.hash ? line : null;
179
+ } catch { return null; }
180
+ }
181
+
182
+ /** One parsed JSONL line, or undefined when it is not a journal entry shape. */
183
+ function parsedEntry(line: string): Record<string, unknown> | undefined {
184
+ try {
185
+ const parsed: unknown = JSON.parse(line);
186
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return undefined;
187
+ const entry = parsed as Record<string, unknown>;
188
+ return typeof entry.seg === "string" && Number.isSafeInteger(entry.seq) && typeof entry.hash === "string" ? entry : undefined;
189
+ } catch { return undefined; }
190
+ }
191
+
192
+ /** Mark every target whose line this blob still holds with a payload that hashes to its `payload_sha256`. */
193
+ function blobMatches(content: Buffer, targets: Map<string, Target>): { target: Target; line: string }[] {
194
+ const matches: { target: Target; line: string }[] = [];
195
+ for (const line of text(content).split("\n")) {
196
+ // Every exposed line is the same entry with `payload` restored; the marker is a cheap prefilter before parsing.
197
+ if (!line.includes('"payload":')) continue;
198
+ const entry = parsedEntry(line);
199
+ if (!entry || entry.payload === undefined || typeof entry.payload_sha256 !== "string") continue;
200
+ if (payloadHash(entry.payload) !== entry.payload_sha256) continue;
201
+ const target = targets.get(`${entry.seg as string}\u0000${String(entry.seq)}\u0000${entry.hash as string}`);
202
+ if (target) matches.push({ target, line });
203
+ }
204
+ return matches;
205
+ }
206
+
207
+ /**
208
+ * Scan the vault repository for purged payloads this repository can still reach. Never throws for an unreadable
209
+ * repository: `clean: null` with the cause in `issues` is the answer, and a partial scan is never `clean: true`.
210
+ */
211
+ export function scanAssertionExposure(home: string, request: ExposureRequest, options: { env?: NodeJS.ProcessEnv } = {}): ExposureScan {
212
+ const env = options.env ?? process.env;
213
+ const { vault, root, denyPaths } = openVault(home, request);
214
+ const receipt: ExposureReceipt = { vault: vault.id, scanned: { commits: 0, blobs: 0, bytes: 0 }, exposed: [], clean: null, truncated: null, issues: [] };
215
+ const scan: ExposureScan = { receipt, root, lines: [] };
216
+ const dropped = { count: 0 };
217
+ const issue = (message: string) => { if (receipt.issues.length < ISSUE_LIMIT) receipt.issues.push(message); else dropped.count++; };
218
+ // The journal is read first, so a denied `assertions/` path refuses here exactly as it refuses `vmem assertions`.
219
+ const journal = readJournal(root, denyPaths);
220
+ const history = evaluateAssertions(journal, { history: true }).items;
221
+ if (request.assertion !== undefined) {
222
+ const found = history.filter(item => item.id === request.assertion);
223
+ if (!found.length) fail("V01", `no verified journal entry belongs to ${request.assertion}; list assertion ids with vmem assertions ${vault.id} --history`);
224
+ if (!found.some(item => item.purged === true)) fail("V01", `${request.assertion} is not purged, so its payload is still in the current journal and no history scan can clear it; purge it with vmem assertions purge ${vault.id} ${request.assertion} --reason TEXT`);
225
+ }
226
+ const targets = purgedTargets(journal, history, request.assertion);
227
+ const top = git(root, ["rev-parse", "--show-toplevel"], env, 1 << 16);
228
+ if (top.status !== 0) {
229
+ if (top.stderr.trim() === "ENOENT") issue("git is not available on PATH; install git and retry");
230
+ else issue("not a git repository");
231
+ return scan;
232
+ }
233
+ if (real(text(top.stdout).trim()) !== root) {
234
+ issue("the registered vault is not the top level of its Git repository; exposure reads the whole repository, so it refuses to scan a parent or an unrelated tree");
235
+ return scan;
236
+ }
237
+ if (!targets.length) { receipt.clean = true; return scan; }
238
+ issue(LOCAL_ONLY);
239
+ const refs = git(root, ["for-each-ref", "--format=%(refname)", "refs/heads", "refs/tags", "refs/remotes"], env, LIST_BUFFER_BYTES);
240
+ if (refs.status !== 0) { issue("git could not list this repository's refs; the scan is incomplete and is not reported clean"); return scan; }
241
+ if (!text(refs.stdout).trim()) { receipt.clean = true; return scan; }
242
+ // Every reachable commit is visited, not only the ones that change `assertions/`: a note-only commit between the
243
+ // payload and the purge still holds the segment blob, and the receipt promises every commit that still holds it.
244
+ const listed = git(root, ["rev-list", `--max-count=${EXPOSURE_COMMIT_BUDGET + 1}`, "--branches", "--tags", "--remotes"], env, LIST_BUFFER_BYTES);
245
+ if (listed.status !== 0) {
246
+ const first = listed.stderr.split(/\r?\n/)[0] ?? "";
247
+ issue(`git could not list this repository's reachable commits (${first.slice(0, 200) || `exit ${listed.status}`}); the scan is incomplete and is not reported clean`);
248
+ return scan;
249
+ }
250
+ let commits = text(listed.stdout).split("\n").map(line => line.trim()).filter(line => OID.test(line));
251
+ if (commits.length > EXPOSURE_COMMIT_BUDGET) {
252
+ commits = commits.slice(0, EXPOSURE_COMMIT_BUDGET);
253
+ receipt.truncated = `commit budget (${EXPOSURE_COMMIT_BUDGET} commits)`;
254
+ }
255
+ const subtrees = assertionsTrees(root, env, commits);
256
+ if (!subtrees) {
257
+ issue("git could not resolve the assertions/ tree of these commits; the scan is incomplete and is not reported clean");
258
+ return scan;
259
+ }
260
+ const byKey = new Map(targets.map(target => [key(target.entry), target]));
261
+ const blobCache = new Map<string, { target: Target; line: string }[]>(), refCache = new Map<string, { refs: string[]; ok: boolean }>();
262
+ const treeCache = new Map<string, TreeBlob[]>();
263
+ let incomplete = false;
264
+ for (const commit of commits) {
265
+ const subtree = subtrees.get(commit);
266
+ // A commit that never had an `assertions/` tree holds no assertion line.
267
+ if (subtree === undefined) continue;
268
+ let blobs = treeCache.get(subtree);
269
+ if (!blobs) {
270
+ // The listing is per tree, not per commit: commits that leave the journal alone share one tree and one read.
271
+ const tree = git(root, ["ls-tree", "-r", "-l", "-z", subtree], env, LIST_BUFFER_BYTES);
272
+ if (tree.status !== 0) {
273
+ incomplete = true;
274
+ issue(`git could not list the assertions/ tree of commit ${commit.slice(0, 12)} (exit ${tree.status}); the scan is incomplete`);
275
+ continue;
276
+ }
277
+ blobs = treeBlobs(tree.stdout);
278
+ treeCache.set(subtree, blobs);
279
+ }
280
+ // The per-commit total is summed from the sizes Git lists, so the bound is crossed without reading a byte.
281
+ if (blobs.reduce((sum, blob) => sum + blob.size, 0) > EXPOSURE_COMMIT_BYTES) { receipt.truncated ??= `64 MiB per-commit budget (commit ${commit.slice(0, 12)})`; continue; }
282
+ receipt.scanned.commits++;
283
+ for (const blob of blobs) {
284
+ if (blob.size > EXPOSURE_BLOB_BYTES) { receipt.truncated ??= `8 MiB per-blob budget (blob ${blob.oid.slice(0, 12)})`; continue; }
285
+ let matches = blobCache.get(blob.oid);
286
+ if (!matches) {
287
+ const content = git(root, ["cat-file", "blob", blob.oid], env, Math.max(blob.size + 64, 1 << 16));
288
+ if (content.status !== 0 || content.stdout.length !== blob.size) {
289
+ incomplete = true;
290
+ issue(`blob ${blob.oid.slice(0, 12)} could not be read (exit ${content.status}); the scan is incomplete`);
291
+ continue;
292
+ }
293
+ receipt.scanned.blobs++; receipt.scanned.bytes += content.stdout.length;
294
+ matches = blobMatches(content.stdout, byKey);
295
+ blobCache.set(blob.oid, matches);
296
+ }
297
+ // One blob can sit unchanged in several commits, and every one of them still holds the line.
298
+ for (const match of matches) {
299
+ match.target.exposed.add(match.line);
300
+ if (!match.target.seen.has(commit)) { match.target.seen.add(commit); match.target.commits.push(commit); }
301
+ }
302
+ }
303
+ }
304
+ for (const target of targets) {
305
+ if (!target.exposed.size) continue;
306
+ const found = target.commits.map(commit => refsContaining(root, env, commit, refCache));
307
+ if (found.some(item => !item.ok)) {
308
+ incomplete = true;
309
+ issue(`git could not list the refs that contain a commit holding a purged line (${target.entry.seg}:${target.entry.seq}); refs are incomplete`);
310
+ }
311
+ const report = journal.segments.find(segment => segment.segment === target.entry.seg);
312
+ const purged = report ? purgedLine(root, report, target.entry) : null;
313
+ const pairs: ExposureLine[] = [];
314
+ let replaceable = purged !== null;
315
+ if (purged !== null) for (const line of target.exposed) {
316
+ if (!expressible(line) || !expressible(purged)) { replaceable = false; pairs.length = 0; break; }
317
+ pairs.push({ exposed: line, purged });
318
+ }
319
+ scan.lines.push(...pairs);
320
+ receipt.exposed.push({ assertion: target.assertion, segment: target.entry.seg, seq: target.entry.seq, hash: target.entry.hash,
321
+ commits: { count: target.commits.length, first: target.commits[target.commits.length - 1]!, last: target.commits[0]! },
322
+ refs: [...new Set(found.flatMap(item => item.refs))].sort(), replaceable, ...(replaceable ? {} : { manual: purged === null ? NO_PURGED_LINE : NOT_EXPRESSIBLE }) });
323
+ }
324
+ if (receipt.truncated !== null) issue(`the scan stopped at the ${receipt.truncated}; a partial scan is never reported clean`);
325
+ if (dropped.count) issue(`${dropped.count} more issue(s) were dropped from this receipt`);
326
+ receipt.clean = receipt.truncated !== null || incomplete ? false : receipt.exposed.length === 0;
327
+ return scan;
328
+ }
329
+
330
+ /**
331
+ * Write the `git filter-repo --replace-text` expressions file for a scan: one literal `exposed==>purged` line per
332
+ * exposed line, mode 0600, a new path only, outside the vault. It holds the payloads it removes, so the receipt
333
+ * says to delete it.
334
+ */
335
+ export function writeReplaceFile(scan: ExposureScan, target: string): ReplaceFileReceipt {
336
+ const receipt = scan.receipt;
337
+ if (receipt.clean !== false) fail("V01", receipt.clean === true
338
+ ? "the exposure scan found no purged payload in reachable history, so there is nothing to replace; no file was written"
339
+ : "the exposure scan did not complete, so no replace file was written; fix the issues it reported and rerun");
340
+ if (!scan.lines.length) fail("V01", "no exposed line can be written as a replace expression; see each unreplaceable row's manual step, and no file was written");
341
+ const absolute = resolve(target), vault = resolve(scan.root);
342
+ const directory = (() => { try { return realpathSync(dirname(absolute)); } catch { return fail("V04", `--replace-file: the directory ${dirname(absolute)} does not exist; create a private directory for the expressions file, then retry`); } })();
343
+ const path = join(directory, basename(absolute));
344
+ if (path === vault || path.startsWith(vault + sep)) fail("V04", "--replace-file must lie outside the vault: the expressions file holds the payloads it removes, so it is never written into the repository it is meant to clean; name a path outside the vault");
345
+ if (existsSync(path)) fail("V04", `--replace-file refuses an existing path (${path}); delete or rename that file first, and never reuse an expressions file across scans`);
346
+ const expressions = scan.lines.map(line => `${line.exposed}==>${line.purged}`).join("\n") + "\n";
347
+ let fd: number;
348
+ try { fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); }
349
+ catch (caught) {
350
+ const code = (caught as NodeJS.ErrnoException).code ?? "unknown error";
351
+ if (code === "EEXIST") fail("V04", `--replace-file refuses an existing path (${path}); delete or rename that file first, and never reuse an expressions file across scans`);
352
+ if (code === "ELOOP") fail("V04", `--replace-file refuses ${path}: it is a symbolic link; name a new regular file outside the vault`);
353
+ fail("V04", `--replace-file could not create ${path} (${code}); check the directory's permissions and free space, then retry`);
354
+ }
355
+ try { writeFileSync(fd, expressions); fsyncSync(fd); }
356
+ catch (caught) {
357
+ closeSync(fd);
358
+ let removed = true;
359
+ try { unlinkSync(path); } catch { removed = false; }
360
+ fail("IO", removed
361
+ ? `the replace file could not be written (${(caught as NodeJS.ErrnoException).code ?? "write failed"}); the partial file was removed and nothing changed`
362
+ : `the replace file could not be written (${(caught as NodeJS.ErrnoException).code ?? "write failed"}); a partial file remains at ${path} and must be deleted, because it holds payload text`);
363
+ }
364
+ closeSync(fd);
365
+ return { path, lines: scan.lines.length, unreplaceable: receipt.exposed.filter(row => !row.replaceable).length,
366
+ delete: "delete this file after the rewrite, and after every failed attempt: it holds the payloads it maps" };
367
+ }
@@ -0,0 +1,86 @@
1
+ /** Disposable, deterministic SQLite projection of a freshly verified assertion journal. */
2
+ import { Database } from "bun:sqlite";
3
+ import { createHash, randomUUID } from "node:crypto";
4
+ import { closeSync, constants, lstatSync, mkdirSync, openSync, realpathSync, renameSync, unlinkSync } from "node:fs";
5
+ import { isAbsolute, join, relative } from "node:path";
6
+ import { MAX_JOURNAL_BYTES, MAX_JOURNAL_ENTRIES, type JournalEntry, type VerifiedJournal } from "./assertions.ts";
7
+ import { canonical } from "./writer.ts";
8
+
9
+ /** The index belongs to the configured home, never to the vault or its source journal. */
10
+ export function assertionIndexPath(home: string, vaultRoot: string, protectedRoots: readonly string[] = [vaultRoot]): string {
11
+ const root = realpathSync(vaultRoot), state = realpathSync(home);
12
+ for (const protectedRoot of [vaultRoot, ...protectedRoots]) {
13
+ let canonicalRoot: string;
14
+ try { canonicalRoot = realpathSync(protectedRoot); }
15
+ catch (error) {
16
+ if (["ENOENT", "ENOTDIR"].includes((error as NodeJS.ErrnoException).code ?? "")) continue;
17
+ throw error;
18
+ }
19
+ let indexDir = join(state, "indexes");
20
+ try { indexDir = realpathSync(indexDir); } catch { /* A missing cache directory has its lexical location. */ }
21
+ for (const candidate of [state, indexDir]) {
22
+ const rel = relative(canonicalRoot, candidate);
23
+ if (!rel || (rel !== ".." && !rel.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`) && !isAbsolute(rel)))
24
+ throw new Error("assertion index: configured home or index directory is inside a vault; move VMEM_HOME outside every registered vault");
25
+ }
26
+ }
27
+ const digest = createHash("sha256").update(root).digest("hex");
28
+ return join(state, "indexes", `assertions-${digest}-v1.sqlite`);
29
+ }
30
+
31
+ /**
32
+ * Rebuild from verified entries in a fixed order. The caller must re-read the journal for each request and must
33
+ * never use this projection as proof of journal integrity or as a substitute for current read policy.
34
+ */
35
+ export function rebuildAssertionIndex(home: string, vaultRoot: string, journal: VerifiedJournal, protectedRoots: readonly string[] = [vaultRoot]): string {
36
+ const target = assertionIndexPath(home, vaultRoot, protectedRoots), dir = join(realpathSync(home), "indexes");
37
+ try { mkdirSync(dir, { mode: 0o700 }); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; }
38
+ const info = lstatSync(dir);
39
+ if (info.isSymbolicLink() || !info.isDirectory()) throw new Error("assertion index: indexes is not a plain directory; replace the link or file with a directory in VMEM_HOME");
40
+ try {
41
+ const existing = lstatSync(target);
42
+ if (!existing.isFile() || existing.isSymbolicLink() || existing.nlink !== 1) throw new Error("assertion index: existing index is linked or not a file; remove the disposable index from VMEM_HOME/indexes");
43
+ } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; }
44
+ const temporary = join(dir, `.assertions-${randomUUID()}.tmp`);
45
+ const fd = openSync(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600);
46
+ closeSync(fd);
47
+ try {
48
+ const db = new Database(temporary);
49
+ try {
50
+ db.exec("PRAGMA journal_mode=MEMORY; PRAGMA synchronous=OFF; CREATE TABLE entries (segment TEXT NOT NULL, seq INTEGER NOT NULL, entry TEXT NOT NULL, PRIMARY KEY(segment, seq)) WITHOUT ROWID;");
51
+ const put = db.prepare("INSERT INTO entries (segment, seq, entry) VALUES (?, ?, ?)");
52
+ db.transaction(() => {
53
+ for (const segment of [...journal.segments].sort((a, b) => a.segment < b.segment ? -1 : a.segment > b.segment ? 1 : 0))
54
+ for (const entry of segment.entries) put.run(segment.segment, entry.seq, canonical(entry));
55
+ })();
56
+ } finally { db.close(); }
57
+ renameSync(temporary, target);
58
+ return target;
59
+ } finally { try { unlinkSync(temporary); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } }
60
+ }
61
+
62
+ /**
63
+ * Read the projection only after verifying the journal. Compare every indexed entry with the verified bytes before
64
+ * using it, so a replaced, stale or corrupt cache can never become an authority over the source journal.
65
+ */
66
+ export function indexedJournal(file: string, verified: VerifiedJournal): VerifiedJournal {
67
+ let db: Database | undefined;
68
+ try {
69
+ const stat = lstatSync(file);
70
+ if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.size > MAX_JOURNAL_BYTES * 4) return verified;
71
+ db = new Database(file, { readonly: true });
72
+ const sizes = db.query("SELECT count(*) AS count, coalesce(sum(length(cast(entry AS blob))), 0) AS bytes FROM entries").get() as { count: number; bytes: number };
73
+ if (sizes.count !== verified.entries || sizes.count > MAX_JOURNAL_ENTRIES || sizes.bytes > MAX_JOURNAL_BYTES) return verified;
74
+ const rows = db.query("SELECT segment, seq, entry FROM entries ORDER BY segment, seq").all() as { segment: string; seq: number; entry: string }[];
75
+ const expected = verified.segments.flatMap(segment => segment.entries.map(entry => ({ segment: segment.segment, entry }))).sort((a, b) => a.segment < b.segment ? -1 : a.segment > b.segment ? 1 : a.entry.seq - b.entry.seq);
76
+ if (rows.length !== expected.length) return verified;
77
+ const entries = new Map<string, JournalEntry[]>();
78
+ for (let i = 0; i < rows.length; i++) {
79
+ const row = rows[i]!, source = expected[i]!;
80
+ if (row.segment !== source.segment || row.seq !== source.entry.seq || row.entry !== canonical(source.entry)) return verified;
81
+ const segmentEntries = entries.get(row.segment) ?? [];
82
+ segmentEntries.push(JSON.parse(row.entry) as JournalEntry); entries.set(row.segment, segmentEntries);
83
+ }
84
+ return { ...verified, segments: verified.segments.map(segment => ({ ...segment, entries: entries.get(segment.segment) ?? [] })) };
85
+ } catch { return verified; } finally { db?.close(); }
86
+ }
@@ -0,0 +1,63 @@
1
+ /** Current-policy projection of assertion content. Never cache the projected result across reads. */
2
+ import { resolve } from "node:path";
3
+ import type { VaultSpec, VmemConfig } from "./config.ts";
4
+ import { parseEvidence, verifyEvidenceEntry, type EvidenceReadBudget, type EvidenceReads } from "./evidence.ts";
5
+ import { canonicalPath, isDenied } from "./platform.ts";
6
+ import { sourceReferences, sourceWorkspaceRoot } from "./source.ts";
7
+ import { screenContent } from "./write-validation.ts";
8
+ import type { AssertionItem } from "./assertions.ts";
9
+
10
+ export type AssertionValue = string | { withheld: "secret" | "pii_policy" };
11
+ export type ScreenedAssertion = Omit<AssertionItem, "value" | "evidence" | "source"> & {
12
+ value?: AssertionValue;
13
+ evidence?: Record<string, unknown>[];
14
+ source?: (string | { withheld: true })[];
15
+ availability?: "unavailable";
16
+ };
17
+
18
+ function withheld(text: string, vault: VaultSpec, config: VmemConfig): "secret" | "pii_policy" | undefined {
19
+ const issues = screenContent(text, vault, config).filter(issue => issue.severity === "error");
20
+ return issues.some(issue => issue.code === "V06") ? "secret" : issues.length ? "pii_policy" : undefined;
21
+ }
22
+ function deniedSource(source: string, vault: VaultSpec, config: VmemConfig): boolean {
23
+ const deny = config.denyPaths ?? [];
24
+ try {
25
+ const ref = sourceReferences(source)[0]!;
26
+ if (ref.kind !== "path") return false;
27
+ const root = sourceWorkspaceRoot(vault);
28
+ return isDenied(ref.value, deny) || (root !== undefined && isDenied(canonicalPath(resolve(root, ref.value)), deny));
29
+ } catch { return true; }
30
+ }
31
+
32
+ /** Screen every field on every read. A denied evidence entry reveals neither its source, quote nor hashes. */
33
+ export function screenAssertion(item: AssertionItem, vault: VaultSpec, config: VmemConfig, reads: EvidenceReads, budget: EvidenceReadBudget): ScreenedAssertion {
34
+ const { value, evidence, source, ...safe } = item;
35
+ const out: ScreenedAssertion = { ...safe };
36
+ if (item.purged) { out.availability = "unavailable"; out.evidence = [{ status: "unavailable" }]; return out; }
37
+ if (typeof value === "string") {
38
+ const reason = withheld(value, vault, config);
39
+ out.value = reason ? { withheld: reason } : value;
40
+ }
41
+ if (Array.isArray(source)) out.source = source.map(entry => typeof entry === "string" && !deniedSource(entry, vault, config) && !withheld(entry, vault, config) ? entry : { withheld: true as const });
42
+ if (evidence !== undefined) {
43
+ if (!Array.isArray(evidence)) { out.evidence = [{ status: "unsupported", reason: "invalid_metadata" }]; return out; }
44
+ // Journal entries add the write-time status; verify again and never treat that old status as current proof.
45
+ const entries = evidence.map(value => {
46
+ if (!value || typeof value !== "object" || Array.isArray(value)) return value;
47
+ const { status: _status, ...entry } = value as Record<string, unknown>;
48
+ return entry;
49
+ });
50
+ const evidenceSources = entries.flatMap(value => value && typeof value === "object" && "source" in value && typeof value.source === "string" ? [value.source] : []);
51
+ const parsed = entries.length ? parseEvidence(entries, evidenceSources.length ? evidenceSources : undefined) : { entries: [] };
52
+ out.evidence = parsed.problem ? [{ status: "unsupported", reason: "invalid_metadata" }] : parsed.entries.map((entry, index) => {
53
+ if ("problem" in entry) return { status: "unsupported", reason: "invalid_metadata" };
54
+ if (deniedSource(entry.source, vault, config) || withheld(entry.source, vault, config) || withheld(entry.quote, vault, config) || (entry.section && withheld(entry.section, vault, config))) return { status: "withheld", reason: "denied_by_current_policy" };
55
+ const result = verifyEvidenceEntry(vault, entry, index, config.denyPaths ?? [], reads, budget);
56
+ if (result.reason === "denied") return { status: "withheld", reason: "denied_by_current_policy" };
57
+ if (result.reason === "read_budget") throw Object.assign(new Error("assertions: evidence read budget exceeded; narrow the query or lower the page limit"), { code: "V09" });
58
+ if (result.reason === "source_not_found") return { source: entry.source, status: "unavailable" };
59
+ return { ...entry, status: result.status, ...(result.reason ? { reason: result.reason } : {}) };
60
+ });
61
+ }
62
+ return out;
63
+ }