@openparachute/vault 0.7.9-rc.4 → 0.7.9-rc.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +4 -0
  2. package/core/src/compact.test.ts +248 -0
  3. package/core/src/delta.test.ts +49 -0
  4. package/core/src/delta.ts +21 -0
  5. package/core/src/doctor.ts +41 -1
  6. package/core/src/history-capture.test.ts +244 -0
  7. package/core/src/history-import.test.ts +82 -0
  8. package/core/src/history-import.ts +157 -0
  9. package/core/src/history.test.ts +229 -0
  10. package/core/src/history.ts +681 -0
  11. package/core/src/mcp-manifest.ts +17 -0
  12. package/core/src/mcp.ts +28 -3
  13. package/core/src/notes.ts +6 -1
  14. package/core/src/portable-md.ts +1 -1
  15. package/core/src/schema-v29-note-versions.test.ts +63 -0
  16. package/core/src/schema-v30-note-deltas.test.ts +51 -0
  17. package/core/src/schema-v31-history-import.test.ts +25 -0
  18. package/core/src/schema.ts +234 -1
  19. package/core/src/store.ts +103 -14
  20. package/core/src/types.ts +35 -2
  21. package/core/src/vendor/fossil-delta.ts +472 -0
  22. package/package.json +1 -1
  23. package/src/cli.ts +10 -1
  24. package/src/config.ts +42 -0
  25. package/src/history-import-mcp.test.ts +94 -0
  26. package/src/history-import-routes.test.ts +91 -0
  27. package/src/history-import.test.ts +318 -0
  28. package/src/history-import.ts +482 -0
  29. package/src/mcp-http.ts +13 -0
  30. package/src/mcp-tools.ts +18 -2
  31. package/src/mirror-config.ts +38 -0
  32. package/src/mirror-manager.ts +6 -0
  33. package/src/mirror-retirement.test.ts +79 -0
  34. package/src/mirror-routes.ts +12 -2
  35. package/src/routes.ts +226 -86
  36. package/src/routing.ts +8 -3
  37. package/src/server.ts +33 -14
  38. package/src/vault-compact-routes.test.ts +257 -0
  39. package/src/vault-history-routes.test.ts +562 -0
  40. package/src/vault-store.ts +13 -0
@@ -0,0 +1,681 @@
1
+ /**
2
+ * Note history stores prior note-row states, inside the caller's write
3
+ * transaction. captureVersion, appendRestoreMarker and pruneVersions open
4
+ * no explicit transaction of their own. Live reads still use notes.content.
5
+ * NULL content is recorded as an empty string; metadata bytes are preserved.
6
+ *
7
+ * Capture sites: Store.updateNote (including SQL append/prepend, transitions,
8
+ * and skipUpdatedAt metadata writes), Store.deleteNote, cascadeRename, and
9
+ * renameTag's content/path rewrites. Creates have no prior state. Schema
10
+ * migrations, restoreNoteTimestamps, mergeTags' timestamp bump, deleteTag's
11
+ * untagging, and blow-away deletion deliberately do not capture.
12
+ */
13
+ import type { Database } from "bun:sqlite";
14
+ import { createHash } from "node:crypto";
15
+ import { encodeDelta, decodeDelta } from "./delta.js";
16
+ import { transaction } from "./txn.js";
17
+
18
+ export type HistoryOp =
19
+ | "import"
20
+ | "update"
21
+ | "append"
22
+ | "prepend"
23
+ | "delete"
24
+ | "restore"
25
+ | "tag-rename"
26
+ | "cascade-rename";
27
+ export interface HistoryPolicy {
28
+ enabled: boolean;
29
+ min_versions: number;
30
+ max_versions: number;
31
+ max_age_days: number;
32
+ deleted_retention_days: number | null;
33
+ compact_enabled: boolean;
34
+ compact_ratio: number;
35
+ compact_min_versions: number;
36
+ compact_run_length: number;
37
+ max_bytes_per_note: number | null;
38
+ compact_budget_ms: number;
39
+ compact_max_notes: number;
40
+ }
41
+ export const DEFAULT_HISTORY_POLICY: HistoryPolicy = {
42
+ enabled: true,
43
+ min_versions: 20,
44
+ max_versions: 100,
45
+ max_age_days: 180,
46
+ deleted_retention_days: null,
47
+ compact_enabled: true,
48
+ compact_ratio: 3,
49
+ compact_min_versions: 10,
50
+ compact_run_length: 24,
51
+ max_bytes_per_note: 8_388_608,
52
+ compact_budget_ms: 250,
53
+ compact_max_notes: 50,
54
+ };
55
+ export function resolveHistoryPolicy(
56
+ partial?: Partial<HistoryPolicy>,
57
+ ): HistoryPolicy {
58
+ const p = { ...DEFAULT_HISTORY_POLICY, ...partial };
59
+ // The floor is the retention promise: clamp the ceiling up, never down.
60
+ p.max_versions = Math.max(1, p.max_versions, p.min_versions);
61
+ p.max_age_days = Math.min(p.max_age_days, 36500);
62
+ if (p.deleted_retention_days !== null)
63
+ p.deleted_retention_days = Math.min(p.deleted_retention_days, 36500);
64
+ p.compact_ratio = Math.max(1, p.compact_ratio);
65
+ p.compact_min_versions = Math.max(2, p.compact_min_versions);
66
+ p.compact_run_length = Math.min(100, Math.max(2, p.compact_run_length));
67
+ if (p.max_bytes_per_note !== null) p.max_bytes_per_note = Math.max(65_536, p.max_bytes_per_note);
68
+ p.compact_budget_ms = Math.min(60_000, Math.max(0, p.compact_budget_ms));
69
+ p.compact_max_notes = Math.min(10_000, Math.max(0, p.compact_max_notes));
70
+ return p;
71
+ }
72
+ export const VERSION_MAX_BYTES = 2_000_000;
73
+ export interface PriorNoteRow {
74
+ id: string;
75
+ content: string | null;
76
+ path: string | null;
77
+ metadata: string | null;
78
+ extension: string | null;
79
+ created_at: string | null;
80
+ }
81
+ export interface VersionRow {
82
+ note_id: string;
83
+ version_ix: number;
84
+ content_hash: string | null;
85
+ path: string | null;
86
+ metadata: Record<string, unknown>;
87
+ extension: string | null;
88
+ created_at: string | null;
89
+ superseded_at: string;
90
+ actor: string | null;
91
+ via: string | null;
92
+ op: HistoryOp;
93
+ content_len: number;
94
+ encoding: string | null;
95
+ }
96
+ type RawVersionRow = Omit<VersionRow, "metadata"> & { metadata: string | null };
97
+ export class HistoryOverflowError extends Error {
98
+ readonly code = "HISTORY_OVERFLOW";
99
+ constructor(
100
+ readonly note_id: string,
101
+ readonly byte_size: number,
102
+ readonly limit: number,
103
+ ) {
104
+ super(
105
+ `Note history exceeds ${limit} bytes: "${note_id}" (${byte_size} bytes)`,
106
+ );
107
+ this.name = "HistoryOverflowError";
108
+ }
109
+ }
110
+ export class HistoryUnrecoverableError extends Error {
111
+ readonly error_type = "history_unrecoverable";
112
+ readonly code = "HISTORY_UNRECOVERABLE";
113
+ constructor(
114
+ readonly note_id: string,
115
+ readonly version_ix: number,
116
+ readonly reason: "overflow" | "delta_orphan" = "overflow",
117
+ ) {
118
+ super(`Version content is unrecoverable: "${note_id}"@${version_ix}`);
119
+ this.name = "HistoryUnrecoverableError";
120
+ }
121
+ }
122
+ export class HistoryNotFoundError extends Error {
123
+ code = "HISTORY_NOT_FOUND" as const;
124
+ note_id: string;
125
+ version_ix: number | null;
126
+ constructor(noteId: string, versionIx: number | null) {
127
+ super(
128
+ versionIx === null
129
+ ? `Note not found: "${noteId}"`
130
+ : `Version not found: "${noteId}"@${versionIx}`,
131
+ );
132
+ this.name = "HistoryNotFoundError";
133
+ this.note_id = noteId;
134
+ this.version_ix = versionIx;
135
+ }
136
+ }
137
+ export function hashContent(text: string): string {
138
+ return createHash("sha256").update(text, "utf8").digest("hex");
139
+ }
140
+ export function byteLength(text: string): number {
141
+ return new TextEncoder().encode(text).length;
142
+ }
143
+ const present = new WeakMap<Database, boolean>();
144
+ export function historyTablesPresent(db: Database): boolean {
145
+ if (present.get(db)) return true;
146
+ const exists = !!db
147
+ .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?")
148
+ .get("note_versions");
149
+ if (exists) present.set(db, true);
150
+ return exists;
151
+ }
152
+ export function readPriorNoteRow(
153
+ db: Database,
154
+ id: string,
155
+ ): PriorNoteRow | null {
156
+ return db
157
+ .prepare(
158
+ "SELECT id, content, path, metadata, extension, created_at FROM notes WHERE id = ?",
159
+ )
160
+ .get(id) as PriorNoteRow | null;
161
+ }
162
+ function nextIndex(db: Database, noteId: string): number {
163
+ return (
164
+ db
165
+ .prepare(
166
+ "SELECT COALESCE(MAX(version_ix), -1) + 1 AS ix FROM note_versions WHERE note_id = ? AND version_ix >= 0",
167
+ )
168
+ .get(noteId) as { ix: number }
169
+ ).ix;
170
+ }
171
+ /**
172
+ * `createNote` does no capture, so a note larger than `VERSION_MAX_BYTES` can exist in any vault (probe H counts them). Every later `updateNote` throws `HistoryOverflowError` **inside** `store.ts:635`'s transaction and rolls the write back — correct, loud, and exactly what CodexJi asked for. But change 9 puts `deleteNote`'s capture inside a transaction too, and if a delete threw the same way the note would be **permanently undeletable**: no update can shrink it, and the only escape (`captureHistory: false`) is exposed on no door. An un-updatable note is a nuisance; an un-deletable one is a trap. So a `delete` capture **always succeeds**: it writes the tombstone with `content_hash = NULL`, the real `content_len`, and `encoding = 'overflow'`. The note's *existence*, size, path, metadata and deletion time are recorded; its bytes are not, and were never recordable. **Updates still throw — unchanged.** (ClaudeJi ruling 6, 2026-09-14 20:35Z.)
173
+ */
174
+ export function captureVersion(
175
+ db: Database,
176
+ prior: PriorNoteRow,
177
+ opts: {
178
+ actor: string | null;
179
+ via: string | null;
180
+ op: HistoryOp;
181
+ policy: HistoryPolicy;
182
+ },
183
+ ): void {
184
+ if (!opts.policy.enabled) return;
185
+ if (!historyTablesPresent(db)) return;
186
+ const text = prior.content ?? "";
187
+ const size = byteLength(text);
188
+ if (size > VERSION_MAX_BYTES) {
189
+ if (opts.op !== "delete")
190
+ throw new HistoryOverflowError(prior.id, size, VERSION_MAX_BYTES);
191
+ const ix = nextIndex(db, prior.id);
192
+ db.prepare(
193
+ `INSERT INTO note_versions (note_id, version_ix, content_hash, path, metadata, extension,
194
+ superseded_at, actor, via, op, content_len, encoding, created_at) VALUES (?, ?, NULL, ?, ?, ?, ?, ?, ?, 'delete', ?, 'overflow', ?)`,
195
+ ).run(
196
+ prior.id,
197
+ ix,
198
+ prior.path,
199
+ prior.metadata,
200
+ prior.extension,
201
+ new Date().toISOString(),
202
+ opts.actor,
203
+ opts.via,
204
+ size,
205
+ prior.created_at,
206
+ );
207
+ pruneVersions(db, prior.id, opts.policy);
208
+ return;
209
+ }
210
+ const hash = hashContent(text);
211
+ db.prepare(
212
+ "INSERT OR IGNORE INTO note_blobs (hash, content, byte_size) VALUES (?, ?, ?)",
213
+ ).run(hash, text, size);
214
+ const ix = nextIndex(db, prior.id);
215
+ db.prepare(
216
+ `INSERT INTO note_versions (note_id, version_ix, content_hash, path, metadata, extension,
217
+ superseded_at, actor, via, op, content_len, encoding, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?)`,
218
+ ).run(
219
+ prior.id,
220
+ ix,
221
+ hash,
222
+ prior.path,
223
+ prior.metadata,
224
+ prior.extension,
225
+ new Date().toISOString(),
226
+ opts.actor,
227
+ opts.via,
228
+ opts.op,
229
+ size,
230
+ prior.created_at,
231
+ );
232
+ pruneVersions(db, prior.id, opts.policy);
233
+ }
234
+ export function appendRestoreMarker(
235
+ db: Database,
236
+ noteId: string,
237
+ tombstone: VersionRow,
238
+ attr: { actor: string | null; via: string | null },
239
+ policy: HistoryPolicy,
240
+ ): void {
241
+ // Copy in SQL so the raw metadata TEXT is not parsed and re-serialized.
242
+ const ix = nextIndex(db, noteId);
243
+ db.prepare(
244
+ `INSERT INTO note_versions (note_id, version_ix, content_hash, path, metadata, extension,
245
+ superseded_at, actor, via, op, content_len, encoding, created_at)
246
+ SELECT note_id, ?, content_hash, path, metadata, extension, ?, ?, ?, 'restore', content_len, encoding, created_at
247
+ FROM note_versions WHERE note_id = ? AND version_ix = ?`,
248
+ ).run(
249
+ ix,
250
+ new Date().toISOString(),
251
+ attr.actor,
252
+ attr.via,
253
+ noteId,
254
+ tombstone.version_ix,
255
+ );
256
+ pruneVersions(db, noteId, policy);
257
+ }
258
+ export function pruneVersions(
259
+ db: Database,
260
+ noteId: string,
261
+ policy: HistoryPolicy,
262
+ now = Date.now(),
263
+ ): { versionsDeleted: number; blobsDeleted: number } {
264
+ const rows = db
265
+ .prepare(
266
+ "SELECT version_ix, superseded_at, op, content_hash FROM note_versions WHERE note_id = ? ORDER BY version_ix DESC",
267
+ )
268
+ .all(noteId) as Pick<
269
+ VersionRow,
270
+ "version_ix" | "superseded_at" | "op" | "content_hash"
271
+ >[];
272
+ const cutoff = new Date(
273
+ now - policy.max_age_days * 86_400_000,
274
+ ).toISOString();
275
+ const doomed = rows.filter(
276
+ (r, i) =>
277
+ i >= policy.min_versions &&
278
+ r.op !== "delete" &&
279
+ (i >= policy.max_versions || r.superseded_at < cutoff),
280
+ );
281
+ let versionsDeleted = 0,
282
+ blobsDeleted = 0;
283
+ for (const r of doomed)
284
+ versionsDeleted += db
285
+ .prepare("DELETE FROM note_versions WHERE note_id = ? AND version_ix = ?")
286
+ .run(noteId, r.version_ix).changes;
287
+ for (const hash of new Set(
288
+ doomed.map((r) => r.content_hash).filter((h) => h !== null),
289
+ )) {
290
+ blobsDeleted += db
291
+ .prepare(
292
+ "DELETE FROM note_blobs WHERE hash = ? AND NOT EXISTS (SELECT 1 FROM note_versions WHERE content_hash = ?) AND NOT EXISTS (SELECT 1 FROM note_blobs WHERE delta_of = ?)",
293
+ )
294
+ .run(hash, hash, hash).changes;
295
+ }
296
+ return { versionsDeleted, blobsDeleted };
297
+ }
298
+ export function gcBlobs(db: Database): { blobsDeleted: number } {
299
+ let blobsDeleted = 0;
300
+ // Depth one needs two deletion waves; bounded even in a corrupt database.
301
+ for (let pass = 0; pass < 4; pass++) {
302
+ const changed = db.prepare("DELETE FROM note_blobs WHERE hash NOT IN (SELECT content_hash FROM note_versions WHERE content_hash IS NOT NULL) AND hash NOT IN (SELECT delta_of FROM note_blobs WHERE delta_of IS NOT NULL)").run().changes;
303
+ blobsDeleted += changed;
304
+ if (!changed) break;
305
+ }
306
+ return { blobsDeleted };
307
+ }
308
+ export function sweepDeletedHistory(
309
+ db: Database,
310
+ policy: HistoryPolicy,
311
+ now = new Date(),
312
+ ): { notesSwept: number; versionsDeleted: number; blobsDeleted: number } {
313
+ if (policy.deleted_retention_days === null)
314
+ return { notesSwept: 0, versionsDeleted: 0, blobsDeleted: 0 };
315
+ const cutoff = new Date(
316
+ now.getTime() - policy.deleted_retention_days * 86_400_000,
317
+ ).toISOString();
318
+ const victims = db
319
+ .prepare(
320
+ `SELECT v.note_id, MAX(v.superseded_at) AS last FROM note_versions v
321
+ LEFT JOIN notes n ON n.id = v.note_id WHERE n.id IS NULL GROUP BY v.note_id HAVING last < ?`,
322
+ )
323
+ .all(cutoff) as { note_id: string }[];
324
+ return transaction(db, () => {
325
+ let versionsDeleted = 0;
326
+ for (const v of victims)
327
+ versionsDeleted += db
328
+ .prepare("DELETE FROM note_versions WHERE note_id = ?")
329
+ .run(v.note_id).changes;
330
+ return { notesSwept: victims.length, versionsDeleted, ...gcBlobs(db) };
331
+ });
332
+ }
333
+ const columns =
334
+ "note_id, version_ix, content_hash, path, metadata, extension, superseded_at, actor, via, op, content_len, encoding, created_at";
335
+ function parseRow(row: RawVersionRow): VersionRow {
336
+ let metadata: Record<string, unknown> = {};
337
+ try {
338
+ metadata = row.metadata ? JSON.parse(row.metadata) : {};
339
+ } catch {}
340
+ return { ...row, metadata };
341
+ }
342
+ export function listVersions(
343
+ db: Database,
344
+ noteId: string,
345
+ opts?: { limit?: number; offset?: number },
346
+ ): VersionRow[] {
347
+ return (
348
+ db
349
+ .prepare(
350
+ `SELECT ${columns} FROM note_versions WHERE note_id = ? ORDER BY version_ix DESC LIMIT ? OFFSET ?`,
351
+ )
352
+ .all(noteId, opts?.limit ?? 50, opts?.offset ?? 0) as RawVersionRow[]
353
+ ).map(parseRow);
354
+ }
355
+ export function getVersion(
356
+ db: Database,
357
+ noteId: string,
358
+ versionIx: number,
359
+ ): (VersionRow & { content: string | null }) | null {
360
+ const row = db
361
+ .prepare(
362
+ `SELECT ${columns
363
+ .split(", ")
364
+ .map((c) => "v." + c)
365
+ .join(", ")}, b.content, b.encoding AS blob_encoding, b.delta_of
366
+ FROM note_versions v LEFT JOIN note_blobs b ON b.hash = v.content_hash WHERE v.note_id = ? AND v.version_ix = ?`,
367
+ )
368
+ .get(noteId, versionIx) as
369
+ | (RawVersionRow & BlobReadRow)
370
+ | null;
371
+ if (!row) return null;
372
+ const { content, blob_encoding, delta_of, ...verCols } = row;
373
+ try {
374
+ const reconstructed = materialise(db, verCols.content_hash!, { content, blob_encoding, delta_of });
375
+ return { ...parseRow(verCols), content: reconstructed, encoding: verCols.encoding ?? blob_encoding };
376
+ } catch (err) {
377
+ if (!(err instanceof HistoryDeltaOrphanError)) throw err;
378
+ console.warn("[history] unrecoverable delta", { hash: err.hash, delta_of: err.delta_of, reason: err.reason });
379
+ throw new HistoryUnrecoverableError(noteId, versionIx, "delta_orphan");
380
+ }
381
+ }
382
+ export function latestTombstone(
383
+ db: Database,
384
+ noteId: string,
385
+ ): VersionRow | null {
386
+ const row = db
387
+ .prepare(
388
+ `SELECT ${columns} FROM note_versions WHERE note_id = ? AND op = 'delete' ORDER BY version_ix DESC LIMIT 1`,
389
+ )
390
+ .get(noteId) as RawVersionRow | null;
391
+ return row ? parseRow(row) : null;
392
+ }
393
+ export function eraseHistory(
394
+ db: Database,
395
+ noteId: string,
396
+ ): { versionsDeleted: number; blobsDeleted: number } {
397
+ return transaction(db, () => {
398
+ const versionsDeleted = db
399
+ .prepare("DELETE FROM note_versions WHERE note_id = ?")
400
+ .run(noteId).changes;
401
+ return { versionsDeleted, ...gcBlobs(db) };
402
+ });
403
+ }
404
+ export function deletedHistoryStats(db: Database): {
405
+ notes: number;
406
+ versions: number;
407
+ bytes: number;
408
+ overflow_tombstones: number;
409
+ } {
410
+ return db
411
+ .prepare(
412
+ `SELECT COUNT(DISTINCT v.note_id) AS notes, COUNT(*) AS versions,
413
+ COALESCE(SUM(b.byte_size), 0) AS bytes,
414
+ COALESCE(SUM(CASE WHEN v.encoding = 'overflow' THEN 1 ELSE 0 END), 0) AS overflow_tombstones
415
+ FROM note_versions v LEFT JOIN notes n ON n.id = v.note_id
416
+ LEFT JOIN note_blobs b ON b.hash = v.content_hash WHERE n.id IS NULL`,
417
+ )
418
+ .get() as {
419
+ notes: number;
420
+ versions: number;
421
+ bytes: number;
422
+ overflow_tombstones: number;
423
+ };
424
+ }
425
+
426
+ export class HistoryDeltaOrphanError extends Error {
427
+ readonly code = "HISTORY_UNRECOVERABLE";
428
+ constructor(readonly hash: string, readonly delta_of: string | null, readonly reason: "unknown_encoding" | "base_missing" | "base_not_whole" | "bad_delta" | "identity_mismatch") {
429
+ super(`History delta cannot be reconstructed: ${reason}`);
430
+ this.name = "HistoryDeltaOrphanError";
431
+ }
432
+ }
433
+ interface BlobReadRow {
434
+ content: string | null;
435
+ blob_encoding: string | null;
436
+ delta_of: string | null;
437
+ }
438
+ function materialise(db: Database, hash: string, row: BlobReadRow): string | null {
439
+ if (row.content === null)
440
+ return null;
441
+ if (row.blob_encoding === null)
442
+ return row.content;
443
+ if (row.blob_encoding !== "fossil-delta")
444
+ throw new HistoryDeltaOrphanError(hash, row.delta_of, "unknown_encoding");
445
+ const base = row.delta_of ? db.prepare("SELECT content, encoding FROM note_blobs WHERE hash = ?").get(row.delta_of) as {
446
+ content: string;
447
+ encoding: string | null;
448
+ } | null : null;
449
+ if (!base)
450
+ throw new HistoryDeltaOrphanError(hash, row.delta_of, "base_missing");
451
+ if (base.encoding !== null)
452
+ throw new HistoryDeltaOrphanError(hash, row.delta_of, "base_not_whole");
453
+ let text: string;
454
+ try {
455
+ text = decodeDelta(base.content, row.content);
456
+ }
457
+ catch {
458
+ throw new HistoryDeltaOrphanError(hash, row.delta_of, "bad_delta");
459
+ }
460
+ if (hashContent(text) !== hash)
461
+ throw new HistoryDeltaOrphanError(hash, row.delta_of, "identity_mismatch");
462
+ return text;
463
+ }
464
+ export function readBlobContent(db: Database, hash: string): string | null {
465
+ const row = db.prepare("SELECT content, encoding AS blob_encoding, delta_of FROM note_blobs WHERE hash = ?").get(hash) as BlobReadRow | null;
466
+ return row ? materialise(db, hash, row) : null;
467
+ }
468
+ export function countNoteVersions(db: Database, noteId: string): number {
469
+ return (db.prepare("SELECT COUNT(*) AS n FROM note_versions WHERE note_id = ?").get(noteId) as {
470
+ n: number;
471
+ }).n;
472
+ }
473
+ /** Attribution: each directly referenced blob once, excluding indirect bases. */
474
+ export function noteHistoryBytes(db: Database, noteId: string): number {
475
+ return (db.prepare(`SELECT COALESCE(SUM(b.byte_size),0) AS n FROM
476
+ (SELECT DISTINCT content_hash FROM note_versions WHERE note_id = ? AND content_hash IS NOT NULL) v
477
+ JOIN note_blobs b ON b.hash = v.content_hash`).get(noteId) as {
478
+ n: number;
479
+ }).n;
480
+ }
481
+ export function historyStorageStats(db: Database) {
482
+ const stats = { whole_blobs: 0, whole_bytes: 0, delta_blobs: 0, delta_bytes: 0, orphan_deltas: 0, unknown_encoding_blobs: 0 };
483
+ if (!historyTablesPresent(db))
484
+ return stats;
485
+ const groups = db.prepare("SELECT encoding, COUNT(*) AS n, COALESCE(SUM(byte_size),0) AS bytes FROM note_blobs GROUP BY encoding").all() as {
486
+ encoding: string | null;
487
+ n: number;
488
+ bytes: number;
489
+ }[];
490
+ for (const g of groups) {
491
+ if (g.encoding === null) {
492
+ stats.whole_blobs = g.n;
493
+ stats.whole_bytes = g.bytes;
494
+ }
495
+ else if (g.encoding === "fossil-delta") {
496
+ stats.delta_blobs = g.n;
497
+ stats.delta_bytes = g.bytes;
498
+ }
499
+ else
500
+ stats.unknown_encoding_blobs += g.n;
501
+ }
502
+ stats.orphan_deltas = (db.prepare(`SELECT COUNT(*) AS n FROM note_blobs d WHERE d.encoding = 'fossil-delta'
503
+ AND (d.delta_of IS NULL OR NOT EXISTS (SELECT 1 FROM note_blobs b WHERE b.hash = d.delta_of AND b.encoding IS NULL))`).get() as {
504
+ n: number;
505
+ }).n;
506
+ return stats;
507
+ }
508
+ export function topNotesByHistoryBytes(db: Database, limit: number): {
509
+ note_id: string;
510
+ bytes: number;
511
+ versions: number;
512
+ }[] {
513
+ return db.prepare(`SELECT v.note_id, SUM(b.byte_size) AS bytes,
514
+ (SELECT COUNT(*) FROM note_versions n WHERE n.note_id = v.note_id) AS versions
515
+ FROM (SELECT DISTINCT note_id, content_hash FROM note_versions WHERE content_hash IS NOT NULL) v
516
+ JOIN note_blobs b ON b.hash = v.content_hash GROUP BY v.note_id ORDER BY bytes DESC LIMIT ?`).all(limit) as {
517
+ note_id: string;
518
+ bytes: number;
519
+ versions: number;
520
+ }[];
521
+ }
522
+ export interface CompactResult {
523
+ blobs_deltified: number;
524
+ blobs_skipped_too_large: number;
525
+ versions_dropped: number;
526
+ bytes_before: number;
527
+ bytes_after: number;
528
+ }
529
+ export interface CompactSummary {
530
+ notes_scanned: number;
531
+ notes_compacted: number;
532
+ notes_failed: number;
533
+ blobs_deltified: number;
534
+ versions_dropped: number;
535
+ bytes_before: number;
536
+ bytes_after: number;
537
+ remaining_candidates: number;
538
+ duration_ms: number;
539
+ stopped_by: "complete" | "budget" | "max_notes" | "disabled";
540
+ }
541
+ export function compactNote(db: Database, noteId: string, policy: HistoryPolicy): CompactResult {
542
+ const result: CompactResult = { blobs_deltified: 0, blobs_skipped_too_large: 0, versions_dropped: 0, bytes_before: 0, bytes_after: 0 };
543
+ if (!policy.enabled || !policy.compact_enabled || !historyTablesPresent(db))
544
+ return result;
545
+ const rows = db.prepare("SELECT version_ix, content_hash, content_len FROM note_versions WHERE note_id = ? AND content_hash IS NOT NULL ORDER BY version_ix DESC").all(noteId) as {
546
+ version_ix: number;
547
+ content_hash: string;
548
+ content_len: number;
549
+ }[];
550
+ const stored = noteHistoryBytes(db, noteId);
551
+ result.bytes_before = result.bytes_after = stored;
552
+ const current = db.prepare("SELECT content FROM notes WHERE id = ?").get(noteId) as {
553
+ content: string | null;
554
+ } | null;
555
+ const newest = current?.content == null
556
+ ? db.prepare("SELECT content_len FROM note_versions WHERE note_id = ? ORDER BY version_ix DESC LIMIT 1").get(noteId) as { content_len: number } | null
557
+ : null;
558
+ const live = current?.content == null ? newest?.content_len ?? 0 : byteLength(current.content);
559
+ const overBytes = policy.max_bytes_per_note !== null && stored > policy.max_bytes_per_note;
560
+ if (!overBytes && rows.length < policy.compact_min_versions)
561
+ return result;
562
+ if (!overBytes && stored <= policy.compact_ratio * Math.max(live, 1))
563
+ return result;
564
+ return transaction(db, () => {
565
+ for (let start = 0; start < rows.length; start += policy.compact_run_length) {
566
+ const run = rows.slice(start, start + policy.compact_run_length);
567
+ const baseRow = db.prepare("SELECT hash, encoding, delta_of FROM note_blobs WHERE hash = ?").get(run[0]!.content_hash) as {
568
+ hash: string;
569
+ encoding: string | null;
570
+ delta_of: string | null;
571
+ } | null;
572
+ if (!baseRow)
573
+ continue;
574
+ const baseHash = baseRow.encoding === null ? baseRow.hash : baseRow.delta_of;
575
+ if (!baseHash)
576
+ continue;
577
+ const base = db.prepare("SELECT encoding FROM note_blobs WHERE hash = ?").get(baseHash) as {
578
+ encoding: string | null;
579
+ } | null;
580
+ if (!base)
581
+ continue;
582
+ // Never propagate a pre-existing depth-two chain into healthy blobs.
583
+ if (base.encoding !== null)
584
+ throw new HistoryDeltaOrphanError(baseRow.hash, baseHash, "base_not_whole");
585
+ const text = readBlobContent(db, baseHash);
586
+ if (text === null)
587
+ continue;
588
+ for (const r of run.slice(1)) {
589
+ if (r.content_hash === baseHash)
590
+ continue;
591
+ const b = db.prepare("SELECT hash, content, byte_size, encoding FROM note_blobs WHERE hash = ?").get(r.content_hash) as {
592
+ hash: string;
593
+ content: string;
594
+ byte_size: number;
595
+ encoding: string | null;
596
+ } | null;
597
+ if (!b || b.encoding !== null)
598
+ continue;
599
+ if (db.prepare("SELECT 1 FROM note_blobs WHERE delta_of = ? LIMIT 1").get(b.hash))
600
+ continue;
601
+ const payload = encodeDelta(text, b.content);
602
+ if (payload.length >= b.byte_size * 0.9) {
603
+ result.blobs_skipped_too_large++;
604
+ continue;
605
+ }
606
+ result.blobs_deltified += db.prepare("UPDATE note_blobs SET content = ?, byte_size = ?, encoding = 'fossil-delta', delta_of = ? WHERE hash = ? AND encoding IS NULL").run(payload, payload.length, baseHash, b.hash).changes;
607
+ }
608
+ }
609
+ if (policy.max_bytes_per_note !== null)
610
+ result.versions_dropped = enforceByteCeiling(db, noteId, policy);
611
+ result.bytes_after = noteHistoryBytes(db, noteId);
612
+ return result;
613
+ });
614
+ }
615
+ function enforceByteCeiling(db: Database, noteId: string, policy: HistoryPolicy): number {
616
+ let dropped = 0;
617
+ while (policy.max_bytes_per_note !== null && noteHistoryBytes(db, noteId) > policy.max_bytes_per_note) {
618
+ const rows = db.prepare("SELECT version_ix, op, content_hash FROM note_versions WHERE note_id = ? ORDER BY version_ix ASC").all(noteId) as Pick<VersionRow, "version_ix" | "op" | "content_hash">[];
619
+ const victim = rows.find((r, i) => r.op !== "delete" && rows.length - 1 - i >= policy.min_versions);
620
+ if (!victim)
621
+ break;
622
+ dropped += db.prepare("DELETE FROM note_versions WHERE note_id = ? AND version_ix = ?").run(noteId, victim.version_ix).changes;
623
+ db.prepare("DELETE FROM note_blobs WHERE hash = ? AND NOT EXISTS (SELECT 1 FROM note_versions WHERE content_hash = ?) AND NOT EXISTS (SELECT 1 FROM note_blobs WHERE delta_of = ?)").run(victim.content_hash, victim.content_hash, victim.content_hash);
624
+ }
625
+ return dropped;
626
+ }
627
+ export function compactVault(db: Database, policy: HistoryPolicy, opts?: {
628
+ noteId?: string;
629
+ budgetMs?: number | null;
630
+ maxNotes?: number | null;
631
+ }): CompactSummary {
632
+ const result: CompactSummary = { notes_scanned: 0, notes_compacted: 0, notes_failed: 0, blobs_deltified: 0, versions_dropped: 0, bytes_before: 0, bytes_after: 0, remaining_candidates: 0, duration_ms: 0, stopped_by: "complete" };
633
+ if (!historyTablesPresent(db) || !policy.enabled || !policy.compact_enabled)
634
+ return { ...result, stopped_by: "disabled" };
635
+ const budget = opts?.budgetMs === undefined ? policy.compact_budget_ms : opts.budgetMs;
636
+ const max = opts?.maxNotes === undefined ? policy.compact_max_notes : opts.maxNotes;
637
+ if (max !== null && max <= 0)
638
+ return { ...result, stopped_by: "max_notes" };
639
+ if (budget !== null && budget <= 0)
640
+ return { ...result, stopped_by: "budget" };
641
+ const started = performance.now();
642
+ const candidates = opts?.noteId !== undefined ? [{ note_id: opts.noteId }] : db.prepare(`SELECT v.note_id, SUM(b.byte_size) AS stored,
643
+ (SELECT COUNT(*) FROM note_versions n WHERE n.note_id = v.note_id AND n.content_hash IS NOT NULL) AS versions,
644
+ COALESCE(LENGTH(CAST(live_note.content AS BLOB)),
645
+ (SELECT content_len FROM note_versions newest WHERE newest.note_id = v.note_id ORDER BY version_ix DESC LIMIT 1),0) AS live
646
+ FROM (SELECT DISTINCT note_id, content_hash FROM note_versions WHERE content_hash IS NOT NULL) v
647
+ JOIN note_blobs b ON b.hash = v.content_hash LEFT JOIN notes live_note ON live_note.id = v.note_id
648
+ GROUP BY v.note_id
649
+ HAVING (versions >= ? AND stored > ? * MAX(live,1)) OR (? IS NOT NULL AND stored > ?)
650
+ ORDER BY stored DESC`).all(policy.compact_min_versions, policy.compact_ratio, policy.max_bytes_per_note, policy.max_bytes_per_note) as {
651
+ note_id: string;
652
+ }[];
653
+ for (const candidate of candidates) {
654
+ if (max !== null && result.notes_scanned >= max) {
655
+ result.stopped_by = "max_notes";
656
+ break;
657
+ }
658
+ // Always attempt one candidate, even when the candidate scan used the budget.
659
+ if (result.notes_scanned > 0 && budget !== null && performance.now() - started >= budget) {
660
+ result.stopped_by = "budget";
661
+ break;
662
+ }
663
+ result.notes_scanned++;
664
+ try {
665
+ const r = compactNote(db, candidate.note_id, policy);
666
+ if (r.blobs_deltified || r.versions_dropped)
667
+ result.notes_compacted++;
668
+ result.blobs_deltified += r.blobs_deltified;
669
+ result.versions_dropped += r.versions_dropped;
670
+ result.bytes_before += r.bytes_before;
671
+ result.bytes_after += r.bytes_after;
672
+ }
673
+ catch (err) {
674
+ result.notes_failed++;
675
+ console.warn("[history] compaction failed", candidate.note_id, err);
676
+ }
677
+ }
678
+ result.remaining_candidates = candidates.length - result.notes_scanned;
679
+ result.duration_ms = performance.now() - started;
680
+ return result;
681
+ }