@openparachute/vault 0.7.9-rc.4 → 0.7.9-rc.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/core/src/compact.test.ts +248 -0
- package/core/src/delta.test.ts +49 -0
- package/core/src/delta.ts +21 -0
- package/core/src/doctor.ts +41 -1
- package/core/src/history-capture.test.ts +244 -0
- package/core/src/history-import.test.ts +82 -0
- package/core/src/history-import.ts +157 -0
- package/core/src/history.test.ts +229 -0
- package/core/src/history.ts +681 -0
- package/core/src/mcp-manifest.ts +17 -0
- package/core/src/mcp.ts +28 -3
- package/core/src/notes.ts +6 -1
- package/core/src/portable-md.ts +1 -1
- package/core/src/schema-v29-note-versions.test.ts +63 -0
- package/core/src/schema-v30-note-deltas.test.ts +51 -0
- package/core/src/schema-v31-history-import.test.ts +25 -0
- package/core/src/schema.ts +234 -1
- package/core/src/store.ts +103 -14
- package/core/src/types.ts +35 -2
- package/core/src/vendor/fossil-delta.ts +472 -0
- package/package.json +1 -1
- package/src/cli.ts +10 -1
- package/src/config.ts +42 -0
- package/src/history-import-mcp.test.ts +94 -0
- package/src/history-import-routes.test.ts +91 -0
- package/src/history-import.test.ts +318 -0
- package/src/history-import.ts +482 -0
- package/src/mcp-http.ts +13 -0
- package/src/mcp-tools.ts +18 -2
- package/src/mirror-config.ts +38 -0
- package/src/mirror-manager.ts +6 -0
- package/src/mirror-retirement.test.ts +79 -0
- package/src/mirror-routes.ts +12 -2
- package/src/routes.ts +226 -86
- package/src/routing.ts +8 -3
- package/src/server.ts +33 -14
- package/src/vault-compact-routes.test.ts +257 -0
- package/src/vault-history-routes.test.ts +562 -0
- package/src/vault-store.ts +13 -0
package/core/src/mcp-manifest.ts
CHANGED
|
@@ -208,6 +208,23 @@ Response shape (vault#550 — three variants, pick by what you passed):
|
|
|
208
208
|
required: ["note_id"],
|
|
209
209
|
description: "Scope results to notes within N hops of an anchor note",
|
|
210
210
|
},
|
|
211
|
+
versions: {
|
|
212
|
+
type: "object",
|
|
213
|
+
properties: {
|
|
214
|
+
note_id: { type: "string", description: "Note ID or path" },
|
|
215
|
+
version_ix: { type: "integer", minimum: 0 },
|
|
216
|
+
origin: { type: "string", enum: ["git-import"] },
|
|
217
|
+
import_ix: { type: "integer", minimum: 0 },
|
|
218
|
+
limit: { type: "integer", minimum: 0, maximum: 200 },
|
|
219
|
+
offset: { type: "integer", minimum: 0 },
|
|
220
|
+
},
|
|
221
|
+
required: ["note_id"],
|
|
222
|
+
oneOf: [
|
|
223
|
+
{ required: ["origin", "import_ix"], not: { required: ["version_ix"] } },
|
|
224
|
+
{ not: { anyOf: [{ required: ["origin"] }, { required: ["import_ix"] }] } },
|
|
225
|
+
],
|
|
226
|
+
description: "Read note history; mutually exclusive with id, search, near, cursor, aggregate, and semantic. Tag-scoped sessions see the SAME visibility enforcement as every other read — a note outside the token's tag scope answers `not_found`, and a deleted note's history is not readable by a scoped session at all. Deleted-note history is also unavailable to unscoped MCP sessions; restore is REST-only.",
|
|
227
|
+
},
|
|
211
228
|
sort: {
|
|
212
229
|
type: "string",
|
|
213
230
|
enum: ["asc", "desc"],
|
package/core/src/mcp.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { getImportedVersion, parseHistorySelector, projectHistoryRow } from "./history-import.js";
|
|
1
2
|
import { Database } from "bun:sqlite";
|
|
2
3
|
import type { Store, Note, QueryOpts, Attachment } from "./types.js";
|
|
3
4
|
import { transactionAsync } from "./txn.js";
|
|
@@ -184,7 +185,7 @@ function requireNoteReference(value: unknown): string {
|
|
|
184
185
|
* match required. Same [[wikilink]] semantics as `resolveWikilink`; exact
|
|
185
186
|
* id/path always wins first.
|
|
186
187
|
*/
|
|
187
|
-
function resolveNote(db: Database, idOrPath: string): Note | null {
|
|
188
|
+
export function resolveNote(db: Database, idOrPath: string): Note | null {
|
|
188
189
|
// Try ID match first (fast, indexed)
|
|
189
190
|
const byId = noteOps.getNote(db, idOrPath);
|
|
190
191
|
if (byId) return byId;
|
|
@@ -692,6 +693,30 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
|
|
|
692
693
|
// stays byte-identical to the no-pagination behavior.
|
|
693
694
|
const contentRange = parseContentRange(params.content_offset, params.content_length);
|
|
694
695
|
|
|
696
|
+
if (params.versions) {
|
|
697
|
+
for (const key of ["search", "near", "cursor", "aggregate", "semantic", "id"] as const) {
|
|
698
|
+
if (params[key] !== undefined) throw new QueryError(`versions is incompatible with ${key}`, "INVALID_QUERY", {
|
|
699
|
+
error_type: "invalid_query", field: "versions", hint: `drop ${key} when using versions`,
|
|
700
|
+
});
|
|
701
|
+
}
|
|
702
|
+
const v = params.versions as { note_id: string; version_ix?: number; origin?: unknown; import_ix?: unknown; limit?: number; offset?: number };
|
|
703
|
+
const selector = parseHistorySelector(v);
|
|
704
|
+
const note = requireNote(db, requireNoteReference(v.note_id));
|
|
705
|
+
if (selector && "import_ix" in selector) {
|
|
706
|
+
const version = getImportedVersion(db, note.id, selector.import_ix);
|
|
707
|
+
if (!version) return { error: "Imported version not found", error_type: "not_found", id: v.note_id, ...selector };
|
|
708
|
+
return projectHistoryRow(version);
|
|
709
|
+
}
|
|
710
|
+
if (selector && "version_ix" in selector) {
|
|
711
|
+
const version = await store.getNoteVersion(note.id, selector.version_ix);
|
|
712
|
+
if (!version) return { error: `Version not found: "${v.note_id}"@${v.version_ix}`, error_type: "not_found", id: v.note_id, version_ix: v.version_ix };
|
|
713
|
+
return version;
|
|
714
|
+
}
|
|
715
|
+
const versions = await store.listNoteVersions(note.id, { limit: Math.max(0, Math.min(v.limit ?? 50, 200)), offset: v.offset ?? 0 });
|
|
716
|
+
const total = await store.countNoteVersions(note.id);
|
|
717
|
+
return { versions: versions.map(projectHistoryRow), total };
|
|
718
|
+
}
|
|
719
|
+
|
|
695
720
|
// --- Single note by ID/path ---
|
|
696
721
|
if (params.id) {
|
|
697
722
|
const note = resolveNote(db, params.id as string);
|
|
@@ -2280,7 +2305,7 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
|
|
|
2280
2305
|
name: "delete-note",
|
|
2281
2306
|
execute: async (params) => {
|
|
2282
2307
|
const note = requireNote(db, requireNoteReference(params.id));
|
|
2283
|
-
await store.deleteNote(note.id);
|
|
2308
|
+
await store.deleteNote(note.id, { actor: writeActor, via: writeVia });
|
|
2284
2309
|
return { deleted: true, id: note.id };
|
|
2285
2310
|
},
|
|
2286
2311
|
},
|
|
@@ -2494,7 +2519,7 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
|
|
|
2494
2519
|
field: "new_name",
|
|
2495
2520
|
});
|
|
2496
2521
|
}
|
|
2497
|
-
const result = await store.renameTag(oldName, newName);
|
|
2522
|
+
const result = await store.renameTag(oldName, newName, { actor: writeActor ?? undefined, via: writeVia ?? undefined });
|
|
2498
2523
|
if ("error" in result) {
|
|
2499
2524
|
if (result.error === "not_found") {
|
|
2500
2525
|
throw structuredError(`rename-tag: tag "${oldName}" not found`, {
|
package/core/src/notes.ts
CHANGED
|
@@ -31,6 +31,7 @@ import {
|
|
|
31
31
|
type SearchMode,
|
|
32
32
|
} from "./search-query.js";
|
|
33
33
|
import { generateUlid } from "./ulid.js";
|
|
34
|
+
import { captureVersion, readPriorNoteRow, DEFAULT_HISTORY_POLICY, type HistoryPolicy } from "./history.js";
|
|
34
35
|
|
|
35
36
|
/**
|
|
36
37
|
* Write-attribution context (vault#298) — the two axes of provenance threaded
|
|
@@ -2390,7 +2391,7 @@ export type RenameTagResult =
|
|
|
2390
2391
|
* (the store wrapper) bust both `_tagHierarchy` and `_schemaConfig`
|
|
2391
2392
|
* after the cascade returns.
|
|
2392
2393
|
*/
|
|
2393
|
-
export function renameTag(db: Database, oldName: string, newName: string): RenameTagResult {
|
|
2394
|
+
export function renameTag(db: Database, oldName: string, newName: string, attr?: { actor?: string | null; via?: string | null }, policy: HistoryPolicy = DEFAULT_HISTORY_POLICY): RenameTagResult {
|
|
2394
2395
|
// Normalize the TARGET so a rename can never create a `#`-prefixed tag. The
|
|
2395
2396
|
// SOURCE (`oldName`) is left LITERAL on purpose — it's the transitional escape
|
|
2396
2397
|
// hatch that lets the `#legacy/*` → `legacy/*` data migration find the
|
|
@@ -2590,6 +2591,8 @@ export function renameTag(db: Database, oldName: string, newName: string): Renam
|
|
|
2590
2591
|
for (const row of candidates) {
|
|
2591
2592
|
const next = rewriteNoteBody(row.content, renames);
|
|
2592
2593
|
if (next === row.content) continue;
|
|
2594
|
+
const prior = readPriorNoteRow(db, row.id);
|
|
2595
|
+
if (prior) captureVersion(db, prior, { actor: attr?.actor ?? null, via: attr?.via ?? null, op: "tag-rename", policy });
|
|
2593
2596
|
updateStmt.run(next, now, nowMs, row.id);
|
|
2594
2597
|
notesRewritten++;
|
|
2595
2598
|
}
|
|
@@ -2612,6 +2615,8 @@ export function renameTag(db: Database, oldName: string, newName: string): Renam
|
|
|
2612
2615
|
for (const row of candidates) {
|
|
2613
2616
|
const next = rewriteTagConfigPath(row.path, renames);
|
|
2614
2617
|
if (next === row.path) continue;
|
|
2618
|
+
const prior = readPriorNoteRow(db, row.id);
|
|
2619
|
+
if (prior) captureVersion(db, prior, { actor: attr?.actor ?? null, via: attr?.via ?? null, op: "tag-rename", policy });
|
|
2615
2620
|
updateStmt.run(next, now, nowMs, row.id);
|
|
2616
2621
|
pathsRenamed++;
|
|
2617
2622
|
}
|
package/core/src/portable-md.ts
CHANGED
|
@@ -1961,7 +1961,7 @@ async function importVaultReplay(
|
|
|
1961
1961
|
const batch = await store.queryNotes({ sort: "asc", limit: EXPORT_BATCH_SIZE });
|
|
1962
1962
|
if (batch.length === 0) break;
|
|
1963
1963
|
for (const note of batch) {
|
|
1964
|
-
await store.deleteNote(note.id);
|
|
1964
|
+
await store.deleteNote(note.id, { captureHistory: false });
|
|
1965
1965
|
stats.notes_wiped++;
|
|
1966
1966
|
}
|
|
1967
1967
|
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { it, expect, beforeEach, afterEach } from "bun:test";
|
|
2
|
+
import { Database } from "bun:sqlite";
|
|
3
|
+
import { initSchema } from "./schema.js";
|
|
4
|
+
import { BunSqliteStore } from "./store.js";
|
|
5
|
+
let db: Database;
|
|
6
|
+
beforeEach(() => {
|
|
7
|
+
db = new Database(":memory:");
|
|
8
|
+
initSchema(db);
|
|
9
|
+
});
|
|
10
|
+
afterEach(() => db.close());
|
|
11
|
+
function dropHistory() {
|
|
12
|
+
db.exec(
|
|
13
|
+
"DROP TABLE IF EXISTS note_versions; DROP TABLE IF EXISTS note_blobs;",
|
|
14
|
+
);
|
|
15
|
+
}
|
|
16
|
+
it("P14a v28 upgrade creates both tables and three indexes without backfill", () => {
|
|
17
|
+
dropHistory();
|
|
18
|
+
db.prepare("UPDATE schema_version SET version=28").run();
|
|
19
|
+
initSchema(db);
|
|
20
|
+
expect(
|
|
21
|
+
db.prepare("SELECT MAX(version) AS version FROM schema_version").get(),
|
|
22
|
+
).toEqual({ version: 31 });
|
|
23
|
+
const tables = db
|
|
24
|
+
.prepare(
|
|
25
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name IN ('note_versions','note_blobs') ORDER BY name",
|
|
26
|
+
)
|
|
27
|
+
.all();
|
|
28
|
+
expect(tables).toEqual([{ name: "note_blobs" }, { name: "note_versions" }]);
|
|
29
|
+
expect(
|
|
30
|
+
db
|
|
31
|
+
.prepare(
|
|
32
|
+
"SELECT name FROM sqlite_master WHERE type='index' AND name IN ('idx_note_versions_note','idx_note_versions_hash','idx_note_versions_superseded')",
|
|
33
|
+
)
|
|
34
|
+
.all(),
|
|
35
|
+
).toHaveLength(3);
|
|
36
|
+
expect(db.prepare("SELECT COUNT(*) AS n FROM note_versions").get()).toEqual({
|
|
37
|
+
n: 0,
|
|
38
|
+
});
|
|
39
|
+
});
|
|
40
|
+
it("P14b repeated init is idempotent", () => {
|
|
41
|
+
initSchema(db);
|
|
42
|
+
initSchema(db);
|
|
43
|
+
expect(db.prepare("SELECT COUNT(*) AS n FROM note_versions").get()).toEqual({
|
|
44
|
+
n: 0,
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
it("P14c rerunning the v29 migration preserves captured rows", async () => {
|
|
48
|
+
const store = new BunSqliteStore(db);
|
|
49
|
+
const n = await store.createNote("a");
|
|
50
|
+
await store.updateNote(n.id, { content: "b" });
|
|
51
|
+
db.prepare("UPDATE schema_version SET version=28").run();
|
|
52
|
+
initSchema(db);
|
|
53
|
+
expect((await store.getNoteVersion(n.id, 0))!.content).toBe("a");
|
|
54
|
+
});
|
|
55
|
+
it("P14d missing tables before the first capture degrade to no history", async () => {
|
|
56
|
+
const store = new BunSqliteStore(db);
|
|
57
|
+
const n = await store.createNote("a");
|
|
58
|
+
// Drop before this handle has probed true. A drop after capture is an
|
|
59
|
+
// accepted throw because historyTablesPresent memoises true per handle.
|
|
60
|
+
dropHistory();
|
|
61
|
+
await store.updateNote(n.id, { content: "b" });
|
|
62
|
+
expect((await store.getNote(n.id))!.content).toBe("b");
|
|
63
|
+
});
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { test, expect } from "bun:test";
|
|
2
|
+
import { Database } from "bun:sqlite";
|
|
3
|
+
import { BunSqliteStore } from "./store.js";
|
|
4
|
+
import { initSchema, SCHEMA_VERSION } from "./schema.js";
|
|
5
|
+
test("P12 v29 opens losslessly, fresh and upgraded columns/indexes agree", async () => {
|
|
6
|
+
const db = new Database(":memory:"), fresh = new Database(":memory:");
|
|
7
|
+
try {
|
|
8
|
+
const store = new BunSqliteStore(db);
|
|
9
|
+
const a = await store.createNote("one"), b = await store.createNote("other");
|
|
10
|
+
await store.updateNote(a.id, { content: "two" });
|
|
11
|
+
await store.updateNote(a.id, { content: "three" });
|
|
12
|
+
await store.deleteNote(a.id);
|
|
13
|
+
await store.updateNote(b.id, { content: "changed" });
|
|
14
|
+
// Remove only v30 additions to recreate the actual v29 table shape.
|
|
15
|
+
db.exec("DROP INDEX IF EXISTS idx_note_blobs_delta_of");
|
|
16
|
+
const cols = (table: string) => db.prepare(`PRAGMA table_info(${table})`).all() as {
|
|
17
|
+
name: string;
|
|
18
|
+
}[];
|
|
19
|
+
if (cols("note_blobs").some(c => c.name === "delta_of"))
|
|
20
|
+
db.exec("ALTER TABLE note_blobs DROP COLUMN delta_of");
|
|
21
|
+
if (cols("note_blobs").some(c => c.name === "encoding"))
|
|
22
|
+
db.exec("ALTER TABLE note_blobs DROP COLUMN encoding");
|
|
23
|
+
if (cols("note_versions").some(c => c.name === "created_at"))
|
|
24
|
+
db.exec("ALTER TABLE note_versions DROP COLUMN created_at");
|
|
25
|
+
db.exec("UPDATE schema_version SET version=29");
|
|
26
|
+
const snap = () => ({ blobs: db.prepare("SELECT hash,content,byte_size FROM note_blobs ORDER BY hash").all(), versions: db.prepare("SELECT note_id,version_ix,content_hash,path,metadata,extension,superseded_at,actor,via,op,content_len,encoding FROM note_versions ORDER BY note_id,version_ix").all() });
|
|
27
|
+
const before = snap();
|
|
28
|
+
expect(() => initSchema(db)).not.toThrow();
|
|
29
|
+
expect(SCHEMA_VERSION).toBe(31);
|
|
30
|
+
expect(snap()).toEqual(before);
|
|
31
|
+
expect(db.prepare("SELECT hash FROM note_blobs WHERE delta_of IS NOT NULL OR encoding IS NOT NULL").all()).toEqual([]);
|
|
32
|
+
expect(db.prepare("SELECT note_id FROM note_versions WHERE created_at IS NOT NULL").all()).toEqual([]);
|
|
33
|
+
initSchema(fresh);
|
|
34
|
+
for (const table of ["note_blobs", "note_versions"]) {
|
|
35
|
+
expect(cols(table).map(c => c.name).sort()).toEqual((fresh.prepare(`PRAGMA table_info(${table})`).all() as {
|
|
36
|
+
name: string;
|
|
37
|
+
}[]).map(c => c.name).sort());
|
|
38
|
+
}
|
|
39
|
+
for (const handle of [db, fresh])
|
|
40
|
+
expect(handle.prepare("SELECT name FROM sqlite_master WHERE name='idx_note_blobs_delta_of'").get()).not.toBeNull();
|
|
41
|
+
expect(db.prepare("PRAGMA integrity_check").get()).toEqual({ integrity_check: "ok" });
|
|
42
|
+
expect(db.prepare("PRAGMA foreign_key_check").all()).toEqual([]);
|
|
43
|
+
initSchema(db);
|
|
44
|
+
expect(snap()).toEqual(before);
|
|
45
|
+
expect(() => db.exec("UPDATE note_blobs SET delta_of='nosuchhash'")).toThrow("FOREIGN KEY");
|
|
46
|
+
}
|
|
47
|
+
finally {
|
|
48
|
+
db.close();
|
|
49
|
+
fresh.close();
|
|
50
|
+
}
|
|
51
|
+
});
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { test, expect } from "bun:test";
|
|
2
|
+
import { Database } from "bun:sqlite";
|
|
3
|
+
import { SqliteStore } from "./store.js";
|
|
4
|
+
import { initSchema } from "./schema.js";
|
|
5
|
+
|
|
6
|
+
test("v30 to v31 retains native history and matches fresh import tables; reopening is idempotent", async () => {
|
|
7
|
+
const db = new Database(":memory:"), fresh = new Database(":memory:");
|
|
8
|
+
try {
|
|
9
|
+
const store = new SqliteStore(db), note = await store.createNote("before");
|
|
10
|
+
await store.updateNote(note.id, { content: "after" });
|
|
11
|
+
const tables = ["history_import_refs", "history_import_receipts", "history_import_runs"];
|
|
12
|
+
for (const table of tables) db.exec(`DROP TABLE ${table}`);
|
|
13
|
+
db.exec("UPDATE schema_version SET version=30");
|
|
14
|
+
const before = db.query("SELECT * FROM note_versions").all();
|
|
15
|
+
initSchema(db); initSchema(fresh); initSchema(db);
|
|
16
|
+
expect(db.query("SELECT MAX(version) AS version FROM schema_version").get()).toEqual({ version: 31 });
|
|
17
|
+
expect(db.query("SELECT * FROM note_versions").all()).toEqual(before);
|
|
18
|
+
for (const table of tables) {
|
|
19
|
+
expect(db.query(`PRAGMA table_info(${table})`).all()).toEqual(fresh.query(`PRAGMA table_info(${table})`).all());
|
|
20
|
+
expect(db.query(`SELECT * FROM ${table}`).all()).toEqual([]);
|
|
21
|
+
}
|
|
22
|
+
expect(db.query("PRAGMA foreign_key_check").all()).toEqual([]);
|
|
23
|
+
expect(db.query("PRAGMA integrity_check").get()).toEqual({ integrity_check: "ok" });
|
|
24
|
+
} finally { db.close(); fresh.close(); }
|
|
25
|
+
});
|
package/core/src/schema.ts
CHANGED
|
@@ -6,7 +6,7 @@ import { transaction } from "./txn.js";
|
|
|
6
6
|
import { timestampToMs } from "./cursor.js";
|
|
7
7
|
import { ensureRelationshipColumn } from "./wikilinks.js";
|
|
8
8
|
|
|
9
|
-
export const SCHEMA_VERSION =
|
|
9
|
+
export const SCHEMA_VERSION = 31;
|
|
10
10
|
|
|
11
11
|
/**
|
|
12
12
|
* Deterministic last-resort epoch for a note whose `updated_at` AND
|
|
@@ -142,6 +142,114 @@ CREATE TABLE IF NOT EXISTS note_vectors (
|
|
|
142
142
|
-- sweep (a model <> ? scan) the backfill/drain use to find pending work.
|
|
143
143
|
CREATE INDEX IF NOT EXISTS idx_note_vectors_stale ON note_vectors(model, content_hash);
|
|
144
144
|
|
|
145
|
+
-- note_blobs + note_versions (v29, vault#524): note-level version history.
|
|
146
|
+
--
|
|
147
|
+
-- STORAGE. A version row is skinny; the BODY lives once per distinct content,
|
|
148
|
+
-- keyed by its sha256, in note_blobs. A metadata-only edit on a 116 KB note
|
|
149
|
+
-- costs one ~300-byte version row and a no-op \`INSERT OR IGNORE\` on the blob —
|
|
150
|
+
-- the whole answer to "does every little update store an entirely new copy?"
|
|
151
|
+
-- Live \`notes.content\` stays on \`notes\`: no join on the hot read path.
|
|
152
|
+
--
|
|
153
|
+
-- encoding IS NULL FOR EVERY ORDINARY ROW IN v29. NULL means "content_hash
|
|
154
|
+
-- names a WHOLE blob". It exists so the PR-2 compactor can rewrite an old row
|
|
155
|
+
-- to reference a delta (setting a format marker) with no second migration and
|
|
156
|
+
-- no version-id change. The ONE non-NULL value v29 ever writes is the literal
|
|
157
|
+
-- 'overflow', on a delete tombstone for a note whose body exceeded
|
|
158
|
+
-- VERSION_MAX_BYTES: the row records that the note WAS deleted and how big it
|
|
159
|
+
-- was, with content_hash NULL because the bytes were never blobbed. A delete
|
|
160
|
+
-- must never be blocked by history (see history.ts captureVersion step 5).
|
|
161
|
+
-- A restore marker copied from that tombstone also retains overflow encoding
|
|
162
|
+
-- and a NULL hash, including after the note has been recreated.
|
|
163
|
+
--
|
|
164
|
+
-- content_hash IS NULLABLE, and it is nullable ONLY for that overflow
|
|
165
|
+
-- tombstone or its copied restore marker. Every consumer must filter NULL —
|
|
166
|
+
-- in particular gcBlobs's \`NOT IN (SELECT content_hash ...)\` MUST carry
|
|
167
|
+
-- \`WHERE content_hash IS NOT NULL\`, or one NULL in the subquery makes the
|
|
168
|
+
-- whole NOT IN evaluate to NULL and the sweep silently deletes nothing.
|
|
169
|
+
--
|
|
170
|
+
-- NO \`REFERENCES notes(id) ON DELETE CASCADE\` — the one deviation from
|
|
171
|
+
-- note_vectors above, deliberate. deleteNote writes a final op='delete' row and
|
|
172
|
+
-- the \`notes\` row then goes away; a cascade would delete the only surviving
|
|
173
|
+
-- copy at the instant it became the only copy. Deleted-note history is reaped
|
|
174
|
+
-- by the configurable sweep (history.deleted_retention_days) or an explicit
|
|
175
|
+
-- erase, never by the FK.
|
|
176
|
+
--
|
|
177
|
+
-- content_hash DOES have a real FK to note_blobs with the default NO ACTION,
|
|
178
|
+
-- so SQLite itself enforces the one invariant refcount GC protects: deleting a
|
|
179
|
+
-- still-referenced blob raises instead of silently orphaning a version.
|
|
180
|
+
-- foreign_keys is turned ON per connection in applyConnectionPragmas
|
|
181
|
+
-- (schema.ts:486) — but inside a SWALLOWING try/catch, so on a handle where
|
|
182
|
+
-- the pragma throws there is no FK enforcement at all and no signal. The
|
|
183
|
+
-- NOT EXISTS guard on every blob delete is therefore the PRIMARY defence and
|
|
184
|
+
-- the FK is the backstop, not the other way round (§4.6).
|
|
185
|
+
-- note_blobs.encoding IS NULL -> \`content\` is the literal body, \`byte_size\`
|
|
186
|
+
-- is its UTF-8 byte length, and \`hash\` = sha256(content). This is every row
|
|
187
|
+
-- v29 ever wrote.
|
|
188
|
+
-- note_blobs.encoding = 'fossil-delta' -> \`content\` is the BASE64 of a Fossil
|
|
189
|
+
-- delta that reconstructs this row's logical body from the body of the blob
|
|
190
|
+
-- named by \`delta_of\`; \`byte_size\` is the length of that base64 payload, i.e.
|
|
191
|
+
-- the bytes actually stored. \`hash\` is STILL sha256 of the LOGICAL body, so
|
|
192
|
+
-- every note_versions.content_hash reference and its FK stay valid and
|
|
193
|
+
-- content-addressed identity is never a delta hash (ClaudeJi R3).
|
|
194
|
+
-- The LOGICAL byte length is note_versions.content_len, which PR 1
|
|
195
|
+
-- denormalised for exactly this moment (PR 1 §2.1: "it must stay readable
|
|
196
|
+
-- after PR 2 deltifies the blob").
|
|
197
|
+
-- ANY OTHER \`encoding\` VALUE IS CORRUPTION. It is not a future format marker to
|
|
198
|
+
-- be tolerated: a read of such a row answers 409 (materialise's
|
|
199
|
+
-- "unknown_encoding" branch) and the doctor counts it (historyStorageStats's
|
|
200
|
+
-- unknown_encoding_blobs). A later format adds its value here AND to both.
|
|
201
|
+
--
|
|
202
|
+
-- DEPTH IS EXACTLY ONE, AND THAT IS AN INVARIANT, NOT A TUNING CHOICE.
|
|
203
|
+
-- delta_of MUST name a row whose \`encoding IS NULL\`. A delta is never the
|
|
204
|
+
-- base of another delta. Rebuilding any version is therefore ONE applyDelta,
|
|
205
|
+
-- measured p95 3.73 ms / p99 4.68 ms on a 116 KB note, against 38.98 ms /
|
|
206
|
+
-- 49.99 ms for a 24-link chain on the same note -- the chain sits ON the
|
|
207
|
+
-- 50 ms budget with no headroom. Star also confines corruption: a damaged
|
|
208
|
+
-- delta loses one version, not every older one. See §11.1 D1.
|
|
209
|
+
--
|
|
210
|
+
-- delta_of carries a REAL self-referential FK with the default NO ACTION, and
|
|
211
|
+
-- it IS enforced on a bun handle (probe E: deleting a referenced base raises
|
|
212
|
+
-- FOREIGN KEY constraint failed). It is the backstop; the NOT EXISTS guards in
|
|
213
|
+
-- pruneVersions and gcBlobs are the primary defence, exactly as for
|
|
214
|
+
-- content_hash in v29 (PR 1 §4.6).
|
|
215
|
+
--
|
|
216
|
+
-- note_versions.created_at (#735) is the captured note's OWN creation time,
|
|
217
|
+
-- copied from notes.created_at at capture. NULL on every pre-v30 row; a
|
|
218
|
+
-- deleted-note restore falls back to the tombstone's superseded_at when it is
|
|
219
|
+
-- NULL, which is precisely v29's behaviour.
|
|
220
|
+
CREATE TABLE IF NOT EXISTS note_blobs (
|
|
221
|
+
hash TEXT PRIMARY KEY,
|
|
222
|
+
content TEXT NOT NULL,
|
|
223
|
+
byte_size INTEGER NOT NULL,
|
|
224
|
+
encoding TEXT,
|
|
225
|
+
delta_of TEXT REFERENCES note_blobs(hash)
|
|
226
|
+
);
|
|
227
|
+
|
|
228
|
+
CREATE TABLE IF NOT EXISTS note_versions (
|
|
229
|
+
note_id TEXT NOT NULL,
|
|
230
|
+
version_ix INTEGER NOT NULL,
|
|
231
|
+
content_hash TEXT REFERENCES note_blobs(hash),
|
|
232
|
+
path TEXT,
|
|
233
|
+
metadata TEXT,
|
|
234
|
+
extension TEXT,
|
|
235
|
+
superseded_at TEXT NOT NULL,
|
|
236
|
+
actor TEXT,
|
|
237
|
+
via TEXT,
|
|
238
|
+
op TEXT NOT NULL,
|
|
239
|
+
content_len INTEGER NOT NULL,
|
|
240
|
+
encoding TEXT,
|
|
241
|
+
created_at TEXT,
|
|
242
|
+
PRIMARY KEY (note_id, version_ix)
|
|
243
|
+
);
|
|
244
|
+
-- list/get (newest first) + the prune walk.
|
|
245
|
+
CREATE INDEX IF NOT EXISTS idx_note_versions_note ON note_versions(note_id, version_ix DESC);
|
|
246
|
+
-- the age half of retention + the deleted-note sweep.
|
|
247
|
+
CREATE INDEX IF NOT EXISTS idx_note_versions_superseded ON note_versions(superseded_at);
|
|
248
|
+
-- refcount GC: "is any version still pointing at this blob?" This index is
|
|
249
|
+
-- what keeps gcBlobs's per-vault-open anti-join off a full table scan; the
|
|
250
|
+
-- boot sweep runs it on an opted-in vault's open path (§4.6).
|
|
251
|
+
CREATE INDEX IF NOT EXISTS idx_note_versions_hash ON note_versions(content_hash);
|
|
252
|
+
|
|
145
253
|
-- tag_schemas (v6) was retired in v14; description + fields lifted onto the
|
|
146
254
|
-- tags row directly. The CREATE TABLE was removed from SCHEMA_SQL after the
|
|
147
255
|
-- v14 data migration drops the table; existing v6+ vaults pick up the
|
|
@@ -629,6 +737,12 @@ export function initSchema(db: Database): void {
|
|
|
629
737
|
// queued an unresolved link, and does not rewrite notes.
|
|
630
738
|
migrateToV28(db);
|
|
631
739
|
|
|
740
|
+
// v29: note-level history, no backfill.
|
|
741
|
+
migrateToV29(db);
|
|
742
|
+
// v30: depth-one history deltas and captured creation time, no backfill.
|
|
743
|
+
migrateToV30(db);
|
|
744
|
+
migrateToV31(db);
|
|
745
|
+
|
|
632
746
|
// Rebuild any generated columns + indexes declared in indexed_fields.
|
|
633
747
|
// No-op for a fresh vault; idempotent on existing vaults.
|
|
634
748
|
rebuildIndexes(db);
|
|
@@ -1756,6 +1870,94 @@ function migrateToV28(db: Database): void {
|
|
|
1756
1870
|
ensureRelationshipColumn(db);
|
|
1757
1871
|
}
|
|
1758
1872
|
|
|
1873
|
+
/** Add history tables and indexes without moving existing note data. */
|
|
1874
|
+
function migrateToV29(db: Database): void {
|
|
1875
|
+
transaction(db, () => {
|
|
1876
|
+
db.exec(`
|
|
1877
|
+
-- note_blobs + note_versions (v29, vault#524): note-level version history.
|
|
1878
|
+
--
|
|
1879
|
+
-- STORAGE. A version row is skinny; the BODY lives once per distinct content,
|
|
1880
|
+
-- keyed by its sha256, in note_blobs. A metadata-only edit on a 116 KB note
|
|
1881
|
+
-- costs one ~300-byte version row and a no-op \`INSERT OR IGNORE\` on the blob —
|
|
1882
|
+
-- the whole answer to "does every little update store an entirely new copy?"
|
|
1883
|
+
-- Live \`notes.content\` stays on \`notes\`: no join on the hot read path.
|
|
1884
|
+
--
|
|
1885
|
+
-- encoding IS NULL FOR EVERY ORDINARY ROW IN v29. NULL means "content_hash
|
|
1886
|
+
-- names a WHOLE blob". It exists so the PR-2 compactor can rewrite an old row
|
|
1887
|
+
-- to reference a delta (setting a format marker) with no second migration and
|
|
1888
|
+
-- no version-id change. The ONE non-NULL value v29 ever writes is the literal
|
|
1889
|
+
-- 'overflow', on a delete tombstone for a note whose body exceeded
|
|
1890
|
+
-- VERSION_MAX_BYTES: the row records that the note WAS deleted and how big it
|
|
1891
|
+
-- was, with content_hash NULL because the bytes were never blobbed. A delete
|
|
1892
|
+
-- must never be blocked by history (see history.ts captureVersion step 5).
|
|
1893
|
+
-- A restore marker copied from that tombstone also retains overflow encoding
|
|
1894
|
+
-- and a NULL hash, including after the note has been recreated.
|
|
1895
|
+
--
|
|
1896
|
+
-- content_hash IS NULLABLE, and it is nullable ONLY for that overflow
|
|
1897
|
+
-- tombstone or its copied restore marker. Every consumer must filter NULL —
|
|
1898
|
+
-- in particular gcBlobs's \`NOT IN (SELECT content_hash ...)\` MUST carry
|
|
1899
|
+
-- \`WHERE content_hash IS NOT NULL\`, or one NULL in the subquery makes the
|
|
1900
|
+
-- whole NOT IN evaluate to NULL and the sweep silently deletes nothing.
|
|
1901
|
+
--
|
|
1902
|
+
-- NO \`REFERENCES notes(id) ON DELETE CASCADE\` — the one deviation from
|
|
1903
|
+
-- note_vectors above, deliberate. deleteNote writes a final op='delete' row and
|
|
1904
|
+
-- the \`notes\` row then goes away; a cascade would delete the only surviving
|
|
1905
|
+
-- copy at the instant it became the only copy. Deleted-note history is reaped
|
|
1906
|
+
-- by the configurable sweep (history.deleted_retention_days) or an explicit
|
|
1907
|
+
-- erase, never by the FK.
|
|
1908
|
+
--
|
|
1909
|
+
-- content_hash DOES have a real FK to note_blobs with the default NO ACTION,
|
|
1910
|
+
-- so SQLite itself enforces the one invariant refcount GC protects: deleting a
|
|
1911
|
+
-- still-referenced blob raises instead of silently orphaning a version.
|
|
1912
|
+
-- foreign_keys is turned ON per connection in applyConnectionPragmas
|
|
1913
|
+
-- (schema.ts:486) — but inside a SWALLOWING try/catch, so on a handle where
|
|
1914
|
+
-- the pragma throws there is no FK enforcement at all and no signal. The
|
|
1915
|
+
-- NOT EXISTS guard on every blob delete is therefore the PRIMARY defence and
|
|
1916
|
+
-- the FK is the backstop, not the other way round (§4.6).
|
|
1917
|
+
CREATE TABLE IF NOT EXISTS note_blobs (
|
|
1918
|
+
hash TEXT PRIMARY KEY,
|
|
1919
|
+
content TEXT NOT NULL,
|
|
1920
|
+
byte_size INTEGER NOT NULL
|
|
1921
|
+
);
|
|
1922
|
+
|
|
1923
|
+
CREATE TABLE IF NOT EXISTS note_versions (
|
|
1924
|
+
note_id TEXT NOT NULL,
|
|
1925
|
+
version_ix INTEGER NOT NULL,
|
|
1926
|
+
content_hash TEXT REFERENCES note_blobs(hash),
|
|
1927
|
+
path TEXT,
|
|
1928
|
+
metadata TEXT,
|
|
1929
|
+
extension TEXT,
|
|
1930
|
+
superseded_at TEXT NOT NULL,
|
|
1931
|
+
actor TEXT,
|
|
1932
|
+
via TEXT,
|
|
1933
|
+
op TEXT NOT NULL,
|
|
1934
|
+
content_len INTEGER NOT NULL,
|
|
1935
|
+
encoding TEXT,
|
|
1936
|
+
PRIMARY KEY (note_id, version_ix)
|
|
1937
|
+
);
|
|
1938
|
+
-- list/get (newest first) + the prune walk.
|
|
1939
|
+
CREATE INDEX IF NOT EXISTS idx_note_versions_note ON note_versions(note_id, version_ix DESC);
|
|
1940
|
+
-- the age half of retention + the deleted-note sweep.
|
|
1941
|
+
CREATE INDEX IF NOT EXISTS idx_note_versions_superseded ON note_versions(superseded_at);
|
|
1942
|
+
-- refcount GC: "is any version still pointing at this blob?" This index is
|
|
1943
|
+
-- what keeps gcBlobs's per-vault-open anti-join off a full table scan; the
|
|
1944
|
+
-- boot sweep runs it on an opted-in vault's open path (§4.6).
|
|
1945
|
+
CREATE INDEX IF NOT EXISTS idx_note_versions_hash ON note_versions(content_hash);
|
|
1946
|
+
`);
|
|
1947
|
+
});
|
|
1948
|
+
}
|
|
1949
|
+
|
|
1950
|
+
/** Add delta representation and creation time without moving prior rows. */
|
|
1951
|
+
function migrateToV30(db: Database): void {
|
|
1952
|
+
transaction(db, () => {
|
|
1953
|
+
if (hasTable(db, "note_blobs") && !hasColumn(db, "note_blobs", "encoding")) db.exec("ALTER TABLE note_blobs ADD COLUMN encoding TEXT");
|
|
1954
|
+
if (hasTable(db, "note_blobs") && !hasColumn(db, "note_blobs", "delta_of")) db.exec("ALTER TABLE note_blobs ADD COLUMN delta_of TEXT REFERENCES note_blobs(hash)");
|
|
1955
|
+
if (hasTable(db, "note_versions") && !hasColumn(db, "note_versions", "created_at")) db.exec("ALTER TABLE note_versions ADD COLUMN created_at TEXT");
|
|
1956
|
+
// SCHEMA_SQL runs before migrations: the index must be created here.
|
|
1957
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_note_blobs_delta_of ON note_blobs(delta_of)");
|
|
1958
|
+
});
|
|
1959
|
+
}
|
|
1960
|
+
|
|
1759
1961
|
function hasTable(db: Database, name: string): boolean {
|
|
1760
1962
|
const row = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(name);
|
|
1761
1963
|
return !!row;
|
|
@@ -1831,3 +2033,34 @@ function migrateFromV2(db: Database): void {
|
|
|
1831
2033
|
// Re-enable FK checks
|
|
1832
2034
|
db.exec("PRAGMA foreign_keys = ON");
|
|
1833
2035
|
}
|
|
2036
|
+
|
|
2037
|
+
/** Import receipts deliberately survive version retention and erasure. */
|
|
2038
|
+
function migrateToV31(db: Database): void {
|
|
2039
|
+
transaction(db, () => db.exec(`
|
|
2040
|
+
CREATE TABLE IF NOT EXISTS history_import_runs (
|
|
2041
|
+
run_id TEXT PRIMARY KEY,
|
|
2042
|
+
source_fingerprint TEXT NOT NULL,
|
|
2043
|
+
tip TEXT NOT NULL,
|
|
2044
|
+
options_digest TEXT NOT NULL,
|
|
2045
|
+
status TEXT NOT NULL CHECK(status IN ('applying', 'complete')),
|
|
2046
|
+
created_at TEXT NOT NULL
|
|
2047
|
+
);
|
|
2048
|
+
CREATE TABLE IF NOT EXISTS history_import_receipts (
|
|
2049
|
+
note_id TEXT PRIMARY KEY,
|
|
2050
|
+
run_id TEXT NOT NULL REFERENCES history_import_runs(run_id),
|
|
2051
|
+
state_digest TEXT NOT NULL,
|
|
2052
|
+
imported_count INTEGER NOT NULL,
|
|
2053
|
+
retained_count INTEGER NOT NULL,
|
|
2054
|
+
pruned_imported INTEGER NOT NULL,
|
|
2055
|
+
pruned_native INTEGER NOT NULL,
|
|
2056
|
+
completed_at TEXT NOT NULL
|
|
2057
|
+
);
|
|
2058
|
+
CREATE TABLE IF NOT EXISTS history_import_refs (
|
|
2059
|
+
note_id TEXT NOT NULL REFERENCES history_import_receipts(note_id),
|
|
2060
|
+
import_ix INTEGER NOT NULL CHECK(import_ix >= 0),
|
|
2061
|
+
source_commit TEXT NOT NULL,
|
|
2062
|
+
source_blob TEXT NOT NULL,
|
|
2063
|
+
PRIMARY KEY(note_id, import_ix)
|
|
2064
|
+
);
|
|
2065
|
+
`));
|
|
2066
|
+
}
|