@openparachute/vault 0.7.1 → 0.7.2-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/src/core.test.ts +70 -4
- package/core/src/cursor-keyset-ms.test.ts +537 -0
- package/core/src/cursor.ts +76 -6
- package/core/src/hooks.ts +9 -0
- package/core/src/mcp.ts +12 -2
- package/core/src/notes.ts +175 -50
- package/core/src/paths.ts +4 -0
- package/core/src/portable-md.test.ts +161 -0
- package/core/src/portable-md.ts +140 -9
- package/core/src/schema.ts +128 -2
- package/core/src/seed-packs.ts +39 -13
- package/core/src/store.ts +12 -2
- package/core/src/txn.test.ts +100 -4
- package/core/src/txn.ts +119 -24
- package/package.json +1 -1
- package/src/cli.ts +6 -0
- package/src/mirror-import.ts +5 -0
- package/src/routes.ts +27 -2
- package/src/vault.test.ts +88 -0
package/core/src/core.test.ts
CHANGED
|
@@ -8,7 +8,7 @@ import { traverseLinks } from "./links.js";
|
|
|
8
8
|
import * as indexedFieldOps from "./indexed-fields.js";
|
|
9
9
|
import { resolveLinkTarget } from "./wikilinks.js";
|
|
10
10
|
import { generateUlid, ULID_REGEX } from "./ulid.js";
|
|
11
|
-
import { getVaultMap, extractH1Title, findNotesByTitle, getNoteByTitle } from "./notes.js";
|
|
11
|
+
import { getVaultMap, extractH1Title, findNotesByTitle, getNoteByTitle, validatePath, PathValidationError } from "./notes.js";
|
|
12
12
|
|
|
13
13
|
let store: SqliteStore;
|
|
14
14
|
let db: Database;
|
|
@@ -663,9 +663,13 @@ describe("renameTag cascade (vault#240 + #247)", async () => {
|
|
|
663
663
|
// vault#555 fix 2 — a rename cascade's inline content rewrite is a real
|
|
664
664
|
// persisted-state change; `updated_at` must move or a cursor/sync-poll
|
|
665
665
|
// loop never sees the rewritten note.
|
|
666
|
-
it("1b. content rewrite bumps updated_at (vault#555)", async () => {
|
|
666
|
+
it("1b. content rewrite bumps updated_at (vault#555) AND updated_at_ms (vault#586)", async () => {
|
|
667
667
|
const note = await store.createNote("Today's #task is important.", { tags: ["task"] });
|
|
668
668
|
const before = note.updatedAt;
|
|
669
|
+
const msOf = (id: string) =>
|
|
670
|
+
(db.prepare("SELECT updated_at_ms FROM notes WHERE id = ?").get(id) as { updated_at_ms: number })
|
|
671
|
+
.updated_at_ms;
|
|
672
|
+
const beforeMs = msOf(note.id);
|
|
669
673
|
await new Promise((r) => setTimeout(r, 5));
|
|
670
674
|
|
|
671
675
|
await store.renameTag("task", "todo");
|
|
@@ -674,6 +678,12 @@ describe("renameTag cascade (vault#240 + #247)", async () => {
|
|
|
674
678
|
expect(fresh!.content).toContain("#todo");
|
|
675
679
|
expect(fresh!.updatedAt).not.toBe(before);
|
|
676
680
|
expect(new Date(fresh!.updatedAt) > new Date(before)).toBe(true);
|
|
681
|
+
// vault#586: the integer keyset key must move in lockstep with updated_at —
|
|
682
|
+
// else a cursor sync-poll never surfaces the rewritten note. Pins the
|
|
683
|
+
// renameTag content-cascade ms hunk (a revert leaves ms stale → RED).
|
|
684
|
+
const afterMs = msOf(note.id);
|
|
685
|
+
expect(afterMs).toBeGreaterThan(beforeMs);
|
|
686
|
+
expect(afterMs).toBe(Date.parse(fresh!.updatedAt!));
|
|
677
687
|
});
|
|
678
688
|
|
|
679
689
|
it("2. cascades sub-tags recursively (task → todo, task/work → todo/work, task/work/client → todo/work/client)", async () => {
|
|
@@ -1543,7 +1553,16 @@ describe("queryNotes", async () => {
|
|
|
1543
1553
|
// Helper: pin a note's updated_at to a known value so cursor math
|
|
1544
1554
|
// doesn't race wall-clock writes from the test harness.
|
|
1545
1555
|
function pinUpdatedAt(id: string, iso: string) {
|
|
1546
|
-
|
|
1556
|
+
// Mirror production: EVERY write maintains updated_at_ms in lockstep with
|
|
1557
|
+
// updated_at (vault#586 — the integer column is the keyset-ordering
|
|
1558
|
+
// authority). A helper that pinned only the string would leave a stale ms
|
|
1559
|
+
// and mis-order the cursor. The pinned values here are canonical `...Z`,
|
|
1560
|
+
// so `Date.parse` is exact.
|
|
1561
|
+
db.prepare("UPDATE notes SET updated_at = ?, updated_at_ms = ? WHERE id = ?").run(
|
|
1562
|
+
iso,
|
|
1563
|
+
Date.parse(iso),
|
|
1564
|
+
id,
|
|
1565
|
+
);
|
|
1547
1566
|
}
|
|
1548
1567
|
|
|
1549
1568
|
it("first call returns notes + a next_cursor string", async () => {
|
|
@@ -1790,7 +1809,12 @@ describe("queryNotes", async () => {
|
|
|
1790
1809
|
|
|
1791
1810
|
describe("ULID ids for new notes (existing IDs unchanged)", () => {
|
|
1792
1811
|
function pinUpdatedAt(id: string, iso: string) {
|
|
1793
|
-
|
|
1812
|
+
// Keep updated_at_ms in lockstep — see the sibling helper's note (vault#586).
|
|
1813
|
+
db.prepare("UPDATE notes SET updated_at = ?, updated_at_ms = ? WHERE id = ?").run(
|
|
1814
|
+
iso,
|
|
1815
|
+
Date.parse(iso),
|
|
1816
|
+
id,
|
|
1817
|
+
);
|
|
1794
1818
|
}
|
|
1795
1819
|
|
|
1796
1820
|
it("new notes get ULID-format ids (26-char Crockford base32)", async () => {
|
|
@@ -7652,3 +7676,45 @@ describe("vault projection (vault#271)", async () => {
|
|
|
7652
7676
|
expect(approxTokens).toBeLessThan(5000);
|
|
7653
7677
|
});
|
|
7654
7678
|
});
|
|
7679
|
+
|
|
7680
|
+
// ---- Path write-validation (vault#589 / FIX 2) ----
|
|
7681
|
+
|
|
7682
|
+
describe("validatePath — reject NUL / '..' at the write surface", () => {
|
|
7683
|
+
const NUL = String.fromCharCode(0);
|
|
7684
|
+
|
|
7685
|
+
it("rejects a NUL byte in the path", () => {
|
|
7686
|
+
expect(() => validatePath(`bad${NUL}path`)).toThrow(PathValidationError);
|
|
7687
|
+
try {
|
|
7688
|
+
validatePath(`x${NUL}y`);
|
|
7689
|
+
throw new Error("expected validatePath to throw");
|
|
7690
|
+
} catch (e: any) {
|
|
7691
|
+
expect(e.code).toBe("INVALID_PATH");
|
|
7692
|
+
expect(e.error_type).toBe("invalid_path");
|
|
7693
|
+
expect(e.reason).toMatch(/NUL/);
|
|
7694
|
+
}
|
|
7695
|
+
});
|
|
7696
|
+
|
|
7697
|
+
it("rejects a '..' path segment (traversal-shaped)", () => {
|
|
7698
|
+
expect(() => validatePath("..")).toThrow(PathValidationError);
|
|
7699
|
+
expect(() => validatePath("../secrets")).toThrow(PathValidationError);
|
|
7700
|
+
expect(() => validatePath("a/../b")).toThrow(PathValidationError);
|
|
7701
|
+
expect(() => validatePath("a/..")).toThrow(PathValidationError);
|
|
7702
|
+
// Backslash form normalizes to '/', so it's caught too.
|
|
7703
|
+
expect(() => validatePath("a\\..\\b")).toThrow(PathValidationError);
|
|
7704
|
+
});
|
|
7705
|
+
|
|
7706
|
+
it("accepts legitimate paths, including ones that merely contain dots", () => {
|
|
7707
|
+
expect(() => validatePath("Projects/Parachute/README")).not.toThrow();
|
|
7708
|
+
expect(() => validatePath("notes/2026-07-09")).not.toThrow();
|
|
7709
|
+
expect(() => validatePath("weird...name")).not.toThrow(); // three literal dots, not a '..' segment
|
|
7710
|
+
expect(() => validatePath("a/.hidden/b")).not.toThrow(); // single-dot-prefixed segment is fine
|
|
7711
|
+
expect(() => validatePath("with spaces and : colon?")).not.toThrow(); // FORBIDDEN_CHARS stay un-enforced at write
|
|
7712
|
+
});
|
|
7713
|
+
|
|
7714
|
+
it("treats a null / undefined / empty path as valid (notes need no path)", () => {
|
|
7715
|
+
expect(() => validatePath(null)).not.toThrow();
|
|
7716
|
+
expect(() => validatePath(undefined)).not.toThrow();
|
|
7717
|
+
expect(() => validatePath("")).not.toThrow();
|
|
7718
|
+
expect(() => validatePath(" ")).not.toThrow(); // normalizes to null
|
|
7719
|
+
});
|
|
7720
|
+
});
|
|
@@ -0,0 +1,537 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cursor keyset ordering on the integer `updated_at_ms` column (schema v26,
|
|
3
|
+
* vault#586).
|
|
4
|
+
*
|
|
5
|
+
* THE BUG this closes: the `(updated_at, id)` cursor keyset used THREE
|
|
6
|
+
* inconsistent orderings of `updated_at` that agree ONLY when every timestamp
|
|
7
|
+
* is canonical `.toISOString()` output —
|
|
8
|
+
* 1. walk order = TEXT-lexicographic `ORDER BY n.updated_at ASC, n.id ASC`,
|
|
9
|
+
* 2. boundary = a canonical-ISO string compared as TEXT (`n.updated_at > ?`),
|
|
10
|
+
* 3. watermark = `Date.parse(...)` millis, which read space-form timestamps
|
|
11
|
+
* in LOCAL time and THREW on unparseable strings.
|
|
12
|
+
* Import stores frontmatter timestamps VERBATIM, so aged/imported vaults carry
|
|
13
|
+
* non-canonical `updated_at` (`2024-11-02 14:30:00` space-form, `+02:00`
|
|
14
|
+
* offset, no-`Z`) — under which the three diverge and cursor pagination
|
|
15
|
+
* silently SKIPPED notes, re-delivered offset rows in an infinite loop, or
|
|
16
|
+
* 400'd the whole walk. v26 introduces an integer `updated_at_ms` column that
|
|
17
|
+
* makes walk-order, boundary, and watermark ONE numeric ordering.
|
|
18
|
+
*
|
|
19
|
+
* Mirrors the real failure: non-canonical values are injected via direct
|
|
20
|
+
* sqlite UPDATE / seeded into a pre-v26 table (that's how they land in prod via
|
|
21
|
+
* import), and the v26 backfill derives `updated_at_ms` UTC-correctly.
|
|
22
|
+
*/
|
|
23
|
+
import { describe, it, expect } from "bun:test";
|
|
24
|
+
import { Database } from "bun:sqlite";
|
|
25
|
+
import { initSchema, SCHEMA_VERSION } from "./schema.js";
|
|
26
|
+
import { timestampToMs, decodeCursor } from "./cursor.js";
|
|
27
|
+
import { SqliteStore } from "./store.js";
|
|
28
|
+
import * as noteOps from "./notes.js";
|
|
29
|
+
|
|
30
|
+
/** Raw read of a note's integer keyset key (not on the Note wire shape). */
|
|
31
|
+
function msOf(db: Database, id: string): number | null {
|
|
32
|
+
return (
|
|
33
|
+
db.prepare("SELECT updated_at_ms FROM notes WHERE id = ?").get(id) as {
|
|
34
|
+
updated_at_ms: number | null;
|
|
35
|
+
}
|
|
36
|
+
).updated_at_ms;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
// timestampToMs — the ONE UTC-correct parser feeding the backfill + writes.
|
|
41
|
+
// ---------------------------------------------------------------------------
|
|
42
|
+
describe("timestampToMs — UTC-correct, never-throws (vault#586)", () => {
|
|
43
|
+
it("canonical `.000Z` round-trips to its epoch", () => {
|
|
44
|
+
expect(timestampToMs("2024-11-02T14:30:00.000Z")).toBe(Date.UTC(2024, 10, 2, 14, 30, 0, 0));
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
it("space-separated `YYYY-MM-DD HH:MM:SS` is read as UTC — NOT shifted by local tz (the live bug)", () => {
|
|
48
|
+
const ms = timestampToMs("2024-11-02 14:30:00");
|
|
49
|
+
// The correct UTC answer, independent of the host timezone.
|
|
50
|
+
expect(ms).toBe(Date.UTC(2024, 10, 2, 14, 30, 0));
|
|
51
|
+
// And identical to the explicit-Z form — proving the space separator did
|
|
52
|
+
// NOT trigger a local-time reinterpretation.
|
|
53
|
+
expect(ms).toBe(timestampToMs("2024-11-02T14:30:00Z"));
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it("zone-less ISO `YYYY-MM-DDTHH:MM:SS` (no Z) is read as UTC, not local", () => {
|
|
57
|
+
expect(timestampToMs("2024-05-05T06:30:00")).toBe(Date.UTC(2024, 4, 5, 6, 30, 0));
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it("explicit `+HH:MM` / `-HH:MM` offsets shift to UTC", () => {
|
|
61
|
+
// 10:00 at +02:00 == 08:00Z; 10:00 at -05:00 == 15:00Z.
|
|
62
|
+
expect(timestampToMs("2024-04-04T10:00:00+02:00")).toBe(Date.UTC(2024, 3, 4, 8, 0, 0));
|
|
63
|
+
expect(timestampToMs("2024-04-04T10:00:00-05:00")).toBe(Date.UTC(2024, 3, 4, 15, 0, 0));
|
|
64
|
+
// Colon-less offset spelling too.
|
|
65
|
+
expect(timestampToMs("2024-04-04T10:00:00+0200")).toBe(Date.UTC(2024, 3, 4, 8, 0, 0));
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
it("fractional seconds truncate (floor) to millisecond precision", () => {
|
|
69
|
+
expect(timestampToMs("2024-01-01T00:00:00.5Z")).toBe(Date.UTC(2024, 0, 1, 0, 0, 0, 500));
|
|
70
|
+
expect(timestampToMs("2024-01-01T00:00:00.123456Z")).toBe(Date.UTC(2024, 0, 1, 0, 0, 0, 123));
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it("second- and minute-precision (no seconds) and date-only forms parse as UTC", () => {
|
|
74
|
+
expect(timestampToMs("2024-11-02")).toBe(Date.UTC(2024, 10, 2, 0, 0, 0));
|
|
75
|
+
expect(timestampToMs("2024-11-02T14:30")).toBe(Date.UTC(2024, 10, 2, 14, 30, 0));
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it("genuinely unparseable / empty / non-string → null (caller supplies a fallback, no throw)", () => {
|
|
79
|
+
expect(timestampToMs("not a timestamp")).toBeNull();
|
|
80
|
+
expect(timestampToMs("")).toBeNull();
|
|
81
|
+
expect(timestampToMs(" ")).toBeNull();
|
|
82
|
+
expect(timestampToMs(null)).toBeNull();
|
|
83
|
+
expect(timestampToMs(undefined)).toBeNull();
|
|
84
|
+
// Never throws, whatever the input.
|
|
85
|
+
expect(() => timestampToMs("2024-13-99 99:99:99xyz")).not.toThrow();
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
// ---------------------------------------------------------------------------
|
|
90
|
+
// Fixture: a pre-v26 vault — the notes table WITHOUT `updated_at_ms`, seeded
|
|
91
|
+
// with a matrix of `updated_at` shapes BEFORE the migration runs (exactly the
|
|
92
|
+
// aged/imported-vault upgrade scenario). `initSchema` drives the whole chain
|
|
93
|
+
// up through migrateToV26, which adds the column and backfills it.
|
|
94
|
+
// ---------------------------------------------------------------------------
|
|
95
|
+
interface SeedRow {
|
|
96
|
+
id: string;
|
|
97
|
+
updated_at: string;
|
|
98
|
+
created_at: string;
|
|
99
|
+
/** Expected backfilled updated_at_ms after migrateToV26. */
|
|
100
|
+
expectedMs: number;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// A deliberate spread of shapes + mixed ULID / legacy-timestamp ids, plus a
|
|
104
|
+
// same-millisecond cluster, an unparseable-updated_at row (falls back to
|
|
105
|
+
// created_at), and a both-unparseable row (falls back to the 0 sentinel).
|
|
106
|
+
const SEED: SeedRow[] = [
|
|
107
|
+
// canonical Z (ULID id)
|
|
108
|
+
{ id: "01J0AAAAAAAAAAAAAAAAAAAAAA", updated_at: "2024-01-01T00:00:00.000Z", created_at: "2024-01-01T00:00:00.000Z", expectedMs: Date.UTC(2024, 0, 1) },
|
|
109
|
+
// second-precision Z, no millis (legacy-format id)
|
|
110
|
+
{ id: "2024-02-02-12-00-00-000000", updated_at: "2024-02-02T12:00:00Z", created_at: "2024-02-02T12:00:00Z", expectedMs: Date.UTC(2024, 1, 2, 12) },
|
|
111
|
+
// SPACE-separated, no zone — the live bug shape
|
|
112
|
+
{ id: "note-space", updated_at: "2024-03-03 08:15:00", created_at: "2024-03-03T08:15:00.000Z", expectedMs: Date.UTC(2024, 2, 3, 8, 15) },
|
|
113
|
+
// +02:00 offset
|
|
114
|
+
{ id: "note-offset-plus", updated_at: "2024-04-04T10:00:00+02:00", created_at: "2024-04-04T08:00:00.000Z", expectedMs: Date.UTC(2024, 3, 4, 8) },
|
|
115
|
+
// -05:00 offset
|
|
116
|
+
{ id: "note-offset-minus", updated_at: "2024-04-04T10:00:00-05:00", created_at: "2024-04-04T15:00:00.000Z", expectedMs: Date.UTC(2024, 3, 4, 15) },
|
|
117
|
+
// zone-less T form
|
|
118
|
+
{ id: "note-zoneless", updated_at: "2024-05-05T06:30:00", created_at: "2024-05-05T06:30:00.000Z", expectedMs: Date.UTC(2024, 4, 5, 6, 30) },
|
|
119
|
+
// same-millisecond cluster (three rows share updated_at → same ms; id breaks the tie)
|
|
120
|
+
{ id: "cluster-c", updated_at: "2024-06-06T00:00:00.000Z", created_at: "2024-06-06T00:00:00.000Z", expectedMs: Date.UTC(2024, 5, 6) },
|
|
121
|
+
{ id: "cluster-a", updated_at: "2024-06-06T00:00:00.000Z", created_at: "2024-06-06T00:00:00.000Z", expectedMs: Date.UTC(2024, 5, 6) },
|
|
122
|
+
{ id: "cluster-b", updated_at: "2024-06-06T00:00:00.000Z", created_at: "2024-06-06T00:00:00.000Z", expectedMs: Date.UTC(2024, 5, 6) },
|
|
123
|
+
// historically-unparseable updated_at → falls back to created_at's ms
|
|
124
|
+
{ id: "note-unparseable", updated_at: "not-a-real-timestamp", created_at: "2024-07-07T00:00:00.000Z", expectedMs: Date.UTC(2024, 6, 7) },
|
|
125
|
+
// BOTH unparseable → the stable 0 sentinel (never NULL, never a throw)
|
|
126
|
+
{ id: "note-both-bad", updated_at: "garbage", created_at: "also-garbage", expectedMs: 0 },
|
|
127
|
+
];
|
|
128
|
+
|
|
129
|
+
function buildPreV26Vault(): Database {
|
|
130
|
+
const db = new Database(":memory:");
|
|
131
|
+
// The pre-v26 notes shape: everything v25 had EXCEPT `updated_at_ms`.
|
|
132
|
+
db.exec(`
|
|
133
|
+
CREATE TABLE notes (
|
|
134
|
+
id TEXT PRIMARY KEY,
|
|
135
|
+
content TEXT DEFAULT '',
|
|
136
|
+
path TEXT,
|
|
137
|
+
metadata TEXT DEFAULT '{}',
|
|
138
|
+
created_at TEXT NOT NULL,
|
|
139
|
+
updated_at TEXT,
|
|
140
|
+
extension TEXT NOT NULL DEFAULT 'md',
|
|
141
|
+
created_by TEXT,
|
|
142
|
+
created_via TEXT,
|
|
143
|
+
last_updated_by TEXT,
|
|
144
|
+
last_updated_via TEXT
|
|
145
|
+
);
|
|
146
|
+
CREATE TABLE schema_version (version INTEGER PRIMARY KEY, applied_at TEXT);
|
|
147
|
+
INSERT INTO schema_version (version, applied_at) VALUES (25, '2026-01-01T00:00:00.000Z');
|
|
148
|
+
`);
|
|
149
|
+
const insert = db.prepare(
|
|
150
|
+
"INSERT INTO notes (id, content, created_at, updated_at) VALUES (?, ?, ?, ?)",
|
|
151
|
+
);
|
|
152
|
+
for (const row of SEED) {
|
|
153
|
+
insert.run(row.id, `content of ${row.id}`, row.created_at, row.updated_at);
|
|
154
|
+
}
|
|
155
|
+
return db;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
describe("migrateToV26 — backfills updated_at_ms UTC-correctly (vault#586)", () => {
|
|
159
|
+
it("has no updated_at_ms column before the migration; adds one after", () => {
|
|
160
|
+
const db = buildPreV26Vault();
|
|
161
|
+
const cols = () =>
|
|
162
|
+
(db.prepare("PRAGMA table_info(notes)").all() as { name: string }[]).map((c) => c.name);
|
|
163
|
+
expect(cols()).not.toContain("updated_at_ms");
|
|
164
|
+
initSchema(db);
|
|
165
|
+
expect(cols()).toContain("updated_at_ms");
|
|
166
|
+
// Chain advanced all the way to the current schema version.
|
|
167
|
+
expect((db.prepare("SELECT MAX(version) AS v FROM schema_version").get() as { v: number }).v).toBe(
|
|
168
|
+
SCHEMA_VERSION,
|
|
169
|
+
);
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
it("backfills each row from its updated_at with a UTC-correct parse (space-form NOT tz-shifted)", () => {
|
|
173
|
+
const db = buildPreV26Vault();
|
|
174
|
+
initSchema(db);
|
|
175
|
+
for (const row of SEED) {
|
|
176
|
+
const got = (
|
|
177
|
+
db.prepare("SELECT updated_at_ms FROM notes WHERE id = ?").get(row.id) as {
|
|
178
|
+
updated_at_ms: number | null;
|
|
179
|
+
}
|
|
180
|
+
).updated_at_ms;
|
|
181
|
+
expect(got).toBe(row.expectedMs);
|
|
182
|
+
}
|
|
183
|
+
// No row left NULL — a NULL key would break the `> ?` keyset predicate.
|
|
184
|
+
const nulls = (
|
|
185
|
+
db.prepare("SELECT COUNT(*) AS n FROM notes WHERE updated_at_ms IS NULL").get() as { n: number }
|
|
186
|
+
).n;
|
|
187
|
+
expect(nulls).toBe(0);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
it("creates the (updated_at_ms, id) keyset index", () => {
|
|
191
|
+
const db = buildPreV26Vault();
|
|
192
|
+
initSchema(db);
|
|
193
|
+
const idx = db
|
|
194
|
+
.prepare("SELECT name FROM sqlite_master WHERE type='index' AND name='idx_notes_updated_ms'")
|
|
195
|
+
.get();
|
|
196
|
+
expect(idx).toBeTruthy();
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
it("the rewritten keyset predicate seeks the (updated_at_ms, id) index — no full scan / temp b-tree", () => {
|
|
200
|
+
const db = buildPreV26Vault();
|
|
201
|
+
initSchema(db);
|
|
202
|
+
// This is the SQL queryNotes now emits in cursor mode (integer keyset).
|
|
203
|
+
const plan = db
|
|
204
|
+
.prepare(
|
|
205
|
+
`EXPLAIN QUERY PLAN
|
|
206
|
+
SELECT n.id FROM notes n
|
|
207
|
+
WHERE (n.updated_at_ms > ? OR (n.updated_at_ms = ? AND n.id > ?))
|
|
208
|
+
ORDER BY n.updated_at_ms ASC, n.id ASC LIMIT 10`,
|
|
209
|
+
)
|
|
210
|
+
.all(0, 0, "") as { detail: string }[];
|
|
211
|
+
const details = plan.map((r) => r.detail).join(" | ");
|
|
212
|
+
expect(details).toContain("idx_notes_updated_ms");
|
|
213
|
+
expect(details).not.toContain("TEMP B-TREE");
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
it("is idempotent — a second initSchema neither re-backfills nor throws", () => {
|
|
217
|
+
const db = buildPreV26Vault();
|
|
218
|
+
initSchema(db);
|
|
219
|
+
expect(() => initSchema(db)).not.toThrow();
|
|
220
|
+
// The unparseable-both row keeps its sentinel; nothing drifted.
|
|
221
|
+
const got = (
|
|
222
|
+
db.prepare("SELECT updated_at_ms FROM notes WHERE id = ?").get("note-both-bad") as {
|
|
223
|
+
updated_at_ms: number | null;
|
|
224
|
+
}
|
|
225
|
+
).updated_at_ms;
|
|
226
|
+
expect(got).toBe(0);
|
|
227
|
+
});
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
// ---------------------------------------------------------------------------
|
|
231
|
+
// The whole point: an aged vault with the full matrix walks a cursor from
|
|
232
|
+
// bootstrap to empty with zero misses, zero duplicates, bounded pages, and no
|
|
233
|
+
// throw — the exact loop that was broken pre-v26.
|
|
234
|
+
// ---------------------------------------------------------------------------
|
|
235
|
+
describe("cursor walk over a matrix of non-canonical timestamps (vault#586)", () => {
|
|
236
|
+
const allIds = SEED.map((r) => r.id).sort();
|
|
237
|
+
|
|
238
|
+
function walk(db: Database, limit: number): { seen: string[]; pages: number; threw: unknown } {
|
|
239
|
+
const seen: string[] = [];
|
|
240
|
+
let cursor = ""; // bootstrap
|
|
241
|
+
let pages = 0;
|
|
242
|
+
// Hard cap well above the honest bound so a broken (looping) walk fails
|
|
243
|
+
// LOUDLY on the page-count assertion instead of spinning forever.
|
|
244
|
+
const hardCap = SEED.length * 4 + 10;
|
|
245
|
+
let threw: unknown = null;
|
|
246
|
+
try {
|
|
247
|
+
while (pages < hardCap) {
|
|
248
|
+
const page = noteOps.queryNotesPaged(db, { cursor, limit });
|
|
249
|
+
pages++;
|
|
250
|
+
if (page.notes.length === 0) break;
|
|
251
|
+
for (const n of page.notes) seen.push(n.id);
|
|
252
|
+
cursor = page.next_cursor;
|
|
253
|
+
}
|
|
254
|
+
} catch (err) {
|
|
255
|
+
threw = err;
|
|
256
|
+
}
|
|
257
|
+
return { seen, pages, threw };
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
it("bootstrap → empty page: every id exactly once, terminates within ceil(N/limit)+2 pages, no throw", () => {
|
|
261
|
+
const db = buildPreV26Vault();
|
|
262
|
+
initSchema(db);
|
|
263
|
+
const limit = 3;
|
|
264
|
+
const { seen, pages, threw } = walk(db, limit);
|
|
265
|
+
|
|
266
|
+
// No page 400'd / threw (unparseable timestamps used to blow up the walk).
|
|
267
|
+
expect(threw).toBeNull();
|
|
268
|
+
// Zero duplicates.
|
|
269
|
+
expect(seen.length).toBe(new Set(seen).size);
|
|
270
|
+
// Zero misses + no extras: the visited set equals ALL ids.
|
|
271
|
+
expect([...seen].sort()).toEqual(allIds);
|
|
272
|
+
// Bounded termination (includes the final empty page).
|
|
273
|
+
const bound = Math.ceil(SEED.length / limit) + 2;
|
|
274
|
+
expect(pages).toBeLessThanOrEqual(bound);
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
it("holds across several page sizes (including limit=1 and a limit > N)", () => {
|
|
278
|
+
for (const limit of [1, 2, 4, 100]) {
|
|
279
|
+
const db = buildPreV26Vault();
|
|
280
|
+
initSchema(db);
|
|
281
|
+
const { seen, threw } = walk(db, limit);
|
|
282
|
+
expect(threw).toBeNull();
|
|
283
|
+
expect(seen.length).toBe(new Set(seen).size);
|
|
284
|
+
expect([...seen].sort()).toEqual(allIds);
|
|
285
|
+
}
|
|
286
|
+
});
|
|
287
|
+
|
|
288
|
+
it("a bare mid-walk sqlite UPDATE of updated_at alone (no ms) would strand a row — the store never does this, and a full walk still completes", () => {
|
|
289
|
+
// Belt-and-suspenders: even after a rogue direct `UPDATE notes SET
|
|
290
|
+
// updated_at` that leaves updated_at_ms untouched (NOT a path the store
|
|
291
|
+
// exposes), the keyset — which reads updated_at_ms — still terminates and
|
|
292
|
+
// never throws. The ms column is authoritative; the string is cosmetic.
|
|
293
|
+
const db = buildPreV26Vault();
|
|
294
|
+
initSchema(db);
|
|
295
|
+
db.prepare("UPDATE notes SET updated_at = ? WHERE id = ?").run(
|
|
296
|
+
"2099-01-01 00:00:00",
|
|
297
|
+
"note-space",
|
|
298
|
+
);
|
|
299
|
+
const { seen, threw } = walk(db, 3);
|
|
300
|
+
expect(threw).toBeNull();
|
|
301
|
+
expect([...seen].sort()).toEqual(allIds);
|
|
302
|
+
});
|
|
303
|
+
});
|
|
304
|
+
|
|
305
|
+
// ---------------------------------------------------------------------------
|
|
306
|
+
// LOAD-BEARING: the ALTER + backfill run inside ONE transaction, so a crash
|
|
307
|
+
// mid-backfill rolls the column back and the next boot re-runs cleanly. A
|
|
308
|
+
// partial backfill left behind (column present, some rows NULL) would be
|
|
309
|
+
// permanent — the `hasColumn` guard would report "done." This test drives the
|
|
310
|
+
// REAL migrateToV26 (via initSchema) and injects a crash on the 2nd backfill
|
|
311
|
+
// UPDATE. Remove the `transaction()` wrap and it goes RED (post-crash the
|
|
312
|
+
// column persists with NULLs, the guard skips, and recovery leaves NULLs).
|
|
313
|
+
// ---------------------------------------------------------------------------
|
|
314
|
+
describe("migrateToV26 is all-or-nothing (crash mid-backfill) (vault#586)", () => {
|
|
315
|
+
it("a crash during the backfill rolls back the ALTER; the next initSchema fully recovers", () => {
|
|
316
|
+
const db = buildPreV26Vault();
|
|
317
|
+
const hasMsColumn = () =>
|
|
318
|
+
(db.prepare("PRAGMA table_info(notes)").all() as { name: string }[]).some(
|
|
319
|
+
(c) => c.name === "updated_at_ms",
|
|
320
|
+
);
|
|
321
|
+
expect(hasMsColumn()).toBe(false);
|
|
322
|
+
|
|
323
|
+
// Patch prepare so the backfill statement's `.run` throws on its 2nd call —
|
|
324
|
+
// a faithful mid-loop interruption AFTER the ALTER (db.exec, untouched) and
|
|
325
|
+
// after one row has already been written inside the transaction.
|
|
326
|
+
const origPrepare = db.prepare.bind(db);
|
|
327
|
+
let runCount = 0;
|
|
328
|
+
let injected = false;
|
|
329
|
+
(db as any).prepare = (sql: string) => {
|
|
330
|
+
const stmt = origPrepare(sql);
|
|
331
|
+
if (/UPDATE notes SET updated_at_ms = \?/.test(sql)) {
|
|
332
|
+
const origRun = stmt.run.bind(stmt);
|
|
333
|
+
(stmt as any).run = (...args: unknown[]) => {
|
|
334
|
+
runCount++;
|
|
335
|
+
if (runCount === 2) {
|
|
336
|
+
injected = true;
|
|
337
|
+
throw new Error("simulated crash mid-backfill");
|
|
338
|
+
}
|
|
339
|
+
return (origRun as (...a: unknown[]) => unknown)(...args);
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
return stmt;
|
|
343
|
+
};
|
|
344
|
+
|
|
345
|
+
expect(() => initSchema(db)).toThrow("simulated crash mid-backfill");
|
|
346
|
+
// Positive control: the injection actually fired (not a vacuous pass).
|
|
347
|
+
expect(injected).toBe(true);
|
|
348
|
+
|
|
349
|
+
// Restore the real prepare for the assertions + recovery run.
|
|
350
|
+
(db as any).prepare = origPrepare;
|
|
351
|
+
|
|
352
|
+
// LOAD-BEARING: the transaction rolled back the ALTER — the column is GONE,
|
|
353
|
+
// so the hasColumn guard correctly re-detects "not migrated". Without the
|
|
354
|
+
// wrap the ALTER would have autocommitted and this reads `true`, the guard
|
|
355
|
+
// skips forever, and the rows below stay NULL.
|
|
356
|
+
expect(hasMsColumn()).toBe(false);
|
|
357
|
+
// schema_version never advanced past the fixture's 25.
|
|
358
|
+
expect((db.prepare("SELECT MAX(version) AS v FROM schema_version").get() as { v: number }).v).toBe(
|
|
359
|
+
25,
|
|
360
|
+
);
|
|
361
|
+
|
|
362
|
+
// A clean restart re-runs the REAL migration and fully backfills — no NULLs.
|
|
363
|
+
initSchema(db);
|
|
364
|
+
expect(hasMsColumn()).toBe(true);
|
|
365
|
+
const nulls = (
|
|
366
|
+
db.prepare("SELECT COUNT(*) AS n FROM notes WHERE updated_at_ms IS NULL").get() as { n: number }
|
|
367
|
+
).n;
|
|
368
|
+
expect(nulls).toBe(0);
|
|
369
|
+
for (const row of SEED) {
|
|
370
|
+
const got = (
|
|
371
|
+
db.prepare("SELECT updated_at_ms FROM notes WHERE id = ?").get(row.id) as {
|
|
372
|
+
updated_at_ms: number | null;
|
|
373
|
+
}
|
|
374
|
+
).updated_at_ms;
|
|
375
|
+
expect(got).toBe(row.expectedMs);
|
|
376
|
+
}
|
|
377
|
+
});
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
// ---------------------------------------------------------------------------
|
|
381
|
+
// SELF-HEAL (review nit 1): a vault whose notes table ALREADY has the column
|
|
382
|
+
// but carries NULL updated_at_ms rows — the shape a schema-v2-era upgrade
|
|
383
|
+
// produces (migrateFromV2's INSERT…SELECT omits the column). A
|
|
384
|
+
// column-presence-only guard would leave those NULL forever (invisible to the
|
|
385
|
+
// keyset). migrateToV26 must backfill WHERE updated_at_ms IS NULL regardless.
|
|
386
|
+
// ---------------------------------------------------------------------------
|
|
387
|
+
describe("migrateToV26 self-heals NULL updated_at_ms on an existing column (vault#586)", () => {
|
|
388
|
+
function buildColumnPresentWithNullRows(): Database {
|
|
389
|
+
const db = new Database(":memory:");
|
|
390
|
+
db.exec(`
|
|
391
|
+
CREATE TABLE notes (
|
|
392
|
+
id TEXT PRIMARY KEY,
|
|
393
|
+
content TEXT DEFAULT '',
|
|
394
|
+
path TEXT,
|
|
395
|
+
metadata TEXT DEFAULT '{}',
|
|
396
|
+
created_at TEXT NOT NULL,
|
|
397
|
+
updated_at TEXT,
|
|
398
|
+
extension TEXT NOT NULL DEFAULT 'md',
|
|
399
|
+
created_by TEXT,
|
|
400
|
+
created_via TEXT,
|
|
401
|
+
last_updated_by TEXT,
|
|
402
|
+
last_updated_via TEXT,
|
|
403
|
+
updated_at_ms INTEGER
|
|
404
|
+
);
|
|
405
|
+
CREATE TABLE schema_version (version INTEGER PRIMARY KEY, applied_at TEXT);
|
|
406
|
+
INSERT INTO schema_version (version, applied_at) VALUES (25, '2026-01-01T00:00:00.000Z');
|
|
407
|
+
`);
|
|
408
|
+
const ins = db.prepare(
|
|
409
|
+
"INSERT INTO notes (id, content, created_at, updated_at, updated_at_ms) VALUES (?, ?, ?, ?, ?)",
|
|
410
|
+
);
|
|
411
|
+
// Two leaked rows: column present, updated_at_ms NULL — one space-form, one
|
|
412
|
+
// unparseable (falls back to created_at).
|
|
413
|
+
ins.run("leak-space", "", "2024-03-03T08:15:00.000Z", "2024-03-03 08:15:00", null);
|
|
414
|
+
ins.run("leak-bad", "", "2024-07-07T00:00:00.000Z", "not-a-timestamp", null);
|
|
415
|
+
// An already-populated row carrying a deliberately-arbitrary ms — the
|
|
416
|
+
// self-heal touches only NULLs, so this must survive UNCHANGED.
|
|
417
|
+
ins.run("already", "", "2024-01-01T00:00:00.000Z", "2024-01-01T00:00:00.000Z", 999);
|
|
418
|
+
return db;
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
it("backfills the NULL rows UTC-correctly, leaves populated rows untouched, ends with zero NULLs", () => {
|
|
422
|
+
const db = buildColumnPresentWithNullRows();
|
|
423
|
+
expect(msOf(db, "leak-space")).toBeNull(); // precondition: leaked NULL
|
|
424
|
+
initSchema(db);
|
|
425
|
+
|
|
426
|
+
expect(
|
|
427
|
+
(db.prepare("SELECT COUNT(*) AS n FROM notes WHERE updated_at_ms IS NULL").get() as { n: number }).n,
|
|
428
|
+
).toBe(0);
|
|
429
|
+
// Space-form healed to UTC (not local-shifted).
|
|
430
|
+
expect(msOf(db, "leak-space")).toBe(Date.UTC(2024, 2, 3, 8, 15));
|
|
431
|
+
// Unparseable updated_at healed via created_at fallback.
|
|
432
|
+
expect(msOf(db, "leak-bad")).toBe(Date.UTC(2024, 6, 7));
|
|
433
|
+
// Already-populated row: NOT re-derived (still the arbitrary 999).
|
|
434
|
+
expect(msOf(db, "already")).toBe(999);
|
|
435
|
+
});
|
|
436
|
+
});
|
|
437
|
+
|
|
438
|
+
// ---------------------------------------------------------------------------
|
|
439
|
+
// WRITE-SITE PINS (review nit 3): assert updated_at_ms actually advances at
|
|
440
|
+
// the restore + rename-cascade sites, so reverting either ms hunk goes RED
|
|
441
|
+
// (previously the whole suite passed with those hunks removed).
|
|
442
|
+
// ---------------------------------------------------------------------------
|
|
443
|
+
describe("write sites maintain updated_at_ms in lockstep (vault#586)", () => {
|
|
444
|
+
it("restoreNoteTimestamps derives updated_at_ms UTC-correctly from a non-canonical updated_at (pins store.ts hunk)", async () => {
|
|
445
|
+
const db = new Database(":memory:");
|
|
446
|
+
const store = new SqliteStore(db);
|
|
447
|
+
const note = await store.createNote("x");
|
|
448
|
+
const createMs = msOf(db, note.id)!;
|
|
449
|
+
|
|
450
|
+
// Import restores a verbatim, non-canonical (space-form) updated_at.
|
|
451
|
+
await store.restoreNoteTimestamps(note.id, "2024-01-01T00:00:00.000Z", "2024-03-03 08:15:00");
|
|
452
|
+
|
|
453
|
+
// updated_at kept verbatim (round-trip fidelity); ms derived UTC-correctly.
|
|
454
|
+
const row = db.prepare("SELECT updated_at, updated_at_ms FROM notes WHERE id = ?").get(note.id) as {
|
|
455
|
+
updated_at: string;
|
|
456
|
+
updated_at_ms: number;
|
|
457
|
+
};
|
|
458
|
+
expect(row.updated_at).toBe("2024-03-03 08:15:00");
|
|
459
|
+
expect(row.updated_at_ms).toBe(Date.UTC(2024, 2, 3, 8, 15));
|
|
460
|
+
// A revert of the store hunk would leave ms at the createNote value.
|
|
461
|
+
expect(row.updated_at_ms).not.toBe(createMs);
|
|
462
|
+
});
|
|
463
|
+
|
|
464
|
+
it("renameTag path-cascade bumps updated_at_ms on a `_tags/<old>` note (pins notes.ts path hunk)", async () => {
|
|
465
|
+
const db = new Database(":memory:");
|
|
466
|
+
const store = new SqliteStore(db);
|
|
467
|
+
// Body has no #task, so ONLY the path cascade touches this note — isolating
|
|
468
|
+
// the path hunk from the content hunk.
|
|
469
|
+
const note = await store.createNote("plain body, no hashtags", {
|
|
470
|
+
path: "_tags/task",
|
|
471
|
+
tags: ["task"],
|
|
472
|
+
});
|
|
473
|
+
const beforeMs = msOf(db, note.id)!;
|
|
474
|
+
await new Promise((r) => setTimeout(r, 5));
|
|
475
|
+
|
|
476
|
+
await store.renameTag("task", "todo");
|
|
477
|
+
|
|
478
|
+
const fresh = await store.getNote(note.id);
|
|
479
|
+
expect(fresh!.path).toBe("_tags/todo");
|
|
480
|
+
const afterMs = msOf(db, note.id)!;
|
|
481
|
+
expect(afterMs).toBeGreaterThan(beforeMs);
|
|
482
|
+
expect(afterMs).toBe(Date.parse(fresh!.updatedAt!));
|
|
483
|
+
});
|
|
484
|
+
});
|
|
485
|
+
|
|
486
|
+
// ---------------------------------------------------------------------------
|
|
487
|
+
// SAME-SNAPSHOT WATERMARK (review nit 2): the watermark is captured in the
|
|
488
|
+
// phase-1 page statement (the one that ordered + sliced the page), not a
|
|
489
|
+
// separate later read. A concurrent writer that bumps a page row's ms AFTER
|
|
490
|
+
// the page is read must NOT make the watermark leapfrog past unseen rows.
|
|
491
|
+
// ---------------------------------------------------------------------------
|
|
492
|
+
describe("queryNotesPaged watermark comes from the phase-1 page snapshot (vault#586)", () => {
|
|
493
|
+
it("a concurrent writer bumping a page row's ms after the page read does NOT advance the watermark past it", () => {
|
|
494
|
+
const db = buildPreV26Vault();
|
|
495
|
+
initSchema(db);
|
|
496
|
+
|
|
497
|
+
// The true max keyset key of the full page (bootstrap, limit > N): the
|
|
498
|
+
// July row. note-offset-plus (an EARLIER row) is what we'll bump.
|
|
499
|
+
const trueMax = Date.UTC(2024, 6, 7); // note-unparseable → created_at fallback
|
|
500
|
+
const bumpTarget = "note-offset-plus";
|
|
501
|
+
|
|
502
|
+
const origPrepare = db.prepare.bind(db);
|
|
503
|
+
let bumped = false;
|
|
504
|
+
(db as any).prepare = (sql: string) => {
|
|
505
|
+
const stmt = origPrepare(sql);
|
|
506
|
+
// Intercept ONLY the phase-1 ordering statement.
|
|
507
|
+
if (/SELECT n\.id, n\.updated_at_ms FROM notes n/.test(sql)) {
|
|
508
|
+
const origAll = stmt.all.bind(stmt);
|
|
509
|
+
(stmt as any).all = (...args: unknown[]) => {
|
|
510
|
+
const rows = (origAll as (...a: unknown[]) => unknown[])(...args);
|
|
511
|
+
// A concurrent writer lands right AFTER the page snapshot is read,
|
|
512
|
+
// shoving the target row's ms far into the future. A watermark that
|
|
513
|
+
// re-read the column in a SECOND statement would leapfrog to it.
|
|
514
|
+
if (!bumped) {
|
|
515
|
+
bumped = true;
|
|
516
|
+
origPrepare("UPDATE notes SET updated_at_ms = ? WHERE id = ?").run(9_000_000_000_000, bumpTarget);
|
|
517
|
+
}
|
|
518
|
+
return rows;
|
|
519
|
+
};
|
|
520
|
+
}
|
|
521
|
+
return stmt;
|
|
522
|
+
};
|
|
523
|
+
|
|
524
|
+
const page = noteOps.queryNotesPaged(db, { cursor: "", limit: 100 });
|
|
525
|
+
(db as any).prepare = origPrepare;
|
|
526
|
+
|
|
527
|
+
// Positive control: the interception actually fired (guards against the
|
|
528
|
+
// phase-1 SQL drifting and silently no-op'ing the test).
|
|
529
|
+
expect(bumped).toBe(true);
|
|
530
|
+
// The watermark reflects the PRE-bump page snapshot — the real max — NOT
|
|
531
|
+
// the injected 9e12. A re-read-based watermark would encode 9e12 and skip
|
|
532
|
+
// every row between trueMax and 9e12 on the next call.
|
|
533
|
+
const wm = decodeCursor(page.next_cursor).last_updated_at;
|
|
534
|
+
expect(wm).toBe(trueMax);
|
|
535
|
+
expect(wm).not.toBe(9_000_000_000_000);
|
|
536
|
+
});
|
|
537
|
+
});
|