@openparachute/vault 0.4.7-rc.2 → 0.4.8-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -10
- package/core/src/connection-pragmas.test.ts +232 -0
- package/core/src/core.test.ts +257 -0
- package/core/src/cursor.test.ts +160 -0
- package/core/src/cursor.ts +272 -0
- package/core/src/mcp.ts +51 -7
- package/core/src/notes.ts +164 -2
- package/core/src/schema.ts +98 -2
- package/core/src/store.ts +11 -1
- package/core/src/types.ts +32 -0
- package/package.json +1 -1
- package/src/auth-status.ts +4 -0
- package/src/auto-transcribe.test.ts +116 -0
- package/src/auto-transcribe.ts +48 -0
- package/src/cli.ts +57 -48
- package/src/config.test.ts +26 -0
- package/src/config.ts +53 -1
- package/src/db.ts +15 -2
- package/src/mcp-install-interactive.test.ts +23 -2
- package/src/mcp-install-interactive.ts +21 -2
- package/src/mcp-install.test.ts +40 -0
- package/src/mcp-tools.ts +17 -1
- package/src/module-config.ts +70 -14
- package/src/module-manifest.test.ts +93 -0
- package/src/module-manifest.ts +94 -0
- package/src/routes.ts +267 -50
- package/src/scribe-discovery.test.ts +77 -0
- package/src/scribe-discovery.ts +91 -0
- package/src/scribe-env.test.ts +66 -1
- package/src/scribe-env.ts +42 -1
- package/src/self-register.test.ts +380 -0
- package/src/self-register.ts +234 -0
- package/src/server.ts +46 -11
- package/src/transcript-note.test.ts +171 -0
- package/src/transcript-note.ts +189 -0
- package/src/transcription-registry.ts +22 -0
- package/src/transcription-worker.test.ts +250 -0
- package/src/transcription-worker.ts +186 -27
- package/src/vault.test.ts +347 -0
package/README.md
CHANGED
|
@@ -67,6 +67,9 @@ A mental model for "where is my data?" and "what can I poke at?" after the one-c
|
|
|
67
67
|
default/
|
|
68
68
|
vault.db # the SQLite database — notes, tags, links, attachments,
|
|
69
69
|
# per-vault tokens, OAuth clients + codes, tag schemas
|
|
70
|
+
vault.db-wal # WAL journal (write-ahead log) — transient, recreated
|
|
71
|
+
# on demand. Carries pending writes between checkpoints.
|
|
72
|
+
vault.db-shm # WAL shared-memory index — transient, recreated on demand.
|
|
70
73
|
vault.yaml # per-vault config — description (sent as MCP session
|
|
71
74
|
# instruction), published_tag override, legacy api_keys
|
|
72
75
|
assets/ # per-vault uploaded attachments (audio, images)
|
|
@@ -76,6 +79,8 @@ A mental model for "where is my data?" and "what can I poke at?" after the one-c
|
|
|
76
79
|
|
|
77
80
|
`config.yaml` is the one file written at 0600 because it holds the bcrypt owner-password hash and the plaintext TOTP secret. `.env` is written with your umask default (typically 0644); if you add webhook API keys there, tighten the mode yourself. SQLite DBs follow your umask.
|
|
78
81
|
|
|
82
|
+
The vault SQLite database runs in **WAL** (write-ahead logging) journal mode for multi-process concurrent access — the daemon, CLI tools, and out-of-process consumers (e.g. `parachute-runner` polling `tag:job`) can read concurrently while the daemon writes, without lock contention. WAL adds two sidecar files alongside `vault.db`: `vault.db-wal` (the journal) and `vault.db-shm` (shared-memory index). Both are recreated on demand; **don't back them up separately** — `parachute-vault backup` snapshots only `vault.db` via `VACUUM INTO`, which produces a consistent full-DB copy without needing the sidecars. If you copy a vault by hand, `vault.db` alone is sufficient on a checkpointed database; if you must capture an in-flight write, copy all three files. If WAL can't be enabled (NFS, some FUSE / Docker volume drivers don't support the `-shm` region), vault logs `[vault] WAL mode could not be enabled` on startup and falls back to the legacy single-writer mode.
|
|
83
|
+
|
|
79
84
|
### Registered externally
|
|
80
85
|
|
|
81
86
|
- **macOS**: a launchd user agent labelled `computer.parachute.vault` (plus `computer.parachute.vault.backup` if you ran `vault backup --schedule`).
|
|
@@ -339,16 +344,45 @@ Webhook servers (scribe, narrate) are stateless — they don't need vault's API
|
|
|
339
344
|
|
|
340
345
|
### On-upload transcription
|
|
341
346
|
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
347
|
+
Two paths feed the same transcription worker; they differ in how the
|
|
348
|
+
result surfaces.
|
|
349
|
+
|
|
350
|
+
**Auto-transcribe (vault#353).** When the operator flips
|
|
351
|
+
`auto_transcribe.enabled: true` in vault's config AND scribe is
|
|
352
|
+
reachable (registered in `~/.parachute/services.json`, or pointed at
|
|
353
|
+
via `SCRIBE_URL`), any audio attachment uploaded to the vault is
|
|
354
|
+
automatically queued for transcription. The worker writes a sibling
|
|
355
|
+
`<attachment-path>.transcript.md` note with the transcript text +
|
|
356
|
+
frontmatter linking back to the audio (`transcript_of`,
|
|
357
|
+
`transcript_status`, `transcript_duration_ms`, etc.). Failures land as
|
|
358
|
+
the same note with `transcript_status: failed` and the cause in
|
|
359
|
+
`transcript_error`; the original audio is preserved. Operators can
|
|
360
|
+
retry a failed transcript via `POST /vault/<name>/api/notes/<note-id>/retry-transcription`.
|
|
361
|
+
|
|
362
|
+
**Explicit `transcribe: true` (legacy, Lens flow).** Callers that already
|
|
363
|
+
know at upload time that an audio file should be transcribed pass
|
|
364
|
+
`transcribe: true` to `POST /api/notes/{id}/attachments`. The vault
|
|
365
|
+
stamps the attachment with `transcribe_status: "pending"` and the note
|
|
366
|
+
with `transcribe_stub: true`. The worker replaces the literal
|
|
367
|
+
`_Transcript pending._` placeholder in the note body with the transcript
|
|
368
|
+
on success (or the whole body if no placeholder is present).
|
|
369
|
+
|
|
370
|
+
Service discovery is automatic — when scribe lands in
|
|
371
|
+
`~/.parachute/services.json` (the canonical hub-maintained registry),
|
|
372
|
+
vault picks up its URL on next restart. The `SCRIBE_URL` env var still
|
|
373
|
+
wins when set. The shared bearer for vault→scribe auth is generated
|
|
374
|
+
once at first boot and persisted to `~/.parachute/vault/.env` as
|
|
375
|
+
`SCRIBE_AUTH_TOKEN`; mirror that value into scribe's
|
|
376
|
+
`SCRIBE_AUTH_TOKEN` env so scribe accepts the Authorization header.
|
|
377
|
+
|
|
378
|
+
The worker POSTs audio as multipart to
|
|
379
|
+
`${SCRIBE_URL}/v1/audio/transcriptions` and expects `{ text: string }`
|
|
380
|
+
back. On success it records `transcript` + `transcribe_done_at` +
|
|
381
|
+
`transcribe_duration_ms` on the attachment row regardless of the
|
|
382
|
+
result-surface path. Failures retry with exponential backoff up to three
|
|
383
|
+
attempts before flipping `transcribe_status` to `"failed"`; 4xx
|
|
384
|
+
responses carrying `error_code` (e.g. `missing_provider`) are treated as
|
|
385
|
+
terminal on the first failure.
|
|
352
386
|
|
|
353
387
|
Per-vault `audio_retention` controls what happens to the audio file on disk once the worker reaches a terminal state. It's readable and mutable at runtime via `GET` / `PATCH /api/vault` (under `config.audio_retention`), or by hand-editing `vault.yaml`.
|
|
354
388
|
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for connection-level pragmas — WAL mode + synchronous=NORMAL +
|
|
3
|
+
* foreign_keys=ON. See applyConnectionPragmas in schema.ts and vault#326.
|
|
4
|
+
*
|
|
5
|
+
* `:memory:` databases land in journal_mode=memory and DO NOT support WAL
|
|
6
|
+
* (the WAL/shm sidecars need a real file). On-disk DBs are the realistic
|
|
7
|
+
* path; we exercise both so the readonly + filesystem-unsupported branches
|
|
8
|
+
* are covered.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { describe, it, expect, beforeEach, afterEach } from "bun:test";
|
|
12
|
+
import { Database } from "bun:sqlite";
|
|
13
|
+
import { mkdtempSync, rmSync } from "fs";
|
|
14
|
+
import { join } from "path";
|
|
15
|
+
import { tmpdir } from "os";
|
|
16
|
+
import { applyConnectionPragmas, initSchema } from "./schema.ts";
|
|
17
|
+
|
|
18
|
+
let dir: string;
|
|
19
|
+
|
|
20
|
+
beforeEach(() => {
|
|
21
|
+
dir = mkdtempSync(join(tmpdir(), "vault-pragma-"));
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
afterEach(() => {
|
|
25
|
+
rmSync(dir, { recursive: true, force: true });
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
function pragma(db: Database, name: string): string | number {
|
|
29
|
+
const row = db.prepare(`PRAGMA ${name}`).get() as Record<string, string | number> | null;
|
|
30
|
+
if (!row) return "";
|
|
31
|
+
// PRAGMA xxx returns a one-key object whose key matches the pragma name.
|
|
32
|
+
const v = Object.values(row)[0];
|
|
33
|
+
return typeof v === "string" ? v.toLowerCase() : (v as number);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
describe("applyConnectionPragmas — on-disk DB", () => {
|
|
37
|
+
it("enables WAL mode on a fresh on-disk database", () => {
|
|
38
|
+
const db = new Database(join(dir, "fresh.db"));
|
|
39
|
+
const result = applyConnectionPragmas(db);
|
|
40
|
+
expect(result.wal).toBe(true);
|
|
41
|
+
expect(result.journalMode).toBe("wal");
|
|
42
|
+
expect(pragma(db, "journal_mode")).toBe("wal");
|
|
43
|
+
db.close();
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it("sets synchronous=NORMAL when WAL succeeds", () => {
|
|
47
|
+
const db = new Database(join(dir, "sync.db"));
|
|
48
|
+
applyConnectionPragmas(db);
|
|
49
|
+
// synchronous values: 0=off, 1=normal, 2=full, 3=extra. NORMAL is 1.
|
|
50
|
+
expect(pragma(db, "synchronous")).toBe(1);
|
|
51
|
+
db.close();
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it("sets wal_autocheckpoint=1000 when WAL succeeds", () => {
|
|
55
|
+
const db = new Database(join(dir, "checkpoint.db"));
|
|
56
|
+
applyConnectionPragmas(db);
|
|
57
|
+
expect(pragma(db, "wal_autocheckpoint")).toBe(1000);
|
|
58
|
+
db.close();
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it("enables foreign_keys", () => {
|
|
62
|
+
const db = new Database(join(dir, "fk.db"));
|
|
63
|
+
applyConnectionPragmas(db);
|
|
64
|
+
expect(pragma(db, "foreign_keys")).toBe(1);
|
|
65
|
+
db.close();
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
it("is idempotent — applying twice yields the same result", () => {
|
|
69
|
+
const dbPath = join(dir, "idem.db");
|
|
70
|
+
const db = new Database(dbPath);
|
|
71
|
+
const a = applyConnectionPragmas(db);
|
|
72
|
+
const b = applyConnectionPragmas(db);
|
|
73
|
+
expect(a).toEqual(b);
|
|
74
|
+
expect(b.wal).toBe(true);
|
|
75
|
+
db.close();
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it("a DB created in DELETE mode is migrated to WAL on next open", () => {
|
|
79
|
+
const dbPath = join(dir, "migrate.db");
|
|
80
|
+
|
|
81
|
+
// First connection — manually force DELETE journal mode (the legacy
|
|
82
|
+
// shape that pre-WAL vaults shipped with). Write some data so we can
|
|
83
|
+
// verify it survives the WAL flip.
|
|
84
|
+
{
|
|
85
|
+
const db = new Database(dbPath);
|
|
86
|
+
db.exec("PRAGMA journal_mode = DELETE");
|
|
87
|
+
db.exec("CREATE TABLE legacy (k TEXT PRIMARY KEY, v TEXT)");
|
|
88
|
+
db.prepare("INSERT INTO legacy (k, v) VALUES (?, ?)").run("hello", "world");
|
|
89
|
+
expect(pragma(db, "journal_mode")).toBe("delete");
|
|
90
|
+
db.close();
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Second connection — apply pragmas. WAL takes effect, existing data
|
|
94
|
+
// intact.
|
|
95
|
+
{
|
|
96
|
+
const db = new Database(dbPath);
|
|
97
|
+
const result = applyConnectionPragmas(db);
|
|
98
|
+
expect(result.wal).toBe(true);
|
|
99
|
+
const row = db.prepare("SELECT v FROM legacy WHERE k = ?").get("hello") as { v: string };
|
|
100
|
+
expect(row.v).toBe("world");
|
|
101
|
+
db.close();
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
describe("applyConnectionPragmas — :memory: DB", () => {
|
|
107
|
+
it("returns wal:false, journalMode='memory' WITHOUT warning", () => {
|
|
108
|
+
// :memory: is bun:sqlite's ephemeral mode — journal_mode comes back as
|
|
109
|
+
// "memory" and WAL can't be enabled. This is an explicit caller choice
|
|
110
|
+
// (test fixtures, throwaway probes), not an operator-visible filesystem
|
|
111
|
+
// limitation, so applyConnectionPragmas suppresses the warning for it
|
|
112
|
+
// to keep test output clean.
|
|
113
|
+
const db = new Database(":memory:");
|
|
114
|
+
|
|
115
|
+
const warnings: string[] = [];
|
|
116
|
+
const origWarn = console.warn;
|
|
117
|
+
console.warn = (msg: string) => warnings.push(msg);
|
|
118
|
+
|
|
119
|
+
try {
|
|
120
|
+
const result = applyConnectionPragmas(db);
|
|
121
|
+
expect(result.wal).toBe(false);
|
|
122
|
+
expect(result.journalMode).toBe("memory");
|
|
123
|
+
expect(warnings.length).toBe(0);
|
|
124
|
+
} finally {
|
|
125
|
+
console.warn = origWarn;
|
|
126
|
+
db.close();
|
|
127
|
+
}
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
it("still enables foreign_keys on the :memory: branch", () => {
|
|
131
|
+
const db = new Database(":memory:");
|
|
132
|
+
applyConnectionPragmas(db);
|
|
133
|
+
expect(pragma(db, "foreign_keys")).toBe(1);
|
|
134
|
+
db.close();
|
|
135
|
+
});
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
describe("applyConnectionPragmas — WAL-unsupported on-disk filesystem (simulated)", () => {
|
|
139
|
+
it("warns once when an on-disk DB lands in a non-WAL, non-memory mode", () => {
|
|
140
|
+
// We can't easily mount an NFS volume in CI, so simulate the
|
|
141
|
+
// unsupported-FS branch: open an on-disk DB and immediately force
|
|
142
|
+
// journal_mode to a non-WAL mode that sticks. PRAGMA journal_mode=WAL
|
|
143
|
+
// will then return that mode (because WAL silently fell back), which
|
|
144
|
+
// is the exact shape the unsupported-FS detection triggers on.
|
|
145
|
+
//
|
|
146
|
+
// We do this by stubbing prepare("PRAGMA journal_mode = WAL") to
|
|
147
|
+
// return { journal_mode: "delete" } — what bun:sqlite returns when
|
|
148
|
+
// SQLite refuses the WAL flip.
|
|
149
|
+
const db = new Database(join(dir, "stub.db"));
|
|
150
|
+
|
|
151
|
+
const origPrepare = db.prepare.bind(db);
|
|
152
|
+
// @ts-expect-error — narrow override for the one PRAGMA we care about
|
|
153
|
+
db.prepare = (sql: string) => {
|
|
154
|
+
if (sql === "PRAGMA journal_mode = WAL") {
|
|
155
|
+
return {
|
|
156
|
+
get: () => ({ journal_mode: "delete" }),
|
|
157
|
+
all: () => [{ journal_mode: "delete" }],
|
|
158
|
+
run: () => ({ changes: 0, lastInsertRowid: 0 }),
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
return origPrepare(sql);
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
const warnings: string[] = [];
|
|
165
|
+
const origWarn = console.warn;
|
|
166
|
+
console.warn = (msg: string) => warnings.push(msg);
|
|
167
|
+
|
|
168
|
+
try {
|
|
169
|
+
const result = applyConnectionPragmas(db);
|
|
170
|
+
expect(result.wal).toBe(false);
|
|
171
|
+
expect(result.journalMode).toBe("delete");
|
|
172
|
+
expect(warnings.length).toBe(1);
|
|
173
|
+
expect(warnings[0]).toContain("WAL mode could not be enabled");
|
|
174
|
+
expect(warnings[0]).toContain("journal_mode=delete");
|
|
175
|
+
|
|
176
|
+
// Second call on the same handle: dedupe by WeakSet, no second warn.
|
|
177
|
+
applyConnectionPragmas(db);
|
|
178
|
+
expect(warnings.length).toBe(1);
|
|
179
|
+
} finally {
|
|
180
|
+
console.warn = origWarn;
|
|
181
|
+
db.close();
|
|
182
|
+
}
|
|
183
|
+
});
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
describe("initSchema integration", () => {
|
|
187
|
+
it("leaves a fresh vault DB in WAL mode after full initialization", () => {
|
|
188
|
+
const db = new Database(join(dir, "vault.db"));
|
|
189
|
+
initSchema(db);
|
|
190
|
+
expect(pragma(db, "journal_mode")).toBe("wal");
|
|
191
|
+
expect(pragma(db, "foreign_keys")).toBe(1);
|
|
192
|
+
expect(pragma(db, "synchronous")).toBe(1);
|
|
193
|
+
db.close();
|
|
194
|
+
});
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
describe("multi-connection concurrency under WAL", () => {
|
|
198
|
+
it("allows a second connection to read while a first holds an open write txn", () => {
|
|
199
|
+
// The whole point of WAL: a reader running concurrently with a writer
|
|
200
|
+
// does NOT block. Under the legacy DELETE journal mode an open write
|
|
201
|
+
// txn locks out readers and a `BEGIN IMMEDIATE` from a sibling
|
|
202
|
+
// connection on the same file errors with SQLITE_BUSY.
|
|
203
|
+
//
|
|
204
|
+
// Setup: writer opens, applies pragmas (flips DB to WAL), inserts a
|
|
205
|
+
// row, BEGINS another txn (uncommitted). Reader opens a second
|
|
206
|
+
// connection to the same file and SELECTs — should succeed instantly
|
|
207
|
+
// with the pre-txn committed state.
|
|
208
|
+
const dbPath = join(dir, "concurrent.db");
|
|
209
|
+
const writer = new Database(dbPath);
|
|
210
|
+
initSchema(writer);
|
|
211
|
+
|
|
212
|
+
writer.exec(`CREATE TABLE k (id INTEGER PRIMARY KEY, v TEXT)`);
|
|
213
|
+
writer.prepare(`INSERT INTO k (id, v) VALUES (?, ?)`).run(1, "committed");
|
|
214
|
+
|
|
215
|
+
// Open a long-running write txn that doesn't commit.
|
|
216
|
+
writer.exec("BEGIN IMMEDIATE");
|
|
217
|
+
writer.prepare(`INSERT INTO k (id, v) VALUES (?, ?)`).run(2, "uncommitted");
|
|
218
|
+
|
|
219
|
+
// Reader: a separate connection (simulating a separate process).
|
|
220
|
+
// Under WAL this read should succeed and see only the committed row.
|
|
221
|
+
const reader = new Database(dbPath, { readonly: true });
|
|
222
|
+
try {
|
|
223
|
+
const rows = reader.prepare("SELECT id, v FROM k ORDER BY id").all() as { id: number; v: string }[];
|
|
224
|
+
expect(rows).toEqual([{ id: 1, v: "committed" }]);
|
|
225
|
+
} finally {
|
|
226
|
+
reader.close();
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
writer.exec("ROLLBACK");
|
|
230
|
+
writer.close();
|
|
231
|
+
});
|
|
232
|
+
});
|
package/core/src/core.test.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { Database } from "bun:sqlite";
|
|
|
3
3
|
import { SqliteStore } from "./store.js";
|
|
4
4
|
import { generateMcpTools } from "./mcp.js";
|
|
5
5
|
import { initSchema } from "./schema.js";
|
|
6
|
+
import { decodeCursor } from "./cursor.js";
|
|
6
7
|
|
|
7
8
|
let store: SqliteStore;
|
|
8
9
|
let db: Database;
|
|
@@ -1312,6 +1313,262 @@ describe("queryNotes", async () => {
|
|
|
1312
1313
|
expect(desc[0].content).toBe("Second");
|
|
1313
1314
|
});
|
|
1314
1315
|
|
|
1316
|
+
// ---- Cursor pagination (vault#313) ----
|
|
1317
|
+
//
|
|
1318
|
+
// Opaque cursors for "since last checked" agent loops. The cursor binds
|
|
1319
|
+
// to the query's filters via sha256 of the result-set-affecting params;
|
|
1320
|
+
// a mismatched cursor raises CursorError. Pagination is keyset on
|
|
1321
|
+
// (updated_at, id) so two notes sharing a millisecond don't get skipped
|
|
1322
|
+
// or doubled across the page boundary.
|
|
1323
|
+
describe("cursor pagination", () => {
|
|
1324
|
+
// Helper: pin a note's updated_at to a known value so cursor math
|
|
1325
|
+
// doesn't race wall-clock writes from the test harness.
|
|
1326
|
+
function pinUpdatedAt(id: string, iso: string) {
|
|
1327
|
+
db.prepare("UPDATE notes SET updated_at = ? WHERE id = ?").run(iso, id);
|
|
1328
|
+
}
|
|
1329
|
+
|
|
1330
|
+
it("first call returns notes + a next_cursor string", async () => {
|
|
1331
|
+
await store.createNote("A", { id: "na" });
|
|
1332
|
+
await store.createNote("B", { id: "nb" });
|
|
1333
|
+
const page = await store.queryNotesPaged({ tags: [], limit: 50 });
|
|
1334
|
+
expect(page.notes.length).toBe(2);
|
|
1335
|
+
expect(typeof page.next_cursor).toBe("string");
|
|
1336
|
+
expect(page.next_cursor.length).toBeGreaterThan(0);
|
|
1337
|
+
});
|
|
1338
|
+
|
|
1339
|
+
it("subsequent call with cursor returns only newer notes", async () => {
|
|
1340
|
+
const a = await store.createNote("first", { id: "p1" });
|
|
1341
|
+
pinUpdatedAt(a.id, "2026-04-01T00:00:00.000Z");
|
|
1342
|
+
const b = await store.createNote("second", { id: "p2" });
|
|
1343
|
+
pinUpdatedAt(b.id, "2026-04-02T00:00:00.000Z");
|
|
1344
|
+
|
|
1345
|
+
const page1 = await store.queryNotesPaged({});
|
|
1346
|
+
// Both notes returned, cursor advances to "second"'s watermark.
|
|
1347
|
+
expect(page1.notes.map((n) => n.id).sort()).toEqual(["p1", "p2"]);
|
|
1348
|
+
|
|
1349
|
+
// No new writes yet — second call should be empty.
|
|
1350
|
+
const page2 = await store.queryNotesPaged({ cursor: page1.next_cursor });
|
|
1351
|
+
expect(page2.notes).toHaveLength(0);
|
|
1352
|
+
|
|
1353
|
+
// Now write a new note; third call should return only it.
|
|
1354
|
+
const c = await store.createNote("third", { id: "p3" });
|
|
1355
|
+
pinUpdatedAt(c.id, "2026-04-03T00:00:00.000Z");
|
|
1356
|
+
const page3 = await store.queryNotesPaged({ cursor: page2.next_cursor });
|
|
1357
|
+
expect(page3.notes.map((n) => n.id)).toEqual(["p3"]);
|
|
1358
|
+
});
|
|
1359
|
+
|
|
1360
|
+
it("next_cursor is always returned, even on an empty result page", async () => {
|
|
1361
|
+
// No notes at all — first call returns empty array but still a cursor.
|
|
1362
|
+
const page = await store.queryNotesPaged({});
|
|
1363
|
+
expect(page.notes).toHaveLength(0);
|
|
1364
|
+
expect(typeof page.next_cursor).toBe("string");
|
|
1365
|
+
expect(page.next_cursor.length).toBeGreaterThan(0);
|
|
1366
|
+
|
|
1367
|
+
// Decode it: empty-page sentinel watermark is millis 0 + empty id.
|
|
1368
|
+
const decoded = decodeCursor(page.next_cursor);
|
|
1369
|
+
expect(decoded.last_updated_at).toBe(0);
|
|
1370
|
+
expect(decoded.last_id).toBe("");
|
|
1371
|
+
});
|
|
1372
|
+
|
|
1373
|
+
it("cursor with stale query_hash raises CursorError (cursor_query_mismatch)", async () => {
|
|
1374
|
+
await store.createNote("a", { tags: ["x"], id: "qm1" });
|
|
1375
|
+
await store.createNote("b", { tags: ["y"], id: "qm2" });
|
|
1376
|
+
|
|
1377
|
+
const page1 = await store.queryNotesPaged({ tags: ["x"] });
|
|
1378
|
+
expect(page1.notes.map((n) => n.id)).toEqual(["qm1"]);
|
|
1379
|
+
|
|
1380
|
+
// Reuse the cursor with a different query — must reject loudly.
|
|
1381
|
+
try {
|
|
1382
|
+
await store.queryNotesPaged({ tags: ["y"], cursor: page1.next_cursor });
|
|
1383
|
+
throw new Error("expected CursorError");
|
|
1384
|
+
} catch (err: any) {
|
|
1385
|
+
expect(err.name).toBe("CursorError");
|
|
1386
|
+
expect(err.code).toBe("cursor_query_mismatch");
|
|
1387
|
+
}
|
|
1388
|
+
});
|
|
1389
|
+
|
|
1390
|
+
it("cursor with malformed payload raises CursorError (cursor_invalid)", async () => {
|
|
1391
|
+
try {
|
|
1392
|
+
await store.queryNotesPaged({ cursor: "not-a-real-cursor-!!!" });
|
|
1393
|
+
throw new Error("expected CursorError");
|
|
1394
|
+
} catch (err: any) {
|
|
1395
|
+
expect(err.name).toBe("CursorError");
|
|
1396
|
+
expect(err.code).toBe("cursor_invalid");
|
|
1397
|
+
}
|
|
1398
|
+
});
|
|
1399
|
+
|
|
1400
|
+
it("tiebreaker: two notes at the same updated_at use id as the secondary sort key", async () => {
|
|
1401
|
+
const ts = "2026-04-15T00:00:00.000Z";
|
|
1402
|
+
const a = await store.createNote("alpha", { id: "tb-a" });
|
|
1403
|
+
const b = await store.createNote("beta", { id: "tb-b" });
|
|
1404
|
+
const c = await store.createNote("gamma", { id: "tb-c" });
|
|
1405
|
+
pinUpdatedAt(a.id, ts);
|
|
1406
|
+
pinUpdatedAt(b.id, ts);
|
|
1407
|
+
pinUpdatedAt(c.id, ts);
|
|
1408
|
+
|
|
1409
|
+
// Page 1 with limit=2: should return a + b (id-ascending tiebreaker).
|
|
1410
|
+
const page1 = await store.queryNotesPaged({ limit: 2 });
|
|
1411
|
+
expect(page1.notes.map((n) => n.id)).toEqual(["tb-a", "tb-b"]);
|
|
1412
|
+
|
|
1413
|
+
// Page 2 with the cursor: should return c (id > "tb-b" at same ms).
|
|
1414
|
+
// Note: c is queried with the same query_hash since limit is
|
|
1415
|
+
// excluded from the hash inputs by design.
|
|
1416
|
+
const page2 = await store.queryNotesPaged({ limit: 2, cursor: page1.next_cursor });
|
|
1417
|
+
expect(page2.notes.map((n) => n.id)).toEqual(["tb-c"]);
|
|
1418
|
+
});
|
|
1419
|
+
|
|
1420
|
+
it("cursor advances correctly when notes share a millisecond on the page boundary", async () => {
|
|
1421
|
+
// Two notes share the exact updated_at the cursor was minted at.
|
|
1422
|
+
// The keyset predicate must include the larger-id one but NOT
|
|
1423
|
+
// duplicate the cursor's own note.
|
|
1424
|
+
const ts = "2026-04-15T12:34:56.789Z";
|
|
1425
|
+
const a = await store.createNote("first-at-ts", { id: "ms-a" });
|
|
1426
|
+
pinUpdatedAt(a.id, ts);
|
|
1427
|
+
|
|
1428
|
+
const page1 = await store.queryNotesPaged({ limit: 50 });
|
|
1429
|
+
expect(page1.notes.map((n) => n.id)).toEqual(["ms-a"]);
|
|
1430
|
+
|
|
1431
|
+
// Now write a note that lands at the EXACT same updated_at (race
|
|
1432
|
+
// window between pages). Its id sorts AFTER "ms-a".
|
|
1433
|
+
const b = await store.createNote("second-at-same-ts", { id: "ms-b" });
|
|
1434
|
+
pinUpdatedAt(b.id, ts);
|
|
1435
|
+
|
|
1436
|
+
const page2 = await store.queryNotesPaged({ cursor: page1.next_cursor });
|
|
1437
|
+
// Must NOT include "ms-a" (we already saw it), MUST include "ms-b"
|
|
1438
|
+
// (its id sorts after the cursor's last_id at the same timestamp).
|
|
1439
|
+
expect(page2.notes.map((n) => n.id)).toEqual(["ms-b"]);
|
|
1440
|
+
});
|
|
1441
|
+
|
|
1442
|
+
it("cursor invariance across reboots: a serialized cursor still resumes correctly", async () => {
|
|
1443
|
+
// Cursors are self-contained — they don't reference any server-side
|
|
1444
|
+
// state. Simulate a reboot by tearing down the DB+store and replaying
|
|
1445
|
+
// the same data; the cursor minted on instance #1 must work against
|
|
1446
|
+
// instance #2 with the same query.
|
|
1447
|
+
const a1 = await store.createNote("note-1", { id: "reboot-1", path: "n1" });
|
|
1448
|
+
pinUpdatedAt(a1.id, "2026-04-01T00:00:00.000Z");
|
|
1449
|
+
const a2 = await store.createNote("note-2", { id: "reboot-2", path: "n2" });
|
|
1450
|
+
pinUpdatedAt(a2.id, "2026-04-02T00:00:00.000Z");
|
|
1451
|
+
|
|
1452
|
+
const page1 = await store.queryNotesPaged({ limit: 1 });
|
|
1453
|
+
expect(page1.notes.map((n) => n.id)).toEqual(["reboot-1"]);
|
|
1454
|
+
const cursor = page1.next_cursor;
|
|
1455
|
+
|
|
1456
|
+
// Simulate a process restart with a fresh DB seeded the same way.
|
|
1457
|
+
db.close();
|
|
1458
|
+
db = new Database(":memory:");
|
|
1459
|
+
store = new SqliteStore(db);
|
|
1460
|
+
const a1b = await store.createNote("note-1", { id: "reboot-1", path: "n1" });
|
|
1461
|
+
pinUpdatedAt(a1b.id, "2026-04-01T00:00:00.000Z");
|
|
1462
|
+
const a2b = await store.createNote("note-2", { id: "reboot-2", path: "n2" });
|
|
1463
|
+
pinUpdatedAt(a2b.id, "2026-04-02T00:00:00.000Z");
|
|
1464
|
+
|
|
1465
|
+
// The cursor is opaque-but-portable: encodes only millis + id + hash,
|
|
1466
|
+
// none of which depend on server-side session state.
|
|
1467
|
+
const page2 = await store.queryNotesPaged({ limit: 1, cursor });
|
|
1468
|
+
expect(page2.notes.map((n) => n.id)).toEqual(["reboot-2"]);
|
|
1469
|
+
});
|
|
1470
|
+
|
|
1471
|
+
it("cursor mode rejects sort: desc (descending iteration would skip new writes)", async () => {
|
|
1472
|
+
await store.createNote("a");
|
|
1473
|
+
const page = await store.queryNotesPaged({});
|
|
1474
|
+
try {
|
|
1475
|
+
await store.queryNotesPaged({ cursor: page.next_cursor, sort: "desc" });
|
|
1476
|
+
throw new Error("expected QueryError");
|
|
1477
|
+
} catch (err: any) {
|
|
1478
|
+
expect(err.name).toBe("QueryError");
|
|
1479
|
+
expect(err.code).toBe("INVALID_QUERY");
|
|
1480
|
+
expect(err.message.toLowerCase()).toContain("ascending");
|
|
1481
|
+
}
|
|
1482
|
+
});
|
|
1483
|
+
|
|
1484
|
+
it("cursor mode rejects orderBy (mutually exclusive with updated_at keyset)", async () => {
|
|
1485
|
+
const { declareField } = await import("./indexed-fields.js");
|
|
1486
|
+
declareField(db, "priority", "INTEGER", "task");
|
|
1487
|
+
await store.createNote("a", { metadata: { priority: 1 } });
|
|
1488
|
+
const page = await store.queryNotesPaged({});
|
|
1489
|
+
try {
|
|
1490
|
+
await store.queryNotesPaged({ cursor: page.next_cursor, orderBy: "priority" });
|
|
1491
|
+
throw new Error("expected QueryError");
|
|
1492
|
+
} catch (err: any) {
|
|
1493
|
+
expect(err.name).toBe("QueryError");
|
|
1494
|
+
expect(err.code).toBe("INVALID_QUERY");
|
|
1495
|
+
expect(err.message.toLowerCase()).toContain("order_by");
|
|
1496
|
+
}
|
|
1497
|
+
});
|
|
1498
|
+
|
|
1499
|
+
it("cursor + tag filter: only notes matching the filter advance the watermark", async () => {
|
|
1500
|
+
// Cursor pagination composes with the rest of the query — the
|
|
1501
|
+
// watermark tracks the last note that matched the filter, not the
|
|
1502
|
+
// last note in the vault.
|
|
1503
|
+
const a = await store.createNote("task-1", { tags: ["task"], id: "ct-1" });
|
|
1504
|
+
pinUpdatedAt(a.id, "2026-04-01T00:00:00.000Z");
|
|
1505
|
+
const b = await store.createNote("not-a-task", { tags: ["other"], id: "ct-2" });
|
|
1506
|
+
pinUpdatedAt(b.id, "2026-04-02T00:00:00.000Z");
|
|
1507
|
+
const c = await store.createNote("task-2", { tags: ["task"], id: "ct-3" });
|
|
1508
|
+
pinUpdatedAt(c.id, "2026-04-03T00:00:00.000Z");
|
|
1509
|
+
|
|
1510
|
+
const page1 = await store.queryNotesPaged({ tags: ["task"] });
|
|
1511
|
+
expect(page1.notes.map((n) => n.id).sort()).toEqual(["ct-1", "ct-3"]);
|
|
1512
|
+
|
|
1513
|
+
const page2 = await store.queryNotesPaged({
|
|
1514
|
+
tags: ["task"],
|
|
1515
|
+
cursor: page1.next_cursor,
|
|
1516
|
+
});
|
|
1517
|
+
expect(page2.notes).toHaveLength(0);
|
|
1518
|
+
});
|
|
1519
|
+
|
|
1520
|
+
it("query_hash is stable across key-order permutations of the same query", async () => {
|
|
1521
|
+
// Two semantically-equivalent queries differing only in JS object key
|
|
1522
|
+
// order must produce the same cursor binding — otherwise an SDK that
|
|
1523
|
+
// reshuffles parameters between calls would silently invalidate the
|
|
1524
|
+
// cursor.
|
|
1525
|
+
const { computeQueryHash } = await import("./cursor.js");
|
|
1526
|
+
const h1 = computeQueryHash({
|
|
1527
|
+
tags: ["a", "b"],
|
|
1528
|
+
path: "Projects",
|
|
1529
|
+
metadata: { status: "open" },
|
|
1530
|
+
});
|
|
1531
|
+
const h2 = computeQueryHash({
|
|
1532
|
+
metadata: { status: "open" },
|
|
1533
|
+
path: "Projects",
|
|
1534
|
+
tags: ["a", "b"],
|
|
1535
|
+
});
|
|
1536
|
+
expect(h1).toBe(h2);
|
|
1537
|
+
|
|
1538
|
+
// ...and tag-array order is irrelevant (different SDKs may sort).
|
|
1539
|
+
const h3 = computeQueryHash({
|
|
1540
|
+
tags: ["b", "a"],
|
|
1541
|
+
path: "Projects",
|
|
1542
|
+
metadata: { status: "open" },
|
|
1543
|
+
});
|
|
1544
|
+
expect(h3).toBe(h1);
|
|
1545
|
+
});
|
|
1546
|
+
|
|
1547
|
+
it("MCP query-notes: cursor mode returns {notes, next_cursor} envelope", async () => {
|
|
1548
|
+
await store.createNote("a", { id: "mcp-a" });
|
|
1549
|
+
await store.createNote("b", { id: "mcp-b" });
|
|
1550
|
+
|
|
1551
|
+
const tools = generateMcpTools(store);
|
|
1552
|
+
const query = tools.find((t) => t.name === "query-notes")!;
|
|
1553
|
+
|
|
1554
|
+
// First call without cursor returns flat array (legacy shape).
|
|
1555
|
+
const firstResult = await query.execute({}) as unknown;
|
|
1556
|
+
expect(Array.isArray(firstResult)).toBe(true);
|
|
1557
|
+
|
|
1558
|
+
// Get a cursor by minting one ourselves via the store.
|
|
1559
|
+
const seed = await store.queryNotesPaged({});
|
|
1560
|
+
|
|
1561
|
+
// Second call with cursor returns the wrapped envelope.
|
|
1562
|
+
const envelope = await query.execute({ cursor: seed.next_cursor }) as any;
|
|
1563
|
+
expect(envelope).toHaveProperty("notes");
|
|
1564
|
+
expect(envelope).toHaveProperty("next_cursor");
|
|
1565
|
+
expect(Array.isArray(envelope.notes)).toBe(true);
|
|
1566
|
+
// No new writes since seed → empty page, cursor still advances.
|
|
1567
|
+
expect(envelope.notes).toHaveLength(0);
|
|
1568
|
+
expect(typeof envelope.next_cursor).toBe("string");
|
|
1569
|
+
});
|
|
1570
|
+
});
|
|
1571
|
+
|
|
1315
1572
|
it("limits results", async () => {
|
|
1316
1573
|
for (let i = 0; i < 5; i++) await store.createNote(`Note ${i}`);
|
|
1317
1574
|
const results = await store.queryNotes({ limit: 3 });
|