@openparachute/vault 0.7.0-rc.5 → 0.7.0-rc.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/src/attribution.test.ts +11 -3
- package/core/src/contract-typed-index.test.ts +228 -11
- package/core/src/core.test.ts +38 -8
- package/core/src/doctor.ts +43 -13
- package/core/src/mcp.ts +66 -26
- package/core/src/notes.ts +85 -20
- package/core/src/query-warnings.ts +110 -0
- package/core/src/schema-defaults.ts +48 -7
- package/core/src/schema.ts +314 -10
- package/core/src/search-fts-v25.test.ts +362 -0
- package/core/src/search-query.ts +0 -0
- package/core/src/seed-packs.ts +20 -13
- package/core/src/store.ts +18 -0
- package/core/src/tag-schemas.ts +121 -6
- package/core/src/types.ts +20 -0
- package/package.json +1 -1
- package/src/contract-search.test.ts +168 -0
- package/src/mcp-query-notes-search-scope.test.ts +53 -0
- package/src/onboarding-seed.test.ts +5 -2
- package/src/routes.ts +42 -53
- package/src/vault.test.ts +33 -27
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Search recall + ranking — schema v25 FTS rebuild (Wave 7 of the
|
|
3
|
+
* Reliability & Usability Program, WS2B/C, vault#551). Covers the
|
|
4
|
+
* MIGRATION-bearing risk this PR carries:
|
|
5
|
+
*
|
|
6
|
+
* - a v24 vault (single-column `content`-only notes_fts) upgrades to the
|
|
7
|
+
* v25 shape (path + content, porter stemming) via `migrateToV25`,
|
|
8
|
+
* idempotently, without touching note data;
|
|
9
|
+
* - a fresh vault gets the v25 shape directly from SCHEMA_SQL;
|
|
10
|
+
* - post-migration, a note's TITLE (path) is searchable — impossible
|
|
11
|
+
* pre-v25;
|
|
12
|
+
* - existing body-text search keeps working (no regression);
|
|
13
|
+
* - porter stemming matches regular-affix variants;
|
|
14
|
+
* - bm25 ranking is weighted so a title match outranks a passing body
|
|
15
|
+
* mention, and every result carries a legible `score`;
|
|
16
|
+
* - the sync triggers keep the index correct across insert/update
|
|
17
|
+
* (content-only, path-only, both)/delete, including notes with no path.
|
|
18
|
+
*
|
|
19
|
+
* See `src/contract-search.test.ts` for the REST/MCP-surface coverage
|
|
20
|
+
* (did_you_mean, the wrapped advanced-mode column-filter hint, sort
|
|
21
|
+
* override parity).
|
|
22
|
+
*/
|
|
23
|
+
import { describe, it, expect, beforeEach } from "bun:test";
|
|
24
|
+
import { Database } from "bun:sqlite";
|
|
25
|
+
import { SqliteStore } from "./store.js";
|
|
26
|
+
import { initSchema, SCHEMA_VERSION } from "./schema.js";
|
|
27
|
+
import * as noteOps from "./notes.js";
|
|
28
|
+
|
|
29
|
+
let store: SqliteStore;
|
|
30
|
+
let db: Database;
|
|
31
|
+
|
|
32
|
+
beforeEach(() => {
|
|
33
|
+
db = new Database(":memory:");
|
|
34
|
+
store = new SqliteStore(db);
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
describe("search — schema v25 FTS rebuild", () => {
|
|
38
|
+
it("bumped SCHEMA_VERSION to at least 25 (notes_fts path+content landed)", () => {
|
|
39
|
+
expect(SCHEMA_VERSION).toBeGreaterThanOrEqual(25);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it("a fresh vault gets the v25 notes_fts shape directly (path + content columns)", () => {
|
|
43
|
+
const cols = (db.prepare("PRAGMA table_info(notes_fts)").all() as { name: string }[]).map(
|
|
44
|
+
(r) => r.name,
|
|
45
|
+
);
|
|
46
|
+
expect(cols).toContain("path");
|
|
47
|
+
expect(cols).toContain("content");
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
it("a fresh vault: a note's title (path) is searchable even with unrelated body content", async () => {
|
|
51
|
+
await store.createNote("nothing relevant in the body here", {
|
|
52
|
+
path: "quarterly-budget-review",
|
|
53
|
+
});
|
|
54
|
+
const hits = await store.searchNotes("budget");
|
|
55
|
+
expect(hits.map((n) => n.path)).toContain("quarterly-budget-review");
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it("a fresh vault: existing body search still works (no regression)", async () => {
|
|
59
|
+
await store.createNote("the widget shipped on time", { path: "unrelated-title" });
|
|
60
|
+
const hits = await store.searchNotes("widget");
|
|
61
|
+
expect(hits.map((n) => n.path)).toContain("unrelated-title");
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
describe("search — v24 → v25 migration (legacy vault upgrade)", () => {
|
|
66
|
+
/**
|
|
67
|
+
* Build a v24-shaped vault by hand: the OLD single-column notes_fts
|
|
68
|
+
* (content only), the OLD triggers (UPDATE OF content only — no path
|
|
69
|
+
* sync), and a handful of notes seeded BEFORE the migration runs —
|
|
70
|
+
* mirroring the real upgrade scenario (existing data, not a fresh DB).
|
|
71
|
+
* `initSchema` then drives the whole migration chain up through v25.
|
|
72
|
+
*/
|
|
73
|
+
function buildLegacyV24Vault(): Database {
|
|
74
|
+
const legacy = new Database(":memory:");
|
|
75
|
+
legacy.exec(`
|
|
76
|
+
CREATE TABLE notes (
|
|
77
|
+
id TEXT PRIMARY KEY,
|
|
78
|
+
content TEXT DEFAULT '',
|
|
79
|
+
path TEXT,
|
|
80
|
+
metadata TEXT DEFAULT '{}',
|
|
81
|
+
created_at TEXT NOT NULL,
|
|
82
|
+
updated_at TEXT,
|
|
83
|
+
extension TEXT NOT NULL DEFAULT 'md',
|
|
84
|
+
created_by TEXT,
|
|
85
|
+
created_via TEXT,
|
|
86
|
+
last_updated_by TEXT,
|
|
87
|
+
last_updated_via TEXT
|
|
88
|
+
);
|
|
89
|
+
CREATE TABLE tags (
|
|
90
|
+
name TEXT PRIMARY KEY,
|
|
91
|
+
description TEXT,
|
|
92
|
+
fields TEXT,
|
|
93
|
+
relationships TEXT,
|
|
94
|
+
parent_names TEXT,
|
|
95
|
+
created_at TEXT,
|
|
96
|
+
updated_at TEXT
|
|
97
|
+
);
|
|
98
|
+
CREATE TABLE note_tags (
|
|
99
|
+
note_id TEXT NOT NULL,
|
|
100
|
+
tag_name TEXT NOT NULL,
|
|
101
|
+
PRIMARY KEY (note_id, tag_name)
|
|
102
|
+
);
|
|
103
|
+
CREATE TABLE indexed_fields (
|
|
104
|
+
field TEXT PRIMARY KEY,
|
|
105
|
+
sqlite_type TEXT NOT NULL,
|
|
106
|
+
declarer_tags TEXT NOT NULL DEFAULT '[]'
|
|
107
|
+
);
|
|
108
|
+
CREATE TABLE schema_version (version INTEGER PRIMARY KEY, applied_at TEXT);
|
|
109
|
+
INSERT INTO schema_version (version, applied_at) VALUES (24, '2026-01-01T00:00:00.000Z');
|
|
110
|
+
|
|
111
|
+
CREATE VIRTUAL TABLE notes_fts USING fts5(
|
|
112
|
+
content,
|
|
113
|
+
content='notes',
|
|
114
|
+
content_rowid='rowid'
|
|
115
|
+
);
|
|
116
|
+
CREATE TRIGGER notes_fts_insert AFTER INSERT ON notes BEGIN
|
|
117
|
+
INSERT INTO notes_fts(rowid, content) VALUES (new.rowid, new.content);
|
|
118
|
+
END;
|
|
119
|
+
CREATE TRIGGER notes_fts_delete AFTER DELETE ON notes BEGIN
|
|
120
|
+
INSERT INTO notes_fts(notes_fts, rowid, content) VALUES('delete', old.rowid, old.content);
|
|
121
|
+
END;
|
|
122
|
+
CREATE TRIGGER notes_fts_update AFTER UPDATE OF content ON notes BEGIN
|
|
123
|
+
INSERT INTO notes_fts(notes_fts, rowid, content) VALUES('delete', old.rowid, old.content);
|
|
124
|
+
INSERT INTO notes_fts(rowid, content) VALUES (new.rowid, new.content);
|
|
125
|
+
END;
|
|
126
|
+
`);
|
|
127
|
+
|
|
128
|
+
// Seed BEFORE migration — real-data-like: some notes with a path, one
|
|
129
|
+
// without (path IS NULL — the coalesce-to-'' path must not throw).
|
|
130
|
+
const insert = legacy.prepare(
|
|
131
|
+
"INSERT INTO notes (id, content, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)",
|
|
132
|
+
);
|
|
133
|
+
insert.run(
|
|
134
|
+
"legacy-1",
|
|
135
|
+
"a dedicated writeup mentioning propolis only once in passing text",
|
|
136
|
+
"beekeeping-notes",
|
|
137
|
+
"2026-01-01T00:00:00.000Z",
|
|
138
|
+
"2026-01-01T00:00:00.000Z",
|
|
139
|
+
);
|
|
140
|
+
insert.run(
|
|
141
|
+
"legacy-2",
|
|
142
|
+
"nothing about the topic here at all",
|
|
143
|
+
"propolis",
|
|
144
|
+
"2026-01-02T00:00:00.000Z",
|
|
145
|
+
"2026-01-02T00:00:00.000Z",
|
|
146
|
+
);
|
|
147
|
+
insert.run(
|
|
148
|
+
"legacy-3",
|
|
149
|
+
"the firefighters responded quickly to the call",
|
|
150
|
+
null,
|
|
151
|
+
"2026-01-03T00:00:00.000Z",
|
|
152
|
+
"2026-01-03T00:00:00.000Z",
|
|
153
|
+
);
|
|
154
|
+
// Manually populate the OLD single-column FTS index (mirrors what the
|
|
155
|
+
// old triggers would have done on real inserts).
|
|
156
|
+
legacy.exec(`INSERT INTO notes_fts(rowid, content) SELECT rowid, content FROM notes`);
|
|
157
|
+
|
|
158
|
+
return legacy;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
it("migrates a v24 vault to the current SCHEMA_VERSION without throwing", () => {
|
|
162
|
+
const legacy = buildLegacyV24Vault();
|
|
163
|
+
expect(() => initSchema(legacy)).not.toThrow();
|
|
164
|
+
const ver = (legacy.prepare("SELECT MAX(version) AS v FROM schema_version").get() as { v: number }).v;
|
|
165
|
+
expect(ver).toBe(SCHEMA_VERSION);
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it("post-migration: notes_fts carries path + content columns", () => {
|
|
169
|
+
const legacy = buildLegacyV24Vault();
|
|
170
|
+
initSchema(legacy);
|
|
171
|
+
const cols = (legacy.prepare("PRAGMA table_info(notes_fts)").all() as { name: string }[]).map(
|
|
172
|
+
(r) => r.name,
|
|
173
|
+
);
|
|
174
|
+
expect(cols).toContain("path");
|
|
175
|
+
expect(cols).toContain("content");
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
it("post-migration: a note's TITLE is searchable (impossible pre-v25)", () => {
|
|
179
|
+
const legacy = buildLegacyV24Vault();
|
|
180
|
+
initSchema(legacy);
|
|
181
|
+
// "beekeeping-notes" (legacy-1's path) never appears in any note's body.
|
|
182
|
+
const hits = noteOps.searchNotes(legacy, "beekeeping");
|
|
183
|
+
expect(hits.map((n) => n.id)).toContain("legacy-1");
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
it("post-migration: existing body search still works (no regression from the rebuild)", () => {
|
|
187
|
+
const legacy = buildLegacyV24Vault();
|
|
188
|
+
initSchema(legacy);
|
|
189
|
+
const hits = noteOps.searchNotes(legacy, "firefighters");
|
|
190
|
+
expect(hits.map((n) => n.id)).toContain("legacy-3");
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
it("post-migration: porter stemming matches a regular-affix variant not present verbatim", () => {
|
|
194
|
+
const legacy = buildLegacyV24Vault();
|
|
195
|
+
initSchema(legacy);
|
|
196
|
+
// legacy-3's body has "firefighters" (plural) — singular query must match.
|
|
197
|
+
const hits = noteOps.searchNotes(legacy, "firefighter");
|
|
198
|
+
expect(hits.map((n) => n.id)).toContain("legacy-3");
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
it("post-migration: a title match ranks ABOVE a passing body-only mention, and both carry a score", () => {
|
|
202
|
+
const legacy = buildLegacyV24Vault();
|
|
203
|
+
initSchema(legacy);
|
|
204
|
+
// legacy-2's path IS "propolis" (title match); legacy-1's body mentions
|
|
205
|
+
// "propolis" once in passing (body-only match). Weighted bm25 must rank
|
|
206
|
+
// the title match first.
|
|
207
|
+
const hits = noteOps.searchNotes(legacy, "propolis");
|
|
208
|
+
const ids = hits.map((n) => n.id);
|
|
209
|
+
expect(ids.indexOf("legacy-2")).toBeLessThan(ids.indexOf("legacy-1"));
|
|
210
|
+
for (const n of hits) {
|
|
211
|
+
expect(typeof n.score).toBe("number");
|
|
212
|
+
}
|
|
213
|
+
const byId = new Map(hits.map((n) => [n.id, n.score!]));
|
|
214
|
+
expect(byId.get("legacy-2")!).toBeGreaterThan(byId.get("legacy-1")!);
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
it("post-migration: a note that had NO path (path IS NULL) survived the rebuild and stays body-searchable", () => {
|
|
218
|
+
const legacy = buildLegacyV24Vault();
|
|
219
|
+
initSchema(legacy);
|
|
220
|
+
const hits = noteOps.searchNotes(legacy, "firefighters");
|
|
221
|
+
expect(hits.some((n) => n.id === "legacy-3")).toBe(true);
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
it("post-migration: the FTS index isn't duplicated or corrupted — an integrity-check passes", () => {
|
|
225
|
+
const legacy = buildLegacyV24Vault();
|
|
226
|
+
initSchema(legacy);
|
|
227
|
+
// FTS5's built-in consistency check: throws if the shadow tables and
|
|
228
|
+
// the external-content table have drifted apart (e.g. double-inserted
|
|
229
|
+
// rows from a repopulation bug).
|
|
230
|
+
expect(() => legacy.exec(`INSERT INTO notes_fts(notes_fts) VALUES('integrity-check')`)).not.toThrow();
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
it("migration is idempotent — running initSchema twice does not duplicate rows or throw", () => {
|
|
234
|
+
const legacy = buildLegacyV24Vault();
|
|
235
|
+
initSchema(legacy);
|
|
236
|
+
expect(() => initSchema(legacy)).not.toThrow();
|
|
237
|
+
const hits = noteOps.searchNotes(legacy, "firefighter");
|
|
238
|
+
// Exactly one match, not duplicated by a second repopulation pass.
|
|
239
|
+
expect(hits.filter((n) => n.id === "legacy-3").length).toBe(1);
|
|
240
|
+
expect(() => legacy.exec(`INSERT INTO notes_fts(notes_fts) VALUES('integrity-check')`)).not.toThrow();
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
it("migration is idempotent on a vault that was ALREADY on v25 (e.g. a fresh vault) — no rebuild, no data loss", async () => {
|
|
244
|
+
// `store`/`db` from the outer beforeEach is already a fresh v25 vault.
|
|
245
|
+
await store.createNote("hello world", { path: "greeting" });
|
|
246
|
+
expect(() => initSchema(db)).not.toThrow();
|
|
247
|
+
const hits = await store.searchNotes("hello");
|
|
248
|
+
expect(hits.length).toBe(1);
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* MUST-FIX (generalist review, #565) — LOAD-BEARING regression guard: the
|
|
253
|
+
* whole DDL + repopulation runs inside ONE `transaction`, so a crash partway
|
|
254
|
+
* through can NEVER leave a recreated-but-EMPTY notes_fts (which the
|
|
255
|
+
* path-column idempotency guard would then treat as "done," leaving search
|
|
256
|
+
* permanently empty).
|
|
257
|
+
*
|
|
258
|
+
* This drives the REAL `migrateToV25` (via `initSchema`) and injects the
|
|
259
|
+
* crash by monkey-patching `db.prepare` to throw on the repopulation
|
|
260
|
+
* `INSERT INTO notes_fts(rowid, path, content)` statement — a faithful
|
|
261
|
+
* mid-migration interruption AFTER the rebuild DDL (which runs via
|
|
262
|
+
* `db.exec`, untouched by the patch) but during repopulation. It does NOT
|
|
263
|
+
* hand-roll its own copy of the DDL: an earlier draft did, which made it a
|
|
264
|
+
* generic "transaction() rolls back DDL" test that stayed green even against
|
|
265
|
+
* the pre-fix unwrapped-DDL code. Driving `initSchema` means the test
|
|
266
|
+
* actually guards `migrateToV25`'s OWN transaction wrapping — verified by
|
|
267
|
+
* reverting the fix locally (DDL moved back outside `transaction`) and
|
|
268
|
+
* confirming this test goes RED (the post-crash `ftsCols()` reads
|
|
269
|
+
* `["path","content"]`, the guard skips on restart, and recovery search is
|
|
270
|
+
* empty).
|
|
271
|
+
*
|
|
272
|
+
* The load-bearing assertion is (a): post-crash `ftsCols() === ["content"]`
|
|
273
|
+
* — the rollback restored the pre-v25 shape. Under the unwrapped-DDL
|
|
274
|
+
* regression the recreated table would still carry `path` here.
|
|
275
|
+
*/
|
|
276
|
+
it("a REAL interrupted migrateToV25 (crash during repopulation) rolls back to the v24 shape — the next initSchema fully recovers, never silent-empty", () => {
|
|
277
|
+
const legacy = buildLegacyV24Vault();
|
|
278
|
+
|
|
279
|
+
const ftsCols = () =>
|
|
280
|
+
(legacy.prepare("PRAGMA table_info(notes_fts)").all() as { name: string }[]).map((c) => c.name);
|
|
281
|
+
const ftsShadowTables = () =>
|
|
282
|
+
(
|
|
283
|
+
legacy
|
|
284
|
+
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'notes_fts%' ORDER BY name")
|
|
285
|
+
.all() as { name: string }[]
|
|
286
|
+
).map((r) => r.name);
|
|
287
|
+
|
|
288
|
+
// Sanity: starts on the OLD single-column shape.
|
|
289
|
+
expect(ftsCols()).toEqual(["content"]);
|
|
290
|
+
const shadowsBefore = ftsShadowTables();
|
|
291
|
+
|
|
292
|
+
// Monkey-patch prepare so migrateToV25's repopulation INSERT throws —
|
|
293
|
+
// everything else (earlier migrations, the guard's PRAGMA, the
|
|
294
|
+
// repopulation SELECT, the DDL via db.exec) delegates to the real prepare.
|
|
295
|
+
const origPrepare = legacy.prepare.bind(legacy);
|
|
296
|
+
let injected = false;
|
|
297
|
+
(legacy as any).prepare = (sql: string) => {
|
|
298
|
+
if (!injected && /INSERT INTO notes_fts\(rowid, path, content\)/.test(sql)) {
|
|
299
|
+
injected = true;
|
|
300
|
+
throw new Error("simulated crash during notes_fts repopulation");
|
|
301
|
+
}
|
|
302
|
+
return origPrepare(sql);
|
|
303
|
+
};
|
|
304
|
+
|
|
305
|
+
// initSchema runs the whole migration chain; migrateToV25 throws mid-repopulation.
|
|
306
|
+
expect(() => initSchema(legacy)).toThrow("simulated crash during notes_fts repopulation");
|
|
307
|
+
// Positive control: the injection actually fired (not a vacuous pass).
|
|
308
|
+
expect(injected).toBe(true);
|
|
309
|
+
|
|
310
|
+
// Restore the real prepare for the assertions + recovery run.
|
|
311
|
+
(legacy as any).prepare = origPrepare;
|
|
312
|
+
|
|
313
|
+
// (a) LOAD-BEARING: rollback restored the v24 single-column shape. Under
|
|
314
|
+
// the pre-fix shape (DDL OUTSIDE the transaction) this reads
|
|
315
|
+
// ["path","content"], the hasColumn(notes_fts,"path") guard skips on
|
|
316
|
+
// restart, and search stays silently empty forever.
|
|
317
|
+
expect(ftsCols()).toEqual(["content"]);
|
|
318
|
+
// schema_version never advanced past the fixture's 24.
|
|
319
|
+
expect(
|
|
320
|
+
(legacy.prepare("SELECT MAX(version) AS v FROM schema_version").get() as { v: number }).v,
|
|
321
|
+
).toBe(24);
|
|
322
|
+
// No orphaned/duplicated FTS shadow tables — the set is exactly what it was.
|
|
323
|
+
expect(ftsShadowTables()).toEqual(shadowsBefore);
|
|
324
|
+
// The restored v24 index is intact + consistent (integrity-check passes).
|
|
325
|
+
expect(() => legacy.exec(`INSERT INTO notes_fts(notes_fts) VALUES('integrity-check')`)).not.toThrow();
|
|
326
|
+
|
|
327
|
+
// (b) A clean restart re-runs the REAL migration and fully populates
|
|
328
|
+
// search — both a title-only term and a body term are findable.
|
|
329
|
+
initSchema(legacy);
|
|
330
|
+
expect(ftsCols()).toContain("path");
|
|
331
|
+
expect(noteOps.searchNotes(legacy, "beekeeping").map((n) => n.id)).toContain("legacy-1"); // title-only
|
|
332
|
+
expect(noteOps.searchNotes(legacy, "firefighters").map((n) => n.id)).toContain("legacy-3"); // body
|
|
333
|
+
expect(() => legacy.exec(`INSERT INTO notes_fts(notes_fts) VALUES('integrity-check')`)).not.toThrow();
|
|
334
|
+
});
|
|
335
|
+
});
|
|
336
|
+
|
|
337
|
+
describe("search — FTS sync triggers keep path + content current (schema v25)", () => {
|
|
338
|
+
it("a note created with no path is indexed fine (empty path, not an error)", async () => {
|
|
339
|
+
const note = await store.createNote("body text only, no title");
|
|
340
|
+
const hits = await store.searchNotes("body");
|
|
341
|
+
expect(hits.map((n) => n.id)).toContain(note.id);
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
it("a path-ONLY update (content untouched) is now synced to the index — pre-v25 this was invisible to notes_fts", async () => {
|
|
345
|
+
const note = await store.createNote("unrelated body text", { path: "oldtitle" });
|
|
346
|
+
expect((await store.searchNotes("oldtitle")).map((n) => n.id)).toContain(note.id);
|
|
347
|
+
await store.updateNote(note.id, { path: "newtitle" });
|
|
348
|
+
expect((await store.searchNotes("newtitle")).map((n) => n.id)).toContain(note.id);
|
|
349
|
+
expect((await store.searchNotes("oldtitle")).map((n) => n.id)).not.toContain(note.id);
|
|
350
|
+
});
|
|
351
|
+
|
|
352
|
+
it("deleting a note removes it from both the path and content index", async () => {
|
|
353
|
+
const note = await store.createNote("nothing in the body matches this term", {
|
|
354
|
+
path: "uniquetitleterm",
|
|
355
|
+
});
|
|
356
|
+
expect((await store.searchNotes("uniquetitleterm")).map((n) => n.id)).toContain(note.id);
|
|
357
|
+
expect((await store.searchNotes("nothing")).map((n) => n.id)).toContain(note.id);
|
|
358
|
+
await store.deleteNote(note.id);
|
|
359
|
+
expect((await store.searchNotes("uniquetitleterm")).map((n) => n.id)).not.toContain(note.id);
|
|
360
|
+
expect((await store.searchNotes("nothing")).map((n) => n.id)).not.toContain(note.id);
|
|
361
|
+
});
|
|
362
|
+
});
|
package/core/src/search-query.ts
CHANGED
|
Binary file
|
package/core/src/seed-packs.ts
CHANGED
|
@@ -468,9 +468,9 @@ update-tag {
|
|
|
468
468
|
tag: "meeting",
|
|
469
469
|
description: "A meeting with notes",
|
|
470
470
|
fields: {
|
|
471
|
-
held_on: { type: "string", indexed: true },
|
|
472
|
-
status: { type: "string", enum: ["scheduled", "done"] },
|
|
473
|
-
rating: { type: "integer" }
|
|
471
|
+
held_on: { type: "string", indexed: true }, // queryable with operators
|
|
472
|
+
status: { type: "string", enum: ["scheduled", "done"], default: "scheduled" }, // explicit default — declare one to auto-fill
|
|
473
|
+
rating: { type: "integer" } // no default — stays unset until written
|
|
474
474
|
}
|
|
475
475
|
}
|
|
476
476
|
\`\`\`
|
|
@@ -490,16 +490,23 @@ A few behaviors worth knowing before you write at scale:
|
|
|
490
490
|
conflict error (re-read, reconcile, retry). For bulk/scripted writes where
|
|
491
491
|
concurrency is known-safe, pass \`force: true\` to waive the *requirement to
|
|
492
492
|
supply* it. \`append\`/\`prepend\`-only updates are exempt (no-conflict-by-design).
|
|
493
|
-
- **A schema field
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
- **Validation is advisory, never blocking
|
|
501
|
-
|
|
502
|
-
|
|
493
|
+
- **A schema field only back-fills when you declare an explicit \`default\`.**
|
|
494
|
+
When a note gets a tag whose schema declares a field with a \`default\`, an
|
|
495
|
+
unset field is filled with THAT value. A field with no \`default\` stays
|
|
496
|
+
genuinely absent — \`query-notes { metadata: { rating: { exists: false } } }\`
|
|
497
|
+
reliably finds notes that never set \`rating\`. Declare \`default\` on a field
|
|
498
|
+
only when "unset" and "explicitly set to X" should read the same; leave it
|
|
499
|
+
off when you need to tell them apart.
|
|
500
|
+
- **Validation is advisory, never blocking — EXCEPT an \`indexed: true\`
|
|
501
|
+
field's type.** A type/enum mismatch on a non-indexed field (or a
|
|
502
|
+
non-\`strict\` constraint on any field) comes back as a \`validation_status\`
|
|
503
|
+
warning and the write still lands — read those warnings and self-correct on
|
|
504
|
+
the next turn. An \`indexed: true\` field's TYPE is always enforced, though:
|
|
505
|
+
writing a string into an indexed \`integer\` field is REJECTED with a
|
|
506
|
+
\`schema_validation\` error, because a bad-typed value would otherwise
|
|
507
|
+
silently poison range queries (\`gt\`/\`gte\`/\`lt\`/\`lte\`) on that field. Mark a
|
|
508
|
+
field \`strict: true\` to enforce its OTHER constraints
|
|
509
|
+
(enum/required/cardinality) the same hard way.
|
|
503
510
|
|
|
504
511
|
(Full design guide, with copy-paste examples: https://parachute.computer/scripting/)
|
|
505
512
|
|
package/core/src/store.ts
CHANGED
|
@@ -741,6 +741,24 @@ export class BunSqliteStore implements Store {
|
|
|
741
741
|
// Throws IndexedFieldError on an invalid identifier (e.g. kebab-case).
|
|
742
742
|
indexedFieldOps.validateFieldName(fieldName);
|
|
743
743
|
}
|
|
744
|
+
// Default-conformance pre-validate (vault#553 Decision B) — mirrors the
|
|
745
|
+
// indexed-type/name checks above: fail BEFORE any persistence so a bad
|
|
746
|
+
// `default` never gets written. Runs over EVERY field in the full
|
|
747
|
+
// (already-merged) `nextFields` map, not just indexed ones — a default
|
|
748
|
+
// that doesn't match its own field's type/enum is a tag-schema error
|
|
749
|
+
// regardless of queryability. This is the REST parity path: MCP's
|
|
750
|
+
// `update-tag` tool ALSO calls `collectTagFieldViolations` before
|
|
751
|
+
// reaching here (bundling every violation into one `TagFieldConflictError`
|
|
752
|
+
// — see that function's doc comment), so a conforming MCP call never
|
|
753
|
+
// trips this; a non-conforming one throws there first. REST has no such
|
|
754
|
+
// pre-check, so this is REST's only gate — same asymmetry as the
|
|
755
|
+
// indexed-type/name checks it sits beside.
|
|
756
|
+
for (const [fieldName, spec] of Object.entries(nextFields ?? {})) {
|
|
757
|
+
const violation = tagSchemaOps.validateFieldDefault(fieldName, spec);
|
|
758
|
+
if (violation) {
|
|
759
|
+
throw new tagSchemaOps.InvalidFieldDefaultError(fieldName, violation.message);
|
|
760
|
+
}
|
|
761
|
+
}
|
|
744
762
|
}
|
|
745
763
|
|
|
746
764
|
// Persist the record + reconcile the indexed-field lifecycle atomically.
|
package/core/src/tag-schemas.ts
CHANGED
|
@@ -44,6 +44,25 @@ export interface TagFieldSchema {
|
|
|
44
44
|
// "one" (scalar, default) or "many" (array). Advisory unless `strict:true`.
|
|
45
45
|
// Distinct from relationship cardinality. vault#299.
|
|
46
46
|
cardinality?: "one" | "many";
|
|
47
|
+
/**
|
|
48
|
+
* Explicit backfill value (vault#553 Decision B). When a note gains this
|
|
49
|
+
* tag and doesn't set the field, `applySchemaDefaults` (core/src/mcp.ts)
|
|
50
|
+
* writes THIS value onto the note. Undeclared (the default) means the
|
|
51
|
+
* field stays ABSENT on notes that don't set it — no more implicit
|
|
52
|
+
* first-enum-value / type-zero-value backfill. This is what makes
|
|
53
|
+
* `exists:false` trustworthy: "never set" and "explicitly set to the
|
|
54
|
+
* default" are now distinguishable states.
|
|
55
|
+
*
|
|
56
|
+
* Must conform to this field's own `type` (and `enum`, when declared) —
|
|
57
|
+
* `upsertTagRecord` (core/src/store.ts, the single chokepoint both REST
|
|
58
|
+
* and MCP funnel through) validates the default BEFORE persisting via
|
|
59
|
+
* {@link validateFieldDefault}; a non-conforming default is a tag-schema
|
|
60
|
+
* error (`invalid_field_default` / `TagFieldViolation` reason
|
|
61
|
+
* `"invalid_default"`), not a silent typo. `undefined` is JSON-serialized
|
|
62
|
+
* away by `JSON.stringify` — an omitted `default` key round-trips as "not
|
|
63
|
+
* declared", never as a stored `null`.
|
|
64
|
+
*/
|
|
65
|
+
default?: unknown;
|
|
47
66
|
}
|
|
48
67
|
|
|
49
68
|
/**
|
|
@@ -147,6 +166,86 @@ export function getTagRecord(db: Database, tag: string): TagRecord | null {
|
|
|
147
166
|
* for a tag-scoped caller (`scrubParentCycleError` in src/tag-scope.ts),
|
|
148
167
|
* same posture as `TagFieldConflictError`.
|
|
149
168
|
*/
|
|
169
|
+
/**
|
|
170
|
+
* Thrown by `upsertTagRecord` (core/src/store.ts's chokepoint — REST
|
|
171
|
+
* `PUT /api/tags/:name` and MCP `update-tag` both funnel through it) when a
|
|
172
|
+
* field's declared `default` (vault#553 Decision B) doesn't conform to that
|
|
173
|
+
* SAME field's own `type` (or `enum`, when declared). A bad default is a
|
|
174
|
+
* tag-schema authoring error, not silently-accepted junk that would poison
|
|
175
|
+
* every note gaining the tag — fail closed, same posture as
|
|
176
|
+
* `IndexedFieldError` and `ParentCycleError`. `error_type` is stamped
|
|
177
|
+
* directly (own-field validation leaf, one caller-facing shape) so the
|
|
178
|
+
* generic MCP domain-error mapping (`src/mcp-http.ts`) and REST's dedicated
|
|
179
|
+
* catch branch both surface it without a bespoke class-specific mapping.
|
|
180
|
+
*/
|
|
181
|
+
export class InvalidFieldDefaultError extends Error {
|
|
182
|
+
code = "INVALID_FIELD_DEFAULT" as const;
|
|
183
|
+
error_type = "invalid_field_default" as const;
|
|
184
|
+
field: string;
|
|
185
|
+
|
|
186
|
+
constructor(field: string, message: string) {
|
|
187
|
+
super(message);
|
|
188
|
+
this.name = "InvalidFieldDefaultError";
|
|
189
|
+
this.field = field;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Validate that a field's declared `default` (vault#553 Decision B) conforms
|
|
195
|
+
* to its own `type` and (when declared) `enum`. Returns `null` when the
|
|
196
|
+
* field has no `default` (the common case) or the default is valid;
|
|
197
|
+
* otherwise a {@link TagFieldViolation} with `reason: "invalid_default"`.
|
|
198
|
+
* Pure — never throws; the two callers (the store chokepoint's fail-fast
|
|
199
|
+
* pre-validate, and {@link collectTagFieldViolations}'s bundled MCP report)
|
|
200
|
+
* each decide how to surface it.
|
|
201
|
+
*
|
|
202
|
+
* Deliberately independent of `schema-defaults.ts`'s `valueMatchesType` (this
|
|
203
|
+
* module doesn't import that one, and vice versa — no cycle either way) so
|
|
204
|
+
* the two modules' type vocabularies can drift without cross-coupling; the
|
|
205
|
+
* rules are kept in lockstep by hand (string/number/integer/boolean/array/
|
|
206
|
+
* object, the SAME six as `TagFieldSchema.type`'s documented vocabulary).
|
|
207
|
+
*/
|
|
208
|
+
export function validateFieldDefault(field: string, spec: TagFieldSchema): TagFieldViolation | null {
|
|
209
|
+
if (spec.default === undefined) return null;
|
|
210
|
+
const value = spec.default;
|
|
211
|
+
const typeOk = defaultMatchesType(value, spec.type);
|
|
212
|
+
if (!typeOk) {
|
|
213
|
+
return {
|
|
214
|
+
field,
|
|
215
|
+
reason: "invalid_default",
|
|
216
|
+
message: `field "${field}" declares default ${JSON.stringify(value)}, which does not match its own type "${spec.type}"`,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
if (spec.enum && spec.enum.length > 0 && typeof value === "string" && !spec.enum.includes(value)) {
|
|
220
|
+
return {
|
|
221
|
+
field,
|
|
222
|
+
reason: "invalid_default",
|
|
223
|
+
message: `field "${field}" declares default "${value}", which is not one of its own enum values [${spec.enum.join(", ")}]`,
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
return null;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/** Same six-type vocabulary as `TagFieldSchema.type`'s doc comment. Unknown/unset types pass (nothing to check against). */
|
|
230
|
+
function defaultMatchesType(value: unknown, type: string): boolean {
|
|
231
|
+
switch (type) {
|
|
232
|
+
case "string":
|
|
233
|
+
return typeof value === "string";
|
|
234
|
+
case "number":
|
|
235
|
+
return typeof value === "number" && Number.isFinite(value);
|
|
236
|
+
case "integer":
|
|
237
|
+
return typeof value === "number" && Number.isInteger(value);
|
|
238
|
+
case "boolean":
|
|
239
|
+
return typeof value === "boolean";
|
|
240
|
+
case "array":
|
|
241
|
+
return Array.isArray(value);
|
|
242
|
+
case "object":
|
|
243
|
+
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
244
|
+
default:
|
|
245
|
+
return true;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
150
249
|
export class ParentCycleError extends Error {
|
|
151
250
|
code = "PARENT_CYCLE" as const;
|
|
152
251
|
error_type = "parent_cycle" as const;
|
|
@@ -403,7 +502,12 @@ function jsonOrNull(value: unknown): string | null {
|
|
|
403
502
|
*/
|
|
404
503
|
export interface TagFieldViolation {
|
|
405
504
|
field: string;
|
|
406
|
-
reason:
|
|
505
|
+
reason:
|
|
506
|
+
| "type_conflict"
|
|
507
|
+
| "indexed_flag_conflict"
|
|
508
|
+
| "unsupported_indexed_type"
|
|
509
|
+
| "invalid_field_name"
|
|
510
|
+
| "invalid_default";
|
|
407
511
|
message: string;
|
|
408
512
|
/**
|
|
409
513
|
* The conflicting declarer tag — present on the cross-tag reasons
|
|
@@ -522,12 +626,17 @@ export function collectCrossTagFieldViolations(
|
|
|
522
626
|
/**
|
|
523
627
|
* Full field-violation collection: {@link collectCrossTagFieldViolations}
|
|
524
628
|
* PLUS the own-field checks (unsupported type for indexing, invalid
|
|
525
|
-
* identifier
|
|
526
|
-
*
|
|
527
|
-
*
|
|
528
|
-
*
|
|
629
|
+
* identifier, and — vault#553 Decision B — a non-conforming `default`).
|
|
630
|
+
* Used by the MCP `update-tag` tool, which — unlike REST — had no prior
|
|
631
|
+
* single-violation status-code contract to preserve for those checks (its
|
|
632
|
+
* old inline loop threw an unstructured `Error` for them, same as
|
|
633
|
+
* everything else pre-#554); collecting everything here is a strict
|
|
529
634
|
* improvement. See {@link collectCrossTagFieldViolations}'s doc comment for
|
|
530
|
-
* why REST calls that narrower function directly instead of this one
|
|
635
|
+
* why REST calls that narrower function directly instead of this one — REST
|
|
636
|
+
* gets the SAME `invalid_default` coverage via `store.upsertTagRecord`'s
|
|
637
|
+
* fail-fast pre-validate (mirrors its indexed-type/name checks), just as a
|
|
638
|
+
* single-violation `InvalidFieldDefaultError` → 400 rather than a bundled
|
|
639
|
+
* `TagFieldConflictError` → 422.
|
|
531
640
|
*/
|
|
532
641
|
export function collectTagFieldViolations(
|
|
533
642
|
db: Database,
|
|
@@ -537,6 +646,12 @@ export function collectTagFieldViolations(
|
|
|
537
646
|
const violations = collectCrossTagFieldViolations(db, tag, incomingFields);
|
|
538
647
|
|
|
539
648
|
for (const [fieldName, spec] of Object.entries(incomingFields)) {
|
|
649
|
+
// Own-field default-conformance check (vault#553 Decision B) — applies
|
|
650
|
+
// to EVERY field (not just indexed ones); a bad default is a tag-schema
|
|
651
|
+
// error regardless of whether the field is queryable.
|
|
652
|
+
const defaultViolation = validateFieldDefault(fieldName, spec);
|
|
653
|
+
if (defaultViolation) violations.push(defaultViolation);
|
|
654
|
+
|
|
540
655
|
if (spec.indexed === true) {
|
|
541
656
|
const mapped = mapFieldType(spec.type);
|
|
542
657
|
if (!mapped) {
|
package/core/src/types.ts
CHANGED
|
@@ -57,6 +57,21 @@ export interface Note {
|
|
|
57
57
|
* for the exact degree semantics (self-loop = 2 under `both`).
|
|
58
58
|
*/
|
|
59
59
|
linkCount?: number;
|
|
60
|
+
/**
|
|
61
|
+
* Full-text search relevance score (vault#551 WS2C — ranking legibility).
|
|
62
|
+
* ONLY present on results from `search=`/`query-notes{search}` — every
|
|
63
|
+
* other read path (structured `queryNotes`, `getNoteById`, ...) leaves
|
|
64
|
+
* this `undefined`. Higher is more relevant — the sign-flipped weighted
|
|
65
|
+
* `bm25(notes_fts, SEARCH_WEIGHT_PATH, SEARCH_WEIGHT_CONTENT)` value (raw
|
|
66
|
+
* SQLite bm25 is negative-is-better; flipped here so external callers get
|
|
67
|
+
* the more intuitive "bigger number wins" convention). Meaningful only
|
|
68
|
+
* for RELATIVE comparison within one result set — the absolute magnitude
|
|
69
|
+
* has no fixed scale and isn't comparable across different queries. When
|
|
70
|
+
* an explicit `sort: "asc"|"desc"` overrides relevance ordering, `score`
|
|
71
|
+
* is still computed and returned (for legibility) even though it no
|
|
72
|
+
* longer determines the result order.
|
|
73
|
+
*/
|
|
74
|
+
score?: number;
|
|
60
75
|
}
|
|
61
76
|
|
|
62
77
|
// ---- Link ----
|
|
@@ -248,6 +263,11 @@ export interface NoteIndex {
|
|
|
248
263
|
preview: string;
|
|
249
264
|
/** Opt-in link degree (see `Note.linkCount`). */
|
|
250
265
|
linkCount?: number;
|
|
266
|
+
/** Full-text search relevance score (see `Note.score`). Carried onto the
|
|
267
|
+
* lean shape too — search's default response IS the lean `NoteIndex[]`
|
|
268
|
+
* (`include_content` is opt-in), so `score` would be invisible in the
|
|
269
|
+
* common case if it only lived on the full `Note` shape. */
|
|
270
|
+
score?: number;
|
|
251
271
|
}
|
|
252
272
|
|
|
253
273
|
/** Link with hydrated note summaries. */
|