@openparachute/vault 0.7.0-rc.6 → 0.7.0-rc.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/src/mcp.ts +26 -3
- package/core/src/notes.ts +85 -20
- package/core/src/query-warnings.ts +110 -0
- package/core/src/schema.ts +168 -9
- package/core/src/search-fts-v25.test.ts +362 -0
- package/core/src/search-query.ts +0 -0
- package/core/src/types.ts +20 -0
- package/package.json +1 -1
- package/src/contract-search.test.ts +168 -0
- package/src/mcp-query-notes-search-scope.test.ts +53 -0
- package/src/routes.ts +24 -1
package/core/src/mcp.ts
CHANGED
|
@@ -5,7 +5,14 @@ import * as noteOps from "./notes.js";
|
|
|
5
5
|
import { filterMetadata, MAX_BATCH_SIZE, validateExtension, ExtensionValidationError } from "./notes.js";
|
|
6
6
|
import { QueryError } from "./query-operators.js";
|
|
7
7
|
import { TAG_EXPAND_MODES, stripTagHash, suggestSimilarTag, type TagExpandMode } from "./tag-hierarchy.js";
|
|
8
|
-
import {
|
|
8
|
+
import {
|
|
9
|
+
collectUnknownTagWarnings,
|
|
10
|
+
emptySearchWarning,
|
|
11
|
+
ignoredParamWarning,
|
|
12
|
+
computeSearchDidYouMean,
|
|
13
|
+
searchDidYouMeanWarning,
|
|
14
|
+
type QueryWarning,
|
|
15
|
+
} from "./query-warnings.js";
|
|
9
16
|
import { SEARCH_MODES, buildLiteralSearchQuery, isValidSearchMode, type SearchMode } from "./search-query.js";
|
|
10
17
|
import * as linkOps from "./links.js";
|
|
11
18
|
import * as tagSchemaOps from "./tag-schemas.js";
|
|
@@ -265,7 +272,9 @@ Response shape (vault#550 — three variants, pick by what you passed):
|
|
|
265
272
|
- Cursor mode (\`cursor\` param present — including \`cursor: ""\` to bootstrap): \`{notes: [...], next_cursor}\`. See \`cursor\` below for the bootstrap flow.
|
|
266
273
|
- Warnings present (e.g. an unrecognized \`tag\`) and NOT in cursor mode: \`{notes: [...], warnings: [...]}\`. Cursor mode + warnings compose: \`{notes, next_cursor, warnings}\`. Absent \`warnings\` key means nothing to flag — don't assume its presence either way.
|
|
267
274
|
|
|
268
|
-
\`search\` is literal-by-default (vault#551): your text is escaped and phrase-quoted before it reaches FTS5, so ordinary punctuation ("didn't", "eleven-day", "18.6") is matched as literal content instead of being parsed as query syntax (a bare hyphen used to mean NOT; an apostrophe or decimal point used to break the parse and silently return \`[]\`). Pass \`search_mode: "advanced"\` to opt back into raw FTS5 syntax (AND/OR/NOT, manual phrase quoting, prefix \`*\`) — a malformed advanced query now throws a structured error instead of silently returning \`[]\`. \`sort\` is honored under \`search\` too: omit it for relevance ranking (default), or pass "asc"/"desc" to order by \`created_at\` instead
|
|
275
|
+
\`search\` is literal-by-default (vault#551): your text is escaped and phrase-quoted before it reaches FTS5, so ordinary punctuation ("didn't", "eleven-day", "18.6") is matched as literal content instead of being parsed as query syntax (a bare hyphen used to mean NOT; an apostrophe or decimal point used to break the parse and silently return \`[]\`). Pass \`search_mode: "advanced"\` to opt back into raw FTS5 syntax (AND/OR/NOT, manual phrase quoting, prefix \`*\`) — a malformed advanced query now throws a structured error instead of silently returning \`[]\`. \`sort\` is honored under \`search\` too: omit it for relevance ranking (default), or pass "asc"/"desc" to order by \`created_at\` instead.
|
|
276
|
+
|
|
277
|
+
\`search\` indexes BOTH a note's title (\`path\`) and its \`content\` (vault#551 WS2C, schema v25) — a title match is weighted far above a passing body mention, so a dedicated note on a topic outranks another note that merely references it. Every result carries a \`score\` field (higher = more relevant; only meaningful as a RELATIVE comparison within one result set). Word matching also stems regular English affixes ("firefighter" matches "firefighters", "microbe" matches "microbes") — irregular plurals with a consonant change ("wolf"/"wolves") aren't covered by stemming. A search that returns ZERO results may carry a \`search_did_you_mean\` warning suggesting the closest indexed term when one looks like a likely typo (only unscoped sessions — tag-scoped tokens never see it, since the suggestion is computed vault-wide).`,
|
|
269
278
|
inputSchema: {
|
|
270
279
|
type: "object",
|
|
271
280
|
properties: {
|
|
@@ -322,7 +331,7 @@ Response shape (vault#550 — three variants, pick by what you passed):
|
|
|
322
331
|
search: {
|
|
323
332
|
type: "string",
|
|
324
333
|
description:
|
|
325
|
-
'Full-text search query. Literal-by-default (vault#551): your text is escaped and phrase-quoted before reaching FTS5, so punctuation ("didn\'t", "eleven-day", "18.6") is matched as literal content rather than parsed as FTS5 query syntax. Pass `search_mode: "advanced"` for raw FTS5 syntax (boolean/phrase/prefix operators). `sort` is honored under search (see below) — default is relevance ranking.',
|
|
334
|
+
'Full-text search query, matched against BOTH a note\'s title (path) and its content — a title match ranks far above a passing body mention. Literal-by-default (vault#551): your text is escaped and phrase-quoted before reaching FTS5, so punctuation ("didn\'t", "eleven-day", "18.6") is matched as literal content rather than parsed as FTS5 query syntax. Pass `search_mode: "advanced"` for raw FTS5 syntax (boolean/phrase/prefix operators). `sort` is honored under search (see below) — default is relevance ranking. Matching stems regular affixes ("firefighter"/"firefighters") but not irregular plurals ("wolf"/"wolves"). Results carry a `score` field (higher = more relevant, relative within this result set only). A zero-result search may carry a `search_did_you_mean` warning (unscoped sessions only).',
|
|
326
335
|
},
|
|
327
336
|
search_mode: {
|
|
328
337
|
type: "string",
|
|
@@ -603,6 +612,20 @@ Response shape (vault#550 — three variants, pick by what you passed):
|
|
|
603
612
|
mode,
|
|
604
613
|
sort: params.sort as "asc" | "desc" | undefined,
|
|
605
614
|
});
|
|
615
|
+
// Zero-result `did_you_mean` (vault#551 WS2B) — cheap (a bounded
|
|
616
|
+
// FTS5-vocabulary scan) and ONLY computed on the already-rare
|
|
617
|
+
// empty-result path, mirroring `unknown_tag`'s did_you_mean.
|
|
618
|
+
// Scope-unaware by construction (same as `collectUnknownTagWarnings`
|
|
619
|
+
// above) — safe here because `applyTagScopeWrappers`'s
|
|
620
|
+
// `query-notes` wrapper (`src/mcp-tools.ts`) strips the ENTIRE
|
|
621
|
+
// `warnings` array for a tag-scoped session before it reaches the
|
|
622
|
+
// caller, so this never leaks out-of-scope vocabulary to one.
|
|
623
|
+
if (results.length === 0) {
|
|
624
|
+
const suggestion = computeSearchDidYouMean(db, params.search as string);
|
|
625
|
+
if (suggestion) {
|
|
626
|
+
queryWarnings.push(searchDidYouMeanWarning(params.search as string, suggestion));
|
|
627
|
+
}
|
|
628
|
+
}
|
|
606
629
|
}
|
|
607
630
|
} else {
|
|
608
631
|
// --- Structured query ---
|
package/core/src/notes.ts
CHANGED
|
@@ -22,7 +22,12 @@ import {
|
|
|
22
22
|
import { getIndexedField, releaseField } from "./indexed-fields.js";
|
|
23
23
|
import { computeExpandedTagCounts, loadTagHierarchy, stripTagHash } from "./tag-hierarchy.js";
|
|
24
24
|
import { chunkForInClause, IN_VIA_JSON_EACH, jsonEachParam } from "./sql-in.js";
|
|
25
|
-
import {
|
|
25
|
+
import {
|
|
26
|
+
buildLiteralSearchQuery,
|
|
27
|
+
SEARCH_WEIGHT_PATH,
|
|
28
|
+
SEARCH_WEIGHT_CONTENT,
|
|
29
|
+
type SearchMode,
|
|
30
|
+
} from "./search-query.js";
|
|
26
31
|
|
|
27
32
|
let idCounter = 0;
|
|
28
33
|
|
|
@@ -1336,12 +1341,41 @@ export function queryNotesPaged(db: Database, opts: QueryOpts): QueryNotesPage {
|
|
|
1336
1341
|
* invariant is ever broken it still surfaces as an honest error, not a
|
|
1337
1342
|
* 500. Signals a vault bug worth reporting.
|
|
1338
1343
|
*/
|
|
1344
|
+
/**
|
|
1345
|
+
* FTS5's own error text for a syntax mistake is written for someone who
|
|
1346
|
+
* already knows FTS5's grammar — "no such column: espresso" for a bare
|
|
1347
|
+
* leading `-espresso` (NOT with no left-hand term — FTS5's `-`/`NOT` is a
|
|
1348
|
+
* BINARY operator, so a lone `-token` with nothing before it gets
|
|
1349
|
+
* misparsed as `column:term` filter syntax and fails looking for a column
|
|
1350
|
+
* named after the token) is confusing/leaky rather than actionable: it
|
|
1351
|
+
* reads as if a column literally named "espresso" was expected, which
|
|
1352
|
+
* exposes FTS5-internal parsing behavior instead of explaining the actual
|
|
1353
|
+
* mistake (vault#551 WS2B item 4 — interim harness finding). Detected
|
|
1354
|
+
* generically off the FTS5 error text (`/no such column:/i`) rather than
|
|
1355
|
+
* re-parsing the query ourselves, since the SAME message also covers the
|
|
1356
|
+
* other real cause (an explicit `column:term` filter naming a column that
|
|
1357
|
+
* isn't one of `notes_fts`'s declared columns, `path`/`content`) — the hint
|
|
1358
|
+
* below covers both without claiming to know which one happened.
|
|
1359
|
+
*/
|
|
1360
|
+
function advancedModeColumnHint(causeMessage: string): string | null {
|
|
1361
|
+
if (!/no such column:/i.test(causeMessage)) return null;
|
|
1362
|
+
return (
|
|
1363
|
+
`FTS5 read part of this query as column-filter syntax ("column:term") or as a ` +
|
|
1364
|
+
`leading "-" with no term to its left — NOT/"-" is a BINARY operator in FTS5 ` +
|
|
1365
|
+
`("good -bad", not "-bad" alone). Indexed columns are "path" and "content". ` +
|
|
1366
|
+
`Add a preceding positive term before a NOT, quote the phrase to search it ` +
|
|
1367
|
+
`literally, or use search_mode:"literal" (the default) to skip advanced syntax entirely.`
|
|
1368
|
+
);
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1339
1371
|
function searchSyntaxError(rawQuery: string, err: unknown, mode: SearchMode): QueryError {
|
|
1340
1372
|
const causeMessage = err instanceof Error ? err.message : String(err);
|
|
1373
|
+
const columnHint = mode === "advanced" ? advancedModeColumnHint(causeMessage) : null;
|
|
1341
1374
|
const hint =
|
|
1342
|
-
|
|
1375
|
+
columnHint ??
|
|
1376
|
+
(mode === "advanced"
|
|
1343
1377
|
? `FTS5 rejected this as advanced query syntax (${causeMessage}). Fix the syntax, or omit search_mode:"advanced" for literal (punctuation-safe) search.`
|
|
1344
|
-
: `FTS5 rejected the escaped literal query (${causeMessage}) — this should be impossible after literal-mode escaping + control-char sanitization; please report it as a vault bug
|
|
1378
|
+
: `FTS5 rejected the escaped literal query (${causeMessage}) — this should be impossible after literal-mode escaping + control-char sanitization; please report it as a vault bug.`);
|
|
1345
1379
|
return new QueryError(`invalid search syntax: ${causeMessage}`, "INVALID_QUERY", {
|
|
1346
1380
|
error_type: "invalid_search_syntax",
|
|
1347
1381
|
field: "search",
|
|
@@ -1377,20 +1411,33 @@ export function searchNotes(
|
|
|
1377
1411
|
ftsQuery = query;
|
|
1378
1412
|
}
|
|
1379
1413
|
|
|
1414
|
+
// Weighted bm25 relevance expression (vault#551 WS2C, schema v25):
|
|
1415
|
+
// `notes_fts` now indexes `path` (title, column 0) then `content` (body,
|
|
1416
|
+
// column 1) — SEARCH_WEIGHT_PATH/SEARCH_WEIGHT_CONTENT bias ranking so a
|
|
1417
|
+
// title match outranks a passing body mention. Raw SQLite bm25 is
|
|
1418
|
+
// negative-is-better; `-1.0 * bm25(...)` flips it so bigger is more
|
|
1419
|
+
// relevant (see `Note.score`'s doc comment for the external contract).
|
|
1420
|
+
// The weights are our own numeric constants (not user input) interpolated
|
|
1421
|
+
// directly — bm25()'s weight arguments are positional per-column
|
|
1422
|
+
// multipliers, not general SQL expressions callers can influence.
|
|
1423
|
+
const scoreExpr = `(-1.0 * bm25(notes_fts, ${SEARCH_WEIGHT_PATH}, ${SEARCH_WEIGHT_CONTENT}))`;
|
|
1424
|
+
|
|
1380
1425
|
// `sort` honored under search (vault#551 WS2A item 3): default stays FTS5
|
|
1381
|
-
// relevance (`
|
|
1382
|
-
// to `created_at` ordering. Checked against
|
|
1383
|
-
// truthiness) so an absent `sort` can
|
|
1384
|
-
// `n.id ${direction}` is appended as a
|
|
1385
|
-
//
|
|
1386
|
-
//
|
|
1387
|
-
//
|
|
1426
|
+
// relevance (the weighted `score` expression); an EXPLICIT
|
|
1427
|
+
// `sort: "asc"|"desc"` switches to `created_at` ordering. Checked against
|
|
1428
|
+
// the literal string values (not truthiness) so an absent `sort` can
|
|
1429
|
+
// never accidentally match a branch. `n.id ${direction}` is appended as a
|
|
1430
|
+
// deterministic tiebreaker in every branch — two notes at the same
|
|
1431
|
+
// created_at millisecond (structured-sort branches) OR with an
|
|
1432
|
+
// identical weighted score (relevance branch — much more likely than
|
|
1433
|
+
// with unweighted `rank`, since many notes share "zero path matches")
|
|
1434
|
+
// would otherwise return in arbitrary/unstable order.
|
|
1388
1435
|
const orderBy =
|
|
1389
1436
|
opts?.sort === "asc"
|
|
1390
1437
|
? "n.created_at ASC, n.id ASC"
|
|
1391
1438
|
: opts?.sort === "desc"
|
|
1392
1439
|
? "n.created_at DESC, n.id DESC"
|
|
1393
|
-
: "
|
|
1440
|
+
: "score DESC, n.id ASC";
|
|
1394
1441
|
|
|
1395
1442
|
if (opts?.tags && opts.tags.length > 0) {
|
|
1396
1443
|
// Canonical-bare-tag guard backstop (vault#XXX) for direct-core callers.
|
|
@@ -1406,14 +1453,14 @@ export function searchNotes(
|
|
|
1406
1453
|
// DISTINCT over full rows. The FTS join itself is 1:1 on rowid.
|
|
1407
1454
|
const tagPlaceholders = searchTags.map(() => "?").join(", ");
|
|
1408
1455
|
const rows = db.prepare(`
|
|
1409
|
-
SELECT n
|
|
1456
|
+
SELECT n.*, ${scoreExpr} AS score FROM notes n
|
|
1410
1457
|
JOIN notes_fts fts ON fts.rowid = n.rowid
|
|
1411
1458
|
WHERE notes_fts MATCH ?
|
|
1412
1459
|
AND n.id IN (SELECT note_id FROM note_tags WHERE tag_name IN (${tagPlaceholders}))
|
|
1413
1460
|
ORDER BY ${orderBy}
|
|
1414
1461
|
LIMIT ?
|
|
1415
|
-
`).all(ftsQuery, ...searchTags, limit) as NoteRow[];
|
|
1416
|
-
return notesWithTags(db, rows);
|
|
1462
|
+
`).all(ftsQuery, ...searchTags, limit) as (NoteRow & { score: number })[];
|
|
1463
|
+
return notesWithTags(db, rows, scoresById(rows));
|
|
1417
1464
|
} catch (err) {
|
|
1418
1465
|
// Surface EVERY FTS5 error structured, never a raw rethrow (vault#551):
|
|
1419
1466
|
// advanced mode expects it (the caller passed raw syntax); literal
|
|
@@ -1429,27 +1476,44 @@ export function searchNotes(
|
|
|
1429
1476
|
|
|
1430
1477
|
try {
|
|
1431
1478
|
const rows = db.prepare(`
|
|
1432
|
-
SELECT n
|
|
1479
|
+
SELECT n.*, ${scoreExpr} AS score FROM notes n
|
|
1433
1480
|
JOIN notes_fts fts ON fts.rowid = n.rowid
|
|
1434
1481
|
WHERE notes_fts MATCH ?
|
|
1435
1482
|
ORDER BY ${orderBy}
|
|
1436
1483
|
LIMIT ?
|
|
1437
|
-
`).all(ftsQuery, limit) as NoteRow[];
|
|
1438
|
-
return notesWithTags(db, rows);
|
|
1484
|
+
`).all(ftsQuery, limit) as (NoteRow & { score: number })[];
|
|
1485
|
+
return notesWithTags(db, rows, scoresById(rows));
|
|
1439
1486
|
} catch (err) {
|
|
1440
1487
|
if (err instanceof QueryError) throw err;
|
|
1441
1488
|
throw searchSyntaxError(query, err, mode);
|
|
1442
1489
|
}
|
|
1443
1490
|
}
|
|
1444
1491
|
|
|
1445
|
-
/**
|
|
1446
|
-
|
|
1492
|
+
/**
|
|
1493
|
+
* Map rows → Notes with tags hydrated in one batched query. `scores`
|
|
1494
|
+
* (vault#551 WS2C) is an optional id → weighted-bm25-score map, attached
|
|
1495
|
+
* onto each returned `Note.score` when present — ONLY `searchNotes`'s two
|
|
1496
|
+
* callers pass it; every other caller (plain `queryNotes`, `getNotesByIds`,
|
|
1497
|
+
* ...) omits it and gets byte-identical output to before `score` existed.
|
|
1498
|
+
*/
|
|
1499
|
+
function notesWithTags(db: Database, rows: NoteRow[], scores?: Map<string, number>): Note[] {
|
|
1447
1500
|
const notes = rows.map(rowToNote);
|
|
1448
1501
|
const tagsById = getNoteTagsForNotes(db, notes.map((n) => n.id));
|
|
1449
|
-
for (const note of notes)
|
|
1502
|
+
for (const note of notes) {
|
|
1503
|
+
note.tags = tagsById.get(note.id) ?? [];
|
|
1504
|
+
if (scores) {
|
|
1505
|
+
const s = scores.get(note.id);
|
|
1506
|
+
if (s !== undefined) note.score = s;
|
|
1507
|
+
}
|
|
1508
|
+
}
|
|
1450
1509
|
return notes;
|
|
1451
1510
|
}
|
|
1452
1511
|
|
|
1512
|
+
/** Build an id → score map from search result rows carrying a `score` column. */
|
|
1513
|
+
function scoresById(rows: (NoteRow & { score: number })[]): Map<string, number> {
|
|
1514
|
+
return new Map(rows.map((r) => [r.id, r.score]));
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1453
1517
|
// ---- Tag Operations ----
|
|
1454
1518
|
|
|
1455
1519
|
export function tagNote(db: Database, noteId: string, tags: string[]): void {
|
|
@@ -2120,6 +2184,7 @@ export function toNoteIndex(note: Note): NoteIndex {
|
|
|
2120
2184
|
metadata: note.metadata,
|
|
2121
2185
|
byteSize,
|
|
2122
2186
|
preview,
|
|
2187
|
+
...(note.score !== undefined ? { score: note.score } : {}),
|
|
2123
2188
|
};
|
|
2124
2189
|
}
|
|
2125
2190
|
|
|
@@ -217,3 +217,113 @@ export function ignoredParamWarning(param: string, reason: string): QueryWarning
|
|
|
217
217
|
param,
|
|
218
218
|
};
|
|
219
219
|
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Cheap zero-result search suggestion (vault#551 WS2B — mirrors the tag
|
|
223
|
+
* `did_you_mean` above via the SAME `suggestSimilarTag` scorer, just over a
|
|
224
|
+
* different candidate pool). Callers MUST only invoke this AFTER a search
|
|
225
|
+
* already returned zero rows — it is never on the hot non-empty path, so
|
|
226
|
+
* the extra vocabulary scan costs nothing in the overwhelmingly common
|
|
227
|
+
* case where the search actually found something.
|
|
228
|
+
*
|
|
229
|
+
* Candidates are drawn from two cheap-to-reach sources, unioned:
|
|
230
|
+
*
|
|
231
|
+
* - the FTS5 vocabulary — a `notes_fts_vocab` `fts5vocab('row')` table
|
|
232
|
+
* created here LAZILY and BEST-EFFORT (not part of the schema/
|
|
233
|
+
* migration — see the try/catch below) — the terms actually indexed,
|
|
234
|
+
* POST-tokenization/stemming (porter). A suggestion therefore
|
|
235
|
+
* sometimes reads as a stemmed form ("propoli" rather than "propolis")
|
|
236
|
+
* rather than the original dictionary word — an accepted tradeoff
|
|
237
|
+
* (documented in docs/HTTP_API.md) rather than maintaining a second,
|
|
238
|
+
* unstemmed index just for spelling suggestions.
|
|
239
|
+
* - tag names, when the caller passes `tagNames` (already cached by the
|
|
240
|
+
* caller's tag hierarchy — free to include).
|
|
241
|
+
*
|
|
242
|
+
* `fts5vocab` is registered by the same FTS5 extension init as `notes_fts`
|
|
243
|
+
* itself (not a separately-enabled module) so it's expected to be
|
|
244
|
+
* available anywhere search already works — but this is NOT exercised
|
|
245
|
+
* against Cloudflare DO SQLite (flagged for the wire reviewer). The WHOLE
|
|
246
|
+
* function is wrapped in try/catch and degrades to "no suggestion" on ANY
|
|
247
|
+
* failure — including a runtime where `fts5vocab` is unavailable — rather
|
|
248
|
+
* than ever throwing; a spelling hint is a nicety, never worth risking the
|
|
249
|
+
* search response itself.
|
|
250
|
+
*
|
|
251
|
+
* Per-token length filtering (`length(term) BETWEEN len-2 AND len+2`) keeps
|
|
252
|
+
* the vocabulary scan bounded on a large vault without a hardcoded row cap
|
|
253
|
+
* that could silently exclude the very term a caller needs (a `LIMIT`
|
|
254
|
+
* ORDER-BY-frequency would bias toward common words, which is backwards
|
|
255
|
+
* for a name/typo lookup — the word you're trying to find is often rare).
|
|
256
|
+
*/
|
|
257
|
+
export function computeSearchDidYouMean(
|
|
258
|
+
db: Database,
|
|
259
|
+
rawQuery: string,
|
|
260
|
+
tagNames?: Iterable<string>,
|
|
261
|
+
): string | undefined {
|
|
262
|
+
try {
|
|
263
|
+
const tokens = rawQuery.trim().split(/\s+/).filter(Boolean);
|
|
264
|
+
if (tokens.length === 0) return undefined;
|
|
265
|
+
|
|
266
|
+
db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts_vocab USING fts5vocab(notes_fts, 'row')");
|
|
267
|
+
const exactStmt = db.prepare("SELECT 1 FROM notes_fts_vocab WHERE term = ? LIMIT 1");
|
|
268
|
+
const rangeStmt = db.prepare("SELECT term FROM notes_fts_vocab WHERE length(term) BETWEEN ? AND ?");
|
|
269
|
+
|
|
270
|
+
let changed = false;
|
|
271
|
+
const corrected = tokens.map((tok) => {
|
|
272
|
+
// Strip leading/trailing punctuation for the lookup (the FTS5
|
|
273
|
+
// vocabulary never contains punctuation) but leave short tokens
|
|
274
|
+
// (numbers, "a", "of", ...) alone — too little signal to safely
|
|
275
|
+
// "correct" without a high false-positive rate.
|
|
276
|
+
const clean = tok.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "");
|
|
277
|
+
if (clean.length < 3) return tok;
|
|
278
|
+
const lower = clean.toLowerCase();
|
|
279
|
+
|
|
280
|
+
// Already indexed verbatim (post-stemming) — no typo signal.
|
|
281
|
+
if (exactStmt.get(lower)) return tok;
|
|
282
|
+
|
|
283
|
+
const lo = Math.max(1, clean.length - 2);
|
|
284
|
+
const hi = clean.length + 2;
|
|
285
|
+
const rows = rangeStmt.all(lo, hi) as { term: string }[];
|
|
286
|
+
const candidates = new Set<string>(rows.map((r) => r.term));
|
|
287
|
+
if (tagNames) for (const t of tagNames) candidates.add(t);
|
|
288
|
+
|
|
289
|
+
const suggestion = suggestSimilarTag(candidates, lower);
|
|
290
|
+
if (suggestion && suggestion.toLowerCase() !== lower) {
|
|
291
|
+
changed = true;
|
|
292
|
+
return suggestion;
|
|
293
|
+
}
|
|
294
|
+
return tok;
|
|
295
|
+
});
|
|
296
|
+
|
|
297
|
+
if (!changed) return undefined;
|
|
298
|
+
return corrected.join(" ");
|
|
299
|
+
} catch {
|
|
300
|
+
return undefined;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* `search_did_you_mean` warning (vault#551 WS2B) — wraps
|
|
306
|
+
* {@link computeSearchDidYouMean}'s suggestion in the standard warning
|
|
307
|
+
* shape, mirroring `unknown_tag`'s `did_you_mean` field. Returns
|
|
308
|
+
* `undefined` (nothing to push) when no suggestion clears the bar — a
|
|
309
|
+
* zero-result search is a perfectly ordinary outcome on its own and does
|
|
310
|
+
* NOT warrant a warning by itself; only an actual spelling suggestion does.
|
|
311
|
+
*
|
|
312
|
+
* Scope-unaware by construction, exactly like `collectUnknownTagWarnings`
|
|
313
|
+
* (see this file's top-of-file doc comment) — the FTS5 vocabulary spans
|
|
314
|
+
* the WHOLE vault regardless of any caller's tag scope. Callers on a
|
|
315
|
+
* tag-scoped session MUST NOT surface this warning directly:
|
|
316
|
+
* `src/mcp-tools.ts`'s `query-notes` wrapper already strips the entire
|
|
317
|
+
* `warnings` array for a scoped caller (so MCP is safe with no extra
|
|
318
|
+
* work); `src/routes.ts` must gate the call itself behind
|
|
319
|
+
* `tagScope.allowed === null`, same as it already does for
|
|
320
|
+
* `collectUnknownTagWarnings`.
|
|
321
|
+
*/
|
|
322
|
+
export function searchDidYouMeanWarning(rawQuery: string, suggestion: string): QueryWarning {
|
|
323
|
+
return {
|
|
324
|
+
code: "search_did_you_mean",
|
|
325
|
+
message: `no results for "${rawQuery}" — did you mean "${suggestion}"?`,
|
|
326
|
+
query: rawQuery,
|
|
327
|
+
did_you_mean: suggestion,
|
|
328
|
+
};
|
|
329
|
+
}
|
package/core/src/schema.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { rebuildIndexes, listIndexedFields } from "./indexed-fields.js";
|
|
|
4
4
|
import { findMixedTypeIndexedFieldNotes } from "./doctor.js";
|
|
5
5
|
import { transaction } from "./txn.js";
|
|
6
6
|
|
|
7
|
-
export const SCHEMA_VERSION =
|
|
7
|
+
export const SCHEMA_VERSION = 25;
|
|
8
8
|
|
|
9
9
|
export const SCHEMA_SQL = `
|
|
10
10
|
-- Notes: the universal record.
|
|
@@ -256,25 +256,43 @@ CREATE TABLE IF NOT EXISTS schema_version (
|
|
|
256
256
|
applied_at TEXT NOT NULL
|
|
257
257
|
);
|
|
258
258
|
|
|
259
|
-
-- Full-text search on note content
|
|
259
|
+
-- Full-text search on note title (path) AND content (v25, vault#551 WS2B/C).
|
|
260
|
+
-- Two columns, declared in this order — bm25(notes_fts, w_path, w_content)
|
|
261
|
+
-- calls in core/src/notes.ts positionally match column 0 = path, column 1 =
|
|
262
|
+
-- content, weighted so a TITLE match outranks a passing body mention (see
|
|
263
|
+
-- SEARCH_WEIGHT_PATH/SEARCH_WEIGHT_CONTENT in core/src/search-query.ts).
|
|
264
|
+
-- Pre-v25, only content was indexed — a note's title/path was completely
|
|
265
|
+
-- unsearchable, which both defeated any title-biased ranking (nothing to
|
|
266
|
+
-- bias) and was a plain recall gap. tokenize='porter unicode61' adds
|
|
267
|
+
-- Porter stemming on top of the v3-era default unicode61 tokenizer so
|
|
268
|
+
-- regular-affix variants (firefighter/firefighters, microbe/microbes) match
|
|
269
|
+
-- each other; irregular plurals with a consonant change (wolf/wolves) are a
|
|
270
|
+
-- known Porter limitation, not fixed by this — see docs/HTTP_API.md.
|
|
271
|
+
-- Existing vaults upgrade via migrateToV25 (rebuild + repopulate); this
|
|
272
|
+
-- CREATE only reaches its new shape directly on a fresh vault.
|
|
260
273
|
CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(
|
|
274
|
+
path,
|
|
261
275
|
content,
|
|
262
276
|
content='notes',
|
|
263
|
-
content_rowid='rowid'
|
|
277
|
+
content_rowid='rowid',
|
|
278
|
+
tokenize='porter unicode61'
|
|
264
279
|
);
|
|
265
280
|
|
|
266
|
-
-- FTS triggers
|
|
281
|
+
-- FTS triggers. UPDATE OF content, path (not content alone, pre-v25) —
|
|
282
|
+
-- either column changing must resync the index now that path is indexed too;
|
|
283
|
+
-- a path-only rename (content untouched) used to be silently invisible to
|
|
284
|
+
-- notes_fts because the trigger's column list didn't include it.
|
|
267
285
|
CREATE TRIGGER IF NOT EXISTS notes_fts_insert AFTER INSERT ON notes BEGIN
|
|
268
|
-
INSERT INTO notes_fts(rowid, content) VALUES (new.rowid, new.content);
|
|
286
|
+
INSERT INTO notes_fts(rowid, path, content) VALUES (new.rowid, COALESCE(new.path, ''), new.content);
|
|
269
287
|
END;
|
|
270
288
|
|
|
271
289
|
CREATE TRIGGER IF NOT EXISTS notes_fts_delete AFTER DELETE ON notes BEGIN
|
|
272
|
-
INSERT INTO notes_fts(notes_fts, rowid, content) VALUES('delete', old.rowid, old.content);
|
|
290
|
+
INSERT INTO notes_fts(notes_fts, rowid, path, content) VALUES('delete', old.rowid, COALESCE(old.path, ''), old.content);
|
|
273
291
|
END;
|
|
274
292
|
|
|
275
|
-
CREATE TRIGGER IF NOT EXISTS notes_fts_update AFTER UPDATE OF content ON notes BEGIN
|
|
276
|
-
INSERT INTO notes_fts(notes_fts, rowid, content) VALUES('delete', old.rowid, old.content);
|
|
277
|
-
INSERT INTO notes_fts(rowid, content) VALUES (new.rowid, new.content);
|
|
293
|
+
CREATE TRIGGER IF NOT EXISTS notes_fts_update AFTER UPDATE OF content, path ON notes BEGIN
|
|
294
|
+
INSERT INTO notes_fts(notes_fts, rowid, path, content) VALUES('delete', old.rowid, COALESCE(old.path, ''), old.content);
|
|
295
|
+
INSERT INTO notes_fts(rowid, path, content) VALUES (new.rowid, COALESCE(new.path, ''), new.content);
|
|
278
296
|
END;
|
|
279
297
|
|
|
280
298
|
-- Indexes
|
|
@@ -507,6 +525,10 @@ export function initSchema(db: Database): void {
|
|
|
507
525
|
// leave the rest in place for `doctor` to surface. See vault#553.
|
|
508
526
|
migrateToV24(db);
|
|
509
527
|
|
|
528
|
+
// Migrate v24 → v25: rebuild notes_fts with path+content columns +
|
|
529
|
+
// porter stemming, repopulate from every existing note. See vault#551.
|
|
530
|
+
migrateToV25(db);
|
|
531
|
+
|
|
510
532
|
// Rebuild any generated columns + indexes declared in indexed_fields.
|
|
511
533
|
// No-op for a fresh vault; idempotent on existing vaults.
|
|
512
534
|
rebuildIndexes(db);
|
|
@@ -1332,6 +1354,143 @@ function migrateToV24(db: Database): void {
|
|
|
1332
1354
|
}
|
|
1333
1355
|
}
|
|
1334
1356
|
|
|
1357
|
+
/**
|
|
1358
|
+
* Migrate v24 → v25: rebuild `notes_fts` to index BOTH `path` (title) and
|
|
1359
|
+
* `content` as separate weighted FTS5 columns, with Porter stemming added
|
|
1360
|
+
* to the tokenizer (vault#551 WS2B/C — the Reliability & Usability
|
|
1361
|
+
* Program's interim-harness finding: a note's title/path was completely
|
|
1362
|
+
* unsearchable pre-v25, which both defeated title-biased ranking and was a
|
|
1363
|
+
* plain recall gap on its own).
|
|
1364
|
+
*
|
|
1365
|
+
* `notes_fts` is an EXTERNAL CONTENT table (`content='notes'`) — the FTS5
|
|
1366
|
+
* index is a side structure over `notes`, not a copy of the data itself, so
|
|
1367
|
+
* changing its column shape means DROP + CREATE, not `ALTER`. FTS5 virtual
|
|
1368
|
+
* tables don't support `ALTER TABLE ... ADD COLUMN` the way ordinary tables
|
|
1369
|
+
* do. The three sync triggers are defined ON `notes` (`AFTER INSERT/UPDATE/
|
|
1370
|
+
* DELETE ON notes`), not on `notes_fts` itself, so dropping `notes_fts`
|
|
1371
|
+
* does NOT drop them automatically — they're dropped explicitly FIRST
|
|
1372
|
+
* (referencing the old single-column shape, they'd fail on the very next
|
|
1373
|
+
* note write if left in place against the new table) and recreated
|
|
1374
|
+
* immediately after in the new two-column form. SCHEMA_SQL carries the
|
|
1375
|
+
* identical CREATE VIRTUAL TABLE + trigger definitions for fresh vaults;
|
|
1376
|
+
* this function is the upgrade path for a vault that already has the old
|
|
1377
|
+
* shape.
|
|
1378
|
+
*
|
|
1379
|
+
* External-content tables start EMPTY after a bare CREATE — unlike a normal
|
|
1380
|
+
* table, there's no data to inherit from the old dropped table (FTS5 stores
|
|
1381
|
+
* its own inverted-index structures, not a row-for-row copy). The
|
|
1382
|
+
* repopulation pass below re-derives every row from `notes` directly, so
|
|
1383
|
+
* this migration is self-contained (it does NOT read from the old
|
|
1384
|
+
* `notes_fts` — that data is gone the moment DROP TABLE runs).
|
|
1385
|
+
*
|
|
1386
|
+
* ALL-OR-NOTHING (generalist review, #565): the entire DROP + CREATE +
|
|
1387
|
+
* trigger-recreate + repopulation sequence runs inside a SINGLE
|
|
1388
|
+
* `transaction`, not just the repopulation loop. A crash partway through
|
|
1389
|
+
* with the DDL committed but the index empty would otherwise be
|
|
1390
|
+
* unrecoverable — the recreated table already has the `path` column, so the
|
|
1391
|
+
* `hasColumn(db, "notes_fts", "path")` guard would report "done" and never
|
|
1392
|
+
* retry, leaving search permanently empty. Wrapping the whole sequence means a
|
|
1393
|
+
* rollback restores the pre-v25 single-column shape (no `path`), so the
|
|
1394
|
+
* guard correctly re-detects "not migrated" and the next boot re-runs it —
|
|
1395
|
+
* correct-by-construction rather than dependent on the DDL fully
|
|
1396
|
+
* completing.
|
|
1397
|
+
*
|
|
1398
|
+
* Idempotent via `hasColumn(db, "notes_fts", "path")` (checked first — a
|
|
1399
|
+
* vault already on the v25 shape, including every fresh vault, no-ops
|
|
1400
|
+
* immediately; `PRAGMA table_info` returns `[]` for a nonexistent table and
|
|
1401
|
+
* the declared columns for an FTS5 virtual table, so the shared helper is
|
|
1402
|
+
* correct here without a separate `hasTable` pre-check).
|
|
1403
|
+
* Cross-runtime: DROP/CREATE VIRTUAL TABLE, triggers, and the repopulation
|
|
1404
|
+
* SELECT/INSERT are all standard FTS5 + SQL surface — no bun-only
|
|
1405
|
+
* functions — but `tokenize='porter unicode61'` and the two-column
|
|
1406
|
+
* external-content shape are new usage for this codebase; flagged for the
|
|
1407
|
+
* wire reviewer to confirm against Cloudflare DO SQLite's FTS5 build (the
|
|
1408
|
+
* hosted door's async Store backend is not shipped yet — see `store.ts`'s
|
|
1409
|
+
* `BunSqliteStore` doc comment — so this is unverified-until-that-lands,
|
|
1410
|
+
* not a regression against a working path). Porter is part of the same
|
|
1411
|
+
* FTS5 extension registration as unicode61 (not a separately-enabled
|
|
1412
|
+
* module) and `PRAGMA table_info` on a virtual table is standard SQLite,
|
|
1413
|
+
* so the expected risk surface is narrow, but it hasn't been exercised
|
|
1414
|
+
* against DO SQLite directly.
|
|
1415
|
+
*/
|
|
1416
|
+
function migrateToV25(db: Database): void {
|
|
1417
|
+
if (!hasTable(db, "notes")) return;
|
|
1418
|
+
if (hasColumn(db, "notes_fts", "path")) return;
|
|
1419
|
+
|
|
1420
|
+
console.log(
|
|
1421
|
+
"[vault] migrating to schema v25 (vault#551): rebuilding notes_fts with path+content columns and porter stemming...",
|
|
1422
|
+
);
|
|
1423
|
+
|
|
1424
|
+
// The ENTIRE sequence — DROP triggers + table, CREATE the new virtual
|
|
1425
|
+
// table, CREATE the three sync triggers, AND repopulate — runs inside ONE
|
|
1426
|
+
// transaction so it's strictly all-or-nothing (generalist review, #565).
|
|
1427
|
+
//
|
|
1428
|
+
// Why this matters: if the DDL ran outside the transaction (only the
|
|
1429
|
+
// repopulation transacted, the pre-review shape), a crash between
|
|
1430
|
+
// "CREATE VIRTUAL TABLE" and the end of repopulation would leave a
|
|
1431
|
+
// recreated-but-EMPTY notes_fts — and the idempotency guard
|
|
1432
|
+
// (`hasColumn(db, "notes_fts", "path")`) would see the new `path` column
|
|
1433
|
+
// and report the migration "done" on the next boot, so search stays
|
|
1434
|
+
// PERMANENTLY empty with no retry. Worse, a crash before the CREATE TRIGGERs would leave
|
|
1435
|
+
// future writes unindexed too. Wrapping the whole thing means a rollback
|
|
1436
|
+
// restores the pre-v25 shape (single `content` column, no `path`), so the
|
|
1437
|
+
// guard correctly reports "not migrated" and the next boot re-runs it
|
|
1438
|
+
// cleanly — the guard becomes correct-by-construction rather than relying
|
|
1439
|
+
// on the DDL having fully completed. The reviewer verified CREATE VIRTUAL
|
|
1440
|
+
// TABLE + CREATE TRIGGER execute fine inside bun's `BEGIN IMMEDIATE …
|
|
1441
|
+
// COMMIT`; a DO backend routes the same block through `transactionSync`
|
|
1442
|
+
// (see core/src/txn.ts) with the identical commit-on-return /
|
|
1443
|
+
// rollback-on-throw contract.
|
|
1444
|
+
let repopulated = 0;
|
|
1445
|
+
transaction(db, () => {
|
|
1446
|
+
db.exec("DROP TRIGGER IF EXISTS notes_fts_insert");
|
|
1447
|
+
db.exec("DROP TRIGGER IF EXISTS notes_fts_delete");
|
|
1448
|
+
db.exec("DROP TRIGGER IF EXISTS notes_fts_update");
|
|
1449
|
+
db.exec("DROP TABLE IF EXISTS notes_fts");
|
|
1450
|
+
|
|
1451
|
+
db.exec(`
|
|
1452
|
+
CREATE VIRTUAL TABLE notes_fts USING fts5(
|
|
1453
|
+
path,
|
|
1454
|
+
content,
|
|
1455
|
+
content='notes',
|
|
1456
|
+
content_rowid='rowid',
|
|
1457
|
+
tokenize='porter unicode61'
|
|
1458
|
+
)
|
|
1459
|
+
`);
|
|
1460
|
+
db.exec(`
|
|
1461
|
+
CREATE TRIGGER notes_fts_insert AFTER INSERT ON notes BEGIN
|
|
1462
|
+
INSERT INTO notes_fts(rowid, path, content) VALUES (new.rowid, COALESCE(new.path, ''), new.content);
|
|
1463
|
+
END
|
|
1464
|
+
`);
|
|
1465
|
+
db.exec(`
|
|
1466
|
+
CREATE TRIGGER notes_fts_delete AFTER DELETE ON notes BEGIN
|
|
1467
|
+
INSERT INTO notes_fts(notes_fts, rowid, path, content) VALUES('delete', old.rowid, COALESCE(old.path, ''), old.content);
|
|
1468
|
+
END
|
|
1469
|
+
`);
|
|
1470
|
+
db.exec(`
|
|
1471
|
+
CREATE TRIGGER notes_fts_update AFTER UPDATE OF content, path ON notes BEGIN
|
|
1472
|
+
INSERT INTO notes_fts(notes_fts, rowid, path, content) VALUES('delete', old.rowid, COALESCE(old.path, ''), old.content);
|
|
1473
|
+
INSERT INTO notes_fts(rowid, path, content) VALUES (new.rowid, COALESCE(new.path, ''), new.content);
|
|
1474
|
+
END
|
|
1475
|
+
`);
|
|
1476
|
+
|
|
1477
|
+
const rows = db.prepare("SELECT rowid, path, content FROM notes").all() as {
|
|
1478
|
+
rowid: number;
|
|
1479
|
+
path: string | null;
|
|
1480
|
+
content: string | null;
|
|
1481
|
+
}[];
|
|
1482
|
+
const insert = db.prepare("INSERT INTO notes_fts(rowid, path, content) VALUES (?, ?, ?)");
|
|
1483
|
+
for (const row of rows) {
|
|
1484
|
+
insert.run(row.rowid, row.path ?? "", row.content ?? "");
|
|
1485
|
+
repopulated++;
|
|
1486
|
+
}
|
|
1487
|
+
});
|
|
1488
|
+
|
|
1489
|
+
console.log(
|
|
1490
|
+
`[vault] migrated to schema v25 (vault#551): notes_fts rebuilt with path+content columns + porter stemming; repopulated ${repopulated} note(s).`,
|
|
1491
|
+
);
|
|
1492
|
+
}
|
|
1493
|
+
|
|
1335
1494
|
function hasTable(db: Database, name: string): boolean {
|
|
1336
1495
|
const row = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(name);
|
|
1337
1496
|
return !!row;
|
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Search recall + ranking — schema v25 FTS rebuild (Wave 7 of the
|
|
3
|
+
* Reliability & Usability Program, WS2B/C, vault#551). Covers the
|
|
4
|
+
* MIGRATION-bearing risk this PR carries:
|
|
5
|
+
*
|
|
6
|
+
* - a v24 vault (single-column `content`-only notes_fts) upgrades to the
|
|
7
|
+
* v25 shape (path + content, porter stemming) via `migrateToV25`,
|
|
8
|
+
* idempotently, without touching note data;
|
|
9
|
+
* - a fresh vault gets the v25 shape directly from SCHEMA_SQL;
|
|
10
|
+
* - post-migration, a note's TITLE (path) is searchable — impossible
|
|
11
|
+
* pre-v25;
|
|
12
|
+
* - existing body-text search keeps working (no regression);
|
|
13
|
+
* - porter stemming matches regular-affix variants;
|
|
14
|
+
* - bm25 ranking is weighted so a title match outranks a passing body
|
|
15
|
+
* mention, and every result carries a legible `score`;
|
|
16
|
+
* - the sync triggers keep the index correct across insert/update
|
|
17
|
+
* (content-only, path-only, both)/delete, including notes with no path.
|
|
18
|
+
*
|
|
19
|
+
* See `src/contract-search.test.ts` for the REST/MCP-surface coverage
|
|
20
|
+
* (did_you_mean, the wrapped advanced-mode column-filter hint, sort
|
|
21
|
+
* override parity).
|
|
22
|
+
*/
|
|
23
|
+
import { describe, it, expect, beforeEach } from "bun:test";
|
|
24
|
+
import { Database } from "bun:sqlite";
|
|
25
|
+
import { SqliteStore } from "./store.js";
|
|
26
|
+
import { initSchema, SCHEMA_VERSION } from "./schema.js";
|
|
27
|
+
import * as noteOps from "./notes.js";
|
|
28
|
+
|
|
29
|
+
let store: SqliteStore;
|
|
30
|
+
let db: Database;
|
|
31
|
+
|
|
32
|
+
beforeEach(() => {
|
|
33
|
+
db = new Database(":memory:");
|
|
34
|
+
store = new SqliteStore(db);
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
describe("search — schema v25 FTS rebuild", () => {
|
|
38
|
+
it("bumped SCHEMA_VERSION to at least 25 (notes_fts path+content landed)", () => {
|
|
39
|
+
expect(SCHEMA_VERSION).toBeGreaterThanOrEqual(25);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it("a fresh vault gets the v25 notes_fts shape directly (path + content columns)", () => {
|
|
43
|
+
const cols = (db.prepare("PRAGMA table_info(notes_fts)").all() as { name: string }[]).map(
|
|
44
|
+
(r) => r.name,
|
|
45
|
+
);
|
|
46
|
+
expect(cols).toContain("path");
|
|
47
|
+
expect(cols).toContain("content");
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
it("a fresh vault: a note's title (path) is searchable even with unrelated body content", async () => {
|
|
51
|
+
await store.createNote("nothing relevant in the body here", {
|
|
52
|
+
path: "quarterly-budget-review",
|
|
53
|
+
});
|
|
54
|
+
const hits = await store.searchNotes("budget");
|
|
55
|
+
expect(hits.map((n) => n.path)).toContain("quarterly-budget-review");
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it("a fresh vault: existing body search still works (no regression)", async () => {
|
|
59
|
+
await store.createNote("the widget shipped on time", { path: "unrelated-title" });
|
|
60
|
+
const hits = await store.searchNotes("widget");
|
|
61
|
+
expect(hits.map((n) => n.path)).toContain("unrelated-title");
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
describe("search — v24 → v25 migration (legacy vault upgrade)", () => {
|
|
66
|
+
/**
|
|
67
|
+
* Build a v24-shaped vault by hand: the OLD single-column notes_fts
|
|
68
|
+
* (content only), the OLD triggers (UPDATE OF content only — no path
|
|
69
|
+
* sync), and a handful of notes seeded BEFORE the migration runs —
|
|
70
|
+
* mirroring the real upgrade scenario (existing data, not a fresh DB).
|
|
71
|
+
* `initSchema` then drives the whole migration chain up through v25.
|
|
72
|
+
*/
|
|
73
|
+
function buildLegacyV24Vault(): Database {
|
|
74
|
+
const legacy = new Database(":memory:");
|
|
75
|
+
legacy.exec(`
|
|
76
|
+
CREATE TABLE notes (
|
|
77
|
+
id TEXT PRIMARY KEY,
|
|
78
|
+
content TEXT DEFAULT '',
|
|
79
|
+
path TEXT,
|
|
80
|
+
metadata TEXT DEFAULT '{}',
|
|
81
|
+
created_at TEXT NOT NULL,
|
|
82
|
+
updated_at TEXT,
|
|
83
|
+
extension TEXT NOT NULL DEFAULT 'md',
|
|
84
|
+
created_by TEXT,
|
|
85
|
+
created_via TEXT,
|
|
86
|
+
last_updated_by TEXT,
|
|
87
|
+
last_updated_via TEXT
|
|
88
|
+
);
|
|
89
|
+
CREATE TABLE tags (
|
|
90
|
+
name TEXT PRIMARY KEY,
|
|
91
|
+
description TEXT,
|
|
92
|
+
fields TEXT,
|
|
93
|
+
relationships TEXT,
|
|
94
|
+
parent_names TEXT,
|
|
95
|
+
created_at TEXT,
|
|
96
|
+
updated_at TEXT
|
|
97
|
+
);
|
|
98
|
+
CREATE TABLE note_tags (
|
|
99
|
+
note_id TEXT NOT NULL,
|
|
100
|
+
tag_name TEXT NOT NULL,
|
|
101
|
+
PRIMARY KEY (note_id, tag_name)
|
|
102
|
+
);
|
|
103
|
+
CREATE TABLE indexed_fields (
|
|
104
|
+
field TEXT PRIMARY KEY,
|
|
105
|
+
sqlite_type TEXT NOT NULL,
|
|
106
|
+
declarer_tags TEXT NOT NULL DEFAULT '[]'
|
|
107
|
+
);
|
|
108
|
+
CREATE TABLE schema_version (version INTEGER PRIMARY KEY, applied_at TEXT);
|
|
109
|
+
INSERT INTO schema_version (version, applied_at) VALUES (24, '2026-01-01T00:00:00.000Z');
|
|
110
|
+
|
|
111
|
+
CREATE VIRTUAL TABLE notes_fts USING fts5(
|
|
112
|
+
content,
|
|
113
|
+
content='notes',
|
|
114
|
+
content_rowid='rowid'
|
|
115
|
+
);
|
|
116
|
+
CREATE TRIGGER notes_fts_insert AFTER INSERT ON notes BEGIN
|
|
117
|
+
INSERT INTO notes_fts(rowid, content) VALUES (new.rowid, new.content);
|
|
118
|
+
END;
|
|
119
|
+
CREATE TRIGGER notes_fts_delete AFTER DELETE ON notes BEGIN
|
|
120
|
+
INSERT INTO notes_fts(notes_fts, rowid, content) VALUES('delete', old.rowid, old.content);
|
|
121
|
+
END;
|
|
122
|
+
CREATE TRIGGER notes_fts_update AFTER UPDATE OF content ON notes BEGIN
|
|
123
|
+
INSERT INTO notes_fts(notes_fts, rowid, content) VALUES('delete', old.rowid, old.content);
|
|
124
|
+
INSERT INTO notes_fts(rowid, content) VALUES (new.rowid, new.content);
|
|
125
|
+
END;
|
|
126
|
+
`);
|
|
127
|
+
|
|
128
|
+
// Seed BEFORE migration — real-data-like: some notes with a path, one
|
|
129
|
+
// without (path IS NULL — the coalesce-to-'' path must not throw).
|
|
130
|
+
const insert = legacy.prepare(
|
|
131
|
+
"INSERT INTO notes (id, content, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)",
|
|
132
|
+
);
|
|
133
|
+
insert.run(
|
|
134
|
+
"legacy-1",
|
|
135
|
+
"a dedicated writeup mentioning propolis only once in passing text",
|
|
136
|
+
"beekeeping-notes",
|
|
137
|
+
"2026-01-01T00:00:00.000Z",
|
|
138
|
+
"2026-01-01T00:00:00.000Z",
|
|
139
|
+
);
|
|
140
|
+
insert.run(
|
|
141
|
+
"legacy-2",
|
|
142
|
+
"nothing about the topic here at all",
|
|
143
|
+
"propolis",
|
|
144
|
+
"2026-01-02T00:00:00.000Z",
|
|
145
|
+
"2026-01-02T00:00:00.000Z",
|
|
146
|
+
);
|
|
147
|
+
insert.run(
|
|
148
|
+
"legacy-3",
|
|
149
|
+
"the firefighters responded quickly to the call",
|
|
150
|
+
null,
|
|
151
|
+
"2026-01-03T00:00:00.000Z",
|
|
152
|
+
"2026-01-03T00:00:00.000Z",
|
|
153
|
+
);
|
|
154
|
+
// Manually populate the OLD single-column FTS index (mirrors what the
|
|
155
|
+
// old triggers would have done on real inserts).
|
|
156
|
+
legacy.exec(`INSERT INTO notes_fts(rowid, content) SELECT rowid, content FROM notes`);
|
|
157
|
+
|
|
158
|
+
return legacy;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
it("migrates a v24 vault to the current SCHEMA_VERSION without throwing", () => {
|
|
162
|
+
const legacy = buildLegacyV24Vault();
|
|
163
|
+
expect(() => initSchema(legacy)).not.toThrow();
|
|
164
|
+
const ver = (legacy.prepare("SELECT MAX(version) AS v FROM schema_version").get() as { v: number }).v;
|
|
165
|
+
expect(ver).toBe(SCHEMA_VERSION);
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it("post-migration: notes_fts carries path + content columns", () => {
|
|
169
|
+
const legacy = buildLegacyV24Vault();
|
|
170
|
+
initSchema(legacy);
|
|
171
|
+
const cols = (legacy.prepare("PRAGMA table_info(notes_fts)").all() as { name: string }[]).map(
|
|
172
|
+
(r) => r.name,
|
|
173
|
+
);
|
|
174
|
+
expect(cols).toContain("path");
|
|
175
|
+
expect(cols).toContain("content");
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
it("post-migration: a note's TITLE is searchable (impossible pre-v25)", () => {
|
|
179
|
+
const legacy = buildLegacyV24Vault();
|
|
180
|
+
initSchema(legacy);
|
|
181
|
+
// "beekeeping-notes" (legacy-1's path) never appears in any note's body.
|
|
182
|
+
const hits = noteOps.searchNotes(legacy, "beekeeping");
|
|
183
|
+
expect(hits.map((n) => n.id)).toContain("legacy-1");
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
it("post-migration: existing body search still works (no regression from the rebuild)", () => {
|
|
187
|
+
const legacy = buildLegacyV24Vault();
|
|
188
|
+
initSchema(legacy);
|
|
189
|
+
const hits = noteOps.searchNotes(legacy, "firefighters");
|
|
190
|
+
expect(hits.map((n) => n.id)).toContain("legacy-3");
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
it("post-migration: porter stemming matches a regular-affix variant not present verbatim", () => {
|
|
194
|
+
const legacy = buildLegacyV24Vault();
|
|
195
|
+
initSchema(legacy);
|
|
196
|
+
// legacy-3's body has "firefighters" (plural) — singular query must match.
|
|
197
|
+
const hits = noteOps.searchNotes(legacy, "firefighter");
|
|
198
|
+
expect(hits.map((n) => n.id)).toContain("legacy-3");
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
it("post-migration: a title match ranks ABOVE a passing body-only mention, and both carry a score", () => {
|
|
202
|
+
const legacy = buildLegacyV24Vault();
|
|
203
|
+
initSchema(legacy);
|
|
204
|
+
// legacy-2's path IS "propolis" (title match); legacy-1's body mentions
|
|
205
|
+
// "propolis" once in passing (body-only match). Weighted bm25 must rank
|
|
206
|
+
// the title match first.
|
|
207
|
+
const hits = noteOps.searchNotes(legacy, "propolis");
|
|
208
|
+
const ids = hits.map((n) => n.id);
|
|
209
|
+
expect(ids.indexOf("legacy-2")).toBeLessThan(ids.indexOf("legacy-1"));
|
|
210
|
+
for (const n of hits) {
|
|
211
|
+
expect(typeof n.score).toBe("number");
|
|
212
|
+
}
|
|
213
|
+
const byId = new Map(hits.map((n) => [n.id, n.score!]));
|
|
214
|
+
expect(byId.get("legacy-2")!).toBeGreaterThan(byId.get("legacy-1")!);
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
it("post-migration: a note that had NO path (path IS NULL) survived the rebuild and stays body-searchable", () => {
|
|
218
|
+
const legacy = buildLegacyV24Vault();
|
|
219
|
+
initSchema(legacy);
|
|
220
|
+
const hits = noteOps.searchNotes(legacy, "firefighters");
|
|
221
|
+
expect(hits.some((n) => n.id === "legacy-3")).toBe(true);
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
it("post-migration: the FTS index isn't duplicated or corrupted — an integrity-check passes", () => {
|
|
225
|
+
const legacy = buildLegacyV24Vault();
|
|
226
|
+
initSchema(legacy);
|
|
227
|
+
// FTS5's built-in consistency check: throws if the shadow tables and
|
|
228
|
+
// the external-content table have drifted apart (e.g. double-inserted
|
|
229
|
+
// rows from a repopulation bug).
|
|
230
|
+
expect(() => legacy.exec(`INSERT INTO notes_fts(notes_fts) VALUES('integrity-check')`)).not.toThrow();
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
it("migration is idempotent — running initSchema twice does not duplicate rows or throw", () => {
|
|
234
|
+
const legacy = buildLegacyV24Vault();
|
|
235
|
+
initSchema(legacy);
|
|
236
|
+
expect(() => initSchema(legacy)).not.toThrow();
|
|
237
|
+
const hits = noteOps.searchNotes(legacy, "firefighter");
|
|
238
|
+
// Exactly one match, not duplicated by a second repopulation pass.
|
|
239
|
+
expect(hits.filter((n) => n.id === "legacy-3").length).toBe(1);
|
|
240
|
+
expect(() => legacy.exec(`INSERT INTO notes_fts(notes_fts) VALUES('integrity-check')`)).not.toThrow();
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
it("migration is idempotent on a vault that was ALREADY on v25 (e.g. a fresh vault) — no rebuild, no data loss", async () => {
|
|
244
|
+
// `store`/`db` from the outer beforeEach is already a fresh v25 vault.
|
|
245
|
+
await store.createNote("hello world", { path: "greeting" });
|
|
246
|
+
expect(() => initSchema(db)).not.toThrow();
|
|
247
|
+
const hits = await store.searchNotes("hello");
|
|
248
|
+
expect(hits.length).toBe(1);
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* MUST-FIX (generalist review, #565) — LOAD-BEARING regression guard: the
|
|
253
|
+
* whole DDL + repopulation runs inside ONE `transaction`, so a crash partway
|
|
254
|
+
* through can NEVER leave a recreated-but-EMPTY notes_fts (which the
|
|
255
|
+
* path-column idempotency guard would then treat as "done," leaving search
|
|
256
|
+
* permanently empty).
|
|
257
|
+
*
|
|
258
|
+
* This drives the REAL `migrateToV25` (via `initSchema`) and injects the
|
|
259
|
+
* crash by monkey-patching `db.prepare` to throw on the repopulation
|
|
260
|
+
* `INSERT INTO notes_fts(rowid, path, content)` statement — a faithful
|
|
261
|
+
* mid-migration interruption AFTER the rebuild DDL (which runs via
|
|
262
|
+
* `db.exec`, untouched by the patch) but during repopulation. It does NOT
|
|
263
|
+
* hand-roll its own copy of the DDL: an earlier draft did, which made it a
|
|
264
|
+
* generic "transaction() rolls back DDL" test that stayed green even against
|
|
265
|
+
* the pre-fix unwrapped-DDL code. Driving `initSchema` means the test
|
|
266
|
+
* actually guards `migrateToV25`'s OWN transaction wrapping — verified by
|
|
267
|
+
* reverting the fix locally (DDL moved back outside `transaction`) and
|
|
268
|
+
* confirming this test goes RED (the post-crash `ftsCols()` reads
|
|
269
|
+
* `["path","content"]`, the guard skips on restart, and recovery search is
|
|
270
|
+
* empty).
|
|
271
|
+
*
|
|
272
|
+
* The load-bearing assertion is (a): post-crash `ftsCols() === ["content"]`
|
|
273
|
+
* — the rollback restored the pre-v25 shape. Under the unwrapped-DDL
|
|
274
|
+
* regression the recreated table would still carry `path` here.
|
|
275
|
+
*/
|
|
276
|
+
it("a REAL interrupted migrateToV25 (crash during repopulation) rolls back to the v24 shape — the next initSchema fully recovers, never silent-empty", () => {
|
|
277
|
+
const legacy = buildLegacyV24Vault();
|
|
278
|
+
|
|
279
|
+
const ftsCols = () =>
|
|
280
|
+
(legacy.prepare("PRAGMA table_info(notes_fts)").all() as { name: string }[]).map((c) => c.name);
|
|
281
|
+
const ftsShadowTables = () =>
|
|
282
|
+
(
|
|
283
|
+
legacy
|
|
284
|
+
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'notes_fts%' ORDER BY name")
|
|
285
|
+
.all() as { name: string }[]
|
|
286
|
+
).map((r) => r.name);
|
|
287
|
+
|
|
288
|
+
// Sanity: starts on the OLD single-column shape.
|
|
289
|
+
expect(ftsCols()).toEqual(["content"]);
|
|
290
|
+
const shadowsBefore = ftsShadowTables();
|
|
291
|
+
|
|
292
|
+
// Monkey-patch prepare so migrateToV25's repopulation INSERT throws —
|
|
293
|
+
// everything else (earlier migrations, the guard's PRAGMA, the
|
|
294
|
+
// repopulation SELECT, the DDL via db.exec) delegates to the real prepare.
|
|
295
|
+
const origPrepare = legacy.prepare.bind(legacy);
|
|
296
|
+
let injected = false;
|
|
297
|
+
(legacy as any).prepare = (sql: string) => {
|
|
298
|
+
if (!injected && /INSERT INTO notes_fts\(rowid, path, content\)/.test(sql)) {
|
|
299
|
+
injected = true;
|
|
300
|
+
throw new Error("simulated crash during notes_fts repopulation");
|
|
301
|
+
}
|
|
302
|
+
return origPrepare(sql);
|
|
303
|
+
};
|
|
304
|
+
|
|
305
|
+
// initSchema runs the whole migration chain; migrateToV25 throws mid-repopulation.
|
|
306
|
+
expect(() => initSchema(legacy)).toThrow("simulated crash during notes_fts repopulation");
|
|
307
|
+
// Positive control: the injection actually fired (not a vacuous pass).
|
|
308
|
+
expect(injected).toBe(true);
|
|
309
|
+
|
|
310
|
+
// Restore the real prepare for the assertions + recovery run.
|
|
311
|
+
(legacy as any).prepare = origPrepare;
|
|
312
|
+
|
|
313
|
+
// (a) LOAD-BEARING: rollback restored the v24 single-column shape. Under
|
|
314
|
+
// the pre-fix shape (DDL OUTSIDE the transaction) this reads
|
|
315
|
+
// ["path","content"], the hasColumn(notes_fts,"path") guard skips on
|
|
316
|
+
// restart, and search stays silently empty forever.
|
|
317
|
+
expect(ftsCols()).toEqual(["content"]);
|
|
318
|
+
// schema_version never advanced past the fixture's 24.
|
|
319
|
+
expect(
|
|
320
|
+
(legacy.prepare("SELECT MAX(version) AS v FROM schema_version").get() as { v: number }).v,
|
|
321
|
+
).toBe(24);
|
|
322
|
+
// No orphaned/duplicated FTS shadow tables — the set is exactly what it was.
|
|
323
|
+
expect(ftsShadowTables()).toEqual(shadowsBefore);
|
|
324
|
+
// The restored v24 index is intact + consistent (integrity-check passes).
|
|
325
|
+
expect(() => legacy.exec(`INSERT INTO notes_fts(notes_fts) VALUES('integrity-check')`)).not.toThrow();
|
|
326
|
+
|
|
327
|
+
// (b) A clean restart re-runs the REAL migration and fully populates
|
|
328
|
+
// search — both a title-only term and a body term are findable.
|
|
329
|
+
initSchema(legacy);
|
|
330
|
+
expect(ftsCols()).toContain("path");
|
|
331
|
+
expect(noteOps.searchNotes(legacy, "beekeeping").map((n) => n.id)).toContain("legacy-1"); // title-only
|
|
332
|
+
expect(noteOps.searchNotes(legacy, "firefighters").map((n) => n.id)).toContain("legacy-3"); // body
|
|
333
|
+
expect(() => legacy.exec(`INSERT INTO notes_fts(notes_fts) VALUES('integrity-check')`)).not.toThrow();
|
|
334
|
+
});
|
|
335
|
+
});
|
|
336
|
+
|
|
337
|
+
describe("search — FTS sync triggers keep path + content current (schema v25)", () => {
|
|
338
|
+
it("a note created with no path is indexed fine (empty path, not an error)", async () => {
|
|
339
|
+
const note = await store.createNote("body text only, no title");
|
|
340
|
+
const hits = await store.searchNotes("body");
|
|
341
|
+
expect(hits.map((n) => n.id)).toContain(note.id);
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
it("a path-ONLY update (content untouched) is now synced to the index — pre-v25 this was invisible to notes_fts", async () => {
|
|
345
|
+
const note = await store.createNote("unrelated body text", { path: "oldtitle" });
|
|
346
|
+
expect((await store.searchNotes("oldtitle")).map((n) => n.id)).toContain(note.id);
|
|
347
|
+
await store.updateNote(note.id, { path: "newtitle" });
|
|
348
|
+
expect((await store.searchNotes("newtitle")).map((n) => n.id)).toContain(note.id);
|
|
349
|
+
expect((await store.searchNotes("oldtitle")).map((n) => n.id)).not.toContain(note.id);
|
|
350
|
+
});
|
|
351
|
+
|
|
352
|
+
it("deleting a note removes it from both the path and content index", async () => {
|
|
353
|
+
const note = await store.createNote("nothing in the body matches this term", {
|
|
354
|
+
path: "uniquetitleterm",
|
|
355
|
+
});
|
|
356
|
+
expect((await store.searchNotes("uniquetitleterm")).map((n) => n.id)).toContain(note.id);
|
|
357
|
+
expect((await store.searchNotes("nothing")).map((n) => n.id)).toContain(note.id);
|
|
358
|
+
await store.deleteNote(note.id);
|
|
359
|
+
expect((await store.searchNotes("uniquetitleterm")).map((n) => n.id)).not.toContain(note.id);
|
|
360
|
+
expect((await store.searchNotes("nothing")).map((n) => n.id)).not.toContain(note.id);
|
|
361
|
+
});
|
|
362
|
+
});
|
package/core/src/search-query.ts
CHANGED
|
Binary file
|
package/core/src/types.ts
CHANGED
|
@@ -57,6 +57,21 @@ export interface Note {
|
|
|
57
57
|
* for the exact degree semantics (self-loop = 2 under `both`).
|
|
58
58
|
*/
|
|
59
59
|
linkCount?: number;
|
|
60
|
+
/**
|
|
61
|
+
* Full-text search relevance score (vault#551 WS2C — ranking legibility).
|
|
62
|
+
* ONLY present on results from `search=`/`query-notes{search}` — every
|
|
63
|
+
* other read path (structured `queryNotes`, `getNoteById`, ...) leaves
|
|
64
|
+
* this `undefined`. Higher is more relevant — the sign-flipped weighted
|
|
65
|
+
* `bm25(notes_fts, SEARCH_WEIGHT_PATH, SEARCH_WEIGHT_CONTENT)` value (raw
|
|
66
|
+
* SQLite bm25 is negative-is-better; flipped here so external callers get
|
|
67
|
+
* the more intuitive "bigger number wins" convention). Meaningful only
|
|
68
|
+
* for RELATIVE comparison within one result set — the absolute magnitude
|
|
69
|
+
* has no fixed scale and isn't comparable across different queries. When
|
|
70
|
+
* an explicit `sort: "asc"|"desc"` overrides relevance ordering, `score`
|
|
71
|
+
* is still computed and returned (for legibility) even though it no
|
|
72
|
+
* longer determines the result order.
|
|
73
|
+
*/
|
|
74
|
+
score?: number;
|
|
60
75
|
}
|
|
61
76
|
|
|
62
77
|
// ---- Link ----
|
|
@@ -248,6 +263,11 @@ export interface NoteIndex {
|
|
|
248
263
|
preview: string;
|
|
249
264
|
/** Opt-in link degree (see `Note.linkCount`). */
|
|
250
265
|
linkCount?: number;
|
|
266
|
+
/** Full-text search relevance score (see `Note.score`). Carried onto the
|
|
267
|
+
* lean shape too — search's default response IS the lean `NoteIndex[]`
|
|
268
|
+
* (`include_content` is opt-in), so `score` would be invisible in the
|
|
269
|
+
* common case if it only lived on the full `Note` shape. */
|
|
270
|
+
score?: number;
|
|
251
271
|
}
|
|
252
272
|
|
|
253
273
|
/** Link with hydrated note summaries. */
|
package/package.json
CHANGED
|
@@ -316,6 +316,111 @@ describe("contract: search — escaping edge cases, tag-scope, warnings (#551)",
|
|
|
316
316
|
});
|
|
317
317
|
});
|
|
318
318
|
|
|
319
|
+
describe("contract: search — recall + ranking legibility (WS2B/C, #551, schema v25)", () => {
|
|
320
|
+
it("a note's TITLE (path) is now searchable — a term appearing ONLY in path, not content", async () => {
|
|
321
|
+
await store.createNote("nothing in this body matches the query term", {
|
|
322
|
+
path: "quarterly-budget-review",
|
|
323
|
+
});
|
|
324
|
+
const res = await search("search=budget&include_content=true");
|
|
325
|
+
expect(res.status).toBe(200);
|
|
326
|
+
const body = await bodyOf(res);
|
|
327
|
+
expect(body.some((n: any) => n.path === "quarterly-budget-review")).toBe(true);
|
|
328
|
+
});
|
|
329
|
+
|
|
330
|
+
it("a title match outranks a passing body-only mention — legible via the score field", async () => {
|
|
331
|
+
await store.createNote("only a passing mention of espresso here, nothing more", {
|
|
332
|
+
path: "coffee-notes",
|
|
333
|
+
});
|
|
334
|
+
await store.createNote("everything about espresso — brewing, grinding, tasting", {
|
|
335
|
+
path: "espresso",
|
|
336
|
+
});
|
|
337
|
+
const res = await search("search=espresso&include_content=true");
|
|
338
|
+
expect(res.status).toBe(200);
|
|
339
|
+
const body = await bodyOf(res);
|
|
340
|
+
const titleMatchIdx = body.findIndex((n: any) => n.path === "espresso");
|
|
341
|
+
const bodyMatchIdx = body.findIndex((n: any) => n.path === "coffee-notes");
|
|
342
|
+
expect(titleMatchIdx).toBeGreaterThanOrEqual(0);
|
|
343
|
+
expect(bodyMatchIdx).toBeGreaterThanOrEqual(0);
|
|
344
|
+
expect(titleMatchIdx).toBeLessThan(bodyMatchIdx);
|
|
345
|
+
for (const n of body) expect(typeof n.score).toBe("number");
|
|
346
|
+
const scoreOf = (path: string) => body.find((n: any) => n.path === path).score;
|
|
347
|
+
expect(scoreOf("espresso")).toBeGreaterThan(scoreOf("coffee-notes"));
|
|
348
|
+
});
|
|
349
|
+
|
|
350
|
+
it("score is present on the LEAN (default, no include_content) shape too", async () => {
|
|
351
|
+
await store.createNote(NOTES.bothWords);
|
|
352
|
+
const res = await search("search=widgets");
|
|
353
|
+
expect(res.status).toBe(200);
|
|
354
|
+
const body = await bodyOf(res);
|
|
355
|
+
expect(body.length).toBeGreaterThan(0);
|
|
356
|
+
for (const n of body) expect(typeof n.score).toBe("number");
|
|
357
|
+
});
|
|
358
|
+
|
|
359
|
+
it("porter stemming: singular query matches a plural-only body word", async () => {
|
|
360
|
+
await store.createNote("the firefighters responded quickly to the call");
|
|
361
|
+
const res = await search("search=firefighter&include_content=true");
|
|
362
|
+
expect(res.status).toBe(200);
|
|
363
|
+
const body = await bodyOf(res);
|
|
364
|
+
expect(body.some((n: any) => n.content.includes("firefighters"))).toBe(true);
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
it("a zero-result search with a nearby indexed term surfaces a search_did_you_mean warning", async () => {
|
|
368
|
+
await store.createNote("a note that discusses Vasquez and the incident report");
|
|
369
|
+
const res = await search("search=Vasqez");
|
|
370
|
+
expect(res.status).toBe(200);
|
|
371
|
+
const body = await bodyOf(res);
|
|
372
|
+
expect(body).toEqual([]);
|
|
373
|
+
const warnings = decodeWarningsHeader(res);
|
|
374
|
+
expect(warnings).not.toBeNull();
|
|
375
|
+
const w = warnings!.find((x: any) => x.code === "search_did_you_mean");
|
|
376
|
+
expect(w).toBeDefined();
|
|
377
|
+
expect(w.did_you_mean).toBe("vasquez");
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
it("a genuinely zero-result search with NO close vocabulary match carries no did_you_mean warning", async () => {
|
|
381
|
+
await store.createNote(NOTES.bothWords);
|
|
382
|
+
const res = await search("search=zzzznonexistentword");
|
|
383
|
+
expect(res.status).toBe(200);
|
|
384
|
+
const body = await bodyOf(res);
|
|
385
|
+
expect(body).toEqual([]);
|
|
386
|
+
const warnings = decodeWarningsHeader(res);
|
|
387
|
+
if (warnings) {
|
|
388
|
+
expect(warnings.some((w: any) => w.code === "search_did_you_mean")).toBe(false);
|
|
389
|
+
}
|
|
390
|
+
});
|
|
391
|
+
|
|
392
|
+
it("did_you_mean is suppressed for a tag-scoped session (no leak of vault-wide vocabulary)", async () => {
|
|
393
|
+
// Out-of-scope note carries the near-match term; the scoped token's
|
|
394
|
+
// own allowlist ("health") sees nothing else. The did_you_mean
|
|
395
|
+
// suggestion is computed against the WHOLE vault's FTS5 vocabulary
|
|
396
|
+
// regardless of scope, so a scoped caller must never see it — same
|
|
397
|
+
// "no leak" stance as `unknown_tag`'s did_you_mean.
|
|
398
|
+
await store.createNote("a note that discusses Vasquez and the incident report", { tags: ["work"] });
|
|
399
|
+
const scopedReq = new Request(`${BASE}/notes?search=Vasqez`, { method: "GET" });
|
|
400
|
+
const res = await handleNotes(scopedReq, store, "", undefined, {
|
|
401
|
+
allowed: new Set(["health"]),
|
|
402
|
+
raw: ["health"],
|
|
403
|
+
});
|
|
404
|
+
expect(res.status).toBe(200);
|
|
405
|
+
const body = await bodyOf(res);
|
|
406
|
+
expect(body).toEqual([]);
|
|
407
|
+
const warnings = decodeWarningsHeader(res);
|
|
408
|
+
expect(warnings?.some((w: any) => w.code === "search_did_you_mean")).toBeFalsy();
|
|
409
|
+
});
|
|
410
|
+
|
|
411
|
+
it('search_mode:"advanced" — a lone leading hyphen gets a caller-actionable hint, not raw "no such column" text', async () => {
|
|
412
|
+
await store.createNote("espresso and coffee content");
|
|
413
|
+
const res = await search(`search=${encodeURIComponent("-espresso")}&search_mode=advanced`);
|
|
414
|
+
expect(res.status).toBe(400);
|
|
415
|
+
const body = await bodyOf(res);
|
|
416
|
+
expect(body.code).toBe("INVALID_QUERY");
|
|
417
|
+
expect(body.error_type).toBe("invalid_search_syntax");
|
|
418
|
+
expect(typeof body.hint).toBe("string");
|
|
419
|
+
expect(body.hint).toMatch(/BINARY operator/i);
|
|
420
|
+
expect(body.hint).not.toMatch(/^no such column/i);
|
|
421
|
+
});
|
|
422
|
+
});
|
|
423
|
+
|
|
319
424
|
describe("contract: search — MCP parity (#551)", () => {
|
|
320
425
|
it("query-notes: literal-by-default finds punctuation content the same as REST", async () => {
|
|
321
426
|
const tools = generateMcpTools(store);
|
|
@@ -356,4 +461,67 @@ describe("contract: search — MCP parity (#551)", () => {
|
|
|
356
461
|
const desc = (await query.execute({ search: "mcpsortprobe", sort: "desc", include_content: true })) as any[];
|
|
357
462
|
expect(desc.map((n: any) => n.content)).toEqual(["mcpsortprobe beta", "mcpsortprobe alpha"]);
|
|
358
463
|
});
|
|
464
|
+
|
|
465
|
+
it("query-notes: title match outranks a body-only mention, and results carry a score (parity with REST)", async () => {
|
|
466
|
+
await store.createNote("only a passing mention of espresso here, nothing more", { path: "coffee-notes" });
|
|
467
|
+
await store.createNote("everything about espresso — brewing, grinding, tasting", { path: "espresso" });
|
|
468
|
+
const tools = generateMcpTools(store);
|
|
469
|
+
const query = tools.find((t) => t.name === "query-notes")!;
|
|
470
|
+
const result = (await query.execute({ search: "espresso" })) as any[];
|
|
471
|
+
const titleIdx = result.findIndex((n: any) => n.path === "espresso");
|
|
472
|
+
const bodyIdx = result.findIndex((n: any) => n.path === "coffee-notes");
|
|
473
|
+
expect(titleIdx).toBeGreaterThanOrEqual(0);
|
|
474
|
+
expect(bodyIdx).toBeGreaterThanOrEqual(0);
|
|
475
|
+
expect(titleIdx).toBeLessThan(bodyIdx);
|
|
476
|
+
for (const n of result) expect(typeof n.score).toBe("number");
|
|
477
|
+
});
|
|
478
|
+
|
|
479
|
+
it("query-notes: a note's title (path) is searchable (parity with REST)", async () => {
|
|
480
|
+
await store.createNote("nothing in this body matches the query term", { path: "quarterly-budget-review" });
|
|
481
|
+
const tools = generateMcpTools(store);
|
|
482
|
+
const query = tools.find((t) => t.name === "query-notes")!;
|
|
483
|
+
const result = (await query.execute({ search: "budget" })) as any[];
|
|
484
|
+
expect(result.some((n: any) => n.path === "quarterly-budget-review")).toBe(true);
|
|
485
|
+
});
|
|
486
|
+
|
|
487
|
+
it("query-notes: porter stemming matches a plural-only body word (parity with REST)", async () => {
|
|
488
|
+
await store.createNote("the firefighters responded quickly to the call");
|
|
489
|
+
const tools = generateMcpTools(store);
|
|
490
|
+
const query = tools.find((t) => t.name === "query-notes")!;
|
|
491
|
+
const result = (await query.execute({ search: "firefighter", include_content: true })) as any[];
|
|
492
|
+
expect(result.some((n: any) => n.content.includes("firefighters"))).toBe(true);
|
|
493
|
+
});
|
|
494
|
+
|
|
495
|
+
it("query-notes: zero-result search with a nearby indexed term surfaces search_did_you_mean", async () => {
|
|
496
|
+
await store.createNote("a note that discusses Vasquez and the incident report");
|
|
497
|
+
const tools = generateMcpTools(store);
|
|
498
|
+
const query = tools.find((t) => t.name === "query-notes")!;
|
|
499
|
+
const result = (await query.execute({ search: "Vasqez" })) as any;
|
|
500
|
+
// Zero notes + a non-empty warnings array — per the response-shape
|
|
501
|
+
// contract (vault#550) that's the `{notes, warnings}` envelope, not a
|
|
502
|
+
// bare array (a bare array can't carry `warnings` at all).
|
|
503
|
+
expect(Array.isArray(result)).toBe(false);
|
|
504
|
+
expect(result.notes).toEqual([]);
|
|
505
|
+
expect(result.warnings).toBeDefined();
|
|
506
|
+
const w = result.warnings.find((x: any) => x.code === "search_did_you_mean");
|
|
507
|
+
expect(w).toBeDefined();
|
|
508
|
+
expect(w.did_you_mean).toBe("vasquez");
|
|
509
|
+
});
|
|
510
|
+
|
|
511
|
+
it('query-notes: search_mode:"advanced" — a lone leading hyphen gets a caller-actionable hint, not raw "no such column" text', async () => {
|
|
512
|
+
await store.createNote("espresso and coffee content");
|
|
513
|
+
const tools = generateMcpTools(store);
|
|
514
|
+
const query = tools.find((t) => t.name === "query-notes")!;
|
|
515
|
+
let err: any;
|
|
516
|
+
try {
|
|
517
|
+
await query.execute({ search: "-espresso", search_mode: "advanced" });
|
|
518
|
+
} catch (e) {
|
|
519
|
+
err = e;
|
|
520
|
+
}
|
|
521
|
+
expect(err).toBeDefined();
|
|
522
|
+
expect(err.error_type).toBe("invalid_search_syntax");
|
|
523
|
+
expect(typeof err.hint).toBe("string");
|
|
524
|
+
expect(err.hint).toMatch(/BINARY operator/i);
|
|
525
|
+
expect(err.hint).not.toMatch(/^no such column/i);
|
|
526
|
+
});
|
|
359
527
|
});
|
|
@@ -134,3 +134,56 @@ describe("MCP query-notes search × tag-scope — combined enforcement (vault#55
|
|
|
134
134
|
}
|
|
135
135
|
});
|
|
136
136
|
});
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* NIT 1 (auth-561 review, #565): did_you_mean on a zero-result search is
|
|
140
|
+
* computed against the WHOLE vault's FTS5 vocabulary regardless of scope
|
|
141
|
+
* (`computeSearchDidYouMean` is scope-unaware by architecture, like every
|
|
142
|
+
* warning core produces). The MCP query-notes wrapper
|
|
143
|
+
* (`applyTagScopeWrappers`, src/mcp-tools.ts) strips the ENTIRE `warnings`
|
|
144
|
+
* array for a tag-scoped session, which covers `search_did_you_mean`
|
|
145
|
+
* generically — but the security property (a scoped token must not learn an
|
|
146
|
+
* out-of-scope note's vocabulary via a spelling suggestion) deserves a
|
|
147
|
+
* direct test, mirroring the REST test in src/contract-search.test.ts.
|
|
148
|
+
*/
|
|
149
|
+
function warningsOf(result: unknown): any[] {
|
|
150
|
+
if (result && typeof result === "object" && "warnings" in result && Array.isArray((result as any).warnings)) {
|
|
151
|
+
return (result as any).warnings;
|
|
152
|
+
}
|
|
153
|
+
return [];
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
describe("MCP query-notes did_you_mean × tag-scope suppression (#565 NIT 1)", () => {
|
|
157
|
+
test("a scoped session's zero-result search does NOT surface an out-of-scope term via search_did_you_mean", async () => {
|
|
158
|
+
seedVault("journal");
|
|
159
|
+
const store = getVaultStore("journal");
|
|
160
|
+
|
|
161
|
+
// The near-match term "Vasquez" lives ONLY in an out-of-scope note.
|
|
162
|
+
await store.createNote("a note that discusses Vasquez and the incident report", { tags: ["work"] });
|
|
163
|
+
|
|
164
|
+
const scopedTool = await queryNotesTool("journal", ["health"]);
|
|
165
|
+
const scopedResult = await scopedTool.execute({ search: "Vasqez" }); // typo → zero results
|
|
166
|
+
expect(idsOf(scopedResult)).toEqual([]);
|
|
167
|
+
// The wrapper strips warnings wholesale for a scoped session — no
|
|
168
|
+
// search_did_you_mean leaking the out-of-scope "vasquez".
|
|
169
|
+
expect(warningsOf(scopedResult).some((w: any) => w.code === "search_did_you_mean")).toBe(false);
|
|
170
|
+
// Belt-and-suspenders: the out-of-scope term must not appear ANYWHERE in
|
|
171
|
+
// the serialized response (no did_you_mean field slipping through under a
|
|
172
|
+
// different shape).
|
|
173
|
+
expect(JSON.stringify(scopedResult).toLowerCase()).not.toContain("vasquez");
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
test("control: an UNSCOPED session's zero-result search DOES surface the suggestion (proves it's suppressed by scope, not simply absent)", async () => {
|
|
177
|
+
seedVault("journal");
|
|
178
|
+
const store = getVaultStore("journal");
|
|
179
|
+
|
|
180
|
+
await store.createNote("a note that discusses Vasquez and the incident report", { tags: ["work"] });
|
|
181
|
+
|
|
182
|
+
const unscopedTool = await queryNotesTool("journal", null);
|
|
183
|
+
const result = await unscopedTool.execute({ search: "Vasqez" });
|
|
184
|
+
expect(idsOf(result)).toEqual([]);
|
|
185
|
+
const w = warningsOf(result).find((x: any) => x.code === "search_did_you_mean");
|
|
186
|
+
expect(w).toBeDefined();
|
|
187
|
+
expect(w.did_you_mean).toBe("vasquez");
|
|
188
|
+
});
|
|
189
|
+
});
|
package/src/routes.ts
CHANGED
|
@@ -13,7 +13,14 @@
|
|
|
13
13
|
|
|
14
14
|
import type { Store, Note, QueryOpts } from "../core/src/types.ts";
|
|
15
15
|
import { TAG_EXPAND_MODES, stripTagHash, suggestSimilarTag, type TagExpandMode } from "../core/src/tag-hierarchy.ts";
|
|
16
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
collectUnknownTagWarnings,
|
|
18
|
+
emptySearchWarning,
|
|
19
|
+
ignoredParamWarning,
|
|
20
|
+
computeSearchDidYouMean,
|
|
21
|
+
searchDidYouMeanWarning,
|
|
22
|
+
type QueryWarning,
|
|
23
|
+
} from "../core/src/query-warnings.ts";
|
|
17
24
|
import { SEARCH_MODES, buildLiteralSearchQuery, isValidSearchMode, type SearchMode } from "../core/src/search-query.ts";
|
|
18
25
|
import { listUnresolvedWikilinks } from "../core/src/wikilinks.ts";
|
|
19
26
|
import { transactionAsync } from "../core/src/txn.ts";
|
|
@@ -1103,6 +1110,22 @@ async function handleNotesInner(
|
|
|
1103
1110
|
}
|
|
1104
1111
|
throw e;
|
|
1105
1112
|
}
|
|
1113
|
+
// Zero-result `did_you_mean` (vault#551 WS2B) — cheap (a bounded
|
|
1114
|
+
// FTS5-vocabulary scan) and ONLY computed on the already-rare
|
|
1115
|
+
// empty-result path, mirroring `unknown_tag`'s did_you_mean.
|
|
1116
|
+
// Gated on `tagScope.allowed === null` (unscoped) — same stance
|
|
1117
|
+
// as `collectUnknownTagWarnings` below: the FTS5 vocabulary spans
|
|
1118
|
+
// the whole vault regardless of scope, so surfacing a suggestion
|
|
1119
|
+
// to a scoped token would leak out-of-scope content's existence
|
|
1120
|
+
// across the scope boundary. Unlike the MCP path, REST enforces
|
|
1121
|
+
// tag-scope inline (no post-hoc wrapper to strip it for us), so
|
|
1122
|
+
// the gate has to live here explicitly.
|
|
1123
|
+
if (rawResults.length === 0 && tagScope.allowed === null) {
|
|
1124
|
+
const suggestion = computeSearchDidYouMean(db, search);
|
|
1125
|
+
if (suggestion) {
|
|
1126
|
+
searchWarnings.push(searchDidYouMeanWarning(search, suggestion));
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1106
1129
|
}
|
|
1107
1130
|
// Tag-scope: drop any result the token isn't permitted to see. Filter
|
|
1108
1131
|
// happens after the store query so an empty post-filter list still
|