@openparachute/vault 0.7.0-rc.6 → 0.7.0-rc.8

package/core/src/query-warnings.ts

@@ -217,3 +217,113 @@ export function ignoredParamWarning(param: string, reason: string): QueryWarning
     param,
   };
 }
+
+/**
+ * Cheap zero-result search suggestion (vault#551 WS2B — mirrors the tag
+ * `did_you_mean` above via the SAME `suggestSimilarTag` scorer, just over a
+ * different candidate pool). Callers MUST only invoke this AFTER a search
+ * already returned zero rows — it is never on the hot non-empty path, so
+ * the extra vocabulary scan costs nothing in the overwhelmingly common
+ * case where the search actually found something.
+ *
+ * Candidates are drawn from two cheap-to-reach sources, unioned:
+ *
+ *   - the FTS5 vocabulary — a `notes_fts_vocab` `fts5vocab('row')` table
+ *     created here LAZILY and BEST-EFFORT (not part of the schema/
+ *     migration — see the try/catch below) — the terms actually indexed,
+ *     POST-tokenization/stemming (porter). A suggestion therefore
+ *     sometimes reads as a stemmed form ("propoli" rather than "propolis")
+ *     rather than the original dictionary word — an accepted tradeoff
+ *     (documented in docs/HTTP_API.md) rather than maintaining a second,
+ *     unstemmed index just for spelling suggestions.
+ *   - tag names, when the caller passes `tagNames` (already cached by the
+ *     caller's tag hierarchy — free to include).
+ *
+ * `fts5vocab` is registered by the same FTS5 extension init as `notes_fts`
+ * itself (not a separately-enabled module) so it's expected to be
+ * available anywhere search already works — but this is NOT exercised
+ * against Cloudflare DO SQLite (flagged for the wire reviewer). The WHOLE
+ * function is wrapped in try/catch and degrades to "no suggestion" on ANY
+ * failure — including a runtime where `fts5vocab` is unavailable — rather
+ * than ever throwing; a spelling hint is a nicety, never worth risking the
+ * search response itself.
+ *
+ * Per-token length filtering (`length(term) BETWEEN len-2 AND len+2`) keeps
+ * the vocabulary scan bounded on a large vault without a hardcoded row cap
+ * that could silently exclude the very term a caller needs (a `LIMIT`
+ * ORDER-BY-frequency would bias toward common words, which is backwards
+ * for a name/typo lookup — the word you're trying to find is often rare).
+ */
+export function computeSearchDidYouMean(
+  db: Database,
+  rawQuery: string,
+  tagNames?: Iterable<string>,
+): string | undefined {
+  try {
+    const tokens = rawQuery.trim().split(/\s+/).filter(Boolean);
+    if (tokens.length === 0) return undefined;
+
+    db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts_vocab USING fts5vocab(notes_fts, 'row')");
+    const exactStmt = db.prepare("SELECT 1 FROM notes_fts_vocab WHERE term = ? LIMIT 1");
+    const rangeStmt = db.prepare("SELECT term FROM notes_fts_vocab WHERE length(term) BETWEEN ? AND ?");
+
+    let changed = false;
+    const corrected = tokens.map((tok) => {
+      // Strip leading/trailing punctuation for the lookup (the FTS5
+      // vocabulary never contains punctuation) but leave short tokens
+      // (numbers, "a", "of", ...) alone — too little signal to safely
+      // "correct" without a high false-positive rate.
+      const clean = tok.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "");
+      if (clean.length < 3) return tok;
+      const lower = clean.toLowerCase();
+
+      // Already indexed verbatim (post-stemming) — no typo signal.
+      if (exactStmt.get(lower)) return tok;
+
+      const lo = Math.max(1, clean.length - 2);
+      const hi = clean.length + 2;
+      const rows = rangeStmt.all(lo, hi) as { term: string }[];
+      const candidates = new Set<string>(rows.map((r) => r.term));
+      if (tagNames) for (const t of tagNames) candidates.add(t);
+
+      const suggestion = suggestSimilarTag(candidates, lower);
+      if (suggestion && suggestion.toLowerCase() !== lower) {
+        changed = true;
+        return suggestion;
+      }
+      return tok;
+    });
+
+    if (!changed) return undefined;
+    return corrected.join(" ");
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * `search_did_you_mean` warning (vault#551 WS2B) — wraps
+ * {@link computeSearchDidYouMean}'s suggestion in the standard warning
+ * shape, mirroring `unknown_tag`'s `did_you_mean` field. Returns
+ * `undefined` (nothing to push) when no suggestion clears the bar — a
+ * zero-result search is a perfectly ordinary outcome on its own and does
+ * NOT warrant a warning by itself; only an actual spelling suggestion does.
+ *
+ * Scope-unaware by construction, exactly like `collectUnknownTagWarnings`
+ * (see this file's top-of-file doc comment) — the FTS5 vocabulary spans
+ * the WHOLE vault regardless of any caller's tag scope. Callers on a
+ * tag-scoped session MUST NOT surface this warning directly:
+ * `src/mcp-tools.ts`'s `query-notes` wrapper already strips the entire
+ * `warnings` array for a scoped caller (so MCP is safe with no extra
+ * work); `src/routes.ts` must gate the call itself behind
+ * `tagScope.allowed === null`, same as it already does for
+ * `collectUnknownTagWarnings`.
+ */
+export function searchDidYouMeanWarning(rawQuery: string, suggestion: string): QueryWarning {
+  return {
+    code: "search_did_you_mean",
+    message: `no results for "${rawQuery}" — did you mean "${suggestion}"?`,
+    query: rawQuery,
+    did_you_mean: suggestion,
+  };
+}

package/core/src/schema.ts

@@ -4,7 +4,7 @@ import { rebuildIndexes, listIndexedFields } from "./indexed-fields.js";
 import { findMixedTypeIndexedFieldNotes } from "./doctor.js";
 import { transaction } from "./txn.js";
 
-export const SCHEMA_VERSION = 24;
+export const SCHEMA_VERSION = 25;
 
 export const SCHEMA_SQL = `
 -- Notes: the universal record.
@@ -256,25 +256,43 @@ CREATE TABLE IF NOT EXISTS schema_version (
   applied_at TEXT NOT NULL
 );
 
--- Full-text search on note content
+-- Full-text search on note title (path) AND content (v25, vault#551 WS2B/C).
+-- Two columns, declared in this order — bm25(notes_fts, w_path, w_content)
+-- calls in core/src/notes.ts positionally match column 0 = path, column 1 =
+-- content, weighted so a TITLE match outranks a passing body mention (see
+-- SEARCH_WEIGHT_PATH/SEARCH_WEIGHT_CONTENT in core/src/search-query.ts).
+-- Pre-v25, only content was indexed — a note's title/path was completely
+-- unsearchable, which both defeated any title-biased ranking (nothing to
+-- bias) and was a plain recall gap. tokenize='porter unicode61' adds
+-- Porter stemming on top of the v3-era default unicode61 tokenizer so
+-- regular-affix variants (firefighter/firefighters, microbe/microbes) match
+-- each other; irregular plurals with a consonant change (wolf/wolves) are a
+-- known Porter limitation, not fixed by this — see docs/HTTP_API.md.
+-- Existing vaults upgrade via migrateToV25 (rebuild + repopulate); this
+-- CREATE only reaches its new shape directly on a fresh vault.
 CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(
+  path,
   content,
   content='notes',
-  content_rowid='rowid'
+  content_rowid='rowid',
+  tokenize='porter unicode61'
 );
 
--- FTS triggers
+-- FTS triggers. UPDATE OF content, path (not content alone, pre-v25) —
+-- either column changing must resync the index now that path is indexed too;
+-- a path-only rename (content untouched) used to be silently invisible to
+-- notes_fts because the trigger's column list didn't include it.
 CREATE TRIGGER IF NOT EXISTS notes_fts_insert AFTER INSERT ON notes BEGIN
-  INSERT INTO notes_fts(rowid, content) VALUES (new.rowid, new.content);
+  INSERT INTO notes_fts(rowid, path, content) VALUES (new.rowid, COALESCE(new.path, ''), new.content);
 END;
 
 CREATE TRIGGER IF NOT EXISTS notes_fts_delete AFTER DELETE ON notes BEGIN
-  INSERT INTO notes_fts(notes_fts, rowid, content) VALUES('delete', old.rowid, old.content);
+  INSERT INTO notes_fts(notes_fts, rowid, path, content) VALUES('delete', old.rowid, COALESCE(old.path, ''), old.content);
 END;
 
-CREATE TRIGGER IF NOT EXISTS notes_fts_update AFTER UPDATE OF content ON notes BEGIN
-  INSERT INTO notes_fts(notes_fts, rowid, content) VALUES('delete', old.rowid, old.content);
-  INSERT INTO notes_fts(rowid, content) VALUES (new.rowid, new.content);
+CREATE TRIGGER IF NOT EXISTS notes_fts_update AFTER UPDATE OF content, path ON notes BEGIN
+  INSERT INTO notes_fts(notes_fts, rowid, path, content) VALUES('delete', old.rowid, COALESCE(old.path, ''), old.content);
+  INSERT INTO notes_fts(rowid, path, content) VALUES (new.rowid, COALESCE(new.path, ''), new.content);
 END;
 
 -- Indexes
@@ -507,6 +525,10 @@ export function initSchema(db: Database): void {
   // leave the rest in place for `doctor` to surface. See vault#553.
   migrateToV24(db);
 
+  // Migrate v24 → v25: rebuild notes_fts with path+content columns +
+  // porter stemming, repopulate from every existing note. See vault#551.
+  migrateToV25(db);
+
   // Rebuild any generated columns + indexes declared in indexed_fields.
   // No-op for a fresh vault; idempotent on existing vaults.
   rebuildIndexes(db);
@@ -1332,6 +1354,143 @@ function migrateToV24(db: Database): void {
   }
 }
 
+/**
+ * Migrate v24 → v25: rebuild `notes_fts` to index BOTH `path` (title) and
+ * `content` as separate weighted FTS5 columns, with Porter stemming added
+ * to the tokenizer (vault#551 WS2B/C — the Reliability & Usability
+ * Program's interim-harness finding: a note's title/path was completely
+ * unsearchable pre-v25, which both defeated title-biased ranking and was a
+ * plain recall gap on its own).
+ *
+ * `notes_fts` is an EXTERNAL CONTENT table (`content='notes'`) — the FTS5
+ * index is a side structure over `notes`, not a copy of the data itself, so
+ * changing its column shape means DROP + CREATE, not `ALTER`. FTS5 virtual
+ * tables don't support `ALTER TABLE ... ADD COLUMN` the way ordinary tables
+ * do. The three sync triggers are defined ON `notes` (`AFTER INSERT/UPDATE/
+ * DELETE ON notes`), not on `notes_fts` itself, so dropping `notes_fts`
+ * does NOT drop them automatically — they're dropped explicitly FIRST
+ * (referencing the old single-column shape, they'd fail on the very next
+ * note write if left in place against the new table) and recreated
+ * immediately after in the new two-column form. SCHEMA_SQL carries the
+ * identical CREATE VIRTUAL TABLE + trigger definitions for fresh vaults;
+ * this function is the upgrade path for a vault that already has the old
+ * shape.
+ *
+ * External-content tables start EMPTY after a bare CREATE — unlike a normal
+ * table, there's no data to inherit from the old dropped table (FTS5 stores
+ * its own inverted-index structures, not a row-for-row copy). The
+ * repopulation pass below re-derives every row from `notes` directly, so
+ * this migration is self-contained (it does NOT read from the old
+ * `notes_fts` — that data is gone the moment DROP TABLE runs).
+ *
+ * ALL-OR-NOTHING (generalist review, #565): the entire DROP + CREATE +
+ * trigger-recreate + repopulation sequence runs inside a SINGLE
+ * `transaction`, not just the repopulation loop. A crash partway through
+ * with the DDL committed but the index empty would otherwise be
+ * unrecoverable — the recreated table already has the `path` column, so the
+ * `hasColumn(db, "notes_fts", "path")` guard would report "done" and never
+ * retry, leaving search permanently empty. Wrapping the whole sequence means a
+ * rollback restores the pre-v25 single-column shape (no `path`), so the
+ * guard correctly re-detects "not migrated" and the next boot re-runs it —
+ * correct-by-construction rather than dependent on the DDL fully
+ * completing.
+ *
+ * Idempotent via `hasColumn(db, "notes_fts", "path")` (checked first — a
+ * vault already on the v25 shape, including every fresh vault, no-ops
+ * immediately; `PRAGMA table_info` returns `[]` for a nonexistent table and
+ * the declared columns for an FTS5 virtual table, so the shared helper is
+ * correct here without a separate `hasTable` pre-check).
+ * Cross-runtime: DROP/CREATE VIRTUAL TABLE, triggers, and the repopulation
+ * SELECT/INSERT are all standard FTS5 + SQL surface — no bun-only
+ * functions — but `tokenize='porter unicode61'` and the two-column
+ * external-content shape are new usage for this codebase; flagged for the
+ * wire reviewer to confirm against Cloudflare DO SQLite's FTS5 build (the
+ * hosted door's async Store backend is not shipped yet — see `store.ts`'s
+ * `BunSqliteStore` doc comment — so this is unverified-until-that-lands,
+ * not a regression against a working path). Porter is part of the same
+ * FTS5 extension registration as unicode61 (not a separately-enabled
+ * module) and `PRAGMA table_info` on a virtual table is standard SQLite,
+ * so the expected risk surface is narrow, but it hasn't been exercised
+ * against DO SQLite directly.
+ */
+function migrateToV25(db: Database): void {
+  if (!hasTable(db, "notes")) return;
+  if (hasColumn(db, "notes_fts", "path")) return;
+
+  console.log(
+    "[vault] migrating to schema v25 (vault#551): rebuilding notes_fts with path+content columns and porter stemming...",
+  );
+
+  // The ENTIRE sequence — DROP triggers + table, CREATE the new virtual
+  // table, CREATE the three sync triggers, AND repopulate — runs inside ONE
+  // transaction so it's strictly all-or-nothing (generalist review, #565).
+  //
+  // Why this matters: if the DDL ran outside the transaction (only the
+  // repopulation transacted, the pre-review shape), a crash between
+  // "CREATE VIRTUAL TABLE" and the end of repopulation would leave a
+  // recreated-but-EMPTY notes_fts — and the idempotency guard
+  // (`hasColumn(db, "notes_fts", "path")`) would see the new `path` column
+  // and report the migration "done" on the next boot, so search stays
+  // PERMANENTLY empty with no retry. Worse, a crash before the CREATE TRIGGERs would leave
+  // future writes unindexed too. Wrapping the whole thing means a rollback
+  // restores the pre-v25 shape (single `content` column, no `path`), so the
+  // guard correctly reports "not migrated" and the next boot re-runs it
+  // cleanly — the guard becomes correct-by-construction rather than relying
+  // on the DDL having fully completed. The reviewer verified CREATE VIRTUAL
+  // TABLE + CREATE TRIGGER execute fine inside bun's `BEGIN IMMEDIATE …
+  // COMMIT`; a DO backend routes the same block through `transactionSync`
+  // (see core/src/txn.ts) with the identical commit-on-return /
+  // rollback-on-throw contract.
+  let repopulated = 0;
+  transaction(db, () => {
+    db.exec("DROP TRIGGER IF EXISTS notes_fts_insert");
+    db.exec("DROP TRIGGER IF EXISTS notes_fts_delete");
+    db.exec("DROP TRIGGER IF EXISTS notes_fts_update");
+    db.exec("DROP TABLE IF EXISTS notes_fts");
+
+    db.exec(`
+      CREATE VIRTUAL TABLE notes_fts USING fts5(
+        path,
+        content,
+        content='notes',
+        content_rowid='rowid',
+        tokenize='porter unicode61'
+      )
+    `);
+    db.exec(`
+      CREATE TRIGGER notes_fts_insert AFTER INSERT ON notes BEGIN
+        INSERT INTO notes_fts(rowid, path, content) VALUES (new.rowid, COALESCE(new.path, ''), new.content);
+      END
+    `);
+    db.exec(`
+      CREATE TRIGGER notes_fts_delete AFTER DELETE ON notes BEGIN
+        INSERT INTO notes_fts(notes_fts, rowid, path, content) VALUES('delete', old.rowid, COALESCE(old.path, ''), old.content);
+      END
+    `);
+    db.exec(`
+      CREATE TRIGGER notes_fts_update AFTER UPDATE OF content, path ON notes BEGIN
+        INSERT INTO notes_fts(notes_fts, rowid, path, content) VALUES('delete', old.rowid, COALESCE(old.path, ''), old.content);
+        INSERT INTO notes_fts(rowid, path, content) VALUES (new.rowid, COALESCE(new.path, ''), new.content);
+      END
+    `);
+
+    const rows = db.prepare("SELECT rowid, path, content FROM notes").all() as {
+      rowid: number;
+      path: string | null;
+      content: string | null;
+    }[];
+    const insert = db.prepare("INSERT INTO notes_fts(rowid, path, content) VALUES (?, ?, ?)");
+    for (const row of rows) {
+      insert.run(row.rowid, row.path ?? "", row.content ?? "");
+      repopulated++;
+    }
+  });
+
+  console.log(
+    `[vault] migrated to schema v25 (vault#551): notes_fts rebuilt with path+content columns + porter stemming; repopulated ${repopulated} note(s).`,
+  );
+}
+
 function hasTable(db: Database, name: string): boolean {
   const row = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(name);
   return !!row;