@openparachute/vault 0.7.0-rc.5 → 0.7.0-rc.7

package/core/src/notes.ts

@@ -22,7 +22,12 @@ import {
 import { getIndexedField, releaseField } from "./indexed-fields.js";
 import { computeExpandedTagCounts, loadTagHierarchy, stripTagHash } from "./tag-hierarchy.js";
 import { chunkForInClause, IN_VIA_JSON_EACH, jsonEachParam } from "./sql-in.js";
-import { buildLiteralSearchQuery, type SearchMode } from "./search-query.js";
+import {
+  buildLiteralSearchQuery,
+  SEARCH_WEIGHT_PATH,
+  SEARCH_WEIGHT_CONTENT,
+  type SearchMode,
+} from "./search-query.js";
 
 let idCounter = 0;
 
@@ -1336,12 +1341,41 @@ export function queryNotesPaged(db: Database, opts: QueryOpts): QueryNotesPage {
  *     invariant is ever broken it still surfaces as an honest error, not a
  *     500. Signals a vault bug worth reporting.
  */
+/**
+ * FTS5's own error text for a syntax mistake is written for someone who
+ * already knows FTS5's grammar — "no such column: espresso" for a bare
+ * leading `-espresso` (NOT with no left-hand term — FTS5's `-`/`NOT` is a
+ * BINARY operator, so a lone `-token` with nothing before it gets
+ * misparsed as `column:term` filter syntax and fails looking for a column
+ * named after the token) is confusing/leaky rather than actionable: it
+ * reads as if a column literally named "espresso" was expected, which
+ * exposes FTS5-internal parsing behavior instead of explaining the actual
+ * mistake (vault#551 WS2B item 4 — interim harness finding). Detected
+ * generically off the FTS5 error text (`/no such column:/i`) rather than
+ * re-parsing the query ourselves, since the SAME message also covers the
+ * other real cause (an explicit `column:term` filter naming a column that
+ * isn't one of `notes_fts`'s declared columns, `path`/`content`) — the hint
+ * below covers both without claiming to know which one happened.
+ */
+function advancedModeColumnHint(causeMessage: string): string | null {
+  if (!/no such column:/i.test(causeMessage)) return null;
+  return (
+    `FTS5 read part of this query as column-filter syntax ("column:term") or as a ` +
+    `leading "-" with no term to its left — NOT/"-" is a BINARY operator in FTS5 ` +
+    `("good -bad", not "-bad" alone). Indexed columns are "path" and "content". ` +
+    `Add a preceding positive term before a NOT, quote the phrase to search it ` +
+    `literally, or use search_mode:"literal" (the default) to skip advanced syntax entirely.`
+  );
+}
+
 function searchSyntaxError(rawQuery: string, err: unknown, mode: SearchMode): QueryError {
   const causeMessage = err instanceof Error ? err.message : String(err);
+  const columnHint = mode === "advanced" ? advancedModeColumnHint(causeMessage) : null;
   const hint =
-    mode === "advanced"
+    columnHint ??
+    (mode === "advanced"
       ? `FTS5 rejected this as advanced query syntax (${causeMessage}). Fix the syntax, or omit search_mode:"advanced" for literal (punctuation-safe) search.`
-      : `FTS5 rejected the escaped literal query (${causeMessage}) — this should be impossible after literal-mode escaping + control-char sanitization; please report it as a vault bug.`;
+      : `FTS5 rejected the escaped literal query (${causeMessage}) — this should be impossible after literal-mode escaping + control-char sanitization; please report it as a vault bug.`);
   return new QueryError(`invalid search syntax: ${causeMessage}`, "INVALID_QUERY", {
     error_type: "invalid_search_syntax",
     field: "search",
@@ -1377,20 +1411,33 @@ export function searchNotes(
     ftsQuery = query;
   }
 
+  // Weighted bm25 relevance expression (vault#551 WS2C, schema v25):
+  // `notes_fts` now indexes `path` (title, column 0) then `content` (body,
+  // column 1) — SEARCH_WEIGHT_PATH/SEARCH_WEIGHT_CONTENT bias ranking so a
+  // title match outranks a passing body mention. Raw SQLite bm25 is
+  // negative-is-better; `-1.0 * bm25(...)` flips it so bigger is more
+  // relevant (see `Note.score`'s doc comment for the external contract).
+  // The weights are our own numeric constants (not user input) interpolated
+  // directly — bm25()'s weight arguments are positional per-column
+  // multipliers, not general SQL expressions callers can influence.
+  const scoreExpr = `(-1.0 * bm25(notes_fts, ${SEARCH_WEIGHT_PATH}, ${SEARCH_WEIGHT_CONTENT}))`;
+
   // `sort` honored under search (vault#551 WS2A item 3): default stays FTS5
-  // relevance (`rank`, unchanged); an EXPLICIT `sort: "asc"|"desc"` switches
-  // to `created_at` ordering. Checked against the literal string values (not
-  // truthiness) so an absent `sort` can never accidentally match a branch.
-  // `n.id ${direction}` is appended as a deterministic tiebreaker — same
-  // rationale as `queryNotes` (two notes at the same created_at millisecond
-  // would otherwise return in arbitrary/unstable order). `rank` needs no
-  // tiebreaker (bm25 score is effectively unique per row).
+  // relevance (the weighted `score` expression); an EXPLICIT
+  // `sort: "asc"|"desc"` switches to `created_at` ordering. Checked against
+  // the literal string values (not truthiness) so an absent `sort` can
+  // never accidentally match a branch. `n.id ${direction}` is appended as a
+  // deterministic tiebreaker in every branch — two notes at the same
+  // created_at millisecond (structured-sort branches) OR with an
+  // identical weighted score (relevance branch — much more likely than
+  // with unweighted `rank`, since many notes share "zero path matches")
+  // would otherwise return in arbitrary/unstable order.
   const orderBy =
     opts?.sort === "asc"
       ? "n.created_at ASC, n.id ASC"
       : opts?.sort === "desc"
         ? "n.created_at DESC, n.id DESC"
-        : "rank";
+        : "score DESC, n.id ASC";
 
   if (opts?.tags && opts.tags.length > 0) {
     // Canonical-bare-tag guard backstop (vault#XXX) for direct-core callers.
@@ -1406,14 +1453,14 @@ export function searchNotes(
       // DISTINCT over full rows. The FTS join itself is 1:1 on rowid.
       const tagPlaceholders = searchTags.map(() => "?").join(", ");
       const rows = db.prepare(`
-        SELECT n.* FROM notes n
+        SELECT n.*, ${scoreExpr} AS score FROM notes n
         JOIN notes_fts fts ON fts.rowid = n.rowid
         WHERE notes_fts MATCH ?
           AND n.id IN (SELECT note_id FROM note_tags WHERE tag_name IN (${tagPlaceholders}))
         ORDER BY ${orderBy}
         LIMIT ?
-      `).all(ftsQuery, ...searchTags, limit) as NoteRow[];
-      return notesWithTags(db, rows);
+      `).all(ftsQuery, ...searchTags, limit) as (NoteRow & { score: number })[];
+      return notesWithTags(db, rows, scoresById(rows));
     } catch (err) {
       // Surface EVERY FTS5 error structured, never a raw rethrow (vault#551):
       // advanced mode expects it (the caller passed raw syntax); literal
@@ -1429,27 +1476,44 @@ export function searchNotes(
 
   try {
     const rows = db.prepare(`
-      SELECT n.* FROM notes n
+      SELECT n.*, ${scoreExpr} AS score FROM notes n
       JOIN notes_fts fts ON fts.rowid = n.rowid
       WHERE notes_fts MATCH ?
       ORDER BY ${orderBy}
       LIMIT ?
-    `).all(ftsQuery, limit) as NoteRow[];
-    return notesWithTags(db, rows);
+    `).all(ftsQuery, limit) as (NoteRow & { score: number })[];
+    return notesWithTags(db, rows, scoresById(rows));
   } catch (err) {
     if (err instanceof QueryError) throw err;
     throw searchSyntaxError(query, err, mode);
   }
 }
 
-/** Map rows → Notes with tags hydrated in one batched query. */
-function notesWithTags(db: Database, rows: NoteRow[]): Note[] {
+/**
+ * Map rows → Notes with tags hydrated in one batched query. `scores`
+ * (vault#551 WS2C) is an optional id → weighted-bm25-score map, attached
+ * onto each returned `Note.score` when present — ONLY `searchNotes`'s two
+ * callers pass it; every other caller (plain `queryNotes`, `getNotesByIds`,
+ * ...) omits it and gets byte-identical output to before `score` existed.
+ */
+function notesWithTags(db: Database, rows: NoteRow[], scores?: Map<string, number>): Note[] {
   const notes = rows.map(rowToNote);
   const tagsById = getNoteTagsForNotes(db, notes.map((n) => n.id));
-  for (const note of notes) note.tags = tagsById.get(note.id) ?? [];
+  for (const note of notes) {
+    note.tags = tagsById.get(note.id) ?? [];
+    if (scores) {
+      const s = scores.get(note.id);
+      if (s !== undefined) note.score = s;
+    }
+  }
   return notes;
 }
 
+/** Build an id → score map from search result rows carrying a `score` column. */
+function scoresById(rows: (NoteRow & { score: number })[]): Map<string, number> {
+  return new Map(rows.map((r) => [r.id, r.score]));
+}
+
 // ---- Tag Operations ----
 
 export function tagNote(db: Database, noteId: string, tags: string[]): void {
@@ -2120,6 +2184,7 @@ export function toNoteIndex(note: Note): NoteIndex {
     metadata: note.metadata,
     byteSize,
     preview,
+    ...(note.score !== undefined ? { score: note.score } : {}),
   };
 }
 

package/core/src/query-warnings.ts

@@ -217,3 +217,113 @@ export function ignoredParamWarning(param: string, reason: string): QueryWarning
     param,
   };
 }
+
+/**
+ * Cheap zero-result search suggestion (vault#551 WS2B — mirrors the tag
+ * `did_you_mean` above via the SAME `suggestSimilarTag` scorer, just over a
+ * different candidate pool). Callers MUST only invoke this AFTER a search
+ * already returned zero rows — it is never on the hot non-empty path, so
+ * the extra vocabulary scan costs nothing in the overwhelmingly common
+ * case where the search actually found something.
+ *
+ * Candidates are drawn from two cheap-to-reach sources, unioned:
+ *
+ *   - the FTS5 vocabulary — a `notes_fts_vocab` `fts5vocab('row')` table
+ *     created here LAZILY and BEST-EFFORT (not part of the schema/
+ *     migration — see the try/catch below) — the terms actually indexed,
+ *     POST-tokenization/stemming (porter). A suggestion therefore
+ *     sometimes reads as a stemmed form ("propoli" rather than "propolis")
+ *     rather than the original dictionary word — an accepted tradeoff
+ *     (documented in docs/HTTP_API.md) rather than maintaining a second,
+ *     unstemmed index just for spelling suggestions.
+ *   - tag names, when the caller passes `tagNames` (already cached by the
+ *     caller's tag hierarchy — free to include).
+ *
+ * `fts5vocab` is registered by the same FTS5 extension init as `notes_fts`
+ * itself (not a separately-enabled module) so it's expected to be
+ * available anywhere search already works — but this is NOT exercised
+ * against Cloudflare DO SQLite (flagged for the wire reviewer). The WHOLE
+ * function is wrapped in try/catch and degrades to "no suggestion" on ANY
+ * failure — including a runtime where `fts5vocab` is unavailable — rather
+ * than ever throwing; a spelling hint is a nicety, never worth risking the
+ * search response itself.
+ *
+ * Per-token length filtering (`length(term) BETWEEN len-2 AND len+2`) keeps
+ * the vocabulary scan bounded on a large vault without a hardcoded row cap
+ * that could silently exclude the very term a caller needs (a `LIMIT`
+ * ORDER-BY-frequency would bias toward common words, which is backwards
+ * for a name/typo lookup — the word you're trying to find is often rare).
+ */
+export function computeSearchDidYouMean(
+  db: Database,
+  rawQuery: string,
+  tagNames?: Iterable<string>,
+): string | undefined {
+  try {
+    const tokens = rawQuery.trim().split(/\s+/).filter(Boolean);
+    if (tokens.length === 0) return undefined;
+
+    db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts_vocab USING fts5vocab(notes_fts, 'row')");
+    const exactStmt = db.prepare("SELECT 1 FROM notes_fts_vocab WHERE term = ? LIMIT 1");
+    const rangeStmt = db.prepare("SELECT term FROM notes_fts_vocab WHERE length(term) BETWEEN ? AND ?");
+
+    let changed = false;
+    const corrected = tokens.map((tok) => {
+      // Strip leading/trailing punctuation for the lookup (the FTS5
+      // vocabulary never contains punctuation) but leave short tokens
+      // (numbers, "a", "of", ...) alone — too little signal to safely
+      // "correct" without a high false-positive rate.
+      const clean = tok.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "");
+      if (clean.length < 3) return tok;
+      const lower = clean.toLowerCase();
+
+      // Already indexed verbatim (post-stemming) — no typo signal.
+      if (exactStmt.get(lower)) return tok;
+
+      const lo = Math.max(1, clean.length - 2);
+      const hi = clean.length + 2;
+      const rows = rangeStmt.all(lo, hi) as { term: string }[];
+      const candidates = new Set<string>(rows.map((r) => r.term));
+      if (tagNames) for (const t of tagNames) candidates.add(t);
+
+      const suggestion = suggestSimilarTag(candidates, lower);
+      if (suggestion && suggestion.toLowerCase() !== lower) {
+        changed = true;
+        return suggestion;
+      }
+      return tok;
+    });
+
+    if (!changed) return undefined;
+    return corrected.join(" ");
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * `search_did_you_mean` warning (vault#551 WS2B) — wraps
+ * {@link computeSearchDidYouMean}'s suggestion in the standard warning
+ * shape, mirroring `unknown_tag`'s `did_you_mean` field. Returns
+ * `undefined` (nothing to push) when no suggestion clears the bar — a
+ * zero-result search is a perfectly ordinary outcome on its own and does
+ * NOT warrant a warning by itself; only an actual spelling suggestion does.
+ *
+ * Scope-unaware by construction, exactly like `collectUnknownTagWarnings`
+ * (see this file's top-of-file doc comment) — the FTS5 vocabulary spans
+ * the WHOLE vault regardless of any caller's tag scope. Callers on a
+ * tag-scoped session MUST NOT surface this warning directly:
+ * `src/mcp-tools.ts`'s `query-notes` wrapper already strips the entire
+ * `warnings` array for a scoped caller (so MCP is safe with no extra
+ * work); `src/routes.ts` must gate the call itself behind
+ * `tagScope.allowed === null`, same as it already does for
+ * `collectUnknownTagWarnings`.
+ */
+export function searchDidYouMeanWarning(rawQuery: string, suggestion: string): QueryWarning {
+  return {
+    code: "search_did_you_mean",
+    message: `no results for "${rawQuery}" — did you mean "${suggestion}"?`,
+    query: rawQuery,
+    did_you_mean: suggestion,
+  };
+}

package/core/src/schema-defaults.ts

@@ -60,6 +60,20 @@ export interface SchemaField {
   type?: "string" | "number" | "integer" | "boolean" | "array" | "object";
   enum?: string[];
   description?: string;
+  /**
+   * Mirrors `TagFieldSchema.indexed` (core/src/tag-schemas.ts) — this field
+   * has a generated column + B-tree index maintained on `notes` (see
+   * core/src/indexed-fields.ts). Parsed here (not just there) because
+   * `validateNote` uses it: an indexed field is a QUERY CONTRACT, not just a
+   * storage hint, so a `type_mismatch` on an indexed field is always
+   * enforced (`strict: true` on that ONE warning) regardless of this field's
+   * own `strict` flag (vault#553 Decision A). Every other constraint
+   * (enum/required/cardinality) on an indexed field stays governed by
+   * `strict` as before — only the TYPE contract is unconditional, because
+   * that's the one a type-mismatched write can silently poison range
+   * queries with (SQLite's TEXT-sorts-above-INTEGER affinity ordering).
+   */
+  indexed?: boolean;
   /**
    * Strict enforcement opt-in (vault#299, Part A). Default `false` — when
    * unset/false, ALL constraints on this field are advisory: violations
@@ -169,6 +183,7 @@ function parseFieldsJson(raw: string | null): Record<string, SchemaField> {
     if (typeof f.type === "string") field.type = f.type as SchemaField["type"];
     if (Array.isArray(f.enum)) field.enum = f.enum.filter((x): x is string => typeof x === "string");
     if (typeof f.description === "string") field.description = f.description;
+    if (f.indexed === true) field.indexed = true;
     if (f.strict === true) field.strict = true;
     if (f.required === true) field.required = true;
     if (f.cardinality === "one" || f.cardinality === "many") field.cardinality = f.cardinality;
@@ -319,6 +334,20 @@ function stringArraysEqual(a: string[] | undefined, b: string[] | undefined): bo
   return true;
 }
 
+/**
+ * Human-readable JSON-shape name for a value, used to name the "got" side of
+ * a `type_mismatch` message (vault#553 — "message names field + expected
+ * type + got type"). Distinct from SQLite's `json_type()` vocabulary
+ * (integer/real/true/false/text/...) — this is the JSON-Schema-ish vocabulary
+ * `SchemaField.type` already uses, so the message reads as "expected X, got
+ * Y" in the SAME words the schema declares.
+ */
+function jsonTypeOf(value: unknown): string {
+  if (value === null) return "null";
+  if (Array.isArray(value)) return "array";
+  return typeof value; // "string" | "number" | "boolean" | "object" | ...
+}
+
 function valueMatchesType(value: unknown, type: SchemaField["type"]): boolean {
   if (type === undefined) return true;
   switch (type) {
@@ -357,11 +386,15 @@ function valueMatchesType(value: unknown, type: SchemaField["type"]): boolean {
  *   → `cardinality_mismatch`
  *
  * Every violation carries `strict: true` iff its field declared `strict:true`
- * (vault#299). The list itself is the SAME whether or not a field is strict —
- * the difference is only the per-warning `strict` flag, which the write path
- * uses to decide block-vs-warn. Under `strict:false` this is byte-identical to
- * the historical advisory behavior PLUS the new `required`/`cardinality`
- * advisory reasons (which fire for any field declaring those, strict or not).
+ * (vault#299) — EXCEPT a `type_mismatch` on an `indexed: true` field, which is
+ * ALWAYS `strict: true` regardless of the field's own `strict` setting
+ * (vault#553 Decision A — an indexed field's type is a query contract, not
+ * just guidance). The list itself is the SAME whether or not a field is
+ * strict — the difference is only the per-warning `strict` flag, which the
+ * write path uses to decide block-vs-warn. Under `strict:false` (and
+ * `indexed:false`) this is byte-identical to the historical advisory
+ * behavior PLUS the new `required`/`cardinality` advisory reasons (which
+ * fire for any field declaring those, strict or not).
  *
  * Fields not declared by any ancestor's schema are ignored entirely.
  */
@@ -397,12 +430,20 @@ export function validateNote(
     if (absent) continue;
 
     if (spec.type && !valueMatchesType(value, spec.type)) {
+      // Decision A (vault#553): an INDEXED field's type is a query
+      // contract — a type-mismatched write poisons range-query ordering
+      // (SQLite's TEXT-sorts-above-INTEGER affinity) regardless of whether
+      // this field opted into `strict`. Force `strict: true` on THIS
+      // warning alone when the field is indexed; every other constraint
+      // (enum/required/cardinality) stays governed by `spec.strict` as
+      // before.
+      const indexedTypeStrict = spec.indexed === true;
       warnings.push({
         field: fieldName,
         schema: sourceTag,
         reason: "type_mismatch",
-        message: `'${fieldName}' should be ${spec.type} (tag '${sourceTag}')`,
-        ...strictFlag,
+        message: `'${fieldName}' should be ${spec.type} (tag '${sourceTag}'), got ${jsonTypeOf(value)}${indexedTypeStrict ? " — indexed fields reject type-mismatched writes (vault#553)" : ""}`,
+        ...(strictFlag.strict || indexedTypeStrict ? { strict: true } : {}),
       });
     }