@openparachute/vault 0.7.2 → 0.7.3-rc.10

package/core/src/notes.semantic-search.test.ts

@@ -0,0 +1,131 @@
+import { describe, it, expect, beforeEach } from "bun:test";
+import { Database } from "bun:sqlite";
+import { SqliteStore } from "./store.js";
+import { semanticSearchNotes } from "./notes.js";
+import { encodeVector, normalize } from "./embedding/vector-codec.js";
+
+const MODEL = "test-model";
+const DIMS = 4;
+
+let db: Database;
+let store: SqliteStore;
+
+beforeEach(() => {
+  db = new Database(":memory:");
+  store = new SqliteStore(db);
+});
+
+/** Insert a chunk_ix=0 vector row for a note under the test model. */
+function insertVector(noteId: string, vector: number[]): void {
+  const encoded = encodeVector(normalize(new Float32Array(vector)));
+  db.prepare(
+    `INSERT INTO note_vectors (note_id, chunk_ix, vector, dims, model, content_hash, embedded_at)
+     VALUES (?, 0, ?, ?, ?, ?, ?)`,
+  ).run(noteId, encoded, DIMS, MODEL, "hash", new Date().toISOString());
+}
+
+function insertChunkVector(noteId: string, ix: number, vector: number[]): void {
+  const encoded = encodeVector(normalize(new Float32Array(vector)));
+  db.prepare(
+    `INSERT INTO note_vectors (note_id, chunk_ix, vector, dims, model, content_hash, embedded_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?)`,
+  ).run(noteId, ix, encoded, DIMS, MODEL, `hash-${ix}`, new Date().toISOString());
+}
+
+describe("semanticSearchNotes", () => {
+  it("ranks notes by cosine similarity, highest first", async () => {
+    const a = await store.createNote("note a", { path: "a" });
+    const b = await store.createNote("note b", { path: "b" });
+    const c = await store.createNote("note c", { path: "c" });
+
+    const query = normalize(new Float32Array([1, 0, 0, 0]));
+    insertVector(a.id, [1, 0, 0, 0]); // identical direction — score ~1
+    insertVector(b.id, [0, 1, 0, 0]); // orthogonal — score ~0
+    insertVector(c.id, [0.9, 0.1, 0, 0]); // close-ish — score high but < a
+
+    const result = semanticSearchNotes(db, query, {}, MODEL);
+    expect(result.notes.map((n) => n.id)).toEqual([a.id, c.id, b.id]);
+    expect(result.notes[0].score).toBeCloseTo(1, 4);
+    expect(result.totalCandidates).toBe(3);
+    expect(result.pendingCount).toBe(0);
+  });
+
+  it("respects limit (top-K)", async () => {
+    const query = normalize(new Float32Array([1, 0, 0, 0]));
+    const ids: string[] = [];
+    for (let i = 0; i < 5; i++) {
+      const n = await store.createNote(`note ${i}`, { path: `n${i}` });
+      ids.push(n.id);
+      insertVector(n.id, [1 - i * 0.01, 0.01 * i, 0, 0]);
+    }
+    const result = semanticSearchNotes(db, query, { limit: 2 }, MODEL);
+    expect(result.notes.length).toBe(2);
+    expect(result.totalCandidates).toBe(5);
+  });
+
+  it("filter-then-rank: structured filters (tags) narrow the candidate set BEFORE ranking", async () => {
+    const query = normalize(new Float32Array([1, 0, 0, 0]));
+    const tagged = await store.createNote("tagged note", { path: "tagged", tags: ["project"] });
+    const untagged = await store.createNote("untagged note", { path: "untagged" });
+    // untagged has the objectively CLOSER vector, but is filtered out by `tags`.
+    insertVector(tagged.id, [0.5, 0.5, 0, 0]);
+    insertVector(untagged.id, [1, 0, 0, 0]);
+
+    const result = semanticSearchNotes(db, query, { tags: ["project"] }, MODEL);
+    expect(result.notes.map((n) => n.id)).toEqual([tagged.id]);
+    expect(result.totalCandidates).toBe(1);
+  });
+
+  it("pendingCount: candidates with no vector row for the active model are excluded from ranking but counted as pending", async () => {
+    const query = normalize(new Float32Array([1, 0, 0, 0]));
+    const embedded = await store.createNote("embedded", { path: "embedded" });
+    const notYetEmbedded = await store.createNote("not yet embedded", { path: "pending" });
+    insertVector(embedded.id, [1, 0, 0, 0]);
+    // notYetEmbedded gets no note_vectors row at all.
+
+    const result = semanticSearchNotes(db, query, {}, MODEL);
+    expect(result.notes.map((n) => n.id)).toEqual([embedded.id]);
+    expect(result.totalCandidates).toBe(2);
+    expect(result.pendingCount).toBe(1);
+  });
+
+  it("ignores vectors recorded under a DIFFERENT model", async () => {
+    const query = normalize(new Float32Array([1, 0, 0, 0]));
+    const note = await store.createNote("note", { path: "n" });
+    const encoded = encodeVector(normalize(new Float32Array([1, 0, 0, 0])));
+    db.prepare(
+      `INSERT INTO note_vectors (note_id, chunk_ix, vector, dims, model, content_hash, embedded_at)
+       VALUES (?, 0, ?, ?, 'stale-model', 'hash', ?)`,
+    ).run(note.id, encoded, DIMS, new Date().toISOString());
+
+    const result = semanticSearchNotes(db, query, {}, MODEL);
+    expect(result.notes).toEqual([]);
+    expect(result.pendingCount).toBe(1); // no row under the ACTIVE model
+  });
+
+  it("best-chunk ranking: a long note's best-matching SECTION outranks a note with weak whole-note similarity", async () => {
+    // This is the exact case chunking exists for (P0's dominant miss
+    // pattern: whole-note embedding dilutes long, multi-topic notes).
+    const query = normalize(new Float32Array([1, 0, 0, 0]));
+
+    const longNote = await store.createNote("a long multi-section note", { path: "long" });
+    // Section 0 is about something unrelated (poor match); section 2 is a
+    // strong match for the query — best-chunk ranking must pick that up.
+    insertChunkVector(longNote.id, 0, [0, 1, 0, 0]); // orthogonal — weak
+    insertChunkVector(longNote.id, 1, [0.3, 0.3, 0.3, 0.3]); // mediocre
+    insertChunkVector(longNote.id, 2, [1, 0, 0, 0]); // strong match
+
+    const shortNote = await store.createNote("a short weakly-related note", { path: "short" });
+    insertVector(shortNote.id, [0.5, 0.5, 0, 0]); // weak whole-note similarity
+
+    const result = semanticSearchNotes(db, query, {}, MODEL);
+    expect(result.notes.map((n) => n.id)).toEqual([longNote.id, shortNote.id]);
+    expect(result.notes[0].score).toBeCloseTo(1, 4);
+  });
+
+  it("returns nothing (no throw) when the candidate set is empty", () => {
+    const query = normalize(new Float32Array([1, 0, 0, 0]));
+    const result = semanticSearchNotes(db, query, { tags: ["nonexistent"] }, MODEL);
+    expect(result).toEqual({ notes: [], pendingCount: 0, totalCandidates: 0 });
+  });
+});

package/core/src/notes.ts

@@ -1,5 +1,7 @@
 import { Database, type SQLQueryBindings } from "bun:sqlite";
-import type { Note, NoteIndex, QueryOpts, QueryNotesPage, VaultStats, VaultMap, AggregateSpec, AggregateRow } from "./types.js";
+import type { Note, NoteIndex, QueryOpts, QueryNotesPage, VaultStats, VaultMap, AggregateSpec, AggregateRow, SemanticSearchResult } from "./types.js";
+import { decodeVector, dot } from "./embedding/vector-codec.js";
+import { nonBlankContentClause } from "./embedding/vectors.js";
 import { normalizePath } from "./paths.js";
 import { transaction } from "./txn.js";
 import {
@@ -23,6 +25,7 @@ import { computeExpandedTagCounts, loadTagHierarchy, stripTagHash } from "./tag-
 import { chunkForInClause, IN_VIA_JSON_EACH, jsonEachParam } from "./sql-in.js";
 import {
   buildLiteralSearchQuery,
+  extractLiteralBoostTerms,
   SEARCH_WEIGHT_PATH,
   SEARCH_WEIGHT_CONTENT,
   type SearchMode,
@@ -962,7 +965,7 @@ function resolveUpdatedAtBoundMs(field: string, value: string): number {
  * semantics a normal query applies, rather than risking the two drifting
  * apart.
  */
-function buildFilterConditions(db: Database, opts: QueryOpts): { conditions: string[]; params: SQLQueryBindings[] } {
+export function buildFilterConditions(db: Database, opts: QueryOpts): { conditions: string[]; params: SQLQueryBindings[] } {
   const conditions: string[] = [];
   const params: SQLQueryBindings[] = [];
 
@@ -1808,6 +1811,61 @@ function searchSyntaxError(rawQuery: string, err: unknown, mode: SearchMode): Qu
   });
 }
 
+/**
+ * Post-rank a search result page (title axis, ratified 2026-07-17) so
+ * notes whose display title (first line of content) contains EVERY boost
+ * term outrank the rest — a title-match tier ahead of a body-only-match
+ * tier. Stable partition: within a tier, the incoming order (FTS5
+ * relevance, or an explicit `sort`) is preserved untouched — two notes
+ * that both match, or both don't, keep their relative order.
+ *
+ * No-migration by design (see the search-seam note on `searchNotes`
+ * below): re-orders the already-fetched page in memory instead of adding
+ * a weighted title column to `notes_fts`, which would need a schema
+ * migration + backfill. `notes_fts`'s existing `SEARCH_WEIGHT_PATH` bm25
+ * weighting (above) already boosts by note PATH — a different axis from
+ * the ratified title model (title = first line of CONTENT, not path).
+ * This function is the one that boosts by the content-derived title.
+ */
+function boostTitleMatches(notes: Note[], terms: string[]): Note[] {
+  if (terms.length === 0) return notes;
+  return notes
+    .map((note, index) => ({ note, index, titleMatch: titleMatchesAllTerms(note.content, terms) }))
+    .sort((a, b) => {
+      if (a.titleMatch === b.titleMatch) return a.index - b.index; // stable
+      return a.titleMatch ? -1 : 1;
+    })
+    .map((x) => x.note);
+}
+
+function titleMatchesAllTerms(content: string | null | undefined, terms: string[]): boolean {
+  const title = computeDisplayTitle(content);
+  if (!title) return false;
+  const lower = title.toLowerCase();
+  return terms.every((t) => lower.includes(t));
+}
+
+/**
+ * Apply the title-boost post-rank to a fetched search page. Skipped
+ * whenever it would fight the caller's intent:
+ *   - an explicit `sort: "asc"|"desc"` means the caller wants chronological
+ *     order, not relevance re-ranking;
+ *   - `search_mode: "advanced"` text carries raw FTS5 syntax (boolean
+ *     operators, column filters, prefix `*`) — naively tokenizing it into
+ *     "terms" would misread operators as content (see
+ *     `extractLiteralBoostTerms`'s doc comment). Only literal-mode queries
+ *     (the default) get boosted today.
+ */
+function applySearchTitleBoost(
+  notes: Note[],
+  mode: SearchMode,
+  rawQuery: string,
+  sort: "asc" | "desc" | undefined,
+): Note[] {
+  if (sort !== undefined || mode !== "literal") return notes;
+  return boostTitleMatches(notes, extractLiteralBoostTerms(rawQuery));
+}
+
 export function searchNotes(
   db: Database,
   query: string,
@@ -1844,6 +1902,14 @@ export function searchNotes(
   // The weights are our own numeric constants (not user input) interpolated
   // directly — bm25()'s weight arguments are positional per-column
   // multipliers, not general SQL expressions callers can influence.
+  //
+  // NOTE: "title" here means the note's `path` — this predates the title
+  // axis (ratified 2026-07-17), under which a note's title is its first
+  // CONTENT line, not its path. `applySearchTitleBoost` (below) is the
+  // mechanism for THAT axis: a post-rank pass over the already-scored page,
+  // not a third FTS column. The two are complementary, not redundant — a
+  // note can have a matching path with a non-matching first line, or vice
+  // versa.
   const scoreExpr = `(-1.0 * bm25(notes_fts, ${SEARCH_WEIGHT_PATH}, ${SEARCH_WEIGHT_CONTENT}))`;
 
   // `sort` honored under search (vault#551 WS2A item 3): default stays FTS5
@@ -1884,7 +1950,7 @@ export function searchNotes(
         ORDER BY ${orderBy}
         LIMIT ?
       `).all(ftsQuery, ...searchTags, limit) as (NoteRow & { score: number })[];
-      return notesWithTags(db, rows, scoresById(rows));
+      return applySearchTitleBoost(notesWithTags(db, rows, scoresById(rows)), mode, query, opts?.sort);
     } catch (err) {
       // Surface EVERY FTS5 error structured, never a raw rethrow (vault#551):
       // advanced mode expects it (the caller passed raw syntax); literal
@@ -1906,13 +1972,121 @@ export function searchNotes(
       ORDER BY ${orderBy}
       LIMIT ?
     `).all(ftsQuery, limit) as (NoteRow & { score: number })[];
-    return notesWithTags(db, rows, scoresById(rows));
+    return applySearchTitleBoost(notesWithTags(db, rows, scoresById(rows)), mode, query, opts?.sort);
   } catch (err) {
     if (err instanceof QueryError) throw err;
     throw searchSyntaxError(query, err, mode);
   }
 }
 
+/**
+ * Semantic-search scan (EXPERIMENTAL — `SEMANTIC-MVP-PLAN.md`). Structural
+ * sibling of `searchNotes`: text-shaped filters compose the same way, but
+ * ranking is cosine similarity over `note_vectors` instead of FTS5 bm25.
+ *
+ * Filter-then-rank (per the plan): `buildFilterConditions` — the SAME
+ * filter builder `queryNotes` uses — narrows to the candidate note set
+ * FIRST (tags/metadata/dates/path/etc. all apply), so "notes about X that
+ * are also #project" is one call, exactly like `search` composes with
+ * structured filters. Only THEN does the brute-force cosine scan run, and
+ * only over that (usually much smaller) candidate set — this is the perf
+ * lever the architecture doc calls out (filter shrinks N before the O(N)
+ * vector work).
+ *
+ * `model` restricts the scan to vectors recorded under the CURRENTLY
+ * configured embedding model — a row left over from a prior model (see
+ * `core/src/embedding/staleness.ts`) never contributes to ranking, so a
+ * model change can't silently blend two incomparable vector spaces.
+ *
+ * Ranking is best-CHUNK-per-note (max cosine over a note's chunks, per the
+ * ratified per-section chunking decision), but the returned unit is
+ * always the NOTE (via `fetchNotesByIdsOrdered`), never a bare chunk —
+ * matching `search`'s "text in → ranked notes out" contract.
+ *
+ * `queryVector` MUST already be L2-normalized (so the dot product below
+ * IS cosine similarity) — `Store.semanticSearch` normalizes right after
+ * embedding the query text; stored vectors are normalized at write time.
+ */
+export function semanticSearchNotes(
+  db: Database,
+  queryVector: Float32Array,
+  opts: QueryOpts,
+  model: string,
+): SemanticSearchResult {
+  const limit = typeof opts.limit === "number" ? opts.limit : 10;
+  const { conditions, params } = buildFilterConditions(db, opts);
+  // Blank/whitespace-only notes are never candidates: they have nothing
+  // embeddable and never get a note_vectors row (see `embedding-worker.ts`'s
+  // chunk filter), so counting one as a "pending" candidate would produce a
+  // phantom embeddings_pending warning that never drains. See
+  // `nonBlankContentClause`'s doc comment.
+  const allConditions = [...conditions, nonBlankContentClause("n.content")];
+  const whereClause = `WHERE ${allConditions.join(" AND ")}`;
+
+  const idRows = db.prepare(`SELECT n.id FROM notes n ${whereClause}`).all(...params) as {
+    id: string;
+  }[];
+  const candidateIds = idRows.map((r) => r.id);
+  if (candidateIds.length === 0) {
+    return { notes: [], pendingCount: 0, totalCandidates: 0 };
+  }
+
+  // Pull every candidate's vector rows for the ACTIVE model, batched to
+  // stay under the DO 100-bound-param cap (same chunking `fetchNotesByIdsOrdered`
+  // uses). A note absent from this map has no fresh vector yet — pending.
+  const vecRowsByNote = new Map<string, { chunk_ix: number; vector: Uint8Array }[]>();
+  for (const idChunk of chunkForInClause(candidateIds)) {
+    const placeholders = idChunk.map(() => "?").join(", ");
+    const rows = db.prepare(
+      `SELECT note_id, chunk_ix, vector FROM note_vectors WHERE model = ? AND note_id IN (${placeholders})`,
+    ).all(model, ...idChunk) as { note_id: string; chunk_ix: number; vector: Uint8Array }[];
+    for (const row of rows) {
+      let list = vecRowsByNote.get(row.note_id);
+      if (!list) {
+        list = [];
+        vecRowsByNote.set(row.note_id, list);
+      }
+      list.push(row);
+    }
+  }
+
+  // Best-chunk-per-note cosine (max over the note's chunks) — a long note
+  // whose 3rd section matches outranks a note with weak whole-note
+  // similarity, which is the whole point of per-section chunking.
+  const scored: { id: string; score: number }[] = [];
+  for (const id of candidateIds) {
+    const rows = vecRowsByNote.get(id);
+    if (!rows || rows.length === 0) continue;
+    let best = -Infinity;
+    for (const row of rows) {
+      const score = dot(decodeVector(row.vector), queryVector);
+      if (score > best) best = score;
+    }
+    scored.push({ id, score: best });
+  }
+  scored.sort((a, b) => b.score - a.score);
+  const top = scored.slice(0, limit);
+
+  // Reuse the existing hydration machinery — same as every other id-list
+  // fetch path (tags batched, no N+1).
+  const hydrated = fetchNotesByIdsOrdered(db, top.map((t) => t.id));
+  const byId = new Map(hydrated.map((n) => [n.id, n]));
+  const scoreById = new Map(top.map((t) => [t.id, t.score]));
+  const notes: Note[] = [];
+  for (const t of top) {
+    const note = byId.get(t.id);
+    if (!note) continue; // defensive: row vanished between the two queries
+    note.score = scoreById.get(t.id);
+    notes.push(note);
+  }
+
+  return {
+    notes,
+    pendingCount: candidateIds.length - vecRowsByNote.size,
+    totalCandidates: candidateIds.length,
+  };
+}
+
 /**
  * Map rows → Notes with tags hydrated in one batched query. `scores`
  * (vault#551 WS2C) is an optional id → weighted-bm25-score map, attached
@@ -2592,10 +2766,142 @@ export function mergeTags(
 /** Max code points in a NoteIndex preview. */
 export const NOTE_INDEX_PREVIEW_LEN = 120;
 
+/** Max code points in a computed `displayTitle` (title axis, ratified 2026-07-17). */
+export const DISPLAY_TITLE_MAX_LEN = 120;
+
+/**
+ * Bounded line-count a leading frontmatter block's closing fence is
+ * searched within (see `computeDisplayTitle`). Frontmatter blocks are a
+ * handful of lines in practice; capping the scan keeps a pathological
+ * "content starts with `---` but never closes it" note O(1)-ish rather
+ * than a full-content scan on every derivation.
+ */
+const FRONTMATTER_SCAN_LINES = 100;
+
+/**
+ * Shared by `computeDisplayTitle` and `computeLede`: given `content` already
+ * split into `lines`, return the index of the first line that counts as the
+ * start of the DOCUMENT body — after a leading closed frontmatter block, if
+ * `content` opens with one (see `computeDisplayTitle`'s frontmatter-skip
+ * doc for the full rationale). `0` when there's no leading frontmatter.
+ */
+function skipLeadingFrontmatter(lines: string[]): number {
+  if (lines[0]?.trim() !== "---") return 0;
+  const scanLimit = Math.min(lines.length, FRONTMATTER_SCAN_LINES);
+  for (let i = 1; i < scanLimit; i++) {
+    if (lines[i]?.trim() === "---") return i + 1;
+  }
+  return 0;
+}
+
+/**
+ * Derive a note's display title: the first non-empty line of `content`,
+ * with a leading markdown heading marker (`#` through `######`) and its
+ * following whitespace stripped, truncated to `DISPLAY_TITLE_MAX_LEN` code
+ * points. `null` when content has no non-empty line (empty or
+ * whitespace/heading-marker-only note).
+ *
+ * The ratified title model (2026-07-17): a note's title IS its first line
+ * — derived from content, NEVER stored as its own column. This function is
+ * the one place that derivation happens; callers (`toNoteIndex` below) call
+ * it fresh at read time. Timestamp-path formatting — what a surface shows
+ * in place of a `null` title — is deliberately NOT this function's job; it
+ * reports the honest content-derived value (or its absence) and leaves
+ * rendering to the caller.
+ *
+ * Frontmatter skip: normal ingestion strips a YAML frontmatter block into
+ * `metadata` before create, so `content` never carries one — but a direct
+ * MCP/REST create can paste raw frontmatter-bearing text
+ * (`---\ntitle: X\n---\n# Real Title`), and the first line of that
+ * DOCUMENT is not the first line of its delimiter. When `content` opens
+ * with a `---` line, derivation starts after the matching CLOSING `---`
+ * (searched within the first `FRONTMATTER_SCAN_LINES` lines). An
+ * unterminated opening `---` falls back to the pre-existing behavior —
+ * scanning from line 0 — so a note whose real first line is literally
+ * `---` isn't mangled.
+ */
+export function computeDisplayTitle(content: string | null | undefined): string | null {
+  if (!content) return null;
+  const lines = content.split("\n");
+  const startIndex = skipLeadingFrontmatter(lines);
+  for (let i = startIndex; i < lines.length; i++) {
+    const stripped = lines[i]!.replace(/^#{1,6}\s*/, "").trim();
+    if (stripped === "") continue;
+    // Iterate by Unicode code points so we don't split surrogate pairs mid-character.
+    const codePoints = Array.from(stripped);
+    return codePoints.length > DISPLAY_TITLE_MAX_LEN
+      ? codePoints.slice(0, DISPLAY_TITLE_MAX_LEN).join("")
+      : stripped;
+  }
+  return null;
+}
+
+/** Max code points in a computed `lede` (summaries-as-content, 2026-07-17 dialogue). */
+export const LEDE_MAX_LEN = 400;
+
+/**
+ * Derive a note's "lede": the first non-empty PARAGRAPH after the title
+ * line (a run of consecutive non-blank lines, whitespace-collapsed to one
+ * line, truncated to `LEDE_MAX_LEN` code points). `null` when there's no
+ * paragraph after the title — a title-only note has no lede to report, and
+ * callers must not fall back to repeating the title itself.
+ *
+ * This is the machinery half of a soft convention (2026-07-17 dialogue):
+ * summaries work better as visible CONTENT — a note's opening paragraph —
+ * than as hidden metadata, because visible text gets corrected by the notes
+ * that reference it while hidden metadata rots unnoticed. Nothing validates
+ * that a note actually opens with a title + lede; this function just reports
+ * what it finds, honestly, using the SAME title-line rule as
+ * `computeDisplayTitle` (including its frontmatter skip) so a caller that
+ * shows both title and lede sees them agree on where the title ends.
+ */
+export function computeLede(content: string | null | undefined): string | null {
+  if (!content) return null;
+  const lines = content.split("\n");
+  const bodyStart = skipLeadingFrontmatter(lines);
+
+  let titleLine = -1;
+  for (let i = bodyStart; i < lines.length; i++) {
+    if (lines[i]!.replace(/^#{1,6}\s*/, "").trim() !== "") {
+      titleLine = i;
+      break;
+    }
+  }
+  if (titleLine === -1) return null; // no title at all — nothing to find a lede after
+
+  let i = titleLine + 1;
+  while (i < lines.length && lines[i]!.trim() === "") i++; // skip blank lines after the title
+
+  const paragraphLines: string[] = [];
+  while (i < lines.length && lines[i]!.trim() !== "") {
+    paragraphLines.push(lines[i]!);
+    i++;
+  }
+  if (paragraphLines.length === 0) return null; // title-only note
+
+  const paragraph = paragraphLines.join(" ").replace(/\s+/g, " ").trim();
+  if (paragraph === "") return null;
+
+  const codePoints = Array.from(paragraph);
+  return codePoints.length > LEDE_MAX_LEN
+    ? codePoints.slice(0, LEDE_MAX_LEN).join("")
+    : paragraph;
+}
+
 /**
  * Convert a full Note into its lean index shape:
- * drops `content`, adds `byteSize` and a whitespace-collapsed `preview`.
- * Shared between the `query-notes` MCP tool, HTTP /notes endpoints, and /graph.
+ * drops `content`, adds `byteSize`, a whitespace-collapsed `preview`, and a
+ * computed `displayTitle`. Shared between the `query-notes` MCP tool, HTTP
+ * /notes endpoints, and /graph.
+ *
+ * Perf note: `displayTitle` costs nothing extra here — every caller of this
+ * function already has the full `Note` (content included) in hand by the
+ * time it's called. `queryNotes`'s two-phase page fetch selects a narrow
+ * `id`-only column list for the ORDER BY/pagination phase, but the second
+ * phase (`fetchNotesByIdsOrdered`) always `SELECT *`s the page's full rows
+ * before `toNoteIndex` ever runs — same as `preview`/`byteSize` today. If a
+ * future lean-list path is added that genuinely avoids a content read, it
+ * must NOT call this function (or `toNoteIndex`) on a content-less row.
  */
 export function toNoteIndex(note: Note): NoteIndex {
   const content = note.content ?? "";
@@ -2622,6 +2928,7 @@ export function toNoteIndex(note: Note): NoteIndex {
     metadata: note.metadata,
     byteSize,
     preview,
+    displayTitle: computeDisplayTitle(note.content),
     ...(note.score !== undefined ? { score: note.score } : {}),
   };
 }
@@ -2980,7 +3287,12 @@ interface NoteRow {
 }
 
 function rowToNote(row: NoteRow): Note {
-  let metadata: Record<string, unknown> | undefined;
+  // `metadata` is always present on the wire — NULL/"{}"/unparseable all
+  // collapse to `{}` rather than an absent key. `tags` is already always
+  // `[]` (see getNote/queryNotes/searchNotes hydration below); this keeps
+  // the note shape internally consistent and fixes a real third-party
+  // crash on clients that assumed `note.metadata` always exists.
+  let metadata: Record<string, unknown> = {};
   if (row.metadata && row.metadata !== "{}") {
     try { metadata = JSON.parse(row.metadata); } catch {}
   }

package/core/src/query-warnings.ts

@@ -219,6 +219,26 @@ export function ignoredParamWarning(param: string, reason: string): QueryWarning
   };
 }
 
+/**
+ * `truncated` warning (vault contracts-brief V1.3) — a structured, non-
+ * cursor list query returned exactly `limit` rows with no watermark to page
+ * from. That's ambiguous on its own: it might be the whole result set
+ * (coincidence), or there might be more rows silently cut off. Combined
+ * with the engine's default order (`created_at ASC` — oldest first,
+ * core/src/notes.ts) this is the concrete failure mode: a bare `GET
+ * /api/notes` on a >limit-note vault returns the OLDEST rows with zero
+ * signal that the NEWEST ones didn't make it. `?cursor=` (bootstrap or
+ * watermark) sidesteps this entirely — the warning only fires in its
+ * absence.
+ */
+export function truncatedResultsWarning(limit: number): QueryWarning {
+  return {
+    code: "truncated",
+    message: `${limit} = limit rows returned; there may be more. Pass ?cursor= to page, or sort=desc for newest-first.`,
+    limit,
+  };
+}
+
 /**
  * Cheap zero-result search suggestion (vault#551 WS2B — mirrors the tag
  * `did_you_mean` above via the SAME `suggestSimilarTag` scorer, just over a
@@ -375,3 +395,23 @@ export function searchDidYouMeanWarning(rawQuery: string, suggestion: string): Q
     did_you_mean: suggestion,
   };
 }
+
+/**
+ * `embeddings_pending` warning (semantic search MVP, EXPERIMENTAL —
+ * `SEMANTIC-MVP-PLAN.md`) — a `semantic: true` query ran and returned
+ * REAL results, but `pending` of the `total_candidates` structured-filter
+ * matches have no vector yet for the active embedding model (a fresh
+ * note, or the backfill sweep hasn't reached it yet). Attached alongside
+ * honest (possibly incomplete) results — NEVER instead of them, and never
+ * a silent keyword fallback: the caller always knows it got a partial
+ * semantic ranking, not a complete one. Callers only push this when
+ * `pending > 0` (the common steady-state case attaches nothing).
+ */
+export function embeddingsPendingWarning(pending: number, totalCandidates: number): QueryWarning {
+  return {
+    code: "embeddings_pending",
+    message: `${pending} of ${totalCandidates} candidate note(s) have no embedding yet for the active model — results may be incomplete until the backfill catches up.`,
+    pending,
+    total_candidates: totalCandidates,
+  };
+}