@openparachute/vault 0.7.2 → 0.7.3-rc.10

package/core/src/embedding/staleness.ts

@@ -0,0 +1,83 @@
+/**
+ * Freshness gate for `note_vectors` (V2 — see `SEMANTIC-MVP-PLAN.md` §3,
+ * "Staleness hook"). Pure diffing logic: given a note's CURRENT chunks
+ * (from `chunkNoteContent`) and the vector rows already on disk, decide
+ * which chunk indices need (re-)embedding and which existing rows are
+ * obsolete (the note shrank, or the configured model changed) and should
+ * be pruned.
+ *
+ * No I/O here — `core/src/notes.ts`/`store.ts` own reading `note_vectors`
+ * and applying the resulting plan (embed the stale ones, delete the
+ * obsolete ones). Keeping this pure makes "no-op edit ≠ re-embed" and
+ * "model change ⇒ full sweep" both directly testable without a database.
+ *
+ * `content_hash` uses `node:crypto`'s `createHash` — already a core
+ * dependency (see `core/src/cursor.ts`), so this doesn't add a new one.
+ */
+
+import { createHash } from "node:crypto";
+import type { Chunk } from "./chunker.js";
+
+/** Deterministic content hash for one chunk's text — the freshness key. */
+export function contentHash(text: string): string {
+  return createHash("sha256").update(text, "utf8").digest("hex");
+}
+
+/** The subset of a `note_vectors` row this module needs to reason about. */
+export interface ExistingVectorRow {
+  chunk_ix: number;
+  model: string;
+  content_hash: string;
+}
+
+export interface StalenessPlan {
+  /**
+   * Chunks that need embedding: either no row exists yet at this
+   * `chunk_ix`, or the existing row's `model`/`content_hash` doesn't
+   * match — a no-op edit (same content, same model) never lands here.
+   */
+  stale: Chunk[];
+  /**
+   * `chunk_ix` values present in `existing` that no longer correspond to
+   * a current chunk (the note shrank) OR belong to a stale `model` — safe
+   * to DELETE. Pruning these is what keeps a shrunk note's leftover
+   * high-index chunk from lingering as a ghost match in the cosine scan.
+   */
+  obsoleteIxs: number[];
+}
+
+/**
+ * Diff a note's current chunk set against its existing `note_vectors`
+ * rows for the ACTIVE `model`. Rows recorded under a different model are
+ * always obsolete (a `note_vectors` row's `model` never mixes with the
+ * configured provider's model within one scan — see
+ * `core/src/notes.ts:semanticSearchNotes`, which filters `WHERE model =
+ * ?`) — so a model change surfaces as "every existing row is obsolete,
+ * every current chunk is stale," i.e. a full re-embed sweep, matching the
+ * plan's "model change → stale sweep (re-embed)."
+ */
+export function planStaleness(
+  existing: ExistingVectorRow[],
+  chunks: Chunk[],
+  model: string,
+): StalenessPlan {
+  const existingByIx = new Map(existing.map((r) => [r.chunk_ix, r]));
+  const currentIxs = new Set(chunks.map((c) => c.ix));
+
+  const stale: Chunk[] = [];
+  for (const c of chunks) {
+    const row = existingByIx.get(c.ix);
+    if (!row || row.model !== model || row.content_hash !== contentHash(c.text)) {
+      stale.push(c);
+    }
+  }
+
+  const obsoleteIxs: number[] = [];
+  for (const row of existing) {
+    if (!currentIxs.has(row.chunk_ix) || row.model !== model) {
+      obsoleteIxs.push(row.chunk_ix);
+    }
+  }
+
+  return { stale, obsoleteIxs };
+}

package/core/src/embedding/vector-codec.test.ts

@@ -0,0 +1,99 @@
+import { describe, it, expect } from "bun:test";
+import { Database } from "bun:sqlite";
+import { normalize, dot, encodeVector, decodeVector } from "./vector-codec.js";
+
+describe("normalize", () => {
+  it("produces a unit-length vector", () => {
+    const v = new Float32Array([3, 4]); // 3-4-5 triangle
+    const n = normalize(v);
+    const len = Math.sqrt(n[0] * n[0] + n[1] * n[1]);
+    expect(len).toBeCloseTo(1, 5);
+    expect(n[0]).toBeCloseTo(0.6, 5);
+    expect(n[1]).toBeCloseTo(0.8, 5);
+  });
+
+  it("does not mutate the input", () => {
+    const v = new Float32Array([3, 4]);
+    normalize(v);
+    expect(v[0]).toBe(3);
+    expect(v[1]).toBe(4);
+  });
+
+  it("guards the all-zero vector instead of dividing by zero", () => {
+    const v = new Float32Array([0, 0, 0]);
+    const n = normalize(v);
+    expect(Array.from(n)).toEqual([0, 0, 0]);
+    expect(n.every((x) => Number.isFinite(x))).toBe(true);
+  });
+});
+
+describe("dot", () => {
+  it("computes the dot product of two vectors", () => {
+    const a = new Float32Array([1, 2, 3]);
+    const b = new Float32Array([4, 5, 6]);
+    expect(dot(a, b)).toBeCloseTo(1 * 4 + 2 * 5 + 3 * 6, 5);
+  });
+
+  it("cosine similarity of two normalized identical vectors is 1", () => {
+    const v = normalize(new Float32Array([1, 2, 3, 4]));
+    expect(dot(v, v)).toBeCloseTo(1, 5);
+  });
+
+  it("cosine similarity of two normalized orthogonal vectors is 0", () => {
+    const a = normalize(new Float32Array([1, 0]));
+    const b = normalize(new Float32Array([0, 1]));
+    expect(dot(a, b)).toBeCloseTo(0, 5);
+  });
+});
+
+describe("encodeVector / decodeVector round-trip", () => {
+  it("round-trips a vector through actual bytes (not a tautological identity check)", () => {
+    const original = new Float32Array([0.1, -0.2, 3.14159, -0.0, 42, -42, 1e-10]);
+    const encoded = encodeVector(original);
+    expect(encoded).toBeInstanceOf(Uint8Array);
+    expect(encoded.byteLength).toBe(original.length * 4);
+    const decoded = decodeVector(encoded);
+    expect(decoded.length).toBe(original.length);
+    for (let i = 0; i < original.length; i++) {
+      expect(decoded[i]).toBeCloseTo(original[i], 5);
+    }
+  });
+
+  it("round-trips through an ACTUAL SQLite BLOB column, not just in-memory bytes", () => {
+    const db = new Database(":memory:");
+    db.exec("CREATE TABLE t (v BLOB)");
+    const original = normalize(new Float32Array(384).map((_, i) => Math.sin(i)));
+    db.prepare("INSERT INTO t (v) VALUES (?)").run(encodeVector(original));
+    const row = db.prepare("SELECT v FROM t").get() as { v: Uint8Array };
+    const decoded = decodeVector(row.v);
+    expect(decoded.length).toBe(original.length);
+    for (let i = 0; i < original.length; i++) {
+      expect(decoded[i]).toBeCloseTo(original[i], 5);
+    }
+    db.close();
+  });
+
+  it("encodeVector copies rather than aliasing the input buffer", () => {
+    const original = new Float32Array([1, 2, 3]);
+    const encoded = encodeVector(original);
+    original[0] = 999;
+    const decoded = decodeVector(encoded);
+    expect(decoded[0]).toBeCloseTo(1, 5); // unaffected by the mutation above
+  });
+
+  it("decodeVector throws on a byte length that isn't a multiple of 4", () => {
+    const bad = new Uint8Array([1, 2, 3]);
+    expect(() => decodeVector(bad)).toThrow(RangeError);
+  });
+
+  it("decodeVector handles a non-4-aligned view into a larger shared buffer", () => {
+    // Simulate what a driver might hand back: a Uint8Array view starting at
+    // a byte offset that isn't a multiple of 4 within a larger buffer.
+    const backing = new ArrayBuffer(20);
+    const view = new Uint8Array(backing, 1, 16); // offset 1 — misaligned for Float32Array
+    const original = new Float32Array([1.5, -2.5, 3.5, 4.5]);
+    view.set(new Uint8Array(original.buffer, original.byteOffset, original.byteLength));
+    const decoded = decodeVector(view);
+    expect(Array.from(decoded)).toEqual(Array.from(original));
+  });
+});

package/core/src/embedding/vector-codec.ts

@@ -0,0 +1,60 @@
+/**
+ * Vector <-> BLOB codec + cosine-similarity math for semantic search.
+ *
+ * `note_vectors.vector` stores a `Float32Array` as raw bytes (a BLOB
+ * column) — the door-agnostic shape the architecture doc settled on (a
+ * BLOB in the same per-vault SQLite both `bun:sqlite` and the Cloudflare
+ * DO shim support natively, vs. a vector extension neither door can share).
+ * Vectors are stored L2-NORMALIZED (see `normalize` below) so that ranking
+ * is a plain dot product — no per-query renormalization of every candidate
+ * row, which matters once a scan is touching thousands of rows.
+ */
+
+/** L2-normalize a vector in place... no — returns a NEW Float32Array (never mutates the input). */
+export function normalize(v: Float32Array): Float32Array {
+  let sumSq = 0;
+  for (let i = 0; i < v.length; i++) sumSq += v[i]! * v[i]!;
+  const norm = Math.sqrt(sumSq) || 1; // guard the all-zero vector (never divide by 0)
+  const out = new Float32Array(v.length);
+  for (let i = 0; i < v.length; i++) out[i] = v[i]! / norm;
+  return out;
+}
+
+/** Dot product of two equal-length vectors. Cosine similarity IFF both are L2-normalized. */
+export function dot(a: Float32Array, b: Float32Array): number {
+  let sum = 0;
+  const n = Math.min(a.length, b.length);
+  for (let i = 0; i < n; i++) sum += a[i]! * b[i]!;
+  return sum;
+}
+
+/**
+ * Encode a `Float32Array` to raw bytes for a SQLite BLOB parameter.
+ * Copies (never returns a view over the input's possibly-shared
+ * `ArrayBuffer`) so the caller's array can be mutated/GC'd freely
+ * afterward without corrupting what's bound to the statement.
+ */
+export function encodeVector(v: Float32Array): Uint8Array {
+  const copy = new Float32Array(v); // copies the values into a fresh buffer
+  return new Uint8Array(copy.buffer, copy.byteOffset, copy.byteLength);
+}
+
+/**
+ * Decode a BLOB column's raw bytes back to a `Float32Array`. `bun:sqlite`
+ * returns BLOB columns as `Uint8Array`; a byte length not a multiple of 4
+ * (corruption, or a caller passing the wrong column) throws rather than
+ * silently truncating the last partial float.
+ */
+export function decodeVector(blob: Uint8Array): Float32Array {
+  if (blob.byteLength % 4 !== 0) {
+    throw new RangeError(
+      `decodeVector: byte length ${blob.byteLength} is not a multiple of 4 (Float32Array) — corrupt or non-vector BLOB`,
+    );
+  }
+  // Copy into a freshly-aligned buffer: `blob` may be a view with a
+  // non-4-aligned `byteOffset` into a larger shared buffer (bun:sqlite's
+  // returned Uint8Array isn't guaranteed aligned), and Float32Array's
+  // constructor requires `byteOffset % 4 === 0` on the input buffer.
+  const copy = new Uint8Array(blob); // copies bytes into a new, 0-offset buffer
+  return new Float32Array(copy.buffer, copy.byteOffset, copy.byteLength / 4);
+}

package/core/src/embedding/vectors.test.ts

@@ -0,0 +1,163 @@
+import { describe, it, expect, beforeEach } from "bun:test";
+import { Database } from "bun:sqlite";
+import { SqliteStore } from "../store.js";
+import {
+  getNoteVectorRows,
+  deleteObsoleteVectorRows,
+  upsertNoteVector,
+  countNotesPendingEmbedding,
+  getNotesPendingEmbedding,
+} from "./vectors.js";
+import { normalize, decodeVector } from "./vector-codec.js";
+
+const MODEL = "test-model";
+
+let db: Database;
+let store: SqliteStore;
+
+beforeEach(() => {
+  db = new Database(":memory:");
+  store = new SqliteStore(db);
+});
+
+describe("upsertNoteVector / getNoteVectorRows", () => {
+  it("a fresh note has zero vector rows", async () => {
+    const note = await store.createNote("hello", { path: "n" });
+    expect(getNoteVectorRows(db, note.id)).toEqual([]);
+  });
+
+  it("upsert inserts a new row, readable back with matching hash/model", async () => {
+    const note = await store.createNote("hello", { path: "n" });
+    upsertNoteVector(db, note.id, { ix: 0, text: "hello" }, normalize(new Float32Array([1, 2, 3])), MODEL, "hash-a");
+    const rows = getNoteVectorRows(db, note.id);
+    expect(rows).toEqual([{ chunk_ix: 0, model: MODEL, content_hash: "hash-a" }]);
+  });
+
+  it("upsert on an existing chunk_ix REPLACES it (re-embed overwrites in place)", async () => {
+    const note = await store.createNote("hello", { path: "n" });
+    upsertNoteVector(db, note.id, { ix: 0, text: "v1" }, normalize(new Float32Array([1, 0, 0])), MODEL, "hash-v1");
+    upsertNoteVector(db, note.id, { ix: 0, text: "v2" }, normalize(new Float32Array([0, 1, 0])), MODEL, "hash-v2");
+
+    const rows = getNoteVectorRows(db, note.id);
+    expect(rows.length).toBe(1);
+    expect(rows[0].content_hash).toBe("hash-v2");
+
+    const raw = db.prepare("SELECT vector FROM note_vectors WHERE note_id = ? AND chunk_ix = 0").get(note.id) as {
+      vector: Uint8Array;
+    };
+    const decoded = decodeVector(raw.vector);
+    expect(decoded[1]).toBeCloseTo(1, 5); // the v2 vector, not v1
+  });
+
+  it("a model change on the same chunk_ix also replaces (never appends a second row)", async () => {
+    const note = await store.createNote("hello", { path: "n" });
+    upsertNoteVector(db, note.id, { ix: 0, text: "x" }, normalize(new Float32Array([1, 0])), "model-a", "hash");
+    upsertNoteVector(db, note.id, { ix: 0, text: "x" }, normalize(new Float32Array([0, 1])), "model-b", "hash");
+    const rows = getNoteVectorRows(db, note.id);
+    expect(rows.length).toBe(1);
+    expect(rows[0].model).toBe("model-b");
+  });
+
+  it("multiple chunk_ix rows coexist independently", async () => {
+    const note = await store.createNote("hello", { path: "n" });
+    upsertNoteVector(db, note.id, { ix: 0, text: "a" }, normalize(new Float32Array([1, 0])), MODEL, "h0");
+    upsertNoteVector(db, note.id, { ix: 1, text: "b" }, normalize(new Float32Array([0, 1])), MODEL, "h1");
+    const rows = getNoteVectorRows(db, note.id).sort((a, b) => a.chunk_ix - b.chunk_ix);
+    expect(rows.map((r) => r.chunk_ix)).toEqual([0, 1]);
+  });
+});
+
+describe("deleteObsoleteVectorRows", () => {
+  it("removes only the specified chunk_ix rows", async () => {
+    const note = await store.createNote("hello", { path: "n" });
+    upsertNoteVector(db, note.id, { ix: 0, text: "a" }, normalize(new Float32Array([1, 0])), MODEL, "h0");
+    upsertNoteVector(db, note.id, { ix: 1, text: "b" }, normalize(new Float32Array([0, 1])), MODEL, "h1");
+    upsertNoteVector(db, note.id, { ix: 2, text: "c" }, normalize(new Float32Array([1, 1])), MODEL, "h2");
+
+    deleteObsoleteVectorRows(db, note.id, [1]);
+
+    const rows = getNoteVectorRows(db, note.id).map((r) => r.chunk_ix).sort();
+    expect(rows).toEqual([0, 2]);
+  });
+
+  it("is a no-op on an empty list (no IN () syntax error)", async () => {
+    const note = await store.createNote("hello", { path: "n" });
+    upsertNoteVector(db, note.id, { ix: 0, text: "a" }, normalize(new Float32Array([1, 0])), MODEL, "h0");
+    expect(() => deleteObsoleteVectorRows(db, note.id, [])).not.toThrow();
+    expect(getNoteVectorRows(db, note.id).length).toBe(1);
+  });
+});
+
+describe("countNotesPendingEmbedding", () => {
+  it("all notes pending when none have vectors", async () => {
+    await store.createNote("a", { path: "a" });
+    await store.createNote("b", { path: "b" });
+    expect(countNotesPendingEmbedding(db, MODEL)).toEqual({ total: 2, pending: 2 });
+  });
+
+  it("a note with at least one vector under the active model is no longer pending", async () => {
+    const a = await store.createNote("a", { path: "a" });
+    await store.createNote("b", { path: "b" });
+    upsertNoteVector(db, a.id, { ix: 0, text: "a" }, normalize(new Float32Array([1, 0])), MODEL, "h");
+    expect(countNotesPendingEmbedding(db, MODEL)).toEqual({ total: 2, pending: 1 });
+  });
+
+  it("a vector under a DIFFERENT model doesn't count toward the active model's embedded set", async () => {
+    const a = await store.createNote("a", { path: "a" });
+    upsertNoteVector(db, a.id, { ix: 0, text: "a" }, normalize(new Float32Array([1, 0])), "old-model", "h");
+    expect(countNotesPendingEmbedding(db, MODEL)).toEqual({ total: 1, pending: 1 });
+  });
+
+  it("zero notes: zero total, zero pending", () => {
+    expect(countNotesPendingEmbedding(db, MODEL)).toEqual({ total: 0, pending: 0 });
+  });
+
+  it("L4: a blank note is excluded from `total` too, not just `pending` — it never counts as pending, and never inflates the denominator either", async () => {
+    const real = await store.createNote("real content", { path: "real" });
+    await store.createNote("", { path: "blank" });
+    // No vector for `real` yet — before this fix `total` would be 2
+    // (counting the blank note), so `pending` would be 2 forever (the
+    // blank note can never get embedded) instead of settling to 0 once
+    // `real` is embedded.
+    expect(countNotesPendingEmbedding(db, MODEL)).toEqual({ total: 1, pending: 1 });
+
+    upsertNoteVector(db, real.id, { ix: 0, text: "real content" }, normalize(new Float32Array([1, 0])), MODEL, "h");
+    expect(countNotesPendingEmbedding(db, MODEL)).toEqual({ total: 1, pending: 0 });
+  });
+});
+
+describe("getNotesPendingEmbedding", () => {
+  it("returns notes with zero vectors under the active model", async () => {
+    const a = await store.createNote("a content", { path: "a" });
+    const b = await store.createNote("b content", { path: "b" });
+    upsertNoteVector(db, a.id, { ix: 0, text: "a content" }, normalize(new Float32Array([1, 0])), MODEL, "h");
+
+    const pending = getNotesPendingEmbedding(db, MODEL);
+    expect(pending.map((n) => n.id)).toEqual([b.id]);
+    expect(pending[0].content).toBe("b content");
+  });
+
+  it("respects an optional limit", async () => {
+    for (let i = 0; i < 5; i++) await store.createNote(`note ${i}`, { path: `n${i}` });
+    const pending = getNotesPendingEmbedding(db, MODEL, 2);
+    expect(pending.length).toBe(2);
+  });
+
+  it("empty array when every note is embedded", async () => {
+    const a = await store.createNote("a", { path: "a" });
+    upsertNoteVector(db, a.id, { ix: 0, text: "a" }, normalize(new Float32Array([1, 0])), MODEL, "h");
+    expect(getNotesPendingEmbedding(db, MODEL)).toEqual([]);
+  });
+
+  it("M3: excludes a blank note — never returned as pending, so the sweep can't hammer it forever", async () => {
+    await store.createNote("", { path: "blank" });
+    const real = await store.createNote("real content", { path: "real" });
+    const pending = getNotesPendingEmbedding(db, MODEL);
+    expect(pending.map((n) => n.id)).toEqual([real.id]);
+  });
+
+  it("M3: excludes a whitespace-only note", async () => {
+    await store.createNote("   \n\t  ", { path: "whitespace" });
+    expect(getNotesPendingEmbedding(db, MODEL)).toEqual([]);
+  });
+});

package/core/src/embedding/vectors.ts

@@ -0,0 +1,135 @@
+/**
+ * `note_vectors` CRUD — the small, pure-SQL surface the embed-on-write
+ * hook (registered in `src/`, self-host; the DO-alarm drain, cloud) uses
+ * to apply a `StalenessPlan` (see `staleness.ts`). Kept separate from
+ * `notes.ts`'s read-side `semanticSearchNotes` scan so the write-path
+ * helpers here have no dependency on the ranking/cosine code, and vice
+ * versa.
+ */
+
+import { Database } from "bun:sqlite";
+import { encodeVector } from "./vector-codec.js";
+import type { Chunk } from "./chunker.js";
+import type { ExistingVectorRow } from "./staleness.js";
+
+/**
+ * SQL fragment: `<colRef>` (a `content` column reference — bare `content`
+ * or an aliased `n.content`) is non-blank. A blank/whitespace-only note
+ * has nothing embeddable (see `embedding-worker.ts`'s chunk filter) and
+ * NEVER gets a `note_vectors` row, so every "which notes still need
+ * embedding" query — the backfill-sweep candidate list
+ * (`getNotesPendingEmbedding`), the progress count
+ * (`countNotesPendingEmbedding`), and the semantic-search scan's own
+ * `pendingCount` (`notes.ts:semanticSearchNotes`) — must exclude it, or a
+ * vault with even one blank note carries a permanent phantom "pending"
+ * count that never drains.
+ *
+ * SQLite's `TRIM(x)` (no second arg) only strips literal SPACE
+ * characters, unlike JS's `.trim()` — a note that's only tabs/newlines
+ * would slip through a bare `TRIM(content) = ''` check. `TRIM(x, y)`'s
+ * second-argument form strips every character IN `y` from both ends, so
+ * passing the space/tab/LF/VT/FF/CR set (`char(32,9,10,11,12,13)`) makes
+ * this match JS whitespace semantics closely enough for a "nothing here"
+ * check (exotic Unicode whitespace isn't covered — not worth the
+ * complexity for a defensive filter, not user-facing validation).
+ */
+export function nonBlankContentClause(colRef: string): string {
+  return `TRIM(COALESCE(${colRef}, ''), char(32,9,10,11,12,13)) != ''`;
+}
+
+/** Read a note's existing `note_vectors` rows — the shape `planStaleness` diffs against. */
+export function getNoteVectorRows(db: Database, noteId: string): ExistingVectorRow[] {
+  return db
+    .prepare("SELECT chunk_ix, model, content_hash FROM note_vectors WHERE note_id = ?")
+    .all(noteId) as ExistingVectorRow[];
+}
+
+/**
+ * Delete specific obsolete chunk rows for a note — the "cheap sync write"
+ * that keeps a shrunk note (or a model change) from leaving a ghost
+ * vector that `semanticSearchNotes` would otherwise still rank against.
+ * No-op on an empty list (avoids an `IN ()` syntax error).
+ */
+export function deleteObsoleteVectorRows(db: Database, noteId: string, obsoleteIxs: number[]): void {
+  if (obsoleteIxs.length === 0) return;
+  const placeholders = obsoleteIxs.map(() => "?").join(", ");
+  db.prepare(`DELETE FROM note_vectors WHERE note_id = ? AND chunk_ix IN (${placeholders})`).run(
+    noteId,
+    ...obsoleteIxs,
+  );
+}
+
+/**
+ * Write (insert or replace) one chunk's embedded vector. Replace-by-PK
+ * semantics — a re-embed (content changed, or the model changed)
+ * overwrites the existing `(note_id, chunk_ix)` row in place; a vault
+ * never carries two models' vectors for the same chunk simultaneously
+ * (see the `note_vectors` schema doc comment).
+ */
+export function upsertNoteVector(
+  db: Database,
+  noteId: string,
+  chunk: Chunk,
+  vector: Float32Array,
+  model: string,
+  contentHash: string,
+): void {
+  db.prepare(
+    `INSERT INTO note_vectors (note_id, chunk_ix, vector, dims, model, content_hash, embedded_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?)
+     ON CONFLICT (note_id, chunk_ix) DO UPDATE SET
+       vector = excluded.vector,
+       dims = excluded.dims,
+       model = excluded.model,
+       content_hash = excluded.content_hash,
+       embedded_at = excluded.embedded_at`,
+  ).run(noteId, chunk.ix, encodeVector(vector), vector.length, model, contentHash, new Date().toISOString());
+}
+
+/**
+ * Count notes that have zero `note_vectors` rows under `model` — the same
+ * "pending" definition `semanticSearchNotes`/`getNotesPendingEmbedding`
+ * use (see `nonBlankContentClause`'s doc for why blank notes are excluded
+ * from `total`, not just `pending` — otherwise `pending` could go
+ * negative, or a vault with only blank notes would report 100% pending
+ * forever). Used by the `embeddings` capability field (`GET /api/vault`)
+ * to report backfill progress without a full semantic scan.
+ */
+export function countNotesPendingEmbedding(db: Database, model: string): { total: number; pending: number } {
+  const total = (
+    db.prepare(`SELECT COUNT(*) AS n FROM notes WHERE ${nonBlankContentClause("content")}`).get() as { n: number }
+  ).n;
+  const embedded = (
+    db
+      .prepare("SELECT COUNT(DISTINCT note_id) AS n FROM note_vectors WHERE model = ?")
+      .get(model) as { n: number }
+  ).n;
+  return { total, pending: Math.max(0, total - embedded) };
+}
+
+/**
+ * Notes with zero `note_vectors` rows under `model` — the actual
+ * candidate list (not just a count) the self-host in-process drain and
+ * the cloud DO-alarm drain walk to backfill/catch up a vault. A note with
+ * SOME but not all chunks embedded (a partial prior embed, e.g. a crash
+ * mid-drain) is NOT returned here — see the module doc on
+ * `countNotesPendingEmbedding` for the same coarse-but-honest tradeoff;
+ * the embed-on-write hook re-derives full freshness per note via
+ * `planStaleness`, so a partially-embedded note self-heals on its next
+ * edit even if the sweep doesn't re-visit it.
+ *
+ * Excludes blank/whitespace-only notes — see `nonBlankContentClause`'s
+ * doc comment for why (they'd otherwise be "pending" FOREVER, hammering
+ * the provider with empty input every sweep pass forever).
+ */
+export function getNotesPendingEmbedding(
+  db: Database,
+  model: string,
+  limit?: number,
+): { id: string; content: string }[] {
+  const sql = `SELECT id, content FROM notes
+    WHERE id NOT IN (SELECT DISTINCT note_id FROM note_vectors WHERE model = ?)
+      AND ${nonBlankContentClause("content")}${typeof limit === "number" ? " LIMIT ?" : ""}`;
+  const params: (string | number)[] = typeof limit === "number" ? [model, limit] : [model];
+  return db.prepare(sql).all(...params) as { id: string; content: string }[];
+}

package/core/src/expand.ts

@@ -3,8 +3,9 @@
  *
  * Used by `query-notes` when `expand_links=true`. Replaces wikilink matches
  * with delimited blocks containing the linked note's content (full mode) or
- * metadata summary (summary mode). Deduplicates across the query and guards
- * against cycles via a shared `expanded` set.
+ * a summary (summary mode): `metadata.summary` when present, else the
+ * target note's lede (see `summaryText`). Deduplicates across the query and
+ * guards against cycles via a shared `expanded` set.
  */
 
 import { Database } from "bun:sqlite";
@@ -152,11 +153,18 @@ function renderSummary(note: Note): string {
   return `<expanded path="${pathAttr}" mode="summary">\n${summary}\n</expanded>`;
 }
 
+/**
+ * `metadata.summary` wins when present (no behavior change for existing
+ * callers). Absent it, falls back to the target note's lede — its opening
+ * paragraph, per `computeLede` — so a note that was never given a
+ * `metadata.summary` still yields something useful in summary-mode
+ * expansion instead of an empty block.
+ */
 function summaryText(note: Note): string {
   const meta = note.metadata as Record<string, unknown> | undefined;
   const s = meta?.summary;
   if (typeof s === "string" && s.trim()) return s.trim();
-  return "";
+  return noteOps.computeLede(note.content) ?? "";
 }
 
 function escapeAttr(s: string): string {

package/core/src/indexed-fields.test.ts

@@ -67,13 +67,16 @@ describe("indexed-fields: module", () => {
     expect(() => validateFieldName("'; DROP TABLE notes; --")).toThrow(IndexedFieldError);
   });
 
-  it("TYPE_MAP covers string/integer/boolean/reference", () => {
+  it("TYPE_MAP covers string/integer/boolean/reference/date", () => {
     expect(TYPE_MAP.string).toBe("TEXT");
     expect(TYPE_MAP.integer).toBe("INTEGER");
     expect(TYPE_MAP.boolean).toBe("INTEGER");
     // vault#typed-reference-field: `reference` stores like `string` — the
     // metadata VALUE (an id/path/title) is what's indexed here.
     expect(TYPE_MAP.reference).toBe("TEXT");
+    // vault#date-field-type: `date` also stores like `string` — an ISO-8601
+    // string, which sorts correctly under a plain TEXT comparison.
+    expect(TYPE_MAP.date).toBe("TEXT");
   });
 
   it("declareField creates column + index on first declaration", () => {
@@ -212,12 +215,15 @@ describe("update-tag: indexed flag", () => {
   });
 
   it("unsupported field type for indexing throws", async () => {
+    // vault#date-field-type: `date` JOINED the indexable subset (stores TEXT,
+    // same as string/reference) — `number` (a float) is the current example
+    // of a recognized-but-unindexable type; see TYPE_MAP.
     expect(() =>
       findTool("update-tag").execute({
         tag: "project",
-        fields: { weird: { type: "date", indexed: true } },
+        fields: { weird: { type: "number", indexed: true } },
       }),
-    ).toThrow(/unsupported type "date"/);
+    ).toThrow(/unsupported type "number"/);
   });
 
   it("invalid field name for indexing throws", async () => {

package/core/src/indexed-fields.ts

@@ -27,13 +27,21 @@ export type SqliteType = "TEXT" | "INTEGER";
 // `string` — because the metadata VALUE (an id/path/title) is what's
 // indexed here; the resolved graph link is a separate concern maintained by
 // `core/src/store.ts`'s write path. See tag-schemas.ts's `VALID_FIELD_TYPES`.
-export type FieldType = "string" | "integer" | "boolean" | "reference";
+// `date` is also stored as TEXT: an indexed `date` field holds an ISO-8601
+// string (validated by `tag-schemas.ts`'s `defaultMatchesType` /
+// `schema-defaults.ts`'s `valueMatchesType`), and ISO-8601 strings are
+// lexicographically comparable — the same reason TEXT-stored `updated_at`
+// range queries already work — so `gt`/`gte`/`lt`/`lte`, `date_filter`, and
+// `order_by` all fall out of the generic TEXT-column machinery with no
+// type-specific SQL.
+export type FieldType = "string" | "integer" | "boolean" | "reference" | "date";
 
 export const TYPE_MAP: Record<FieldType, SqliteType> = {
   string: "TEXT",
   integer: "INTEGER",
   boolean: "INTEGER",
   reference: "TEXT",
+  date: "TEXT",
 };
 
 export interface IndexedField {