mcp-memory-bucket 0.13.0 → 0.14.0

package/dist/src/store/embeddings.js

@@ -0,0 +1,138 @@
+import { bumpChunksVersion, embedTextFor, setChunksChangedListener } from './chunks.js';
+export const DEFAULT_EMBED_MODEL = 'Xenova/bge-small-en-v1.5';
+const BGE_QUERY_PREFIX = 'Represent this sentence for searching relevant passages: ';
+/**
+ * A local sentence-embedding model run in-process through transformers.js (ONNX, CPU) — no LLM and
+ * no network after the one-time model download into `cacheDir`. The package is an optional
+ * dependency loaded on first use, so a machine where it failed to install only loses the semantic
+ * half of bucket_recall rather than failing to start.
+ */
+export function createTransformersEmbedder(opts) {
+    const model = opts.model ?? DEFAULT_EMBED_MODEL;
+    const isBge = /bge/i.test(model);
+    let extractor;
+    const load = () => {
+        extractor ??= (async () => {
+            const transformers = (await import('@huggingface/transformers'));
+            // The default cache sits inside node_modules, which an `npx` run throws away with the package.
+            transformers.env.cacheDir = opts.cacheDir;
+            return transformers.pipeline('feature-extraction', model, { dtype: 'q8' });
+        })();
+        // A failed load (offline first run) must not be cached forever — let the next call retry.
+        extractor.catch(() => (extractor = undefined));
+        return extractor;
+    };
+    return {
+        model,
+        async embed(texts, kind) {
+            const run = await load();
+            const inputs = kind === 'query' && isBge ? texts.map((t) => BGE_QUERY_PREFIX + t) : texts;
+            const output = await run(inputs, { pooling: isBge ? 'cls' : 'mean', normalize: true });
+            const dim = output.dims[output.dims.length - 1];
+            return texts.map((_, i) => output.data.slice(i * dim, (i + 1) * dim));
+        },
+    };
+}
+export function vectorToBlob(vector) {
+    return Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
+}
+export function blobToVector(blob) {
+    // Copy: a Buffer's byteOffset into its pool is not guaranteed to be 4-byte aligned.
+    return new Float32Array(blob.buffer.slice(blob.byteOffset, blob.byteOffset + blob.byteLength));
+}
+const BATCH_SIZE = 16;
+const RETRY_AFTER_FAILURE_MS = 5 * 60_000;
+/**
+ * Embeds every chunk that has no vector for the current model, in the background, a batch at a
+ * time. Indexing never waits on it: upsertFile writes chunks synchronously with embedding NULL and
+ * this catches up, so lexical search works immediately and semantic search fills in behind it.
+ */
+export class EmbeddingIndexer {
+    db;
+    embedder;
+    running = false;
+    rerun = false;
+    modelLoaded = false;
+    lastError;
+    retryTimer;
+    constructor(db, embedder) {
+        this.db = db;
+        this.embedder = embedder;
+    }
+    /** Subscribes to chunk writes and embeds whatever is already pending. */
+    start() {
+        setChunksChangedListener(() => this.kick());
+        this.kick();
+    }
+    stop() {
+        setChunksChangedListener(undefined);
+        clearTimeout(this.retryTimer);
+    }
+    /** True once the model has produced output — i.e. a query can be embedded without a cold load or a download. */
+    isReady() {
+        return this.modelLoaded;
+    }
+    status() {
+        const { total, embedded } = this.db
+            .prepare(`SELECT COUNT(*) AS total, COALESCE(SUM(embedding IS NOT NULL AND embed_model = ?), 0) AS embedded FROM chunks`)
+            .get(this.embedder.model);
+        return { model: this.embedder.model, embedded, total, error: this.lastError };
+    }
+    kick() {
+        if (this.running) {
+            this.rerun = true;
+            return;
+        }
+        this.running = true;
+        this.run()
+            .then(() => {
+            this.lastError = undefined;
+        })
+            .catch((err) => {
+            this.lastError = err.message;
+            console.error(`[memory-bucket] embedding indexer stopped (semantic search unavailable, lexical still works): ${this.lastError}`);
+            clearTimeout(this.retryTimer);
+            this.retryTimer = setTimeout(() => this.kick(), RETRY_AFTER_FAILURE_MS);
+            this.retryTimer.unref();
+        })
+            .finally(() => {
+            this.running = false;
+            if (this.rerun) {
+                this.rerun = false;
+                if (!this.lastError)
+                    this.kick();
+            }
+        });
+    }
+    /** Resolves once nothing is pending — for tests and one-shot scripts; the server itself never awaits this. */
+    async drain() {
+        while (this.running)
+            await new Promise((resolve) => setTimeout(resolve, 5));
+        if (this.lastError)
+            throw new Error(this.lastError);
+    }
+    async run() {
+        const pending = this.db.prepare(`SELECT id, hash, context, heading, text FROM chunks WHERE embedding IS NULL OR embed_model IS NOT ? LIMIT ${BATCH_SIZE}`);
+        // Guarded by hash: a chunk rewritten while its batch was in flight must not receive the old text's vector.
+        const store = this.db.prepare(`UPDATE chunks SET embedding = ?, embed_model = ? WHERE id = ? AND hash = ?`);
+        for (;;) {
+            const batch = pending.all(this.embedder.model);
+            if (batch.length === 0) {
+                // Everything was embedded by an earlier process: load the model anyway, or isReady() stays
+                // false and every query falls back to lexical despite a complete index.
+                if (!this.modelLoaded) {
+                    await this.embedder.embed(['warm up'], 'query');
+                    this.modelLoaded = true;
+                }
+                return;
+            }
+            const vectors = await this.embedder.embed(batch.map(embedTextFor), 'passage');
+            this.modelLoaded = true;
+            this.db.transaction(() => {
+                batch.forEach((row, i) => store.run(vectorToBlob(vectors[i]), this.embedder.model, row.id, row.hash));
+            })();
+            bumpChunksVersion();
+            await new Promise((resolve) => setImmediate(resolve));
+        }
+    }
+}

package/dist/src/store/recall.js

@@ -0,0 +1,228 @@
+import path from 'node:path';
+import { getChunksVersion } from './chunks.js';
+import { blobToVector } from './embeddings.js';
+import { SEARCH_BM25 } from './search.js';
+import { correctWords } from './spelling.js';
+const STOPWORDS = new Set('a an and are as at be but by can do does for from how i if in into is it my not of on or so that the their then there this to was we what when where which who why will with you your'.split(' '));
+/**
+ * A natural-language question as an FTS5 query: its content words OR'd together. OR (not FTS5's
+ * default AND) because a question rarely shares every word with its answer — bm25 already ranks a
+ * chunk matching more of the words above one matching fewer.
+ */
+export function questionToFtsQuery(question, corrections) {
+    const words = [...new Set(question.toLowerCase().match(/[\p{L}\p{N}_]+/gu) ?? [])];
+    const content = words.filter((w) => !STOPWORDS.has(w));
+    const terms = new Set(content.length > 0 ? content : words);
+    // A corrected spelling is added, not swapped in: the original may be a real word this corpus simply lacks.
+    for (const term of [...terms])
+        if (corrections?.has(term))
+            terms.add(corrections.get(term));
+    return terms.size > 0 ? [...terms].map((t) => `"${t}"`).join(' OR ') : null;
+}
+const WORD_RE = /[\p{L}\p{N}_]+/gu;
+/** `query` with each misspelled word swapped for its correction — what gets embedded, since a typo shifts an embedding toward nothing in particular. */
+function respell(query, corrections) {
+    return corrections.size === 0 ? query : query.replace(WORD_RE, (word) => corrections.get(word.toLowerCase()) ?? word);
+}
+const CANDIDATES_PER_LIST = 30;
+const DOC_CANDIDATES = 10;
+/** Reciprocal-rank-fusion constant — the conventional 60; damps how much a rank-1 hit outweighs a rank-5 one. */
+const RRF_K = 60;
+const MAX_CHUNKS_PER_DOC = 3;
+/** Fused scores below this fraction of the best hit's are dropped rather than used to fill the token budget. */
+const RELATIVE_SCORE_FLOOR = 0.3;
+const CHARS_PER_TOKEN = 4;
+const HEADER_TOKENS = 20;
+let vectorCache;
+function loadVectors(db, model) {
+    const version = getChunksVersion();
+    if (!vectorCache || vectorCache.version !== version || vectorCache.model !== model) {
+        const rows = db.prepare(`SELECT id, embedding FROM chunks WHERE embedding IS NOT NULL AND embed_model = ?`).all(model);
+        vectorCache = { version, model, ids: rows.map((r) => r.id), vectors: rows.map((r) => blobToVector(r.embedding)) };
+    }
+    return vectorCache;
+}
+/** Chunks by descending cosine similarity (vectors are unit-length, so a dot product). Brute force — a personal corpus is thousands of chunks, not millions. */
+function vectorSearch(db, model, query, limit) {
+    const { ids, vectors } = loadVectors(db, model);
+    const scored = [];
+    for (let i = 0; i < vectors.length; i++) {
+        const vector = vectors[i];
+        if (vector.length !== query.length)
+            continue;
+        let dot = 0;
+        for (let d = 0; d < vector.length; d++)
+            dot += vector[d] * query[d];
+        scored.push({ id: ids[i], similarity: dot });
+    }
+    return scored.sort((a, b) => b.similarity - a.similarity).slice(0, limit);
+}
+/**
+ * Cosine similarity a chunk must reach to count as a semantic match on its own. Rank fusion in
+ * recall() needs no such cut-off, but a result LIST does: similarity search always returns its
+ * nearest chunks, however unrelated. Tuned for the default bge-small model, where unrelated text
+ * scores 0.50–0.58 and a real match 0.60+; override when using another model.
+ */
+const SEMANTIC_MIN_SIMILARITY = Number(process.env.MEMORY_BUCKET_SEMANTIC_MIN_SIMILARITY) || 0.6;
+/** Beyond the floor, a doc must also be this close to the best match — keeps one strong hit from dragging in the merely-adjacent. */
+const SEMANTIC_WINDOW = 0.05;
+const SEMANTIC_MAX_DOCS = 5;
+/**
+ * Docs whose best chunk is semantically close to `query`, best first — the doc-level counterpart
+ * of recall()'s vector list, for the web UI's result list (which shows docs, not passages).
+ */
+export async function semanticDocMatches(db, embedder, query) {
+    const corrections = correctWords(db, query.toLowerCase().match(WORD_RE) ?? []);
+    const [queryVector] = await embedder.embed([respell(query, corrections)], 'query');
+    const hits = vectorSearch(db, embedder.model, queryVector, 50);
+    const floor = Math.max(SEMANTIC_MIN_SIMILARITY, (hits[0]?.similarity ?? 0) - SEMANTIC_WINDOW);
+    const chunkDoc = db.prepare(`SELECT ref_table, ref_id, ref_folder FROM chunks WHERE id = ?`);
+    const docs = new Map();
+    for (const hit of hits) {
+        if (hit.similarity < floor || docs.size >= SEMANTIC_MAX_DOCS)
+            break;
+        const doc = chunkDoc.get(hit.id);
+        if (doc && !docs.has(docKey(doc)))
+            docs.set(docKey(doc), { ...doc, similarity: hit.similarity });
+    }
+    return [...docs.values()];
+}
+const docKey = (c) => `${c.ref_table}\0${c.ref_folder}\0${c.ref_id}`;
+/**
+ * Retrieves the passages most relevant to a natural-language query, packed into a token budget.
+ * Three ranked lists are fused by reciprocal rank: chunk-level BM25, chunk-level embedding
+ * similarity (when an embedder is ready), and doc-level weighted BM25 — the last so a query that
+ * names a skill, key, tag or trigger phrase lifts that doc's chunks even when the body never
+ * repeats the word. Rank fusion needs no score calibration between the three.
+ */
+export async function recall(db, query, opts = {}) {
+    const { maxTokens = 2000, table, hidden, embedder } = opts;
+    const fused = new Map();
+    const add = (chunkId, rank) => fused.set(chunkId, (fused.get(chunkId) ?? 0) + 1 / (RRF_K + rank));
+    const corrections = correctWords(db, query.toLowerCase().match(WORD_RE) ?? []);
+    const ftsQuery = questionToFtsQuery(query, corrections);
+    const tableFilter = table ? `AND c.ref_table = '${table}'` : '';
+    let docRanks = [];
+    if (ftsQuery) {
+        const lexical = db
+            .prepare(`SELECT c.id FROM chunk_fts JOIN chunks c ON c.id = chunk_fts.rowid
+         WHERE chunk_fts MATCH ? ${tableFilter}
+         ORDER BY bm25(chunk_fts, 3.0, 1.0) LIMIT ${CANDIDATES_PER_LIST}`)
+            .all(ftsQuery);
+        lexical.forEach((row, i) => add(row.id, i + 1));
+        docRanks = db
+            .prepare(`SELECT ref_table, ref_id, ref_folder FROM search_index
+         WHERE search_index MATCH ? ${table ? `AND ref_table = '${table}'` : ''}
+         ORDER BY ${SEARCH_BM25} LIMIT ${DOC_CANDIDATES}`)
+            .all(ftsQuery);
+    }
+    let mode = 'lexical';
+    if (embedder) {
+        const [queryVector] = await embedder.embed([respell(query, corrections)], 'query');
+        const semantic = vectorSearch(db, embedder.model, queryVector, CANDIDATES_PER_LIST * (table ? 4 : 1));
+        if (semantic.length > 0)
+            mode = 'hybrid';
+        semantic.forEach((hit, i) => add(hit.id, i + 1));
+    }
+    // Doc-level boost: every candidate chunk of a doc-level hit gains that doc's rank; a doc with no
+    // candidate chunk yet (the query matched only its name/key/tags) contributes its opening chunk.
+    const firstChunkOf = db.prepare(`SELECT id FROM chunks WHERE ref_table = ? AND ref_id = ? AND ref_folder = ? ORDER BY ord LIMIT 1`);
+    const chunkById = db.prepare(`SELECT id, ref_table, ref_id, ref_folder, ord, heading, text FROM chunks WHERE id = ?`);
+    const rows = new Map();
+    for (const id of fused.keys()) {
+        const row = chunkById.get(id);
+        if (row && (!table || row.ref_table === table))
+            rows.set(id, row);
+        else
+            fused.delete(id);
+    }
+    docRanks.forEach((doc, i) => {
+        const key = docKey(doc);
+        const members = [...rows.values()].filter((row) => docKey(row) === key);
+        if (members.length === 0) {
+            const first = firstChunkOf.get(doc.ref_table, doc.ref_id, doc.ref_folder);
+            const row = first && chunkById.get(first.id);
+            if (row) {
+                rows.set(row.id, row);
+                members.push(row);
+            }
+        }
+        for (const row of members)
+            add(row.id, i + 1);
+    });
+    // Resolve each candidate's parent doc once — for its public handle, and to drop paused/hidden docs.
+    const skillMeta = db.prepare(`SELECT folder, paused FROM skills WHERE id = ? AND folder = ?`);
+    const memoryMeta = db.prepare(`SELECT folder, key, paused FROM memory_docs WHERE source_path = ?`);
+    const docs = new Map();
+    const resolveDoc = (row) => {
+        const key = docKey(row);
+        if (!docs.has(key)) {
+            const meta = (row.ref_table === 'skills' ? skillMeta.get(row.ref_id, row.ref_folder) : memoryMeta.get(row.ref_id));
+            const visible = meta && !meta.paused && !(hidden?.[row.ref_table] ?? []).includes(meta.folder);
+            docs.set(key, visible ? { id: row.ref_table === 'skills' ? row.ref_id : path.basename(row.ref_id), folder: meta.folder, key: meta.key } : null);
+        }
+        return docs.get(key);
+    };
+    const ranked = [...rows.values()]
+        .map((row) => ({ row, score: fused.get(row.id) ?? 0, doc: resolveDoc(row) }))
+        .filter((c) => c.doc !== null)
+        .sort((a, b) => b.score - a.score);
+    const floor = (ranked[0]?.score ?? 0) * RELATIVE_SCORE_FLOOR;
+    const perDoc = new Map();
+    const chunks = [];
+    let budget = maxTokens;
+    let truncated = false;
+    for (const { row, score, doc } of ranked) {
+        if (score < floor)
+            break;
+        const key = docKey(row);
+        if ((perDoc.get(key) ?? 0) >= MAX_CHUNKS_PER_DOC)
+            continue;
+        let text = row.text;
+        const cost = () => HEADER_TOKENS + Math.ceil((row.heading.length + text.length) / CHARS_PER_TOKEN);
+        if (cost() > budget) {
+            truncated = true;
+            // Always return something for the best hit, even under a budget smaller than one chunk.
+            if (chunks.length > 0)
+                continue;
+            text = text.slice(0, Math.max(0, (budget - HEADER_TOKENS) * CHARS_PER_TOKEN - row.heading.length)) + '…';
+        }
+        budget -= cost();
+        perDoc.set(key, (perDoc.get(key) ?? 0) + 1);
+        chunks.push({ ref_table: row.ref_table, id: doc.id, folder: doc.folder, key: doc.key, heading: row.heading, ord: row.ord, text, score });
+    }
+    return { mode, chunks, truncated, corrections: Object.fromEntries(corrections) };
+}
+/**
+ * Renders a recall result as compact markdown rather than JSON — passages are mostly multi-line
+ * text, and JSON-escaping every newline and quote costs tokens for nothing. Chunks of the same doc
+ * are grouped (docs in best-hit order) and shown in document order.
+ */
+export function formatRecall(result, note) {
+    if (result.chunks.length === 0)
+        return `No matches.${note ? ` ${note}` : ''}`;
+    const groups = new Map();
+    for (const chunk of result.chunks) {
+        const key = `${chunk.ref_table}\0${chunk.folder}\0${chunk.id}`;
+        groups.set(key, [...(groups.get(key) ?? []), chunk]);
+    }
+    const corrected = Object.entries(result.corrections).map(([from, to]) => `"${from}" read as "${to}"`).join(', ');
+    const lines = [
+        `${result.mode} search, ${result.chunks.length} passages from ${groups.size} docs${result.truncated ? ' (more matched — raise max_tokens)' : ''}${corrected ? `; ${corrected}` : ''}${note ? `. ${note}` : ''}`,
+    ];
+    for (const group of groups.values()) {
+        const first = group[0];
+        const label = first.ref_table === 'skills' ? `skill ${first.id}` : `memory ${first.id}`;
+        const details = [first.folder && `folder: ${first.folder}`, first.key && `key: ${first.key}`].filter(Boolean).join(', ');
+        lines.push('', `## ${label}${details ? ` (${details})` : ''}`);
+        let previousHeading = '';
+        for (const chunk of group.sort((a, b) => a.ord - b.ord)) {
+            // Consecutive pieces of one oversized section share a heading — print it once.
+            if (chunk.heading && chunk.heading !== previousHeading)
+                lines.push(`### ${chunk.heading}`);
+            previousHeading = chunk.heading;
+            lines.push(chunk.text, '');
+        }
+    }
+    return lines.join('\n').trimEnd();
+}

package/dist/src/store/search.js

@@ -1,3 +1,4 @@
+import { correctWords } from './spelling.js';
 /** A quoted FTS5 syntax error, translated into something an agent can act on without seeing raw SQLite internals. */
 export class SearchQueryError extends Error {
     constructor(query, cause) {
@@ -28,27 +29,66 @@ export function sanitizeFtsQuery(query) {
     })
         .join(' ');
 }
+/**
+ * bm25() over search_index with per-column weights, in column order (ref_table, ref_id, ref_folder,
+ * description, body, tags, key, filename, triggers) — a hit in a doc's name/key/trigger phrases
+ * says far more about what the doc IS than the same word appearing somewhere in a long body.
+ */
+export const SEARCH_BM25 = 'bm25(search_index, 0, 0, 0, 4.0, 1.0, 3.0, 6.0, 8.0, 6.0)';
+/** Best-matching column's excerpt (-1), not a fixed column — a hit in tags/key alone still shows why it matched. */
+export const SEARCH_SNIPPET = `snippet(search_index, -1, '<<', '>>', '…', 20)`;
+/** True when the caller used FTS5 syntax (quotes, AND/OR/NOT, prefix*, parentheses, column filters) and so meant the query literally. */
+export function hasExplicitSyntax(query) {
+    const tokens = query.match(/\S+/g) ?? [];
+    return tokens.some((t) => FTS_BOOLEAN_KEYWORDS.has(t.toUpperCase()) || /["*():^]/.test(t));
+}
+/**
+ * Runs `run` with the sanitized query and, only when that matches nothing and the query is plain
+ * bare words, loosens it in two steps: first the words OR'd instead of FTS5's implicit AND (bm25
+ * ranks docs matching more of them first), then with misspelled words replaced by the closest word
+ * in the corpus (see spelling.ts). A query using explicit syntax is never loosened.
+ */
+export function withOrFallback(db, query, offset, run) {
+    const strict = sanitizeFtsQuery(query);
+    const rows = run(strict, false);
+    if (rows.length > 0 || hasExplicitSyntax(query))
+        return rows;
+    // An empty page past the first is not "no AND matches" — don't start serving looser rows mid-pagination.
+    if (offset > 0 && run(strict, true).length > 0)
+        return rows;
+    const tokens = query.match(/\S+/g) ?? [];
+    if (tokens.length > 1) {
+        const relaxed = run(tokens.map((t) => sanitizeFtsQuery(t)).join(' OR '), false);
+        if (relaxed.length > 0)
+            return relaxed;
+    }
+    const corrections = correctWords(db, tokens);
+    if (corrections.size === 0)
+        return rows;
+    return run(tokens.map((t) => sanitizeFtsQuery(corrections.get(t) ?? t)).join(' OR '), false);
+}
 /**
  * Full-text search over the shared FTS5 index, optionally scoped to one ref_table.
  * `query` is passed through as raw FTS5 MATCH syntax — supports `AND`/`OR`/`NOT`,
  * `"exact phrases"`, and `prefix*` — so callers get grep-like power for free.
- * Ranked by bm25() (lower is better, so we negate for a "higher is better" score).
+ * Ranked by weighted bm25() (lower is better, so we negate for a "higher is better" score); bare
+ * words that match nothing together fall back to OR, then to typo correction (see withOrFallback).
  * Throws SearchQueryError on malformed FTS5 syntax instead of a raw SQLite error.
  */
 export function searchIndex(db, query, opts = {}) {
     const { table, limit = 20, offset = 0 } = opts;
-    const sanitized = sanitizeFtsQuery(query);
     try {
-        const rows = db
-            .prepare(`SELECT ref_table, ref_id,
-                snippet(search_index, 3, '<<', '>>', '…', 20) AS snippet,
-                -bm25(search_index) AS score
-         FROM search_index
-         WHERE search_index MATCH ? ${table ? 'AND ref_table = ?' : ''}
-         ORDER BY bm25(search_index)
-         LIMIT ? OFFSET ?`)
-            .all(...(table ? [sanitized, table, limit, offset] : [sanitized, limit, offset]));
-        return rows;
+        const statement = db.prepare(`SELECT ref_table, ref_id,
+              ${SEARCH_SNIPPET} AS snippet,
+              -${SEARCH_BM25} AS score
+       FROM search_index
+       WHERE search_index MATCH ? ${table ? 'AND ref_table = ?' : ''}
+       ORDER BY ${SEARCH_BM25}
+       LIMIT ? OFFSET ?`);
+        return withOrFallback(db, query, offset, (ftsQuery, probe) => {
+            const page = probe ? [1, 0] : [limit, offset];
+            return statement.all(...(table ? [ftsQuery, table, ...page] : [ftsQuery, ...page]));
+        });
     }
     catch (err) {
         throw new SearchQueryError(query, err);
@@ -123,24 +163,22 @@ export function searchByDate(db, start, end, opts = {}) {
  * per table); use skill_search/memory_search directly when filtering by those.
  */
 export function searchCombined(db, query, limit = 20, offset = 0) {
-    const sanitized = sanitizeFtsQuery(query);
     try {
-        const rows = db
-            .prepare(`SELECT ref_table, ref_id AS id,
-                COALESCE(s.description, m.description) AS description,
-                COALESCE(s.folder, m.folder) AS folder,
-                snippet(search_index, 3, '<<', '>>', '…', 20) AS snippet,
-                -bm25(search_index) AS score,
-                si.owner AS shared_owner,
-                si.role AS shared_role
-         FROM search_index
-         LEFT JOIN skills s ON search_index.ref_table = 'skills' AND s.id = search_index.ref_id AND s.folder = search_index.ref_folder
-         LEFT JOIN memory_docs m ON search_index.ref_table = 'memory_docs' AND m.source_path = search_index.ref_id
-         LEFT JOIN shared_items si ON si.status = 'active' AND si.mirror_path = COALESCE(s.source_path, m.source_path)
-         WHERE search_index MATCH ?
-         ORDER BY bm25(search_index)
-         LIMIT ? OFFSET ?`)
-            .all(sanitized, limit, offset);
+        const statement = db.prepare(`SELECT ref_table, ref_id AS id,
+              COALESCE(s.description, m.description) AS description,
+              COALESCE(s.folder, m.folder) AS folder,
+              ${SEARCH_SNIPPET} AS snippet,
+              -${SEARCH_BM25} AS score,
+              si.owner AS shared_owner,
+              si.role AS shared_role
+       FROM search_index
+       LEFT JOIN skills s ON search_index.ref_table = 'skills' AND s.id = search_index.ref_id AND s.folder = search_index.ref_folder
+       LEFT JOIN memory_docs m ON search_index.ref_table = 'memory_docs' AND m.source_path = search_index.ref_id
+       LEFT JOIN shared_items si ON si.status = 'active' AND si.mirror_path = COALESCE(s.source_path, m.source_path)
+       WHERE search_index MATCH ?
+       ORDER BY ${SEARCH_BM25}
+       LIMIT ? OFFSET ?`);
+        const rows = withOrFallback(db, query, offset, (ftsQuery, probe) => statement.all(ftsQuery, ...(probe ? [1, 0] : [limit, offset])));
         return rows.map((row) => ({
             ...row,
             shared_owner: row.shared_owner ?? undefined,

package/dist/src/store/spelling.js

@@ -0,0 +1,54 @@
+import { distance } from 'fastest-levenshtein';
+import { getChunksVersion } from './chunks.js';
+let vocabulary;
+/** Every distinct (unstemmed) word in the corpus with how many chunks contain it — from term_vocab, reloaded when chunks change. */
+function loadVocabulary(db) {
+    const version = getChunksVersion();
+    if (!vocabulary || vocabulary.version !== version || vocabulary.db !== db) {
+        const terms = db.prepare(`SELECT term, doc AS docs FROM term_vocab`).all();
+        vocabulary = { version, db, terms, known: new Set(terms.map((t) => t.term)) };
+    }
+    return vocabulary;
+}
+/**
+ * The corpus word a misspelled query word most likely meant, or undefined when the word is fine
+ * (it, or a stemmed form of it, occurs somewhere) or nothing is close enough. Short words and
+ * anything with a digit are left alone — an edit away from a 3-letter word or a ticket number is
+ * usually a different word, not a typo. Ties go to the word found in more chunks.
+ */
+export function suggestCorrection(db, word) {
+    const lower = word.toLowerCase();
+    if (lower.length < 4 || !/^\p{L}+$/u.test(lower))
+        return undefined;
+    const { terms, known } = loadVocabulary(db);
+    if (known.has(lower))
+        return undefined;
+    // Not in the unstemmed vocabulary, but the porter-stemmed indexes may still match it ("deploying" → "deploy").
+    const phrase = `"${lower}"`;
+    if (db.prepare(`SELECT 1 FROM chunk_fts WHERE chunk_fts MATCH ? LIMIT 1`).get(phrase))
+        return undefined;
+    if (db.prepare(`SELECT 1 FROM search_index WHERE search_index MATCH ? LIMIT 1`).get(phrase))
+        return undefined;
+    const maxDistance = lower.length <= 5 ? 1 : 2;
+    let best;
+    for (const { term, docs } of terms) {
+        if (Math.abs(term.length - lower.length) > maxDistance)
+            continue;
+        const d = distance(lower, term);
+        if (d > maxDistance)
+            continue;
+        if (!best || d < best.distance || (d === best.distance && docs > best.docs))
+            best = { term, docs, distance: d };
+    }
+    return best?.term;
+}
+/** Misspelled word → correction, for each of `words` that has one. */
+export function correctWords(db, words) {
+    const corrections = new Map();
+    for (const word of words) {
+        const correction = suggestCorrection(db, word);
+        if (correction)
+            corrections.set(word, correction);
+    }
+    return corrections;
+}

package/dist/src/store/sync.js

@@ -3,6 +3,7 @@ import path from 'node:path';
 import chokidar, {} from 'chokidar';
 import { readMarkdownFile } from './markdown-file.js';
 import { flattenTags } from './db.js';
+import { writeChunks, deleteChunks, chunkContext } from './chunks.js';
 import { extractDates, toLocalDate } from './date-extract.js';
 import { slugify } from './slug.js';
 import { normalizeKey } from '../types.js';
@@ -228,7 +229,11 @@ folderOverride) {
     // construction), so '' (unscoped) is safe there.
     const refFolder = scopedById ? folder : '';
     db.prepare(`DELETE FROM search_index WHERE ref_table = ? AND ref_id = ? AND ref_folder = ?`).run(spec.table, id, refFolder);
-    db.prepare(`INSERT INTO search_index (ref_table, ref_id, ref_folder, description, body, tags, key, filename) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(spec.table, id, refFolder, String(row.description ?? ''), parsed.body, flattenTags(String(row.tags ?? '[]')), String(row.key ?? row.skill_group ?? ''), path.basename(filePath));
+    db.prepare(`INSERT INTO search_index (ref_table, ref_id, ref_folder, description, body, tags, key, filename, triggers) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(spec.table, id, refFolder, String(row.description ?? ''), parsed.body, flattenTags(String(row.tags ?? '[]')), String(row.key ?? row.skill_group ?? ''), 
+    // A skill's file is always SKILL.md, so its name goes here instead — otherwise the name is
+    // only searchable when the description or body happens to repeat it.
+    scopedById ? id : path.basename(filePath), flattenTags(String(row.trigger_phrases ?? '[]')));
+    writeChunks(db, spec.table, id, refFolder, chunkContext(spec.table, row), String(row.description ?? ''), parsed.body);
     db.prepare(`DELETE FROM doc_dates WHERE ref_table = ? AND ref_id = ? AND ref_folder = ?`).run(spec.table, id, refFolder);
     const dates = new Set(extractDates(parsed.body));
     // created_at is a UTC instant; convert to the OS-local calendar date so it
@@ -255,6 +260,7 @@ export function removeFile(db, table, filePath) {
         const refFolder = table === 'skills' ? existing.folder : '';
         db.prepare(`DELETE FROM search_index WHERE ref_table = ? AND ref_id = ? AND ref_folder = ?`).run(table, existing.id, refFolder);
         db.prepare(`DELETE FROM doc_dates WHERE ref_table = ? AND ref_id = ? AND ref_folder = ?`).run(table, existing.id, refFolder);
+        deleteChunks(db, table, existing.id, refFolder);
     }
 }
 /** Full scan of all configured source dirs — used once at startup before the watcher takes over. */
@@ -296,6 +302,7 @@ export function unregisterFolder(db, table, folderName) {
     db.prepare(`DELETE FROM ${table} WHERE folder = ?`).run(folderName);
     for (const row of rows) {
         db.prepare(`DELETE FROM search_index WHERE ref_table = ? AND ref_id = ?`).run(table, row.id);
+        deleteChunks(db, table, row.id, table === 'skills' ? folderName : '');
     }
 }
 export function* walkMarkdownFiles(dir) {