mcp-memory-bucket 0.13.0 → 0.14.0

package/README.md

@@ -45,6 +45,53 @@ the outdated ones deprecated" becomes `skill_search("deploy")` to find
 candidates, then `skill_bulk_update(names, { deprecated: true })` to flag
 the stale ones in one call — no need to touch each file individually.
 
+## Retrieval
+
+`bucket_recall` is the retrieval entry point: it takes a plain-language
+question and returns the most relevant *passages* — heading-level sections,
+with their text — packed into a `max_tokens` budget, so an agent gets the
+one section it needs instead of searching, then loading a whole document.
+No LLM is involved anywhere in indexing or retrieval:
+
+- **Chunking.** Each doc body is split along its markdown headings (code
+  fences kept intact, oversized sections cut at paragraph boundaries) and
+  every chunk remembers its heading path.
+- **Keyword ranking.** SQLite FTS5 with BM25, at two levels: per chunk, and
+  per doc with column weights so a hit in a skill's name, a memory doc's
+  key or filename, tags, or trigger phrases outranks the same word deep in
+  a body.
+- **Semantic ranking.** A small sentence-embedding model
+  (`Xenova/bge-small-en-v1.5`, ~33 MB) runs locally on CPU via
+  transformers.js and embeds each chunk in the background, so a paraphrase
+  with no shared keyword ("step back after a mistake" → an undo/redo skill)
+  still matches. Only chunks whose text changed are re-embedded.
+- **Fusion.** The three ranked lists are merged by reciprocal rank fusion.
+- **Typo correction.** A query word found nowhere in the corpus is matched
+  to the closest word that is (edit distance 1–2, by word length), so
+  "tooltp" finds the tooltip skill. `bucket_recall` reports the correction
+  in its first line.
+
+The model downloads once, on first start, into
+`<base dir>/.memory-bucket-models`; until it has finished (or if the machine
+is offline, or the optional `@huggingface/transformers` dependency failed to
+install) `bucket_recall` answers from keyword ranking alone and says so in
+its first line. Environment variables:
+
+| Variable | Effect |
+| --- | --- |
+| `MEMORY_BUCKET_EMBEDDINGS=off` | Never load a model; keyword ranking only. |
+| `MEMORY_BUCKET_EMBED_MODEL` | A different transformers.js feature-extraction model, e.g. `Xenova/multilingual-e5-small` for non-English content. Changing it re-embeds every chunk. |
+| `MEMORY_BUCKET_SEMANTIC_MIN_SIMILARITY` | Cosine similarity a doc needs to appear in the web UI's results on meaning alone (default `0.6`, tuned for the default model — other models score on a different scale). |
+
+`skill_search` / `memory_search` / `bucket_search` remain the keyword tools
+that return a list of *docs* (and take metadata filters). Bare words must all
+match; if no doc contains all of them, the query is retried matching any,
+then with typo correction. Explicit syntax (quotes, `AND`/`OR`/`NOT`,
+`prefix*`) is always taken literally.
+
+The web UI's search box uses the same keyword search and, for a plain-words
+query, also lists docs that match by meaning alone.
+
 ## Run
 
 ### Via npx

package/dist/src/memory/repository.js

@@ -5,7 +5,7 @@ import { writeMarkdownFile, formatMarkdownFile } from '../store/markdown-file.js
 import { resolveWithinBase } from '../store/safe-path.js';
 import { uniqueFilename } from '../attachments/storage.js';
 import { upsertFile, removeFile, scanSingleFolder, unregisterFolder, memorySyncSpec } from '../store/sync.js';
-import { SearchQueryError, sanitizeFtsQuery } from '../store/search.js';
+import { SearchQueryError, SEARCH_BM25, SEARCH_SNIPPET, withOrFallback } from '../store/search.js';
 import { applyBodyEdits } from '../shared/body-edits.js';
 import { attachmentsDirFor } from '../attachments/storage.js';
 import { rebaseFolderPath } from '../config.js';
@@ -401,7 +401,7 @@ export class MemoryRepository {
     search(query, opts = {}) {
         const { docType, status, folder, tag, limit = 20, offset = 0, includePaused = false } = opts;
         const conditions = [];
-        const params = [sanitizeFtsQuery(query)];
+        const params = [];
         if (docType) {
             conditions.push('m.doc_type = ?');
             params.push(docType);
@@ -427,23 +427,21 @@ export class MemoryRepository {
             conditions.push(`m.folder NOT IN (${hidden.map(() => '?').join(', ')})`);
             params.push(...hidden);
         }
-        params.push(limit, offset);
         try {
             // shared_items is LEFT JOINed on m.source_path (== mirror_path for an item-level share —
             // see remote/shared-items.ts) to surface owner/role on any hit that came from a share.
-            const rows = this.db
-                .prepare(`SELECT m.source_path, m.key, m.description, m.doc_type, m.folder,
-                  snippet(search_index, 3, '<<', '>>', '…', 20) AS snippet,
-                  -bm25(search_index) AS score,
+            const statement = this.db.prepare(`SELECT m.source_path, m.key, m.description, m.doc_type, m.folder,
+                  ${SEARCH_SNIPPET} AS snippet,
+                  -${SEARCH_BM25} AS score,
                   si.owner AS shared_owner,
                   si.role AS shared_role
            FROM search_index
            JOIN memory_docs m ON m.source_path = search_index.ref_id
            LEFT JOIN shared_items si ON si.status = 'active' AND si.mirror_path = m.source_path
            WHERE search_index.ref_table = 'memory_docs' AND search_index MATCH ? ${conditions.map((c) => `AND ${c}`).join(' ')}
-           ORDER BY bm25(search_index)
-           LIMIT ? OFFSET ?`)
-                .all(...params);
+           ORDER BY ${SEARCH_BM25}
+           LIMIT ? OFFSET ?`);
+            const rows = withOrFallback(this.db, query, offset, (ftsQuery, probe) => statement.all(ftsQuery, ...params, ...(probe ? [1, 0] : [limit, offset])));
             return rows.map(({ source_path, shared_owner, shared_role, ...rest }) => ({
                 ...rest,
                 filename: path.basename(source_path),

package/dist/src/memory/tools.js

@@ -33,7 +33,7 @@ export function registerMemoryTools(mcp, repo) {
         const keys = repo.listKeys(key_prefix);
         return { content: [{ type: 'text', text: JSON.stringify(keys, null, 2) }] };
     });
-    mcp.tool('memory_search', 'Full-text search over memory doc description/body/tags (grep/find-like, ranked by relevance) — unlike memory_get\'s exact-key lookup, this searches by content across all keys. `query` is raw SQLite FTS5 MATCH syntax: bare words, "exact phrases", prefix* wildcards, AND/OR/NOT boolean operators; hyphenated/punctuated terms must be quoted, e.g. "blue-green". Can be combined with doc_type/status/tag filters. Returns ranked hits with a highlighted snippet, not the full body — call memory_get(key) or fetch by id for that. Paused docs are hidden by default — pass include_paused to see them.', {
+    mcp.tool('memory_search', 'Full-text search over memory doc description/body/tags (grep/find-like, ranked by relevance) — unlike memory_get\'s exact-key lookup, this searches by content across all keys. `query` is SQLite FTS5 MATCH syntax: bare words (all must match; falls back to any-word when nothing matches all), "exact phrases", prefix* wildcards, AND/OR/NOT. Can be combined with doc_type/status/tag filters. Returns ranked hits with a highlighted snippet, not the full body — call memory_get(key) or fetch by id for that. Paused docs are hidden by default — pass include_paused to see them.', {
         query: z.string().describe('FTS5 match expression, e.g. `migration AND rollback` or `"blue green"`'),
         doc_type: z.enum(MEMORY_DOC_TYPES).optional(),
         status: statusSchema(MEMORY_STATUS_DEFAULTS).optional(),

package/dist/src/server.js

@@ -14,6 +14,7 @@ import { registerSkillTools } from './skills/tools.js';
 import { registerMemoryTools } from './memory/tools.js';
 import { registerRelocateTool } from './shared/relocate-tool.js';
 import { registerSearchTool } from './shared/search-tool.js';
+import { EmbeddingIndexer, createTransformersEmbedder } from './store/embeddings.js';
 import { registerBucketFolderTools } from './shared/bucket-folder-tool.js';
 import { registerBucketShareTools } from './shared/bucket-share-tool.js';
 import { syncSharedItemsIndex } from './remote/shared-items.js';
@@ -113,6 +114,14 @@ const memoryWatcher = watchSources(db, memorySpec);
 skillRepo.setWatcher(skillWatcher);
 memoryRepo.setWatcher(memoryWatcher);
 const attachmentRepo = new AttachmentRepository(memoryRepo, skillRepo, db);
+// Semantic half of bucket_recall: a local embedding model (no LLM, no network after its one-time
+// download into baseDir) fills in chunk vectors in the background. Started after the scans above so
+// its first pass sees every chunk; anything indexed later wakes it through the chunk-write listener.
+// MEMORY_BUCKET_EMBEDDINGS=off keeps retrieval purely lexical.
+const embeddingIndexer = process.env.MEMORY_BUCKET_EMBEDDINGS === 'off'
+    ? undefined
+    : new EmbeddingIndexer(db, createTransformersEmbedder({ cacheDir: path.join(config.baseDir, '.memory-bucket-models'), model: process.env.MEMORY_BUCKET_EMBED_MODEL }));
+embeddingIndexer?.start();
 // Remote sources have no filesystem to watch (chokidar doesn't apply) - a
 // fixed-interval poller stands in for watchSources above, pulling changes
 // into each remote source's local mirror directory and reusing the exact
@@ -174,13 +183,16 @@ if (config.skillFolders.length === 0 && config.memoryFolders.length === 0) {
 // this server — surfaced both in serverInfo.description and instructions so
 // clients that expose either to the model can make that association.
 const SERVER_DESCRIPTION = 'Also known as "memory bucket", "mem bucket", or "skill bucket" — if the user refers to this server by any of those names, they mean this one.';
-const SERVER_INSTRUCTIONS = `${SERVER_DESCRIPTION} Exposes skill_* (reusable coding patterns, stored as agentskills.io-standard SKILL.md folders) and memory_* (point-in-time working context — plans, specs, SQL, session summaries — looked up by key) tools, plus shared relocate/bucket_search/bucket_*_folder tools. Use skill_search/memory_search/bucket_search for full-text search over body content (not just metadata) — bucket_search when you don't know which bucket something landed in. Use bucket_list_folders to see what named source directories (folders) are configured before passing a folder argument elsewhere, and bucket_create_folder/bucket_delete_folder to register or unregister one (bucket_delete_folder works for both local and remote folders). bucket_list_remote_folders/bucket_connect_remote_folder connect a folderfoo folder as a new source, once the user is logged in via the web UI. Most operations have a _bulk_ variant (bulk_get/bulk_create/bulk_update/bulk_delete/bulk_rename, relocate_bulk) that take a list and return per-item success/failure — prefer these over looping single calls when acting on more than one item. A memory doc's key can be changed in place via memory_update(id, key: ...) — no separate rename tool needed. Before calling any *_create/*_update/relocate tool, call skill_get("memory-bucket-authoring") first to learn the exact frontmatter schema — don't guess the shape. Also exposes memory_channel_read/memory_channel_post/list_memory_channels — a SEPARATE, ephemeral in-memory layer for live cross-agent coordination (a shared scratchpad/discussion channel by name), never written to disk and never indexed by memory_search/bucket_search; do not confuse these with the persisted memory_* docs above.`;
+const SERVER_INSTRUCTIONS = `${SERVER_DESCRIPTION} Exposes skill_* (reusable coding patterns, stored as agentskills.io-standard SKILL.md folders) and memory_* (point-in-time working context — plans, specs, SQL, session summaries — looked up by key) tools, plus shared relocate/bucket_search/bucket_*_folder tools. Use bucket_recall to retrieve relevant passages for a natural-language question (hybrid keyword + semantic, token-budgeted) — the default way to look something up. Use skill_search/memory_search/bucket_search for keyword search returning a list of docs — the first two when filtering by metadata, bucket_search when you don't know which bucket something landed in. Use bucket_list_folders to see what named source directories (folders) are configured before passing a folder argument elsewhere, and bucket_create_folder/bucket_delete_folder to register or unregister one (bucket_delete_folder works for both local and remote folders). bucket_list_remote_folders/bucket_connect_remote_folder connect a folderfoo folder as a new source, once the user is logged in via the web UI. Most operations have a _bulk_ variant (bulk_get/bulk_create/bulk_update/bulk_delete/bulk_rename, relocate_bulk) that take a list and return per-item success/failure — prefer these over looping single calls when acting on more than one item. A memory doc's key can be changed in place via memory_update(id, key: ...) — no separate rename tool needed. Before calling any *_create/*_update/relocate tool, call skill_get("memory-bucket-authoring") first to learn the exact frontmatter schema — don't guess the shape. Also exposes memory_channel_read/memory_channel_post/list_memory_channels — a SEPARATE, ephemeral in-memory layer for live cross-agent coordination (a shared scratchpad/discussion channel by name), never written to disk and never indexed by memory_search/bucket_search; do not confuse these with the persisted memory_* docs above.`;
 function buildMcpServer() {
     const server = new McpServer({ name: 'memory-bucket', version: '0.1.0', description: SERVER_DESCRIPTION }, { capabilities: {}, instructions: SERVER_INSTRUCTIONS });
     registerSkillTools(server, skillRepo);
     registerMemoryTools(server, memoryRepo);
     registerRelocateTool(server, skillRepo, memoryRepo);
-    registerSearchTool(server, db);
+    registerSearchTool(server, db, {
+        hiddenFolders: () => ({ skills: skillRepo.hiddenFolderNames(), memory_docs: memoryRepo.hiddenFolderNames() }),
+        indexer: embeddingIndexer,
+    });
     registerBucketFolderTools(server, config, skillRepo, memoryRepo, db, skillSpec, memorySpec, identity);
     registerBucketShareTools(server, config, skillRepo, memoryRepo, db, identity);
     registerAttachmentTools(server, attachmentRepo);
@@ -191,7 +203,7 @@ function buildMcpServer() {
 startChannelSweep((name) => console.error(`[memory-bucket] sweeping idle memory channel: ${name}`));
 const app = express();
 app.use(express.json());
-app.use(buildWebRouter(db, config, skillRepo, memoryRepo, skillSpec, memorySpec, identity, { skill: remoteSkillPoller, memory: remoteMemoryPoller }, attachmentRepo));
+app.use(buildWebRouter(db, config, skillRepo, memoryRepo, skillSpec, memorySpec, identity, { skill: remoteSkillPoller, memory: remoteMemoryPoller }, attachmentRepo, embeddingIndexer));
 app.use(express.static(path.join(packageRoot, 'dist', 'client')));
 app.post('/mcp', async (req, res) => {
     const server = buildMcpServer();
@@ -219,6 +231,7 @@ app.delete('/mcp', methodNotAllowed);
 app.listen(PORT, () => {
     console.error(`[memory-bucket] MCP server listening on http://localhost:${PORT}/mcp`);
     console.error(`[memory-bucket] UI available at http://localhost:${PORT}`);
+    console.error(`[memory-bucket] semantic search: ${embeddingIndexer ? embeddingIndexer.embedder.model : 'off (MEMORY_BUCKET_EMBEDDINGS=off)'}`);
     console.error(`[memory-bucket] folderfoo integration: ${config.folderfooMode}${config.folderfooHost ? ` (${config.folderfooHost})` : ''}` +
         (config.folderfooMode === 'off' ? ' — pass --folderfoo-mode dev|cloud (or set FOLDERFOO_MODE) to enable' : ''));
     console.error(`[memory-bucket] skill folders: ${config.skillFolders.map((f) => `${f.name}=${f.path}`).join(', ') || '(none)'}`);

package/dist/src/shared/bucket-folder-tool.js

@@ -2,6 +2,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { z } from 'zod';
 import { saveFolder, saveRemoteFolder, removeFolder as removeFolderFromConfig, sanitizeFolderName, mirrorDirFor, } from '../config.js';
+import { pruneOrphanChunks } from '../store/chunks.js';
 import { initialScan } from '../store/sync.js';
 import { listFolders as listFolderfooFolders } from '../remote/folderfoo-client.js';
 import { pollOne } from '../remote/remote-sync.js';
@@ -135,6 +136,7 @@ export function registerBucketFolderTools(mcp, config, skillRepo, memoryRepo, db
             initialScan(db, skillSpec);
             initialScan(db, memorySpec);
             syncSharedItemsIndex(db, skillSpec, memorySpec, identity.current().username);
+            pruneOrphanChunks(db); // chunks survive the wipe so unchanged embeddings carry over — see pruneOrphanChunks
             const skillCount = db.prepare(`SELECT COUNT(*) AS n FROM skills`).get().n;
             const memoryCount = db.prepare(`SELECT COUNT(*) AS n FROM memory_docs`).get().n;
             return {

package/dist/src/shared/search-tool.js

@@ -1,7 +1,34 @@
 import { z } from 'zod';
 import { searchCombined, searchByDate, SearchQueryError } from '../store/search.js';
-export function registerSearchTool(mcp, db) {
-    mcp.tool('bucket_search', 'Full-text search across BOTH skills and memory docs in one ranked list — use this when you don\'t know (or don\'t care) which bucket something landed in. `query` is raw SQLite FTS5 MATCH syntax: bare words, "exact phrases", prefix* wildcards, AND/OR/NOT boolean operators; hyphenated/punctuated terms must be quoted, e.g. "blue-green". For filtering by doc_type/status/tag, use skill_search or memory_search instead. Returns ranked hits with a highlighted snippet, not full body — call skill_get/memory_get on a hit for that.', {
+import { recall, formatRecall } from '../store/recall.js';
+export function registerSearchTool(mcp, db, deps) {
+    mcp.tool('bucket_recall', 'Answers "what do I have about X" in one call: takes a plain natural-language question (no query syntax) and returns the most relevant PASSAGES — heading-level sections with their full text — from skills and memory docs, ranked by hybrid keyword + semantic similarity and packed into max_tokens. Prefer this over *_search followed by *_get when you want content rather than a list of docs; use skill_get/memory_get only when a returned passage shows you need its whole document. Paused docs are excluded.', {
+        query: z.string().describe('a question or topic in plain words, e.g. `how do I position a tooltip so it flips at the viewport edge`'),
+        max_tokens: z.number().int().min(200).max(20000).optional().describe('budget for returned passage text, default 2000'),
+        bucket: z.enum(['skills', 'memory_docs']).optional().describe('restrict to one bucket; omit to search both'),
+    }, async ({ query, max_tokens, bucket }) => {
+        try {
+            const { indexer } = deps;
+            const result = await recall(db, query, {
+                maxTokens: max_tokens,
+                table: bucket,
+                hidden: deps.hiddenFolders(),
+                embedder: indexer?.isReady() ? indexer.embedder : undefined,
+            });
+            let note;
+            if (indexer && result.mode === 'lexical') {
+                const status = indexer.status();
+                note = status.error
+                    ? `Semantic index unavailable (${status.error}) — keyword matches only.`
+                    : `Semantic index still building (${status.embedded}/${status.total} passages) — keyword matches only for now.`;
+            }
+            return { content: [{ type: 'text', text: formatRecall(result, note) }] };
+        }
+        catch (err) {
+            return { content: [{ type: 'text', text: err.message }], isError: true };
+        }
+    });
+    mcp.tool('bucket_search', 'Keyword search across BOTH skills and memory docs in one ranked list of DOCS (for passages, use bucket_recall) — use this to locate a doc when you don\'t know which bucket it landed in. `query` is SQLite FTS5 MATCH syntax: bare words (all must match; falls back to any-word when nothing matches all), "exact phrases", prefix* wildcards, AND/OR/NOT. For filtering by doc_type/status/tag, use skill_search or memory_search instead. Returns ranked hits with a highlighted snippet, not full body — call skill_get/memory_get on a hit for that.', {
         query: z.string().describe('FTS5 match expression, e.g. `deploy AND rollback` or `"blue green"`'),
         limit: z.number().int().positive().max(100).optional(),
         offset: z.number().int().nonnegative().optional(),

package/dist/src/skills/repository.js

@@ -5,7 +5,7 @@ import { writeMarkdownFile, formatMarkdownFile } from '../store/markdown-file.js
 import { assertValidSkillName } from '../store/skill-name.js';
 import { resolveWithinBase } from '../store/safe-path.js';
 import { upsertFile, removeFile, scanSingleFolder, unregisterFolder, skillSyncSpec, walkSkillSiblingFiles } from '../store/sync.js';
-import { SearchQueryError, sanitizeFtsQuery } from '../store/search.js';
+import { SearchQueryError, SEARCH_BM25, SEARCH_SNIPPET, withOrFallback } from '../store/search.js';
 import { applyBodyEdits } from '../shared/body-edits.js';
 import { rebaseFolderPath } from '../config.js';
 import { readFile as readRemoteFile, writeFile as writeRemoteFile, writeBinaryFile as writeRemoteBinaryFile, trashFile as trashRemoteFile, trashFolder as trashRemoteFolder, renameFolder as renameRemoteFolder, joinRemoteFolderPath, assertRemoteFolderExists } from '../remote/folderfoo-client.js';
@@ -339,7 +339,7 @@ export class SkillRepository {
     search(query, opts = {}) {
         const { folder, status, owner, tag, group, limit = 20, offset = 0, includePaused = false } = opts;
         const conditions = [];
-        const params = [sanitizeFtsQuery(query)];
+        const params = [];
         if (folder) {
             conditions.push('s.folder = ?');
             params.push(folder);
@@ -368,23 +368,21 @@ export class SkillRepository {
             conditions.push(`s.folder NOT IN (${hidden.map(() => '?').join(', ')})`);
             params.push(...hidden);
         }
-        params.push(limit, offset);
         try {
             // shared_items is LEFT JOINed on s.source_path (== mirror_path for an item-level share — see
             // remote/shared-items.ts) — same reasoning as MemoryRepository.search's identical join.
-            const rows = this.db
-                .prepare(`SELECT s.id AS name, s.description, s.folder,
-                  snippet(search_index, 3, '<<', '>>', '…', 20) AS snippet,
-                  -bm25(search_index) AS score,
+            const statement = this.db.prepare(`SELECT s.id AS name, s.description, s.folder,
+                  ${SEARCH_SNIPPET} AS snippet,
+                  -${SEARCH_BM25} AS score,
                   si.owner AS shared_owner,
                   si.role AS shared_role
            FROM search_index
            JOIN skills s ON s.id = search_index.ref_id AND s.folder = search_index.ref_folder
            LEFT JOIN shared_items si ON si.status = 'active' AND si.mirror_path = s.source_path
            WHERE search_index.ref_table = 'skills' AND search_index MATCH ? ${conditions.map((c) => `AND ${c}`).join(' ')}
-           ORDER BY bm25(search_index)
-           LIMIT ? OFFSET ?`)
-                .all(...params);
+           ORDER BY ${SEARCH_BM25}
+           LIMIT ? OFFSET ?`);
+            const rows = withOrFallback(this.db, query, offset, (ftsQuery, probe) => statement.all(ftsQuery, ...params, ...(probe ? [1, 0] : [limit, offset])));
             return rows.map(({ shared_owner, shared_role, ...rest }) => ({
                 ...rest,
                 shared_owner: shared_owner ?? undefined,

package/dist/src/skills/tools.js

@@ -23,7 +23,7 @@ export function registerSkillTools(mcp, repo) {
         const items = repo.list(query, folder, { includePaused: include_paused, group });
         return { content: [{ type: 'text', text: JSON.stringify(items, null, 2) }] };
     });
-    mcp.tool('skill_search', 'Full-text search over skill description/body/tags (grep/find-like, ranked by relevance) — unlike skill_list\'s substring metadata filter, this searches the full markdown body. `query` is raw SQLite FTS5 MATCH syntax: bare words, "exact phrases", prefix* wildcards, AND/OR/NOT boolean operators; hyphenated/punctuated terms must be quoted, e.g. "blue-green". Can be combined with status/owner/tag filters. Returns ranked hits with a highlighted snippet, not the full body — call skill_get on a hit\'s name for that. Paused skills are hidden by default — pass include_paused to see them.', {
+    mcp.tool('skill_search', 'Full-text search over skill description/body/tags (grep/find-like, ranked by relevance) — unlike skill_list\'s substring metadata filter, this searches the full markdown body. `query` is SQLite FTS5 MATCH syntax: bare words (all must match; falls back to any-word when nothing matches all), "exact phrases", prefix* wildcards, AND/OR/NOT. Can be combined with status/owner/tag filters. Returns ranked hits with a highlighted snippet, not the full body — call skill_get on a hit\'s name for that. Paused skills are hidden by default — pass include_paused to see them.', {
         query: z.string().describe('FTS5 match expression, e.g. `deploy AND rollback` or `"blue green"`'),
         status: statusSchema(SKILL_STATUS_DEFAULTS).optional(),
         owner: z.string().optional(),

package/dist/src/store/chunker.js

@@ -0,0 +1,112 @@
+/** Upper bound per chunk — sized so heading + context + text stays inside a small embedding model's 512-token window. */
+export const MAX_CHUNK_CHARS = 1400;
+/** Sections shorter than this fold into the chunk before them rather than standing alone as a near-empty hit. */
+const MIN_CHUNK_CHARS = 200;
+const HEADING_RE = /^(#{1,6})\s+(.+?)\s*#*\s*$/;
+const FENCE_RE = /^\s*(```|~~~)/;
+/**
+ * Splits a markdown body into retrieval-sized chunks along its heading structure, so a search hit
+ * can return the one relevant section instead of the whole document. Fenced code blocks are never
+ * split on a `#` line inside them, and an oversized section is cut at paragraph boundaries (never
+ * mid-fence) rather than at an arbitrary character offset.
+ */
+export function chunkMarkdown(body) {
+    const sections = [{ level: 0, title: '', path: '', lines: [] }];
+    const stack = [];
+    let fence = null;
+    for (const line of body.split(/\r?\n/)) {
+        const fenceMatch = FENCE_RE.exec(line);
+        if (fenceMatch) {
+            if (!fence)
+                fence = fenceMatch[1];
+            else if (fenceMatch[1] === fence)
+                fence = null;
+        }
+        const heading = fence ? null : HEADING_RE.exec(line);
+        if (heading) {
+            const level = heading[1].length;
+            while (stack.length > 0 && stack[stack.length - 1].level >= level)
+                stack.pop();
+            stack.push({ level, title: heading[2] });
+            sections.push({ level, title: heading[2], path: stack.map((s) => s.title).join(' › '), lines: [] });
+        }
+        else {
+            sections[sections.length - 1].lines.push(line);
+        }
+    }
+    const chunks = [];
+    for (const section of sections) {
+        const text = section.lines.join('\n').trim();
+        if (!text)
+            continue;
+        const previous = chunks[chunks.length - 1];
+        if (previous && text.length < MIN_CHUNK_CHARS && previous.text.length + text.length <= MAX_CHUNK_CHARS) {
+            // Keep the folded section's own heading inline — its text no longer has a heading path of its own.
+            previous.text += `\n\n${'#'.repeat(section.level)} ${section.title}\n${text}`;
+            continue;
+        }
+        for (const piece of splitOversized(text))
+            chunks.push({ heading: section.path, text: piece });
+    }
+    return chunks.map((chunk, ord) => ({ ord, ...chunk }));
+}
+/** Greedily packs paragraphs up to MAX_CHUNK_CHARS; a single paragraph over the limit is cut by lines. */
+function splitOversized(text) {
+    if (text.length <= MAX_CHUNK_CHARS)
+        return [text];
+    const pieces = [];
+    let current = '';
+    const flush = () => {
+        if (current.trim())
+            pieces.push(current.trim());
+        current = '';
+    };
+    for (const paragraph of splitParagraphs(text)) {
+        if (paragraph.length > MAX_CHUNK_CHARS) {
+            flush();
+            for (const line of paragraph.split('\n')) {
+                if (current.length + line.length + 1 > MAX_CHUNK_CHARS)
+                    flush();
+                // A single line longer than the limit (minified JSON, a long URL list) still has to be cut somewhere.
+                for (let i = 0; i < line.length; i += MAX_CHUNK_CHARS) {
+                    current += (current ? '\n' : '') + line.slice(i, i + MAX_CHUNK_CHARS);
+                    if (line.length > MAX_CHUNK_CHARS)
+                        flush();
+                }
+            }
+            flush();
+            continue;
+        }
+        if (current.length + paragraph.length + 2 > MAX_CHUNK_CHARS)
+            flush();
+        current += (current ? '\n\n' : '') + paragraph;
+    }
+    flush();
+    return pieces;
+}
+/** Blank-line-separated paragraphs, treating a fenced code block (blank lines included) as one paragraph. */
+function splitParagraphs(text) {
+    const paragraphs = [];
+    let current = [];
+    let fence = null;
+    for (const line of text.split('\n')) {
+        const fenceMatch = FENCE_RE.exec(line);
+        if (fenceMatch) {
+            if (!fence)
+                fence = fenceMatch[1];
+            else if (fenceMatch[1] === fence)
+                fence = null;
+        }
+        if (!fence && line.trim() === '') {
+            if (current.length > 0)
+                paragraphs.push(current.join('\n'));
+            current = [];
+        }
+        else {
+            current.push(line);
+        }
+    }
+    if (current.length > 0)
+        paragraphs.push(current.join('\n'));
+    return paragraphs;
+}

package/dist/src/store/chunks.js

@@ -0,0 +1,125 @@
+import crypto from 'node:crypto';
+import { chunkMarkdown } from './chunker.js';
+import { flattenTags } from './db.js';
+/**
+ * Bumped on every write to `chunks` (content or embedding) — lets the in-memory vector cache in
+ * recall.ts know when to reload, and wakes the embedding indexer via the listener below.
+ */
+let chunksVersion = 0;
+let onChunksChanged;
+export function getChunksVersion() {
+    return chunksVersion;
+}
+export function bumpChunksVersion() {
+    chunksVersion++;
+}
+/** Registers the single callback fired whenever chunk CONTENT changes (not on embedding writes) — see EmbeddingIndexer.start. */
+export function setChunksChangedListener(listener) {
+    onChunksChanged = listener;
+}
+/** The text an embedding is computed from: doc-level context + heading path + chunk text, so a chunk stays meaningful out of its document. */
+export function embedTextFor(chunk) {
+    return [chunk.context, chunk.heading, chunk.text].filter(Boolean).join('\n');
+}
+/** Doc-level line prepended to every chunk's embedding input — name/key, description, and a skill's trigger phrases. */
+export function chunkContext(table, row) {
+    const label = table === 'skills' ? row.id : row.key;
+    const triggers = table === 'skills' ? flattenTags(String(row.trigger_phrases ?? '[]')) : '';
+    return [`${label ?? ''}: ${row.description ?? ''}`, triggers].filter(Boolean).join(' — ');
+}
+/**
+ * Replaces one doc's chunks (and their chunk_fts rows) with a fresh split of `body`. An embedding
+ * is carried over to any new chunk whose embedded text is byte-identical to an old one, so editing
+ * one section of a long doc re-embeds only that section. A doc with an empty body still gets one
+ * chunk (its description) so it stays reachable by semantic search.
+ */
+export function writeChunks(db, table, id, refFolder, context, description, body) {
+    const old = db
+        .prepare(`SELECT id, hash, embedding, embed_model FROM chunks WHERE ref_table = ? AND ref_id = ? AND ref_folder = ?`)
+        .all(table, id, refFolder);
+    const reusable = new Map(old.filter((o) => o.embedding).map((o) => [o.hash, o]));
+    deleteChunkRows(db, old.map((o) => o.id));
+    let chunks = chunkMarkdown(body);
+    if (chunks.length === 0)
+        chunks = [{ ord: 0, heading: '', text: description }];
+    const insertChunk = db.prepare(`INSERT INTO chunks (ref_table, ref_id, ref_folder, ord, context, heading, text, hash, embedding, embed_model) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
+    const insertFts = db.prepare(`INSERT INTO chunk_fts (rowid, heading, body) VALUES (?, ?, ?)`);
+    const insertTerms = db.prepare(`INSERT INTO term_index (rowid, text) VALUES (?, ?)`);
+    for (const chunk of chunks) {
+        const hash = crypto.createHash('sha1').update(embedTextFor({ context, ...chunk })).digest('hex');
+        const kept = reusable.get(hash);
+        const { lastInsertRowid } = insertChunk.run(table, id, refFolder, chunk.ord, context, chunk.heading, chunk.text, hash, kept?.embedding ?? null, kept?.embed_model ?? null);
+        insertFts.run(lastInsertRowid, chunk.heading, chunk.text);
+        insertTerms.run(lastInsertRowid, embedTextFor({ context, ...chunk }));
+    }
+    chunksVersion++;
+    onChunksChanged?.();
+}
+export function deleteChunks(db, table, id, refFolder) {
+    const rows = db.prepare(`SELECT id FROM chunks WHERE ref_table = ? AND ref_id = ? AND ref_folder = ?`).all(table, id, refFolder);
+    deleteChunkRows(db, rows.map((r) => r.id));
+    chunksVersion++;
+}
+function deleteChunkRows(db, ids) {
+    const deleteFts = db.prepare(`DELETE FROM chunk_fts WHERE rowid = ?`);
+    const deleteChunk = db.prepare(`DELETE FROM chunks WHERE id = ?`);
+    const deleteTerms = db.prepare(`DELETE FROM term_index WHERE rowid = ?`);
+    for (const id of ids) {
+        deleteFts.run(id);
+        deleteTerms.run(id);
+        deleteChunk.run(id);
+    }
+}
+/**
+ * Drops chunks whose parent doc row is gone. A cache rebuild wipes skills/memory_docs and rescans
+ * but deliberately leaves `chunks` alone so writeChunks can carry every unchanged embedding over —
+ * this then removes whatever the rescan did not re-claim.
+ */
+export function pruneOrphanChunks(db) {
+    const orphans = db
+        .prepare(`SELECT c.id FROM chunks c WHERE
+         (c.ref_table = 'skills' AND NOT EXISTS (SELECT 1 FROM skills s WHERE s.id = c.ref_id AND s.folder = c.ref_folder))
+         OR (c.ref_table = 'memory_docs' AND NOT EXISTS (SELECT 1 FROM memory_docs m WHERE m.source_path = c.ref_id))`)
+        .all();
+    if (orphans.length === 0)
+        return;
+    deleteChunkRows(db, orphans.map((o) => o.id));
+    chunksVersion++;
+}
+/**
+ * Chunks every cached doc that has none yet — cache files created before `chunks` existed, whose
+ * rows upsertFile's mtime check would otherwise skip forever. Safe to call every startup.
+ */
+export function backfillChunks(db) {
+    const skills = db
+        .prepare(`SELECT id, folder, description, trigger_phrases, body FROM skills s
+       WHERE NOT EXISTS (SELECT 1 FROM chunks c WHERE c.ref_table = 'skills' AND c.ref_id = s.id AND c.ref_folder = s.folder)`)
+        .all();
+    const memoryDocs = db
+        .prepare(`SELECT source_path, key, description, body FROM memory_docs m
+       WHERE NOT EXISTS (SELECT 1 FROM chunks c WHERE c.ref_table = 'memory_docs' AND c.ref_id = m.source_path)`)
+        .all();
+    if (skills.length === 0 && memoryDocs.length === 0)
+        return;
+    db.transaction(() => {
+        for (const row of skills)
+            writeChunks(db, 'skills', row.id, row.folder, chunkContext('skills', row), row.description, row.body);
+        for (const row of memoryDocs)
+            writeChunks(db, 'memory_docs', row.source_path, '', chunkContext('memory_docs', row), row.description, row.body);
+    })();
+}
+/** Fills term_index for cache files whose chunks were written before it existed. Safe to call every startup. */
+export function backfillTermIndex(db) {
+    const { n } = db.prepare(`SELECT COUNT(*) AS n FROM term_index`).get();
+    if (n > 0)
+        return;
+    const rows = db.prepare(`SELECT id, context, heading, text FROM chunks`).all();
+    if (rows.length === 0)
+        return;
+    const insert = db.prepare(`INSERT INTO term_index (rowid, text) VALUES (?, ?)`);
+    db.transaction(() => {
+        for (const row of rows)
+            insert.run(row.id, embedTextFor(row));
+    })();
+    chunksVersion++;
+}

package/dist/src/store/db.js

@@ -1,4 +1,5 @@
 import Database from 'better-sqlite3';
+import { backfillChunks, backfillTermIndex } from './chunks.js';
 export function openCache(dbPath) {
     const db = new Database(dbPath);
     db.pragma('journal_mode = WAL');
@@ -53,10 +54,39 @@ export function openCache(dbPath) {
       body,
       tags,
       key,             -- memory docs' key; also holds a skill's group (skills never had a key, so this is safe to share)
-      filename,
+      filename,        -- memory docs' on-disk filename; for a skill, its name (its file is always SKILL.md)
+      triggers,        -- a skill's trigger_phrases; '' for memory docs
       tokenize = 'porter unicode61'
     );
 
+    -- Heading-level pieces of each doc's body (see chunker.ts) — the retrieval unit for bucket_recall.
+    -- embedding is a unit-length Float32 vector, NULL until the background indexer reaches the row
+    -- (or forever, when embeddings are off); embed_model records which model produced it.
+    CREATE TABLE IF NOT EXISTS chunks (
+      id INTEGER PRIMARY KEY,
+      ref_table TEXT NOT NULL,
+      ref_id TEXT NOT NULL,
+      ref_folder TEXT NOT NULL DEFAULT '',
+      ord INTEGER NOT NULL,
+      context TEXT NOT NULL,
+      heading TEXT NOT NULL,
+      text TEXT NOT NULL,
+      hash TEXT NOT NULL,
+      embedding BLOB,
+      embed_model TEXT
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_chunks_ref ON chunks(ref_table, ref_id, ref_folder);
+
+    -- rowid == chunks.id
+    CREATE VIRTUAL TABLE IF NOT EXISTS chunk_fts USING fts5(heading, body, tokenize = 'porter unicode61');
+
+    -- Unstemmed words per chunk (rowid == chunks.id), kept only so term_vocab can list the corpus
+    -- vocabulary for typo correction (see spelling.ts) — the stemmed indexes above hold word stems
+    -- ("migrat"), which a misspelling cannot be compared against. Stores no text and no positions.
+    CREATE VIRTUAL TABLE IF NOT EXISTS term_index USING fts5(text, content = '', contentless_delete = 1, detail = none, tokenize = 'unicode61');
+    CREATE VIRTUAL TABLE IF NOT EXISTS term_vocab USING fts5vocab(term_index, 'row');
+
     CREATE TABLE IF NOT EXISTS doc_dates (
       ref_table TEXT NOT NULL,
       ref_id TEXT NOT NULL,
@@ -115,6 +145,8 @@ export function openCache(dbPath) {
     ]);
     ensureSearchIndexHasKeyColumn(db);
     backfillSearchIndex(db);
+    backfillChunks(db);
+    backfillTermIndex(db);
     return db;
 }
 /**
@@ -133,7 +165,7 @@ export function openCache(dbPath) {
  */
 function ensureSearchIndexHasKeyColumn(db) {
     const cols = db.prepare(`PRAGMA table_info(search_index)`).all();
-    if (cols.length === 0 || (cols.some((c) => c.name === 'key') && cols.some((c) => c.name === 'ref_folder') && cols.some((c) => c.name === 'filename')))
+    if (cols.length === 0 || ['key', 'ref_folder', 'filename', 'triggers'].every((name) => cols.some((c) => c.name === name)))
         return; // fresh table, or already migrated
     db.exec(`DROP TABLE search_index`);
     db.exec(`
@@ -146,6 +178,7 @@ function ensureSearchIndexHasKeyColumn(db) {
       tags,
       key,
       filename,
+      triggers,
       tokenize = 'porter unicode61'
     );
   `);
@@ -209,18 +242,18 @@ function backfillSearchIndex(db) {
     const { count: indexed } = db.prepare(`SELECT COUNT(*) as count FROM search_index`).get();
     if (indexed > 0)
         return;
-    const skillRows = db.prepare(`SELECT id, folder, description, body, tags, skill_group FROM skills`).all();
+    const skillRows = db.prepare(`SELECT id, folder, description, body, tags, skill_group, trigger_phrases FROM skills`).all();
     const memoryRows = db.prepare(`SELECT source_path AS id, key, description, body, tags FROM memory_docs`).all();
     if (skillRows.length === 0 && memoryRows.length === 0)
         return;
-    const insert = db.prepare(`INSERT INTO search_index (ref_table, ref_id, ref_folder, description, body, tags, key, filename) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
+    const insert = db.prepare(`INSERT INTO search_index (ref_table, ref_id, ref_folder, description, body, tags, key, filename, triggers) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`);
     const insertAll = db.transaction(() => {
         for (const row of skillRows) {
-            insert.run('skills', row.id, row.folder, row.description, row.body, flattenTags(row.tags), row.skill_group ?? '', '');
+            insert.run('skills', row.id, row.folder, row.description, row.body, flattenTags(row.tags), row.skill_group ?? '', row.id, flattenTags(row.trigger_phrases));
         }
         for (const row of memoryRows) {
             // row.id is source_path (see the SELECT above) — filename is its basename.
-            insert.run('memory_docs', row.id, '', row.description, row.body, flattenTags(row.tags), row.key, row.id.split('/').pop() ?? row.id);
+            insert.run('memory_docs', row.id, '', row.description, row.body, flattenTags(row.tags), row.key, row.id.split('/').pop() ?? row.id, '');
         }
     });
     insertAll();