@wei840222/qmd 2026.9.6 → 2026.9.25

package/dist/store.js

@@ -25,6 +25,9 @@ import { analyzeCjkSync, containsCjk } from "./search/cjk-analyzer.js";
 import { ExpansionPolicyError, parseExpansionDirective, resolveExpansionPolicy, } from "./search/query-expansion.js";
 import { LlamaCpp, getDefaultLlamaCpp, formatQueryForEmbedding, formatDocForEmbedding, withLLMSessionForLlm, DEFAULT_EMBED_MODEL_URI, DEFAULT_RERANK_MODEL_URI, DEFAULT_GENERATE_MODEL_URI, } from "./llm.js";
 const readOnlyDatabases = new WeakSet();
+import { METADATA_EXTRACTION_VERSION } from "./metadata.js";
+import { compileMetadataFilter } from "./metadata-filter.js";
+import { initializeMetadataSchema, syncDocumentMetadata, countDocumentsPendingMetadata, getMetadataByFilepath, parseMetadataJson, } from "./metadata-store.js";
 // =============================================================================
 // Configuration
 // =============================================================================
@@ -925,18 +928,15 @@ export function normalizeCjkForFTS(text) {
     return text.replace(CJK_RUN_PATTERN, run => ` ${Array.from(run).join(' ')} `);
 }
 function sanitizeFTS5Phrase(phrase) {
-    // Dotted tokens (1.0.21, 2026.4.10) are indexed as adjacent parts by the
-    // porter unicode61 tokenizer. Stripping the dots would produce "1021",
-    // which never matches — split them into phrase terms instead (#757).
+    // A quoted phrase is matched against tokens the porter unicode61 tokenizer
+    // produced, and that tokenizer splits document text on every separator.
+    // Deleting the separators here instead would collapse "1.0.21" to "1021" and
+    // "PIO-1384" to "pio1384", tokens no document holds, so the query returns
+    // nothing with no error (#757 for dots, #916 for the rest). Split on the same
+    // separators the tokenizer does and emit the parts as adjacent phrase terms.
     return normalizeCjkForFTS(phrase)
         .split(/\s+/)
-        .flatMap(t => {
-        if (isDottedToken(t)) {
-            return t.split('.').map(p => sanitizeFTS5Term(p)).filter(p => p);
-        }
-        const sanitized = sanitizeFTS5Term(t);
-        return sanitized ? [sanitized] : [];
-    })
+        .flatMap(t => splitFTS5CompoundTerm(t))
         .join(' ');
 }
 function getUserVersion(db) {
@@ -1402,6 +1402,9 @@ function initializeDatabase(db) {
   `);
     ensureEmbeddingIdentitySchema(db);
     ensureContentVectorsStatusIndex(db);
+    // Document metadata — extraction state plus normalized value rows for
+    // metadata filtering. Keyed by document identity, not content hash.
+    initializeMetadataSchema(db);
     // Store collections — makes the DB self-contained (no external config needed)
     db.exec(`
     CREATE TABLE IF NOT EXISTS store_collections (
@@ -1730,7 +1733,7 @@ export async function reindexCollection(store, collectionPath, globPattern, coll
         return !parts.some(part => part.startsWith("."));
     });
     const total = files.length;
-    let indexed = 0, updated = 0, unchanged = 0, processed = 0;
+    let indexed = 0, updated = 0, unchanged = 0, processed = 0, metadataErrors = 0;
     const skippedFiles = [];
     const seenPaths = new Set();
     // Literal paths of every file in this scan. Passed to the legacy-path
@@ -1772,8 +1775,12 @@ export async function reindexCollection(store, collectionPath, globPattern, coll
         const hash = await hashContent(content);
         const title = extractTitle(content, relativeFile);
         const existing = findOrMigrateLegacyDocument(db, collectionName, path, livePaths);
+        let documentId;
+        let contentChanged = true;
         if (existing) {
+            documentId = existing.id;
             if (existing.hash === hash) {
+                contentChanged = false;
                 if (existing.title !== title) {
                     updateDocumentTitle(db, existing.id, title, now);
                     updated++;
@@ -1791,8 +1798,12 @@ export async function reindexCollection(store, collectionPath, globPattern, coll
         else {
             indexed++;
             const stat = statSync(filepath);
-            insertDocumentWithContent(db, hash, content, now, collectionName, path, title, stat ? new Date(stat.birthtime).toISOString() : now, stat ? new Date(stat.mtime).toISOString() : now);
+            documentId = insertDocumentWithContent(db, hash, content, now, collectionName, path, title, stat ? new Date(stat.birthtime).toISOString() : now, stat ? new Date(stat.mtime).toISOString() : now);
         }
+        // Unchanged content still backfills missing or stale extraction state.
+        const extraction = syncDocumentMetadata(db, documentId, content, path, contentChanged ? undefined : { onlyIfStale: true });
+        if (extraction?.error)
+            metadataErrors++;
         processed++;
         options?.onProgress?.({ file: relativeFile, current: processed, total });
     }
@@ -1806,7 +1817,7 @@ export async function reindexCollection(store, collectionPath, globPattern, coll
         }
     }
     const orphanedCleaned = cleanupOrphanedContent(db);
-    return { indexed, updated, unchanged, removed, orphanedCleaned, skipped: skippedFiles.length, skippedFiles };
+    return { indexed, updated, unchanged, removed, orphanedCleaned, skipped: skippedFiles.length, skippedFiles, metadataErrors };
 }
 function validatePositiveIntegerOption(name, value, fallback) {
     if (value === undefined)
@@ -2464,7 +2475,7 @@ export async function generateEmbeddings(store, options) {
                         pos: chunk.pos,
                         tokens: chunk.tokenUpperBound,
                     }))
-                    : await chunkDocumentByTokens(doc.body, undefined, undefined, undefined, doc.path, options?.chunkStrategy, session.signal);
+                    : await chunkDocumentByTokensWithLlm((llm ?? getLlm(store) ?? getDefaultLlamaCpp()), doc.body, undefined, undefined, undefined, doc.path, options?.chunkStrategy, session.signal);
                 if (embeddingLease) {
                     pruneEmbeddingRowsOutsideLayout(db, doc.hash, model, fingerprint, chunks.length, embeddingLease);
                 }
@@ -2641,11 +2652,11 @@ export function createStore(dbPath, options = {}) {
     const commitPendingDocumentInsert = (collectionName, path, title, hash, createdAt, modifiedAt) => {
         const pending = pendingContent.get(hash);
         if (!pending) {
-            insertDocument(db, collectionName, path, title, hash, createdAt, modifiedAt);
-            return;
+            return insertDocument(db, collectionName, path, title, hash, createdAt, modifiedAt);
         }
-        insertDocumentWithContent(db, hash, pending.content, pending.createdAt, collectionName, path, title, createdAt, modifiedAt);
+        const docId = insertDocumentWithContent(db, hash, pending.content, pending.createdAt, collectionName, path, title, createdAt, modifiedAt);
         pendingContent.delete(hash);
+        return docId;
     };
     const commitPendingDocumentUpdate = (documentId, title, hash, modifiedAt) => {
         const pending = pendingContent.get(hash);
@@ -2693,9 +2704,9 @@ export function createStore(dbPath, options = {}) {
         resolveVirtualPath: (virtualPath) => resolveVirtualPath(db, virtualPath),
         toVirtualPath: (absolutePath) => toVirtualPath(db, absolutePath),
         // Search
-        searchCharFTS: (query, limit, collectionName) => searchCharFTS(db, query, limit, collectionName),
-        searchFTS: (query, limit, collectionName) => searchFTS(db, query, limit, collectionName),
-        searchVec: (query, model, limit, collectionFilter, session, precomputedEmbedding) => searchVec(db, query, model, limit, collectionFilter, session, precomputedEmbedding, store.embeddingProvider, store.authorizeRemoteRequest, store.llm),
+        searchCharFTS: (query, limit, collectionName, filter) => searchCharFTS(db, query, limit, collectionName, filter),
+        searchFTS: (query, limit, collectionName, filter) => searchFTS(db, query, limit, collectionName, filter),
+        searchVec: (query, model, limit, collectionFilter, session, precomputedEmbedding, filter) => searchVec(db, query, model, limit, collectionFilter, session, precomputedEmbedding, store.embeddingProvider, store.authorizeRemoteRequest, store.llm, filter),
         // Query expansion & reranking
         expandQuery: (query, model, expansionContext, options) => expandQuery(query, model ?? store.localLlm?.generateModelName ?? store.llm?.generateModelName ?? DEFAULT_QUERY_MODEL, db, expansionContext, store.llm, options),
         invalidateExpansionCache: (query, expansionContext, options) => deleteExpansionCacheEntry(db, query, store.localLlm?.generateModelName ?? store.llm?.generateModelName ?? DEFAULT_QUERY_MODEL, expansionContext, options),
@@ -2724,6 +2735,30 @@ export function createStore(dbPath, options = {}) {
         getActiveDocumentPaths: (collectionName) => getActiveDocumentPaths(db, collectionName),
         // Vector/embedding operations
         getHashesForEmbedding: () => getHashesForEmbedding(db),
+        insertEmbedding: (hash, seq, pos, embedding, model, embeddedAt, totalChunks, fingerprint, lease) => {
+            const state = db.prepare(`
+        SELECT status, fingerprint, model, generation, lease_owner, lease_expires_at
+        FROM embedding_index_state
+        WHERE singleton = 1
+      `).get();
+            if (!state && !lease) {
+                withLazyContentVectorMigration(db, () => {
+                    db.prepare(`
+            INSERT OR REPLACE INTO content_vectors (hash, seq, pos, model, embed_fingerprint, total_chunks, embedded_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+          `).run(hash, seq, pos, model, fingerprint ?? "", totalChunks ?? 1, embeddedAt);
+                    const hasCollectionCol = db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get();
+                    if (hasCollectionCol?.sql?.includes("collection")) {
+                        db.prepare(`INSERT OR REPLACE INTO vectors_vec (hash_seq, collection, embedding) VALUES (?, ?, ?)`).run(`${hash}_${seq}`, "", embedding);
+                    }
+                    else {
+                        db.prepare(`INSERT OR REPLACE INTO vectors_vec (hash_seq, embedding) VALUES (?, ?)`).run(`${hash}_${seq}`, embedding);
+                    }
+                });
+                return;
+            }
+            insertEmbedding(db, hash, seq, pos, embedding, model, embeddedAt, totalChunks, fingerprint, lease);
+        },
     };
     return store;
 }
@@ -2891,7 +2926,7 @@ export async function maybeAdoptLegacyEmbeddingFingerprint(store, model = DEFAUL
     const title = extractTitle(sample.body, sample.path);
     const llm = getLlm(store);
     return await withLLMSessionForLlm(llm, async (session) => {
-        const chunks = await chunkDocumentByTokens(sample.body, undefined, undefined, undefined, sample.path, undefined, session.signal);
+        const chunks = await chunkDocumentByTokensWithLlm(llm, sample.body, undefined, undefined, undefined, sample.path, undefined, session.signal);
         const chunk = chunks[sample.seq];
         if (!chunk) {
             return { checked: true, adopted: 0, reason: `sample chunk ${expectedHashSeq} no longer exists` };
@@ -3197,9 +3232,10 @@ function rebuildDocumentFTS(db, documentId) {
 }
 /**
  * Insert a new document into the documents table.
+ * Returns the document's id so callers can attach document-scoped state.
  */
 export function insertDocument(db, collectionName, path, title, hash, createdAt, modifiedAt) {
-    runCjkSynchronizedMutation(db, () => {
+    return runCjkSynchronizedMutation(db, () => {
         db.prepare(`
       INSERT INTO documents (collection, path, title, hash, created_at, modified_at, active)
       VALUES (?, ?, ?, ?, ?, ?, 1)
@@ -3210,15 +3246,17 @@ export function insertDocument(db, collectionName, path, title, hash, createdAt,
         active = 1
     `).run(collectionName, path, title, hash, createdAt, modifiedAt);
         const row = db.prepare(`SELECT id FROM documents WHERE collection = ? AND path = ?`).get(collectionName, path);
-        if (row)
-            rebuildDocumentFTS(db, row.id);
+        if (!row)
+            throw new Error(`Document row missing after insert: ${collectionName}/${path}`);
+        rebuildDocumentFTS(db, row.id);
+        return row.id;
     });
 }
 /** Insert immutable content and its document row in the same lexical transaction. */
 export function insertDocumentWithContent(db, hash, content, contentCreatedAt, collectionName, path, title, documentCreatedAt, modifiedAt) {
-    runCjkSynchronizedMutation(db, () => {
+    return runCjkSynchronizedMutation(db, () => {
         insertContent(db, hash, content, contentCreatedAt);
-        insertDocument(db, collectionName, path, title, hash, documentCreatedAt, modifiedAt);
+        return insertDocument(db, collectionName, path, title, hash, documentCreatedAt, modifiedAt);
     });
 }
 /**
@@ -3425,8 +3463,7 @@ function stripUnpairedSurrogates(text) {
  * When filepath and chunkStrategy are provided, uses AST-aware break points
  * for supported code files.
  */
-export async function chunkDocumentByTokens(content, maxTokens = CHUNK_SIZE_TOKENS, overlapTokens = CHUNK_OVERLAP_TOKENS, windowTokens = CHUNK_WINDOW_TOKENS, filepath, chunkStrategy = "regex", signal) {
-    const llm = getDefaultLlamaCpp();
+async function chunkDocumentByTokensWithLlm(llm, content, maxTokens = CHUNK_SIZE_TOKENS, overlapTokens = CHUNK_OVERLAP_TOKENS, windowTokens = CHUNK_WINDOW_TOKENS, filepath, chunkStrategy = "regex", signal) {
     // Use moderate chars/token estimate (prose ~4, code ~2, mixed ~3)
     // If chunks exceed limit, they'll be re-split with actual ratio
     const avgCharsPerToken = 3;
@@ -3500,6 +3537,9 @@ export async function chunkDocumentByTokens(content, maxTokens = CHUNK_SIZE_TOKE
     }
     return results;
 }
+export async function chunkDocumentByTokens(content, maxTokens = CHUNK_SIZE_TOKENS, overlapTokens = CHUNK_OVERLAP_TOKENS, windowTokens = CHUNK_WINDOW_TOKENS, filepath, chunkStrategy = "regex", signal) {
+    return await chunkDocumentByTokensWithLlm(getDefaultLlamaCpp(), content, maxTokens, overlapTokens, windowTokens, filepath, chunkStrategy, signal);
+}
 // =============================================================================
 // Fuzzy matching
 // =============================================================================
@@ -3962,38 +4002,29 @@ export function sanitizeFTS5Term(term) {
     return term.replace(/[^\p{L}\p{N}'_]/gu, '').toLowerCase();
 }
 /**
- * Check if a token is a hyphenated compound word (e.g., multi-agent, DEC-0054, gpt-4).
- * Returns true if the token contains internal hyphens between word/digit characters.
- */
-function isHyphenatedToken(token) {
-    return /^[\p{L}\p{N}][\p{L}\p{N}'-]*-[\p{L}\p{N}][\p{L}\p{N}'-]*$/u.test(token);
-}
-/**
- * Sanitize a hyphenated term into an FTS5 phrase by splitting on hyphens
- * and sanitizing each part. Returns the parts joined by spaces for use
- * inside FTS5 quotes: "multi agent" matches "multi-agent" in porter tokenizer.
+ * A run of characters the FTS tokenizer treats as a separator.
+ *
+ * `documents_fts` is tokenized with `porter unicode61`, which starts a new
+ * token at every character that is not a letter or a digit. Underscore is one
+ * of those, but it is deliberately kept here rather than split on: FTS5 applies
+ * the same tokenizer to a quoted phrase, so leaving `apply_secrets` intact lets
+ * it split symmetrically into `apply secrets` on both sides, and that is the
+ * behaviour #305 shipped. The apostrophe is kept for the same reason.
  */
-function sanitizeHyphenatedTerm(term) {
-    return term.split('-').map(t => sanitizeFTS5Term(t)).filter(t => t).join(' ');
-}
+const FTS5_SEPARATOR_RUN = /[^\p{L}\p{N}'_]+/u;
 /**
- * Check if a token is a dotted version/version-like string (e.g., 2026.4.10, 3.14.0).
- * Returns true if splitting on dots yields at least 2 non-empty parts consisting of
- * word/digit characters only. This avoids incorrectly splitting tokens with leading/
- * trailing dots. Version strings like "2026.4.10" split into ["2026","4","10"] (3 parts).
- */
-function isDottedToken(token) {
-    const parts = token.split('.');
-    return parts.length >= 2 && parts.every(p => p.length > 0 && /^[\p{L}\p{N}_]+$/u.test(p));
-}
-/**
- * Sanitize a dotted term into individual FTS5 tokens joined with AND.
- * e.g. "2026.4.10" → '"2026"* AND "4"* AND "10"*'
- * The AND ensures all parts must appear, matching how the porter tokenizer
- * indexes dotted strings.
+ * Split one query term the way the tokenizer split the document text, and
+ * sanitize each part.
+ *
+ * `PIO-1384` becomes ["pio", "1384"], `src/lib/i18n.ts` becomes
+ * ["src", "lib", "i18n", "ts"], and a term with no separator in it comes back
+ * as a single part. Callers join the parts into an FTS5 phrase, which is what
+ * makes the parts have to be adjacent in the document rather than merely all
+ * present. Parts that sanitize to nothing are dropped, so a term that is all
+ * punctuation yields an empty list and the caller skips it.
  */
-function sanitizeDottedTerm(term) {
-    return term.split('.').map(t => sanitizeFTS5Term(t)).filter(t => t).map(t => `"${t}"*`).join(' AND ');
+function splitFTS5CompoundTerm(term) {
+    return term.split(FTS5_SEPARATOR_RUN).map(p => sanitizeFTS5Term(p)).filter(p => p);
 }
 /**
  * Parse lex query syntax into FTS5 query.
@@ -4001,7 +4032,8 @@ function sanitizeDottedTerm(term) {
  * Supports:
  * - Quoted phrases: "exact phrase" → "exact phrase" (exact match)
  * - Negation: -term or -"phrase" → uses FTS5 NOT operator
- * - Hyphenated tokens: multi-agent, DEC-0054, gpt-4 → treated as phrases
+ * - Terms holding a separator: multi-agent, DEC-0054, gpt-4, 2026.4.10,
+ *   src/lib/i18n.ts, @tobilu/qmd → treated as phrases over their parts
  * - Plain terms: term → "term"* (prefix match)
  *
  * FTS5 NOT is a binary operator: `term1 NOT term2` means "match term1 but not term2".
@@ -4018,6 +4050,8 @@ function sanitizeDottedTerm(term) {
  *   multi-agent memory      → "multi agent" AND "memory"*
  *   DEC-0054               → "dec 0054"
  *   -multi-agent            → NOT "multi agent"
+ *   "DEC-0054"              → "dec 0054"
+ *   src/lib/i18n.ts         → "src lib i18n ts"
  */
 function buildFTS5Query(query) {
     const positive = [];
@@ -4061,41 +4095,7 @@ function buildFTS5Query(query) {
             while (i < s.length && !/[\s"]/.test(s[i]))
                 i++;
             const term = s.slice(start, i);
-            // Handle hyphenated tokens: multi-agent, DEC-0054, gpt-4
-            // These get split into phrase queries so FTS5 porter tokenizer matches them.
-            if (isHyphenatedToken(term)) {
-                const sanitized = sanitizeHyphenatedTerm(term);
-                if (sanitized) {
-                    const ftsPhrase = `"${sanitized}"`; // Phrase match (no prefix)
-                    if (negated) {
-                        negative.push(ftsPhrase);
-                    }
-                    else {
-                        positive.push(ftsPhrase);
-                    }
-                }
-            }
-            else if (isDottedToken(term)) {
-                // Handle dotted version strings: 2026.4.10, 3.14.0, v1.2.3
-                // The porter tokenizer splits on dots, so the index has individual tokens.
-                // We AND all parts together so the query matches documents containing all parts.
-                const sanitized = sanitizeDottedTerm(term);
-                if (sanitized) {
-                    // sanitizeDottedTerm already wraps each part in quotes with prefix match
-                    if (negated) {
-                        // Wrap multi-token AND expression in parens for NOT negation
-                        negative.push(`(${sanitized})`);
-                    }
-                    else {
-                        // Flatten individual AND'd terms into the positive list so they combine
-                        // correctly with other terms (avoids double-wrapping in outer AND).
-                        for (const part of sanitized.split(' AND ')) {
-                            positive.push(part.trim());
-                        }
-                    }
-                }
-            }
-            else if (containsCjk(term)) {
+            if (containsCjk(term)) {
                 const sanitized = sanitizeFTS5Phrase(term);
                 if (sanitized) {
                     const ftsPhrase = `"${sanitized}"`; // CJK phrase over character tokens
@@ -4108,9 +4108,16 @@ function buildFTS5Query(query) {
                 }
             }
             else {
-                const sanitized = sanitizeFTS5Term(term);
-                if (sanitized) {
-                    const ftsTerm = `"${sanitized}"*`; // Prefix match
+                // Any separator inside the term (multi-agent, DEC-0054, 2026.4.10,
+                // src/lib/i18n.ts, @tobilu/qmd) split it at index time too, so the term
+                // has to be matched as the phrase those parts form. A term with no
+                // separator is one part and keeps its prefix match, which is what makes
+                // a plain word still match longer words that start with it.
+                const parts = splitFTS5CompoundTerm(term);
+                if (parts.length > 0) {
+                    const ftsTerm = parts.length > 1
+                        ? `"${parts.join(' ')}"` // Phrase match (no prefix)
+                        : `"${parts[0]}"*`; // Prefix match
                     if (negated) {
                         negative.push(ftsTerm);
                     }
@@ -4170,7 +4177,28 @@ function normalizeCollectionFilter(filter) {
     const names = typeof filter === "string" ? [filter] : filter;
     return [...new Set(names.filter(name => name.length > 0))];
 }
-function searchFtsChannel(db, table, ftsQuery, limit, collectionFilter) {
+function scopedCollectionNames(scope) {
+    if (scope == null)
+        return undefined;
+    const names = (typeof scope === "string" ? [scope] : Array.from(scope))
+        .map(n => n.trim())
+        .filter(n => n.length > 0);
+    return names.length > 0 ? names : undefined;
+}
+function mergeSearchResultsByScore(lists, limit) {
+    const best = new Map();
+    for (const list of lists) {
+        for (const r of list) {
+            const prev = best.get(r.filepath);
+            if (!prev || r.score > prev.score)
+                best.set(r.filepath, r);
+        }
+    }
+    return Array.from(best.values())
+        .sort((a, b) => b.score - a.score)
+        .slice(0, limit);
+}
+function searchFtsChannel(db, table, ftsQuery, limit, collectionFilter, filter) {
     // Keep collection membership inside the ranked FTS candidate set. Filtering
     // after LIMIT can lose every matching row from a smaller collection.
     const params = [ftsQuery];
@@ -4182,7 +4210,10 @@ function searchFtsChannel(db, table, ftsQuery, limit, collectionFilter) {
         ? `AND filtered_d.active = 1 AND filtered_d.collection IN (${collections.map(() => "?").join(", ")})`
         : "";
     params.push(...collections);
-    params.push(limit);
+    // When filtering by metadata, fetch extra candidates from the FTS index
+    // since some will be filtered out. Without a filter we can fetch exactly the requested limit.
+    const ftsLimit = filter ? limit * 10 : limit;
+    params.push(ftsLimit);
     let sql = `
     WITH fts_matches AS (
       SELECT ${table}.rowid AS rowid, bm25(${table}, 1.5, 4.0, 1.0) as bm25_score
@@ -4199,12 +4230,21 @@ function searchFtsChannel(db, table, ftsQuery, limit, collectionFilter) {
       d.title,
       content.doc as body,
       d.hash,
-      fm.bm25_score
+      fm.bm25_score,
+      dm.metadata_json
     FROM fts_matches fm
     JOIN documents d ON d.id = fm.rowid
     JOIN content ON content.hash = d.hash
+    LEFT JOIN document_metadata dm ON dm.document_id = d.id
     WHERE d.active = 1
   `;
+    if (filter) {
+        // Only documents with current, error-free extraction can match — an
+        // unprocessed document must not accidentally satisfy `exists: false`.
+        const compiledFilter = compileMetadataFilter(filter, "d");
+        sql += ` AND dm.extraction_version = ${METADATA_EXTRACTION_VERSION} AND dm.extraction_error IS NULL AND ${compiledFilter.sql}`;
+        params.push(...compiledFilter.params);
+    }
     // bm25 lower is better; sort ascending.
     sql += ` ORDER BY fm.bm25_score ASC LIMIT ?`;
     params.push(limit);
@@ -4227,6 +4267,7 @@ function searchFtsChannel(db, table, ftsQuery, limit, collectionFilter) {
             bodyLength: row.body.length,
             body: row.body,
             context: getContextForFile(db, row.filepath),
+            metadata: parseMetadataJson(row.metadata_json),
             score,
             source: "fts",
         };
@@ -4238,17 +4279,21 @@ function buildCjkSignalQuery(signal) {
         return null;
     return terms.map(term => `"${term.replace(/"/g, '""')}"`).join(" AND ");
 }
-export function searchCharFTS(db, query, limit = 20, collectionFilter) {
+export function searchCharFTS(db, query, limit = 20, collectionFilter, filter) {
     if (!readOnlyDatabases.has(db))
         repairDirtyCjkCharFallback(db);
     const charQuery = buildFTS5Query(query);
     if (!charQuery)
         return [];
-    return searchFtsChannel(db, "documents_fts", charQuery, limit, collectionFilter);
+    return searchFtsChannel(db, "documents_fts", charQuery, limit, collectionFilter, filter);
 }
-export function searchFTS(db, query, limit = 20, collectionFilter) {
+export function searchFTS(db, query, limit = 20, collectionFilter, filter) {
+    const names = scopedCollectionNames(collectionFilter);
+    if (names && names.length > 1) {
+        return mergeSearchResultsByScore(names.map(name => searchFTS(db, query, limit, name, filter)), limit);
+    }
     if (!containsCjk(query))
-        return searchCharFTS(db, query, limit, collectionFilter);
+        return searchCharFTS(db, query, limit, collectionFilter, filter);
     const charQuery = buildFTS5Query(query);
     if (!charQuery)
         return [];
@@ -4257,7 +4302,7 @@ export function searchFTS(db, query, limit = 20, collectionFilter) {
     const candidateDepth = Math.max(limit, CJK_LEXICAL_CANDIDATE_DEPTH);
     const rankedChannels = [{
             channel: "char",
-            results: searchCharFTS(db, query, candidateDepth, collectionFilter),
+            results: searchCharFTS(db, query, candidateDepth, collectionFilter, filter),
         }];
     let secondaryReason = null;
     try {
@@ -4293,7 +4338,7 @@ export function searchFTS(db, query, limit = 20, collectionFilter) {
         }
         rankedChannels.push({
             channel: entry.channel,
-            results: searchFtsChannel(db, entry.table, ftsQuery, candidateDepth, collectionFilter),
+            results: searchFtsChannel(db, entry.table, ftsQuery, candidateDepth, collectionFilter, filter),
         });
         channels.push({ channel: entry.channel, status: "used" });
     }
@@ -4333,6 +4378,49 @@ export function searchFTS(db, query, limit = 20, collectionFilter) {
 // =============================================================================
 // Vector Search
 // =============================================================================
+/** sqlite-vec rejects k above this in MATCH queries (v0.1.9). */
+const SQLITE_VEC_MAX_K = 4096;
+/**
+ * Max filter-eligible vectors for an exact cosine scan. Above this we fall
+ * back to global ANN with a capped over-fetch. Exact scan avoids the
+ * post-filter starvation of small eligible sets — originally small
+ * collections (#791, #803), now also selective metadata filters; ANN remains
+ * for very large eligible sets where a full scan would be expensive.
+ */
+const FILTERED_VEC_EXACT_SCAN_MAX = 20_000;
+const VEC_HASH_SEQ_IN_CHUNK = 400;
+/**
+ * Exact cosine-distance scan over a known set of hash_seq keys.
+ * Uses vec_distance_cosine with chunked IN lists (no JOIN with vectors_vec).
+ */
+function exactVecScanByHashSeq(db, embedding, hashSeqs, limit) {
+    if (hashSeqs.length === 0 || limit <= 0)
+        return [];
+    const queryVec = new Float32Array(embedding);
+    // Over-fetch a bit so multi-chunk docs can still yield `limit` unique files.
+    const fetchLimit = Math.max(limit * 3, limit);
+    const scored = [];
+    for (let i = 0; i < hashSeqs.length; i += VEC_HASH_SEQ_IN_CHUNK) {
+        const chunk = hashSeqs.slice(i, i + VEC_HASH_SEQ_IN_CHUNK);
+        const placeholders = chunk.map(() => "?").join(",");
+        const rows = db.prepare(`
+      SELECT hash_seq, vec_distance_cosine(embedding, ?) AS distance
+      FROM vectors_vec
+      WHERE hash_seq IN (${placeholders})
+    `).all(queryVec, ...chunk);
+        scored.push(...rows);
+    }
+    scored.sort((a, b) => a.distance - b.distance);
+    return scored.slice(0, fetchLimit);
+}
+function annVecScan(db, embedding, k) {
+    const vecK = Math.max(1, Math.min(SQLITE_VEC_MAX_K, k));
+    return db.prepare(`
+    SELECT hash_seq, distance
+    FROM vectors_vec
+    WHERE embedding MATCH ? AND k = ?
+  `).all(new Float32Array(embedding), vecK);
+}
 function resolveReadyProviderEmbeddingIdentity(db, provider) {
     const storedIdentity = readStoredEmbeddingIdentity(db);
     if (!storedIdentity || storedIdentity.providerId !== provider.providerId
@@ -4373,10 +4461,29 @@ function hasSearchableVectorIndex(store) {
     return storedIdentity !== undefined
         && inspectEmbeddingIndexState(store.db, storedIdentity).status === "ready";
 }
-export async function searchVec(db, query, model, limit = 20, collectionFilter, session, precomputedEmbedding, provider, authorizeRemoteRequest, llmOverride) {
+export async function searchVec(db, query, model, limit = 20, collectionFilter, session, precomputedEmbedding, providerOrLlm, authorizeRemoteRequestOrFilter, llmOverride, metadataFilter) {
     const tableExists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get();
     if (!tableExists)
         return [];
+    // Disambiguate overloaded arguments
+    let provider;
+    let authorizeRemoteRequest;
+    let filter = metadataFilter;
+    let effectiveLlmOverride = llmOverride;
+    if (providerOrLlm && typeof providerOrLlm === "object") {
+        if ("providerId" in providerOrLlm) {
+            provider = providerOrLlm;
+        }
+        else {
+            effectiveLlmOverride = providerOrLlm;
+        }
+    }
+    if (typeof authorizeRemoteRequestOrFilter === "function") {
+        authorizeRemoteRequest = authorizeRemoteRequestOrFilter;
+    }
+    else if (typeof authorizeRemoteRequestOrFilter === "object" && authorizeRemoteRequestOrFilter !== null && !filter) {
+        filter = authorizeRemoteRequestOrFilter;
+    }
     if (provider && model !== provider.model) {
         throw new Error(`Embedding model ${model} does not match borrowed provider model ${provider.model}.`);
     }
@@ -4405,13 +4512,19 @@ export async function searchVec(db, query, model, limit = 20, collectionFilter,
         })).vector;
     }
     else if (!embedding) {
-        embedding = await getEmbedding(query, model, true, session, llmOverride);
+        embedding = await getEmbedding(query, model, true, session, effectiveLlmOverride);
     }
     if (!embedding)
         return [];
+    // Multi-collection union: search each collection separately to avoid starvation
+    const names = scopedCollectionNames(collectionFilter);
+    if (names && names.length > 1) {
+        const lists = await Promise.all(names.map(name => searchVec(db, query, model, limit, name, session, embedding, provider ?? effectiveLlmOverride, authorizeRemoteRequest ?? filter, effectiveLlmOverride, filter)));
+        return mergeSearchResultsByScore(lists, limit);
+    }
     const activeFingerprint = providerIdentity
         ? providerIdentity.fingerprint
-        : storedIdentity.fingerprint;
+        : storedIdentity?.fingerprint;
     const toSearchResult = (row, body, distance) => {
         return {
             filepath: row.filepath,
@@ -4424,56 +4537,87 @@ export async function searchVec(db, query, model, limit = 20, collectionFilter,
             bodyLength: body.length,
             body,
             context: getContextForFile(db, row.filepath),
+            metadata: parseMetadataJson(row.metadata_json),
             score: 1 - distance, // Cosine similarity = 1 - cosine distance
             source: "vec",
             chunkPos: row.pos,
         };
     };
     const collections = normalizeCollectionFilter(collectionFilter);
-    const queryVec = embedding instanceof Float32Array ? embedding : new Float32Array(embedding);
-    const SQLITE_VEC_MAX_K = 4096;
-    let collectionFilterSql = "";
-    const queryParams = [queryVec];
-    if (collections.length === 1) {
-        collectionFilterSql = "AND collection = ?";
-        queryParams.push(collections[0]);
-    }
-    else if (collections.length > 1) {
-        const placeholders = collections.map(() => "?").join(", ");
-        collectionFilterSql = `AND collection IN (${placeholders})`;
-        queryParams.push(...collections);
-    }
-    // Request generous k to account for multi-chunk deduplication per document
-    const fetchK = Math.min(SQLITE_VEC_MAX_K, Math.max(limit * 15, 60));
-    queryParams.push(fetchK);
     let vecRows;
-    try {
-        vecRows = withLazyContentVectorMigration(db, () => db.prepare(`
-        SELECT hash_seq, collection, distance
-        FROM vectors_vec
-        WHERE embedding MATCH ?
-          ${collectionFilterSql}
-          AND k = ?
-      `).all(...queryParams));
+    if (filter) {
+        let eligibleSql = `
+      SELECT DISTINCT cv.hash || '_' || cv.seq AS hash_seq
+      FROM content_vectors cv
+      JOIN documents d ON d.hash = cv.hash AND d.active = 1
+      JOIN document_metadata dm ON dm.document_id = d.id
+    `;
+        const eligibleConditions = [];
+        const eligibleParams = [];
+        if (activeFingerprint) {
+            eligibleConditions.push(`cv.model = ? AND cv.embed_fingerprint = ?`);
+            eligibleParams.push(model, activeFingerprint);
+        }
+        if (collections.length === 1) {
+            eligibleConditions.push(`d.collection = ?`);
+            eligibleParams.push(collections[0]);
+        }
+        else if (collections.length > 1) {
+            eligibleConditions.push(`d.collection IN (${collections.map(() => "?").join(", ")})`);
+            eligibleParams.push(...collections);
+        }
+        const compiledFilter = compileMetadataFilter(filter, "d");
+        eligibleConditions.push(`dm.extraction_version = ${METADATA_EXTRACTION_VERSION}`);
+        eligibleConditions.push(`dm.extraction_error IS NULL`);
+        eligibleConditions.push(compiledFilter.sql);
+        eligibleParams.push(...compiledFilter.params);
+        eligibleSql += ` WHERE ${eligibleConditions.join(" AND ")}`;
+        const eligibleHashSeqs = withLazyContentVectorMigration(db, () => db.prepare(eligibleSql).all(...eligibleParams)).map((r) => r.hash_seq);
+        if (eligibleHashSeqs.length === 0)
+            return [];
+        if (eligibleHashSeqs.length <= FILTERED_VEC_EXACT_SCAN_MAX) {
+            vecRows = exactVecScanByHashSeq(db, embedding, eligibleHashSeqs, limit);
+        }
+        else {
+            vecRows = annVecScan(db, embedding, Math.max(limit * 30, limit * 3));
+        }
     }
-    catch {
-        // Fallback if table lacks collection column before migration
-        vecRows = withLazyContentVectorMigration(db, () => db.prepare(`
-        SELECT hash_seq, distance
-        FROM vectors_vec
-        WHERE embedding MATCH ?
-          AND k = ?
-      `).all(queryVec, fetchK));
+    else {
+        // No metadata filter: use fast ANN scan (with collection pushdown if supported)
+        const queryVec = embedding instanceof Float32Array ? embedding : new Float32Array(embedding);
+        let collectionFilterSql = "";
+        const queryParams = [queryVec];
+        if (collections.length === 1) {
+            collectionFilterSql = "AND collection = ?";
+            queryParams.push(collections[0]);
+        }
+        else if (collections.length > 1) {
+            const placeholders = collections.map(() => "?").join(", ");
+            collectionFilterSql = `AND collection IN (${placeholders})`;
+            queryParams.push(...collections);
+        }
+        const fetchK = Math.min(SQLITE_VEC_MAX_K, Math.max(limit * 15, 60));
+        queryParams.push(fetchK);
+        try {
+            vecRows = withLazyContentVectorMigration(db, () => db.prepare(`
+          SELECT hash_seq, distance
+          FROM vectors_vec
+          WHERE embedding MATCH ?
+            ${collectionFilterSql}
+            AND k = ?
+        `).all(...queryParams));
+        }
+        catch {
+            // Fallback if table lacks collection column before migration
+            vecRows = annVecScan(db, embedding, fetchK);
+        }
     }
     if (vecRows.length === 0)
         return [];
     const keys = vecRows.map(r => r.hash_seq);
     const distMap = new Map(vecRows.map(r => [r.hash_seq, r.distance]));
     const placeholders = keys.map(() => "?").join(", ");
-    const collectionScope = collections.length > 0
-        ? `AND d.collection IN (${collections.map(() => "?").join(", ")})`
-        : "";
-    const metaRows = withLazyContentVectorMigration(db, () => db.prepare(`
+    let metaSql = `
     SELECT
       cv.hash || '_' || cv.seq AS hash_seq,
       cv.hash,
@@ -4481,14 +4625,32 @@ export async function searchVec(db, query, model, limit = 20, collectionFilter,
       'qmd://' || d.collection || '/' || d.path AS filepath,
       d.collection || '/' || d.path AS display_path,
       d.title,
-      d.collection
+      d.collection,
+      dm.metadata_json
     FROM content_vectors cv
     JOIN documents d ON d.hash = cv.hash AND d.active = 1
-    WHERE cv.model = ?
-      AND cv.embed_fingerprint = ?
-      ${collectionScope}
-      AND (cv.hash || '_' || cv.seq) IN (${placeholders})
-  `).all(model, activeFingerprint, ...collections, ...keys));
+    LEFT JOIN document_metadata dm ON dm.document_id = d.id
+    WHERE (cv.hash || '_' || cv.seq) IN (${placeholders})
+  `;
+    const metaParams = [...keys];
+    if (activeFingerprint) {
+        metaSql += ` AND cv.model = ? AND cv.embed_fingerprint = ?`;
+        metaParams.push(model, activeFingerprint);
+    }
+    if (collections.length === 1) {
+        metaSql += ` AND d.collection = ?`;
+        metaParams.push(collections[0]);
+    }
+    else if (collections.length > 1) {
+        metaSql += ` AND d.collection IN (${collections.map(() => "?").join(", ")})`;
+        metaParams.push(...collections);
+    }
+    if (filter) {
+        const compiledFilter = compileMetadataFilter(filter, "d");
+        metaSql += ` AND dm.extraction_version = ${METADATA_EXTRACTION_VERSION} AND dm.extraction_error IS NULL AND ${compiledFilter.sql}`;
+        metaParams.push(...compiledFilter.params);
+    }
+    const metaRows = withLazyContentVectorMigration(db, () => db.prepare(metaSql).all(...metaParams));
     const byFile = new Map();
     for (const m of metaRows) {
         const dist = distMap.get(m.hash_seq);
@@ -5318,6 +5480,7 @@ export function getStatusReadOnly(db, needsEmbedding) {
         totalDocuments: totalDocs,
         needsEmbedding,
         hasVectorIndex: hasVectors,
+        pendingMetadata: countDocumentsPendingMetadata(db),
         collections,
     };
 }
@@ -5443,6 +5606,15 @@ export function addLineNumbers(text, startLine = 1) {
     const lines = text.split('\n');
     return lines.map((line, i) => `${startLine + i}: ${line}`).join('\n');
 }
+/**
+ * Attach canonical metadata to final search results with one batch query.
+ * Runs after RRF/reranking so metadata is never duplicated through the
+ * intermediate ranked lists.
+ */
+function attachResultMetadata(db, results) {
+    const metadataByFilepath = getMetadataByFilepath(db, results.map(r => r.file));
+    return results.map(r => ({ ...r, metadata: metadataByFilepath.get(r.file) ?? {} }));
+}
 /**
  * RRF list weights for hybridQuery.
  *
@@ -5477,6 +5649,7 @@ export async function hybridQuery(store, query, options) {
     const collectionFilter = options?.collections && options.collections.length > 0
         ? options.collections
         : options?.collection;
+    const filter = options?.filter;
     const explain = options?.explain ?? false;
     const expansionContext = options?.expansionContext;
     const rerankContext = options?.rerankContext;
@@ -5491,9 +5664,9 @@ export async function hybridQuery(store, query, options) {
     // When either context is provided, disable strong-signal bypass — the obvious BM25
     // match may not be what the caller wants (e.g. "performance" with context
     // "web page load times" should NOT shortcut to a sports-performance doc).
-    // Pass collection directly into FTS query (filter at SQL level, not post-hoc)
+    // Pass collection and metadata filter directly into FTS query (filter at SQL level, not post-hoc)
     const parsedDirective = parseExpansionDirective(query);
-    const initialFts = store.searchFTS(parsedDirective.query, 20, collectionFilter);
+    const initialFts = store.searchFTS(parsedDirective.query, 20, collectionFilter, filter);
     const strongSignal = getLexicalStrongSignal(initialFts);
     const topScore = strongSignal.topScore;
     let expansionDecision;
@@ -5561,7 +5734,7 @@ export async function hybridQuery(store, query, options) {
     // 3a: Run FTS for all lex expansions right away (no LLM needed)
     for (const q of expanded) {
         if (q.type === 'lex') {
-            const ftsResults = store.searchFTS(q.query, 20, collectionFilter);
+            const ftsResults = store.searchFTS(q.query, 20, collectionFilter, filter);
             if (ftsResults.length > 0) {
                 for (const r of ftsResults)
                     docidMap.set(r.filepath, r.docid);
@@ -5593,7 +5766,7 @@ export async function hybridQuery(store, query, options) {
             const embedding = embeddings[i];
             if (!embedding || embedding.length === 0)
                 continue;
-            const vecResults = await store.searchVec(vecQueries[i].text, embedModel, 20, collectionFilter, undefined, embedding);
+            const vecResults = await store.searchVec(vecQueries[i].text, embedModel, 20, collectionFilter, undefined, embedding, filter);
             if (vecResults.length > 0) {
                 for (const r of vecResults)
                     docidMap.set(r.filepath, r.docid);
@@ -5660,7 +5833,7 @@ export async function hybridQuery(store, query, options) {
     if (skipRerank) {
         // Skip LLM reranking — return candidates scored by RRF only
         const seenFiles = new Set();
-        return candidates
+        const rrfResults = candidates
             .map((cand, i) => {
             const chunkInfo = docChunkMap.get(cand.file);
             const bestIdx = chunkInfo?.bestIdx ?? 0;
@@ -5705,6 +5878,7 @@ export async function hybridQuery(store, query, options) {
         })
             .filter(r => r.score >= minScore)
             .slice(0, limit);
+        return attachResultMetadata(store.db, rrfResults);
     }
     // Step 6: Rerank chunks (NOT full bodies)
     const chunksToRerank = [];
@@ -5771,7 +5945,7 @@ export async function hybridQuery(store, query, options) {
     }).sort((a, b) => b.score - a.score);
     // Step 8: Dedup by file (safety net — prevents duplicate output)
     const seenFiles = new Set();
-    return blended
+    const finalResults = blended
         .filter(r => {
         if (seenFiles.has(r.file))
             return false;
@@ -5780,6 +5954,7 @@ export async function hybridQuery(store, query, options) {
     })
         .filter(r => r.score >= minScore)
         .slice(0, limit);
+    return attachResultMetadata(store.db, finalResults);
 }
 /**
  * Vector-only semantic search with query expansion.
@@ -5794,6 +5969,7 @@ export async function vectorSearchQuery(store, query, options) {
     const limit = options?.limit ?? 10;
     const minScore = options?.minScore ?? 0.3;
     const collection = options?.collection;
+    const filter = options?.filter;
     const expansionContext = options?.expansionContext;
     const includeHyde = options?.includeHyde ?? true;
     if (!hasSearchableVectorIndex(store))
@@ -5809,7 +5985,7 @@ export async function vectorSearchQuery(store, query, options) {
     const queryTexts = [query, ...vecExpanded.map(q => q.query)];
     const allResults = new Map();
     for (const q of queryTexts) {
-        const vecResults = await store.searchVec(q, embedModel, limit, collection);
+        const vecResults = await store.searchVec(q, embedModel, limit, collection, undefined, undefined, filter);
         for (const r of vecResults) {
             const existing = allResults.get(r.filepath);
             if (!existing || r.score > existing.score) {
@@ -5821,6 +5997,7 @@ export async function vectorSearchQuery(store, query, options) {
                     score: r.score,
                     context: store.getContextForFile(r.filepath),
                     docid: r.docid,
+                    metadata: r.metadata,
                 });
             }
         }
@@ -5857,6 +6034,7 @@ export async function structuredSearch(store, searches, options) {
     const skipRerank = options?.skipRerank ?? false;
     const hooks = options?.hooks;
     const collections = options?.collections;
+    const filter = options?.filter;
     if (searches.length === 0)
         return [];
     // Validate queries before executing
@@ -5889,7 +6067,7 @@ export async function structuredSearch(store, searches, options) {
     for (const [searchIndex, search] of searches.entries()) {
         if (search.type === 'lex') {
             for (const coll of collectionList) {
-                const ftsResults = store.searchFTS(search.query, 20, coll);
+                const ftsResults = store.searchFTS(search.query, 20, coll, filter);
                 if (ftsResults.length > 0) {
                     for (const r of ftsResults)
                         docidMap.set(r.filepath, r.docid);
@@ -5922,7 +6100,7 @@ export async function structuredSearch(store, searches, options) {
                 if (!embedding || embedding.length === 0)
                     continue;
                 for (const coll of collectionList) {
-                    const vecResults = await store.searchVec(vecSearches[i].search.query, embedModel, 20, coll, undefined, embedding);
+                    const vecResults = await store.searchVec(vecSearches[i].search.query, embedModel, 20, coll, undefined, embedding, filter);
                     if (vecResults.length > 0) {
                         for (const r of vecResults)
                             docidMap.set(r.filepath, r.docid);
@@ -5985,7 +6163,7 @@ export async function structuredSearch(store, searches, options) {
     if (skipRerank) {
         // Skip LLM reranking — return candidates scored by RRF only
         const seenFiles = new Set();
-        return candidates
+        const rrfResults = candidates
             .map((cand, i) => {
             const chunkInfo = docChunkMap.get(cand.file);
             const bestIdx = chunkInfo?.bestIdx ?? 0;
@@ -6030,6 +6208,7 @@ export async function structuredSearch(store, searches, options) {
         })
             .filter(r => r.score >= minScore)
             .slice(0, limit);
+        return attachResultMetadata(store.db, rrfResults);
     }
     // Step 5: Rerank chunks
     const chunksToRerank = [];
@@ -6095,7 +6274,7 @@ export async function structuredSearch(store, searches, options) {
     }).sort((a, b) => b.score - a.score);
     // Step 7: Dedup by file
     const seenFiles = new Set();
-    return blended
+    const finalResults = blended
         .filter(r => {
         if (seenFiles.has(r.file))
             return false;
@@ -6104,4 +6283,5 @@ export async function structuredSearch(store, searches, options) {
     })
         .filter(r => r.score >= minScore)
         .slice(0, limit);
+    return attachResultMetadata(store.db, finalResults);
 }