@tobilu/qmd 1.1.5 → 2.0.0

package/dist/store.js

@@ -13,9 +13,10 @@
 import { openDatabase, loadSqliteVec } from "./db.js";
 import picomatch from "picomatch";
 import { createHash } from "crypto";
-import { realpathSync, statSync, mkdirSync } from "node:fs";
-import { LlamaCpp, getDefaultLlamaCpp, formatQueryForEmbedding, formatDocForEmbedding, } from "./llm.js";
-import { findContextForPath as collectionsFindContextForPath, addContext as collectionsAddContext, removeContext as collectionsRemoveContext, listAllContexts as collectionsListAllContexts, getCollection, listCollections as collectionsListCollections, addCollection as collectionsAddCollection, removeCollection as collectionsRemoveCollection, renameCollection as collectionsRenameCollection, setGlobalContext, loadConfig as collectionsLoadConfig, } from "./collections.js";
+import { readFileSync, realpathSync, statSync, mkdirSync } from "node:fs";
+// Note: node:path resolve is not imported — we export our own cross-platform resolve()
+import fastGlob from "fast-glob";
+import { LlamaCpp, getDefaultLlamaCpp, formatQueryForEmbedding, formatDocForEmbedding, withLLMSessionForLlm, } from "./llm.js";
 // =============================================================================
 // Configuration
 // =============================================================================
@@ -35,6 +36,13 @@ export const CHUNK_OVERLAP_CHARS = CHUNK_OVERLAP_TOKENS * 4; // 540 chars
 // Search window for finding optimal break points (in tokens, ~200 tokens)
 export const CHUNK_WINDOW_TOKENS = 200;
 export const CHUNK_WINDOW_CHARS = CHUNK_WINDOW_TOKENS * 4; // 800 chars
+/**
+ * Get the LlamaCpp instance for a store — prefers the store's own instance,
+ * falls back to the global singleton.
+ */
+function getLlm(store) {
+    return store.llm ?? getDefaultLlamaCpp();
+}
 /**
  * Patterns for detecting break points in markdown documents.
  * Higher scores indicate better places to split.
@@ -442,8 +450,8 @@ export function resolveVirtualPath(db, virtualPath) {
  * Returns null if the file is not in any indexed collection.
  */
 export function toVirtualPath(db, absolutePath) {
-    // Get all collections from YAML config
-    const collections = collectionsListCollections();
+    // Get all collections from DB
+    const collections = getStoreCollections(db);
     // Find which collection this absolute path belongs to
     for (const coll of collections) {
         if (absolutePath.startsWith(coll.path + '/') || absolutePath === coll.path) {
@@ -556,6 +564,25 @@ function initializeDatabase(db) {
       embedded_at TEXT NOT NULL,
       PRIMARY KEY (hash, seq)
     )
+  `);
+    // Store collections — makes the DB self-contained (no external config needed)
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS store_collections (
+      name TEXT PRIMARY KEY,
+      path TEXT NOT NULL,
+      pattern TEXT NOT NULL DEFAULT '**/*.md',
+      ignore_patterns TEXT,
+      include_by_default INTEGER DEFAULT 1,
+      update_command TEXT,
+      context TEXT
+    )
+  `);
+    // Store config — key-value metadata (e.g. config_hash for sync optimization)
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS store_config (
+      key TEXT PRIMARY KEY,
+      value TEXT
+    )
   `);
     // FTS - index filepath (collection/path), title, and content
     db.exec(`
@@ -600,6 +627,141 @@ function initializeDatabase(db) {
     END
   `);
 }
+function rowToNamedCollection(row) {
+    return {
+        name: row.name,
+        path: row.path,
+        pattern: row.pattern,
+        ...(row.ignore_patterns ? { ignore: JSON.parse(row.ignore_patterns) } : {}),
+        ...(row.include_by_default === 0 ? { includeByDefault: false } : {}),
+        ...(row.update_command ? { update: row.update_command } : {}),
+        ...(row.context ? { context: JSON.parse(row.context) } : {}),
+    };
+}
+export function getStoreCollections(db) {
+    const rows = db.prepare(`SELECT * FROM store_collections`).all();
+    return rows.map(rowToNamedCollection);
+}
+export function getStoreCollection(db, name) {
+    const row = db.prepare(`SELECT * FROM store_collections WHERE name = ?`).get(name);
+    if (row == null)
+        return null;
+    return rowToNamedCollection(row);
+}
+export function getStoreGlobalContext(db) {
+    const row = db.prepare(`SELECT value FROM store_config WHERE key = 'global_context'`).get();
+    if (row == null)
+        return undefined;
+    return row.value || undefined;
+}
+export function getStoreContexts(db) {
+    const results = [];
+    // Global context
+    const globalCtx = getStoreGlobalContext(db);
+    if (globalCtx) {
+        results.push({ collection: "*", path: "/", context: globalCtx });
+    }
+    // Collection contexts
+    const rows = db.prepare(`SELECT name, context FROM store_collections WHERE context IS NOT NULL`).all();
+    for (const row of rows) {
+        const ctxMap = JSON.parse(row.context);
+        for (const [path, context] of Object.entries(ctxMap)) {
+            results.push({ collection: row.name, path, context });
+        }
+    }
+    return results;
+}
+export function upsertStoreCollection(db, name, collection) {
+    db.prepare(`
+    INSERT INTO store_collections (name, path, pattern, ignore_patterns, include_by_default, update_command, context)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(name) DO UPDATE SET
+      path = excluded.path,
+      pattern = excluded.pattern,
+      ignore_patterns = excluded.ignore_patterns,
+      include_by_default = excluded.include_by_default,
+      update_command = excluded.update_command,
+      context = excluded.context
+  `).run(name, collection.path, collection.pattern || '**/*.md', collection.ignore ? JSON.stringify(collection.ignore) : null, collection.includeByDefault === false ? 0 : 1, collection.update || null, collection.context ? JSON.stringify(collection.context) : null);
+}
+export function deleteStoreCollection(db, name) {
+    const result = db.prepare(`DELETE FROM store_collections WHERE name = ?`).run(name);
+    return result.changes > 0;
+}
+export function renameStoreCollection(db, oldName, newName) {
+    // Check target doesn't exist
+    const existing = db.prepare(`SELECT name FROM store_collections WHERE name = ?`).get(newName);
+    if (existing != null) {
+        throw new Error(`Collection '${newName}' already exists`);
+    }
+    const result = db.prepare(`UPDATE store_collections SET name = ? WHERE name = ?`).run(newName, oldName);
+    return result.changes > 0;
+}
+export function updateStoreContext(db, collectionName, path, text) {
+    const row = db.prepare(`SELECT context FROM store_collections WHERE name = ?`).get(collectionName);
+    if (row == null)
+        return false;
+    const ctxMap = row.context ? JSON.parse(row.context) : {};
+    ctxMap[path] = text;
+    db.prepare(`UPDATE store_collections SET context = ? WHERE name = ?`).run(JSON.stringify(ctxMap), collectionName);
+    return true;
+}
+export function removeStoreContext(db, collectionName, path) {
+    const row = db.prepare(`SELECT context FROM store_collections WHERE name = ?`).get(collectionName);
+    if (row == null)
+        return false;
+    if (!row.context)
+        return false;
+    const ctxMap = JSON.parse(row.context);
+    if (!(path in ctxMap))
+        return false;
+    delete ctxMap[path];
+    const newCtx = Object.keys(ctxMap).length > 0 ? JSON.stringify(ctxMap) : null;
+    db.prepare(`UPDATE store_collections SET context = ? WHERE name = ?`).run(newCtx, collectionName);
+    return true;
+}
+export function setStoreGlobalContext(db, value) {
+    if (value === undefined) {
+        db.prepare(`DELETE FROM store_config WHERE key = 'global_context'`).run();
+    }
+    else {
+        db.prepare(`INSERT INTO store_config (key, value) VALUES ('global_context', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`).run(value);
+    }
+}
+/**
+ * Sync external config (YAML/inline) into SQLite store_collections.
+ * External config always wins. Skips sync if config hash hasn't changed.
+ */
+export function syncConfigToDb(db, config) {
+    // Check config hash — skip sync if unchanged
+    const configJson = JSON.stringify(config);
+    const hash = createHash('sha256').update(configJson).digest('hex');
+    const existingHash = db.prepare(`SELECT value FROM store_config WHERE key = 'config_hash'`).get();
+    if (existingHash != null && existingHash.value === hash) {
+        return; // Config unchanged, skip sync
+    }
+    // Sync collections
+    const configNames = new Set(Object.keys(config.collections));
+    for (const [name, coll] of Object.entries(config.collections)) {
+        upsertStoreCollection(db, name, coll);
+    }
+    // Delete collections not in config
+    const dbCollections = db.prepare(`SELECT name FROM store_collections`).all();
+    for (const row of dbCollections) {
+        if (!configNames.has(row.name)) {
+            db.prepare(`DELETE FROM store_collections WHERE name = ?`).run(row.name);
+        }
+    }
+    // Sync global context
+    if (config.global_context !== undefined) {
+        setStoreGlobalContext(db, config.global_context);
+    }
+    else {
+        setStoreGlobalContext(db, undefined);
+    }
+    // Save config hash
+    db.prepare(`INSERT INTO store_config (key, value) VALUES ('config_hash', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`).run(hash);
+}
 export function isSqliteVecAvailable() {
     return _sqliteVecAvailable === true;
 }
@@ -620,6 +782,199 @@ function ensureVecTableInternal(db, dimensions) {
     }
     db.exec(`CREATE VIRTUAL TABLE vectors_vec USING vec0(hash_seq TEXT PRIMARY KEY, embedding float[${dimensions}] distance_metric=cosine)`);
 }
+/**
+ * Re-index a single collection by scanning the filesystem and updating the database.
+ * Pure function — no console output, no db lifecycle management.
+ */
+export async function reindexCollection(store, collectionPath, globPattern, collectionName, options) {
+    const db = store.db;
+    const now = new Date().toISOString();
+    const excludeDirs = ["node_modules", ".git", ".cache", "vendor", "dist", "build"];
+    const allIgnore = [
+        ...excludeDirs.map(d => `**/${d}/**`),
+        ...(options?.ignorePatterns || []),
+    ];
+    const allFiles = await fastGlob(globPattern, {
+        cwd: collectionPath,
+        onlyFiles: true,
+        followSymbolicLinks: false,
+        dot: false,
+        ignore: allIgnore,
+    });
+    // Filter hidden files/folders
+    const files = allFiles.filter(file => {
+        const parts = file.split("/");
+        return !parts.some(part => part.startsWith("."));
+    });
+    const total = files.length;
+    let indexed = 0, updated = 0, unchanged = 0, processed = 0;
+    const seenPaths = new Set();
+    for (const relativeFile of files) {
+        const filepath = getRealPath(resolve(collectionPath, relativeFile));
+        const path = handelize(relativeFile);
+        seenPaths.add(path);
+        let content;
+        try {
+            content = readFileSync(filepath, "utf-8");
+        }
+        catch {
+            processed++;
+            options?.onProgress?.({ file: relativeFile, current: processed, total });
+            continue;
+        }
+        if (!content.trim()) {
+            processed++;
+            continue;
+        }
+        const hash = await hashContent(content);
+        const title = extractTitle(content, relativeFile);
+        const existing = findActiveDocument(db, collectionName, path);
+        if (existing) {
+            if (existing.hash === hash) {
+                if (existing.title !== title) {
+                    updateDocumentTitle(db, existing.id, title, now);
+                    updated++;
+                }
+                else {
+                    unchanged++;
+                }
+            }
+            else {
+                insertContent(db, hash, content, now);
+                const stat = statSync(filepath);
+                updateDocument(db, existing.id, title, hash, stat ? new Date(stat.mtime).toISOString() : now);
+                updated++;
+            }
+        }
+        else {
+            indexed++;
+            insertContent(db, hash, content, now);
+            const stat = statSync(filepath);
+            insertDocument(db, collectionName, path, title, hash, stat ? new Date(stat.birthtime).toISOString() : now, stat ? new Date(stat.mtime).toISOString() : now);
+        }
+        processed++;
+        options?.onProgress?.({ file: relativeFile, current: processed, total });
+    }
+    // Deactivate documents that no longer exist
+    const allActive = getActiveDocumentPaths(db, collectionName);
+    let removed = 0;
+    for (const path of allActive) {
+        if (!seenPaths.has(path)) {
+            deactivateDocument(db, collectionName, path);
+            removed++;
+        }
+    }
+    const orphanedCleaned = cleanupOrphanedContent(db);
+    return { indexed, updated, unchanged, removed, orphanedCleaned };
+}
+/**
+ * Generate vector embeddings for documents that need them.
+ * Pure function — no console output, no db lifecycle management.
+ * Uses the store's LlamaCpp instance if set, otherwise the global singleton.
+ */
+export async function generateEmbeddings(store, options) {
+    const db = store.db;
+    const model = options?.model ?? DEFAULT_EMBED_MODEL;
+    const now = new Date().toISOString();
+    if (options?.force) {
+        clearAllEmbeddings(db);
+    }
+    const hashesToEmbed = getHashesForEmbedding(db);
+    if (hashesToEmbed.length === 0) {
+        return { docsProcessed: 0, chunksEmbedded: 0, errors: 0, durationMs: 0 };
+    }
+    const allChunks = [];
+    for (const item of hashesToEmbed) {
+        const encoder = new TextEncoder();
+        const bodyBytes = encoder.encode(item.body).length;
+        if (bodyBytes === 0)
+            continue;
+        const title = extractTitle(item.body, item.path);
+        const chunks = await chunkDocumentByTokens(item.body);
+        for (let seq = 0; seq < chunks.length; seq++) {
+            allChunks.push({
+                hash: item.hash,
+                title,
+                text: chunks[seq].text,
+                seq,
+                pos: chunks[seq].pos,
+                tokens: chunks[seq].tokens,
+                bytes: encoder.encode(chunks[seq].text).length,
+            });
+        }
+    }
+    if (allChunks.length === 0) {
+        return { docsProcessed: 0, chunksEmbedded: 0, errors: 0, durationMs: 0 };
+    }
+    const totalBytes = allChunks.reduce((sum, chk) => sum + chk.bytes, 0);
+    const totalChunks = allChunks.length;
+    const totalDocs = hashesToEmbed.length;
+    const startTime = Date.now();
+    // Use store's LlamaCpp or global singleton, wrapped in a session
+    const llm = getLlm(store);
+    const sessionOptions = { maxDuration: 30 * 60 * 1000, name: 'generateEmbeddings' };
+    // Create a session manager for this llm instance
+    const result = await withLLMSessionForLlm(llm, async (session) => {
+        // Get embedding dimensions from first chunk
+        const firstChunk = allChunks[0];
+        const firstText = formatDocForEmbedding(firstChunk.text, firstChunk.title);
+        const firstResult = await session.embed(firstText);
+        if (!firstResult) {
+            throw new Error("Failed to get embedding dimensions from first chunk");
+        }
+        store.ensureVecTable(firstResult.embedding.length);
+        let chunksEmbedded = 0, errors = 0, bytesProcessed = 0;
+        const BATCH_SIZE = 32;
+        for (let batchStart = 0; batchStart < allChunks.length; batchStart += BATCH_SIZE) {
+            const batchEnd = Math.min(batchStart + BATCH_SIZE, allChunks.length);
+            const batch = allChunks.slice(batchStart, batchEnd);
+            const texts = batch.map(chunk => formatDocForEmbedding(chunk.text, chunk.title));
+            try {
+                const embeddings = await session.embedBatch(texts);
+                for (let i = 0; i < batch.length; i++) {
+                    const chunk = batch[i];
+                    const embedding = embeddings[i];
+                    if (embedding) {
+                        insertEmbedding(db, chunk.hash, chunk.seq, chunk.pos, new Float32Array(embedding.embedding), model, now);
+                        chunksEmbedded++;
+                    }
+                    else {
+                        errors++;
+                    }
+                    bytesProcessed += chunk.bytes;
+                }
+            }
+            catch {
+                // Batch failed — try individual embeddings as fallback
+                for (const chunk of batch) {
+                    try {
+                        const text = formatDocForEmbedding(chunk.text, chunk.title);
+                        const result = await session.embed(text);
+                        if (result) {
+                            insertEmbedding(db, chunk.hash, chunk.seq, chunk.pos, new Float32Array(result.embedding), model, now);
+                            chunksEmbedded++;
+                        }
+                        else {
+                            errors++;
+                        }
+                    }
+                    catch {
+                        errors++;
+                    }
+                    bytesProcessed += chunk.bytes;
+                }
+            }
+            options?.onProgress?.({ chunksEmbedded, totalChunks, bytesProcessed, totalBytes, errors });
+        }
+        return { chunksEmbedded, errors };
+    }, sessionOptions);
+    return {
+        docsProcessed: totalDocs,
+        chunksEmbedded: result.chunksEmbedded,
+        errors: result.errors,
+        durationMs: Date.now() - startTime,
+    };
+}
 /**
  * Create a new store instance with the given database path.
  * If no path is provided, uses the default path (~/.cache/qmd/index.sqlite).
@@ -631,7 +986,7 @@ export function createStore(dbPath) {
     const resolvedPath = dbPath || getDefaultDbPath();
     const db = openDatabase(resolvedPath);
     initializeDatabase(db);
-    return {
+    const store = {
         db,
         dbPath: resolvedPath,
         close: () => db.close(),
@@ -667,8 +1022,8 @@ export function createStore(dbPath) {
         searchFTS: (query, limit, collectionName) => searchFTS(db, query, limit, collectionName),
         searchVec: (query, model, limit, collectionName, session, precomputedEmbedding) => searchVec(db, query, model, limit, collectionName, session, precomputedEmbedding),
         // Query expansion & reranking
-        expandQuery: (query, model, intent) => expandQuery(query, model, db, intent),
-        rerank: (query, documents, model, intent) => rerank(query, documents, model, db, intent),
+        expandQuery: (query, model, intent) => expandQuery(query, model, db, intent, store.llm),
+        rerank: (query, documents, model, intent) => rerank(query, documents, model, db, intent, store.llm),
         // Document retrieval
         findDocument: (filename, options) => findDocument(db, filename, options),
         getDocumentBody: (doc, fromLine, maxLines) => getDocumentBody(db, doc, fromLine, maxLines),
@@ -690,6 +1045,7 @@ export function createStore(dbPath) {
         clearAllEmbeddings: () => clearAllEmbeddings(db),
         insertEmbedding: (hash, seq, pos, embedding, model, embeddedAt) => insertEmbedding(db, hash, seq, pos, embedding, model, embeddedAt),
     };
+    return store;
 }
 /**
  * Extract short docid from a full hash (first 6 characters).
@@ -1188,15 +1544,15 @@ export function matchFilesByGlob(db, pattern) {
  * @returns Context string or null if no context is defined
  */
 export function getContextForPath(db, collectionName, path) {
-    const config = collectionsLoadConfig();
-    const coll = getCollection(collectionName);
+    const coll = getStoreCollection(db, collectionName);
     if (!coll)
         return null;
     // Collect ALL matching contexts (global + all path prefixes)
     const contexts = [];
     // Add global context if present
-    if (config.global_context) {
-        contexts.push(config.global_context);
+    const globalCtx = getStoreGlobalContext(db);
+    if (globalCtx) {
+        contexts.push(globalCtx);
     }
     // Add all matching path contexts (from most general to most specific)
     if (coll.context) {
@@ -1221,15 +1577,14 @@ export function getContextForPath(db, collectionName, path) {
 }
 /**
  * Get context for a file path (virtual or filesystem).
- * Resolves the collection and relative path using the YAML collections config.
+ * Resolves the collection and relative path from the DB store_collections table.
  */
 export function getContextForFile(db, filepath) {
     // Handle undefined or null filepath
     if (!filepath)
         return null;
-    // Get all collections from YAML config
-    const collections = collectionsListCollections();
-    const config = collectionsLoadConfig();
+    // Get all collections from DB
+    const collections = getStoreCollections(db);
     // Parse virtual path format: qmd://collection/path
     let collectionName = null;
     let relativePath = null;
@@ -1256,8 +1611,8 @@ export function getContextForFile(db, filepath) {
         if (!collectionName || relativePath === null)
             return null;
     }
-    // Get the collection from config
-    const coll = getCollection(collectionName);
+    // Get the collection from DB
+    const coll = getStoreCollection(db, collectionName);
     if (!coll)
         return null;
     // Verify this document exists in the database
@@ -1272,8 +1627,9 @@ export function getContextForFile(db, filepath) {
     // Collect ALL matching contexts (global + all path prefixes)
     const contexts = [];
     // Add global context if present
-    if (config.global_context) {
-        contexts.push(config.global_context);
+    const globalCtx = getStoreGlobalContext(db);
+    if (globalCtx) {
+        contexts.push(globalCtx);
     }
     // Add all matching path contexts (from most general to most specific)
     if (coll.context) {
@@ -1297,11 +1653,10 @@ export function getContextForFile(db, filepath) {
     return contexts.length > 0 ? contexts.join('\n\n') : null;
 }
 /**
- * Get collection by name from YAML config.
- * Returns collection metadata from ~/.config/qmd/index.yml
+ * Get collection by name from DB store_collections table.
  */
 export function getCollectionByName(db, name) {
-    const collection = getCollection(name);
+    const collection = getStoreCollection(db, name);
     if (!collection)
         return null;
     return {
@@ -1312,10 +1667,10 @@ export function getCollectionByName(db, name) {
 }
 /**
  * List all collections with document counts from database.
- * Merges YAML config with database statistics.
+ * Merges store_collections config with database statistics.
  */
 export function listCollections(db) {
-    const collections = collectionsListCollections();
+    const collections = getStoreCollections(db);
     // Get document counts from database for each collection
     const result = collections.map(coll => {
         const stats = db.prepare(`
@@ -1333,6 +1688,7 @@ export function listCollections(db) {
             doc_count: stats?.doc_count || 0,
             active_count: stats?.active_count || 0,
             last_modified: stats?.last_modified || null,
+            includeByDefault: coll.includeByDefault !== false,
         };
     });
     return result;
@@ -1349,8 +1705,8 @@ export function removeCollection(db, collectionName) {
     DELETE FROM content
     WHERE hash NOT IN (SELECT DISTINCT hash FROM documents WHERE active = 1)
   `).run();
-    // Remove from YAML config (returns true if found and removed)
-    collectionsRemoveCollection(collectionName);
+    // Remove from store_collections
+    deleteStoreCollection(db, collectionName);
     return {
         deletedDocs: docResult.changes,
         cleanedHashes: cleanupResult.changes
@@ -1364,8 +1720,8 @@ export function renameCollection(db, oldName, newName) {
     // Update all documents with the new collection name in database
     db.prepare(`UPDATE documents SET collection = ? WHERE collection = ?`)
         .run(newName, oldName);
-    // Rename in YAML config
-    collectionsRenameCollection(oldName, newName);
+    // Rename in store_collections
+    renameStoreCollection(db, oldName, newName);
 }
 // =============================================================================
 // Context Management Operations
@@ -1379,16 +1735,16 @@ export function insertContext(db, collectionId, pathPrefix, context) {
     if (!coll) {
         throw new Error(`Collection with id ${collectionId} not found`);
     }
-    // Use collections.ts to add context
-    collectionsAddContext(coll.name, pathPrefix, context);
+    // Add context to store_collections
+    updateStoreContext(db, coll.name, pathPrefix, context);
 }
 /**
  * Delete a context for a specific collection and path prefix.
  * Returns the number of contexts deleted.
  */
 export function deleteContext(db, collectionName, pathPrefix) {
-    // Use collections.ts to remove context
-    const success = collectionsRemoveContext(collectionName, pathPrefix);
+    // Remove context from store_collections
+    const success = removeStoreContext(db, collectionName, pathPrefix);
     return success ? 1 : 0;
 }
 /**
@@ -1398,12 +1754,12 @@ export function deleteContext(db, collectionName, pathPrefix) {
 export function deleteGlobalContexts(db) {
     let deletedCount = 0;
     // Remove global context
-    setGlobalContext(undefined);
+    setStoreGlobalContext(db, undefined);
     deletedCount++;
     // Remove root context (empty string) from all collections
-    const collections = collectionsListCollections();
+    const collections = getStoreCollections(db);
     for (const coll of collections) {
-        const success = collectionsRemoveContext(coll.name, '');
+        const success = removeStoreContext(db, coll.name, '');
         if (success) {
             deletedCount++;
         }
@@ -1415,7 +1771,7 @@ export function deleteGlobalContexts(db) {
  * Returns contexts ordered by collection name, then by path prefix length (longest first).
  */
 export function listPathContexts(db) {
-    const allContexts = collectionsListAllContexts();
+    const allContexts = getStoreContexts(db);
     // Convert to expected format and sort
     return allContexts.map(ctx => ({
         collection_name: ctx.collection,
@@ -1438,7 +1794,7 @@ export function listPathContexts(db) {
  * Get all collections (name only - from YAML config).
  */
 export function getAllCollections(db) {
-    const collections = collectionsListCollections();
+    const collections = getStoreCollections(db);
     return collections.map(c => ({ name: c.name }));
 }
 /**
@@ -1446,11 +1802,11 @@ export function getAllCollections(db) {
  * Returns collections that have no context entries at all (not even root context).
  */
 export function getCollectionsWithoutContext(db) {
-    // Get all collections from YAML config
-    const yamlCollections = collectionsListCollections();
+    // Get all collections from DB
+    const allCollections = getStoreCollections(db);
     // Filter to those without context
     const collectionsWithoutContext = [];
-    for (const coll of yamlCollections) {
+    for (const coll of allCollections) {
         // Check if collection has any context
         if (!coll.context || Object.keys(coll.context).length === 0) {
             // Get doc count from database
@@ -1478,13 +1834,13 @@ export function getTopLevelPathsWithoutContext(db, collectionName) {
     SELECT DISTINCT path FROM documents
     WHERE collection = ? AND active = 1
   `).all(collectionName);
-    // Get existing contexts for this collection from YAML
-    const yamlColl = getCollection(collectionName);
-    if (!yamlColl)
+    // Get existing contexts for this collection from DB
+    const dbColl = getStoreCollection(db, collectionName);
+    if (!dbColl)
         return [];
     const contextPrefixes = new Set();
-    if (yamlColl.context) {
-        for (const prefix of Object.keys(yamlColl.context)) {
+    if (dbColl.context) {
+        for (const prefix of Object.keys(dbColl.context)) {
             contextPrefixes.add(prefix);
         }
     }
@@ -1754,12 +2110,12 @@ export async function searchVec(db, query, model, limit = 20, collectionName, se
 // =============================================================================
 // Embeddings
 // =============================================================================
-async function getEmbedding(text, model, isQuery, session) {
+async function getEmbedding(text, model, isQuery, session, llmOverride) {
     // Format text using the appropriate prompt template
     const formattedText = isQuery ? formatQueryForEmbedding(text, model) : formatDocForEmbedding(text, undefined, model);
     const result = session
         ? await session.embed(formattedText, { model, isQuery })
-        : await getDefaultLlamaCpp().embed(formattedText, { model, isQuery });
+        : await (llmOverride ?? getDefaultLlamaCpp()).embed(formattedText, { model, isQuery });
     return result?.embedding || null;
 }
 /**
@@ -1798,26 +2154,33 @@ export function insertEmbedding(db, hash, seq, pos, embedding, model, embeddedAt
 // =============================================================================
 // Query expansion
 // =============================================================================
-export async function expandQuery(query, model = DEFAULT_QUERY_MODEL, db, intent) {
+export async function expandQuery(query, model = DEFAULT_QUERY_MODEL, db, intent, llmOverride) {
     // Check cache first — stored as JSON preserving types
     const cacheKey = getCacheKey("expandQuery", { query, model, ...(intent && { intent }) });
     const cached = getCachedResult(db, cacheKey);
     if (cached) {
         try {
-            return JSON.parse(cached);
+            const parsed = JSON.parse(cached);
+            // Migrate old cache format: { type, text } → { type, query }
+            if (parsed.length > 0 && parsed[0].query) {
+                return parsed;
+            }
+            else if (parsed.length > 0 && parsed[0].text) {
+                return parsed.map((r) => ({ type: r.type, query: r.text }));
+            }
         }
         catch {
             // Old cache format (pre-typed, newline-separated text) — re-expand
         }
     }
-    const llm = getDefaultLlamaCpp();
+    const llm = llmOverride ?? getDefaultLlamaCpp();
     // Note: LlamaCpp uses hardcoded model, model parameter is ignored
     const results = await llm.expandQuery(query, { intent });
     // Map Queryable[] → ExpandedQuery[] (same shape, decoupled from llm.ts internals).
     // Filter out entries that duplicate the original query text.
     const expanded = results
         .filter(r => r.text !== query)
-        .map(r => ({ type: r.type, text: r.text }));
+        .map(r => ({ type: r.type, query: r.text }));
     if (expanded.length > 0) {
         setCachedResult(db, cacheKey, JSON.stringify(expanded));
     }
@@ -1826,7 +2189,7 @@ export async function expandQuery(query, model = DEFAULT_QUERY_MODEL, db, intent
 // =============================================================================
 // Reranking
 // =============================================================================
-export async function rerank(query, documents, model = DEFAULT_RERANK_MODEL, db, intent) {
+export async function rerank(query, documents, model = DEFAULT_RERANK_MODEL, db, intent, llmOverride) {
     // Prepend intent to rerank query so the reranker scores with domain context
     const rerankQuery = intent ? `${intent}\n\n${query}` : query;
     const cachedResults = new Map();
@@ -1849,7 +2212,7 @@ export async function rerank(query, documents, model = DEFAULT_RERANK_MODEL, db,
     }
     // Rerank uncached documents using LlamaCpp
     if (uncachedDocsByChunk.size > 0) {
-        const llm = getDefaultLlamaCpp();
+        const llm = llmOverride ?? getDefaultLlamaCpp();
         const uncachedDocs = [...uncachedDocsByChunk.values()];
         const rerankResult = await llm.rerank(rerankQuery, uncachedDocs, { model });
         // Cache results by chunk text so identical chunks across files are scored once.
@@ -2026,9 +2389,9 @@ export function findDocument(db, filename, options = {}) {
       LIMIT 1
     `).get(`%${filepath}`);
     }
-    // Try to match by absolute path (requires looking up collection paths from YAML)
+    // Try to match by absolute path (requires looking up collection paths from DB)
     if (!doc && !filepath.startsWith('qmd://')) {
-        const collections = collectionsListCollections();
+        const collections = getStoreCollections(db);
         for (const coll of collections) {
             let relativePath = null;
             // If filepath is absolute and starts with collection path, extract relative part
@@ -2088,9 +2451,9 @@ export function getDocumentBody(db, doc, fromLine, maxLines) {
       WHERE 'qmd://' || d.collection || '/' || d.path = ? AND d.active = 1
     `).get(filepath);
     }
-    // Try absolute path by looking up in YAML collections
+    // Try absolute path by looking up in DB store_collections
     if (!row) {
-        const collections = collectionsListCollections();
+        const collections = getStoreCollections(db);
         for (const coll of collections) {
             if (filepath.startsWith(coll.path + '/')) {
                 const relativePath = filepath.slice(coll.path.length + 1);
@@ -2219,23 +2582,27 @@ export function findDocuments(db, pattern, options = {}) {
 // Status
 // =============================================================================
 export function getStatus(db) {
-    // Load collections from YAML
-    const yamlCollections = collectionsListCollections();
-    // Get document counts and last update times for each collection
-    const collections = yamlCollections.map(col => {
-        const stats = db.prepare(`
-      SELECT
-        COUNT(*) as active_count,
-        MAX(modified_at) as last_doc_update
-      FROM documents
-      WHERE collection = ? AND active = 1
-    `).get(col.name);
+    // DB is source of truth for collections — config provides supplementary metadata
+    const dbCollections = db.prepare(`
+    SELECT
+      collection as name,
+      COUNT(*) as active_count,
+      MAX(modified_at) as last_doc_update
+    FROM documents
+    WHERE active = 1
+    GROUP BY collection
+  `).all();
+    // Build a lookup from store_collections for path/pattern metadata
+    const storeCollections = getStoreCollections(db);
+    const configLookup = new Map(storeCollections.map(c => [c.name, { path: c.path, pattern: c.pattern }]));
+    const collections = dbCollections.map(row => {
+        const config = configLookup.get(row.name);
         return {
-            name: col.name,
-            path: col.path,
-            pattern: col.pattern,
-            documents: stats.active_count,
-            lastUpdated: stats.last_doc_update || new Date().toISOString(),
+            name: row.name,
+            path: config?.path ?? null,
+            pattern: config?.pattern ?? null,
+            documents: row.active_count,
+            lastUpdated: row.last_doc_update || new Date().toISOString(),
         };
     });
     // Sort by last update time (most recent first)
@@ -2382,6 +2749,7 @@ export async function hybridQuery(store, query, options) {
     const collection = options?.collection;
     const explain = options?.explain ?? false;
     const intent = options?.intent;
+    const skipRerank = options?.skipRerank ?? false;
     const hooks = options?.hooks;
     const rankedLists = [];
     const rankedListMeta = [];
@@ -2425,7 +2793,7 @@ export async function hybridQuery(store, query, options) {
     // 3a: Run FTS for all lex expansions right away (no LLM needed)
     for (const q of expanded) {
         if (q.type === 'lex') {
-            const ftsResults = store.searchFTS(q.text, 20, collection);
+            const ftsResults = store.searchFTS(q.query, 20, collection);
             if (ftsResults.length > 0) {
                 for (const r of ftsResults)
                     docidMap.set(r.filepath, r.docid);
@@ -2433,7 +2801,7 @@ export async function hybridQuery(store, query, options) {
                     file: r.filepath, displayPath: r.displayPath,
                     title: r.title, body: r.body || "", score: r.score,
                 })));
-                rankedListMeta.push({ source: "fts", queryType: "lex", query: q.text });
+                rankedListMeta.push({ source: "fts", queryType: "lex", query: q.query });
             }
         }
     }
@@ -2444,11 +2812,11 @@ export async function hybridQuery(store, query, options) {
         ];
         for (const q of expanded) {
             if (q.type === 'vec' || q.type === 'hyde') {
-                vecQueries.push({ text: q.text, queryType: q.type });
+                vecQueries.push({ text: q.query, queryType: q.type });
             }
         }
         // Batch embed all vector queries in a single call
-        const llm = getDefaultLlamaCpp();
+        const llm = getLlm(store);
         const textsToEmbed = vecQueries.map(q => formatQueryForEmbedding(q.text));
         hooks?.onEmbedStart?.(textsToEmbed.length);
         const embedStart = Date.now();
@@ -2486,7 +2854,6 @@ export async function hybridQuery(store, query, options) {
     // Reranking full bodies is O(tokens) — the critical perf lesson that motivated this refactor.
     const queryTerms = query.toLowerCase().split(/\s+/).filter(t => t.length > 2);
     const intentTerms = intent ? extractIntentTerms(intent) : [];
-    const chunksToRerank = [];
     const docChunkMap = new Map();
     for (const cand of candidates) {
         const chunks = chunkDocument(cand.body);
@@ -2508,10 +2875,65 @@ export async function hybridQuery(store, query, options) {
                 bestIdx = i;
             }
         }
-        chunksToRerank.push({ file: cand.file, text: chunks[bestIdx].text });
         docChunkMap.set(cand.file, { chunks, bestIdx });
     }
+    if (skipRerank) {
+        // Skip LLM reranking — return candidates scored by RRF only
+        const seenFiles = new Set();
+        return candidates
+            .map((cand, i) => {
+            const chunkInfo = docChunkMap.get(cand.file);
+            const bestIdx = chunkInfo?.bestIdx ?? 0;
+            const bestChunk = chunkInfo?.chunks[bestIdx]?.text || cand.body || "";
+            const bestChunkPos = chunkInfo?.chunks[bestIdx]?.pos || 0;
+            const rrfRank = i + 1;
+            const rrfScore = 1 / rrfRank;
+            const trace = rrfTraceByFile?.get(cand.file);
+            const explainData = explain ? {
+                ftsScores: trace?.contributions.filter(c => c.source === "fts").map(c => c.backendScore) ?? [],
+                vectorScores: trace?.contributions.filter(c => c.source === "vec").map(c => c.backendScore) ?? [],
+                rrf: {
+                    rank: rrfRank,
+                    positionScore: rrfScore,
+                    weight: 1.0,
+                    baseScore: trace?.baseScore ?? 0,
+                    topRankBonus: trace?.topRankBonus ?? 0,
+                    totalScore: trace?.totalScore ?? 0,
+                    contributions: trace?.contributions ?? [],
+                },
+                rerankScore: 0,
+                blendedScore: rrfScore,
+            } : undefined;
+            return {
+                file: cand.file,
+                displayPath: cand.displayPath,
+                title: cand.title,
+                body: cand.body,
+                bestChunk,
+                bestChunkPos,
+                score: rrfScore,
+                context: store.getContextForFile(cand.file),
+                docid: docidMap.get(cand.file) || "",
+                ...(explainData ? { explain: explainData } : {}),
+            };
+        })
+            .filter(r => {
+            if (seenFiles.has(r.file))
+                return false;
+            seenFiles.add(r.file);
+            return true;
+        })
+            .filter(r => r.score >= minScore)
+            .slice(0, limit);
+    }
     // Step 6: Rerank chunks (NOT full bodies)
+    const chunksToRerank = [];
+    for (const cand of candidates) {
+        const chunkInfo = docChunkMap.get(cand.file);
+        if (chunkInfo) {
+            chunksToRerank.push({ file: cand.file, text: chunkInfo.chunks[chunkInfo.bestIdx].text });
+        }
+    }
     hooks?.onRerankStart?.(chunksToRerank.length);
     const rerankStart = Date.now();
     const reranked = await store.rerank(query, chunksToRerank, undefined, intent);
@@ -2602,7 +3024,7 @@ export async function vectorSearchQuery(store, query, options) {
     const vecExpanded = allExpanded.filter(q => q.type !== 'lex');
     options?.hooks?.onExpand?.(query, vecExpanded, Date.now() - expandStart);
     // Run original + vec/hyde expanded through vector, sequentially — concurrent embed() hangs
-    const queryTexts = [query, ...vecExpanded.map(q => q.text)];
+    const queryTexts = [query, ...vecExpanded.map(q => q.query)];
     const allResults = new Map();
     for (const q of queryTexts) {
         const vecResults = await store.searchVec(q, DEFAULT_EMBED_MODEL, limit, collection);
@@ -2650,6 +3072,7 @@ export async function structuredSearch(store, searches, options) {
     const candidateLimit = options?.candidateLimit ?? RERANK_CANDIDATE_LIMIT;
     const explain = options?.explain ?? false;
     const intent = options?.intent;
+    const skipRerank = options?.skipRerank ?? false;
     const hooks = options?.hooks;
     const collections = options?.collections;
     if (searches.length === 0)
@@ -2704,7 +3127,7 @@ export async function structuredSearch(store, searches, options) {
     if (hasVectors) {
         const vecSearches = searches.filter((s) => s.type === 'vec' || s.type === 'hyde');
         if (vecSearches.length > 0) {
-            const llm = getDefaultLlamaCpp();
+            const llm = getLlm(store);
             const textsToEmbed = vecSearches.map(s => formatQueryForEmbedding(s.query));
             hooks?.onEmbedStart?.(textsToEmbed.length);
             const embedStart = Date.now();
@@ -2750,7 +3173,6 @@ export async function structuredSearch(store, searches, options) {
         || searches[0]?.query || "";
     const queryTerms = primaryQuery.toLowerCase().split(/\s+/).filter(t => t.length > 2);
     const intentTerms = intent ? extractIntentTerms(intent) : [];
-    const chunksToRerank = [];
     const docChunkMap = new Map();
     for (const cand of candidates) {
         const chunks = chunkDocument(cand.body);
@@ -2772,10 +3194,65 @@ export async function structuredSearch(store, searches, options) {
                 bestIdx = i;
             }
         }
-        chunksToRerank.push({ file: cand.file, text: chunks[bestIdx].text });
         docChunkMap.set(cand.file, { chunks, bestIdx });
     }
+    if (skipRerank) {
+        // Skip LLM reranking — return candidates scored by RRF only
+        const seenFiles = new Set();
+        return candidates
+            .map((cand, i) => {
+            const chunkInfo = docChunkMap.get(cand.file);
+            const bestIdx = chunkInfo?.bestIdx ?? 0;
+            const bestChunk = chunkInfo?.chunks[bestIdx]?.text || cand.body || "";
+            const bestChunkPos = chunkInfo?.chunks[bestIdx]?.pos || 0;
+            const rrfRank = i + 1;
+            const rrfScore = 1 / rrfRank;
+            const trace = rrfTraceByFile?.get(cand.file);
+            const explainData = explain ? {
+                ftsScores: trace?.contributions.filter(c => c.source === "fts").map(c => c.backendScore) ?? [],
+                vectorScores: trace?.contributions.filter(c => c.source === "vec").map(c => c.backendScore) ?? [],
+                rrf: {
+                    rank: rrfRank,
+                    positionScore: rrfScore,
+                    weight: 1.0,
+                    baseScore: trace?.baseScore ?? 0,
+                    topRankBonus: trace?.topRankBonus ?? 0,
+                    totalScore: trace?.totalScore ?? 0,
+                    contributions: trace?.contributions ?? [],
+                },
+                rerankScore: 0,
+                blendedScore: rrfScore,
+            } : undefined;
+            return {
+                file: cand.file,
+                displayPath: cand.displayPath,
+                title: cand.title,
+                body: cand.body,
+                bestChunk,
+                bestChunkPos,
+                score: rrfScore,
+                context: store.getContextForFile(cand.file),
+                docid: docidMap.get(cand.file) || "",
+                ...(explainData ? { explain: explainData } : {}),
+            };
+        })
+            .filter(r => {
+            if (seenFiles.has(r.file))
+                return false;
+            seenFiles.add(r.file);
+            return true;
+        })
+            .filter(r => r.score >= minScore)
+            .slice(0, limit);
+    }
     // Step 5: Rerank chunks
+    const chunksToRerank = [];
+    for (const cand of candidates) {
+        const chunkInfo = docChunkMap.get(cand.file);
+        if (chunkInfo) {
+            chunksToRerank.push({ file: cand.file, text: chunkInfo.chunks[chunkInfo.bestIdx].text });
+        }
+    }
     hooks?.onRerankStart?.(chunksToRerank.length);
     const rerankStart2 = Date.now();
     const reranked = await store.rerank(primaryQuery, chunksToRerank, undefined, intent);