@tobilu/qmd 1.1.6 → 2.0.1

package/dist/maintenance.js

@@ -0,0 +1,37 @@
+/**
+ * Maintenance - Database cleanup operations for QMD.
+ *
+ * Wraps low-level store operations that the CLI needs for housekeeping.
+ * Takes an internal Store in the constructor — allowed to access DB directly.
+ */
+import { vacuumDatabase, cleanupOrphanedContent, cleanupOrphanedVectors, deleteLLMCache, deleteInactiveDocuments, clearAllEmbeddings, } from "./store.js";
+export class Maintenance {
+    store;
+    constructor(store) {
+        this.store = store;
+    }
+    /** Run VACUUM on the SQLite database to reclaim space */
+    vacuum() {
+        vacuumDatabase(this.store.db);
+    }
+    /** Remove content rows that are no longer referenced by any document */
+    cleanupOrphanedContent() {
+        return cleanupOrphanedContent(this.store.db);
+    }
+    /** Remove vector embeddings for content that no longer exists */
+    cleanupOrphanedVectors() {
+        return cleanupOrphanedVectors(this.store.db);
+    }
+    /** Clear the LLM response cache (query expansion, reranking) */
+    clearLLMCache() {
+        return deleteLLMCache(this.store.db);
+    }
+    /** Delete documents marked as inactive (removed from filesystem) */
+    deleteInactiveDocs() {
+        return deleteInactiveDocuments(this.store.db);
+    }
+    /** Clear all vector embeddings (forces re-embedding) */
+    clearEmbeddings() {
+        clearAllEmbeddings(this.store.db);
+    }
+}

package/dist/{mcp.js → mcp/server.js}

@@ -14,9 +14,7 @@ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"
 import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
 import { isInitializeRequest } from "@modelcontextprotocol/sdk/types.js";
 import { z } from "zod";
-import { createStore, extractSnippet, addLineNumbers, structuredSearch, DEFAULT_MULTI_GET_MAX_BYTES, } from "./store.js";
-import { getCollection, getGlobalContext, getDefaultCollectionNames } from "./collections.js";
-import { disposeDefaultLlamaCpp } from "./llm.js";
+import { createStore, extractSnippet, addLineNumbers, getDefaultDbPath, DEFAULT_MULTI_GET_MAX_BYTES, } from "../index.js";
 // =============================================================================
 // Helper functions
 // =============================================================================
@@ -49,11 +47,12 @@ function formatSearchSummary(results, query) {
  * Injected into the LLM's system prompt via MCP initialize response —
  * gives the LLM immediate context about what's searchable without a tool call.
  */
-function buildInstructions(store) {
-    const status = store.getStatus();
+async function buildInstructions(store) {
+    const status = await store.getStatus();
+    const contexts = await store.listContexts();
+    const globalCtx = await store.getGlobalContext();
     const lines = [];
     // --- What is this? ---
-    const globalCtx = getGlobalContext();
     lines.push(`QMD is your local search engine over ${status.totalDocuments} markdown documents.`);
     if (globalCtx)
         lines.push(`Context: ${globalCtx}`);
@@ -62,9 +61,9 @@ function buildInstructions(store) {
         lines.push("");
         lines.push("Collections (scope with `collection` parameter):");
         for (const col of status.collections) {
-            const collConfig = getCollection(col.name);
-            const rootCtx = collConfig?.context?.[""] || collConfig?.context?.["/"];
-            const desc = rootCtx ? ` — ${rootCtx}` : "";
+            // Find root context for this collection
+            const rootCtx = contexts.find(c => c.collection === col.name && (c.path === "" || c.path === "/"));
+            const desc = rootCtx ? ` — ${rootCtx.context}` : "";
             lines.push(`  - "${col.name}" (${col.documents} docs)${desc}`);
         }
     }
@@ -108,8 +107,10 @@ function buildInstructions(store) {
  * Create an MCP server with all QMD tools, resources, and prompts registered.
  * Shared by both stdio and HTTP transports.
  */
-function createMcpServer(store) {
-    const server = new McpServer({ name: "qmd", version: "0.9.9" }, { instructions: buildInstructions(store) });
+async function createMcpServer(store) {
+    const server = new McpServer({ name: "qmd", version: "0.9.9" }, { instructions: await buildInstructions(store) });
+    // Pre-fetch default collection names for search tools
+    const defaultCollectionNames = await store.getDefaultCollectionNames();
     // ---------------------------------------------------------------------------
     // Resource: qmd://{path} - read-only access to documents by path
     // Note: No list() - documents are discovered via search tools
@@ -122,43 +123,20 @@ function createMcpServer(store) {
         // Decode URL-encoded path (MCP clients send encoded URIs)
         const pathStr = Array.isArray(path) ? path.join('/') : (path || '');
         const decodedPath = decodeURIComponent(pathStr);
-        // Parse virtual path: collection/relative/path
-        const parts = decodedPath.split('/');
-        const collection = parts[0] || '';
-        const relativePath = parts.slice(1).join('/');
-        // Find document by collection and path, join with content table
-        let doc = store.db.prepare(`
-        SELECT d.collection, d.path, d.title, c.doc as body
-        FROM documents d
-        JOIN content c ON c.hash = d.hash
-        WHERE d.collection = ? AND d.path = ? AND d.active = 1
-      `).get(collection, relativePath);
-        // Try suffix match if exact match fails
-        if (!doc) {
-            doc = store.db.prepare(`
-          SELECT d.collection, d.path, d.title, c.doc as body
-          FROM documents d
-          JOIN content c ON c.hash = d.hash
-          WHERE d.path LIKE ? AND d.active = 1
-          LIMIT 1
-        `).get(`%${relativePath}`);
-        }
-        if (!doc) {
+        // Use SDK to find document — findDocument handles collection/path resolution
+        const result = await store.get(decodedPath, { includeBody: true });
+        if ("error" in result) {
             return { contents: [{ uri: uri.href, text: `Document not found: ${decodedPath}` }] };
         }
-        // Construct virtual path for context lookup
-        const virtualPath = `qmd://${doc.collection}/${doc.path}`;
-        const context = store.getContextForFile(virtualPath);
-        let text = addLineNumbers(doc.body); // Default to line numbers
-        if (context) {
-            text = `<!-- Context: ${context} -->\n\n` + text;
+        let text = addLineNumbers(result.body || ""); // Default to line numbers
+        if (result.context) {
+            text = `<!-- Context: ${result.context} -->\n\n` + text;
         }
-        const displayName = `${doc.collection}/${doc.path}`;
         return {
             contents: [{
                     uri: uri.href,
-                    name: displayName,
-                    title: doc.title || doc.path,
+                    name: result.displayPath,
+                    title: result.title || result.displayPath,
                     mimeType: "text/markdown",
                     text,
                 }],
@@ -243,17 +221,17 @@ Intent-aware lex (C++ performance, not sports):
         },
     }, async ({ searches, limit, minScore, candidateLimit, collections, intent }) => {
         // Map to internal format
-        const subSearches = searches.map(s => ({
+        const queries = searches.map(s => ({
             type: s.type,
             query: s.query,
         }));
         // Use default collections if none specified
-        const effectiveCollections = collections ?? getDefaultCollectionNames();
-        const results = await structuredSearch(store, subSearches, {
+        const effectiveCollections = collections ?? defaultCollectionNames;
+        const results = await store.search({
+            queries,
             collections: effectiveCollections.length > 0 ? effectiveCollections : undefined,
             limit,
             minScore,
-            candidateLimit,
             intent,
         });
         // Use first lex or vec query for snippet extraction
@@ -298,7 +276,7 @@ Intent-aware lex (C++ performance, not sports):
             parsedFromLine = parseInt(colonMatch[1], 10);
             lookup = lookup.slice(0, -colonMatch[0].length);
         }
-        const result = store.findDocument(lookup, { includeBody: false });
+        const result = await store.get(lookup, { includeBody: false });
         if ("error" in result) {
             let msg = `Document not found: ${file}`;
             if (result.similarFiles.length > 0) {
@@ -309,7 +287,7 @@ Intent-aware lex (C++ performance, not sports):
                 isError: true,
             };
         }
-        const body = store.getDocumentBody(result, parsedFromLine, maxLines) ?? "";
+        const body = await store.getDocumentBody(result.filepath, { fromLine: parsedFromLine, maxLines }) ?? "";
         let text = body;
         if (lineNumbers) {
             const startLine = parsedFromLine || 1;
@@ -345,7 +323,7 @@ Intent-aware lex (C++ performance, not sports):
             lineNumbers: z.boolean().optional().default(false).describe("Add line numbers to output (format: 'N: content')"),
         },
     }, async ({ pattern, maxLines, maxBytes, lineNumbers }) => {
-        const { docs, errors } = store.findDocuments(pattern, { includeBody: true, maxBytes: maxBytes || DEFAULT_MULTI_GET_MAX_BYTES });
+        const { docs, errors } = await store.multiGet(pattern, { includeBody: true, maxBytes: maxBytes || DEFAULT_MULTI_GET_MAX_BYTES });
         if (docs.length === 0 && errors.length === 0) {
             return {
                 content: [{ type: "text", text: `No files matched pattern: ${pattern}` }],
@@ -400,7 +378,7 @@ Intent-aware lex (C++ performance, not sports):
         annotations: { readOnlyHint: true, openWorldHint: false },
         inputSchema: {},
     }, async () => {
-        const status = store.getStatus();
+        const status = await store.getStatus();
         const summary = [
             `QMD Index Status:`,
             `  Total documents: ${status.totalDocuments}`,
@@ -422,8 +400,8 @@ Intent-aware lex (C++ performance, not sports):
 // Transport: stdio (default)
 // =============================================================================
 export async function startMcpServer() {
-    const store = createStore();
-    const server = createMcpServer(store);
+    const store = await createStore({ dbPath: getDefaultDbPath() });
+    const server = await createMcpServer(store);
     const transport = new StdioServerTransport();
     await server.connect(transport);
 }
@@ -432,7 +410,9 @@ export async function startMcpServer() {
  * Binds to localhost only. Returns a handle for shutdown and port discovery.
  */
 export async function startMcpHttpServer(port, options) {
-    const store = createStore();
+    const store = await createStore({ dbPath: getDefaultDbPath() });
+    // Pre-fetch default collection names for REST endpoint
+    const defaultCollectionNames = await store.getDefaultCollectionNames();
     // Session map: each client gets its own McpServer + Transport pair (MCP spec requirement).
     // The store is shared — it's stateless SQLite, safe for concurrent access.
     const sessions = new Map();
@@ -445,7 +425,7 @@ export async function startMcpHttpServer(port, options) {
                 log(`${ts()} New session ${sessionId} (${sessions.size} active)`);
             },
         });
-        const server = createMcpServer(store);
+        const server = await createMcpServer(store);
         await server.connect(transport);
         transport.onclose = () => {
             if (transport.sessionId) {
@@ -513,17 +493,18 @@ export async function startMcpHttpServer(port, options) {
                     return;
                 }
                 // Map to internal format
-                const subSearches = params.searches.map((s) => ({
+                const queries = params.searches.map((s) => ({
                     type: s.type,
                     query: String(s.query || ""),
                 }));
                 // Use default collections if none specified
-                const effectiveCollections = params.collections ?? getDefaultCollectionNames();
-                const results = await structuredSearch(store, subSearches, {
+                const effectiveCollections = params.collections ?? defaultCollectionNames;
+                const results = await store.search({
+                    queries,
                     collections: effectiveCollections.length > 0 ? effectiveCollections : undefined,
                     limit: params.limit ?? 10,
                     minScore: params.minScore ?? 0,
-                    candidateLimit: params.candidateLimit,
+                    intent: params.intent,
                 });
                 // Use first lex or vec query for snippet extraction
                 const primaryQuery = params.searches.find((s) => s.type === 'lex')?.query
@@ -649,8 +630,7 @@ export async function startMcpHttpServer(port, options) {
         }
         sessions.clear();
         httpServer.close();
-        store.close();
-        await disposeDefaultLlamaCpp();
+        await store.close();
     };
     process.on("SIGTERM", async () => {
         console.error("Shutting down (SIGTERM)...");
@@ -666,6 +646,6 @@ export async function startMcpHttpServer(port, options) {
     return { httpServer, port: actualPort, stop };
 }
 // Run if this is the main module
-if (fileURLToPath(import.meta.url) === process.argv[1] || process.argv[1]?.endsWith("/mcp.ts") || process.argv[1]?.endsWith("/mcp.js")) {
+if (fileURLToPath(import.meta.url) === process.argv[1] || process.argv[1]?.endsWith("/server.ts") || process.argv[1]?.endsWith("/server.js")) {
     startMcpServer().catch(console.error);
 }

package/dist/store.d.ts

@@ -11,7 +11,8 @@
  *   const store = createStore();
  */
 import type { Database } from "./db.js";
-import { formatQueryForEmbedding, formatDocForEmbedding, type ILLMSession } from "./llm.js";
+import { LlamaCpp, formatQueryForEmbedding, formatDocForEmbedding, type ILLMSession } from "./llm.js";
+import type { NamedCollection, Collection, CollectionConfig } from "./collections.js";
 export declare const DEFAULT_EMBED_MODEL = "embeddinggemma";
 export declare const DEFAULT_RERANK_MODEL = "ExpedientFalcon/qwen3-reranker:0.6b-q8_0";
 export declare const DEFAULT_QUERY_MODEL = "Qwen/Qwen3-1.7B";
@@ -88,7 +89,9 @@ export declare const RERANK_CANDIDATE_LIMIT = 40;
  */
 export type ExpandedQuery = {
     type: 'lex' | 'vec' | 'hyde';
-    text: string;
+    query: string;
+    /** Optional line number for error reporting (CLI parser) */
+    line?: number;
 };
 export declare function homedir(): string;
 /**
@@ -164,10 +167,33 @@ export declare function resolveVirtualPath(db: Database, virtualPath: string): s
  */
 export declare function toVirtualPath(db: Database, absolutePath: string): string | null;
 export declare function verifySqliteVecLoaded(db: Database): void;
+export declare function getStoreCollections(db: Database): NamedCollection[];
+export declare function getStoreCollection(db: Database, name: string): NamedCollection | null;
+export declare function getStoreGlobalContext(db: Database): string | undefined;
+export declare function getStoreContexts(db: Database): Array<{
+    collection: string;
+    path: string;
+    context: string;
+}>;
+export declare function upsertStoreCollection(db: Database, name: string, collection: Omit<Collection, 'pattern'> & {
+    pattern?: string;
+}): void;
+export declare function deleteStoreCollection(db: Database, name: string): boolean;
+export declare function renameStoreCollection(db: Database, oldName: string, newName: string): boolean;
+export declare function updateStoreContext(db: Database, collectionName: string, path: string, text: string): boolean;
+export declare function removeStoreContext(db: Database, collectionName: string, path: string): boolean;
+export declare function setStoreGlobalContext(db: Database, value: string | undefined): void;
+/**
+ * Sync external config (YAML/inline) into SQLite store_collections.
+ * External config always wins. Skips sync if config hash hasn't changed.
+ */
+export declare function syncConfigToDb(db: Database, config: CollectionConfig): void;
 export declare function isSqliteVecAvailable(): boolean;
 export type Store = {
     db: Database;
     dbPath: string;
+    /** Optional LlamaCpp instance for this store (overrides the global singleton) */
+    llm?: LlamaCpp;
     close: () => void;
     ensureVecTable: (dimensions: number) => void;
     getHashesNeedingEmbedding: () => number;
@@ -252,6 +278,49 @@ export type Store = {
     clearAllEmbeddings: () => void;
     insertEmbedding: (hash: string, seq: number, pos: number, embedding: Float32Array, model: string, embeddedAt: string) => void;
 };
+export type ReindexProgress = {
+    file: string;
+    current: number;
+    total: number;
+};
+export type ReindexResult = {
+    indexed: number;
+    updated: number;
+    unchanged: number;
+    removed: number;
+    orphanedCleaned: number;
+};
+/**
+ * Re-index a single collection by scanning the filesystem and updating the database.
+ * Pure function — no console output, no db lifecycle management.
+ */
+export declare function reindexCollection(store: Store, collectionPath: string, globPattern: string, collectionName: string, options?: {
+    ignorePatterns?: string[];
+    onProgress?: (info: ReindexProgress) => void;
+}): Promise<ReindexResult>;
+export type EmbedProgress = {
+    chunksEmbedded: number;
+    totalChunks: number;
+    bytesProcessed: number;
+    totalBytes: number;
+    errors: number;
+};
+export type EmbedResult = {
+    docsProcessed: number;
+    chunksEmbedded: number;
+    errors: number;
+    durationMs: number;
+};
+/**
+ * Generate vector embeddings for documents that need them.
+ * Pure function — no console output, no db lifecycle management.
+ * Uses the store's LlamaCpp instance if set, otherwise the global singleton.
+ */
+export declare function generateEmbeddings(store: Store, options?: {
+    force?: boolean;
+    model?: string;
+    onProgress?: (info: EmbedProgress) => void;
+}): Promise<EmbedResult>;
 /**
  * Create a new store instance with the given database path.
  * If no path is provided, uses the default path (~/.cache/qmd/index.sqlite).
@@ -352,8 +421,8 @@ export type MultiGetResult = {
 };
 export type CollectionInfo = {
     name: string;
-    path: string;
-    pattern: string;
+    path: string | null;
+    pattern: string | null;
     documents: number;
     lastUpdated: string;
 };
@@ -491,12 +560,11 @@ export declare function matchFilesByGlob(db: Database, pattern: string): {
 export declare function getContextForPath(db: Database, collectionName: string, path: string): string | null;
 /**
  * Get context for a file path (virtual or filesystem).
- * Resolves the collection and relative path using the YAML collections config.
+ * Resolves the collection and relative path from the DB store_collections table.
  */
 export declare function getContextForFile(db: Database, filepath: string): string | null;
 /**
- * Get collection by name from YAML config.
- * Returns collection metadata from ~/.config/qmd/index.yml
+ * Get collection by name from DB store_collections table.
  */
 export declare function getCollectionByName(db: Database, name: string): {
     name: string;
@@ -505,7 +573,7 @@ export declare function getCollectionByName(db: Database, name: string): {
 } | null;
 /**
  * List all collections with document counts from database.
- * Merges YAML config with database statistics.
+ * Merges store_collections config with database statistics.
  */
 export declare function listCollections(db: Database): {
     name: string;
@@ -514,6 +582,7 @@ export declare function listCollections(db: Database): {
     doc_count: number;
     active_count: number;
     last_modified: string | null;
+    includeByDefault: boolean;
 }[];
 /**
  * Remove a collection and clean up its documents.
@@ -598,11 +667,11 @@ export declare function clearAllEmbeddings(db: Database): void;
  * The hash_seq key is formatted as "hash_seq" for the vectors_vec table.
  */
 export declare function insertEmbedding(db: Database, hash: string, seq: number, pos: number, embedding: Float32Array, model: string, embeddedAt: string): void;
-export declare function expandQuery(query: string, model: string | undefined, db: Database, intent?: string): Promise<ExpandedQuery[]>;
+export declare function expandQuery(query: string, model: string | undefined, db: Database, intent?: string, llmOverride?: LlamaCpp): Promise<ExpandedQuery[]>;
 export declare function rerank(query: string, documents: {
     file: string;
     text: string;
-}[], model: string | undefined, db: Database, intent?: string): Promise<{
+}[], model: string | undefined, db: Database, intent?: string, llmOverride?: LlamaCpp): Promise<{
     file: string;
     score: number;
 }[]>;
@@ -693,6 +762,7 @@ export interface HybridQueryOptions {
     candidateLimit?: number;
     explain?: boolean;
     intent?: string;
+    skipRerank?: boolean;
     hooks?: SearchHooks;
 }
 export interface HybridQueryResult {
@@ -756,14 +826,6 @@ export declare function vectorSearchQuery(store: Store, query: string, options?:
  * A single sub-search in a structured search request.
  * Matches the format used in QMD training data.
  */
-export interface StructuredSubSearch {
-    /** Search type: 'lex' for BM25, 'vec' for semantic, 'hyde' for hypothetical */
-    type: 'lex' | 'vec' | 'hyde';
-    /** The search query text */
-    query: string;
-    /** Optional line number for error reporting (CLI parser) */
-    line?: number;
-}
 export interface StructuredSearchOptions {
     collections?: string[];
     limit?: number;
@@ -772,6 +834,8 @@ export interface StructuredSearchOptions {
     explain?: boolean;
     /** Domain intent hint for disambiguation — steers reranking and chunk selection */
     intent?: string;
+    /** Skip LLM reranking, use only RRF scores */
+    skipRerank?: boolean;
     hooks?: SearchHooks;
 }
 /**
@@ -792,4 +856,4 @@ export interface StructuredSearchOptions {
  * better query variations than our small local model, especially for
  * domain-specific or nuanced queries.
  */
-export declare function structuredSearch(store: Store, searches: StructuredSubSearch[], options?: StructuredSearchOptions): Promise<HybridQueryResult[]>;
+export declare function structuredSearch(store: Store, searches: ExpandedQuery[], options?: StructuredSearchOptions): Promise<HybridQueryResult[]>;