@tobilu/qmd 2.1.0 → 2.5.1

package/dist/mcp/server.d.ts

@@ -6,7 +6,10 @@
  *
  * Follows MCP spec 2025-06-18 for proper response types.
  */
-export declare function startMcpServer(): Promise<void>;
+export type McpStartupOptions = {
+    dbPath?: string;
+};
+export declare function startMcpServer(options?: McpStartupOptions): Promise<void>;
 export type HttpServerHandle = {
     httpServer: import("http").Server;
     port: number;
@@ -16,6 +19,6 @@ export type HttpServerHandle = {
  * Start MCP server over Streamable HTTP (JSON responses, no SSE).
  * Binds to localhost only. Returns a handle for shutdown and port discovery.
  */
-export declare function startMcpHttpServer(port: number, options?: {
+export declare function startMcpHttpServer(port: number, options?: ({
     quiet?: boolean;
-}): Promise<HttpServerHandle>;
+} & McpStartupOptions)): Promise<HttpServerHandle>;

package/dist/mcp/server.js

@@ -19,6 +19,7 @@ import { z } from "zod";
 import { existsSync } from "fs";
 import { createStore, extractSnippet, addLineNumbers, getDefaultDbPath, DEFAULT_MULTI_GET_MAX_BYTES, } from "../index.js";
 import { getConfigPath } from "../collections.js";
+import { enableProductionMode } from "../store.js";
 // =============================================================================
 // Helper functions
 // =============================================================================
@@ -63,7 +64,6 @@ function getPackageVersion() {
  */
 async function buildInstructions(store) {
     const status = await store.getStatus();
-    const contexts = await store.listContexts();
     const globalCtx = await store.getGlobalContext();
     const lines = [];
     // --- What is this? ---
@@ -71,15 +71,13 @@ async function buildInstructions(store) {
     if (globalCtx)
         lines.push(`Context: ${globalCtx}`);
     // --- What's searchable? ---
+    // Emit names only — the per-collection doc counts and descriptions can run to ~1.5 KB
+    // across a dozen collections, and the same info is available on demand via the `status` tool.
     if (status.collections.length > 0) {
         lines.push("");
-        lines.push("Collections (scope with `collection` parameter):");
-        for (const col of status.collections) {
-            // Find root context for this collection
-            const rootCtx = contexts.find(c => c.collection === col.name && (c.path === "" || c.path === "/"));
-            const desc = rootCtx ? ` — ${rootCtx.context}` : "";
-            lines.push(`  - "${col.name}" (${col.documents} docs)${desc}`);
-        }
+        const names = status.collections.map(c => c.name).join(", ");
+        lines.push(`Collections (scope with \`collection\` parameter): ${names}`);
+        lines.push("Call the `status` tool for collection descriptions, paths, and per-collection doc counts.");
     }
     // --- Capability gaps ---
     if (!status.hasVectorIndex) {
@@ -169,6 +167,8 @@ async function createMcpServer(store) {
         title: "Query",
         description: `Search the knowledge base using a query document — one or more typed sub-queries combined for best recall.
 
+Each result includes a \`line\` field with the absolute 1-indexed line of the best match in the source markdown. To read more context around a hit, call \`get(file, fromLine = max(1, line - 20), maxLines = 80, lineNumbers = true)\`.
+
 ## Query Types
 
 **lex** — BM25 keyword search. Fast, exact, no LLM needed.
@@ -247,6 +247,7 @@ Intent-aware lex (C++ performance, not sports):
             collections: effectiveCollections.length > 0 ? effectiveCollections : undefined,
             limit,
             minScore,
+            candidateLimit,
             rerank,
             intent,
         });
@@ -255,13 +256,14 @@ Intent-aware lex (C++ performance, not sports):
             || searches.find(s => s.type === 'vec')?.query
             || searches[0]?.query || "";
         const filtered = results.map(r => {
-            const { line, snippet } = extractSnippet(r.bestChunk, primaryQuery, 300, undefined, undefined, intent);
+            const { line, snippet } = extractSnippet(r.body, primaryQuery, 300, r.bestChunkPos, r.bestChunk.length, intent);
             return {
                 docid: `#${r.docid}`,
                 file: r.displayPath,
                 title: r.title,
                 score: Math.round(r.score * 100) / 100,
                 context: r.context,
+                line,
                 snippet: addLineNumbers(snippet, line),
             };
         });
@@ -292,6 +294,8 @@ Intent-aware lex (C++ performance, not sports):
             parsedFromLine = parseInt(colonMatch[1], 10);
             lookup = lookup.slice(0, -colonMatch[0].length);
         }
+        if (parsedFromLine !== undefined)
+            parsedFromLine = Math.max(1, parsedFromLine);
         const result = await store.get(lookup, { includeBody: false });
         if ("error" in result) {
             let msg = `Document not found: ${file}`;
@@ -412,13 +416,16 @@ Intent-aware lex (C++ performance, not sports):
     });
     return server;
 }
-// =============================================================================
-// Transport: stdio (default)
-// =============================================================================
-export async function startMcpServer() {
+export async function startMcpServer(options = {}) {
+    // Opt into production mode when the MCP server is actually started, not
+    // when this module is merely imported for its exports. Importing the module
+    // at the top level flipped the global production flag and broke test
+    // isolation for downstream suites that expect the default (development)
+    // database path behaviour.
+    enableProductionMode();
     const configPath = getConfigPath();
     const store = await createStore({
-        dbPath: getDefaultDbPath(),
+        dbPath: options.dbPath ?? getDefaultDbPath(),
         ...(existsSync(configPath) ? { configPath } : {}),
     });
     const server = await createMcpServer(store);
@@ -429,10 +436,14 @@ export async function startMcpServer() {
  * Start MCP server over Streamable HTTP (JSON responses, no SSE).
  * Binds to localhost only. Returns a handle for shutdown and port discovery.
  */
-export async function startMcpHttpServer(port, options) {
+export async function startMcpHttpServer(port, options = {}) {
+    // See startMcpServer() for the rationale — flip production mode here so the
+    // HTTP transport resolves the real database path, without leaking state into
+    // callers that only import this module for its exports (e.g. tests).
+    enableProductionMode();
     const configPath = getConfigPath();
     const store = await createStore({
-        dbPath: getDefaultDbPath(),
+        dbPath: options.dbPath ?? getDefaultDbPath(),
         ...(existsSync(configPath) ? { configPath } : {}),
     });
     // Pre-fetch default collection names for REST endpoint
@@ -466,7 +477,7 @@ export async function startMcpHttpServer(port, options) {
     }
     /** Extract a human-readable label from a JSON-RPC body */
     function describeRequest(body) {
-        const method = body?.method ?? "unknown";
+        const method = typeof body.method === "string" ? body.method : "unknown";
         if (method === "tools/call") {
             const tool = body.params?.name ?? "?";
             const args = body.params?.arguments;
@@ -517,31 +528,35 @@ export async function startMcpHttpServer(port, options) {
                     return;
                 }
                 // Map to internal format
-                const queries = params.searches.map((s) => ({
+                const searches = params.searches;
+                const queries = searches.map((s) => ({
                     type: s.type,
                     query: String(s.query || ""),
                 }));
                 // Use default collections if none specified
-                const effectiveCollections = params.collections ?? defaultCollectionNames;
+                const effectiveCollections = Array.isArray(params.collections) ? params.collections.map(String) : defaultCollectionNames;
                 const results = await store.search({
                     queries,
                     collections: effectiveCollections.length > 0 ? effectiveCollections : undefined,
-                    limit: params.limit ?? 10,
-                    minScore: params.minScore ?? 0,
-                    intent: params.intent,
+                    limit: typeof params.limit === "number" ? params.limit : 10,
+                    minScore: typeof params.minScore === "number" ? params.minScore : 0,
+                    candidateLimit: typeof params.candidateLimit === "number" ? params.candidateLimit : undefined,
+                    intent: typeof params.intent === "string" ? params.intent : undefined,
+                    rerank: typeof params.rerank === "boolean" ? params.rerank : undefined,
                 });
                 // Use first lex or vec query for snippet extraction
-                const primaryQuery = params.searches.find((s) => s.type === 'lex')?.query
-                    || params.searches.find((s) => s.type === 'vec')?.query
-                    || params.searches[0]?.query || "";
+                const primaryQuery = searches.find((s) => s.type === 'lex')?.query
+                    || searches.find((s) => s.type === 'vec')?.query
+                    || searches[0]?.query || "";
                 const formatted = results.map(r => {
-                    const { line, snippet } = extractSnippet(r.bestChunk, primaryQuery, 300);
+                    const { line, snippet } = extractSnippet(r.body, String(primaryQuery), 300, r.bestChunkPos, r.bestChunk.length, typeof params.intent === "string" ? params.intent : undefined);
                     return {
                         docid: `#${r.docid}`,
                         file: r.displayPath,
                         title: r.title,
                         score: Math.round(r.score * 100) / 100,
                         context: r.context,
+                        line,
                         snippet: addLineNumbers(snippet, line),
                     };
                 });

package/dist/paths.d.ts

@@ -0,0 +1 @@
+export declare function qmdHomedir(): string;

package/dist/paths.js

@@ -0,0 +1,4 @@
+import { homedir as osHomedir } from "node:os";
+export function qmdHomedir() {
+    return process.env.HOME || process.env.USERPROFILE || osHomedir() || "/tmp";
+}

package/dist/store.d.ts

@@ -13,9 +13,9 @@
 import type { Database } from "./db.js";
 import { LlamaCpp, formatQueryForEmbedding, formatDocForEmbedding, type ILLMSession } from "./llm.js";
 import type { NamedCollection, Collection, CollectionConfig } from "./collections.js";
-export declare const DEFAULT_EMBED_MODEL = "embeddinggemma";
-export declare const DEFAULT_RERANK_MODEL = "ExpedientFalcon/qwen3-reranker:0.6b-q8_0";
-export declare const DEFAULT_QUERY_MODEL = "Qwen/Qwen3-1.7B";
+export declare const DEFAULT_EMBED_MODEL = "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf";
+export declare const DEFAULT_RERANK_MODEL = "hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf";
+export declare const DEFAULT_QUERY_MODEL = "hf:tobil/qmd-query-expansion-1.7B-gguf/qmd-query-expansion-1.7B-q4_k_m.gguf";
 export declare const DEFAULT_GLOB = "**/*.md";
 export declare const DEFAULT_MULTI_GET_MAX_BYTES: number;
 export declare const DEFAULT_EMBED_MAX_DOCS_PER_BATCH = 64;
@@ -26,6 +26,7 @@ export declare const CHUNK_SIZE_CHARS: number;
 export declare const CHUNK_OVERLAP_CHARS: number;
 export declare const CHUNK_WINDOW_TOKENS = 200;
 export declare const CHUNK_WINDOW_CHARS: number;
+export declare function getEmbeddingFingerprint(model?: string): string;
 /**
  * A potential break point in the document with a base score indicating quality.
  */
@@ -142,6 +143,7 @@ export declare function getRealPath(path: string): string;
 export type VirtualPath = {
     collectionName: string;
     path: string;
+    indexName?: string;
 };
 /**
  * Normalize explicit virtual path formats to standard qmd:// format.
@@ -164,7 +166,7 @@ export declare function parseVirtualPath(virtualPath: string): VirtualPath | nul
 /**
  * Build a virtual path from collection name and relative path.
  */
-export declare function buildVirtualPath(collectionName: string, path: string): string;
+export declare function buildVirtualPath(collectionName: string, path: string, indexName?: string): string;
 /**
  * Check if a path is explicitly a virtual path.
  * Only recognizes explicit virtual path formats:
@@ -185,6 +187,12 @@ export declare function resolveVirtualPath(db: Database, virtualPath: string): s
  */
 export declare function toVirtualPath(db: Database, absolutePath: string): string | null;
 export declare function verifySqliteVecLoaded(db: Database): void;
+/**
+ * FTS5's unicode61 tokenizer does not segment CJK text into searchable words.
+ * Normalize CJK runs by spacing every character so exact CJK queries can be
+ * translated into phrase queries while Latin text keeps the default tokenizer.
+ */
+export declare function normalizeCjkForFTS(text: string): string;
 export declare function getStoreCollections(db: Database): NamedCollection[];
 export declare function getStoreCollection(db: Database, name: string): NamedCollection | null;
 export declare function getStoreGlobalContext(db: Database): string | undefined;
@@ -214,9 +222,9 @@ export type Store = {
     llm?: LlamaCpp;
     close: () => void;
     ensureVecTable: (dimensions: number) => void;
-    getHashesNeedingEmbedding: () => number;
-    getIndexHealth: () => IndexHealthInfo;
-    getStatus: () => IndexStatus;
+    getHashesNeedingEmbedding: (model?: string) => number;
+    getIndexHealth: (model?: string) => IndexHealthInfo;
+    getStatus: (model?: string) => IndexStatus;
     getCacheKey: typeof getCacheKey;
     getCachedResult: (cacheKey: string) => string | null;
     setCachedResult: (cacheKey: string, result: string) => void;
@@ -284,6 +292,11 @@ export type Store = {
         hash: string;
         title: string;
     } | null;
+    findOrMigrateLegacyDocument: (collectionName: string, path: string) => {
+        id: number;
+        hash: string;
+        title: string;
+    } | null;
     updateDocumentTitle: (documentId: number, title: string, modifiedAt: string) => void;
     updateDocument: (documentId: number, title: string, hash: string, modifiedAt: string) => void;
     deactivateDocument: (collectionName: string, path: string) => void;
@@ -294,7 +307,7 @@ export type Store = {
         path: string;
     }[];
     clearAllEmbeddings: () => void;
-    insertEmbedding: (hash: string, seq: number, pos: number, embedding: Float32Array, model: string, embeddedAt: string) => void;
+    insertEmbedding: (hash: string, seq: number, pos: number, embedding: Float32Array, model: string, embeddedAt: string, totalChunks?: number, fingerprint?: string) => void;
 };
 export type ReindexProgress = {
     file: string;
@@ -316,22 +329,38 @@ export declare function reindexCollection(store: Store, collectionPath: string,
     ignorePatterns?: string[];
     onProgress?: (info: ReindexProgress) => void;
 }): Promise<ReindexResult>;
+export type EmbedFailure = {
+    path: string;
+    hash: string;
+    seq: number;
+    attempts: number;
+    reason: string;
+};
 export type EmbedProgress = {
     chunksEmbedded: number;
     totalChunks: number;
     bytesProcessed: number;
     totalBytes: number;
+    /** Active failed chunks still awaiting a successful retry. */
     errors: number;
+    failures?: EmbedFailure[];
 };
 export type EmbedResult = {
     docsProcessed: number;
     chunksEmbedded: number;
+    /** Active failed chunks that did not recover after retries. */
     errors: number;
+    failures?: EmbedFailure[];
     durationMs: number;
 };
 export type EmbedOptions = {
     force?: boolean;
     model?: string;
+    /**
+     * Restrict embedding to documents in a single collection.
+     * When omitted, all pending documents across every collection are embedded.
+     */
+    collection?: string;
     maxDocsPerBatch?: number;
     maxBatchBytes?: number;
     chunkStrategy?: ChunkStrategy;
@@ -454,13 +483,19 @@ export type IndexStatus = {
     hasVectorIndex: boolean;
     collections: CollectionInfo[];
 };
-export declare function getHashesNeedingEmbedding(db: Database): number;
+export declare function getHashesNeedingEmbedding(db: Database, collection?: string, model?: string): number;
 export type IndexHealthInfo = {
     needsEmbedding: number;
     totalDocs: number;
     daysStale: number | null;
 };
-export declare function getIndexHealth(db: Database): IndexHealthInfo;
+export type LegacyFingerprintAdoptionResult = {
+    checked: boolean;
+    adopted: number;
+    reason: string;
+};
+export declare function maybeAdoptLegacyEmbeddingFingerprint(store: Store, model?: string): Promise<LegacyFingerprintAdoptionResult>;
+export declare function getIndexHealth(db: Database, model?: string): IndexHealthInfo;
 export declare function getCacheKey(url: string, body: object): string;
 export declare function getCachedResult(db: Database, cacheKey: string): string | null;
 export declare function setCachedResult(db: Database, cacheKey: string, result: string): void;
@@ -476,7 +511,9 @@ export declare function deleteLLMCache(db: Database): number;
  */
 export declare function deleteInactiveDocuments(db: Database): number;
 /**
- * Remove orphaned content hashes that are not referenced by any active document.
+ * Remove orphaned content hashes that are not referenced by any document.
+ * Inactive documents are soft-deleted tombstones, so their content rows must
+ * remain referenced until deleteInactiveDocuments() hard-deletes them.
  * Returns the number of orphaned content hashes deleted.
  */
 export declare function cleanupOrphanedContent(db: Database): number;
@@ -509,6 +546,20 @@ export declare function findActiveDocument(db: Database, collectionName: string,
     hash: string;
     title: string;
 } | null;
+/**
+ * Find an active document, falling back to a case-insensitive path match.
+ * If found under a different casing, renames it in-place and rebuilds the
+ * FTS entry. Embeddings are keyed by content hash, so the rename is
+ * safe — no re-embedding required.
+ *
+ * @internal Used by reindexCollection and indexFiles during qmd update.
+ * Returns null if the document does not exist under either path.
+ */
+export declare function findOrMigrateLegacyDocument(db: Database, collectionName: string, path: string): {
+    id: number;
+    hash: string;
+    title: string;
+} | null;
 /**
  * Update the title and modified_at timestamp for a document.
  */
@@ -694,16 +745,28 @@ export declare function searchVec(db: Database, query: string, model: string, li
  * Get all unique content hashes that need embeddings (from active documents).
  * Returns hash, document body, and a sample path for display purposes.
  */
-export declare function getHashesForEmbedding(db: Database): {
+export declare function getHashesForEmbedding(db: Database, model?: string): {
     hash: string;
     body: string;
     path: string;
 }[];
 /**
- * Clear all embeddings from the database (force re-index).
- * Deletes all rows from content_vectors and drops the vectors_vec table.
+ * Clear embeddings for the whole index, or just for one collection.
+ *
+ * When `collection` is omitted the entire content_vectors table is emptied and
+ * the vectors_vec virtual table is dropped (it is recreated with the right
+ * dimensions on the next embed run).
+ *
+ * When `collection` is provided, only vectors whose hash is referenced
+ * exclusively by active documents in that collection are removed. Hashes
+ * shared with active documents in other collections are left in place so
+ * vector search keeps working there (content_vectors is keyed globally by
+ * content hash; identical document bodies across collections share a row).
+ * vectors_vec is preserved so other collections keep working unless the scoped
+ * clear empties content_vectors entirely, in which case it is dropped so the
+ * next embed can recreate the table with the current dimensions.
  */
-export declare function clearAllEmbeddings(db: Database): void;
+export declare function clearAllEmbeddings(db: Database, collection?: string): void;
 /**
  * Insert a single embedding into both content_vectors and vectors_vec tables.
  * The hash_seq key is formatted as "hash_seq" for the vectors_vec table.
@@ -714,7 +777,7 @@ export declare function clearAllEmbeddings(db: Database): void;
  * vectors_vec uses DELETE + INSERT instead of INSERT OR REPLACE because sqlite-vec's
  * vec0 virtual tables silently ignore the OR REPLACE conflict clause.
  */
-export declare function insertEmbedding(db: Database, hash: string, seq: number, pos: number, embedding: Float32Array, model: string, embeddedAt: string): void;
+export declare function insertEmbedding(db: Database, hash: string, seq: number, pos: number, embedding: Float32Array, model: string, embeddedAt: string, totalChunks?: number, fingerprint?: string): void;
 export declare function expandQuery(query: string, model: string | undefined, db: Database, intent?: string, llmOverride?: LlamaCpp): Promise<ExpandedQuery[]>;
 export declare function rerank(query: string, documents: {
     file: string;
@@ -759,7 +822,7 @@ export declare function findDocuments(db: Database, pattern: string, options?: {
     docs: MultiGetResult[];
     errors: string[];
 };
-export declare function getStatus(db: Database): IndexStatus;
+export declare function getStatus(db: Database, model?: string): IndexStatus;
 export type SnippetResult = {
     line: number;
     snippet: string;
@@ -831,6 +894,18 @@ export type RankedListMeta = {
     queryType: "original" | "lex" | "vec" | "hyde";
     query: string;
 };
+/**
+ * RRF list weights for hybridQuery.
+ *
+ * Original-query retrieval paths are the primary evidence and get 2x weight:
+ * - original FTS
+ * - original vector search
+ *
+ * Expansion-derived lists (lex/vec/hyde) stay at 1x regardless of list order,
+ * so a lex expansion inserted before original vector search cannot steal the
+ * original vector boost.
+ */
+export declare function getHybridRrfWeights(rankedListMeta: RankedListMeta[]): number[];
 /**
  * Hybrid search: BM25 + vector + query expansion + RRF + chunked reranking.
  *