@tobilu/qmd 1.1.1 → 1.1.2

package/dist/store.js

@@ -706,15 +706,24 @@ export function getDocid(hash) {
  * - Preserve folder structure (a/b/c/d.md stays structured)
  * - Preserve file extension
  */
+/** Replace emoji/symbol codepoints with their hex representation (e.g. 🐘 → 1f418) */
+function emojiToHex(str) {
+    return str.replace(/(?:\p{So}\p{Mn}?|\p{Sk})+/gu, (run) => {
+        // Split the run into individual emoji and convert each to hex, dash-separated
+        return [...run].filter(c => /\p{So}|\p{Sk}/u.test(c))
+            .map(c => c.codePointAt(0).toString(16)).join('-');
+    });
+}
 export function handelize(path) {
     if (!path || path.trim() === '') {
         throw new Error('handelize: path cannot be empty');
     }
     // Allow route-style "$" filenames while still rejecting paths with no usable content.
+    // Emoji (\p{So}) counts as valid content — they get converted to hex codepoints below.
     const segments = path.split('/').filter(Boolean);
     const lastSegment = segments[segments.length - 1] || '';
     const filenameWithoutExt = lastSegment.replace(/\.[^.]+$/, '');
-    const hasValidContent = /[\p{L}\p{N}$]/u.test(filenameWithoutExt);
+    const hasValidContent = /[\p{L}\p{N}\p{So}\p{Sk}$]/u.test(filenameWithoutExt);
     if (!hasValidContent) {
         throw new Error(`handelize: path "${path}" has no valid filename content`);
     }
@@ -724,6 +733,8 @@ export function handelize(path) {
         .split('/')
         .map((segment, idx, arr) => {
         const isLastSegment = idx === arr.length - 1;
+        // Convert emoji to hex codepoints before cleaning
+        segment = emojiToHex(segment);
         if (isLastSegment) {
             // For the filename (last segment), preserve the extension
             const extMatch = segment.match(/(\.[a-z0-9]+)$/i);
@@ -1745,7 +1756,7 @@ export async function searchVec(db, query, model, limit = 20, collectionName, se
 // =============================================================================
 async function getEmbedding(text, model, isQuery, session) {
     // Format text using the appropriate prompt template
-    const formattedText = isQuery ? formatQueryForEmbedding(text) : formatDocForEmbedding(text);
+    const formattedText = isQuery ? formatQueryForEmbedding(text, model) : formatDocForEmbedding(text, undefined, model);
     const result = session
         ? await session.embed(formattedText, { model, isQuery })
         : await getDefaultLlamaCpp().embed(formattedText, { model, isQuery });
@@ -1817,35 +1828,40 @@ export async function expandQuery(query, model = DEFAULT_QUERY_MODEL, db) {
 // =============================================================================
 export async function rerank(query, documents, model = DEFAULT_RERANK_MODEL, db) {
     const cachedResults = new Map();
-    const uncachedDocs = [];
+    const uncachedDocsByChunk = new Map();
     // Check cache for each document
     // Cache key includes chunk text — different queries can select different chunks
     // from the same file, and the reranker score depends on which chunk was sent.
+    // File path is excluded from the new cache key because the reranker score
+    // depends on the chunk content, not where it came from.
     for (const doc of documents) {
-        const cacheKey = getCacheKey("rerank", { query, file: doc.file, model, chunk: doc.text });
-        const cached = getCachedResult(db, cacheKey);
+        const cacheKey = getCacheKey("rerank", { query, model, chunk: doc.text });
+        const legacyCacheKey = getCacheKey("rerank", { query, file: doc.file, model, chunk: doc.text });
+        const cached = getCachedResult(db, cacheKey) ?? getCachedResult(db, legacyCacheKey);
         if (cached !== null) {
-            cachedResults.set(doc.file, parseFloat(cached));
+            cachedResults.set(doc.text, parseFloat(cached));
         }
         else {
-            uncachedDocs.push({ file: doc.file, text: doc.text });
+            uncachedDocsByChunk.set(doc.text, { file: doc.file, text: doc.text });
         }
     }
     // Rerank uncached documents using LlamaCpp
-    if (uncachedDocs.length > 0) {
+    if (uncachedDocsByChunk.size > 0) {
         const llm = getDefaultLlamaCpp();
+        const uncachedDocs = [...uncachedDocsByChunk.values()];
         const rerankResult = await llm.rerank(query, uncachedDocs, { model });
-        // Cache results — use original doc.text for cache key (result.file lacks chunk text)
-        const textByFile = new Map(documents.map(d => [d.file, d.text]));
+        // Cache results by chunk text so identical chunks across files are scored once.
+        const textByFile = new Map(uncachedDocs.map(d => [d.file, d.text]));
         for (const result of rerankResult.results) {
-            const cacheKey = getCacheKey("rerank", { query, file: result.file, model, chunk: textByFile.get(result.file) || "" });
+            const chunk = textByFile.get(result.file) || "";
+            const cacheKey = getCacheKey("rerank", { query, model, chunk });
             setCachedResult(db, cacheKey, result.score.toString());
-            cachedResults.set(result.file, result.score);
+            cachedResults.set(chunk, result.score);
         }
     }
     // Return all results sorted by score
     return documents
-        .map(doc => ({ file: doc.file, score: cachedResults.get(doc.file) || 0 }))
+        .map(doc => ({ file: doc.file, score: cachedResults.get(doc.text) || 0 }))
         .sort((a, b) => b.score - a.score);
 }
 // =============================================================================
@@ -1890,6 +1906,65 @@ export function reciprocalRankFusion(resultLists, weights = [], k = 60) {
         .sort((a, b) => b.rrfScore - a.rrfScore)
         .map(e => ({ ...e.result, score: e.rrfScore }));
 }
+/**
+ * Build per-document RRF contribution traces for explain/debug output.
+ */
+export function buildRrfTrace(resultLists, weights = [], listMeta = [], k = 60) {
+    const traces = new Map();
+    for (let listIdx = 0; listIdx < resultLists.length; listIdx++) {
+        const list = resultLists[listIdx];
+        if (!list)
+            continue;
+        const weight = weights[listIdx] ?? 1.0;
+        const meta = listMeta[listIdx] ?? {
+            source: "fts",
+            queryType: "original",
+            query: "",
+        };
+        for (let rank0 = 0; rank0 < list.length; rank0++) {
+            const result = list[rank0];
+            if (!result)
+                continue;
+            const rank = rank0 + 1; // 1-indexed rank for explain output
+            const contribution = weight / (k + rank);
+            const existing = traces.get(result.file);
+            const detail = {
+                listIndex: listIdx,
+                source: meta.source,
+                queryType: meta.queryType,
+                query: meta.query,
+                rank,
+                weight,
+                backendScore: result.score,
+                rrfContribution: contribution,
+            };
+            if (existing) {
+                existing.baseScore += contribution;
+                existing.topRank = Math.min(existing.topRank, rank);
+                existing.contributions.push(detail);
+            }
+            else {
+                traces.set(result.file, {
+                    contributions: [detail],
+                    baseScore: contribution,
+                    topRank: rank,
+                    topRankBonus: 0,
+                    totalScore: 0,
+                });
+            }
+        }
+    }
+    for (const trace of traces.values()) {
+        let bonus = 0;
+        if (trace.topRank === 1)
+            bonus = 0.05;
+        else if (trace.topRank <= 3)
+            bonus = 0.02;
+        trace.topRankBonus = bonus;
+        trace.totalScore = trace.baseScore + bonus;
+    }
+    return traces;
+}
 /**
  * Find a document by filename/path, docid (#hash), or with fuzzy matching.
  * Returns document metadata without body by default.
@@ -2264,8 +2339,10 @@ export async function hybridQuery(store, query, options) {
     const minScore = options?.minScore ?? 0;
     const candidateLimit = options?.candidateLimit ?? RERANK_CANDIDATE_LIMIT;
     const collection = options?.collection;
+    const explain = options?.explain ?? false;
     const hooks = options?.hooks;
     const rankedLists = [];
+    const rankedListMeta = [];
     const docidMap = new Map(); // filepath -> docid
     const hasVectors = !!store.db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get();
     // Step 1: BM25 probe — strong signal skips expensive LLM expansion
@@ -2293,6 +2370,7 @@ export async function hybridQuery(store, query, options) {
             file: r.filepath, displayPath: r.displayPath,
             title: r.title, body: r.body || "", score: r.score,
         })));
+        rankedListMeta.push({ source: "fts", queryType: "original", query });
     }
     // Step 3: Route searches by query type
     //
@@ -2310,17 +2388,18 @@ export async function hybridQuery(store, query, options) {
                     file: r.filepath, displayPath: r.displayPath,
                     title: r.title, body: r.body || "", score: r.score,
                 })));
+                rankedListMeta.push({ source: "fts", queryType: "lex", query: q.text });
             }
         }
     }
     // 3b: Collect all texts that need vector search (original query + vec/hyde expansions)
     if (hasVectors) {
         const vecQueries = [
-            { text: query, isOriginal: true },
+            { text: query, queryType: "original" },
         ];
         for (const q of expanded) {
             if (q.type === 'vec' || q.type === 'hyde') {
-                vecQueries.push({ text: q.text, isOriginal: false });
+                vecQueries.push({ text: q.text, queryType: q.type });
             }
         }
         // Batch embed all vector queries in a single call
@@ -2343,12 +2422,18 @@ export async function hybridQuery(store, query, options) {
                     file: r.filepath, displayPath: r.displayPath,
                     title: r.title, body: r.body || "", score: r.score,
                 })));
+                rankedListMeta.push({
+                    source: "vec",
+                    queryType: vecQueries[i].queryType,
+                    query: vecQueries[i].text,
+                });
             }
         }
     }
     // Step 4: RRF fusion — first 2 lists (original FTS + first vec) get 2x weight
     const weights = rankedLists.map((_, i) => i < 2 ? 2.0 : 1.0);
     const fused = reciprocalRankFusion(rankedLists, weights);
+    const rrfTraceByFile = explain ? buildRrfTrace(rankedLists, weights, rankedListMeta) : null;
     const candidates = fused.slice(0, candidateLimit);
     if (candidates.length === 0)
         return [];
@@ -2402,6 +2487,22 @@ export async function hybridQuery(store, query, options) {
         const bestIdx = chunkInfo?.bestIdx ?? 0;
         const bestChunk = chunkInfo?.chunks[bestIdx]?.text || candidate?.body || "";
         const bestChunkPos = chunkInfo?.chunks[bestIdx]?.pos || 0;
+        const trace = rrfTraceByFile?.get(r.file);
+        const explainData = explain ? {
+            ftsScores: trace?.contributions.filter(c => c.source === "fts").map(c => c.backendScore) ?? [],
+            vectorScores: trace?.contributions.filter(c => c.source === "vec").map(c => c.backendScore) ?? [],
+            rrf: {
+                rank: rrfRank,
+                positionScore: rrfScore,
+                weight: rrfWeight,
+                baseScore: trace?.baseScore ?? 0,
+                topRankBonus: trace?.topRankBonus ?? 0,
+                totalScore: trace?.totalScore ?? 0,
+                contributions: trace?.contributions ?? [],
+            },
+            rerankScore: r.score,
+            blendedScore,
+        } : undefined;
         return {
             file: r.file,
             displayPath: candidate?.displayPath || "",
@@ -2412,6 +2513,7 @@ export async function hybridQuery(store, query, options) {
             score: blendedScore,
             context: store.getContextForFile(r.file),
             docid: docidMap.get(r.file) || "",
+            ...(explainData ? { explain: explainData } : {}),
         };
     }).sort((a, b) => b.score - a.score);
     // Step 8: Dedup by file (safety net — prevents duplicate output)
@@ -2494,6 +2596,7 @@ export async function structuredSearch(store, searches, options) {
     const limit = options?.limit ?? 10;
     const minScore = options?.minScore ?? 0;
     const candidateLimit = options?.candidateLimit ?? RERANK_CANDIDATE_LIMIT;
+    const explain = options?.explain ?? false;
     const hooks = options?.hooks;
     const collections = options?.collections;
     if (searches.length === 0)
@@ -2518,6 +2621,7 @@ export async function structuredSearch(store, searches, options) {
         }
     }
     const rankedLists = [];
+    const rankedListMeta = [];
     const docidMap = new Map(); // filepath -> docid
     const hasVectors = !!store.db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get();
     // Helper to run search across collections (or all if undefined)
@@ -2534,13 +2638,18 @@ export async function structuredSearch(store, searches, options) {
                         file: r.filepath, displayPath: r.displayPath,
                         title: r.title, body: r.body || "", score: r.score,
                     })));
+                    rankedListMeta.push({
+                        source: "fts",
+                        queryType: "lex",
+                        query: search.query,
+                    });
                 }
             }
         }
     }
     // Step 2: Batch embed and run vector searches for vec/hyde
     if (hasVectors) {
-        const vecSearches = searches.filter(s => s.type === 'vec' || s.type === 'hyde');
+        const vecSearches = searches.filter((s) => s.type === 'vec' || s.type === 'hyde');
         if (vecSearches.length > 0) {
             const llm = getDefaultLlamaCpp();
             const textsToEmbed = vecSearches.map(s => formatQueryForEmbedding(s.query));
@@ -2561,6 +2670,11 @@ export async function structuredSearch(store, searches, options) {
                             file: r.filepath, displayPath: r.displayPath,
                             title: r.title, body: r.body || "", score: r.score,
                         })));
+                        rankedListMeta.push({
+                            source: "vec",
+                            queryType: vecSearches[i].type,
+                            query: vecSearches[i].query,
+                        });
                     }
                 }
             }
@@ -2571,6 +2685,7 @@ export async function structuredSearch(store, searches, options) {
     // Step 3: RRF fusion — first list gets 2x weight (assume caller ordered by importance)
     const weights = rankedLists.map((_, i) => i === 0 ? 2.0 : 1.0);
     const fused = reciprocalRankFusion(rankedLists, weights);
+    const rrfTraceByFile = explain ? buildRrfTrace(rankedLists, weights, rankedListMeta) : null;
     const candidates = fused.slice(0, candidateLimit);
     if (candidates.length === 0)
         return [];
@@ -2627,6 +2742,22 @@ export async function structuredSearch(store, searches, options) {
         const bestIdx = chunkInfo?.bestIdx ?? 0;
         const bestChunk = chunkInfo?.chunks[bestIdx]?.text || candidate?.body || "";
         const bestChunkPos = chunkInfo?.chunks[bestIdx]?.pos || 0;
+        const trace = rrfTraceByFile?.get(r.file);
+        const explainData = explain ? {
+            ftsScores: trace?.contributions.filter(c => c.source === "fts").map(c => c.backendScore) ?? [],
+            vectorScores: trace?.contributions.filter(c => c.source === "vec").map(c => c.backendScore) ?? [],
+            rrf: {
+                rank: rrfRank,
+                positionScore: rrfScore,
+                weight: rrfWeight,
+                baseScore: trace?.baseScore ?? 0,
+                topRankBonus: trace?.topRankBonus ?? 0,
+                totalScore: trace?.totalScore ?? 0,
+                contributions: trace?.contributions ?? [],
+            },
+            rerankScore: r.score,
+            blendedScore,
+        } : undefined;
         return {
             file: r.file,
             displayPath: candidate?.displayPath || "",
@@ -2637,6 +2768,7 @@ export async function structuredSearch(store, searches, options) {
             score: blendedScore,
             context: store.getContextForFile(r.file),
             docid: docidMap.get(r.file) || "",
+            ...(explainData ? { explain: explainData } : {}),
         };
     }).sort((a, b) => b.score - a.score);
     // Step 7: Dedup by file

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tobilu/qmd",
-  "version": "1.1.1",
+  "version": "1.1.2",
   "description": "Query Markup Documents - On-device hybrid search for markdown files with BM25, vector search, and LLM reranking",
   "type": "module",
   "bin": {
@@ -39,7 +39,7 @@
     "@modelcontextprotocol/sdk": "^1.25.1",
     "better-sqlite3": "^11.0.0",
     "fast-glob": "^3.3.0",
-    "node-llama-cpp": "^3.14.5",
+    "node-llama-cpp": "^3.17.1",
     "picomatch": "^4.0.0",
     "sqlite-vec": "^0.1.7-alpha.2",
     "yaml": "^2.8.2",
@@ -48,8 +48,9 @@
   "optionalDependencies": {
     "sqlite-vec-darwin-arm64": "^0.1.7-alpha.2",
     "sqlite-vec-darwin-x64": "^0.1.7-alpha.2",
+    "sqlite-vec-linux-arm64": "^0.1.7-alpha.2",
     "sqlite-vec-linux-x64": "^0.1.7-alpha.2",
-    "sqlite-vec-win32-x64": "^0.1.7-alpha.2"
+    "sqlite-vec-windows-x64": "^0.1.7-alpha.2"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.0",