@tobilu/qmd 1.1.1 → 1.1.2

package/dist/qmd.js

@@ -74,19 +74,24 @@ const cursor = {
 // Ensure cursor is restored on exit
 process.on('SIGINT', () => { cursor.show(); process.exit(130); });
 process.on('SIGTERM', () => { cursor.show(); process.exit(143); });
-// Terminal progress bar using OSC 9;4 escape sequence
+// Terminal progress bar using OSC 9;4 escape sequence (TTY only)
+const isTTY = process.stderr.isTTY;
 const progress = {
     set(percent) {
-        process.stderr.write(`\x1b]9;4;1;${Math.round(percent)}\x07`);
+        if (isTTY)
+            process.stderr.write(`\x1b]9;4;1;${Math.round(percent)}\x07`);
     },
     clear() {
-        process.stderr.write(`\x1b]9;4;0\x07`);
+        if (isTTY)
+            process.stderr.write(`\x1b]9;4;0\x07`);
     },
     indeterminate() {
-        process.stderr.write(`\x1b]9;4;3\x07`);
+        if (isTTY)
+            process.stderr.write(`\x1b]9;4;3\x07`);
     },
     error() {
-        process.stderr.write(`\x1b]9;4;2\x07`);
+        if (isTTY)
+            process.stderr.write(`\x1b]9;4;2\x07`);
     },
 };
 // Format seconds into human-readable ETA
@@ -398,7 +403,7 @@ async function updateCollections() {
                 process.exit(1);
             }
         }
-        await indexFiles(col.pwd, col.glob_pattern, col.name, true);
+        await indexFiles(col.pwd, col.glob_pattern, col.name, true, yamlCol?.ignore);
         console.log("");
     }
     // Check if any documents need embedding (show once at end)
@@ -1103,6 +1108,9 @@ function collectionList() {
         const excludeTag = excluded ? ` ${c.yellow}[excluded]${c.reset}` : '';
         console.log(`${c.cyan}${coll.name}${c.reset} ${c.dim}(qmd://${coll.name}/)${c.reset}${excludeTag}`);
         console.log(`  ${c.dim}Pattern:${c.reset}  ${coll.glob_pattern}`);
+        if (yamlColl?.ignore?.length) {
+            console.log(`  ${c.dim}Ignore:${c.reset}   ${yamlColl.ignore.join(', ')}`);
+        }
         console.log(`  ${c.dim}Files:${c.reset}    ${coll.active_count}`);
         console.log(`  ${c.dim}Updated:${c.reset}  ${timeAgo}`);
         console.log();
@@ -1138,7 +1146,8 @@ async function collectionAdd(pwd, globPattern, name) {
     addCollection(collName, pwd, globPattern);
     // Create the collection and index files
     console.log(`Creating collection '${collName}'...`);
-    await indexFiles(pwd, globPattern, collName);
+    const newColl = getCollectionFromYaml(collName);
+    await indexFiles(pwd, globPattern, collName, false, newColl?.ignore);
     console.log(`${c.green}✓${c.reset} Collection '${collName}' created successfully`);
 }
 function collectionRemove(name) {
@@ -1179,7 +1188,7 @@ function collectionRename(oldName, newName) {
     console.log(`${c.green}✓${c.reset} Renamed collection '${oldName}' to '${newName}'`);
     console.log(`  Virtual paths updated: ${c.cyan}qmd://${oldName}/${c.reset} → ${c.cyan}qmd://${newName}/${c.reset}`);
 }
-async function indexFiles(pwd, globPattern = DEFAULT_GLOB, collectionName, suppressEmbedNotice = false) {
+async function indexFiles(pwd, globPattern = DEFAULT_GLOB, collectionName, suppressEmbedNotice = false, ignorePatterns) {
     const db = getDb();
     const resolvedPwd = pwd || getPwd();
     const now = new Date().toISOString();
@@ -1192,12 +1201,16 @@ async function indexFiles(pwd, globPattern = DEFAULT_GLOB, collectionName, suppr
     }
     console.log(`Collection: ${resolvedPwd} (${globPattern})`);
     progress.indeterminate();
+    const allIgnore = [
+        ...excludeDirs.map(d => `**/${d}/**`),
+        ...(ignorePatterns || []),
+    ];
     const allFiles = await fastGlob(globPattern, {
         cwd: resolvedPwd,
         onlyFiles: true,
         followSymbolicLinks: false,
         dot: false,
-        ignore: excludeDirs.map(d => `**/${d}/**`),
+        ignore: allIgnore,
     });
     // Filter hidden files/folders (dot: false handles top-level but not nested)
     const files = allFiles.filter(file => {
@@ -1205,11 +1218,11 @@ async function indexFiles(pwd, globPattern = DEFAULT_GLOB, collectionName, suppr
         return !parts.some(part => part.startsWith("."));
     });
     const total = files.length;
-    if (total === 0) {
+    const hasNoFiles = total === 0;
+    if (hasNoFiles) {
         progress.clear();
         console.log("No files found matching pattern.");
-        closeDb();
-        return;
+        // Continue so the deactivation pass can mark previously indexed docs as inactive.
     }
     let indexed = 0, updated = 0, unchanged = 0, processed = 0;
     const seenPaths = new Set();
@@ -1218,7 +1231,16 @@ async function indexFiles(pwd, globPattern = DEFAULT_GLOB, collectionName, suppr
         const filepath = getRealPath(resolve(resolvedPwd, relativeFile));
         const path = handelize(relativeFile); // Normalize path for token-friendliness
         seenPaths.add(path);
-        const content = readFileSync(filepath, "utf-8");
+        let content;
+        try {
+            content = readFileSync(filepath, "utf-8");
+        }
+        catch (err) {
+            // Skip files that can't be read (e.g. iCloud evicted files returning EAGAIN)
+            processed++;
+            progress.set((processed / total) * 100);
+            continue;
+        }
         // Skip empty files - nothing useful to index
         if (!content.trim()) {
             processed++;
@@ -1260,7 +1282,8 @@ async function indexFiles(pwd, globPattern = DEFAULT_GLOB, collectionName, suppr
         const rate = processed / elapsed;
         const remaining = (total - processed) / rate;
         const eta = processed > 2 ? ` ETA: ${formatETA(remaining)}` : "";
-        process.stderr.write(`\rIndexing: ${processed}/${total}${eta}        `);
+        if (isTTY)
+            process.stderr.write(`\rIndexing: ${processed}/${total}${eta}        `);
     }
     // Deactivate documents in this collection that no longer exist
     const allActive = getActiveDocumentPaths(db, collectionName);
@@ -1423,7 +1446,8 @@ async function vectorIndex(model = DEFAULT_EMBED_MODEL, force = false) {
             const throughput = `${formatBytes(bytesPerSec)}/s`;
             const eta = elapsed > 2 ? formatETA(etaSec) : "...";
             const errStr = errors > 0 ? ` ${c.yellow}${errors} err${c.reset}` : "";
-            process.stderr.write(`\r${c.cyan}${bar}${c.reset} ${c.bold}${percentStr}%${c.reset} ${c.dim}${chunksEmbedded}/${totalChunks}${c.reset}${errStr} ${c.dim}${throughput} ETA ${eta}${c.reset}   `);
+            if (isTTY)
+                process.stderr.write(`\r${c.cyan}${bar}${c.reset} ${c.bold}${percentStr}%${c.reset} ${c.dim}${chunksEmbedded}/${totalChunks}${c.reset}${errStr} ${c.dim}${throughput} ETA ${eta}${c.reset}   `);
         }
         progress.clear();
         cursor.show();
@@ -1496,6 +1520,9 @@ function formatScore(score) {
         return `${c.yellow}${pct}%${c.reset}`;
     return `${c.dim}${pct}%${c.reset}`;
 }
+function formatExplainNumber(value) {
+    return value.toFixed(4);
+}
 // Shorten directory path for display - relative to $HOME (used for context paths, not documents)
 function shortPath(dirpath) {
     const home = homedir();
@@ -1504,10 +1531,33 @@ function shortPath(dirpath) {
     }
     return dirpath;
 }
+// Emit format-safe empty output for search commands.
+function printEmptySearchResults(format, reason = "no_results") {
+    if (format === "json") {
+        console.log("[]");
+        return;
+    }
+    if (format === "csv") {
+        console.log("docid,score,file,title,context,line,snippet");
+        return;
+    }
+    if (format === "xml") {
+        console.log("<results></results>");
+        return;
+    }
+    if (format === "md" || format === "files") {
+        return;
+    }
+    if (reason === "min_score") {
+        console.log("No results found above minimum score threshold.");
+        return;
+    }
+    console.log("No results found.");
+}
 function outputResults(results, query, opts) {
     const filtered = results.filter(r => r.score >= opts.minScore).slice(0, opts.limit);
     if (filtered.length === 0) {
-        console.log("No results found above minimum score threshold.");
+        printEmptySearchResults(opts.format, "min_score");
         return;
     }
     // Helper to create qmd:// URI from displayPath
@@ -1532,6 +1582,7 @@ function outputResults(results, query, opts) {
                 ...(row.context && { context: row.context }),
                 ...(body && { body }),
                 ...(snippet && { snippet }),
+                ...(opts.explain && row.explain && { explain: row.explain }),
             };
         });
         console.log(JSON.stringify(output, null, 2));
@@ -1570,6 +1621,27 @@ function outputResults(results, query, opts) {
             // Line 4: Score
             const score = formatScore(row.score);
             console.log(`Score: ${c.bold}${score}${c.reset}`);
+            if (opts.explain && row.explain) {
+                const explain = row.explain;
+                const ftsScores = explain.ftsScores.length > 0
+                    ? explain.ftsScores.map(formatExplainNumber).join(", ")
+                    : "none";
+                const vecScores = explain.vectorScores.length > 0
+                    ? explain.vectorScores.map(formatExplainNumber).join(", ")
+                    : "none";
+                const contribSummary = explain.rrf.contributions
+                    .slice()
+                    .sort((a, b) => b.rrfContribution - a.rrfContribution)
+                    .slice(0, 3)
+                    .map(c => `${c.source}/${c.queryType}#${c.rank}:${formatExplainNumber(c.rrfContribution)}`)
+                    .join(" | ");
+                console.log(`${c.dim}Explain: fts=[${ftsScores}] vec=[${vecScores}]${c.reset}`);
+                console.log(`${c.dim}  RRF: total=${formatExplainNumber(explain.rrf.totalScore)} base=${formatExplainNumber(explain.rrf.baseScore)} bonus=${formatExplainNumber(explain.rrf.topRankBonus)} rank=${explain.rrf.rank}${c.reset}`);
+                console.log(`${c.dim}  Blend: ${Math.round(explain.rrf.weight * 100)}%*${formatExplainNumber(explain.rrf.positionScore)} + ${Math.round((1 - explain.rrf.weight) * 100)}%*${formatExplainNumber(explain.rerankScore)} = ${formatExplainNumber(explain.blendedScore)}${c.reset}`);
+                if (contribSummary.length > 0) {
+                    console.log(`${c.dim}  Top RRF contributions: ${contribSummary}${c.reset}`);
+                }
+            }
             console.log();
             // Snippet with highlighting (diff-style header included)
             let displaySnippet = opts.lineNumbers ? addLineNumbers(snippet, line) : snippet;
@@ -1735,12 +1807,7 @@ function search(query, opts) {
     }));
     closeDb();
     if (resultsWithContext.length === 0) {
-        if (opts.format === "json") {
-            console.log("[]");
-        }
-        else {
-            console.log("No results found.");
-        }
+        printEmptySearchResults(opts.format);
         return;
     }
     outputResults(resultsWithContext, query, opts);
@@ -1789,12 +1856,7 @@ async function vectorSearch(query, opts, _model = DEFAULT_EMBED_MODEL) {
         }
         closeDb();
         if (results.length === 0) {
-            if (opts.format === "json") {
-                console.log("[]");
-            }
-            else {
-                console.log("No results found.");
-            }
+            printEmptySearchResults(opts.format);
             return;
         }
         outputResults(results.map(r => ({
@@ -1835,6 +1897,8 @@ async function querySearch(query, opts, _embedModel = DEFAULT_EMBED_MODEL, _rera
                 collections: singleCollection ? [singleCollection] : undefined,
                 limit: opts.all ? 500 : (opts.limit || 10),
                 minScore: opts.minScore || 0,
+                candidateLimit: opts.candidateLimit,
+                explain: !!opts.explain,
                 hooks: {
                     onEmbedStart: (count) => {
                         process.stderr.write(`${c.dim}Embedding ${count} ${count === 1 ? 'query' : 'queries'}...${c.reset}`);
@@ -1859,6 +1923,8 @@ async function querySearch(query, opts, _embedModel = DEFAULT_EMBED_MODEL, _rera
                 collection: singleCollection,
                 limit: opts.all ? 500 : (opts.limit || 10),
                 minScore: opts.minScore || 0,
+                candidateLimit: opts.candidateLimit,
+                explain: !!opts.explain,
                 hooks: {
                     onStrongSignal: (score) => {
                         process.stderr.write(`${c.dim}Strong BM25 signal (${score.toFixed(2)}) — skipping expansion${c.reset}\n`);
@@ -1897,12 +1963,7 @@ async function querySearch(query, opts, _embedModel = DEFAULT_EMBED_MODEL, _rera
         }
         closeDb();
         if (results.length === 0) {
-            if (opts.format === "json") {
-                console.log("[]");
-            }
-            else {
-                console.log("No results found.");
-            }
+            printEmptySearchResults(opts.format);
             return;
         }
         // Use first lex/vec query for output context, or original query
@@ -1919,6 +1980,7 @@ async function querySearch(query, opts, _embedModel = DEFAULT_EMBED_MODEL, _rera
             score: r.score,
             context: r.context,
             docid: r.docid,
+            explain: r.explain,
         })), displayQuery, { ...opts, limit: results.length });
     }, { maxDuration: 10 * 60 * 1000, name: 'querySearch' });
 }
@@ -1947,6 +2009,7 @@ function parseCLI() {
             xml: { type: "boolean" },
             files: { type: "boolean" },
             json: { type: "boolean" },
+            explain: { type: "boolean" },
             collection: { type: "string", short: "c", multiple: true }, // Filter by collection(s)
             // Collection options
             name: { type: "string" }, // collection name
@@ -1961,6 +2024,8 @@ function parseCLI() {
             from: { type: "string" }, // start line
             "max-bytes": { type: "string" }, // max bytes for multi-get
             "line-numbers": { type: "boolean" }, // add line numbers to output
+            // Query options
+            "candidate-limit": { type: "string", short: "C" },
             // MCP HTTP transport options
             http: { type: "boolean" },
             daemon: { type: "boolean" },
@@ -1999,6 +2064,8 @@ function parseCLI() {
         all: isAll,
         collection: values.collection,
         lineNumbers: !!values["line-numbers"],
+        candidateLimit: values["candidate-limit"] ? parseInt(String(values["candidate-limit"]), 10) : undefined,
+        explain: !!values.explain,
     };
     return {
         command: positionals[0] || "",
@@ -2094,7 +2161,9 @@ function showHelp() {
     console.log("  --all                      - Return all matches (pair with --min-score)");
     console.log("  --min-score <num>          - Minimum similarity score");
     console.log("  --full                     - Output full document instead of snippet");
+    console.log("  -C, --candidate-limit <n>  - Max candidates to rerank (default 40, lower = faster)");
     console.log("  --line-numbers             - Include line numbers in output");
+    console.log("  --explain                  - Include retrieval score traces (query --json/CLI)");
     console.log("  --files | --json | --csv | --md | --xml  - Output format");
     console.log("  -c, --collection <name>    - Filter by one or more collections");
     console.log("");

package/dist/store.d.ts

@@ -280,15 +280,6 @@ export type DocumentResult = {
  * Extract short docid from a full hash (first 6 characters).
  */
 export declare function getDocid(hash: string): string;
-/**
- * Handelize a filename to be more token-friendly.
- * - Convert triple underscore `___` to `/` (folder separator)
- * - Convert to lowercase
- * - Replace sequences of non-word chars (except /) with single dash
- * - Remove leading/trailing dashes from path segments
- * - Preserve folder structure (a/b/c/d.md stays structured)
- * - Preserve file extension
- */
 export declare function handelize(path: string): string;
 /**
  * Search result extends DocumentResult with score and source info
@@ -308,6 +299,38 @@ export type RankedResult = {
     body: string;
     score: number;
 };
+export type RRFContributionTrace = {
+    listIndex: number;
+    source: "fts" | "vec";
+    queryType: "original" | "lex" | "vec" | "hyde";
+    query: string;
+    rank: number;
+    weight: number;
+    backendScore: number;
+    rrfContribution: number;
+};
+export type RRFScoreTrace = {
+    contributions: RRFContributionTrace[];
+    baseScore: number;
+    topRank: number;
+    topRankBonus: number;
+    totalScore: number;
+};
+export type HybridQueryExplain = {
+    ftsScores: number[];
+    vectorScores: number[];
+    rrf: {
+        rank: number;
+        positionScore: number;
+        weight: number;
+        baseScore: number;
+        topRankBonus: number;
+        totalScore: number;
+        contributions: RRFContributionTrace[];
+    };
+    rerankScore: number;
+    blendedScore: number;
+};
 /**
  * Error result when document is not found
  */
@@ -584,6 +607,10 @@ export declare function rerank(query: string, documents: {
     score: number;
 }[]>;
 export declare function reciprocalRankFusion(resultLists: RankedResult[][], weights?: number[], k?: number): RankedResult[];
+/**
+ * Build per-document RRF contribution traces for explain/debug output.
+ */
+export declare function buildRrfTrace(resultLists: RankedResult[][], weights?: number[], listMeta?: RankedListMeta[], k?: number): Map<string, RRFScoreTrace>;
 /**
  * Find a document by filename/path, docid (#hash), or with fuzzy matching.
  * Returns document metadata without body by default.
@@ -654,6 +681,7 @@ export interface HybridQueryOptions {
     limit?: number;
     minScore?: number;
     candidateLimit?: number;
+    explain?: boolean;
     hooks?: SearchHooks;
 }
 export interface HybridQueryResult {
@@ -666,7 +694,13 @@ export interface HybridQueryResult {
     score: number;
     context: string | null;
     docid: string;
+    explain?: HybridQueryExplain;
 }
+export type RankedListMeta = {
+    source: "fts" | "vec";
+    queryType: "original" | "lex" | "vec" | "hyde";
+    query: string;
+};
 /**
  * Hybrid search: BM25 + vector + query expansion + RRF + chunked reranking.
  *
@@ -723,6 +757,7 @@ export interface StructuredSearchOptions {
     limit?: number;
     minScore?: number;
     candidateLimit?: number;
+    explain?: boolean;
     /** Future: domain intent hint for routing/boosting */
     intent?: string;
     hooks?: SearchHooks;