@tobilu/qmd 1.0.7 → 1.1.2

package/dist/qmd.js

@@ -1,14 +1,15 @@
+#!/usr/bin/env node
 import { openDatabase } from "./db.js";
 import fastGlob from "fast-glob";
 import { execSync, spawn as nodeSpawn } from "child_process";
 import { fileURLToPath } from "url";
 import { dirname, join as pathJoin } from "path";
 import { parseArgs } from "util";
-import { readFileSync, statSync, existsSync, unlinkSync, writeFileSync, openSync, closeSync, mkdirSync } from "fs";
-import { getPwd, getRealPath, homedir, resolve, enableProductionMode, searchFTS, extractSnippet, getContextForFile, getContextForPath, listCollections, removeCollection, renameCollection, findSimilarFiles, findDocumentByDocid, isDocid, matchFilesByGlob, getHashesNeedingEmbedding, getHashesForEmbedding, clearAllEmbeddings, insertEmbedding, getStatus, hashContent, extractTitle, formatDocForEmbedding, chunkDocumentByTokens, clearCache, getCacheKey, getCachedResult, setCachedResult, getIndexHealth, parseVirtualPath, buildVirtualPath, isVirtualPath, resolveVirtualPath, toVirtualPath, insertContent, insertDocument, findActiveDocument, updateDocumentTitle, updateDocument, deactivateDocument, getActiveDocumentPaths, cleanupOrphanedContent, deleteLLMCache, deleteInactiveDocuments, cleanupOrphanedVectors, vacuumDatabase, getCollectionsWithoutContext, getTopLevelPathsWithoutContext, handelize, hybridQuery, vectorSearchQuery, addLineNumbers, DEFAULT_EMBED_MODEL, DEFAULT_RERANK_MODEL, DEFAULT_GLOB, DEFAULT_MULTI_GET_MAX_BYTES, createStore, getDefaultDbPath, } from "./store.js";
+import { readFileSync, realpathSync, statSync, existsSync, unlinkSync, writeFileSync, openSync, closeSync, mkdirSync } from "fs";
+import { getPwd, getRealPath, homedir, resolve, enableProductionMode, searchFTS, extractSnippet, getContextForFile, getContextForPath, listCollections, removeCollection, renameCollection, findSimilarFiles, findDocumentByDocid, isDocid, matchFilesByGlob, getHashesNeedingEmbedding, getHashesForEmbedding, clearAllEmbeddings, insertEmbedding, getStatus, hashContent, extractTitle, formatDocForEmbedding, chunkDocumentByTokens, clearCache, getCacheKey, getCachedResult, setCachedResult, getIndexHealth, parseVirtualPath, buildVirtualPath, isVirtualPath, resolveVirtualPath, toVirtualPath, insertContent, insertDocument, findActiveDocument, updateDocumentTitle, updateDocument, deactivateDocument, getActiveDocumentPaths, cleanupOrphanedContent, deleteLLMCache, deleteInactiveDocuments, cleanupOrphanedVectors, vacuumDatabase, getCollectionsWithoutContext, getTopLevelPathsWithoutContext, handelize, hybridQuery, vectorSearchQuery, structuredSearch, addLineNumbers, DEFAULT_EMBED_MODEL, DEFAULT_RERANK_MODEL, DEFAULT_GLOB, DEFAULT_MULTI_GET_MAX_BYTES, createStore, getDefaultDbPath, } from "./store.js";
 import { disposeDefaultLlamaCpp, getDefaultLlamaCpp, withLLMSession, pullModels, DEFAULT_EMBED_MODEL_URI, DEFAULT_GENERATE_MODEL_URI, DEFAULT_RERANK_MODEL_URI, DEFAULT_MODEL_CACHE_DIR } from "./llm.js";
 import { formatSearchResults, formatDocuments, escapeXml, escapeCSV, } from "./formatter.js";
-import { getCollection as getCollectionFromYaml, listCollections as yamlListCollections, addContext as yamlAddContext, removeContext as yamlRemoveContext, setGlobalContext, listAllContexts, setConfigIndexName, } from "./collections.js";
+import { getCollection as getCollectionFromYaml, listCollections as yamlListCollections, getDefaultCollectionNames, addContext as yamlAddContext, removeContext as yamlRemoveContext, setGlobalContext, listAllContexts, setConfigIndexName, } from "./collections.js";
 // Enable production mode - allows using default database path
 // Tests must set INDEX_PATH or use createStore() with explicit path
 enableProductionMode();
@@ -73,19 +74,24 @@ const cursor = {
 // Ensure cursor is restored on exit
 process.on('SIGINT', () => { cursor.show(); process.exit(130); });
 process.on('SIGTERM', () => { cursor.show(); process.exit(143); });
-// Terminal progress bar using OSC 9;4 escape sequence
+// Terminal progress bar using OSC 9;4 escape sequence (TTY only)
+const isTTY = process.stderr.isTTY;
 const progress = {
     set(percent) {
-        process.stderr.write(`\x1b]9;4;1;${Math.round(percent)}\x07`);
+        if (isTTY)
+            process.stderr.write(`\x1b]9;4;1;${Math.round(percent)}\x07`);
     },
     clear() {
-        process.stderr.write(`\x1b]9;4;0\x07`);
+        if (isTTY)
+            process.stderr.write(`\x1b]9;4;0\x07`);
     },
     indeterminate() {
-        process.stderr.write(`\x1b]9;4;3\x07`);
+        if (isTTY)
+            process.stderr.write(`\x1b]9;4;3\x07`);
     },
     error() {
-        process.stderr.write(`\x1b]9;4;2\x07`);
+        if (isTTY)
+            process.stderr.write(`\x1b]9;4;2\x07`);
     },
 };
 // Format seconds into human-readable ETA
@@ -155,6 +161,11 @@ function formatTimeAgo(date) {
     const days = Math.floor(hours / 24);
     return `${days}d ago`;
 }
+function formatMs(ms) {
+    if (ms < 1000)
+        return `${ms}ms`;
+    return `${(ms / 1000).toFixed(1)}s`;
+}
 function formatBytes(bytes) {
     if (bytes < 1024)
         return `${bytes} B`;
@@ -308,6 +319,37 @@ async function showStatus() {
     catch {
         // Don't fail status if LLM init fails
     }
+    // Tips section
+    const tips = [];
+    // Check for collections without context
+    const collectionsWithoutContext = collections.filter(col => {
+        const contexts = contextsByCollection.get(col.name) || [];
+        return contexts.length === 0;
+    });
+    if (collectionsWithoutContext.length > 0) {
+        const names = collectionsWithoutContext.map(c => c.name).slice(0, 3).join(', ');
+        const more = collectionsWithoutContext.length > 3 ? ` +${collectionsWithoutContext.length - 3} more` : '';
+        tips.push(`Add context to collections for better search results: ${names}${more}`);
+        tips.push(`  ${c.dim}qmd context add qmd://<name>/ "What this collection contains"${c.reset}`);
+        tips.push(`  ${c.dim}qmd context add qmd://<name>/meeting-notes "Weekly team meeting notes"${c.reset}`);
+    }
+    // Check for collections without update commands
+    const collectionsWithoutUpdate = collections.filter(col => {
+        const yamlCol = getCollectionFromYaml(col.name);
+        return !yamlCol?.update;
+    });
+    if (collectionsWithoutUpdate.length > 0 && collections.length > 1) {
+        const names = collectionsWithoutUpdate.map(c => c.name).slice(0, 3).join(', ');
+        const more = collectionsWithoutUpdate.length > 3 ? ` +${collectionsWithoutUpdate.length - 3} more` : '';
+        tips.push(`Add update commands to keep collections fresh: ${names}${more}`);
+        tips.push(`  ${c.dim}qmd collection update-cmd <name> 'git stash && git pull --rebase --ff-only && git stash pop'${c.reset}`);
+    }
+    if (tips.length > 0) {
+        console.log(`\n${c.bold}Tips${c.reset}`);
+        for (const tip of tips) {
+            console.log(`  ${tip}`);
+        }
+    }
     closeDb();
 }
 async function updateCollections() {
@@ -361,7 +403,7 @@ async function updateCollections() {
                 process.exit(1);
             }
         }
-        await indexFiles(col.pwd, col.glob_pattern, col.name, true);
+        await indexFiles(col.pwd, col.glob_pattern, col.name, true, yamlCol?.ignore);
         console.log("");
     }
     // Check if any documents need embedding (show once at end)
@@ -533,49 +575,6 @@ function contextRemove(pathArg) {
     }
     console.log(`${c.green}✓${c.reset} Removed context for: qmd://${detected.collectionName}/${detected.relativePath}`);
 }
-function contextCheck() {
-    const db = getDb();
-    // Get collections without any context
-    const collectionsWithoutContext = getCollectionsWithoutContext(db);
-    // Get all collections to check for missing path contexts
-    const allCollections = listCollections(db);
-    if (collectionsWithoutContext.length === 0 && allCollections.length > 0) {
-        // Check if all collections have contexts
-        console.log(`\n${c.green}✓${c.reset} ${c.bold}All collections have context configured${c.reset}\n`);
-    }
-    if (collectionsWithoutContext.length > 0) {
-        console.log(`\n${c.yellow}Collections without any context:${c.reset}\n`);
-        for (const coll of collectionsWithoutContext) {
-            console.log(`${c.cyan}${coll.name}${c.reset} ${c.dim}(${coll.doc_count} documents)${c.reset}`);
-            console.log(`  ${c.dim}Suggestion: qmd context add qmd://${coll.name}/ "Description of ${coll.name}"${c.reset}\n`);
-        }
-    }
-    // Check for top-level paths without context within collections that DO have context
-    const collectionsWithContext = allCollections.filter(c => c && !collectionsWithoutContext.some(cwc => cwc.name === c.name));
-    let hasPathSuggestions = false;
-    for (const coll of collectionsWithContext) {
-        if (!coll)
-            continue;
-        const missingPaths = getTopLevelPathsWithoutContext(db, coll.name);
-        if (missingPaths.length > 0) {
-            if (!hasPathSuggestions) {
-                console.log(`${c.yellow}Top-level directories without context:${c.reset}\n`);
-                hasPathSuggestions = true;
-            }
-            console.log(`${c.cyan}${coll.name}${c.reset}`);
-            for (const path of missingPaths) {
-                console.log(`  ${path}`);
-                console.log(`    ${c.dim}Suggestion: qmd context add qmd://${coll.name}/${path} "Description of ${path}"${c.reset}`);
-            }
-            console.log('');
-        }
-    }
-    if (collectionsWithoutContext.length === 0 && !hasPathSuggestions) {
-        console.log(`${c.dim}All collections and major paths have context configured.${c.reset}`);
-        console.log(`${c.dim}Use 'qmd context list' to see all configured contexts.${c.reset}\n`);
-    }
-    closeDb();
-}
 function getDocument(filename, fromLine, maxLines, lineNumbers) {
     const db = getDb();
     // Parse :linenum suffix from filename (e.g., "file.md:100")
@@ -1103,8 +1102,15 @@ function collectionList() {
     for (const coll of collections) {
         const updatedAt = coll.last_modified ? new Date(coll.last_modified) : new Date();
         const timeAgo = formatTimeAgo(updatedAt);
-        console.log(`${c.cyan}${coll.name}${c.reset} ${c.dim}(qmd://${coll.name}/)${c.reset}`);
+        // Get YAML config to check includeByDefault
+        const yamlColl = getCollectionFromYaml(coll.name);
+        const excluded = yamlColl?.includeByDefault === false;
+        const excludeTag = excluded ? ` ${c.yellow}[excluded]${c.reset}` : '';
+        console.log(`${c.cyan}${coll.name}${c.reset} ${c.dim}(qmd://${coll.name}/)${c.reset}${excludeTag}`);
         console.log(`  ${c.dim}Pattern:${c.reset}  ${coll.glob_pattern}`);
+        if (yamlColl?.ignore?.length) {
+            console.log(`  ${c.dim}Ignore:${c.reset}   ${yamlColl.ignore.join(', ')}`);
+        }
         console.log(`  ${c.dim}Files:${c.reset}    ${coll.active_count}`);
         console.log(`  ${c.dim}Updated:${c.reset}  ${timeAgo}`);
         console.log();
@@ -1140,7 +1146,8 @@ async function collectionAdd(pwd, globPattern, name) {
     addCollection(collName, pwd, globPattern);
     // Create the collection and index files
     console.log(`Creating collection '${collName}'...`);
-    await indexFiles(pwd, globPattern, collName);
+    const newColl = getCollectionFromYaml(collName);
+    await indexFiles(pwd, globPattern, collName, false, newColl?.ignore);
     console.log(`${c.green}✓${c.reset} Collection '${collName}' created successfully`);
 }
 function collectionRemove(name) {
@@ -1181,7 +1188,7 @@ function collectionRename(oldName, newName) {
     console.log(`${c.green}✓${c.reset} Renamed collection '${oldName}' to '${newName}'`);
     console.log(`  Virtual paths updated: ${c.cyan}qmd://${oldName}/${c.reset} → ${c.cyan}qmd://${newName}/${c.reset}`);
 }
-async function indexFiles(pwd, globPattern = DEFAULT_GLOB, collectionName, suppressEmbedNotice = false) {
+async function indexFiles(pwd, globPattern = DEFAULT_GLOB, collectionName, suppressEmbedNotice = false, ignorePatterns) {
     const db = getDb();
     const resolvedPwd = pwd || getPwd();
     const now = new Date().toISOString();
@@ -1194,12 +1201,16 @@ async function indexFiles(pwd, globPattern = DEFAULT_GLOB, collectionName, suppr
     }
     console.log(`Collection: ${resolvedPwd} (${globPattern})`);
     progress.indeterminate();
+    const allIgnore = [
+        ...excludeDirs.map(d => `**/${d}/**`),
+        ...(ignorePatterns || []),
+    ];
     const allFiles = await fastGlob(globPattern, {
         cwd: resolvedPwd,
         onlyFiles: true,
         followSymbolicLinks: false,
         dot: false,
-        ignore: excludeDirs.map(d => `**/${d}/**`),
+        ignore: allIgnore,
     });
     // Filter hidden files/folders (dot: false handles top-level but not nested)
     const files = allFiles.filter(file => {
@@ -1207,11 +1218,11 @@ async function indexFiles(pwd, globPattern = DEFAULT_GLOB, collectionName, suppr
         return !parts.some(part => part.startsWith("."));
     });
     const total = files.length;
-    if (total === 0) {
+    const hasNoFiles = total === 0;
+    if (hasNoFiles) {
         progress.clear();
         console.log("No files found matching pattern.");
-        closeDb();
-        return;
+        // Continue so the deactivation pass can mark previously indexed docs as inactive.
     }
     let indexed = 0, updated = 0, unchanged = 0, processed = 0;
     const seenPaths = new Set();
@@ -1220,7 +1231,16 @@ async function indexFiles(pwd, globPattern = DEFAULT_GLOB, collectionName, suppr
         const filepath = getRealPath(resolve(resolvedPwd, relativeFile));
         const path = handelize(relativeFile); // Normalize path for token-friendliness
         seenPaths.add(path);
-        const content = readFileSync(filepath, "utf-8");
+        let content;
+        try {
+            content = readFileSync(filepath, "utf-8");
+        }
+        catch (err) {
+            // Skip files that can't be read (e.g. iCloud evicted files returning EAGAIN)
+            processed++;
+            progress.set((processed / total) * 100);
+            continue;
+        }
         // Skip empty files - nothing useful to index
         if (!content.trim()) {
             processed++;
@@ -1262,7 +1282,8 @@ async function indexFiles(pwd, globPattern = DEFAULT_GLOB, collectionName, suppr
         const rate = processed / elapsed;
         const remaining = (total - processed) / rate;
         const eta = processed > 2 ? ` ETA: ${formatETA(remaining)}` : "";
-        process.stderr.write(`\rIndexing: ${processed}/${total}${eta}        `);
+        if (isTTY)
+            process.stderr.write(`\rIndexing: ${processed}/${total}${eta}        `);
     }
     // Deactivate documents in this collection that no longer exist
     const allActive = getActiveDocumentPaths(db, collectionName);
@@ -1425,7 +1446,8 @@ async function vectorIndex(model = DEFAULT_EMBED_MODEL, force = false) {
             const throughput = `${formatBytes(bytesPerSec)}/s`;
             const eta = elapsed > 2 ? formatETA(etaSec) : "...";
             const errStr = errors > 0 ? ` ${c.yellow}${errors} err${c.reset}` : "";
-            process.stderr.write(`\r${c.cyan}${bar}${c.reset} ${c.bold}${percentStr}%${c.reset} ${c.dim}${chunksEmbedded}/${totalChunks}${c.reset}${errStr} ${c.dim}${throughput} ETA ${eta}${c.reset}   `);
+            if (isTTY)
+                process.stderr.write(`\r${c.cyan}${bar}${c.reset} ${c.bold}${percentStr}%${c.reset} ${c.dim}${chunksEmbedded}/${totalChunks}${c.reset}${errStr} ${c.dim}${throughput} ETA ${eta}${c.reset}   `);
         }
         progress.clear();
         cursor.show();
@@ -1498,6 +1520,9 @@ function formatScore(score) {
         return `${c.yellow}${pct}%${c.reset}`;
     return `${c.dim}${pct}%${c.reset}`;
 }
+function formatExplainNumber(value) {
+    return value.toFixed(4);
+}
 // Shorten directory path for display - relative to $HOME (used for context paths, not documents)
 function shortPath(dirpath) {
     const home = homedir();
@@ -1506,10 +1531,33 @@ function shortPath(dirpath) {
     }
     return dirpath;
 }
+// Emit format-safe empty output for search commands.
+function printEmptySearchResults(format, reason = "no_results") {
+    if (format === "json") {
+        console.log("[]");
+        return;
+    }
+    if (format === "csv") {
+        console.log("docid,score,file,title,context,line,snippet");
+        return;
+    }
+    if (format === "xml") {
+        console.log("<results></results>");
+        return;
+    }
+    if (format === "md" || format === "files") {
+        return;
+    }
+    if (reason === "min_score") {
+        console.log("No results found above minimum score threshold.");
+        return;
+    }
+    console.log("No results found.");
+}
 function outputResults(results, query, opts) {
     const filtered = results.filter(r => r.score >= opts.minScore).slice(0, opts.limit);
     if (filtered.length === 0) {
-        console.log("No results found above minimum score threshold.");
+        printEmptySearchResults(opts.format, "min_score");
         return;
     }
     // Helper to create qmd:// URI from displayPath
@@ -1534,6 +1582,7 @@ function outputResults(results, query, opts) {
                 ...(row.context && { context: row.context }),
                 ...(body && { body }),
                 ...(snippet && { snippet }),
+                ...(opts.explain && row.explain && { explain: row.explain }),
             };
         });
         console.log(JSON.stringify(output, null, 2));
@@ -1572,6 +1621,27 @@ function outputResults(results, query, opts) {
             // Line 4: Score
             const score = formatScore(row.score);
             console.log(`Score: ${c.bold}${score}${c.reset}`);
+            if (opts.explain && row.explain) {
+                const explain = row.explain;
+                const ftsScores = explain.ftsScores.length > 0
+                    ? explain.ftsScores.map(formatExplainNumber).join(", ")
+                    : "none";
+                const vecScores = explain.vectorScores.length > 0
+                    ? explain.vectorScores.map(formatExplainNumber).join(", ")
+                    : "none";
+                const contribSummary = explain.rrf.contributions
+                    .slice()
+                    .sort((a, b) => b.rrfContribution - a.rrfContribution)
+                    .slice(0, 3)
+                    .map(c => `${c.source}/${c.queryType}#${c.rank}:${formatExplainNumber(c.rrfContribution)}`)
+                    .join(" | ");
+                console.log(`${c.dim}Explain: fts=[${ftsScores}] vec=[${vecScores}]${c.reset}`);
+                console.log(`${c.dim}  RRF: total=${formatExplainNumber(explain.rrf.totalScore)} base=${formatExplainNumber(explain.rrf.baseScore)} bonus=${formatExplainNumber(explain.rrf.topRankBonus)} rank=${explain.rrf.rank}${c.reset}`);
+                console.log(`${c.dim}  Blend: ${Math.round(explain.rrf.weight * 100)}%*${formatExplainNumber(explain.rrf.positionScore)} + ${Math.round((1 - explain.rrf.weight) * 100)}%*${formatExplainNumber(explain.rerankScore)} = ${formatExplainNumber(explain.blendedScore)}${c.reset}`);
+                if (contribSummary.length > 0) {
+                    console.log(`${c.dim}  Top RRF contributions: ${contribSummary}${c.reset}`);
+                }
+            }
             console.log();
             // Snippet with highlighting (diff-style header included)
             let displaySnippet = opts.lineNumbers ? addLineNumbers(snippet, line) : snippet;
@@ -1627,7 +1697,11 @@ function outputResults(results, query, opts) {
 }
 // Resolve -c collection filter: supports single string, array, or undefined.
 // Returns validated collection names (exits on unknown collection).
-function resolveCollectionFilter(raw) {
+function resolveCollectionFilter(raw, useDefaults = false) {
+    // If no filter specified and useDefaults is true, use default collections
+    if (!raw && useDefaults) {
+        return getDefaultCollectionNames();
+    }
     if (!raw)
         return [];
     const names = Array.isArray(raw) ? raw : [raw];
@@ -1653,10 +1727,69 @@ function filterByCollections(results, collectionNames) {
         return prefixes.some(p => path.startsWith(p));
     });
 }
+/**
+ * Parse structured search query syntax.
+ * Lines starting with lex:, vec:, or hyde: are routed directly.
+ * Plain lines without prefix go through query expansion.
+ *
+ * Returns null if this is a plain query (single line, no prefix).
+ * Returns StructuredSubSearch[] if structured syntax detected.
+ * Throws if multiple plain lines (ambiguous).
+ *
+ * Examples:
+ *   "CAP theorem"                    -> null (plain query, use expansion)
+ *   "lex: CAP theorem"               -> [{ type: 'lex', query: 'CAP theorem' }]
+ *   "lex: CAP\nvec: consistency"     -> [{ type: 'lex', ... }, { type: 'vec', ... }]
+ *   "CAP\nconsistency"               -> throws (multiple plain lines)
+ */
+function parseStructuredQuery(query) {
+    const rawLines = query.split('\n').map((line, idx) => ({
+        raw: line,
+        trimmed: line.trim(),
+        number: idx + 1,
+    })).filter(line => line.trimmed.length > 0);
+    if (rawLines.length === 0)
+        return null;
+    const prefixRe = /^(lex|vec|hyde):\s*/i;
+    const expandRe = /^expand:\s*/i;
+    const typed = [];
+    for (const line of rawLines) {
+        if (expandRe.test(line.trimmed)) {
+            if (rawLines.length > 1) {
+                throw new Error(`Line ${line.number} starts with expand:, but query documents cannot mix expand with typed lines. Submit a single expand query instead.`);
+            }
+            const text = line.trimmed.replace(expandRe, '').trim();
+            if (!text) {
+                throw new Error('expand: query must include text.');
+            }
+            return null; // treat as standalone expand query
+        }
+        const match = line.trimmed.match(prefixRe);
+        if (match) {
+            const type = match[1].toLowerCase();
+            const text = line.trimmed.slice(match[0].length).trim();
+            if (!text) {
+                throw new Error(`Line ${line.number} (${type}:) must include text.`);
+            }
+            if (/\r|\n/.test(text)) {
+                throw new Error(`Line ${line.number} (${type}:) contains a newline. Keep each query on a single line.`);
+            }
+            typed.push({ type, query: text, line: line.number });
+            continue;
+        }
+        if (rawLines.length === 1) {
+            // Single plain line -> implicit expand
+            return null;
+        }
+        throw new Error(`Line ${line.number} is missing a lex:/vec:/hyde: prefix. Each line in a query document must start with one.`);
+    }
+    return typed.length > 0 ? typed : null;
+}
 function search(query, opts) {
     const db = getDb();
     // Validate collection filter (supports multiple -c flags)
-    const collectionNames = resolveCollectionFilter(opts.collection);
+    // Use default collections if none specified
+    const collectionNames = resolveCollectionFilter(opts.collection, true);
     const singleCollection = collectionNames.length === 1 ? collectionNames[0] : undefined;
     // Use large limit for --all, otherwise fetch more than needed and let outputResults filter
     const fetchLimit = opts.all ? 100000 : Math.max(50, opts.limit * 2);
@@ -1674,12 +1807,7 @@ function search(query, opts) {
     }));
     closeDb();
     if (resultsWithContext.length === 0) {
-        if (opts.format === "json") {
-            console.log("[]");
-        }
-        else {
-            console.log("No results found.");
-        }
+        printEmptySearchResults(opts.format);
         return;
     }
     outputResults(resultsWithContext, query, opts);
@@ -1703,7 +1831,8 @@ function logExpansionTree(originalQuery, expanded) {
 async function vectorSearch(query, opts, _model = DEFAULT_EMBED_MODEL) {
     const store = getStore();
     // Validate collection filter (supports multiple -c flags)
-    const collectionNames = resolveCollectionFilter(opts.collection);
+    // Use default collections if none specified
+    const collectionNames = resolveCollectionFilter(opts.collection, true);
     const singleCollection = collectionNames.length === 1 ? collectionNames[0] : undefined;
     checkIndexHealth(store.db);
     await withLLMSession(async () => {
@@ -1727,12 +1856,7 @@ async function vectorSearch(query, opts, _model = DEFAULT_EMBED_MODEL) {
         }
         closeDb();
         if (results.length === 0) {
-            if (opts.format === "json") {
-                console.log("[]");
-            }
-            else {
-                console.log("No results found.");
-            }
+            printEmptySearchResults(opts.format);
             return;
         }
         outputResults(results.map(r => ({
@@ -1749,31 +1873,87 @@ async function vectorSearch(query, opts, _model = DEFAULT_EMBED_MODEL) {
 async function querySearch(query, opts, _embedModel = DEFAULT_EMBED_MODEL, _rerankModel = DEFAULT_RERANK_MODEL) {
     const store = getStore();
     // Validate collection filter (supports multiple -c flags)
-    const collectionNames = resolveCollectionFilter(opts.collection);
+    // Use default collections if none specified
+    const collectionNames = resolveCollectionFilter(opts.collection, true);
     const singleCollection = collectionNames.length === 1 ? collectionNames[0] : undefined;
     checkIndexHealth(store.db);
+    // Check for structured query syntax (lex:/vec:/hyde: prefixes)
+    const structuredQueries = parseStructuredQuery(query);
     await withLLMSession(async () => {
-        let results = await hybridQuery(store, query, {
-            collection: singleCollection,
-            limit: opts.all ? 500 : (opts.limit || 10),
-            minScore: opts.minScore || 0,
-            hooks: {
-                onStrongSignal: (score) => {
-                    process.stderr.write(`${c.dim}Strong BM25 signal (${score.toFixed(2)}) — skipping expansion${c.reset}\n`);
-                },
-                onExpand: (original, expanded) => {
-                    logExpansionTree(original, expanded);
-                    process.stderr.write(`${c.dim}Searching ${expanded.length + 1} queries...${c.reset}\n`);
-                },
-                onRerankStart: (chunkCount) => {
-                    process.stderr.write(`${c.dim}Reranking ${chunkCount} chunks...${c.reset}\n`);
-                    progress.indeterminate();
+        let results;
+        if (structuredQueries) {
+            // Structured search — user provided their own query expansions
+            const typeLabels = structuredQueries.map(s => s.type).join('+');
+            process.stderr.write(`${c.dim}Structured search: ${structuredQueries.length} queries (${typeLabels})${c.reset}\n`);
+            // Log each sub-query
+            for (const s of structuredQueries) {
+                let preview = s.query.replace(/\n/g, ' ');
+                if (preview.length > 72)
+                    preview = preview.substring(0, 69) + '...';
+                process.stderr.write(`${c.dim}├─ ${s.type}: ${preview}${c.reset}\n`);
+            }
+            process.stderr.write(`${c.dim}└─ Searching...${c.reset}\n`);
+            results = await structuredSearch(store, structuredQueries, {
+                collections: singleCollection ? [singleCollection] : undefined,
+                limit: opts.all ? 500 : (opts.limit || 10),
+                minScore: opts.minScore || 0,
+                candidateLimit: opts.candidateLimit,
+                explain: !!opts.explain,
+                hooks: {
+                    onEmbedStart: (count) => {
+                        process.stderr.write(`${c.dim}Embedding ${count} ${count === 1 ? 'query' : 'queries'}...${c.reset}`);
+                    },
+                    onEmbedDone: (ms) => {
+                        process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
+                    },
+                    onRerankStart: (chunkCount) => {
+                        process.stderr.write(`${c.dim}Reranking ${chunkCount} chunks...${c.reset}`);
+                        progress.indeterminate();
+                    },
+                    onRerankDone: (ms) => {
+                        progress.clear();
+                        process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
+                    },
                 },
-                onRerankDone: () => {
-                    progress.clear();
+            });
+        }
+        else {
+            // Standard hybrid query with automatic expansion
+            results = await hybridQuery(store, query, {
+                collection: singleCollection,
+                limit: opts.all ? 500 : (opts.limit || 10),
+                minScore: opts.minScore || 0,
+                candidateLimit: opts.candidateLimit,
+                explain: !!opts.explain,
+                hooks: {
+                    onStrongSignal: (score) => {
+                        process.stderr.write(`${c.dim}Strong BM25 signal (${score.toFixed(2)}) — skipping expansion${c.reset}\n`);
+                    },
+                    onExpandStart: () => {
+                        process.stderr.write(`${c.dim}Expanding query...${c.reset}`);
+                    },
+                    onExpand: (original, expanded, ms) => {
+                        process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
+                        logExpansionTree(original, expanded);
+                        process.stderr.write(`${c.dim}Searching ${expanded.length + 1} queries...${c.reset}\n`);
+                    },
+                    onEmbedStart: (count) => {
+                        process.stderr.write(`${c.dim}Embedding ${count} ${count === 1 ? 'query' : 'queries'}...${c.reset}`);
+                    },
+                    onEmbedDone: (ms) => {
+                        process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
+                    },
+                    onRerankStart: (chunkCount) => {
+                        process.stderr.write(`${c.dim}Reranking ${chunkCount} chunks...${c.reset}`);
+                        progress.indeterminate();
+                    },
+                    onRerankDone: (ms) => {
+                        progress.clear();
+                        process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
+                    },
                 },
-            },
-        });
+            });
+        }
         // Post-filter for multi-collection
         if (collectionNames.length > 1) {
             results = results.filter(r => {
@@ -1783,14 +1963,13 @@ async function querySearch(query, opts, _embedModel = DEFAULT_EMBED_MODEL, _rera
         }
         closeDb();
         if (results.length === 0) {
-            if (opts.format === "json") {
-                console.log("[]");
-            }
-            else {
-                console.log("No results found.");
-            }
+            printEmptySearchResults(opts.format);
             return;
         }
+        // Use first lex/vec query for output context, or original query
+        const displayQuery = structuredQueries
+            ? (structuredQueries.find(s => s.type === 'lex')?.query || structuredQueries.find(s => s.type === 'vec')?.query || query)
+            : query;
         // Map to CLI output format — use bestChunk for snippet display
         outputResults(results.map(r => ({
             file: r.file,
@@ -1801,7 +1980,8 @@ async function querySearch(query, opts, _embedModel = DEFAULT_EMBED_MODEL, _rera
             score: r.score,
             context: r.context,
             docid: r.docid,
-        })), query, { ...opts, limit: results.length });
+            explain: r.explain,
+        })), displayQuery, { ...opts, limit: results.length });
     }, { maxDuration: 10 * 60 * 1000, name: 'querySearch' });
 }
 // Parse CLI arguments using util.parseArgs
@@ -1818,6 +1998,7 @@ function parseCLI() {
             },
             help: { type: "boolean", short: "h" },
             version: { type: "boolean", short: "v" },
+            skill: { type: "boolean" },
             // Search options
             n: { type: "string" },
             "min-score": { type: "string" },
@@ -1828,6 +2009,7 @@ function parseCLI() {
             xml: { type: "boolean" },
             files: { type: "boolean" },
             json: { type: "boolean" },
+            explain: { type: "boolean" },
             collection: { type: "string", short: "c", multiple: true }, // Filter by collection(s)
             // Collection options
             name: { type: "string" }, // collection name
@@ -1842,6 +2024,8 @@ function parseCLI() {
             from: { type: "string" }, // start line
             "max-bytes": { type: "string" }, // max bytes for multi-get
             "line-numbers": { type: "boolean" }, // add line numbers to output
+            // Query options
+            "candidate-limit": { type: "string", short: "C" },
             // MCP HTTP transport options
             http: { type: "boolean" },
             daemon: { type: "boolean" },
@@ -1880,6 +2064,8 @@ function parseCLI() {
         all: isAll,
         collection: values.collection,
         lineNumbers: !!values["line-numbers"],
+        candidateLimit: values["candidate-limit"] ? parseInt(String(values["candidate-limit"]), 10) : undefined,
+        explain: !!values.explain,
     };
     return {
         command: positionals[0] || "",
@@ -1889,50 +2075,102 @@ function parseCLI() {
         values,
     };
 }
+function showSkill() {
+    const scriptDir = dirname(fileURLToPath(import.meta.url));
+    const relativePath = pathJoin("skills", "qmd", "SKILL.md");
+    const skillPath = pathJoin(scriptDir, "..", relativePath);
+    console.log(`QMD Skill (${relativePath})`);
+    console.log(`Location: ${skillPath}`);
+    console.log("");
+    if (!existsSync(skillPath)) {
+        console.error("SKILL.md not found. If you built from source, ensure skills/qmd/SKILL.md exists.");
+        return;
+    }
+    const content = readFileSync(skillPath, "utf-8");
+    process.stdout.write(content.endsWith("\n") ? content : content + "\n");
+}
 function showHelp() {
+    console.log("qmd — Quick Markdown Search");
+    console.log("");
     console.log("Usage:");
-    console.log("  qmd collection add [path] --name <name> --mask <pattern>  - Create/index collection");
-    console.log("  qmd collection list           - List all collections with details");
-    console.log("  qmd collection remove <name>  - Remove a collection by name");
-    console.log("  qmd collection rename <old> <new>  - Rename a collection");
-    console.log("  qmd ls [collection[/path]]    - List collections or files in a collection");
-    console.log("  qmd context add [path] \"text\" - Add context for path (defaults to current dir)");
-    console.log("  qmd context list              - List all contexts");
-    console.log("  qmd context rm <path>         - Remove context");
-    console.log("  qmd get <file>[:line] [-l N] [--from N]  - Get document (optionally from line, max N lines)");
-    console.log("  qmd multi-get <pattern> [-l N] [--max-bytes N]  - Get multiple docs by glob or comma-separated list");
-    console.log("  qmd status                    - Show index status and collections");
-    console.log("  qmd update [--pull]           - Re-index all collections (--pull: git pull first)");
-    console.log("  qmd embed [-f]                - Create vector embeddings (900 tokens/chunk, 15% overlap)");
-    console.log("  qmd cleanup                   - Remove cache and orphaned data, vacuum DB");
-    console.log("  qmd query <query>             - Search with query expansion + reranking (recommended)");
-    console.log("  qmd search <query>            - Full-text keyword search (BM25, no LLM)");
-    console.log("  qmd vsearch <query>           - Vector similarity search (no reranking)");
-    console.log("  qmd mcp                       - Start MCP server (stdio transport)");
-    console.log("  qmd mcp --http [--port N]     - Start MCP server (HTTP transport, default port 8181)");
-    console.log("  qmd mcp --http --daemon       - Start MCP server as background daemon");
-    console.log("  qmd mcp stop                  - Stop background MCP daemon");
+    console.log("  qmd <command> [options]");
+    console.log("");
+    console.log("Primary commands:");
+    console.log("  qmd query <query>             - Hybrid search with auto expansion + reranking (recommended)");
+    console.log("  qmd query 'lex:..\\nvec:...'   - Structured query document (you provide lex/vec/hyde lines)");
+    console.log("  qmd search <query>            - Full-text BM25 keywords (no LLM)");
+    console.log("  qmd vsearch <query>           - Vector similarity only");
+    console.log("  qmd get <file>[:line] [-l N]  - Show a single document, optional line slice");
+    console.log("  qmd multi-get <pattern>       - Batch fetch via glob or comma-separated list");
+    console.log("  qmd mcp                       - Start the MCP server (stdio transport for AI agents)");
+    console.log("");
+    console.log("Collections & context:");
+    console.log("  qmd collection add/list/remove/rename/show   - Manage indexed folders");
+    console.log("  qmd context add/list/rm                      - Attach human-written summaries");
+    console.log("  qmd ls [collection[/path]]                   - Inspect indexed files");
+    console.log("");
+    console.log("Maintenance:");
+    console.log("  qmd status                    - View index + collection health");
+    console.log("  qmd update [--pull]           - Re-index collections (optionally git pull first)");
+    console.log("  qmd embed [-f]                - Generate/refresh vector embeddings");
+    console.log("  qmd cleanup                   - Clear caches, vacuum DB");
+    console.log("");
+    console.log("Query syntax (qmd query):");
+    console.log("  QMD queries are either a single expand query (no prefix) or a multi-line");
+    console.log("  document where every line is typed with lex:, vec:, or hyde:. This grammar");
+    console.log("  matches the docs in docs/SYNTAX.md and is enforced in the CLI.");
+    console.log("");
+    const grammar = [
+        `query          = expand_query | query_document ;`,
+        `expand_query   = text | explicit_expand ;`,
+        `explicit_expand= "expand:" text ;`,
+        `query_document = { typed_line } ;`,
+        `typed_line     = type ":" text newline ;`,
+        `type           = "lex" | "vec" | "hyde" ;`,
+        `text           = quoted_phrase | plain_text ;`,
+        `quoted_phrase  = '"' { character } '"' ;`,
+        `plain_text     = { character } ;`,
+        `newline        = "\\n" ;`,
+    ];
+    console.log("  Grammar:");
+    for (const line of grammar) {
+        console.log(`    ${line}`);
+    }
+    console.log("");
+    console.log("  Examples:");
+    console.log("    qmd query \"how does auth work\"                # single-line → implicit expand");
+    console.log("    qmd query $'lex: CAP theorem\\nvec: consistency'  # typed query document");
+    console.log("    qmd query $'lex: \"exact matches\" sports -baseball'  # phrase + negation lex search");
+    console.log("    qmd query $'hyde: Hypothetical answer text'       # hyde-only document");
+    console.log("");
+    console.log("  Constraints:");
+    console.log("    - Standalone expand queries cannot mix with typed lines.");
+    console.log("    - Query documents allow only lex:, vec:, or hyde: prefixes.");
+    console.log("    - Each typed line must be single-line text with balanced quotes.");
+    console.log("");
+    console.log("AI agents & integrations:");
+    console.log("  - Run `qmd mcp` to expose the MCP server (stdio) to agents/IDEs.");
+    console.log("  - `qmd --skill` prints the packaged skills/qmd/SKILL.md (path + contents).");
+    console.log("  - Advanced: `qmd mcp --http ...` and `qmd mcp --http --daemon` are optional for custom transports.");
     console.log("");
     console.log("Global options:");
-    console.log("  --index <name>             - Use custom index name (default: index)");
+    console.log("  --index <name>             - Use a named index (default: index)");
     console.log("");
     console.log("Search options:");
-    console.log("  -n <num>                   - Number of results (default: 5, or 20 for --files)");
-    console.log("  --all                      - Return all matches (use with --min-score to filter)");
+    console.log("  -n <num>                   - Max results (default 5, or 20 for --files/--json)");
+    console.log("  --all                      - Return all matches (pair with --min-score)");
     console.log("  --min-score <num>          - Minimum similarity score");
     console.log("  --full                     - Output full document instead of snippet");
-    console.log("  --line-numbers             - Add line numbers to output");
-    console.log("  --files                    - Output docid,score,filepath,context (default: 20 results)");
-    console.log("  --json                     - JSON output with snippets (default: 20 results)");
-    console.log("  --csv                      - CSV output with snippets");
-    console.log("  --md                       - Markdown output");
-    console.log("  --xml                      - XML output");
-    console.log("  -c, --collection <name>    - Filter results to a specific collection");
+    console.log("  -C, --candidate-limit <n>  - Max candidates to rerank (default 40, lower = faster)");
+    console.log("  --line-numbers             - Include line numbers in output");
+    console.log("  --explain                  - Include retrieval score traces (query --json/CLI)");
+    console.log("  --files | --json | --csv | --md | --xml  - Output format");
+    console.log("  -c, --collection <name>    - Filter by one or more collections");
     console.log("");
     console.log("Multi-get options:");
     console.log("  -l <num>                   - Maximum lines per file");
-    console.log("  --max-bytes <num>          - Skip files larger than N bytes (default: 10240)");
-    console.log("  --json/--csv/--md/--xml/--files - Output format (same as search)");
+    console.log("  --max-bytes <num>          - Skip files larger than N bytes (default 10240)");
+    console.log("  --json/--csv/--md/--xml/--files - Same formats as search");
     console.log("");
     console.log(`Index: ${getDbPath()}`);
 }
@@ -1951,12 +2189,22 @@ async function showVersion() {
     console.log(`qmd ${versionStr}`);
 }
 // Main CLI - only run if this is the main module
-if (fileURLToPath(import.meta.url) === process.argv[1] || process.argv[1]?.endsWith("/qmd.ts") || process.argv[1]?.endsWith("/qmd.js")) {
+const __filename = fileURLToPath(import.meta.url);
+const argv1 = process.argv[1];
+const isMain = argv1 === __filename
+    || argv1?.endsWith("/qmd.ts")
+    || argv1?.endsWith("/qmd.js")
+    || (argv1 != null && realpathSync(argv1) === __filename);
+if (isMain) {
     const cli = parseCLI();
     if (cli.values.version) {
         await showVersion();
         process.exit(0);
     }
+    if (cli.values.skill) {
+        showSkill();
+        process.exit(0);
+    }
     if (!cli.command || cli.values.help) {
         showHelp();
         process.exit(cli.values.help ? 0 : 1);
@@ -1965,13 +2213,12 @@ if (fileURLToPath(import.meta.url) === process.argv[1] || process.argv[1]?.endsW
         case "context": {
             const subcommand = cli.args[0];
             if (!subcommand) {
-                console.error("Usage: qmd context <add|list|check|rm>");
+                console.error("Usage: qmd context <add|list|rm>");
                 console.error("");
                 console.error("Commands:");
                 console.error("  qmd context add [path] \"text\"  - Add context (defaults to current dir)");
                 console.error("  qmd context add / \"text\"       - Add global context to all collections");
                 console.error("  qmd context list                - List all contexts");
-                console.error("  qmd context check               - Check for missing contexts");
                 console.error("  qmd context rm <path>           - Remove context");
                 process.exit(1);
             }
@@ -2013,10 +2260,6 @@ if (fileURLToPath(import.meta.url) === process.argv[1] || process.argv[1]?.endsW
                     contextList();
                     break;
                 }
-                case "check": {
-                    contextCheck();
-                    break;
-                }
                 case "rm":
                 case "remove": {
                     if (cli.args.length < 2 || !cli.args[1]) {
@@ -2031,7 +2274,7 @@ if (fileURLToPath(import.meta.url) === process.argv[1] || process.argv[1]?.endsW
                 }
                 default:
                     console.error(`Unknown subcommand: ${subcommand}`);
-                    console.error("Available: add, list, check, rm");
+                    console.error("Available: add, list, rm");
                     process.exit(1);
             }
             break;
@@ -2096,9 +2339,99 @@ if (fileURLToPath(import.meta.url) === process.argv[1] || process.argv[1]?.endsW
                     collectionRename(cli.args[1], cli.args[2]);
                     break;
                 }
+                case "set-update":
+                case "update-cmd": {
+                    const name = cli.args[1];
+                    const cmd = cli.args.slice(2).join(' ') || null;
+                    if (!name) {
+                        console.error("Usage: qmd collection update-cmd <name> [command]");
+                        console.error("  Set the command to run before indexing (e.g., 'git pull')");
+                        console.error("  Omit command to clear it");
+                        process.exit(1);
+                    }
+                    const { updateCollectionSettings, getCollection } = await import("./collections.js");
+                    const col = getCollection(name);
+                    if (!col) {
+                        console.error(`Collection not found: ${name}`);
+                        process.exit(1);
+                    }
+                    updateCollectionSettings(name, { update: cmd });
+                    if (cmd) {
+                        console.log(`✓ Set update command for '${name}': ${cmd}`);
+                    }
+                    else {
+                        console.log(`✓ Cleared update command for '${name}'`);
+                    }
+                    break;
+                }
+                case "include":
+                case "exclude": {
+                    const name = cli.args[1];
+                    if (!name) {
+                        console.error(`Usage: qmd collection ${subcommand} <name>`);
+                        console.error(`  ${subcommand === 'include' ? 'Include' : 'Exclude'} collection in default queries`);
+                        process.exit(1);
+                    }
+                    const { updateCollectionSettings, getCollection } = await import("./collections.js");
+                    const col = getCollection(name);
+                    if (!col) {
+                        console.error(`Collection not found: ${name}`);
+                        process.exit(1);
+                    }
+                    const include = subcommand === 'include';
+                    updateCollectionSettings(name, { includeByDefault: include });
+                    console.log(`✓ Collection '${name}' ${include ? 'included in' : 'excluded from'} default queries`);
+                    break;
+                }
+                case "show":
+                case "info": {
+                    const name = cli.args[1];
+                    if (!name) {
+                        console.error("Usage: qmd collection show <name>");
+                        process.exit(1);
+                    }
+                    const { getCollection } = await import("./collections.js");
+                    const col = getCollection(name);
+                    if (!col) {
+                        console.error(`Collection not found: ${name}`);
+                        process.exit(1);
+                    }
+                    console.log(`Collection: ${name}`);
+                    console.log(`  Path:     ${col.path}`);
+                    console.log(`  Pattern:  ${col.pattern}`);
+                    console.log(`  Include:  ${col.includeByDefault !== false ? 'yes (default)' : 'no'}`);
+                    if (col.update) {
+                        console.log(`  Update:   ${col.update}`);
+                    }
+                    if (col.context) {
+                        const ctxCount = Object.keys(col.context).length;
+                        console.log(`  Contexts: ${ctxCount}`);
+                    }
+                    break;
+                }
+                case "help":
+                case undefined: {
+                    console.log("Usage: qmd collection <command> [options]");
+                    console.log("");
+                    console.log("Commands:");
+                    console.log("  list                      List all collections");
+                    console.log("  add <path> [--name NAME]  Add a collection");
+                    console.log("  remove <name>             Remove a collection");
+                    console.log("  rename <old> <new>        Rename a collection");
+                    console.log("  show <name>               Show collection details");
+                    console.log("  update-cmd <name> [cmd]   Set pre-update command (e.g., 'git pull')");
+                    console.log("  include <name>            Include in default queries");
+                    console.log("  exclude <name>            Exclude from default queries");
+                    console.log("");
+                    console.log("Examples:");
+                    console.log("  qmd collection add ~/notes --name notes");
+                    console.log("  qmd collection update-cmd brain 'git pull'");
+                    console.log("  qmd collection exclude archive");
+                    process.exit(0);
+                }
                 default:
                     console.error(`Unknown subcommand: ${subcommand}`);
-                    console.error("Available: list, add, remove, rename");
+                    console.error("Run 'qmd collection help' for usage");
                     process.exit(1);
             }
             break;