@jcode.labs/ragmir 2.17.1 → 2.18.0

package/dist/query.js

@@ -1,37 +1,40 @@
-import { recordAccess } from "./access-log.js";
+import { channel } from "node:diagnostics_channel";
+import { MatchQuery, Operator, PhraseQuery, } from "@lancedb/lancedb";
+import { flushAccessLog, recordAccess } from "./access-log.js";
+import { citationForCoordinates, stripCitationCoordinates } from "./citation.js";
 import { loadConfig } from "./config.js";
-import { VECTOR_DISTANCE_METRIC } from "./defaults.js";
+import { MAX_SEARCH_TOP_K, VECTOR_DISTANCE_METRIC } from "./defaults.js";
 import { embedText } from "./embeddings.js";
 import { RagmirError } from "./errors.js";
-import { getIndexFreshnessWarning } from "./index-diagnostics.js";
+import { acquireGenerationReadLease } from "./generation-retention.js";
+import { indexFreshnessWarning } from "./index-diagnostics.js";
 import { operationSignal, throwIfAborted } from "./operation.js";
 import { sanitizeRetrievalQuery } from "./query-sanitizer.js";
-import { openRowsTable, readIndexManifest } from "./store.js";
+import { candidatePassesAbstention, queryEvidence, rankHybridRows, rankingPolicyFingerprint, rankingPolicyFor, tokensAreLexicallyRelated, } from "./ranking.js";
+import { closeIndexReadSnapshot, loadIndexReadSnapshot } from "./store.js";
 import { tokenize } from "./text.js";
+import { configureAdaptiveVectorQuery, vectorIndexManifestCompatible } from "./vector-index.js";
+import { runWorkload } from "./workload.js";
 const VECTOR_CANDIDATE_POLICY = {
     fast: { minimum: 40, multiplier: 3, maxChunksPerSource: 1 },
     balanced: { minimum: 80, multiplier: 4, maxChunksPerSource: 2 },
     quality: { minimum: 200, multiplier: 8, maxChunksPerSource: 4 },
     custom: { minimum: 80, multiplier: 4, maxChunksPerSource: 2 },
 };
-/**
- * Reciprocal Rank Fusion (Cormack et al. 2009). Each candidate scores
- * `weight / (RRF_K + rank)` per retriever it appears in, summed across
- * retrievers. Rank-only fusion removes the score-calibration problem of
- * weighted-sum fusion: the BM25 and vector score distributions never need to
- * be normalized against each other.
- *
- * Equal weights let exact lexical evidence rescue a relevant document that is
- * absent from the bounded vector candidate pool.
- */
-const RRF_K = 60;
-const RRF_VECTOR_WEIGHT = 1;
-const RRF_LEXICAL_WEIGHT = 1;
-const BM25_K1 = 1.2;
-const BM25_B = 0.75;
+const LEXICAL_CANDIDATE_POLICY = {
+    fast: { minimum: 100, multiplier: 10 },
+    balanced: { minimum: 250, multiplier: 20 },
+    quality: { minimum: 500, multiplier: 40 },
+    custom: { minimum: 250, multiplier: 20 },
+};
 const MAX_CONTEXT_RADIUS = 3;
-const MIN_FUZZY_TOKEN_LENGTH = 7;
-const MIN_TRIGRAM_DICE_SIMILARITY = 0.5;
+const MAX_VECTOR_CANDIDATES = 1_000;
+const MAX_LEXICAL_CANDIDATES = 4_000;
+const FULL_TEXT_INDEX_NAME = "searchText_idx";
+const LEXICAL_IDENTIFIER_PATTERN = /[\p{L}\p{N}]+(?:[-_.][\p{L}\p{N}]+)+/gu;
+const SOURCE_PATH_QUERY_PATTERN = /^[\p{L}\p{N}_. /\\-]+\.[a-z0-9]{1,12}$/iu;
+export const QUERY_EXPLANATION_DIAGNOSTICS_CHANNEL = "ragmir:query-explanation";
+const queryExplanationDiagnostics = channel(QUERY_EXPLANATION_DIAGNOSTICS_CHANNEL);
 const SEARCH_COLUMNS = [
     "source",
     "relativePath",
@@ -45,50 +48,88 @@ const SEARCH_COLUMNS = [
     "lineEnd",
     "pageStart",
     "pageEnd",
+    "locationKind",
+    "locationStart",
+    "locationEnd",
+    "locationLabel",
+    "cellStart",
+    "cellEnd",
 ];
 const VECTOR_SEARCH_COLUMNS = [...SEARCH_COLUMNS, "_distance"];
 const FTS_SEARCH_COLUMNS = [...SEARCH_COLUMNS, "_score"];
 export async function search(query, options = {}) {
     const config = await loadConfig(String(options.cwd ?? process.cwd()));
-    return searchWithConfig(query, options, config);
+    const results = await searchWithConfig(query, options, config);
+    await flushAccessLog(config);
+    return results;
 }
-export async function searchWithConfig(query, options, config, connection, activeSignal) {
+export async function searchWithConfig(query, options, config, connection, activeSignal, suppliedSnapshot, generationLeaseHeld = false) {
+    const signal = activeSignal ?? operationSignal(options);
+    return runWorkload(config, "search", signal, async ({ queueTimeMs }) => {
+        const ownsSnapshot = suppliedSnapshot === undefined;
+        const snapshot = suppliedSnapshot ?? (await loadIndexReadSnapshot(config, connection));
+        let lease;
+        try {
+            if (!generationLeaseHeld && snapshot.table) {
+                lease = await acquireGenerationReadLease(snapshot.tableName, config);
+            }
+            return await searchWithinGeneration(query, options, config, snapshot, signal, queueTimeMs);
+        }
+        finally {
+            await lease?.release().catch(() => undefined);
+            if (ownsSnapshot) {
+                closeIndexReadSnapshot(snapshot, config);
+            }
+        }
+    });
+}
+async function searchWithinGeneration(query, options, config, snapshot, activeSignal, workloadQueueMs = 0) {
     const signal = activeSignal ?? operationSignal(options);
     const topK = normalizeTopK(options.topK ?? config.topK);
     const defaultContextRadius = config.retrievalProfile === "quality" ? 1 : 0;
     const contextRadius = normalizeContextRadius(options.contextRadius ?? defaultContextRadius);
     throwIfAborted(signal);
-    const table = await openRowsTable(config, connection);
+    const table = snapshot.table;
     throwIfAborted(signal);
     if (!table) {
         return [];
     }
-    await assertIndexFreshness(config);
+    assertIndexFreshness(config, snapshot.manifest);
     const sanitized = sanitizeRetrievalQuery(query);
-    const queryTokens = tokenize(sanitized.query);
+    const evidence = queryEvidence(sanitized.query);
+    const queryTokens = evidence.tokens;
     if (!sanitized.query || queryTokens.length === 0) {
         return [];
     }
     const retrievalPredicate = searchPredicate(options.includePaths, options.excludePaths, options.contextPaths);
-    const [vector, textRows] = await Promise.all([
-        embedText(sanitized.query, config),
-        lexicalCandidateRows(table, sanitized.query, config.hybridTextScanLimit, retrievalPredicate),
+    let embeddingQueueMs = 0;
+    const [vector, lexicalCandidates] = await Promise.all([
+        embedText(sanitized.query, config, signal, ({ queueTimeMs }) => {
+            embeddingQueueMs = queueTimeMs;
+        }),
+        lexicalCandidateRows(table, sanitized.query, {
+            ftsLimit: lexicalCandidateLimit(topK, config.retrievalProfile),
+            fallbackLimit: config.hybridTextScanLimit,
+            indexedChunkCount: snapshot.manifest?.chunkCount ?? 0,
+            minimumResults: topK,
+            pathPredicate: retrievalPredicate,
+        }),
     ]);
     throwIfAborted(signal);
-    await assertVectorIndexCompatibility(config, vector.length);
-    const vectorQuery = table.vectorSearch(vector).select(VECTOR_SEARCH_COLUMNS);
+    const manifest = assertVectorIndexCompatibility(config, snapshot.manifest, vector.length);
+    const vectorQuery = configureAdaptiveVectorQuery(table.vectorSearch(vector).select(VECTOR_SEARCH_COLUMNS), manifest.vectorIndex, options.vectorSearchMode === "exact");
     const vectorRows = (await (retrievalPredicate
         ? vectorQuery.where(retrievalPredicate)
         : vectorQuery)
         .limit(vectorCandidateLimit(topK, config.retrievalProfile))
         .toArray());
     throwIfAborted(signal);
-    const rankedRows = rankHybridRows(sanitized.query, vectorRows, textRows);
-    const relevantRows = config.embeddingProvider === "local-hash"
-        ? rankedRows.filter((ranked) => hasLexicalOverlap(queryTokens, ranked.row.searchText))
-        : rankedRows;
-    const rows = diversifyRows(relevantRows, topK, config.retrievalProfile);
-    const contextByRow = await contextChunksByRow(table, rows, contextRadius);
+    const rankingPolicy = rankingPolicyFor(config.embeddingProvider, config.retrievalProfile);
+    const rankedRows = rankHybridRows(sanitized.query, vectorRows, lexicalCandidates.rows, rankingPolicy);
+    const relevantRows = rankedRows.filter((ranked) => lexicalCandidates.exactPathMatches.has(rowKey(ranked.row)) ||
+        candidatePassesAbstention(evidence, ranked.row, rankingPolicy));
+    const rows = diversifyRows(relevantRows, topK, config.retrievalProfile, lexicalCandidates.exactPathMatches);
+    const contextByRow = await contextChunksByRow(table, rows, contextRadius, retrievalPredicate);
     throwIfAborted(signal);
     const results = rows.map((row) => {
         const vectorDistance = typeof row.row._distance === "number" ? row.row._distance : null;
@@ -102,8 +143,8 @@ export async function searchWithConfig(query, options, config, connection, activ
             distance: vectorDistance,
             charStart: nullableNumber(row.row.charStart),
             charEnd: nullableNumber(row.row.charEnd),
-            lineStart: nullableNumber(row.row.lineStart),
-            lineEnd: nullableNumber(row.row.lineEnd),
+            lineStart: nullableLineNumber(row.row.lineStart),
+            lineEnd: nullableLineNumber(row.row.lineEnd),
             pageStart: nullablePageNumber(row.row.pageStart),
             pageEnd: nullablePageNumber(row.row.pageEnd),
             context: contextByRow.get(rowKey(row.row)) ?? [],
@@ -118,13 +159,25 @@ export async function searchWithConfig(query, options, config, connection, activ
                         lexicalRank: row.lexicalRank,
                         vectorDistance,
                         lexicalBackendScore: row.lexicalBackendScore,
-                        matchedTerms: row.matchedTerms,
+                        lexicalBackend: lexicalCandidates.backend,
+                        lexicalFallbackActivated: lexicalCandidates.fallbackActivated,
+                        lexicalFallbackReason: lexicalCandidates.fallbackReason,
+                        lexicalExactPathMatch: lexicalCandidates.exactPathMatches.has(rowKey(row.row)),
+                        lexicalCandidateLimit: lexicalCandidates.candidateLimit,
+                        lexicalCandidatesMaterialized: lexicalCandidates.candidatesMaterialized,
+                        lexicalQueryVariants: lexicalCandidates.queryVariants,
+                        lexicalIndexedRows: lexicalCandidates.indexedRows,
+                        lexicalUnindexedRows: lexicalCandidates.unindexedRows,
+                        lexicalCoverage: lexicalCandidates.coverage,
+                        workloadQueueMs: workloadQueueMs + embeddingQueueMs,
+                        rankingPolicyFingerprint: rankingPolicyFingerprint(rankingPolicy),
+                        matchedTerms: matchedQueryTerms(config.projectRoot, queryTokens, row.row.searchText),
                     },
                 }
                 : {}),
         };
     });
-    await recordAccess(config, {
+    void recordAccess(config, {
         action: "search",
         query: sanitized.query,
         topK,
@@ -132,11 +185,11 @@ export async function searchWithConfig(query, options, config, connection, activ
     });
     return results;
 }
-function diversifyRows(rows, topK, profile) {
+function diversifyRows(rows, topK, profile, exactPathMatches) {
     const uniqueRows = [];
     const textIndexes = new Map();
     for (const row of rows) {
-        const textKey = `${row.row.contextPath}\0${row.row.text.replace(/\s+/gu, " ").trim().toLowerCase()}`;
+        const textKey = row.row.text.replace(/\s+/gu, " ").trim().toLowerCase();
         const existingIndex = textIndexes.get(textKey);
         if (existingIndex === undefined) {
             textIndexes.set(textKey, uniqueRows.length);
@@ -144,23 +197,43 @@ function diversifyRows(rows, topK, profile) {
             continue;
         }
         const existing = uniqueRows[existingIndex];
-        if (existing && preferCanonicalPath(row.row.relativePath, existing.row.relativePath)) {
+        const existingIsExact = existing ? exactPathMatches.has(rowKey(existing.row)) : false;
+        const candidateIsExact = exactPathMatches.has(rowKey(row.row));
+        if (existing &&
+            (candidateIsExact !== existingIsExact
+                ? candidateIsExact
+                : preferCanonicalPath(row.row.relativePath, existing.row.relativePath))) {
             uniqueRows[existingIndex] = row;
         }
     }
     const selected = [];
+    const selectedKeys = new Set();
     const perSource = new Map();
     const maxChunksPerSource = VECTOR_CANDIDATE_POLICY[profile].maxChunksPerSource;
-    for (const row of uniqueRows) {
+    const appendRow = (row, enforceSourceCap) => {
+        const key = rowKey(row.row);
+        if (selectedKeys.has(key)) {
+            return;
+        }
         if (selected.some((candidate) => overlapsSourceSpan(candidate.row, row.row))) {
-            continue;
+            return;
         }
         const sourceCount = perSource.get(row.row.relativePath) ?? 0;
-        if (sourceCount >= maxChunksPerSource) {
-            continue;
+        if (enforceSourceCap && sourceCount >= maxChunksPerSource) {
+            return;
         }
         selected.push(row);
+        selectedKeys.add(key);
         perSource.set(row.row.relativePath, sourceCount + 1);
+    };
+    for (const row of uniqueRows) {
+        appendRow(row, true);
+        if (selected.length >= topK) {
+            return selected;
+        }
+    }
+    for (const row of uniqueRows) {
+        appendRow(row, false);
         if (selected.length >= topK) {
             break;
         }
@@ -185,63 +258,23 @@ function preferCanonicalPath(candidate, current) {
     return (candidateDepth < currentDepth ||
         (candidateDepth === currentDepth && candidate.length < current.length));
 }
-function hasLexicalOverlap(queryTokens, text) {
-    const textTokens = tokenize(text);
-    return queryTokens.some((queryToken) => textTokens.some((textToken) => tokensAreLexicallyRelated(queryToken, textToken)));
-}
-function tokensAreLexicallyRelated(queryToken, textToken) {
-    if (queryToken === textToken) {
-        return true;
-    }
-    if (queryToken.length >= 4 &&
-        textToken.length >= 4 &&
-        sharedPrefixLength(queryToken, textToken) >= 4) {
-        return true;
-    }
-    if (queryToken.length < MIN_FUZZY_TOKEN_LENGTH ||
-        textToken.length < MIN_FUZZY_TOKEN_LENGTH ||
-        Math.abs(queryToken.length - textToken.length) > 1 ||
-        !/^[a-z0-9_-]+$/u.test(queryToken) ||
-        !/^[a-z0-9_-]+$/u.test(textToken)) {
-        return false;
-    }
-    return trigramDiceSimilarity(queryToken, textToken) >= MIN_TRIGRAM_DICE_SIMILARITY;
-}
-function trigramDiceSimilarity(left, right) {
-    const leftTrigrams = tokenTrigrams(left);
-    const rightTrigrams = tokenTrigrams(right);
-    let shared = 0;
-    for (const trigram of leftTrigrams) {
-        if (rightTrigrams.has(trigram)) {
-            shared += 1;
-        }
-    }
-    return (2 * shared) / (leftTrigrams.size + rightTrigrams.size);
-}
-function tokenTrigrams(token) {
-    return new Set(Array.from({ length: token.length - 2 }, (_value, index) => token.slice(index, index + 3)));
-}
-function sharedPrefixLength(left, right) {
-    const limit = Math.min(left.length, right.length);
-    let index = 0;
-    while (index < limit && left[index] === right[index]) {
-        index += 1;
-    }
-    return index;
-}
 export function vectorCandidateLimit(topK, profile = "balanced") {
     const policy = VECTOR_CANDIDATE_POLICY[profile];
-    return Math.max(policy.minimum, topK * policy.multiplier);
+    return Math.min(MAX_VECTOR_CANDIDATES, Math.max(policy.minimum, topK * policy.multiplier));
+}
+export function lexicalCandidateLimit(topK, profile = "balanced") {
+    const policy = LEXICAL_CANDIDATE_POLICY[profile];
+    return Math.min(MAX_LEXICAL_CANDIDATES, Math.max(policy.minimum, topK * policy.multiplier));
 }
 export async function ask(query, options = {}) {
     const config = await loadConfig(String(options.cwd ?? process.cwd()));
     return askWithConfig(query, options, config);
 }
-export async function askWithConfig(query, options, config, connection) {
+export async function askWithConfig(query, options, config, connection, snapshot) {
     const signal = operationSignal(options);
     throwIfAborted(signal);
-    const sources = await searchWithConfig(query, options, config, connection, signal);
-    const staleWarning = await getIndexFreshnessWarning(config);
+    const sources = await searchWithConfig(query, options, config, connection, signal, snapshot);
+    const staleWarning = null;
     throwIfAborted(signal);
     if (sources.length === 0) {
         return {
@@ -266,13 +299,30 @@ export async function expandCitation(citation, options = {}) {
     const config = await loadConfig(String(options.cwd ?? process.cwd()));
     return expandCitationWithConfig(citation, options, config);
 }
-export async function expandCitationWithConfig(citation, options, config, connection) {
+export async function expandCitationWithConfig(citation, options, config, connection, suppliedSnapshot) {
+    const ownsSnapshot = suppliedSnapshot === undefined;
+    const snapshot = suppliedSnapshot ?? (await loadIndexReadSnapshot(config, connection));
+    let lease;
+    try {
+        if (snapshot.table) {
+            lease = await acquireGenerationReadLease(snapshot.tableName, config);
+        }
+        return await expandCitationWithinGeneration(citation, options, config, snapshot);
+    }
+    finally {
+        await lease?.release().catch(() => undefined);
+        if (ownsSnapshot) {
+            closeIndexReadSnapshot(snapshot, config);
+        }
+    }
+}
+async function expandCitationWithinGeneration(citation, options, config, snapshot) {
     const signal = operationSignal(options);
     throwIfAborted(signal);
     const requestedCitation = citation.trim();
     const target = parseCitationTarget(requestedCitation);
     const contextRadius = normalizeContextRadius(options.contextRadius);
-    const table = await openRowsTable(config, connection);
+    const table = snapshot.table;
     throwIfAborted(signal);
     if (!table) {
         return {
@@ -284,11 +334,10 @@ export async function expandCitationWithConfig(citation, options, config, connec
             passages: [],
         };
     }
-    const manifest = await readIndexManifest(config);
-    if (!manifest) {
+    if (!snapshot.manifest) {
         throw new Error("Index manifest is missing. Rebuild with `rgr ingest --rebuild` before expanding citations.");
     }
-    const freshnessWarning = await getIndexFreshnessWarning(config);
+    const freshnessWarning = indexFreshnessWarning(config, snapshot.manifest);
     if (freshnessWarning) {
         throw new Error(freshnessWarning);
     }
@@ -336,25 +385,115 @@ function retrievalOnlyAnswer(sources) {
         snippets,
     ].join("\n");
 }
-async function lexicalCandidateRows(table, query, limit, pathPredicate) {
-    const ftsQuery = lexicalQuery(query);
-    if (ftsQuery) {
+async function lexicalCandidateRows(table, query, options) {
+    const ftsQueries = lexicalQuery(query);
+    const stats = await table.indexStats(FULL_TEXT_INDEX_NAME).catch(() => undefined);
+    let fallbackReason = "fts-index-unavailable";
+    if (ftsQueries && stats) {
         try {
-            const searchQuery = table.search(ftsQuery, "fts", "searchText").select(FTS_SEARCH_COLUMNS);
-            return (await (pathPredicate ? searchQuery.where(pathPredicate) : searchQuery)
-                .limit(limit)
-                .toArray());
+            const sourcePathQuery = sourcePathPredicateForQuery(query);
+            const sourcePathRows = sourcePathQuery
+                ? await executeSourcePathQuery(table, sourcePathQuery, options.ftsLimit, options.pathPredicate)
+                : [];
+            const primaryRows = await executeFullTextQuery(table, ftsQueries.primary, options.ftsLimit, options.pathPredicate);
+            const rows = [];
+            const initialRows = [...sourcePathRows, ...primaryRows];
+            const seenRows = new Set();
+            for (const row of initialRows) {
+                const key = rowKey(row);
+                if (!seenRows.has(key)) {
+                    seenRows.add(key);
+                    rows.push(row);
+                }
+            }
+            let executedVariants = 1 + Number(sourcePathQuery !== null);
+            if (rows.length < options.minimumResults) {
+                for (const supplementalQuery of ftsQueries.supplemental) {
+                    const supplementalRows = await executeFullTextQuery(table, supplementalQuery, options.ftsLimit, options.pathPredicate).catch(() => []);
+                    executedVariants += 1;
+                    for (const row of supplementalRows) {
+                        const key = rowKey(row);
+                        if (seenRows.has(key)) {
+                            continue;
+                        }
+                        seenRows.add(key);
+                        rows.push(row);
+                        if (rows.length >= options.ftsLimit) {
+                            break;
+                        }
+                    }
+                    if (rows.length >= options.minimumResults || rows.length >= options.ftsLimit) {
+                        break;
+                    }
+                }
+            }
+            const indexedRows = stats.numIndexedRows;
+            const unindexedRows = stats.numUnindexedRows;
+            const coveredRows = indexedRows + unindexedRows;
+            const exactPathMatches = new Set(sourcePathQuery?.exactRelativePath
+                ? sourcePathRows
+                    .filter((row) => row.relativePath === sourcePathQuery.exactRelativePath)
+                    .map(rowKey)
+                : []);
+            return {
+                rows,
+                exactPathMatches,
+                backend: "fts",
+                fallbackActivated: false,
+                fallbackReason: null,
+                candidateLimit: options.ftsLimit,
+                candidatesMaterialized: rows.length,
+                queryVariants: executedVariants,
+                indexedRows,
+                unindexedRows,
+                coverage: coveredRows === 0 ? 1 : indexedRows / coveredRows,
+            };
         }
         catch {
-            // Older indexes may not have the FTS index yet. Keep retrieval usable and
-            // let doctor/index freshness tell the operator to rebuild.
+            fallbackReason = "fts-query-failed";
+            // A complete bounded scan remains safe for small or explicitly bounded corpora.
         }
     }
+    if (options.fallbackLimit < options.indexedChunkCount) {
+        throw new RagmirError("INDEX_UNAVAILABLE", `Full-text search is unavailable and the fallback limit would scan only ${options.fallbackLimit} of ${options.indexedChunkCount} chunks. Run \`rgr storage optimize\` or rebuild the index before searching.`, { retryable: true });
+    }
     const fallbackQuery = table.query().select(SEARCH_COLUMNS);
-    return (await (pathPredicate ? fallbackQuery.where(pathPredicate) : fallbackQuery)
+    const rows = (await (options.pathPredicate
+        ? fallbackQuery.where(options.pathPredicate)
+        : fallbackQuery)
+        .limit(options.fallbackLimit)
+        .toArray());
+    return {
+        rows,
+        exactPathMatches: new Set(),
+        backend: "fallback",
+        fallbackActivated: true,
+        fallbackReason,
+        candidateLimit: options.fallbackLimit,
+        candidatesMaterialized: rows.length,
+        queryVariants: 0,
+        indexedRows: 0,
+        unindexedRows: options.indexedChunkCount,
+        coverage: options.indexedChunkCount === 0 ? 1 : Math.min(1, rows.length / options.indexedChunkCount),
+    };
+}
+async function executeFullTextQuery(table, query, limit, pathPredicate) {
+    const searchQuery = table.search(query, "fts", "searchText").select(FTS_SEARCH_COLUMNS);
+    return (await (pathPredicate ? searchQuery.where(pathPredicate) : searchQuery)
         .limit(limit)
         .toArray());
 }
+async function executeSourcePathQuery(table, sourcePathQuery, limit, retrievalPredicate) {
+    const predicate = retrievalPredicate
+        ? `(${sourcePathQuery.predicate}) AND (${retrievalPredicate})`
+        : sourcePathQuery.predicate;
+    return (await table.query().select(SEARCH_COLUMNS).where(predicate).limit(limit).toArray()).map((row) => ({
+        ...row,
+        _score: sourcePathQuery.exactRelativePath === row.relativePath
+            ? Number.MAX_SAFE_INTEGER
+            : Number.MAX_SAFE_INTEGER - 1,
+    }));
+}
 function searchPredicate(includePaths, excludePaths, contextPaths) {
     const includes = normalizePathPrefixes(includePaths);
     const excludes = normalizePathPrefixes(excludePaths);
@@ -393,41 +532,68 @@ function contextPathPredicate(prefix) {
         `starts_with(contextPath, ${sqlString(`${prefix}[`)})`,
     ].join(" OR ");
 }
-async function contextChunksByRow(table, rows, requestedRadius) {
+async function contextChunksByRow(table, rows, requestedRadius, retrievalPredicate) {
     const radius = Math.min(MAX_CONTEXT_RADIUS, Math.max(0, requestedRadius));
     if (radius === 0 || rows.length === 0) {
         return new Map();
     }
-    const predicates = rows.map(({ row }) => {
-        const minChunk = Math.max(0, row.chunkIndex - radius);
-        const maxChunk = row.chunkIndex + radius;
-        return `(relativePath = ${sqlString(row.relativePath)} AND chunkIndex >= ${minChunk} AND chunkIndex <= ${maxChunk})`;
-    });
+    const ranges = contextRangesByPath(rows, radius);
+    const predicates = [...ranges.entries()].flatMap(([relativePath, pathRanges]) => pathRanges.map(({ minimum, maximum }) => `(relativePath = ${sqlString(relativePath)} AND chunkIndex >= ${minimum} AND chunkIndex <= ${maximum})`));
+    const rangePredicate = predicates.join(" OR ");
+    const hydrationPredicate = retrievalPredicate
+        ? `(${rangePredicate}) AND (${retrievalPredicate})`
+        : rangePredicate;
     const allContextRows = (await table
         .query()
         .select(SEARCH_COLUMNS)
-        .where(predicates.join(" OR "))
+        .where(hydrationPredicate)
         .toArray());
+    const contextRowsByPath = new Map();
+    for (const contextRow of allContextRows) {
+        const pathRows = contextRowsByPath.get(contextRow.relativePath) ?? [];
+        pathRows.push(contextRow);
+        contextRowsByPath.set(contextRow.relativePath, pathRows);
+    }
     const contexts = new Map();
     for (const { row } of rows) {
         const minChunk = Math.max(0, row.chunkIndex - radius);
         const maxChunk = row.chunkIndex + radius;
-        const contextRows = allContextRows.filter((candidate) => candidate.relativePath === row.relativePath &&
-            candidate.chunkIndex >= minChunk &&
-            candidate.chunkIndex <= maxChunk);
+        const contextRows = (contextRowsByPath.get(row.relativePath) ?? []).filter((candidate) => candidate.chunkIndex >= minChunk && candidate.chunkIndex <= maxChunk);
         contexts.set(rowKey(row), contextRows.sort(compareChunkRows).map(contextChunkForRow));
     }
     return contexts;
 }
-async function assertVectorIndexCompatibility(config, vectorDimension) {
-    const manifest = await readIndexManifest(config);
+function contextRangesByPath(rows, radius) {
+    const rangesByPath = new Map();
+    for (const rankedRow of rows) {
+        const pathRows = rangesByPath.get(rankedRow.row.relativePath) ?? [];
+        pathRows.push(rankedRow);
+        rangesByPath.set(rankedRow.row.relativePath, pathRows);
+    }
+    return new Map([...rangesByPath.entries()].map(([relativePath, pathRows]) => {
+        const ranges = pathRows
+            .map(({ row }) => ({
+            minimum: Math.max(0, row.chunkIndex - radius),
+            maximum: row.chunkIndex + radius,
+        }))
+            .sort((left, right) => left.minimum - right.minimum);
+        const merged = [];
+        for (const range of ranges) {
+            const previous = merged.at(-1);
+            if (!previous || range.minimum > previous.maximum + 1) {
+                merged.push({ ...range });
+            }
+            else {
+                previous.maximum = Math.max(previous.maximum, range.maximum);
+            }
+        }
+        return [relativePath, merged];
+    }));
+}
+function assertVectorIndexCompatibility(config, manifest, vectorDimension) {
     if (!manifest) {
         throw new Error("Index manifest is missing. Rebuild with `rgr ingest --rebuild` before searching.");
     }
-    const freshnessWarning = await getIndexFreshnessWarning(config);
-    if (freshnessWarning) {
-        throw new Error(freshnessWarning);
-    }
     if (manifest.vectorDimension !== undefined && manifest.vectorDimension !== vectorDimension) {
         throw new Error(`Index vector dimension is ${manifest.vectorDimension} but the active embedding produced ${vectorDimension}. Rebuild with \`rgr ingest --rebuild\`.`);
     }
@@ -435,6 +601,13 @@ async function assertVectorIndexCompatibility(config, vectorDimension) {
         manifest.vectorDistanceMetric !== VECTOR_DISTANCE_METRIC) {
         throw new Error(`Index vector distance metric is ${manifest.vectorDistanceMetric} but Ragmir expects ${VECTOR_DISTANCE_METRIC}. Rebuild with \`rgr ingest --rebuild\`.`);
     }
+    if (!manifest.vectorIndex) {
+        throw new Error("Index vector strategy metadata is missing. Rebuild with `rgr ingest --rebuild`.");
+    }
+    if (!vectorIndexManifestCompatible(manifest.vectorIndex, config, vectorDimension)) {
+        throw new Error("Index vector strategy is incompatible or incomplete. Rebuild with `rgr ingest --rebuild` or repair coverage with `rgr storage optimize`.");
+    }
+    return { ...manifest, vectorIndex: manifest.vectorIndex };
 }
 function answerText(source) {
     if (source.context.length === 0) {
@@ -449,160 +622,25 @@ function contextChunkForRow(row) {
         text: row.text,
         charStart: nullableNumber(row.charStart),
         charEnd: nullableNumber(row.charEnd),
-        lineStart: nullableNumber(row.lineStart),
-        lineEnd: nullableNumber(row.lineEnd),
+        lineStart: nullableLineNumber(row.lineStart),
+        lineEnd: nullableLineNumber(row.lineEnd),
         pageStart: nullablePageNumber(row.pageStart),
         pageEnd: nullablePageNumber(row.pageEnd),
         citation: citationForRow(row),
     };
 }
-/**
- * Reciprocal Rank Fusion of vector and lexical retrievers. Rank-only: each
- * candidate scores `1/(RRF_K + rank)` per retriever it appears in, summed.
- * This removes the score-calibration problem of weighted-sum fusion (the BM25
- * and vector score distributions have nothing in common) and is the standard
- * hybrid retrieval approach.
- *
- * Ranks are 0-based internally; a candidate absent from a retriever contributes
- * nothing from that retriever. Tie-breaks keep the result deterministic.
- */
-function rankHybridRows(query, vectorRows, textRows) {
-    const queryTokens = tokenize(query);
-    const rows = mergeRows(vectorRows, textRows);
-    // Vector ranks: lower _distance is better, so sort ascending then assign rank 0..
-    const vectorRanked = [...vectorRows]
-        .filter((row) => Number.isFinite(rowDistance(row)))
-        .sort((a, b) => rowDistance(a) - rowDistance(b));
-    const vectorRanks = new Map();
-    vectorRanked.forEach((row, index) => {
-        vectorRanks.set(rowKey(row), index);
-    });
-    // LanceDB FTS already returns BM25-ranked rows. The fallback scan computes
-    // BM25 locally when an older index has no text index.
-    const ftsRows = textRows.filter((row) => typeof row._score === "number");
-    const lexicalRanked = ftsRows.length > 0
-        ? ftsRows
-            .sort((a, b) => (b._score ?? 0) - (a._score ?? 0))
-            .map((row) => [rowKey(row), row._score ?? 0])
-        : [...bm25Scores(queryTokens, rows).entries()]
-            .filter(([, score]) => score > 0)
-            .sort((a, b) => b[1] - a[1]);
-    const lexicalRanks = new Map();
-    lexicalRanked.forEach(([key], index) => {
-        lexicalRanks.set(key, index);
-    });
-    const lexicalScores = new Map(lexicalRanked);
-    return rows
-        .map((row) => {
-        const key = rowKey(row);
-        const vectorRank = vectorRanks.get(key);
-        const lexicalRank = lexicalRanks.get(key);
-        let combinedScore = 0;
-        let vectorScore = 0;
-        let lexicalScore = 0;
-        if (vectorRank !== undefined) {
-            vectorScore = RRF_VECTOR_WEIGHT / (RRF_K + vectorRank);
-            combinedScore += vectorScore;
-        }
-        if (lexicalRank !== undefined) {
-            lexicalScore = RRF_LEXICAL_WEIGHT / (RRF_K + lexicalRank);
-            combinedScore += lexicalScore;
-        }
-        return {
-            row,
-            vectorScore,
-            lexicalScore,
-            combinedScore,
-            vectorRank: vectorRank === undefined ? null : vectorRank + 1,
-            lexicalRank: lexicalRank === undefined ? null : lexicalRank + 1,
-            lexicalBackendScore: lexicalScores.get(key) ?? null,
-            matchedTerms: matchedQueryTerms(queryTokens, row.searchText),
-        };
-    })
-        .filter((ranked) => ranked.combinedScore > 0)
-        .sort((a, b) => {
-        const scoreDelta = b.combinedScore - a.combinedScore;
-        if (scoreDelta !== 0) {
-            return scoreDelta;
-        }
-        const distanceDelta = rowDistance(a.row) - rowDistance(b.row);
-        if (Number.isFinite(distanceDelta) && distanceDelta !== 0) {
-            return distanceDelta;
-        }
-        return (a.row.relativePath.localeCompare(b.row.relativePath) || a.row.chunkIndex - b.row.chunkIndex);
-    });
-}
-function matchedQueryTerms(queryTokens, text) {
+function matchedQueryTerms(projectRoot, queryTokens, text) {
+    if (queryExplanationDiagnostics.hasSubscribers) {
+        queryExplanationDiagnostics.publish({ projectRoot });
+    }
     const textTokens = tokenize(text);
     return [...new Set(queryTokens)].filter((queryToken) => textTokens.some((textToken) => tokensAreLexicallyRelated(queryToken, textToken)));
 }
-function mergeRows(vectorRows, textRows) {
-    const rows = new Map();
-    for (const row of textRows) {
-        rows.set(rowKey(row), row);
-    }
-    for (const row of vectorRows) {
-        const existing = rows.get(rowKey(row));
-        if (!existing) {
-            rows.set(rowKey(row), row);
-            continue;
-        }
-        const merged = { ...existing, ...row };
-        if (existing._score !== undefined) {
-            merged._score = existing._score;
-        }
-        rows.set(rowKey(row), merged);
-    }
-    return [...rows.values()];
-}
-function bm25Scores(queryTokens, rows) {
-    const scores = new Map();
-    if (queryTokens.length === 0 || rows.length === 0) {
-        return scores;
-    }
-    const uniqueQueryTokens = [...new Set(queryTokens)];
-    const documents = rows.map((row) => {
-        const tokens = tokenize(row.searchText);
-        const frequencies = new Map();
-        for (const token of tokens) {
-            frequencies.set(token, (frequencies.get(token) ?? 0) + 1);
-        }
-        return { row, tokens, frequencies };
-    });
-    const averageLength = documents.reduce((sum, document) => sum + document.tokens.length, 0) / documents.length || 1;
-    const documentFrequencies = new Map();
-    for (const token of uniqueQueryTokens) {
-        documentFrequencies.set(token, documents.filter((document) => document.frequencies.has(token)).length);
-    }
-    for (const document of documents) {
-        let score = 0;
-        for (const token of uniqueQueryTokens) {
-            const frequency = document.frequencies.get(token) ?? 0;
-            if (frequency === 0) {
-                continue;
-            }
-            const documentFrequency = documentFrequencies.get(token) ?? 0;
-            const inverseDocumentFrequency = Math.log(1 + (documents.length - documentFrequency + 0.5) / (documentFrequency + 0.5));
-            const denominator = frequency + BM25_K1 * (1 - BM25_B + BM25_B * (document.tokens.length / averageLength));
-            score += inverseDocumentFrequency * ((frequency * (BM25_K1 + 1)) / denominator);
-        }
-        if (score > 0) {
-            scores.set(rowKey(document.row), score);
-        }
-    }
-    return scores;
-}
-function rowDistance(row) {
-    return typeof row._distance === "number" && row._distance >= 0
-        ? row._distance
-        : Number.POSITIVE_INFINITY;
-}
-async function assertIndexFreshness(config) {
-    const manifest = await readIndexManifest(config);
+function assertIndexFreshness(config, manifest) {
     if (!manifest) {
         throw new Error("Index manifest is missing. Rebuild with `rgr ingest --rebuild` before searching.");
     }
-    const freshnessWarning = await getIndexFreshnessWarning(config);
+    const freshnessWarning = indexFreshnessWarning(config, manifest);
     if (freshnessWarning) {
         throw new Error(freshnessWarning);
     }
@@ -618,8 +656,7 @@ function parseCitationTarget(citation) {
     }
     const chunkIndex = Number.parseInt(chunkIndexText, 10);
     let relativePath = citation.slice(0, match.index);
-    relativePath = relativePath.replace(/:L\d+-L\d+$/u, "");
-    relativePath = relativePath.replace(/:p\d+(?:-p\d+)?$/u, "");
+    relativePath = stripCitationCoordinates(relativePath);
     if (!relativePath) {
         throw new Error("Citation must include a source path before its chunk suffix.");
     }
@@ -638,23 +675,70 @@ function normalizeTopK(topK) {
     if (!Number.isSafeInteger(topK) || topK <= 0) {
         throw new RagmirError("INVALID_ARGUMENT", "topK must be a positive integer.");
     }
+    if (topK > MAX_SEARCH_TOP_K) {
+        throw new RagmirError("INVALID_ARGUMENT", `topK must be at most ${MAX_SEARCH_TOP_K}.`);
+    }
     return topK;
 }
 function citationForRow(row) {
-    const lineStart = nullableNumber(row.lineStart);
-    const lineEnd = nullableNumber(row.lineEnd);
-    const pageStart = nullablePageNumber(row.pageStart);
-    const pageEnd = nullablePageNumber(row.pageEnd);
-    const pageSegment = pageStart === null
-        ? ""
-        : `:p${pageStart}${pageEnd !== null && pageEnd !== pageStart ? `-p${pageEnd}` : ""}`;
-    if (lineStart === null || lineEnd === null) {
-        return `${row.relativePath}${pageSegment}#${row.chunkIndex}`;
-    }
-    return `${row.relativePath}${pageSegment}:L${lineStart}-L${lineEnd}#${row.chunkIndex}`;
+    return citationForCoordinates(row);
 }
 function lexicalQuery(query) {
-    return [...new Set(tokenize(query))].join(" ");
+    const tokens = [...new Set(tokenize(query))];
+    if (tokens.length === 0) {
+        return null;
+    }
+    const joined = tokens.join(" ");
+    const supplemental = [];
+    if (tokens.length > 1) {
+        supplemental.push(new PhraseQuery(joined, "searchText"));
+    }
+    const identifierTerms = [...query.matchAll(LEXICAL_IDENTIFIER_PATTERN)]
+        .map((match) => match[0])
+        .filter(Boolean);
+    for (const identifier of [...new Set(identifierTerms)]) {
+        supplemental.push(new MatchQuery(identifier, "searchText", {
+            boost: 2,
+            ...(isFuzzyLexicalTerm(identifier) ? { fuzziness: 1, prefixLength: 3 } : {}),
+        }));
+    }
+    const rareTerms = tokens
+        .filter(isFuzzyLexicalTerm)
+        .sort((left, right) => right.length - left.length || left.localeCompare(right))
+        .slice(0, 3);
+    for (const term of rareTerms) {
+        supplemental.push(new MatchQuery(term, "searchText", {
+            boost: 1.25,
+            fuzziness: 1,
+            prefixLength: 3,
+        }));
+    }
+    return {
+        primary: new MatchQuery(joined, "searchText", { operator: Operator.Or }),
+        supplemental,
+    };
+}
+function isFuzzyLexicalTerm(term) {
+    return term.length >= 7 && /^[a-z0-9_.-]+$/u.test(term);
+}
+function sourcePathPredicateForQuery(query) {
+    const normalized = query.trim().replaceAll("\\", "/").replace(/^\.\//u, "");
+    if (!SOURCE_PATH_QUERY_PATTERN.test(normalized)) {
+        return null;
+    }
+    const fileName = normalized.split("/").at(-1);
+    if (!fileName) {
+        return null;
+    }
+    const clauses = new Set([
+        `relativePath = ${sqlString(normalized)}`,
+        `relativePath = ${sqlString(fileName)}`,
+        `ends_with(relativePath, ${sqlString(`/${fileName}`)})`,
+    ]);
+    return {
+        predicate: `(${[...clauses].join(" OR ")})`,
+        exactRelativePath: normalized.includes("/") ? normalized : null,
+    };
 }
 function compareChunkRows(a, b) {
     return a.chunkIndex - b.chunkIndex;
@@ -662,6 +746,9 @@ function compareChunkRows(a, b) {
 function nullableNumber(value) {
     return typeof value === "number" && Number.isFinite(value) ? value : null;
 }
+function nullableLineNumber(value) {
+    return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : null;
+}
 function nullablePageNumber(value) {
     return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : null;
 }