md-semantic-search 1.0.3 → 2.0.0

package/src/search.ts

@@ -1,17 +1,18 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import crypto from 'node:crypto';
-import { embed, cosine, decodeVec, globToRegExp, walkMarkdown, ModelDescriptor } from './core.js';
+import { embed, cosine, decodeVec, globToRegExp, walkMarkdown, fetchTimeoutMs, ModelDescriptor } from './core.js';
 import { validateIndexEnvelope, validateNumericVector } from './index-format.js';
 import { rerankScores } from './rerank.js';
 import { bm25Scores, matchingTerms, tokenize, fuzzyTitleAliasScores, LexicalIndex } from './lexical.js';
 import { collapseResults } from './collapse.js';
 import { DocumentMetadata } from './frontmatter.js';
 import { trainIVF, searchIVFCandidates, deserializeIVF, IVFIndex, ANN_THRESHOLD, DEFAULT_NPROBE } from './ivf.js';
+import { buildHnsw, searchHnswCandidates, deserializeHnsw, HnswIndex, DEFAULT_HNSW_EF } from './hnsw.js';
 import { buildRelationshipGraph, computePageRank, expandGraphNeighborhood, RelationshipGraph } from './wikilinks.js';
 import { evaluateFilter, FilterNode } from './filter.js';
 import { deserializeBinaryIndex } from './binary-format.js';
-import { dequantizeFromInt8 } from './quantization.js';
+import { dequantizeFromInt8, int8Norm, asymmetricCosineInt8CachedNorm } from './quantization.js';
 import { buildDiskKey, diskQueryGet, diskQueryPut } from './query-cache-disk.js';
 
 export { tokenize } from './lexical.js';
@@ -62,6 +63,73 @@ export function rrf(
   return fused;
 }
 
+/**
+ * issue #191: bounded top-k selection over a dense sweep — a capped min-heap
+ * instead of materializing one object per chunk and fully sorting. Ties break
+ * toward the smaller idx, which is exactly what a stable sort by
+ * (score desc) over idx-ordered input produces, so the selected pool and its
+ * order are identical to `[...items].sort((a, b) => b.score - a.score).slice(0, k)`.
+ */
+export function topKByScore(
+  n: number,
+  scoreAt: (i: number) => number,
+  k: number,
+): Array<{ idx: number; score: number }> {
+  if (k <= 0 || n <= 0) return [];
+  const cap = Math.min(k, n);
+  const scores = new Array<number>(cap);
+  const idxs = new Array<number>(cap);
+  let size = 0;
+
+  /** true when (aScore, aIdx) is a WORSE keeper than (bScore, bIdx). */
+  const worse = (aScore: number, aIdx: number, bScore: number, bIdx: number): boolean =>
+    aScore < bScore || (aScore === bScore && aIdx > bIdx);
+
+  const siftDown = (pos: number): void => {
+    for (;;) {
+      const left = pos * 2 + 1;
+      if (left >= size) return;
+      const right = left + 1;
+      const child = right < size && worse(scores[right], idxs[right], scores[left], idxs[left]) ? right : left;
+      if (!worse(scores[child], idxs[child], scores[pos], idxs[pos])) return;
+      const ts = scores[pos]; scores[pos] = scores[child]; scores[child] = ts;
+      const ti = idxs[pos]; idxs[pos] = idxs[child]; idxs[child] = ti;
+      pos = child;
+    }
+  };
+
+  for (let i = 0; i < n; i++) {
+    const s = scoreAt(i);
+    if (size < cap) {
+      let pos = size++;
+      scores[pos] = s;
+      idxs[pos] = i;
+      while (pos > 0) {
+        const parent = (pos - 1) >> 1;
+        if (!worse(scores[pos], idxs[pos], scores[parent], idxs[parent])) break;
+        const ts = scores[pos]; scores[pos] = scores[parent]; scores[parent] = ts;
+        const ti = idxs[pos]; idxs[pos] = idxs[parent]; idxs[parent] = ti;
+        pos = parent;
+      }
+    } else if (worse(scores[0], idxs[0], s, i)) {
+      scores[0] = s;
+      idxs[0] = i;
+      siftDown(0);
+    }
+  }
+
+  const out: Array<{ idx: number; score: number }> = [];
+  while (size > 0) {
+    out.push({ idx: idxs[0], score: scores[0] });
+    size--;
+    scores[0] = scores[size];
+    idxs[0] = idxs[size];
+    siftDown(0);
+  }
+  out.sort((a, b) => b.score - a.score || a.idx - b.idx);
+  return out;
+}
+
 export interface IndexChunk {
   file: string;
   title: string;
@@ -179,12 +247,23 @@ export interface RuntimeIndexState {
   model: ModelDescriptor;
   queryCache: QueryEmbeddingCache;
   ivf?: IVFIndex;
+  /** issue #201: optional HNSW graph sidecar (hnsw.json), parallel to ivf. */
+  hnsw?: HnswIndex;
   graph?: RelationshipGraph;
   pageRank?: Map<string, number>;
+  /**
+   * issue #190: precomputed sqrt(sum(c^2)) for int8 vectors (zero-copy binary
+   * load only) — lets the dense sweep compute the dot product alone.
+   */
+  chunkNorms?: Float32Array;
 }
 
-export function getRuntimeGraph(runtime: RuntimeIndexState): { graph: RelationshipGraph; pageRank: Map<string, number> } {
-  if (runtime.graph && runtime.pageRank) {
+/** issue #190 tool/test seam: the cached int8 sweep norms, when present. */
+export function getRuntimeChunkNorms(index: IndexFile): Float32Array | undefined {
+  return runtimeIndexStates.get(index)?.chunkNorms;
+}
+
+export function getRuntimeGraph(runtime: RuntimeIndexState): { graph: RelationshipGraph; pageRank: Map<string, number> } {  if (runtime.graph && runtime.pageRank) {
     return { graph: runtime.graph, pageRank: runtime.pageRank };
   }
   const docsMap = new Map<string, { file: string; title?: string; meta?: DocumentMetadata; text?: string }>();
@@ -212,12 +291,14 @@ export function getRuntimeGraph(runtime: RuntimeIndexState): { graph: Relationsh
 /**
  * Issue #127 pure helpers — exported for deterministic unit tests.
  */
-/** Frontmatter `created` → `updated` → file mtime; null when nothing parses. */
+/** Frontmatter `updated` → `created` → file mtime; null when nothing parses.
+ *  Issue #164: `updated` wins — long-lived notes that were recently revised
+ *  must not be penalized by their original creation date. */
 export function resolvePassageAgeMs(
   meta: { created?: unknown; updated?: unknown } | undefined,
   absFile?: string,
 ): number | null {
-  for (const v of [meta?.created, meta?.updated]) {
+  for (const v of [meta?.updated, meta?.created]) {
     if (typeof v === 'string') {
       const t = Date.parse(v);
       if (Number.isFinite(t)) return t;
@@ -257,6 +338,9 @@ export async function hydePassage(
     const res = await fetchImpl(endpoint.replace(/\/$/, '') + '/chat/completions', {
       method: 'POST',
       headers: { 'content-type': 'application/json' },
+      // issue #184: HyDE must degrade fast on a hung endpoint, not hold a
+      // serve slot — failure paths return null below
+      signal: AbortSignal.timeout(fetchTimeoutMs()),
       body: JSON.stringify({
         ...(model ? { model } : {}),
         messages: [
@@ -276,6 +360,67 @@ export async function hydePassage(
 
 const runtimeIndexStates = new WeakMap<IndexFile, RuntimeIndexState>();
 
+/**
+ * issue #203: LLM paraphrases for multi-query expansion. ONE OpenAI-compatible
+ * chat completion asking for N alternative phrasings; returns clean, deduped,
+ * non-empty variants (the original query excluded). Returns [] on ANY failure —
+ * multi-query must degrade to the single-query baseline silently, exactly like
+ * HyDE. Injectable fetch for tests.
+ */
+export async function paraphraseQueries(
+  endpoint: string,
+  query: string,
+  count: number,
+  model?: string,
+  fetchImpl: typeof fetch = globalThis.fetch,
+): Promise<string[]> {
+  if (!(count > 0)) return [];
+  const capped = Math.min(Math.floor(count), 5);
+  try {
+    const res = await fetchImpl(endpoint.replace(/\/$/, '') + '/chat/completions', {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      signal: AbortSignal.timeout(fetchTimeoutMs()),
+      body: JSON.stringify({
+        ...(model ? { model } : {}),
+        messages: [
+          {
+            role: 'system',
+            content:
+              `Write ${capped} alternative phrasings of the user's search query. ` +
+              'One per line, no numbering, no quotes, keep the same language and meaning. Output only the paraphrases.',
+          },
+          { role: 'user', content: query },
+        ],
+      }),
+    });
+    if (!res.ok) return [];
+    const data: any = await res.json();
+    const text = data?.choices?.[0]?.message?.content;
+    if (typeof text !== 'string' || !text.trim()) return [];
+    const seen = new Set([query.trim().toLowerCase()]);
+    const out: string[] = [];
+    for (const raw of text.split('\n')) {
+      // strip list markers, surrounding quotes, and decorative dashes LLMs
+      // like to pad output with; a bare "— junk —" line then fails the
+      // length floor instead of passing as a paraphrase
+      const line = raw
+        .replace(/^\s*(?:[-*\d]+[.)]?\s+)/, '')
+        .replace(/^["'«“”\s—–-]+|["'»“”\s—–-]+$/g, '')
+        .trim();
+      if (line.length < 3 || line.length > MAX_QUERY_LENGTH) continue;
+      const key = line.toLowerCase();
+      if (seen.has(key)) continue;
+      seen.add(key);
+      out.push(line);
+      if (out.length >= capped) break;
+    }
+    return out;
+  } catch {
+    return [];
+  }
+}
+
 /** Minimal EN+RU stopword set for PRF expansion (issue #125). */
 const EXPANSION_STOPWORDS = new Set([
   'the', 'a', 'an', 'and', 'or', 'of', 'to', 'in', 'on', 'for', 'with', 'is', 'are',
@@ -320,6 +465,7 @@ function snapshotRuntimeIndex(
     dim: number | undefined;
     model: ModelDescriptor;
     ivf?: IVFIndex;
+    hnsw?: HnswIndex;
     rawVectorsBuffer?: Float32Array | Int8Array;
   }
 ): RuntimeIndexState {
@@ -360,19 +506,32 @@ function snapshotRuntimeIndex(
 
   let lexicalState: LexicalState = { kind: 'legacy-overlap' };
   if (validation.schema >= 3 && index.lexical) {
-    const postings: Record<string, Array<[number, number]>> = Object.create(null);
-    for (const [term, posting] of Object.entries(index.lexical.postings)) {
-      postings[term] = posting.map(([docId, frequency]) => [docId, frequency]);
-    }
+    // issue #192: the per-term [docId, tf] pairs are treated as immutable
+    // everywhere (bm25Scores/matchingTerms only read), so sharing them skips
+    // the O(terms × postings) deep copy on every load/reindex. The two mutable
+    // top levels are still copied so caller-side mutations of the loaded index
+    // cannot reach the runtime state.
     lexicalState = {
       kind: 'persisted-bm25',
       lexical: {
         format: index.lexical.format,
         documentLengths: [...index.lexical.documentLengths],
-        postings,
+        postings: { ...index.lexical.postings },
       },
     };
   }
+
+  // issue #190: int8 chunk norms are constant per chunk — compute them once at
+  // load instead of inside every query's dense sweep
+  let chunkNorms: Float32Array | undefined;
+  if (validation.rawVectorsBuffer instanceof Int8Array && dim > 0 && count > 0) {
+    chunkNorms = new Float32Array(count);
+    const buffer = validation.rawVectorsBuffer;
+    for (let i = 0; i < count; i++) {
+      chunkNorms[i] = int8Norm(buffer.subarray(i * dim, (i + 1) * dim));
+    }
+  }
+
   return {
     schema: validation.schema,
     db: index.db,
@@ -382,6 +541,8 @@ function snapshotRuntimeIndex(
     model: { ...validation.model },
     queryCache: new QueryEmbeddingCache(100),
     ivf: validation.ivf,
+    hnsw: validation.hnsw,
+    chunkNorms,
   };
 }
 
@@ -430,7 +591,14 @@ function warnIfStale(index: IndexFile): void {
   );
 }
 
-export function loadIndex(indexDir: string): { index: IndexFile; model: ModelDescriptor } {
+/**
+ * Sentinel (issue #187): vectors.bin failed its sidecar check — not corrupt,
+ * just unverified — so loadIndex falls back to vectors.json instead of throwing.
+ */
+const BIN_SIDECAR_STALE = Symbol('bin-sidecar-stale');
+
+export function loadIndex(indexDir: string, opts: { /** skip the (expensive) whole-tree staleness walk — deferred to search() */ checkStale?: boolean } = {}): { index: IndexFile; model: ModelDescriptor } {
+  const { checkStale = true } = opts;
   const vectorsBinPath = path.join(indexDir, 'vectors.bin');
   const vectorsPath = path.join(indexDir, 'vectors.json');
 
@@ -442,7 +610,9 @@ export function loadIndex(indexDir: string): { index: IndexFile; model: ModelDes
       try {
         const binStat = fs.statSync(vectorsBinPath);
         const jsonStat = fs.statSync(vectorsPath);
-        if (binStat.mtimeMs >= jsonStat.mtimeMs) {
+        // issue #187: strict `>` — written in the same millisecond, vectors.json
+        // (the format of record) wins over a possibly half-published vectors.bin
+        if (binStat.mtimeMs > jsonStat.mtimeMs) {
           useBinary = true;
         }
       } catch {
@@ -459,7 +629,14 @@ export function loadIndex(indexDir: string): { index: IndexFile; model: ModelDes
         const expectedHash = fs.readFileSync(shaBinPath, 'utf8').trim().split(/\s+/)[0];
         const actualHash = crypto.createHash('sha256').update(binBuffer).digest('hex');
         if (expectedHash && actualHash !== expectedHash) {
-          throw new Error('vectors.bin integrity check failed (SHA-256 mismatch) — run `mdss index` to rebuild.');
+          // issue #187: vectors.bin is a derived cache of vectors.json — a
+          // stale sidecar (crash between the two renames) must not brick the
+          // daemon when the format of record is available. Fall through.
+          process.stderr.write(
+            'warning: vectors.bin.sha256 does not match vectors.bin — ignoring the binary cache and loading vectors.json. ' +
+            'Run `mdss check --fix` to rebuild the binary cache.\n'
+          );
+          throw BIN_SIDECAR_STALE;
         }
       }
       const deserialized = deserializeBinaryIndex(binBuffer);
@@ -468,7 +645,9 @@ export function loadIndex(indexDir: string): { index: IndexFile; model: ModelDes
       const { model, dim: validatedDim } = validated;
       const expectedDim = validatedDim ?? deserialized.dim;
 
-      warnIfStale(index);
+      // issue #192: the staleness walk is a synchronous whole-tree traversal —
+      // when the caller defers it (search()), it runs AFTER results exist
+      if (checkStale) warnIfStale(index);
 
       const ivfPath = path.join(indexDir, 'ivf.json');
       let ivf: IVFIndex | undefined;
@@ -478,6 +657,16 @@ export function loadIndex(indexDir: string): { index: IndexFile; model: ModelDes
           ivf = deserializeIVF(raw);
         } catch {}
       }
+      // issue #201: optional HNSW sidecar — corrupt file degrades to brute
+      // force exactly like a corrupt ivf.json
+      const hnswPath = path.join(indexDir, 'hnsw.json');
+      let hnsw: HnswIndex | undefined;
+      if (fs.existsSync(hnswPath)) {
+        try {
+          const raw = JSON.parse(fs.readFileSync(hnswPath, 'utf8'));
+          hnsw = deserializeHnsw(raw, index.chunks.length);
+        } catch {}
+      }
 
       runtimeIndexStates.set(
         index,
@@ -486,14 +675,17 @@ export function loadIndex(indexDir: string): { index: IndexFile; model: ModelDes
           dim: expectedDim,
           model,
           ivf,
+          hnsw,
           rawVectorsBuffer: deserialized.rawVectorsBuffer,
         })
       );
 
       return { index, model };
     } catch (e: any) {
-      if (e.message?.includes('integrity check failed')) throw e;
-      if (!fs.existsSync(vectorsPath)) throw e;
+      if (e === BIN_SIDECAR_STALE) {
+        // fall through to vectors.json — the format of record (#187)
+      } else if (e.message?.includes('integrity check failed')) throw e;
+      else if (!fs.existsSync(vectorsPath)) throw e;
       // If vectors.bin was corrupt or unreadable, fall through to vectors.json fallback
     }
   }
@@ -518,10 +710,19 @@ export function loadIndex(indexDir: string): { index: IndexFile; model: ModelDes
       const expectedHash = shaContent.split(/\s+/)[0];
       const actualHash = crypto.createHash('sha256').update(rawBytes).digest('hex');
       if (expectedHash && actualHash !== expectedHash) {
-        throw new Error(`vectors.json integrity check failed (SHA-256 mismatch) — run \`mdss index\` to rebuild.`);
+        // issue #187: atomicWrite renames the payload and its sidecar in two
+        // separate steps, so a crash mid-publish leaves a VALID payload with a
+        // STALE sidecar. The envelope above already parsed and validated —
+        // vectors.json is the format of record, so load it with a loud
+        // warning instead of bricking search/serve/mcp until a manual fix.
+        // A payload that cannot parse or validate still fails hard (above).
+        process.stderr.write(
+          'warning: vectors.json.sha256 does not match vectors.json (stale sidecar after a crash mid-publish?) — ' +
+          'the payload itself is valid and will be loaded. Run `mdss check --fix` to refresh the sidecar.\n'
+        );
       }
     } catch (e: any) {
-      if (e.message.includes('integrity check failed')) throw e;
+      if (e.message?.includes('integrity check failed')) throw e;
     }
   }
 
@@ -557,7 +758,7 @@ export function loadIndex(indexDir: string): { index: IndexFile; model: ModelDes
     }
   }
 
-  warnIfStale(index);
+  if (checkStale) warnIfStale(index);
 
   const ivfPath = path.join(indexDir, 'ivf.json');
   let ivf: IVFIndex | undefined;
@@ -567,8 +768,16 @@ export function loadIndex(indexDir: string): { index: IndexFile; model: ModelDes
       ivf = deserializeIVF(raw);
     } catch {}
   }
+  const hnswPath2 = path.join(indexDir, 'hnsw.json');
+  let hnsw: HnswIndex | undefined;
+  if (fs.existsSync(hnswPath2)) {
+    try {
+      const raw = JSON.parse(fs.readFileSync(hnswPath2, 'utf8'));
+      hnsw = deserializeHnsw(raw, index.chunks.length);
+    } catch {}
+  }
 
-  runtimeIndexStates.set(index, snapshotRuntimeIndex(index, { schema: validated.schema, dim: expectedDim, model, ivf }));
+  runtimeIndexStates.set(index, snapshotRuntimeIndex(index, { schema: validated.schema, dim: expectedDim, model, ivf, hnsw }));
 
   return { index, model };
 }
@@ -615,10 +824,23 @@ export interface SearchOptions {
   llmEndpoint?: string;
   /** issue #125: model name for the HyDE chat call. */
   llmModel?: string;
-  ann?: boolean;
+  /**
+   * issue #203: multi-query expansion — N total query variants (original +
+   * N-1 LLM paraphrases), each embedded, rankings fused with union-RRF.
+   * Requires `llmEndpoint`; without one (or on any LLM failure) it degrades
+   * to the single-query baseline. 0/1/undefined = off.
+   */
+  multiQuery?: number;
+  /** issue #203: reranker registry alias (`bge-reranker-base` default, `gte-reranker-modernbert`, ...). */
+  rerankModel?: string;
+  /** issue #201: ANN engine — `true` = default engine, or explicit 'ivf' | 'hnsw'. */
+  ann?: boolean | 'ivf' | 'hnsw';
   nprobe?: number;
+  /** issue #201: HNSW ef-search breadth (default 64). */
+  ef?: number;
   graphBoost?: number;
   filter?: string | FilterNode;
+  dimensions?: number;
 }
 
 export interface SearchResultHit {
@@ -669,11 +891,20 @@ export async function searchIndex(opts: SearchOptions): Promise<SearchResultHit[
   }
 
   const embedFn = opts.embedFn ?? embed;
-  const rerankFn = opts.rerankFn ?? rerankScores;
+  // issue #203: the registry alias rides the DEFAULT rerank path; an injected
+  // rerankFn (tests/serve) keeps its 4-arg signature and ignores it.
+  const rerankFn = opts.rerankFn ?? ((query: string, texts: string[], dir: string, off: boolean) =>
+    rerankScores(query, texts, dir, off, opts.rerankModel));
   const { index } = loaded;
   const runtime = validateRuntimeIndex(index);
   const { chunks, db, lexicalState, expectedDim } = runtime;
 
+  if (opts.dimensions !== undefined && typeof opts.dimensions === 'number' && expectedDim !== undefined && opts.dimensions !== expectedDim) {
+    throw new Error(
+      `requested search dimensions (${opts.dimensions}) does not match index dimension (${expectedDim}) — run \`mdss index --dimensions ${opts.dimensions}\` to rebuild`
+    );
+  }
+
   const pathRes = pathFilter ? (Array.isArray(pathFilter) ? pathFilter : [pathFilter]).map(globToRegExp) : [];
   let sinceMs: number | undefined;
   if (since !== undefined) {
@@ -758,60 +989,168 @@ export async function searchIndex(opts: SearchOptions): Promise<SearchResultHit[
     }
   }
 
-  let qVec: Float32Array | number[];
-  if (opts.useQueryCache !== false && runtime.queryCache) {
-    const normQuery = effectiveQuery.trim().toLowerCase();
-    const cacheKey = `${runtime.model.id}:${runtime.model.revision || 'main'}:${expectedDim || 0}:${normQuery}`;
-    // L2 disk layer (issue #114): persists across processes under cacheDir;
-    // corrupt/missing files degrade silently to a normal embed.
-    const diskKey = buildDiskKey(runtime.model.id, runtime.model.revision || 'main', expectedDim, normQuery);
-    const useDisk = opts.queryDiskCache !== false;
-    qVec = await runtime.queryCache.getOrCompute(cacheKey, async () => {
-      if (useDisk) {
-        const fromDisk = diskQueryGet(cacheDir, diskKey, expectedDim);
-        if (fromDisk) return fromDisk;
-      }
-      const [v] = await embedFn([effectiveQuery], 'query', runtime.model, cacheDir, offline);
-      if ((Array.isArray(v) || v instanceof Float32Array) && expectedDim !== undefined && v.length !== expectedDim) {
-        throw new Error(`query vector has ${v.length} dims, expected ${expectedDim} - run \`mdss index\` to rebuild`);
-      }
-      validateNumericVector(v, undefined, 'query vector');
-      if (useDisk) diskQueryPut(cacheDir, diskKey, v);
-      return v;
-    });
-  } else {
-    const [v] = await embedFn([effectiveQuery], 'query', runtime.model, cacheDir, offline);
+  // issue #203: one embed helper for the original query AND every multi-query
+  // paraphrase — both ride the same in-memory + disk query-embedding cache.
+  const embedFor = async (text: string): Promise<Float32Array | number[]> => {
+    if (opts.useQueryCache !== false && runtime.queryCache) {
+      const normQuery = text.trim().toLowerCase();
+      const cacheKey = `${runtime.model.id}:${runtime.model.revision || 'main'}:${expectedDim || 0}:${normQuery}`;
+      // L2 disk layer (issue #114): persists across processes under cacheDir;
+      // corrupt/missing files degrade silently to a normal embed.
+      const diskKey = buildDiskKey(runtime.model.id, runtime.model.revision || 'main', expectedDim, normQuery);
+      const useDisk = opts.queryDiskCache !== false;
+      return runtime.queryCache.getOrCompute(cacheKey, async () => {
+        if (useDisk) {
+          const fromDisk = diskQueryGet(cacheDir, diskKey, expectedDim);
+          if (fromDisk) return fromDisk;
+        }
+        const [v] = await embedFn([text], 'query', runtime.model, cacheDir, offline);
+        if ((Array.isArray(v) || v instanceof Float32Array) && expectedDim !== undefined && v.length !== expectedDim) {
+          throw new Error(`query vector has ${v.length} dims, expected ${expectedDim} - run \`mdss index\` to rebuild`);
+        }
+        validateNumericVector(v, undefined, 'query vector');
+        if (useDisk) diskQueryPut(cacheDir, diskKey, v);
+        return v;
+      });
+    }
+    const [v] = await embedFn([text], 'query', runtime.model, cacheDir, offline);
     if ((Array.isArray(v) || v instanceof Float32Array) && expectedDim !== undefined && v.length !== expectedDim) {
       throw new Error(`query vector has ${v.length} dims, expected ${expectedDim} — run \`mdss index\` to rebuild`);
     }
     validateNumericVector(v, undefined, 'query vector');
-    qVec = v;
+    return v;
+  };
+
+  const qVec = await embedFor(effectiveQuery);
+
+  // issue #203: multi-query expansion — N total variants (original first),
+  // each embedded, dense rankings fused by union-RRF below. Paraphrases come
+  // from the LLM bridge; without an endpoint (or on any failure) the list is
+  // empty and the search is byte-identical to the single-query baseline.
+  const multiQueryN = Number.isFinite(opts.multiQuery) ? Math.floor(Number(opts.multiQuery)) : 0;
+  const paraphrases =
+    multiQueryN >= 2 && opts.llmEndpoint
+      ? await paraphraseQueries(opts.llmEndpoint, query, multiQueryN - 1, opts.llmModel)
+      : [];
+  const qVecs: Array<Float32Array | number[]> = [qVec];
+  for (const p of paraphrases.slice(0, multiQueryN - 1)) {
+    try {
+      qVecs.push(await embedFor(p));
+    } catch {
+      // a paraphrase that fails to embed (e.g. dim mismatch) is dropped, not fatal
+    }
   }
   const queryTerms = [...new Set(tokenize(query))];
 
+  const maxPerDoc = opts.maxPerFile || opts.maxPerDoc;
+  // Issue #127: time-decay recency boost — multiply the fused score by
+  // 0.5^(ageDays / halfLifeDays), post-RRF and pre-collapse. Age source
+  // priority: frontmatter `updated` → `created` → file mtime (issue #164). Missing dates
+  // get factor 1 (evergreen content stays reachable via lexical strength).
+  const recencyHalfLife = Number(opts.recency) > 0 ? Number(opts.recency) : 0;
+  // Issue #163: recency rescores AFTER the pool slice, so the pool must be
+  // expanded like the maxPerDoc path — otherwise candidates ranked k+1..n
+  // by raw RRF are dropped before the fresh-document boost can lift them.
+  // issue #191: the pool must be known BEFORE the sweep for the bounded top-k.
+  const pool = rerank
+    ? (rerankPool || (maxPerDoc && maxPerDoc > 0 ? Math.max(50, k * 5) : Math.max(20, k * 3)))
+    : (maxPerDoc && maxPerDoc > 0 || recencyHalfLife > 0 ? Math.max(50, k * 5) : k);
+
   let semantic: Array<{ idx: number; score: number }>;
-  const useAnn = opts.ann ?? (runtime.ivf !== undefined && candidates.length >= ANN_THRESHOLD);
-  if (useAnn && candidates.length >= ANN_THRESHOLD) {
-    if (!runtime.ivf) {
-      const rawVecs = chunks.map((c) => (c.vec instanceof Float32Array ? c.vec : dequantizeFromInt8(c.vec)));
-      runtime.ivf = trainIVF(rawVecs);
-    }
-    const qVecF32 = qVec instanceof Float32Array ? qVec : Float32Array.from(qVec);
-    const probedSet = new Set(searchIVFCandidates(qVecF32, runtime.ivf, opts.nprobe ?? DEFAULT_NPROBE));
+  let cosByIdx: Map<number, number> | undefined;
+  // issue #201: engine selection — explicit 'ivf'/'hnsw' wins; `true` picks
+  // whichever sidecar exists (hnsw preferred, then ivf); otherwise the
+  // automatic threshold path (sidecar present AND corpus above ANN_THRESHOLD)
+  const annOpt = opts.ann;
+  const autoAnn = annOpt === undefined || annOpt === false
+    ? Boolean(runtime.hnsw || runtime.ivf) && candidates.length >= ANN_THRESHOLD
+    : candidates.length >= ANN_THRESHOLD;
+  const useAnn = Boolean(annOpt) || autoAnn;
+  const annEngine: 'ivf' | 'hnsw' | undefined = useAnn
+    ? (annOpt === 'hnsw' || (annOpt !== 'ivf' && runtime.hnsw) ? 'hnsw' : 'ivf')
+    : undefined;
+  // issue #190: with cached int8 norms the sweep computes the dot product alone
+  const norms = runtime.chunkNorms;
+  const sweepScore = (chunk: RuntimeChunk, idx: number): number =>
+    norms && chunk.vec instanceof Int8Array
+      ? asymmetricCosineInt8CachedNorm(qVec as Float32Array, chunk.vec, norms[idx])
+      : cosine(qVec, chunk.vec);
+  // issue #203: ANN probing unions the candidate sets of every query variant —
+  // a paraphrase must not recall into chunks the original's probe excluded
+  const probed = useAnn && candidates.length >= ANN_THRESHOLD
+    ? (() => {
+        const probedUnion = new Set<number>();
+        for (const vec of qVecs) {
+          const qVecF32 = vec instanceof Float32Array ? vec : Float32Array.from(vec);
+          if (annEngine === 'hnsw') {
+            // issue #201: HNSW graph search — built on the hot path only when
+            // the sidecar is missing (same semantics as the IVF sync train)
+            if (!runtime.hnsw) {
+              const rawVecs = chunks.map((c) => (c.vec instanceof Float32Array ? c.vec : dequantizeFromInt8(c.vec)));
+              runtime.hnsw = buildHnsw(rawVecs, {
+                vectors: rawVecs,
+                distance: (a, b) => {
+                  const bv = rawVecs[b];
+                  return 1 - (a as Float32Array).reduce((sum, x, i) => sum + x * (bv as ArrayLike<number>)[i], 0);
+                },
+              });
+            }
+            for (const idx of searchHnswCandidates(qVecF32, runtime.hnsw, chunks.map((c) => c.vec), { ef: opts.ef ?? DEFAULT_HNSW_EF })) {
+              probedUnion.add(idx);
+            }
+          } else {
+            if (!runtime.ivf) {
+              const rawVecs = chunks.map((c) => (c.vec instanceof Float32Array ? c.vec : dequantizeFromInt8(c.vec)));
+              runtime.ivf = trainIVF(rawVecs);
+            }
+            for (const idx of searchIVFCandidates(qVecF32, runtime.ivf, opts.nprobe ?? DEFAULT_NPROBE)) {
+              probedUnion.add(idx);
+            }
+          }
+        }
+        return probedUnion;
+      })()
+    : null;
+  if (qVecs.length > 1) {
+    // issue #203: multi-query — one dense sweep per variant, rankings fused by
+    // union-RRF. `cos` stays anchored to the ORIGINAL query's cosine so the
+    // hit shape (and score display) keeps its meaning.
+    const sweepWith = (vec: Float32Array | number[]) => (chunk: RuntimeChunk, idx: number): number =>
+      norms && chunk.vec instanceof Int8Array
+        ? asymmetricCosineInt8CachedNorm(vec as Float32Array, chunk.vec, norms[idx])
+        : cosine(vec, chunk.vec);
+    const perVectorRankings = qVecs.map((vec) =>
+      candidates.map(({ chunk, idx }) => ({
+        idx,
+        score: probed && !probed.has(idx) ? -1 : sweepWith(vec)(chunk, idx),
+      })),
+    );
+    const fusedSemantic = rrf(perVectorRankings, 60);
+    semantic = [...fusedSemantic.entries()].map(([idx, score]) => ({ idx, score }));
+    cosByIdx = new Map(perVectorRankings[0].map((s) => [s.idx, s.score]));
+  } else if (semanticOnly && !(graphBoost > 0)) {
+    // issue #191: the pure-semantic ranking only needs the top pool — select
+    // it with a bounded min-heap in one pass (no per-chunk objects, no full
+    // sort). Equivalent to [...candidates].map(score).sort(desc).slice(0, pool)
+    // with the stable tie-break toward the smaller idx.
+    semantic = topKByScore(
+      candidates.length,
+      (i) => {
+        const { chunk, idx } = candidates[i];
+        if (probed && !probed.has(idx)) return -1;
+        return sweepScore(chunk, idx);
+      },
+      pool,
+    );
+  } else {
     semantic = candidates.map(({ chunk, idx }) => {
-      if (!probedSet.has(idx)) {
+      if (probed && !probed.has(idx)) {
         return { idx, score: -1 };
       }
-      return { idx, score: cosine(qVec, chunk.vec) };
-    });
-  } else {
-    semantic = candidates.map(({ chunk, idx }) => {
-      return { idx, score: cosine(qVec, chunk.vec) };
+      return { idx, score: sweepScore(chunk, idx) };
     });
+    cosByIdx = new Map(semantic.map((s) => [s.idx, s.score]));
   }
-  const cosByIdx = new Map(semantic.map((s) => [s.idx, s.score]));
-
-  const pool = rerank ? rerankPool || Math.max(20, k * 3) : k;
 
   let graphRanking: Array<{ idx: number; score: number }> = [];
   let graphMap: Map<number, number> | undefined;
@@ -843,14 +1182,14 @@ export async function searchIndex(opts: SearchOptions): Promise<SearchResultHit[
     if (graphBoost > 0) {
       const fused = rrf([semantic, graphRanking], 60, [1, graphBoost]);
       ranked = [...fused.entries()]
-        .map(([idx, fscore]) => ({ idx, fscore, cos: cosByIdx.get(idx)! }))
+        .map(([idx, fscore]) => ({ idx, fscore, cos: cosByIdx!.get(idx)! }))
         .sort((a, b) => b.fscore - a.fscore)
         .slice(0, pool);
     } else {
-      ranked = [...semantic]
-        .sort((a, b) => b.score - a.score)
-        .slice(0, pool)
-        .map((s) => ({ idx: s.idx, fscore: s.score, cos: s.score }));
+      // issue #191: `semantic` is already the top pool, ordered (score desc, idx asc);
+      // issue #203: with multi-query the pool scores are RRF — cos stays the
+      // original query's cosine via cosByIdx
+      ranked = semantic.map((s) => ({ idx: s.idx, fscore: s.score, cos: cosByIdx ? cosByIdx.get(s.idx)! : s.score }));
     }
   } else {
     const kw =
@@ -871,23 +1210,18 @@ export async function searchIndex(opts: SearchOptions): Promise<SearchResultHit[
     if (graphBoost > 0) {
       const fused = rrf([semantic, kw, fuzzy, graphRanking], 60, [1, 1, 1, graphBoost]);
       ranked = [...fused.entries()]
-        .map(([idx, fscore]) => ({ idx, fscore, cos: cosByIdx.get(idx)! }))
+        .map(([idx, fscore]) => ({ idx, fscore, cos: cosByIdx!.get(idx)! }))
         .sort((a, b) => b.fscore - a.fscore)
         .slice(0, pool);
     } else {
       const fused = rrf([semantic, kw, fuzzy]);
       ranked = [...fused.entries()]
-        .map(([idx, fscore]) => ({ idx, fscore, cos: cosByIdx.get(idx)! }))
+        .map(([idx, fscore]) => ({ idx, fscore, cos: cosByIdx!.get(idx)! }))
         .sort((a, b) => b.fscore - a.fscore)
         .slice(0, pool);
     }
   }
 
-  // Issue #127: time-decay recency boost — multiply the fused score by
-  // 0.5^(ageDays / halfLifeDays), post-RRF and pre-collapse. Age source
-  // priority: frontmatter `created` → `updated` → file mtime. Missing dates
-  // get factor 1 (evergreen content stays reachable via lexical strength).
-  const recencyHalfLife = Number(opts.recency) > 0 ? Number(opts.recency) : 0;
   if (recencyHalfLife > 0) {
     const now = Date.now();
     const ageCache = new Map<string, number | null>();
@@ -910,8 +1244,10 @@ export async function searchIndex(opts: SearchOptions): Promise<SearchResultHit[
     const scores = await rerankFn(query, texts, cacheDir, offline);
     ranked = ranked
       .map((r, i) => ({ ...r, rerank: scores[i] ?? -Infinity }))
-      .sort((a, b) => b.rerank - a.rerank)
-      .slice(0, k);
+      .sort((a, b) => b.rerank - a.rerank);
+    if (!maxPerDoc || maxPerDoc <= 0) {
+      ranked = ranked.slice(0, k);
+    }
   }
 
   const hasMetaFields = (m?: DocumentMetadata) =>
@@ -974,18 +1310,24 @@ export async function searchIndex(opts: SearchOptions): Promise<SearchResultHit[
     };
   });
 
-  const maxPerDoc = opts.maxPerFile || opts.maxPerDoc;
   if (maxPerDoc && maxPerDoc > 0) {
     return collapseResults(hits, (h) => h.meta?.canonicalRef || h.file, maxPerDoc).slice(0, k);
   }
-  return hits;
+  // Issue #163: with the pool expanded for recency the final k-truncation
+  // moves here (it used to be the pool slice itself). Paths that already
+  // truncate (rerank, maxPerDoc collapse) keep their behavior.
+  return hits.slice(0, k);
 }
 
 export type OneShotSearchOptions = Omit<SearchOptions, 'loaded'> & { indexDir: string };
 
 export async function search(opts: OneShotSearchOptions): Promise<SearchResultHit[]> {
   const { indexDir, ...rest } = opts;
-  const loaded = loadIndex(indexDir);
-  return searchIndex({ ...rest, loaded });
+  // issue #192: the staleness check walks the entire tree synchronously —
+  // defer it until AFTER the results are computed so it never delays them
+  const loaded = loadIndex(indexDir, { checkStale: false });
+  const hits = await searchIndex({ ...rest, loaded });
+  warnIfStale(loaded.index);
+  return hits;
 }