md-semantic-search 1.0.3 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/README.md +132 -16
  2. package/RESEARCH.md +207 -3
  3. package/bin/cli.mjs +449 -13
  4. package/dist/binary-format.js +9 -0
  5. package/dist/completions.js +11 -1
  6. package/dist/core.d.ts +42 -4
  7. package/dist/core.js +208 -14
  8. package/dist/federation.d.ts +16 -1
  9. package/dist/federation.js +33 -16
  10. package/dist/filter.d.ts +7 -0
  11. package/dist/filter.js +13 -0
  12. package/dist/hnsw.d.ts +96 -0
  13. package/dist/hnsw.js +326 -0
  14. package/dist/index-format.js +8 -1
  15. package/dist/index.d.ts +4 -4
  16. package/dist/index.js +4 -4
  17. package/dist/indexer.d.ts +18 -1
  18. package/dist/indexer.js +92 -18
  19. package/dist/ivf.d.ts +14 -7
  20. package/dist/ivf.js +126 -18
  21. package/dist/lexical.d.ts +10 -0
  22. package/dist/lexical.js +98 -15
  23. package/dist/markdown-parser.d.ts +62 -0
  24. package/dist/markdown-parser.js +227 -30
  25. package/dist/mcp.d.ts +229 -7
  26. package/dist/mcp.js +743 -44
  27. package/dist/models.d.ts +8 -2
  28. package/dist/models.js +84 -32
  29. package/dist/providers.d.ts +56 -0
  30. package/dist/providers.js +305 -0
  31. package/dist/quantization.d.ts +12 -0
  32. package/dist/quantization.js +54 -0
  33. package/dist/rag.d.ts +132 -7
  34. package/dist/rag.js +314 -38
  35. package/dist/repair.d.ts +1 -1
  36. package/dist/repair.js +53 -2
  37. package/dist/rerank.d.ts +51 -6
  38. package/dist/rerank.js +69 -10
  39. package/dist/search.d.ts +49 -3
  40. package/dist/search.js +387 -78
  41. package/dist/serve.d.ts +35 -0
  42. package/dist/serve.js +574 -58
  43. package/dist/tui.d.ts +14 -0
  44. package/dist/tui.js +132 -15
  45. package/dist/watcher.js +23 -12
  46. package/dist/webui.d.ts +3 -3
  47. package/dist/webui.js +225 -5
  48. package/dist/wikilinks.d.ts +31 -0
  49. package/dist/wikilinks.js +185 -1
  50. package/package.json +3 -3
  51. package/src/binary-format.ts +13 -0
  52. package/src/completions.ts +11 -1
  53. package/src/core.ts +249 -13
  54. package/src/federation.ts +51 -17
  55. package/src/filter.ts +14 -0
  56. package/src/hnsw.ts +414 -0
  57. package/src/index-format.ts +7 -1
  58. package/src/index.ts +7 -2
  59. package/src/indexer.ts +112 -19
  60. package/src/ivf.ts +157 -20
  61. package/src/lexical.ts +91 -13
  62. package/src/markdown-parser.ts +279 -37
  63. package/src/mcp.ts +820 -47
  64. package/src/models.ts +97 -35
  65. package/src/providers.ts +345 -0
  66. package/src/quantization.ts +61 -0
  67. package/src/rag.ts +451 -34
  68. package/src/repair.ts +51 -2
  69. package/src/rerank.ts +121 -14
  70. package/src/search.ts +424 -82
  71. package/src/serve.ts +589 -66
  72. package/src/tui.ts +142 -16
  73. package/src/watcher.ts +21 -10
  74. package/src/webui.ts +225 -5
  75. package/src/wikilinks.ts +220 -1
package/dist/search.js CHANGED
@@ -1,16 +1,17 @@
1
1
  import fs from 'node:fs';
2
2
  import path from 'node:path';
3
3
  import crypto from 'node:crypto';
4
- import { embed, cosine, decodeVec, globToRegExp, walkMarkdown } from './core.js';
4
+ import { embed, cosine, decodeVec, globToRegExp, walkMarkdown, fetchTimeoutMs } from './core.js';
5
5
  import { validateIndexEnvelope, validateNumericVector } from './index-format.js';
6
6
  import { rerankScores } from './rerank.js';
7
7
  import { bm25Scores, matchingTerms, tokenize, fuzzyTitleAliasScores } from './lexical.js';
8
8
  import { collapseResults } from './collapse.js';
9
9
  import { trainIVF, searchIVFCandidates, deserializeIVF, ANN_THRESHOLD, DEFAULT_NPROBE } from './ivf.js';
10
+ import { buildHnsw, searchHnswCandidates, deserializeHnsw, DEFAULT_HNSW_EF } from './hnsw.js';
10
11
  import { buildRelationshipGraph, computePageRank, expandGraphNeighborhood } from './wikilinks.js';
11
12
  import { evaluateFilter } from './filter.js';
12
13
  import { deserializeBinaryIndex } from './binary-format.js';
13
- import { dequantizeFromInt8 } from './quantization.js';
14
+ import { dequantizeFromInt8, int8Norm, asymmetricCosineInt8CachedNorm } from './quantization.js';
14
15
  import { buildDiskKey, diskQueryGet, diskQueryPut } from './query-cache-disk.js';
15
16
  export { tokenize } from './lexical.js';
16
17
  const chunkTokens = new WeakMap();
@@ -57,6 +58,76 @@ export function rrf(rankings, k = 60, weights) {
57
58
  }
58
59
  return fused;
59
60
  }
61
+ /**
62
+ * issue #191: bounded top-k selection over a dense sweep — a capped min-heap
63
+ * instead of materializing one object per chunk and fully sorting. Ties break
64
+ * toward the smaller idx, which is exactly what a stable sort by
65
+ * (score desc) over idx-ordered input produces, so the selected pool and its
66
+ * order are identical to `[...items].sort((a, b) => b.score - a.score).slice(0, k)`.
67
+ */
68
+ export function topKByScore(n, scoreAt, k) {
69
+ if (k <= 0 || n <= 0)
70
+ return [];
71
+ const cap = Math.min(k, n);
72
+ const scores = new Array(cap);
73
+ const idxs = new Array(cap);
74
+ let size = 0;
75
+ /** true when (aScore, aIdx) is a WORSE keeper than (bScore, bIdx). */
76
+ const worse = (aScore, aIdx, bScore, bIdx) => aScore < bScore || (aScore === bScore && aIdx > bIdx);
77
+ const siftDown = (pos) => {
78
+ for (;;) {
79
+ const left = pos * 2 + 1;
80
+ if (left >= size)
81
+ return;
82
+ const right = left + 1;
83
+ const child = right < size && worse(scores[right], idxs[right], scores[left], idxs[left]) ? right : left;
84
+ if (!worse(scores[child], idxs[child], scores[pos], idxs[pos]))
85
+ return;
86
+ const ts = scores[pos];
87
+ scores[pos] = scores[child];
88
+ scores[child] = ts;
89
+ const ti = idxs[pos];
90
+ idxs[pos] = idxs[child];
91
+ idxs[child] = ti;
92
+ pos = child;
93
+ }
94
+ };
95
+ for (let i = 0; i < n; i++) {
96
+ const s = scoreAt(i);
97
+ if (size < cap) {
98
+ let pos = size++;
99
+ scores[pos] = s;
100
+ idxs[pos] = i;
101
+ while (pos > 0) {
102
+ const parent = (pos - 1) >> 1;
103
+ if (!worse(scores[pos], idxs[pos], scores[parent], idxs[parent]))
104
+ break;
105
+ const ts = scores[pos];
106
+ scores[pos] = scores[parent];
107
+ scores[parent] = ts;
108
+ const ti = idxs[pos];
109
+ idxs[pos] = idxs[parent];
110
+ idxs[parent] = ti;
111
+ pos = parent;
112
+ }
113
+ }
114
+ else if (worse(scores[0], idxs[0], s, i)) {
115
+ scores[0] = s;
116
+ idxs[0] = i;
117
+ siftDown(0);
118
+ }
119
+ }
120
+ const out = [];
121
+ while (size > 0) {
122
+ out.push({ idx: idxs[0], score: scores[0] });
123
+ size--;
124
+ scores[0] = scores[size];
125
+ idxs[0] = idxs[size];
126
+ siftDown(0);
127
+ }
128
+ out.sort((a, b) => b.score - a.score || a.idx - b.idx);
129
+ return out;
130
+ }
60
131
  export class QueryEmbeddingCache {
61
132
  maxSize;
62
133
  cache;
@@ -125,6 +196,10 @@ export class QueryEmbeddingCache {
125
196
  return this.cache.size;
126
197
  }
127
198
  }
199
+ /** issue #190 tool/test seam: the cached int8 sweep norms, when present. */
200
+ export function getRuntimeChunkNorms(index) {
201
+ return runtimeIndexStates.get(index)?.chunkNorms;
202
+ }
128
203
  export function getRuntimeGraph(runtime) {
129
204
  if (runtime.graph && runtime.pageRank) {
130
205
  return { graph: runtime.graph, pageRank: runtime.pageRank };
@@ -154,9 +229,11 @@ export function getRuntimeGraph(runtime) {
154
229
  /**
155
230
  * Issue #127 pure helpers — exported for deterministic unit tests.
156
231
  */
157
- /** Frontmatter `created` → `updated` → file mtime; null when nothing parses. */
232
+ /** Frontmatter `updated` → `created` → file mtime; null when nothing parses.
233
+ * Issue #164: `updated` wins — long-lived notes that were recently revised
234
+ * must not be penalized by their original creation date. */
158
235
  export function resolvePassageAgeMs(meta, absFile) {
159
- for (const v of [meta?.created, meta?.updated]) {
236
+ for (const v of [meta?.updated, meta?.created]) {
160
237
  if (typeof v === 'string') {
161
238
  const t = Date.parse(v);
162
239
  if (Number.isFinite(t))
@@ -193,6 +270,9 @@ export async function hydePassage(endpoint, query, model, fetchImpl = globalThis
193
270
  const res = await fetchImpl(endpoint.replace(/\/$/, '') + '/chat/completions', {
194
271
  method: 'POST',
195
272
  headers: { 'content-type': 'application/json' },
273
+ // issue #184: HyDE must degrade fast on a hung endpoint, not hold a
274
+ // serve slot — failure paths return null below
275
+ signal: AbortSignal.timeout(fetchTimeoutMs()),
196
276
  body: JSON.stringify({
197
277
  ...(model ? { model } : {}),
198
278
  messages: [
@@ -212,6 +292,66 @@ export async function hydePassage(endpoint, query, model, fetchImpl = globalThis
212
292
  }
213
293
  }
214
294
  const runtimeIndexStates = new WeakMap();
295
+ /**
296
+ * issue #203: LLM paraphrases for multi-query expansion. ONE OpenAI-compatible
297
+ * chat completion asking for N alternative phrasings; returns clean, deduped,
298
+ * non-empty variants (the original query excluded). Returns [] on ANY failure —
299
+ * multi-query must degrade to the single-query baseline silently, exactly like
300
+ * HyDE. Injectable fetch for tests.
301
+ */
302
+ export async function paraphraseQueries(endpoint, query, count, model, fetchImpl = globalThis.fetch) {
303
+ if (!(count > 0))
304
+ return [];
305
+ const capped = Math.min(Math.floor(count), 5);
306
+ try {
307
+ const res = await fetchImpl(endpoint.replace(/\/$/, '') + '/chat/completions', {
308
+ method: 'POST',
309
+ headers: { 'content-type': 'application/json' },
310
+ signal: AbortSignal.timeout(fetchTimeoutMs()),
311
+ body: JSON.stringify({
312
+ ...(model ? { model } : {}),
313
+ messages: [
314
+ {
315
+ role: 'system',
316
+ content: `Write ${capped} alternative phrasings of the user's search query. ` +
317
+ 'One per line, no numbering, no quotes, keep the same language and meaning. Output only the paraphrases.',
318
+ },
319
+ { role: 'user', content: query },
320
+ ],
321
+ }),
322
+ });
323
+ if (!res.ok)
324
+ return [];
325
+ const data = await res.json();
326
+ const text = data?.choices?.[0]?.message?.content;
327
+ if (typeof text !== 'string' || !text.trim())
328
+ return [];
329
+ const seen = new Set([query.trim().toLowerCase()]);
330
+ const out = [];
331
+ for (const raw of text.split('\n')) {
332
+ // strip list markers, surrounding quotes, and decorative dashes LLMs
333
+ // like to pad output with; a bare "— junk —" line then fails the
334
+ // length floor instead of passing as a paraphrase
335
+ const line = raw
336
+ .replace(/^\s*(?:[-*\d]+[.)]?\s+)/, '')
337
+ .replace(/^["'«“”\s—–-]+|["'»“”\s—–-]+$/g, '')
338
+ .trim();
339
+ if (line.length < 3 || line.length > MAX_QUERY_LENGTH)
340
+ continue;
341
+ const key = line.toLowerCase();
342
+ if (seen.has(key))
343
+ continue;
344
+ seen.add(key);
345
+ out.push(line);
346
+ if (out.length >= capped)
347
+ break;
348
+ }
349
+ return out;
350
+ }
351
+ catch {
352
+ return [];
353
+ }
354
+ }
215
355
  /** Minimal EN+RU stopword set for PRF expansion (issue #125). */
216
356
  const EXPANSION_STOPWORDS = new Set([
217
357
  'the', 'a', 'an', 'and', 'or', 'of', 'to', 'in', 'on', 'for', 'with', 'is', 'are',
@@ -283,19 +423,30 @@ function snapshotRuntimeIndex(index, validation) {
283
423
  });
284
424
  let lexicalState = { kind: 'legacy-overlap' };
285
425
  if (validation.schema >= 3 && index.lexical) {
286
- const postings = Object.create(null);
287
- for (const [term, posting] of Object.entries(index.lexical.postings)) {
288
- postings[term] = posting.map(([docId, frequency]) => [docId, frequency]);
289
- }
426
+ // issue #192: the per-term [docId, tf] pairs are treated as immutable
427
+ // everywhere (bm25Scores/matchingTerms only read), so sharing them skips
428
+ // the O(terms × postings) deep copy on every load/reindex. The two mutable
429
+ // top levels are still copied so caller-side mutations of the loaded index
430
+ // cannot reach the runtime state.
290
431
  lexicalState = {
291
432
  kind: 'persisted-bm25',
292
433
  lexical: {
293
434
  format: index.lexical.format,
294
435
  documentLengths: [...index.lexical.documentLengths],
295
- postings,
436
+ postings: { ...index.lexical.postings },
296
437
  },
297
438
  };
298
439
  }
440
+ // issue #190: int8 chunk norms are constant per chunk — compute them once at
441
+ // load instead of inside every query's dense sweep
442
+ let chunkNorms;
443
+ if (validation.rawVectorsBuffer instanceof Int8Array && dim > 0 && count > 0) {
444
+ chunkNorms = new Float32Array(count);
445
+ const buffer = validation.rawVectorsBuffer;
446
+ for (let i = 0; i < count; i++) {
447
+ chunkNorms[i] = int8Norm(buffer.subarray(i * dim, (i + 1) * dim));
448
+ }
449
+ }
299
450
  return {
300
451
  schema: validation.schema,
301
452
  db: index.db,
@@ -305,6 +456,8 @@ function snapshotRuntimeIndex(index, validation) {
305
456
  model: { ...validation.model },
306
457
  queryCache: new QueryEmbeddingCache(100),
307
458
  ivf: validation.ivf,
459
+ hnsw: validation.hnsw,
460
+ chunkNorms,
308
461
  };
309
462
  }
310
463
  function validateRuntimeIndex(index) {
@@ -354,7 +507,13 @@ function warnIfStale(index) {
354
507
  const mins = Math.max(1, Math.round((newest - builtMs) / 60000));
355
508
  process.stderr.write(`warning: index is ${mins} min older than the newest change in ${index.db}; run \`mdss index\` to refresh.\n`);
356
509
  }
357
- export function loadIndex(indexDir) {
510
+ /**
511
+ * Sentinel (issue #187): vectors.bin failed its sidecar check — not corrupt,
512
+ * just unverified — so loadIndex falls back to vectors.json instead of throwing.
513
+ */
514
+ const BIN_SIDECAR_STALE = Symbol('bin-sidecar-stale');
515
+ export function loadIndex(indexDir, opts = {}) {
516
+ const { checkStale = true } = opts;
358
517
  const vectorsBinPath = path.join(indexDir, 'vectors.bin');
359
518
  const vectorsPath = path.join(indexDir, 'vectors.json');
360
519
  let useBinary = false;
@@ -366,7 +525,9 @@ export function loadIndex(indexDir) {
366
525
  try {
367
526
  const binStat = fs.statSync(vectorsBinPath);
368
527
  const jsonStat = fs.statSync(vectorsPath);
369
- if (binStat.mtimeMs >= jsonStat.mtimeMs) {
528
+ // issue #187: strict `>` — written in the same millisecond, vectors.json
529
+ // (the format of record) wins over a possibly half-published vectors.bin
530
+ if (binStat.mtimeMs > jsonStat.mtimeMs) {
370
531
  useBinary = true;
371
532
  }
372
533
  }
@@ -383,7 +544,12 @@ export function loadIndex(indexDir) {
383
544
  const expectedHash = fs.readFileSync(shaBinPath, 'utf8').trim().split(/\s+/)[0];
384
545
  const actualHash = crypto.createHash('sha256').update(binBuffer).digest('hex');
385
546
  if (expectedHash && actualHash !== expectedHash) {
386
- throw new Error('vectors.bin integrity check failed (SHA-256 mismatch) — run `mdss index` to rebuild.');
547
+ // issue #187: vectors.bin is a derived cache of vectors.json — a
548
+ // stale sidecar (crash between the two renames) must not brick the
549
+ // daemon when the format of record is available. Fall through.
550
+ process.stderr.write('warning: vectors.bin.sha256 does not match vectors.bin — ignoring the binary cache and loading vectors.json. ' +
551
+ 'Run `mdss check --fix` to rebuild the binary cache.\n');
552
+ throw BIN_SIDECAR_STALE;
387
553
  }
388
554
  }
389
555
  const deserialized = deserializeBinaryIndex(binBuffer);
@@ -391,7 +557,10 @@ export function loadIndex(indexDir) {
391
557
  const index = validated.index;
392
558
  const { model, dim: validatedDim } = validated;
393
559
  const expectedDim = validatedDim ?? deserialized.dim;
394
- warnIfStale(index);
560
+ // issue #192: the staleness walk is a synchronous whole-tree traversal —
561
+ // when the caller defers it (search()), it runs AFTER results exist
562
+ if (checkStale)
563
+ warnIfStale(index);
395
564
  const ivfPath = path.join(indexDir, 'ivf.json');
396
565
  let ivf;
397
566
  if (fs.existsSync(ivfPath)) {
@@ -401,19 +570,34 @@ export function loadIndex(indexDir) {
401
570
  }
402
571
  catch { }
403
572
  }
573
+ // issue #201: optional HNSW sidecar — corrupt file degrades to brute
574
+ // force exactly like a corrupt ivf.json
575
+ const hnswPath = path.join(indexDir, 'hnsw.json');
576
+ let hnsw;
577
+ if (fs.existsSync(hnswPath)) {
578
+ try {
579
+ const raw = JSON.parse(fs.readFileSync(hnswPath, 'utf8'));
580
+ hnsw = deserializeHnsw(raw, index.chunks.length);
581
+ }
582
+ catch { }
583
+ }
404
584
  runtimeIndexStates.set(index, snapshotRuntimeIndex(index, {
405
585
  schema: validated.schema,
406
586
  dim: expectedDim,
407
587
  model,
408
588
  ivf,
589
+ hnsw,
409
590
  rawVectorsBuffer: deserialized.rawVectorsBuffer,
410
591
  }));
411
592
  return { index, model };
412
593
  }
413
594
  catch (e) {
414
- if (e.message?.includes('integrity check failed'))
595
+ if (e === BIN_SIDECAR_STALE) {
596
+ // fall through to vectors.json — the format of record (#187)
597
+ }
598
+ else if (e.message?.includes('integrity check failed'))
415
599
  throw e;
416
- if (!fs.existsSync(vectorsPath))
600
+ else if (!fs.existsSync(vectorsPath))
417
601
  throw e;
418
602
  // If vectors.bin was corrupt or unreadable, fall through to vectors.json fallback
419
603
  }
@@ -438,11 +622,18 @@ export function loadIndex(indexDir) {
438
622
  const expectedHash = shaContent.split(/\s+/)[0];
439
623
  const actualHash = crypto.createHash('sha256').update(rawBytes).digest('hex');
440
624
  if (expectedHash && actualHash !== expectedHash) {
441
- throw new Error(`vectors.json integrity check failed (SHA-256 mismatch) — run \`mdss index\` to rebuild.`);
625
+ // issue #187: atomicWrite renames the payload and its sidecar in two
626
+ // separate steps, so a crash mid-publish leaves a VALID payload with a
627
+ // STALE sidecar. The envelope above already parsed and validated —
628
+ // vectors.json is the format of record, so load it with a loud
629
+ // warning instead of bricking search/serve/mcp until a manual fix.
630
+ // A payload that cannot parse or validate still fails hard (above).
631
+ process.stderr.write('warning: vectors.json.sha256 does not match vectors.json (stale sidecar after a crash mid-publish?) — ' +
632
+ 'the payload itself is valid and will be loaded. Run `mdss check --fix` to refresh the sidecar.\n');
442
633
  }
443
634
  }
444
635
  catch (e) {
445
- if (e.message.includes('integrity check failed'))
636
+ if (e.message?.includes('integrity check failed'))
446
637
  throw e;
447
638
  }
448
639
  }
@@ -475,7 +666,8 @@ export function loadIndex(indexDir) {
475
666
  expectedDim ??= length;
476
667
  }
477
668
  }
478
- warnIfStale(index);
669
+ if (checkStale)
670
+ warnIfStale(index);
479
671
  const ivfPath = path.join(indexDir, 'ivf.json');
480
672
  let ivf;
481
673
  if (fs.existsSync(ivfPath)) {
@@ -485,7 +677,16 @@ export function loadIndex(indexDir) {
485
677
  }
486
678
  catch { }
487
679
  }
488
- runtimeIndexStates.set(index, snapshotRuntimeIndex(index, { schema: validated.schema, dim: expectedDim, model, ivf }));
680
+ const hnswPath2 = path.join(indexDir, 'hnsw.json');
681
+ let hnsw;
682
+ if (fs.existsSync(hnswPath2)) {
683
+ try {
684
+ const raw = JSON.parse(fs.readFileSync(hnswPath2, 'utf8'));
685
+ hnsw = deserializeHnsw(raw, index.chunks.length);
686
+ }
687
+ catch { }
688
+ }
689
+ runtimeIndexStates.set(index, snapshotRuntimeIndex(index, { schema: validated.schema, dim: expectedDim, model, ivf, hnsw }));
489
690
  return { index, model };
490
691
  }
491
692
  export const MAX_QUERY_LENGTH = 2048;
@@ -496,10 +697,15 @@ export async function searchIndex(opts) {
496
697
  throw new Error(`query exceeds maximum length of ${MAX_QUERY_LENGTH} characters`);
497
698
  }
498
699
  const embedFn = opts.embedFn ?? embed;
499
- const rerankFn = opts.rerankFn ?? rerankScores;
700
+ // issue #203: the registry alias rides the DEFAULT rerank path; an injected
701
+ // rerankFn (tests/serve) keeps its 4-arg signature and ignores it.
702
+ const rerankFn = opts.rerankFn ?? ((query, texts, dir, off) => rerankScores(query, texts, dir, off, opts.rerankModel));
500
703
  const { index } = loaded;
501
704
  const runtime = validateRuntimeIndex(index);
502
705
  const { chunks, db, lexicalState, expectedDim } = runtime;
706
+ if (opts.dimensions !== undefined && typeof opts.dimensions === 'number' && expectedDim !== undefined && opts.dimensions !== expectedDim) {
707
+ throw new Error(`requested search dimensions (${opts.dimensions}) does not match index dimension (${expectedDim}) — run \`mdss index --dimensions ${opts.dimensions}\` to rebuild`);
708
+ }
503
709
  const pathRes = pathFilter ? (Array.isArray(pathFilter) ? pathFilter : [pathFilter]).map(globToRegExp) : [];
504
710
  let sinceMs;
505
711
  if (since !== undefined) {
@@ -594,62 +800,162 @@ export async function searchIndex(opts) {
594
800
  }
595
801
  }
596
802
  }
597
- let qVec;
598
- if (opts.useQueryCache !== false && runtime.queryCache) {
599
- const normQuery = effectiveQuery.trim().toLowerCase();
600
- const cacheKey = `${runtime.model.id}:${runtime.model.revision || 'main'}:${expectedDim || 0}:${normQuery}`;
601
- // L2 disk layer (issue #114): persists across processes under cacheDir;
602
- // corrupt/missing files degrade silently to a normal embed.
603
- const diskKey = buildDiskKey(runtime.model.id, runtime.model.revision || 'main', expectedDim, normQuery);
604
- const useDisk = opts.queryDiskCache !== false;
605
- qVec = await runtime.queryCache.getOrCompute(cacheKey, async () => {
606
- if (useDisk) {
607
- const fromDisk = diskQueryGet(cacheDir, diskKey, expectedDim);
608
- if (fromDisk)
609
- return fromDisk;
610
- }
611
- const [v] = await embedFn([effectiveQuery], 'query', runtime.model, cacheDir, offline);
612
- if ((Array.isArray(v) || v instanceof Float32Array) && expectedDim !== undefined && v.length !== expectedDim) {
613
- throw new Error(`query vector has ${v.length} dims, expected ${expectedDim} - run \`mdss index\` to rebuild`);
614
- }
615
- validateNumericVector(v, undefined, 'query vector');
616
- if (useDisk)
617
- diskQueryPut(cacheDir, diskKey, v);
618
- return v;
619
- });
620
- }
621
- else {
622
- const [v] = await embedFn([effectiveQuery], 'query', runtime.model, cacheDir, offline);
803
+ // issue #203: one embed helper for the original query AND every multi-query
804
+ // paraphrase — both ride the same in-memory + disk query-embedding cache.
805
+ const embedFor = async (text) => {
806
+ if (opts.useQueryCache !== false && runtime.queryCache) {
807
+ const normQuery = text.trim().toLowerCase();
808
+ const cacheKey = `${runtime.model.id}:${runtime.model.revision || 'main'}:${expectedDim || 0}:${normQuery}`;
809
+ // L2 disk layer (issue #114): persists across processes under cacheDir;
810
+ // corrupt/missing files degrade silently to a normal embed.
811
+ const diskKey = buildDiskKey(runtime.model.id, runtime.model.revision || 'main', expectedDim, normQuery);
812
+ const useDisk = opts.queryDiskCache !== false;
813
+ return runtime.queryCache.getOrCompute(cacheKey, async () => {
814
+ if (useDisk) {
815
+ const fromDisk = diskQueryGet(cacheDir, diskKey, expectedDim);
816
+ if (fromDisk)
817
+ return fromDisk;
818
+ }
819
+ const [v] = await embedFn([text], 'query', runtime.model, cacheDir, offline);
820
+ if ((Array.isArray(v) || v instanceof Float32Array) && expectedDim !== undefined && v.length !== expectedDim) {
821
+ throw new Error(`query vector has ${v.length} dims, expected ${expectedDim} - run \`mdss index\` to rebuild`);
822
+ }
823
+ validateNumericVector(v, undefined, 'query vector');
824
+ if (useDisk)
825
+ diskQueryPut(cacheDir, diskKey, v);
826
+ return v;
827
+ });
828
+ }
829
+ const [v] = await embedFn([text], 'query', runtime.model, cacheDir, offline);
623
830
  if ((Array.isArray(v) || v instanceof Float32Array) && expectedDim !== undefined && v.length !== expectedDim) {
624
831
  throw new Error(`query vector has ${v.length} dims, expected ${expectedDim} — run \`mdss index\` to rebuild`);
625
832
  }
626
833
  validateNumericVector(v, undefined, 'query vector');
627
- qVec = v;
834
+ return v;
835
+ };
836
+ const qVec = await embedFor(effectiveQuery);
837
+ // issue #203: multi-query expansion — N total variants (original first),
838
+ // each embedded, dense rankings fused by union-RRF below. Paraphrases come
839
+ // from the LLM bridge; without an endpoint (or on any failure) the list is
840
+ // empty and the search is byte-identical to the single-query baseline.
841
+ const multiQueryN = Number.isFinite(opts.multiQuery) ? Math.floor(Number(opts.multiQuery)) : 0;
842
+ const paraphrases = multiQueryN >= 2 && opts.llmEndpoint
843
+ ? await paraphraseQueries(opts.llmEndpoint, query, multiQueryN - 1, opts.llmModel)
844
+ : [];
845
+ const qVecs = [qVec];
846
+ for (const p of paraphrases.slice(0, multiQueryN - 1)) {
847
+ try {
848
+ qVecs.push(await embedFor(p));
849
+ }
850
+ catch {
851
+ // a paraphrase that fails to embed (e.g. dim mismatch) is dropped, not fatal
852
+ }
628
853
  }
629
854
  const queryTerms = [...new Set(tokenize(query))];
855
+ const maxPerDoc = opts.maxPerFile || opts.maxPerDoc;
856
+ // Issue #127: time-decay recency boost — multiply the fused score by
857
+ // 0.5^(ageDays / halfLifeDays), post-RRF and pre-collapse. Age source
858
+ // priority: frontmatter `updated` → `created` → file mtime (issue #164). Missing dates
859
+ // get factor 1 (evergreen content stays reachable via lexical strength).
860
+ const recencyHalfLife = Number(opts.recency) > 0 ? Number(opts.recency) : 0;
861
+ // Issue #163: recency rescores AFTER the pool slice, so the pool must be
862
+ // expanded like the maxPerDoc path — otherwise candidates ranked k+1..n
863
+ // by raw RRF are dropped before the fresh-document boost can lift them.
864
+ // issue #191: the pool must be known BEFORE the sweep for the bounded top-k.
865
+ const pool = rerank
866
+ ? (rerankPool || (maxPerDoc && maxPerDoc > 0 ? Math.max(50, k * 5) : Math.max(20, k * 3)))
867
+ : (maxPerDoc && maxPerDoc > 0 || recencyHalfLife > 0 ? Math.max(50, k * 5) : k);
630
868
  let semantic;
631
- const useAnn = opts.ann ?? (runtime.ivf !== undefined && candidates.length >= ANN_THRESHOLD);
632
- if (useAnn && candidates.length >= ANN_THRESHOLD) {
633
- if (!runtime.ivf) {
634
- const rawVecs = chunks.map((c) => (c.vec instanceof Float32Array ? c.vec : dequantizeFromInt8(c.vec)));
635
- runtime.ivf = trainIVF(rawVecs);
636
- }
637
- const qVecF32 = qVec instanceof Float32Array ? qVec : Float32Array.from(qVec);
638
- const probedSet = new Set(searchIVFCandidates(qVecF32, runtime.ivf, opts.nprobe ?? DEFAULT_NPROBE));
639
- semantic = candidates.map(({ chunk, idx }) => {
640
- if (!probedSet.has(idx)) {
641
- return { idx, score: -1 };
869
+ let cosByIdx;
870
+ // issue #201: engine selection — explicit 'ivf'/'hnsw' wins; `true` picks
871
+ // whichever sidecar exists (hnsw preferred, then ivf); otherwise the
872
+ // automatic threshold path (sidecar present AND corpus above ANN_THRESHOLD)
873
+ const annOpt = opts.ann;
874
+ const autoAnn = annOpt === undefined || annOpt === false
875
+ ? Boolean(runtime.hnsw || runtime.ivf) && candidates.length >= ANN_THRESHOLD
876
+ : candidates.length >= ANN_THRESHOLD;
877
+ const useAnn = Boolean(annOpt) || autoAnn;
878
+ const annEngine = useAnn
879
+ ? (annOpt === 'hnsw' || (annOpt !== 'ivf' && runtime.hnsw) ? 'hnsw' : 'ivf')
880
+ : undefined;
881
+ // issue #190: with cached int8 norms the sweep computes the dot product alone
882
+ const norms = runtime.chunkNorms;
883
+ const sweepScore = (chunk, idx) => norms && chunk.vec instanceof Int8Array
884
+ ? asymmetricCosineInt8CachedNorm(qVec, chunk.vec, norms[idx])
885
+ : cosine(qVec, chunk.vec);
886
+ // issue #203: ANN probing unions the candidate sets of every query variant —
887
+ // a paraphrase must not recall into chunks the original's probe excluded
888
+ const probed = useAnn && candidates.length >= ANN_THRESHOLD
889
+ ? (() => {
890
+ const probedUnion = new Set();
891
+ for (const vec of qVecs) {
892
+ const qVecF32 = vec instanceof Float32Array ? vec : Float32Array.from(vec);
893
+ if (annEngine === 'hnsw') {
894
+ // issue #201: HNSW graph search — built on the hot path only when
895
+ // the sidecar is missing (same semantics as the IVF sync train)
896
+ if (!runtime.hnsw) {
897
+ const rawVecs = chunks.map((c) => (c.vec instanceof Float32Array ? c.vec : dequantizeFromInt8(c.vec)));
898
+ runtime.hnsw = buildHnsw(rawVecs, {
899
+ vectors: rawVecs,
900
+ distance: (a, b) => {
901
+ const bv = rawVecs[b];
902
+ return 1 - a.reduce((sum, x, i) => sum + x * bv[i], 0);
903
+ },
904
+ });
905
+ }
906
+ for (const idx of searchHnswCandidates(qVecF32, runtime.hnsw, chunks.map((c) => c.vec), { ef: opts.ef ?? DEFAULT_HNSW_EF })) {
907
+ probedUnion.add(idx);
908
+ }
909
+ }
910
+ else {
911
+ if (!runtime.ivf) {
912
+ const rawVecs = chunks.map((c) => (c.vec instanceof Float32Array ? c.vec : dequantizeFromInt8(c.vec)));
913
+ runtime.ivf = trainIVF(rawVecs);
914
+ }
915
+ for (const idx of searchIVFCandidates(qVecF32, runtime.ivf, opts.nprobe ?? DEFAULT_NPROBE)) {
916
+ probedUnion.add(idx);
917
+ }
918
+ }
642
919
  }
643
- return { idx, score: cosine(qVec, chunk.vec) };
644
- });
920
+ return probedUnion;
921
+ })()
922
+ : null;
923
+ if (qVecs.length > 1) {
924
+ // issue #203: multi-query — one dense sweep per variant, rankings fused by
925
+ // union-RRF. `cos` stays anchored to the ORIGINAL query's cosine so the
926
+ // hit shape (and score display) keeps its meaning.
927
+ const sweepWith = (vec) => (chunk, idx) => norms && chunk.vec instanceof Int8Array
928
+ ? asymmetricCosineInt8CachedNorm(vec, chunk.vec, norms[idx])
929
+ : cosine(vec, chunk.vec);
930
+ const perVectorRankings = qVecs.map((vec) => candidates.map(({ chunk, idx }) => ({
931
+ idx,
932
+ score: probed && !probed.has(idx) ? -1 : sweepWith(vec)(chunk, idx),
933
+ })));
934
+ const fusedSemantic = rrf(perVectorRankings, 60);
935
+ semantic = [...fusedSemantic.entries()].map(([idx, score]) => ({ idx, score }));
936
+ cosByIdx = new Map(perVectorRankings[0].map((s) => [s.idx, s.score]));
937
+ }
938
+ else if (semanticOnly && !(graphBoost > 0)) {
939
+ // issue #191: the pure-semantic ranking only needs the top pool — select
940
+ // it with a bounded min-heap in one pass (no per-chunk objects, no full
941
+ // sort). Equivalent to [...candidates].map(score).sort(desc).slice(0, pool)
942
+ // with the stable tie-break toward the smaller idx.
943
+ semantic = topKByScore(candidates.length, (i) => {
944
+ const { chunk, idx } = candidates[i];
945
+ if (probed && !probed.has(idx))
946
+ return -1;
947
+ return sweepScore(chunk, idx);
948
+ }, pool);
645
949
  }
646
950
  else {
647
951
  semantic = candidates.map(({ chunk, idx }) => {
648
- return { idx, score: cosine(qVec, chunk.vec) };
952
+ if (probed && !probed.has(idx)) {
953
+ return { idx, score: -1 };
954
+ }
955
+ return { idx, score: sweepScore(chunk, idx) };
649
956
  });
957
+ cosByIdx = new Map(semantic.map((s) => [s.idx, s.score]));
650
958
  }
651
- const cosByIdx = new Map(semantic.map((s) => [s.idx, s.score]));
652
- const pool = rerank ? rerankPool || Math.max(20, k * 3) : k;
653
959
  let graphRanking = [];
654
960
  let graphMap;
655
961
  if (graphBoost > 0) {
@@ -684,10 +990,10 @@ export async function searchIndex(opts) {
684
990
  .slice(0, pool);
685
991
  }
686
992
  else {
687
- ranked = [...semantic]
688
- .sort((a, b) => b.score - a.score)
689
- .slice(0, pool)
690
- .map((s) => ({ idx: s.idx, fscore: s.score, cos: s.score }));
993
+ // issue #191: `semantic` is already the top pool, ordered (score desc, idx asc);
994
+ // issue #203: with multi-query the pool scores are RRF — cos stays the
995
+ // original query's cosine via cosByIdx
996
+ ranked = semantic.map((s) => ({ idx: s.idx, fscore: s.score, cos: cosByIdx ? cosByIdx.get(s.idx) : s.score }));
691
997
  }
692
998
  }
693
999
  else {
@@ -716,11 +1022,6 @@ export async function searchIndex(opts) {
716
1022
  .slice(0, pool);
717
1023
  }
718
1024
  }
719
- // Issue #127: time-decay recency boost — multiply the fused score by
720
- // 0.5^(ageDays / halfLifeDays), post-RRF and pre-collapse. Age source
721
- // priority: frontmatter `created` → `updated` → file mtime. Missing dates
722
- // get factor 1 (evergreen content stays reachable via lexical strength).
723
- const recencyHalfLife = Number(opts.recency) > 0 ? Number(opts.recency) : 0;
724
1025
  if (recencyHalfLife > 0) {
725
1026
  const now = Date.now();
726
1027
  const ageCache = new Map();
@@ -743,8 +1044,10 @@ export async function searchIndex(opts) {
743
1044
  const scores = await rerankFn(query, texts, cacheDir, offline);
744
1045
  ranked = ranked
745
1046
  .map((r, i) => ({ ...r, rerank: scores[i] ?? -Infinity }))
746
- .sort((a, b) => b.rerank - a.rerank)
747
- .slice(0, k);
1047
+ .sort((a, b) => b.rerank - a.rerank);
1048
+ if (!maxPerDoc || maxPerDoc <= 0) {
1049
+ ranked = ranked.slice(0, k);
1050
+ }
748
1051
  }
749
1052
  const hasMetaFields = (m) => Boolean(m &&
750
1053
  (m.tags?.length > 0 ||
@@ -798,14 +1101,20 @@ export async function searchIndex(opts) {
798
1101
  ...(rerank ? { rerankScore: +(r.rerank ?? 0).toFixed(4) } : {}),
799
1102
  };
800
1103
  });
801
- const maxPerDoc = opts.maxPerFile || opts.maxPerDoc;
802
1104
  if (maxPerDoc && maxPerDoc > 0) {
803
1105
  return collapseResults(hits, (h) => h.meta?.canonicalRef || h.file, maxPerDoc).slice(0, k);
804
1106
  }
805
- return hits;
1107
+ // Issue #163: with the pool expanded for recency the final k-truncation
1108
+ // moves here (it used to be the pool slice itself). Paths that already
1109
+ // truncate (rerank, maxPerDoc collapse) keep their behavior.
1110
+ return hits.slice(0, k);
806
1111
  }
807
1112
  export async function search(opts) {
808
1113
  const { indexDir, ...rest } = opts;
809
- const loaded = loadIndex(indexDir);
810
- return searchIndex({ ...rest, loaded });
1114
+ // issue #192: the staleness check walks the entire tree synchronously —
1115
+ // defer it until AFTER the results are computed so it never delays them
1116
+ const loaded = loadIndex(indexDir, { checkStale: false });
1117
+ const hits = await searchIndex({ ...rest, loaded });
1118
+ warnIfStale(loaded.index);
1119
+ return hits;
811
1120
  }