md-semantic-search 1.0.3 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +132 -16
- package/RESEARCH.md +207 -3
- package/bin/cli.mjs +449 -13
- package/dist/binary-format.js +9 -0
- package/dist/completions.js +11 -1
- package/dist/core.d.ts +42 -4
- package/dist/core.js +208 -14
- package/dist/federation.d.ts +16 -1
- package/dist/federation.js +33 -16
- package/dist/filter.d.ts +7 -0
- package/dist/filter.js +13 -0
- package/dist/hnsw.d.ts +96 -0
- package/dist/hnsw.js +326 -0
- package/dist/index-format.js +8 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.js +4 -4
- package/dist/indexer.d.ts +18 -1
- package/dist/indexer.js +92 -18
- package/dist/ivf.d.ts +14 -7
- package/dist/ivf.js +126 -18
- package/dist/lexical.d.ts +10 -0
- package/dist/lexical.js +98 -15
- package/dist/markdown-parser.d.ts +62 -0
- package/dist/markdown-parser.js +227 -30
- package/dist/mcp.d.ts +229 -7
- package/dist/mcp.js +743 -44
- package/dist/models.d.ts +8 -2
- package/dist/models.js +84 -32
- package/dist/providers.d.ts +56 -0
- package/dist/providers.js +305 -0
- package/dist/quantization.d.ts +12 -0
- package/dist/quantization.js +54 -0
- package/dist/rag.d.ts +132 -7
- package/dist/rag.js +314 -38
- package/dist/repair.d.ts +1 -1
- package/dist/repair.js +53 -2
- package/dist/rerank.d.ts +51 -6
- package/dist/rerank.js +69 -10
- package/dist/search.d.ts +49 -3
- package/dist/search.js +387 -78
- package/dist/serve.d.ts +35 -0
- package/dist/serve.js +574 -58
- package/dist/tui.d.ts +14 -0
- package/dist/tui.js +132 -15
- package/dist/watcher.js +23 -12
- package/dist/webui.d.ts +3 -3
- package/dist/webui.js +225 -5
- package/dist/wikilinks.d.ts +31 -0
- package/dist/wikilinks.js +185 -1
- package/package.json +3 -3
- package/src/binary-format.ts +13 -0
- package/src/completions.ts +11 -1
- package/src/core.ts +249 -13
- package/src/federation.ts +51 -17
- package/src/filter.ts +14 -0
- package/src/hnsw.ts +414 -0
- package/src/index-format.ts +7 -1
- package/src/index.ts +7 -2
- package/src/indexer.ts +112 -19
- package/src/ivf.ts +157 -20
- package/src/lexical.ts +91 -13
- package/src/markdown-parser.ts +279 -37
- package/src/mcp.ts +820 -47
- package/src/models.ts +97 -35
- package/src/providers.ts +345 -0
- package/src/quantization.ts +61 -0
- package/src/rag.ts +451 -34
- package/src/repair.ts +51 -2
- package/src/rerank.ts +121 -14
- package/src/search.ts +424 -82
- package/src/serve.ts +589 -66
- package/src/tui.ts +142 -16
- package/src/watcher.ts +21 -10
- package/src/webui.ts +225 -5
- package/src/wikilinks.ts +220 -1
package/dist/search.js
CHANGED
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import crypto from 'node:crypto';
|
|
4
|
-
import { embed, cosine, decodeVec, globToRegExp, walkMarkdown } from './core.js';
|
|
4
|
+
import { embed, cosine, decodeVec, globToRegExp, walkMarkdown, fetchTimeoutMs } from './core.js';
|
|
5
5
|
import { validateIndexEnvelope, validateNumericVector } from './index-format.js';
|
|
6
6
|
import { rerankScores } from './rerank.js';
|
|
7
7
|
import { bm25Scores, matchingTerms, tokenize, fuzzyTitleAliasScores } from './lexical.js';
|
|
8
8
|
import { collapseResults } from './collapse.js';
|
|
9
9
|
import { trainIVF, searchIVFCandidates, deserializeIVF, ANN_THRESHOLD, DEFAULT_NPROBE } from './ivf.js';
|
|
10
|
+
import { buildHnsw, searchHnswCandidates, deserializeHnsw, DEFAULT_HNSW_EF } from './hnsw.js';
|
|
10
11
|
import { buildRelationshipGraph, computePageRank, expandGraphNeighborhood } from './wikilinks.js';
|
|
11
12
|
import { evaluateFilter } from './filter.js';
|
|
12
13
|
import { deserializeBinaryIndex } from './binary-format.js';
|
|
13
|
-
import { dequantizeFromInt8 } from './quantization.js';
|
|
14
|
+
import { dequantizeFromInt8, int8Norm, asymmetricCosineInt8CachedNorm } from './quantization.js';
|
|
14
15
|
import { buildDiskKey, diskQueryGet, diskQueryPut } from './query-cache-disk.js';
|
|
15
16
|
export { tokenize } from './lexical.js';
|
|
16
17
|
const chunkTokens = new WeakMap();
|
|
@@ -57,6 +58,76 @@ export function rrf(rankings, k = 60, weights) {
|
|
|
57
58
|
}
|
|
58
59
|
return fused;
|
|
59
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* issue #191: bounded top-k selection over a dense sweep — a capped min-heap
|
|
63
|
+
* instead of materializing one object per chunk and fully sorting. Ties break
|
|
64
|
+
* toward the smaller idx, which is exactly what a stable sort by
|
|
65
|
+
* (score desc) over idx-ordered input produces, so the selected pool and its
|
|
66
|
+
* order are identical to `[...items].sort((a, b) => b.score - a.score).slice(0, k)`.
|
|
67
|
+
*/
|
|
68
|
+
export function topKByScore(n, scoreAt, k) {
|
|
69
|
+
if (k <= 0 || n <= 0)
|
|
70
|
+
return [];
|
|
71
|
+
const cap = Math.min(k, n);
|
|
72
|
+
const scores = new Array(cap);
|
|
73
|
+
const idxs = new Array(cap);
|
|
74
|
+
let size = 0;
|
|
75
|
+
/** true when (aScore, aIdx) is a WORSE keeper than (bScore, bIdx). */
|
|
76
|
+
const worse = (aScore, aIdx, bScore, bIdx) => aScore < bScore || (aScore === bScore && aIdx > bIdx);
|
|
77
|
+
const siftDown = (pos) => {
|
|
78
|
+
for (;;) {
|
|
79
|
+
const left = pos * 2 + 1;
|
|
80
|
+
if (left >= size)
|
|
81
|
+
return;
|
|
82
|
+
const right = left + 1;
|
|
83
|
+
const child = right < size && worse(scores[right], idxs[right], scores[left], idxs[left]) ? right : left;
|
|
84
|
+
if (!worse(scores[child], idxs[child], scores[pos], idxs[pos]))
|
|
85
|
+
return;
|
|
86
|
+
const ts = scores[pos];
|
|
87
|
+
scores[pos] = scores[child];
|
|
88
|
+
scores[child] = ts;
|
|
89
|
+
const ti = idxs[pos];
|
|
90
|
+
idxs[pos] = idxs[child];
|
|
91
|
+
idxs[child] = ti;
|
|
92
|
+
pos = child;
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
for (let i = 0; i < n; i++) {
|
|
96
|
+
const s = scoreAt(i);
|
|
97
|
+
if (size < cap) {
|
|
98
|
+
let pos = size++;
|
|
99
|
+
scores[pos] = s;
|
|
100
|
+
idxs[pos] = i;
|
|
101
|
+
while (pos > 0) {
|
|
102
|
+
const parent = (pos - 1) >> 1;
|
|
103
|
+
if (!worse(scores[pos], idxs[pos], scores[parent], idxs[parent]))
|
|
104
|
+
break;
|
|
105
|
+
const ts = scores[pos];
|
|
106
|
+
scores[pos] = scores[parent];
|
|
107
|
+
scores[parent] = ts;
|
|
108
|
+
const ti = idxs[pos];
|
|
109
|
+
idxs[pos] = idxs[parent];
|
|
110
|
+
idxs[parent] = ti;
|
|
111
|
+
pos = parent;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
else if (worse(scores[0], idxs[0], s, i)) {
|
|
115
|
+
scores[0] = s;
|
|
116
|
+
idxs[0] = i;
|
|
117
|
+
siftDown(0);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
const out = [];
|
|
121
|
+
while (size > 0) {
|
|
122
|
+
out.push({ idx: idxs[0], score: scores[0] });
|
|
123
|
+
size--;
|
|
124
|
+
scores[0] = scores[size];
|
|
125
|
+
idxs[0] = idxs[size];
|
|
126
|
+
siftDown(0);
|
|
127
|
+
}
|
|
128
|
+
out.sort((a, b) => b.score - a.score || a.idx - b.idx);
|
|
129
|
+
return out;
|
|
130
|
+
}
|
|
60
131
|
export class QueryEmbeddingCache {
|
|
61
132
|
maxSize;
|
|
62
133
|
cache;
|
|
@@ -125,6 +196,10 @@ export class QueryEmbeddingCache {
|
|
|
125
196
|
return this.cache.size;
|
|
126
197
|
}
|
|
127
198
|
}
|
|
199
|
+
/** issue #190 tool/test seam: the cached int8 sweep norms, when present. */
|
|
200
|
+
export function getRuntimeChunkNorms(index) {
|
|
201
|
+
return runtimeIndexStates.get(index)?.chunkNorms;
|
|
202
|
+
}
|
|
128
203
|
export function getRuntimeGraph(runtime) {
|
|
129
204
|
if (runtime.graph && runtime.pageRank) {
|
|
130
205
|
return { graph: runtime.graph, pageRank: runtime.pageRank };
|
|
@@ -154,9 +229,11 @@ export function getRuntimeGraph(runtime) {
|
|
|
154
229
|
/**
|
|
155
230
|
* Issue #127 pure helpers — exported for deterministic unit tests.
|
|
156
231
|
*/
|
|
157
|
-
/** Frontmatter `
|
|
232
|
+
/** Frontmatter `updated` → `created` → file mtime; null when nothing parses.
|
|
233
|
+
* Issue #164: `updated` wins — long-lived notes that were recently revised
|
|
234
|
+
* must not be penalized by their original creation date. */
|
|
158
235
|
export function resolvePassageAgeMs(meta, absFile) {
|
|
159
|
-
for (const v of [meta?.
|
|
236
|
+
for (const v of [meta?.updated, meta?.created]) {
|
|
160
237
|
if (typeof v === 'string') {
|
|
161
238
|
const t = Date.parse(v);
|
|
162
239
|
if (Number.isFinite(t))
|
|
@@ -193,6 +270,9 @@ export async function hydePassage(endpoint, query, model, fetchImpl = globalThis
|
|
|
193
270
|
const res = await fetchImpl(endpoint.replace(/\/$/, '') + '/chat/completions', {
|
|
194
271
|
method: 'POST',
|
|
195
272
|
headers: { 'content-type': 'application/json' },
|
|
273
|
+
// issue #184: HyDE must degrade fast on a hung endpoint, not hold a
|
|
274
|
+
// serve slot — failure paths return null below
|
|
275
|
+
signal: AbortSignal.timeout(fetchTimeoutMs()),
|
|
196
276
|
body: JSON.stringify({
|
|
197
277
|
...(model ? { model } : {}),
|
|
198
278
|
messages: [
|
|
@@ -212,6 +292,66 @@ export async function hydePassage(endpoint, query, model, fetchImpl = globalThis
|
|
|
212
292
|
}
|
|
213
293
|
}
|
|
214
294
|
const runtimeIndexStates = new WeakMap();
|
|
295
|
+
/**
|
|
296
|
+
* issue #203: LLM paraphrases for multi-query expansion. ONE OpenAI-compatible
|
|
297
|
+
* chat completion asking for N alternative phrasings; returns clean, deduped,
|
|
298
|
+
* non-empty variants (the original query excluded). Returns [] on ANY failure —
|
|
299
|
+
* multi-query must degrade to the single-query baseline silently, exactly like
|
|
300
|
+
* HyDE. Injectable fetch for tests.
|
|
301
|
+
*/
|
|
302
|
+
export async function paraphraseQueries(endpoint, query, count, model, fetchImpl = globalThis.fetch) {
|
|
303
|
+
if (!(count > 0))
|
|
304
|
+
return [];
|
|
305
|
+
const capped = Math.min(Math.floor(count), 5);
|
|
306
|
+
try {
|
|
307
|
+
const res = await fetchImpl(endpoint.replace(/\/$/, '') + '/chat/completions', {
|
|
308
|
+
method: 'POST',
|
|
309
|
+
headers: { 'content-type': 'application/json' },
|
|
310
|
+
signal: AbortSignal.timeout(fetchTimeoutMs()),
|
|
311
|
+
body: JSON.stringify({
|
|
312
|
+
...(model ? { model } : {}),
|
|
313
|
+
messages: [
|
|
314
|
+
{
|
|
315
|
+
role: 'system',
|
|
316
|
+
content: `Write ${capped} alternative phrasings of the user's search query. ` +
|
|
317
|
+
'One per line, no numbering, no quotes, keep the same language and meaning. Output only the paraphrases.',
|
|
318
|
+
},
|
|
319
|
+
{ role: 'user', content: query },
|
|
320
|
+
],
|
|
321
|
+
}),
|
|
322
|
+
});
|
|
323
|
+
if (!res.ok)
|
|
324
|
+
return [];
|
|
325
|
+
const data = await res.json();
|
|
326
|
+
const text = data?.choices?.[0]?.message?.content;
|
|
327
|
+
if (typeof text !== 'string' || !text.trim())
|
|
328
|
+
return [];
|
|
329
|
+
const seen = new Set([query.trim().toLowerCase()]);
|
|
330
|
+
const out = [];
|
|
331
|
+
for (const raw of text.split('\n')) {
|
|
332
|
+
// strip list markers, surrounding quotes, and decorative dashes LLMs
|
|
333
|
+
// like to pad output with; a bare "— junk —" line then fails the
|
|
334
|
+
// length floor instead of passing as a paraphrase
|
|
335
|
+
const line = raw
|
|
336
|
+
.replace(/^\s*(?:[-*\d]+[.)]?\s+)/, '')
|
|
337
|
+
.replace(/^["'«“”\s—–-]+|["'»“”\s—–-]+$/g, '')
|
|
338
|
+
.trim();
|
|
339
|
+
if (line.length < 3 || line.length > MAX_QUERY_LENGTH)
|
|
340
|
+
continue;
|
|
341
|
+
const key = line.toLowerCase();
|
|
342
|
+
if (seen.has(key))
|
|
343
|
+
continue;
|
|
344
|
+
seen.add(key);
|
|
345
|
+
out.push(line);
|
|
346
|
+
if (out.length >= capped)
|
|
347
|
+
break;
|
|
348
|
+
}
|
|
349
|
+
return out;
|
|
350
|
+
}
|
|
351
|
+
catch {
|
|
352
|
+
return [];
|
|
353
|
+
}
|
|
354
|
+
}
|
|
215
355
|
/** Minimal EN+RU stopword set for PRF expansion (issue #125). */
|
|
216
356
|
const EXPANSION_STOPWORDS = new Set([
|
|
217
357
|
'the', 'a', 'an', 'and', 'or', 'of', 'to', 'in', 'on', 'for', 'with', 'is', 'are',
|
|
@@ -283,19 +423,30 @@ function snapshotRuntimeIndex(index, validation) {
|
|
|
283
423
|
});
|
|
284
424
|
let lexicalState = { kind: 'legacy-overlap' };
|
|
285
425
|
if (validation.schema >= 3 && index.lexical) {
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
426
|
+
// issue #192: the per-term [docId, tf] pairs are treated as immutable
|
|
427
|
+
// everywhere (bm25Scores/matchingTerms only read), so sharing them skips
|
|
428
|
+
// the O(terms × postings) deep copy on every load/reindex. The two mutable
|
|
429
|
+
// top levels are still copied so caller-side mutations of the loaded index
|
|
430
|
+
// cannot reach the runtime state.
|
|
290
431
|
lexicalState = {
|
|
291
432
|
kind: 'persisted-bm25',
|
|
292
433
|
lexical: {
|
|
293
434
|
format: index.lexical.format,
|
|
294
435
|
documentLengths: [...index.lexical.documentLengths],
|
|
295
|
-
postings,
|
|
436
|
+
postings: { ...index.lexical.postings },
|
|
296
437
|
},
|
|
297
438
|
};
|
|
298
439
|
}
|
|
440
|
+
// issue #190: int8 chunk norms are constant per chunk — compute them once at
|
|
441
|
+
// load instead of inside every query's dense sweep
|
|
442
|
+
let chunkNorms;
|
|
443
|
+
if (validation.rawVectorsBuffer instanceof Int8Array && dim > 0 && count > 0) {
|
|
444
|
+
chunkNorms = new Float32Array(count);
|
|
445
|
+
const buffer = validation.rawVectorsBuffer;
|
|
446
|
+
for (let i = 0; i < count; i++) {
|
|
447
|
+
chunkNorms[i] = int8Norm(buffer.subarray(i * dim, (i + 1) * dim));
|
|
448
|
+
}
|
|
449
|
+
}
|
|
299
450
|
return {
|
|
300
451
|
schema: validation.schema,
|
|
301
452
|
db: index.db,
|
|
@@ -305,6 +456,8 @@ function snapshotRuntimeIndex(index, validation) {
|
|
|
305
456
|
model: { ...validation.model },
|
|
306
457
|
queryCache: new QueryEmbeddingCache(100),
|
|
307
458
|
ivf: validation.ivf,
|
|
459
|
+
hnsw: validation.hnsw,
|
|
460
|
+
chunkNorms,
|
|
308
461
|
};
|
|
309
462
|
}
|
|
310
463
|
function validateRuntimeIndex(index) {
|
|
@@ -354,7 +507,13 @@ function warnIfStale(index) {
|
|
|
354
507
|
const mins = Math.max(1, Math.round((newest - builtMs) / 60000));
|
|
355
508
|
process.stderr.write(`warning: index is ${mins} min older than the newest change in ${index.db}; run \`mdss index\` to refresh.\n`);
|
|
356
509
|
}
|
|
357
|
-
|
|
510
|
+
/**
|
|
511
|
+
* Sentinel (issue #187): vectors.bin failed its sidecar check — not corrupt,
|
|
512
|
+
* just unverified — so loadIndex falls back to vectors.json instead of throwing.
|
|
513
|
+
*/
|
|
514
|
+
const BIN_SIDECAR_STALE = Symbol('bin-sidecar-stale');
|
|
515
|
+
export function loadIndex(indexDir, opts = {}) {
|
|
516
|
+
const { checkStale = true } = opts;
|
|
358
517
|
const vectorsBinPath = path.join(indexDir, 'vectors.bin');
|
|
359
518
|
const vectorsPath = path.join(indexDir, 'vectors.json');
|
|
360
519
|
let useBinary = false;
|
|
@@ -366,7 +525,9 @@ export function loadIndex(indexDir) {
|
|
|
366
525
|
try {
|
|
367
526
|
const binStat = fs.statSync(vectorsBinPath);
|
|
368
527
|
const jsonStat = fs.statSync(vectorsPath);
|
|
369
|
-
|
|
528
|
+
// issue #187: strict `>` — written in the same millisecond, vectors.json
|
|
529
|
+
// (the format of record) wins over a possibly half-published vectors.bin
|
|
530
|
+
if (binStat.mtimeMs > jsonStat.mtimeMs) {
|
|
370
531
|
useBinary = true;
|
|
371
532
|
}
|
|
372
533
|
}
|
|
@@ -383,7 +544,12 @@ export function loadIndex(indexDir) {
|
|
|
383
544
|
const expectedHash = fs.readFileSync(shaBinPath, 'utf8').trim().split(/\s+/)[0];
|
|
384
545
|
const actualHash = crypto.createHash('sha256').update(binBuffer).digest('hex');
|
|
385
546
|
if (expectedHash && actualHash !== expectedHash) {
|
|
386
|
-
|
|
547
|
+
// issue #187: vectors.bin is a derived cache of vectors.json — a
|
|
548
|
+
// stale sidecar (crash between the two renames) must not brick the
|
|
549
|
+
// daemon when the format of record is available. Fall through.
|
|
550
|
+
process.stderr.write('warning: vectors.bin.sha256 does not match vectors.bin — ignoring the binary cache and loading vectors.json. ' +
|
|
551
|
+
'Run `mdss check --fix` to rebuild the binary cache.\n');
|
|
552
|
+
throw BIN_SIDECAR_STALE;
|
|
387
553
|
}
|
|
388
554
|
}
|
|
389
555
|
const deserialized = deserializeBinaryIndex(binBuffer);
|
|
@@ -391,7 +557,10 @@ export function loadIndex(indexDir) {
|
|
|
391
557
|
const index = validated.index;
|
|
392
558
|
const { model, dim: validatedDim } = validated;
|
|
393
559
|
const expectedDim = validatedDim ?? deserialized.dim;
|
|
394
|
-
|
|
560
|
+
// issue #192: the staleness walk is a synchronous whole-tree traversal —
|
|
561
|
+
// when the caller defers it (search()), it runs AFTER results exist
|
|
562
|
+
if (checkStale)
|
|
563
|
+
warnIfStale(index);
|
|
395
564
|
const ivfPath = path.join(indexDir, 'ivf.json');
|
|
396
565
|
let ivf;
|
|
397
566
|
if (fs.existsSync(ivfPath)) {
|
|
@@ -401,19 +570,34 @@ export function loadIndex(indexDir) {
|
|
|
401
570
|
}
|
|
402
571
|
catch { }
|
|
403
572
|
}
|
|
573
|
+
// issue #201: optional HNSW sidecar — corrupt file degrades to brute
|
|
574
|
+
// force exactly like a corrupt ivf.json
|
|
575
|
+
const hnswPath = path.join(indexDir, 'hnsw.json');
|
|
576
|
+
let hnsw;
|
|
577
|
+
if (fs.existsSync(hnswPath)) {
|
|
578
|
+
try {
|
|
579
|
+
const raw = JSON.parse(fs.readFileSync(hnswPath, 'utf8'));
|
|
580
|
+
hnsw = deserializeHnsw(raw, index.chunks.length);
|
|
581
|
+
}
|
|
582
|
+
catch { }
|
|
583
|
+
}
|
|
404
584
|
runtimeIndexStates.set(index, snapshotRuntimeIndex(index, {
|
|
405
585
|
schema: validated.schema,
|
|
406
586
|
dim: expectedDim,
|
|
407
587
|
model,
|
|
408
588
|
ivf,
|
|
589
|
+
hnsw,
|
|
409
590
|
rawVectorsBuffer: deserialized.rawVectorsBuffer,
|
|
410
591
|
}));
|
|
411
592
|
return { index, model };
|
|
412
593
|
}
|
|
413
594
|
catch (e) {
|
|
414
|
-
if (e
|
|
595
|
+
if (e === BIN_SIDECAR_STALE) {
|
|
596
|
+
// fall through to vectors.json — the format of record (#187)
|
|
597
|
+
}
|
|
598
|
+
else if (e.message?.includes('integrity check failed'))
|
|
415
599
|
throw e;
|
|
416
|
-
if (!fs.existsSync(vectorsPath))
|
|
600
|
+
else if (!fs.existsSync(vectorsPath))
|
|
417
601
|
throw e;
|
|
418
602
|
// If vectors.bin was corrupt or unreadable, fall through to vectors.json fallback
|
|
419
603
|
}
|
|
@@ -438,11 +622,18 @@ export function loadIndex(indexDir) {
|
|
|
438
622
|
const expectedHash = shaContent.split(/\s+/)[0];
|
|
439
623
|
const actualHash = crypto.createHash('sha256').update(rawBytes).digest('hex');
|
|
440
624
|
if (expectedHash && actualHash !== expectedHash) {
|
|
441
|
-
|
|
625
|
+
// issue #187: atomicWrite renames the payload and its sidecar in two
|
|
626
|
+
// separate steps, so a crash mid-publish leaves a VALID payload with a
|
|
627
|
+
// STALE sidecar. The envelope above already parsed and validated —
|
|
628
|
+
// vectors.json is the format of record, so load it with a loud
|
|
629
|
+
// warning instead of bricking search/serve/mcp until a manual fix.
|
|
630
|
+
// A payload that cannot parse or validate still fails hard (above).
|
|
631
|
+
process.stderr.write('warning: vectors.json.sha256 does not match vectors.json (stale sidecar after a crash mid-publish?) — ' +
|
|
632
|
+
'the payload itself is valid and will be loaded. Run `mdss check --fix` to refresh the sidecar.\n');
|
|
442
633
|
}
|
|
443
634
|
}
|
|
444
635
|
catch (e) {
|
|
445
|
-
if (e.message
|
|
636
|
+
if (e.message?.includes('integrity check failed'))
|
|
446
637
|
throw e;
|
|
447
638
|
}
|
|
448
639
|
}
|
|
@@ -475,7 +666,8 @@ export function loadIndex(indexDir) {
|
|
|
475
666
|
expectedDim ??= length;
|
|
476
667
|
}
|
|
477
668
|
}
|
|
478
|
-
|
|
669
|
+
if (checkStale)
|
|
670
|
+
warnIfStale(index);
|
|
479
671
|
const ivfPath = path.join(indexDir, 'ivf.json');
|
|
480
672
|
let ivf;
|
|
481
673
|
if (fs.existsSync(ivfPath)) {
|
|
@@ -485,7 +677,16 @@ export function loadIndex(indexDir) {
|
|
|
485
677
|
}
|
|
486
678
|
catch { }
|
|
487
679
|
}
|
|
488
|
-
|
|
680
|
+
const hnswPath2 = path.join(indexDir, 'hnsw.json');
|
|
681
|
+
let hnsw;
|
|
682
|
+
if (fs.existsSync(hnswPath2)) {
|
|
683
|
+
try {
|
|
684
|
+
const raw = JSON.parse(fs.readFileSync(hnswPath2, 'utf8'));
|
|
685
|
+
hnsw = deserializeHnsw(raw, index.chunks.length);
|
|
686
|
+
}
|
|
687
|
+
catch { }
|
|
688
|
+
}
|
|
689
|
+
runtimeIndexStates.set(index, snapshotRuntimeIndex(index, { schema: validated.schema, dim: expectedDim, model, ivf, hnsw }));
|
|
489
690
|
return { index, model };
|
|
490
691
|
}
|
|
491
692
|
export const MAX_QUERY_LENGTH = 2048;
|
|
@@ -496,10 +697,15 @@ export async function searchIndex(opts) {
|
|
|
496
697
|
throw new Error(`query exceeds maximum length of ${MAX_QUERY_LENGTH} characters`);
|
|
497
698
|
}
|
|
498
699
|
const embedFn = opts.embedFn ?? embed;
|
|
499
|
-
|
|
700
|
+
// issue #203: the registry alias rides the DEFAULT rerank path; an injected
|
|
701
|
+
// rerankFn (tests/serve) keeps its 4-arg signature and ignores it.
|
|
702
|
+
const rerankFn = opts.rerankFn ?? ((query, texts, dir, off) => rerankScores(query, texts, dir, off, opts.rerankModel));
|
|
500
703
|
const { index } = loaded;
|
|
501
704
|
const runtime = validateRuntimeIndex(index);
|
|
502
705
|
const { chunks, db, lexicalState, expectedDim } = runtime;
|
|
706
|
+
if (opts.dimensions !== undefined && typeof opts.dimensions === 'number' && expectedDim !== undefined && opts.dimensions !== expectedDim) {
|
|
707
|
+
throw new Error(`requested search dimensions (${opts.dimensions}) does not match index dimension (${expectedDim}) — run \`mdss index --dimensions ${opts.dimensions}\` to rebuild`);
|
|
708
|
+
}
|
|
503
709
|
const pathRes = pathFilter ? (Array.isArray(pathFilter) ? pathFilter : [pathFilter]).map(globToRegExp) : [];
|
|
504
710
|
let sinceMs;
|
|
505
711
|
if (since !== undefined) {
|
|
@@ -594,62 +800,162 @@ export async function searchIndex(opts) {
|
|
|
594
800
|
}
|
|
595
801
|
}
|
|
596
802
|
}
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
if (
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
803
|
+
// issue #203: one embed helper for the original query AND every multi-query
|
|
804
|
+
// paraphrase — both ride the same in-memory + disk query-embedding cache.
|
|
805
|
+
const embedFor = async (text) => {
|
|
806
|
+
if (opts.useQueryCache !== false && runtime.queryCache) {
|
|
807
|
+
const normQuery = text.trim().toLowerCase();
|
|
808
|
+
const cacheKey = `${runtime.model.id}:${runtime.model.revision || 'main'}:${expectedDim || 0}:${normQuery}`;
|
|
809
|
+
// L2 disk layer (issue #114): persists across processes under cacheDir;
|
|
810
|
+
// corrupt/missing files degrade silently to a normal embed.
|
|
811
|
+
const diskKey = buildDiskKey(runtime.model.id, runtime.model.revision || 'main', expectedDim, normQuery);
|
|
812
|
+
const useDisk = opts.queryDiskCache !== false;
|
|
813
|
+
return runtime.queryCache.getOrCompute(cacheKey, async () => {
|
|
814
|
+
if (useDisk) {
|
|
815
|
+
const fromDisk = diskQueryGet(cacheDir, diskKey, expectedDim);
|
|
816
|
+
if (fromDisk)
|
|
817
|
+
return fromDisk;
|
|
818
|
+
}
|
|
819
|
+
const [v] = await embedFn([text], 'query', runtime.model, cacheDir, offline);
|
|
820
|
+
if ((Array.isArray(v) || v instanceof Float32Array) && expectedDim !== undefined && v.length !== expectedDim) {
|
|
821
|
+
throw new Error(`query vector has ${v.length} dims, expected ${expectedDim} - run \`mdss index\` to rebuild`);
|
|
822
|
+
}
|
|
823
|
+
validateNumericVector(v, undefined, 'query vector');
|
|
824
|
+
if (useDisk)
|
|
825
|
+
diskQueryPut(cacheDir, diskKey, v);
|
|
826
|
+
return v;
|
|
827
|
+
});
|
|
828
|
+
}
|
|
829
|
+
const [v] = await embedFn([text], 'query', runtime.model, cacheDir, offline);
|
|
623
830
|
if ((Array.isArray(v) || v instanceof Float32Array) && expectedDim !== undefined && v.length !== expectedDim) {
|
|
624
831
|
throw new Error(`query vector has ${v.length} dims, expected ${expectedDim} — run \`mdss index\` to rebuild`);
|
|
625
832
|
}
|
|
626
833
|
validateNumericVector(v, undefined, 'query vector');
|
|
627
|
-
|
|
834
|
+
return v;
|
|
835
|
+
};
|
|
836
|
+
const qVec = await embedFor(effectiveQuery);
|
|
837
|
+
// issue #203: multi-query expansion — N total variants (original first),
|
|
838
|
+
// each embedded, dense rankings fused by union-RRF below. Paraphrases come
|
|
839
|
+
// from the LLM bridge; without an endpoint (or on any failure) the list is
|
|
840
|
+
// empty and the search is byte-identical to the single-query baseline.
|
|
841
|
+
const multiQueryN = Number.isFinite(opts.multiQuery) ? Math.floor(Number(opts.multiQuery)) : 0;
|
|
842
|
+
const paraphrases = multiQueryN >= 2 && opts.llmEndpoint
|
|
843
|
+
? await paraphraseQueries(opts.llmEndpoint, query, multiQueryN - 1, opts.llmModel)
|
|
844
|
+
: [];
|
|
845
|
+
const qVecs = [qVec];
|
|
846
|
+
for (const p of paraphrases.slice(0, multiQueryN - 1)) {
|
|
847
|
+
try {
|
|
848
|
+
qVecs.push(await embedFor(p));
|
|
849
|
+
}
|
|
850
|
+
catch {
|
|
851
|
+
// a paraphrase that fails to embed (e.g. dim mismatch) is dropped, not fatal
|
|
852
|
+
}
|
|
628
853
|
}
|
|
629
854
|
const queryTerms = [...new Set(tokenize(query))];
|
|
855
|
+
const maxPerDoc = opts.maxPerFile || opts.maxPerDoc;
|
|
856
|
+
// Issue #127: time-decay recency boost — multiply the fused score by
|
|
857
|
+
// 0.5^(ageDays / halfLifeDays), post-RRF and pre-collapse. Age source
|
|
858
|
+
// priority: frontmatter `updated` → `created` → file mtime (issue #164). Missing dates
|
|
859
|
+
// get factor 1 (evergreen content stays reachable via lexical strength).
|
|
860
|
+
const recencyHalfLife = Number(opts.recency) > 0 ? Number(opts.recency) : 0;
|
|
861
|
+
// Issue #163: recency rescores AFTER the pool slice, so the pool must be
|
|
862
|
+
// expanded like the maxPerDoc path — otherwise candidates ranked k+1..n
|
|
863
|
+
// by raw RRF are dropped before the fresh-document boost can lift them.
|
|
864
|
+
// issue #191: the pool must be known BEFORE the sweep for the bounded top-k.
|
|
865
|
+
const pool = rerank
|
|
866
|
+
? (rerankPool || (maxPerDoc && maxPerDoc > 0 ? Math.max(50, k * 5) : Math.max(20, k * 3)))
|
|
867
|
+
: (maxPerDoc && maxPerDoc > 0 || recencyHalfLife > 0 ? Math.max(50, k * 5) : k);
|
|
630
868
|
let semantic;
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
869
|
+
let cosByIdx;
|
|
870
|
+
// issue #201: engine selection — explicit 'ivf'/'hnsw' wins; `true` picks
|
|
871
|
+
// whichever sidecar exists (hnsw preferred, then ivf); otherwise the
|
|
872
|
+
// automatic threshold path (sidecar present AND corpus above ANN_THRESHOLD)
|
|
873
|
+
const annOpt = opts.ann;
|
|
874
|
+
const autoAnn = annOpt === undefined || annOpt === false
|
|
875
|
+
? Boolean(runtime.hnsw || runtime.ivf) && candidates.length >= ANN_THRESHOLD
|
|
876
|
+
: candidates.length >= ANN_THRESHOLD;
|
|
877
|
+
const useAnn = Boolean(annOpt) || autoAnn;
|
|
878
|
+
const annEngine = useAnn
|
|
879
|
+
? (annOpt === 'hnsw' || (annOpt !== 'ivf' && runtime.hnsw) ? 'hnsw' : 'ivf')
|
|
880
|
+
: undefined;
|
|
881
|
+
// issue #190: with cached int8 norms the sweep computes the dot product alone
|
|
882
|
+
const norms = runtime.chunkNorms;
|
|
883
|
+
const sweepScore = (chunk, idx) => norms && chunk.vec instanceof Int8Array
|
|
884
|
+
? asymmetricCosineInt8CachedNorm(qVec, chunk.vec, norms[idx])
|
|
885
|
+
: cosine(qVec, chunk.vec);
|
|
886
|
+
// issue #203: ANN probing unions the candidate sets of every query variant —
|
|
887
|
+
// a paraphrase must not recall into chunks the original's probe excluded
|
|
888
|
+
const probed = useAnn && candidates.length >= ANN_THRESHOLD
|
|
889
|
+
? (() => {
|
|
890
|
+
const probedUnion = new Set();
|
|
891
|
+
for (const vec of qVecs) {
|
|
892
|
+
const qVecF32 = vec instanceof Float32Array ? vec : Float32Array.from(vec);
|
|
893
|
+
if (annEngine === 'hnsw') {
|
|
894
|
+
// issue #201: HNSW graph search — built on the hot path only when
|
|
895
|
+
// the sidecar is missing (same semantics as the IVF sync train)
|
|
896
|
+
if (!runtime.hnsw) {
|
|
897
|
+
const rawVecs = chunks.map((c) => (c.vec instanceof Float32Array ? c.vec : dequantizeFromInt8(c.vec)));
|
|
898
|
+
runtime.hnsw = buildHnsw(rawVecs, {
|
|
899
|
+
vectors: rawVecs,
|
|
900
|
+
distance: (a, b) => {
|
|
901
|
+
const bv = rawVecs[b];
|
|
902
|
+
return 1 - a.reduce((sum, x, i) => sum + x * bv[i], 0);
|
|
903
|
+
},
|
|
904
|
+
});
|
|
905
|
+
}
|
|
906
|
+
for (const idx of searchHnswCandidates(qVecF32, runtime.hnsw, chunks.map((c) => c.vec), { ef: opts.ef ?? DEFAULT_HNSW_EF })) {
|
|
907
|
+
probedUnion.add(idx);
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
else {
|
|
911
|
+
if (!runtime.ivf) {
|
|
912
|
+
const rawVecs = chunks.map((c) => (c.vec instanceof Float32Array ? c.vec : dequantizeFromInt8(c.vec)));
|
|
913
|
+
runtime.ivf = trainIVF(rawVecs);
|
|
914
|
+
}
|
|
915
|
+
for (const idx of searchIVFCandidates(qVecF32, runtime.ivf, opts.nprobe ?? DEFAULT_NPROBE)) {
|
|
916
|
+
probedUnion.add(idx);
|
|
917
|
+
}
|
|
918
|
+
}
|
|
642
919
|
}
|
|
643
|
-
return
|
|
644
|
-
})
|
|
920
|
+
return probedUnion;
|
|
921
|
+
})()
|
|
922
|
+
: null;
|
|
923
|
+
if (qVecs.length > 1) {
|
|
924
|
+
// issue #203: multi-query — one dense sweep per variant, rankings fused by
|
|
925
|
+
// union-RRF. `cos` stays anchored to the ORIGINAL query's cosine so the
|
|
926
|
+
// hit shape (and score display) keeps its meaning.
|
|
927
|
+
const sweepWith = (vec) => (chunk, idx) => norms && chunk.vec instanceof Int8Array
|
|
928
|
+
? asymmetricCosineInt8CachedNorm(vec, chunk.vec, norms[idx])
|
|
929
|
+
: cosine(vec, chunk.vec);
|
|
930
|
+
const perVectorRankings = qVecs.map((vec) => candidates.map(({ chunk, idx }) => ({
|
|
931
|
+
idx,
|
|
932
|
+
score: probed && !probed.has(idx) ? -1 : sweepWith(vec)(chunk, idx),
|
|
933
|
+
})));
|
|
934
|
+
const fusedSemantic = rrf(perVectorRankings, 60);
|
|
935
|
+
semantic = [...fusedSemantic.entries()].map(([idx, score]) => ({ idx, score }));
|
|
936
|
+
cosByIdx = new Map(perVectorRankings[0].map((s) => [s.idx, s.score]));
|
|
937
|
+
}
|
|
938
|
+
else if (semanticOnly && !(graphBoost > 0)) {
|
|
939
|
+
// issue #191: the pure-semantic ranking only needs the top pool — select
|
|
940
|
+
// it with a bounded min-heap in one pass (no per-chunk objects, no full
|
|
941
|
+
// sort). Equivalent to [...candidates].map(score).sort(desc).slice(0, pool)
|
|
942
|
+
// with the stable tie-break toward the smaller idx.
|
|
943
|
+
semantic = topKByScore(candidates.length, (i) => {
|
|
944
|
+
const { chunk, idx } = candidates[i];
|
|
945
|
+
if (probed && !probed.has(idx))
|
|
946
|
+
return -1;
|
|
947
|
+
return sweepScore(chunk, idx);
|
|
948
|
+
}, pool);
|
|
645
949
|
}
|
|
646
950
|
else {
|
|
647
951
|
semantic = candidates.map(({ chunk, idx }) => {
|
|
648
|
-
|
|
952
|
+
if (probed && !probed.has(idx)) {
|
|
953
|
+
return { idx, score: -1 };
|
|
954
|
+
}
|
|
955
|
+
return { idx, score: sweepScore(chunk, idx) };
|
|
649
956
|
});
|
|
957
|
+
cosByIdx = new Map(semantic.map((s) => [s.idx, s.score]));
|
|
650
958
|
}
|
|
651
|
-
const cosByIdx = new Map(semantic.map((s) => [s.idx, s.score]));
|
|
652
|
-
const pool = rerank ? rerankPool || Math.max(20, k * 3) : k;
|
|
653
959
|
let graphRanking = [];
|
|
654
960
|
let graphMap;
|
|
655
961
|
if (graphBoost > 0) {
|
|
@@ -684,10 +990,10 @@ export async function searchIndex(opts) {
|
|
|
684
990
|
.slice(0, pool);
|
|
685
991
|
}
|
|
686
992
|
else {
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
993
|
+
// issue #191: `semantic` is already the top pool, ordered (score desc, idx asc);
|
|
994
|
+
// issue #203: with multi-query the pool scores are RRF — cos stays the
|
|
995
|
+
// original query's cosine via cosByIdx
|
|
996
|
+
ranked = semantic.map((s) => ({ idx: s.idx, fscore: s.score, cos: cosByIdx ? cosByIdx.get(s.idx) : s.score }));
|
|
691
997
|
}
|
|
692
998
|
}
|
|
693
999
|
else {
|
|
@@ -716,11 +1022,6 @@ export async function searchIndex(opts) {
|
|
|
716
1022
|
.slice(0, pool);
|
|
717
1023
|
}
|
|
718
1024
|
}
|
|
719
|
-
// Issue #127: time-decay recency boost — multiply the fused score by
|
|
720
|
-
// 0.5^(ageDays / halfLifeDays), post-RRF and pre-collapse. Age source
|
|
721
|
-
// priority: frontmatter `created` → `updated` → file mtime. Missing dates
|
|
722
|
-
// get factor 1 (evergreen content stays reachable via lexical strength).
|
|
723
|
-
const recencyHalfLife = Number(opts.recency) > 0 ? Number(opts.recency) : 0;
|
|
724
1025
|
if (recencyHalfLife > 0) {
|
|
725
1026
|
const now = Date.now();
|
|
726
1027
|
const ageCache = new Map();
|
|
@@ -743,8 +1044,10 @@ export async function searchIndex(opts) {
|
|
|
743
1044
|
const scores = await rerankFn(query, texts, cacheDir, offline);
|
|
744
1045
|
ranked = ranked
|
|
745
1046
|
.map((r, i) => ({ ...r, rerank: scores[i] ?? -Infinity }))
|
|
746
|
-
.sort((a, b) => b.rerank - a.rerank)
|
|
747
|
-
|
|
1047
|
+
.sort((a, b) => b.rerank - a.rerank);
|
|
1048
|
+
if (!maxPerDoc || maxPerDoc <= 0) {
|
|
1049
|
+
ranked = ranked.slice(0, k);
|
|
1050
|
+
}
|
|
748
1051
|
}
|
|
749
1052
|
const hasMetaFields = (m) => Boolean(m &&
|
|
750
1053
|
(m.tags?.length > 0 ||
|
|
@@ -798,14 +1101,20 @@ export async function searchIndex(opts) {
|
|
|
798
1101
|
...(rerank ? { rerankScore: +(r.rerank ?? 0).toFixed(4) } : {}),
|
|
799
1102
|
};
|
|
800
1103
|
});
|
|
801
|
-
const maxPerDoc = opts.maxPerFile || opts.maxPerDoc;
|
|
802
1104
|
if (maxPerDoc && maxPerDoc > 0) {
|
|
803
1105
|
return collapseResults(hits, (h) => h.meta?.canonicalRef || h.file, maxPerDoc).slice(0, k);
|
|
804
1106
|
}
|
|
805
|
-
|
|
1107
|
+
// Issue #163: with the pool expanded for recency the final k-truncation
|
|
1108
|
+
// moves here (it used to be the pool slice itself). Paths that already
|
|
1109
|
+
// truncate (rerank, maxPerDoc collapse) keep their behavior.
|
|
1110
|
+
return hits.slice(0, k);
|
|
806
1111
|
}
|
|
807
1112
|
export async function search(opts) {
|
|
808
1113
|
const { indexDir, ...rest } = opts;
|
|
809
|
-
|
|
810
|
-
|
|
1114
|
+
// issue #192: the staleness check walks the entire tree synchronously —
|
|
1115
|
+
// defer it until AFTER the results are computed so it never delays them
|
|
1116
|
+
const loaded = loadIndex(indexDir, { checkStale: false });
|
|
1117
|
+
const hits = await searchIndex({ ...rest, loaded });
|
|
1118
|
+
warnIfStale(loaded.index);
|
|
1119
|
+
return hits;
|
|
811
1120
|
}
|