md-semantic-search 1.0.3 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -3
- package/RESEARCH.md +25 -3
- package/bin/cli.mjs +255 -8
- package/dist/completions.js +6 -1
- package/dist/core.d.ts +26 -0
- package/dist/core.js +104 -1
- package/dist/index-format.js +8 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.js +4 -4
- package/dist/indexer.d.ts +1 -0
- package/dist/indexer.js +5 -5
- package/dist/ivf.d.ts +14 -7
- package/dist/ivf.js +126 -18
- package/dist/mcp.d.ts +143 -4
- package/dist/mcp.js +493 -22
- package/dist/models.d.ts +1 -1
- package/dist/models.js +52 -31
- package/dist/rag.d.ts +72 -7
- package/dist/rag.js +191 -13
- package/dist/rerank.d.ts +20 -4
- package/dist/rerank.js +11 -2
- package/dist/search.d.ts +1 -0
- package/dist/search.js +3 -0
- package/dist/serve.js +97 -1
- package/dist/tui.d.ts +8 -0
- package/dist/tui.js +80 -13
- package/dist/webui.d.ts +1 -1
- package/dist/webui.js +24 -3
- package/dist/wikilinks.d.ts +31 -0
- package/dist/wikilinks.js +185 -1
- package/package.json +1 -1
- package/src/completions.ts +6 -1
- package/src/core.ts +134 -1
- package/src/index-format.ts +7 -1
- package/src/index.ts +7 -2
- package/src/indexer.ts +8 -4
- package/src/ivf.ts +157 -20
- package/src/mcp.ts +507 -23
- package/src/models.ts +58 -33
- package/src/rag.ts +279 -17
- package/src/rerank.ts +40 -6
- package/src/search.ts +7 -0
- package/src/serve.ts +91 -1
- package/src/tui.ts +86 -14
- package/src/webui.ts +24 -3
- package/src/wikilinks.ts +220 -1
package/dist/core.js
CHANGED
|
@@ -258,11 +258,36 @@ export function prepareEmbeddingRequest(model, texts, kind) {
|
|
|
258
258
|
options: { pooling: model.pooling ?? 'mean', normalize: model.normalize !== false },
|
|
259
259
|
};
|
|
260
260
|
}
|
|
261
|
+
export function truncateAndNormalizeVector(vec, targetDim, normalize = true) {
|
|
262
|
+
const len = Math.min(vec.length, targetDim);
|
|
263
|
+
const out = new Array(len);
|
|
264
|
+
let norm = 0;
|
|
265
|
+
for (let i = 0; i < len; i++) {
|
|
266
|
+
const val = vec[i];
|
|
267
|
+
out[i] = val;
|
|
268
|
+
norm += val * val;
|
|
269
|
+
}
|
|
270
|
+
if (normalize) {
|
|
271
|
+
norm = Math.sqrt(norm);
|
|
272
|
+
if (norm > 0) {
|
|
273
|
+
for (let i = 0; i < len; i++) {
|
|
274
|
+
out[i] /= norm;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
return out;
|
|
279
|
+
}
|
|
261
280
|
export async function embed(texts, kind, model, cacheDir, offline = false, retryOpts = {}) {
|
|
262
281
|
const ext = await getExtractor(model, cacheDir, offline, retryOpts);
|
|
263
282
|
const { input, options } = prepareEmbeddingRequest(model, texts, kind);
|
|
264
283
|
const out = await ext(input, options);
|
|
265
|
-
|
|
284
|
+
const list = out.tolist();
|
|
285
|
+
const targetDim = model.dim;
|
|
286
|
+
if (targetDim && list.length > 0 && list[0].length > targetDim) {
|
|
287
|
+
const doNorm = model.normalize !== false;
|
|
288
|
+
return list.map((vec) => truncateAndNormalizeVector(vec, targetDim, doNorm));
|
|
289
|
+
}
|
|
290
|
+
return list;
|
|
266
291
|
}
|
|
267
292
|
export function cosine(a, b, bOffset = 0) {
|
|
268
293
|
if (b instanceof Int8Array) {
|
|
@@ -489,3 +514,81 @@ export function parseFile(absPath, dbDir, maxChunk, raw) {
|
|
|
489
514
|
};
|
|
490
515
|
});
|
|
491
516
|
}
|
|
517
|
+
/**
|
|
518
|
+
* Line-range retrieval helper (issue #140).
|
|
519
|
+
* Retrieves an exact line span from disk or fallback chunk data with EOF clamping.
|
|
520
|
+
*/
|
|
521
|
+
export function getDocLines(opts) {
|
|
522
|
+
const { file, fromLine = 1, maxLines, dbDir, chunks } = opts;
|
|
523
|
+
if (!file || typeof file !== 'string') {
|
|
524
|
+
throw new Error('getDocLines: "file" parameter is required');
|
|
525
|
+
}
|
|
526
|
+
const normalizedFile = file.split(/[/\\]/).join('/');
|
|
527
|
+
let rawContent = null;
|
|
528
|
+
if (dbDir) {
|
|
529
|
+
const candidatePath = path.resolve(dbDir, file);
|
|
530
|
+
try {
|
|
531
|
+
assertSafePath(candidatePath, [dbDir]);
|
|
532
|
+
if (fs.existsSync(candidatePath) && fs.statSync(candidatePath).isFile()) {
|
|
533
|
+
rawContent = fs.readFileSync(candidatePath, 'utf8');
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
catch (e) {
|
|
537
|
+
if (e.message?.includes('path traversal guard'))
|
|
538
|
+
throw e;
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
if (rawContent === null && Array.isArray(chunks)) {
|
|
542
|
+
const matchingChunks = chunks.filter((c) => c.file === file || c.file === normalizedFile || c.file.split(/[/\\]/).join('/') === normalizedFile);
|
|
543
|
+
if (matchingChunks.length > 0) {
|
|
544
|
+
matchingChunks.sort((a, b) => (a.startLine || 0) - (b.startLine || 0));
|
|
545
|
+
const maxEnd = Math.max(...matchingChunks.map((c) => c.endLine || c.startLine || 1));
|
|
546
|
+
const lineMap = new Array(maxEnd).fill('');
|
|
547
|
+
for (const ch of matchingChunks) {
|
|
548
|
+
const start = (ch.startLine || 1) - 1;
|
|
549
|
+
const chunkLines = (ch.text || '').split(/\r?\n/);
|
|
550
|
+
for (let i = 0; i < chunkLines.length; i++) {
|
|
551
|
+
if (start + i < lineMap.length) {
|
|
552
|
+
lineMap[start + i] = chunkLines[i];
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
rawContent = lineMap.join('\n');
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
if (rawContent === null) {
|
|
560
|
+
throw new Error(`File not found: ${file}`);
|
|
561
|
+
}
|
|
562
|
+
const lines = rawContent.split(/\r?\n/);
|
|
563
|
+
const totalLines = lines.length;
|
|
564
|
+
let from = typeof fromLine === 'number' && !isNaN(fromLine) ? Math.floor(fromLine) : 1;
|
|
565
|
+
if (from < 1)
|
|
566
|
+
from = 1;
|
|
567
|
+
let selected;
|
|
568
|
+
let to;
|
|
569
|
+
if (totalLines === 0) {
|
|
570
|
+
from = 1;
|
|
571
|
+
to = 0;
|
|
572
|
+
selected = [];
|
|
573
|
+
}
|
|
574
|
+
else if (from > totalLines) {
|
|
575
|
+
from = totalLines;
|
|
576
|
+
to = totalLines;
|
|
577
|
+
selected = [lines[totalLines - 1]];
|
|
578
|
+
}
|
|
579
|
+
else {
|
|
580
|
+
const count = typeof maxLines === 'number' && !isNaN(maxLines) && maxLines > 0
|
|
581
|
+
? Math.min(Math.floor(maxLines), totalLines - from + 1)
|
|
582
|
+
: totalLines - from + 1;
|
|
583
|
+
to = from + count - 1;
|
|
584
|
+
selected = lines.slice(from - 1, to);
|
|
585
|
+
}
|
|
586
|
+
return {
|
|
587
|
+
file: normalizedFile,
|
|
588
|
+
fromLine: from,
|
|
589
|
+
toLine: to,
|
|
590
|
+
lineCount: selected.length,
|
|
591
|
+
totalLines,
|
|
592
|
+
text: selected.join('\n'),
|
|
593
|
+
};
|
|
594
|
+
}
|
package/dist/index-format.js
CHANGED
|
@@ -90,7 +90,14 @@ export function validateIndexEnvelope(value, location, options = {}) {
|
|
|
90
90
|
else if (modelName !== undefined && modelName !== null && typeof modelName !== 'string') {
|
|
91
91
|
throw new Error(`${location} model metadata must be a string${REBUILD}`);
|
|
92
92
|
}
|
|
93
|
-
|
|
93
|
+
let model;
|
|
94
|
+
try {
|
|
95
|
+
const candidateDim = typeof index.dim === 'number' && index.dim > 0 ? index.dim : undefined;
|
|
96
|
+
model = resolveModel(modelName, candidateDim);
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
model = resolveModel(modelName);
|
|
100
|
+
}
|
|
94
101
|
const dim = resolveIndexDimension(index.dim, model.dim);
|
|
95
102
|
if (schema < 3) {
|
|
96
103
|
return { index, schema, model, dim };
|
package/dist/index.d.ts
CHANGED
|
@@ -23,16 +23,16 @@ export { buildIndex, chunkHash } from './indexer.js';
|
|
|
23
23
|
export { search, searchIndex, loadIndex, tokenize, keywordScores, rrf, QueryEmbeddingCache, } from './search.js';
|
|
24
24
|
export { resolveModel, MODELS, DEFAULT_MODEL, normalizeAdapter, embeddingAdapterFingerprint, legacyEmbeddingAdapterFingerprint, } from './models.js';
|
|
25
25
|
export { getReranker, rerankScores, RERANK_MODEL, } from './rerank.js';
|
|
26
|
-
export { embed, getExtractor, cosine, walkMarkdown, parseFile, chunkMarkdown, splitFrontmatter, extractTitle, globToRegExp, encodeVec, decodeVec, isBinaryIndex, } from './core.js';
|
|
27
|
-
export { MCP_TOOLS, handleMcpRequest, startMcpServer, } from './mcp.js';
|
|
26
|
+
export { embed, getExtractor, cosine, truncateAndNormalizeVector, walkMarkdown, parseFile, chunkMarkdown, splitFrontmatter, extractTitle, globToRegExp, encodeVec, decodeVec, isBinaryIndex, getDocLines, GetDocLinesOptions, GetDocLinesResult, } from './core.js';
|
|
27
|
+
export { MCP_TOOLS, MCP_RESOURCE_TEMPLATES, MCP_PROMPTS, handleMcpRequest, startMcpServer, } from './mcp.js';
|
|
28
28
|
export { trainIVF, searchIVFCandidates, serializeIVF, deserializeIVF, ANN_THRESHOLD, DEFAULT_NPROBE, } from './ivf.js';
|
|
29
|
-
export { extractLinks, resolveLinks, buildRelationshipGraph, getRelatedNotes, computePageRank, expandGraphNeighborhood, } from './wikilinks.js';
|
|
29
|
+
export { extractLinks, resolveLinks, buildRelationshipGraph, getRelatedNotes, computePageRank, expandGraphNeighborhood, findRelatedNotes, FindRelatedNotesOptions, RelatedNoteHit, RelatedNoteResult, } from './wikilinks.js';
|
|
30
30
|
export { parseFilter, tokenizeFilter, evaluateFilter, FilterNode, } from './filter.js';
|
|
31
31
|
export { serializeBinaryIndex, deserializeBinaryIndex, readBinaryHeader, BINARY_MAGIC, BINARY_HEADER_SIZE, } from './binary-format.js';
|
|
32
32
|
export { quantizeToInt8, dequantizeFromInt8, asymmetricCosineInt8, INT8_SCALE, INT8_INV_SCALE, } from './quantization.js';
|
|
33
33
|
export { searchFederated, FederatedSearchOptions, FederatedSearchResultHit, VaultConfig, } from './federation.js';
|
|
34
34
|
export { createFileWatcher, FileWatcher, WatcherOptions, } from './watcher.js';
|
|
35
|
-
export { askQuestion, synthesizeAnswer, extractAnswerFallback, AskOptions, SynthesisResult, Citation, DEFAULT_RAG_SYSTEM_PROMPT, } from './rag.js';
|
|
35
|
+
export { askQuestion, synthesizeAnswer, extractAnswerFallback, chatTurn, loadChatSession, saveChatSession, listChatSessions, getSessionsDir, AskOptions, SynthesisResult, Citation, DEFAULT_RAG_SYSTEM_PROMPT, ChatSession, ChatTurn, ChatSourceManifestItem, ChatTurnOptions, } from './rag.js';
|
|
36
36
|
export { extractKeyphraseTags, summarizePassage, analyzeNote, NoteAnalysis, } from './summarize.js';
|
|
37
37
|
export { runTui, TuiOptions } from './tui.js';
|
|
38
38
|
export { generateCompletion, FLAGS, COMMANDS, COMPLETION_SHELLS, } from './completions.js';
|
package/dist/index.js
CHANGED
|
@@ -23,16 +23,16 @@ export { buildIndex, chunkHash } from './indexer.js';
|
|
|
23
23
|
export { search, searchIndex, loadIndex, tokenize, keywordScores, rrf, QueryEmbeddingCache, } from './search.js';
|
|
24
24
|
export { resolveModel, MODELS, DEFAULT_MODEL, normalizeAdapter, embeddingAdapterFingerprint, legacyEmbeddingAdapterFingerprint, } from './models.js';
|
|
25
25
|
export { getReranker, rerankScores, RERANK_MODEL, } from './rerank.js';
|
|
26
|
-
export { embed, getExtractor, cosine, walkMarkdown, parseFile, chunkMarkdown, splitFrontmatter, extractTitle, globToRegExp, encodeVec, decodeVec, isBinaryIndex, } from './core.js';
|
|
27
|
-
export { MCP_TOOLS, handleMcpRequest, startMcpServer, } from './mcp.js';
|
|
26
|
+
export { embed, getExtractor, cosine, truncateAndNormalizeVector, walkMarkdown, parseFile, chunkMarkdown, splitFrontmatter, extractTitle, globToRegExp, encodeVec, decodeVec, isBinaryIndex, getDocLines, } from './core.js';
|
|
27
|
+
export { MCP_TOOLS, MCP_RESOURCE_TEMPLATES, MCP_PROMPTS, handleMcpRequest, startMcpServer, } from './mcp.js';
|
|
28
28
|
export { trainIVF, searchIVFCandidates, serializeIVF, deserializeIVF, ANN_THRESHOLD, DEFAULT_NPROBE, } from './ivf.js';
|
|
29
|
-
export { extractLinks, resolveLinks, buildRelationshipGraph, getRelatedNotes, computePageRank, expandGraphNeighborhood, } from './wikilinks.js';
|
|
29
|
+
export { extractLinks, resolveLinks, buildRelationshipGraph, getRelatedNotes, computePageRank, expandGraphNeighborhood, findRelatedNotes, } from './wikilinks.js';
|
|
30
30
|
export { parseFilter, tokenizeFilter, evaluateFilter, } from './filter.js';
|
|
31
31
|
export { serializeBinaryIndex, deserializeBinaryIndex, readBinaryHeader, BINARY_MAGIC, BINARY_HEADER_SIZE, } from './binary-format.js';
|
|
32
32
|
export { quantizeToInt8, dequantizeFromInt8, asymmetricCosineInt8, INT8_SCALE, INT8_INV_SCALE, } from './quantization.js';
|
|
33
33
|
export { searchFederated, } from './federation.js';
|
|
34
34
|
export { createFileWatcher, } from './watcher.js';
|
|
35
|
-
export { askQuestion, synthesizeAnswer, extractAnswerFallback, DEFAULT_RAG_SYSTEM_PROMPT, } from './rag.js';
|
|
35
|
+
export { askQuestion, synthesizeAnswer, extractAnswerFallback, chatTurn, loadChatSession, saveChatSession, listChatSessions, getSessionsDir, DEFAULT_RAG_SYSTEM_PROMPT, } from './rag.js';
|
|
36
36
|
export { extractKeyphraseTags, summarizePassage, analyzeNote, } from './summarize.js';
|
|
37
37
|
export { runTui } from './tui.js';
|
|
38
38
|
export { generateCompletion, FLAGS, COMMANDS, COMPLETION_SHELLS, } from './completions.js';
|
package/dist/indexer.d.ts
CHANGED
package/dist/indexer.js
CHANGED
|
@@ -79,13 +79,13 @@ export function chunkHash(model, chunk) {
|
|
|
79
79
|
return sha256(input);
|
|
80
80
|
}
|
|
81
81
|
export async function buildIndex(opts) {
|
|
82
|
-
const { db, indexDir, cacheDir, modelName, ignore = [], log = () => { }, offline = false, embedFn = embed, _lockHeld = false, maxRetries = 3, onProgress, workers = 1, ann = false, quantize = 'fp32', } = opts;
|
|
82
|
+
const { db, indexDir, cacheDir, modelName, dimensions, ignore = [], log = () => { }, offline = false, embedFn = embed, _lockHeld = false, maxRetries = 3, onProgress, workers = 1, ann = false, quantize = 'fp32', } = opts;
|
|
83
83
|
if (_lockHeld)
|
|
84
|
-
return _buildIndexInner({ db, indexDir, cacheDir, modelName, ignore, log, offline, embedFn, maxRetries, onProgress, workers, ann, quantize });
|
|
85
|
-
return withIndexLock(indexDir, () => _buildIndexInner({ db, indexDir, cacheDir, modelName, ignore, log, offline, embedFn, maxRetries, onProgress, workers, ann, quantize }));
|
|
84
|
+
return _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions, ignore, log, offline, embedFn, maxRetries, onProgress, workers, ann, quantize });
|
|
85
|
+
return withIndexLock(indexDir, () => _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions, ignore, log, offline, embedFn, maxRetries, onProgress, workers, ann, quantize }));
|
|
86
86
|
}
|
|
87
|
-
async function _buildIndexInner({ db, indexDir, cacheDir, modelName, ignore = [], log = () => { }, offline = false, embedFn = embed, maxRetries = 3, onProgress, workers = 1, ann = false, quantize = 'fp32', }) {
|
|
88
|
-
const model = resolveModel(modelName);
|
|
87
|
+
async function _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions, ignore = [], log = () => { }, offline = false, embedFn = embed, maxRetries = 3, onProgress, workers = 1, ann = false, quantize = 'fp32', }) {
|
|
88
|
+
const model = resolveModel(modelName, dimensions);
|
|
89
89
|
const modelIdentity = `${model.id}@${model.revision || 'main'}`;
|
|
90
90
|
const adapterFingerprint = embeddingAdapterFingerprint(model);
|
|
91
91
|
const legacyAdapterFingerprint = legacyEmbeddingAdapterFingerprint(model);
|
package/dist/ivf.d.ts
CHANGED
|
@@ -10,15 +10,22 @@ export interface SerializedIVF {
|
|
|
10
10
|
centroids: string;
|
|
11
11
|
clusters: number[][];
|
|
12
12
|
}
|
|
13
|
+
export interface TrainIVFOptions {
|
|
14
|
+
k?: number;
|
|
15
|
+
maxIterations?: number;
|
|
16
|
+
seeding?: 'kmeans++' | 'spread';
|
|
17
|
+
seed?: number;
|
|
18
|
+
onMetric?: (metric: {
|
|
19
|
+
emptyClusters: number;
|
|
20
|
+
iterations: number;
|
|
21
|
+
}) => void;
|
|
22
|
+
}
|
|
13
23
|
export declare const ANN_THRESHOLD = 500;
|
|
14
24
|
export declare const DEFAULT_NPROBE = 8;
|
|
15
25
|
/**
|
|
16
|
-
* Train a Spherical K-Means IVF index over normalized vectors.
|
|
26
|
+
* Train a Spherical K-Means IVF index over normalized vectors using k-means++ seeding.
|
|
17
27
|
*/
|
|
18
|
-
export declare function trainIVF(vectors: Float32Array[], options?:
|
|
19
|
-
k?: number;
|
|
20
|
-
maxIterations?: number;
|
|
21
|
-
}): IVFIndex;
|
|
28
|
+
export declare function trainIVF(vectors: Float32Array[], options?: TrainIVFOptions): IVFIndex;
|
|
22
29
|
/**
|
|
23
30
|
* Probe top-nprobe closest centroids and return candidate chunk indices.
|
|
24
31
|
*/
|
|
@@ -28,6 +35,6 @@ export declare function searchIVFCandidates(queryVec: Float32Array, ivf: IVFInde
|
|
|
28
35
|
*/
|
|
29
36
|
export declare function serializeIVF(ivf: IVFIndex): SerializedIVF;
|
|
30
37
|
/**
|
|
31
|
-
* Deserialize JSON object back to IVFIndex.
|
|
38
|
+
* Deserialize JSON object back to IVFIndex with hardened validation.
|
|
32
39
|
*/
|
|
33
|
-
export declare function deserializeIVF(raw:
|
|
40
|
+
export declare function deserializeIVF(raw: unknown, expectedChunksCount?: number): IVFIndex;
|
package/dist/ivf.js
CHANGED
|
@@ -2,7 +2,19 @@ import { cosine } from './core.js';
|
|
|
2
2
|
export const ANN_THRESHOLD = 500;
|
|
3
3
|
export const DEFAULT_NPROBE = 8;
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
5
|
+
* Fast, deterministic pseudo-random number generator (Mulberry32).
|
|
6
|
+
*/
|
|
7
|
+
function mulberry32(seed) {
|
|
8
|
+
let s = seed >>> 0;
|
|
9
|
+
return () => {
|
|
10
|
+
s = (s + 0x6d2b79f5) >>> 0;
|
|
11
|
+
let t = Math.imul(s ^ (s >>> 15), 1 | s);
|
|
12
|
+
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
|
13
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Train a Spherical K-Means IVF index over normalized vectors using k-means++ seeding.
|
|
6
18
|
*/
|
|
7
19
|
export function trainIVF(vectors, options = {}) {
|
|
8
20
|
const n = vectors.length;
|
|
@@ -13,16 +25,62 @@ export function trainIVF(vectors, options = {}) {
|
|
|
13
25
|
const targetK = options.k ?? Math.min(256, Math.max(2, Math.floor(Math.sqrt(n))));
|
|
14
26
|
const k = Math.min(targetK, n);
|
|
15
27
|
const maxIterations = options.maxIterations ?? 20;
|
|
16
|
-
|
|
28
|
+
const seeding = options.seeding ?? 'kmeans++';
|
|
17
29
|
const centroids = new Float32Array(k * dim);
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
const
|
|
21
|
-
|
|
30
|
+
if (seeding === 'spread' || n <= k) {
|
|
31
|
+
// Deterministic spread
|
|
32
|
+
const step = Math.floor(n / k);
|
|
33
|
+
for (let c = 0; c < k; c++) {
|
|
34
|
+
const src = vectors[c * step];
|
|
35
|
+
centroids.set(src, c * dim);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
// Deterministic k-means++ seeding with D^2 weighted probability
|
|
40
|
+
const rng = mulberry32(options.seed ?? 42);
|
|
41
|
+
const firstIdx = Math.floor(rng() * n);
|
|
42
|
+
centroids.set(vectors[firstIdx], 0);
|
|
43
|
+
const minDistSq = new Float64Array(n);
|
|
44
|
+
for (let i = 0; i < n; i++) {
|
|
45
|
+
const cos = cosine(vectors[i], centroids, 0);
|
|
46
|
+
minDistSq[i] = Math.max(0, 1 - cos);
|
|
47
|
+
}
|
|
48
|
+
for (let c = 1; c < k; c++) {
|
|
49
|
+
let sumDistSq = 0;
|
|
50
|
+
for (let i = 0; i < n; i++) {
|
|
51
|
+
sumDistSq += minDistSq[i];
|
|
52
|
+
}
|
|
53
|
+
let chosenIdx = -1;
|
|
54
|
+
if (sumDistSq > 1e-12) {
|
|
55
|
+
const target = rng() * sumDistSq;
|
|
56
|
+
let cumulative = 0;
|
|
57
|
+
for (let i = 0; i < n; i++) {
|
|
58
|
+
cumulative += minDistSq[i];
|
|
59
|
+
if (cumulative >= target) {
|
|
60
|
+
chosenIdx = i;
|
|
61
|
+
break;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
if (chosenIdx === -1 || chosenIdx >= n) {
|
|
66
|
+
chosenIdx = Math.floor(rng() * n);
|
|
67
|
+
}
|
|
68
|
+
centroids.set(vectors[chosenIdx], c * dim);
|
|
69
|
+
for (let i = 0; i < n; i++) {
|
|
70
|
+
const cos = cosine(vectors[i], centroids, c * dim);
|
|
71
|
+
const distSq = Math.max(0, 1 - cos);
|
|
72
|
+
if (distSq < minDistSq[i]) {
|
|
73
|
+
minDistSq[i] = distSq;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
22
77
|
}
|
|
23
78
|
let clusters = Array.from({ length: k }, () => []);
|
|
24
79
|
const assignments = new Int32Array(n).fill(-1);
|
|
80
|
+
let totalEmptyClusters = 0;
|
|
81
|
+
let finalIterations = 0;
|
|
25
82
|
for (let iter = 0; iter < maxIterations; iter++) {
|
|
83
|
+
finalIterations = iter + 1;
|
|
26
84
|
let changed = 0;
|
|
27
85
|
const newClusters = Array.from({ length: k }, () => []);
|
|
28
86
|
// 1. Assign each vector to closest centroid (highest cosine / dot product)
|
|
@@ -44,11 +102,13 @@ export function trainIVF(vectors, options = {}) {
|
|
|
44
102
|
newClusters[bestCluster].push(i);
|
|
45
103
|
}
|
|
46
104
|
clusters = newClusters;
|
|
47
|
-
// 2. Recompute centroids
|
|
105
|
+
// 2. Recompute centroids & rescue empty clusters
|
|
106
|
+
let emptyCount = 0;
|
|
48
107
|
for (let c = 0; c < k; c++) {
|
|
49
108
|
const members = clusters[c];
|
|
50
109
|
if (members.length === 0) {
|
|
51
|
-
|
|
110
|
+
emptyCount++;
|
|
111
|
+
// Reassign to a vector from the largest cluster
|
|
52
112
|
let largest = 0;
|
|
53
113
|
for (let j = 1; j < k; j++) {
|
|
54
114
|
if (clusters[j].length > clusters[largest].length)
|
|
@@ -82,9 +142,11 @@ export function trainIVF(vectors, options = {}) {
|
|
|
82
142
|
centroids[cOffset + d] /= norm;
|
|
83
143
|
}
|
|
84
144
|
}
|
|
145
|
+
totalEmptyClusters = emptyCount;
|
|
85
146
|
if (changed === 0)
|
|
86
147
|
break;
|
|
87
148
|
}
|
|
149
|
+
options.onMetric?.({ emptyClusters: totalEmptyClusters, iterations: finalIterations });
|
|
88
150
|
return { dim, k, centroids, clusters };
|
|
89
151
|
}
|
|
90
152
|
/**
|
|
@@ -126,19 +188,65 @@ export function serializeIVF(ivf) {
|
|
|
126
188
|
};
|
|
127
189
|
}
|
|
128
190
|
/**
|
|
129
|
-
* Deserialize JSON object back to IVFIndex.
|
|
191
|
+
* Deserialize JSON object back to IVFIndex with hardened validation.
|
|
130
192
|
*/
|
|
131
|
-
export function deserializeIVF(raw) {
|
|
132
|
-
if (!raw || typeof raw !== 'object') {
|
|
133
|
-
throw new Error('invalid IVF payload: expected object');
|
|
193
|
+
export function deserializeIVF(raw, expectedChunksCount) {
|
|
194
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
|
|
195
|
+
throw new Error('invalid IVF payload: expected JSON object — run `mdss index` to rebuild');
|
|
196
|
+
}
|
|
197
|
+
const obj = raw;
|
|
198
|
+
const dim = obj.dim;
|
|
199
|
+
const k = obj.k;
|
|
200
|
+
if (typeof dim !== 'number' ||
|
|
201
|
+
!Number.isSafeInteger(dim) ||
|
|
202
|
+
dim <= 0 ||
|
|
203
|
+
typeof k !== 'number' ||
|
|
204
|
+
!Number.isSafeInteger(k) ||
|
|
205
|
+
k <= 0) {
|
|
206
|
+
throw new Error('invalid IVF payload: invalid dim or k — run `mdss index` to rebuild');
|
|
207
|
+
}
|
|
208
|
+
if (typeof obj.centroids !== 'string' || obj.centroids.length === 0) {
|
|
209
|
+
throw new Error('invalid IVF payload: centroids must be a base64 string — run `mdss index` to rebuild');
|
|
134
210
|
}
|
|
135
|
-
const
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
throw new Error('invalid IVF payload: invalid dim or k');
|
|
211
|
+
const buf = Buffer.from(obj.centroids, 'base64');
|
|
212
|
+
if (buf.byteLength % 4 !== 0 || buf.byteLength !== k * dim * 4) {
|
|
213
|
+
throw new Error(`invalid IVF payload: centroids buffer has ${buf.byteLength} bytes, expected ${k * dim * 4} bytes — run \`mdss index\` to rebuild`);
|
|
139
214
|
}
|
|
140
|
-
const buf = Buffer.from(raw.centroids, 'base64');
|
|
141
215
|
const centroids = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
|
142
|
-
|
|
216
|
+
for (let i = 0; i < centroids.length; i++) {
|
|
217
|
+
if (!Number.isFinite(centroids[i])) {
|
|
218
|
+
throw new Error(`invalid IVF payload: non-finite centroid value at index ${i} — run \`mdss index\` to rebuild`);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
if (!Array.isArray(obj.clusters)) {
|
|
222
|
+
throw new Error('invalid IVF payload: clusters must be an array — run `mdss index` to rebuild');
|
|
223
|
+
}
|
|
224
|
+
if (obj.clusters.length !== k) {
|
|
225
|
+
throw new Error(`invalid IVF payload: clusters array has length ${obj.clusters.length}, expected ${k} — run \`mdss index\` to rebuild`);
|
|
226
|
+
}
|
|
227
|
+
const clusters = new Array(k);
|
|
228
|
+
for (let c = 0; c < k; c++) {
|
|
229
|
+
const list = obj.clusters[c];
|
|
230
|
+
if (!Array.isArray(list)) {
|
|
231
|
+
throw new Error(`invalid IVF payload: cluster ${c} must be an array — run \`mdss index\` to rebuild`);
|
|
232
|
+
}
|
|
233
|
+
const validatedCluster = new Array(list.length);
|
|
234
|
+
const seen = new Set();
|
|
235
|
+
for (let j = 0; j < list.length; j++) {
|
|
236
|
+
const idx = list[j];
|
|
237
|
+
if (typeof idx !== 'number' || !Number.isSafeInteger(idx) || idx < 0) {
|
|
238
|
+
throw new Error(`invalid IVF payload: cluster ${c} contains non-integer index "${idx}" — run \`mdss index\` to rebuild`);
|
|
239
|
+
}
|
|
240
|
+
if (expectedChunksCount !== undefined && idx >= expectedChunksCount) {
|
|
241
|
+
throw new Error(`invalid IVF payload: chunk index ${idx} out of range [0, ${expectedChunksCount}) — run \`mdss index\` to rebuild`);
|
|
242
|
+
}
|
|
243
|
+
if (seen.has(idx)) {
|
|
244
|
+
throw new Error(`invalid IVF payload: duplicate chunk index ${idx} in cluster ${c} — run \`mdss index\` to rebuild`);
|
|
245
|
+
}
|
|
246
|
+
seen.add(idx);
|
|
247
|
+
validatedCluster[j] = idx;
|
|
248
|
+
}
|
|
249
|
+
clusters[c] = validatedCluster;
|
|
250
|
+
}
|
|
143
251
|
return { dim, k, centroids, clusters };
|
|
144
252
|
}
|
package/dist/mcp.d.ts
CHANGED
|
@@ -7,6 +7,10 @@ import readline from 'node:readline';
|
|
|
7
7
|
export declare const MCP_TOOLS: ({
|
|
8
8
|
name: string;
|
|
9
9
|
description: string;
|
|
10
|
+
annotations: {
|
|
11
|
+
readOnly: boolean;
|
|
12
|
+
idempotent: boolean;
|
|
13
|
+
};
|
|
10
14
|
inputSchema: {
|
|
11
15
|
type: string;
|
|
12
16
|
properties: {
|
|
@@ -44,14 +48,23 @@ export declare const MCP_TOOLS: ({
|
|
|
44
48
|
type: string;
|
|
45
49
|
description: string;
|
|
46
50
|
};
|
|
47
|
-
file?: undefined;
|
|
48
51
|
heading?: undefined;
|
|
52
|
+
fromLine?: undefined;
|
|
53
|
+
maxLines?: undefined;
|
|
54
|
+
file?: undefined;
|
|
55
|
+
direction?: undefined;
|
|
56
|
+
semantic?: undefined;
|
|
57
|
+
sessionId?: undefined;
|
|
49
58
|
};
|
|
50
59
|
required: string[];
|
|
51
60
|
};
|
|
52
61
|
} | {
|
|
53
62
|
name: string;
|
|
54
63
|
description: string;
|
|
64
|
+
annotations: {
|
|
65
|
+
readOnly: boolean;
|
|
66
|
+
idempotent: boolean;
|
|
67
|
+
};
|
|
55
68
|
inputSchema: {
|
|
56
69
|
type: string;
|
|
57
70
|
properties: {
|
|
@@ -67,15 +80,60 @@ export declare const MCP_TOOLS: ({
|
|
|
67
80
|
type: string;
|
|
68
81
|
description: string;
|
|
69
82
|
};
|
|
83
|
+
fromLine?: undefined;
|
|
84
|
+
maxLines?: undefined;
|
|
85
|
+
direction?: undefined;
|
|
86
|
+
semantic?: undefined;
|
|
70
87
|
path?: undefined;
|
|
71
88
|
query?: undefined;
|
|
72
89
|
k?: undefined;
|
|
90
|
+
sessionId?: undefined;
|
|
73
91
|
};
|
|
74
92
|
required: string[];
|
|
75
93
|
};
|
|
76
94
|
} | {
|
|
77
95
|
name: string;
|
|
78
96
|
description: string;
|
|
97
|
+
annotations: {
|
|
98
|
+
readOnly: boolean;
|
|
99
|
+
idempotent: boolean;
|
|
100
|
+
};
|
|
101
|
+
inputSchema: {
|
|
102
|
+
type: string;
|
|
103
|
+
properties: {
|
|
104
|
+
vaults?: undefined;
|
|
105
|
+
tag?: undefined;
|
|
106
|
+
filter?: undefined;
|
|
107
|
+
graphBoost?: undefined;
|
|
108
|
+
heading?: undefined;
|
|
109
|
+
file: {
|
|
110
|
+
type: string;
|
|
111
|
+
description: string;
|
|
112
|
+
};
|
|
113
|
+
fromLine: {
|
|
114
|
+
type: string;
|
|
115
|
+
description: string;
|
|
116
|
+
};
|
|
117
|
+
maxLines: {
|
|
118
|
+
type: string;
|
|
119
|
+
description: string;
|
|
120
|
+
};
|
|
121
|
+
direction?: undefined;
|
|
122
|
+
semantic?: undefined;
|
|
123
|
+
path?: undefined;
|
|
124
|
+
query?: undefined;
|
|
125
|
+
k?: undefined;
|
|
126
|
+
sessionId?: undefined;
|
|
127
|
+
};
|
|
128
|
+
required: string[];
|
|
129
|
+
};
|
|
130
|
+
} | {
|
|
131
|
+
name: string;
|
|
132
|
+
description: string;
|
|
133
|
+
annotations: {
|
|
134
|
+
readOnly: boolean;
|
|
135
|
+
idempotent: boolean;
|
|
136
|
+
};
|
|
79
137
|
inputSchema: {
|
|
80
138
|
type: string;
|
|
81
139
|
properties: {
|
|
@@ -83,20 +141,69 @@ export declare const MCP_TOOLS: ({
|
|
|
83
141
|
tag?: undefined;
|
|
84
142
|
filter?: undefined;
|
|
85
143
|
graphBoost?: undefined;
|
|
86
|
-
file?: undefined;
|
|
87
144
|
heading?: undefined;
|
|
145
|
+
fromLine?: undefined;
|
|
146
|
+
maxLines?: undefined;
|
|
147
|
+
file: {
|
|
148
|
+
type: string;
|
|
149
|
+
description: string;
|
|
150
|
+
};
|
|
151
|
+
k: {
|
|
152
|
+
type: string;
|
|
153
|
+
description: string;
|
|
154
|
+
};
|
|
155
|
+
direction: {
|
|
156
|
+
type: string;
|
|
157
|
+
enum: string[];
|
|
158
|
+
description: string;
|
|
159
|
+
};
|
|
160
|
+
semantic: {
|
|
161
|
+
type: string;
|
|
162
|
+
description: string;
|
|
163
|
+
};
|
|
164
|
+
path?: undefined;
|
|
165
|
+
query?: undefined;
|
|
166
|
+
sessionId?: undefined;
|
|
167
|
+
};
|
|
168
|
+
required: string[];
|
|
169
|
+
};
|
|
170
|
+
} | {
|
|
171
|
+
name: string;
|
|
172
|
+
description: string;
|
|
173
|
+
annotations: {
|
|
174
|
+
readOnly: boolean;
|
|
175
|
+
idempotent: boolean;
|
|
176
|
+
};
|
|
177
|
+
inputSchema: {
|
|
178
|
+
type: string;
|
|
179
|
+
properties: {
|
|
180
|
+
vaults?: undefined;
|
|
181
|
+
tag?: undefined;
|
|
182
|
+
filter?: undefined;
|
|
183
|
+
graphBoost?: undefined;
|
|
184
|
+
heading?: undefined;
|
|
185
|
+
fromLine?: undefined;
|
|
186
|
+
maxLines?: undefined;
|
|
187
|
+
file?: undefined;
|
|
188
|
+
direction?: undefined;
|
|
189
|
+
semantic?: undefined;
|
|
88
190
|
path: {
|
|
89
191
|
type: string;
|
|
90
192
|
description: string;
|
|
91
193
|
};
|
|
92
194
|
query?: undefined;
|
|
93
195
|
k?: undefined;
|
|
196
|
+
sessionId?: undefined;
|
|
94
197
|
};
|
|
95
198
|
required?: undefined;
|
|
96
199
|
};
|
|
97
200
|
} | {
|
|
98
201
|
name: string;
|
|
99
202
|
description: string;
|
|
203
|
+
annotations: {
|
|
204
|
+
readOnly: boolean;
|
|
205
|
+
idempotent: boolean;
|
|
206
|
+
};
|
|
100
207
|
inputSchema: {
|
|
101
208
|
type: string;
|
|
102
209
|
properties: {
|
|
@@ -104,17 +211,26 @@ export declare const MCP_TOOLS: ({
|
|
|
104
211
|
tag?: undefined;
|
|
105
212
|
filter?: undefined;
|
|
106
213
|
graphBoost?: undefined;
|
|
107
|
-
file?: undefined;
|
|
108
214
|
heading?: undefined;
|
|
215
|
+
fromLine?: undefined;
|
|
216
|
+
maxLines?: undefined;
|
|
217
|
+
file?: undefined;
|
|
218
|
+
direction?: undefined;
|
|
219
|
+
semantic?: undefined;
|
|
109
220
|
path?: undefined;
|
|
110
221
|
query?: undefined;
|
|
111
222
|
k?: undefined;
|
|
223
|
+
sessionId?: undefined;
|
|
112
224
|
};
|
|
113
225
|
required?: undefined;
|
|
114
226
|
};
|
|
115
227
|
} | {
|
|
116
228
|
name: string;
|
|
117
229
|
description: string;
|
|
230
|
+
annotations: {
|
|
231
|
+
readOnly: boolean;
|
|
232
|
+
idempotent: boolean;
|
|
233
|
+
};
|
|
118
234
|
inputSchema: {
|
|
119
235
|
type: string;
|
|
120
236
|
properties: {
|
|
@@ -122,8 +238,12 @@ export declare const MCP_TOOLS: ({
|
|
|
122
238
|
tag?: undefined;
|
|
123
239
|
filter?: undefined;
|
|
124
240
|
graphBoost?: undefined;
|
|
125
|
-
file?: undefined;
|
|
126
241
|
heading?: undefined;
|
|
242
|
+
fromLine?: undefined;
|
|
243
|
+
maxLines?: undefined;
|
|
244
|
+
file?: undefined;
|
|
245
|
+
direction?: undefined;
|
|
246
|
+
semantic?: undefined;
|
|
127
247
|
path?: undefined;
|
|
128
248
|
query: {
|
|
129
249
|
type: string;
|
|
@@ -133,10 +253,29 @@ export declare const MCP_TOOLS: ({
|
|
|
133
253
|
type: string;
|
|
134
254
|
description: string;
|
|
135
255
|
};
|
|
256
|
+
sessionId: {
|
|
257
|
+
type: string;
|
|
258
|
+
description: string;
|
|
259
|
+
};
|
|
136
260
|
};
|
|
137
261
|
required: string[];
|
|
138
262
|
};
|
|
139
263
|
})[];
|
|
264
|
+
export declare const MCP_RESOURCE_TEMPLATES: {
|
|
265
|
+
uriTemplate: string;
|
|
266
|
+
name: string;
|
|
267
|
+
description: string;
|
|
268
|
+
mimeType: string;
|
|
269
|
+
}[];
|
|
270
|
+
export declare const MCP_PROMPTS: {
|
|
271
|
+
name: string;
|
|
272
|
+
description: string;
|
|
273
|
+
arguments: {
|
|
274
|
+
name: string;
|
|
275
|
+
description: string;
|
|
276
|
+
required: boolean;
|
|
277
|
+
}[];
|
|
278
|
+
}[];
|
|
140
279
|
export declare function handleMcpRequest(req: any, state: {
|
|
141
280
|
loaded: any;
|
|
142
281
|
cacheDir: string;
|