md-semantic-search 1.0.3 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +132 -16
- package/RESEARCH.md +207 -3
- package/bin/cli.mjs +449 -13
- package/dist/binary-format.js +9 -0
- package/dist/completions.js +11 -1
- package/dist/core.d.ts +42 -4
- package/dist/core.js +208 -14
- package/dist/federation.d.ts +16 -1
- package/dist/federation.js +33 -16
- package/dist/filter.d.ts +7 -0
- package/dist/filter.js +13 -0
- package/dist/hnsw.d.ts +96 -0
- package/dist/hnsw.js +326 -0
- package/dist/index-format.js +8 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.js +4 -4
- package/dist/indexer.d.ts +18 -1
- package/dist/indexer.js +92 -18
- package/dist/ivf.d.ts +14 -7
- package/dist/ivf.js +126 -18
- package/dist/lexical.d.ts +10 -0
- package/dist/lexical.js +98 -15
- package/dist/markdown-parser.d.ts +62 -0
- package/dist/markdown-parser.js +227 -30
- package/dist/mcp.d.ts +229 -7
- package/dist/mcp.js +743 -44
- package/dist/models.d.ts +8 -2
- package/dist/models.js +84 -32
- package/dist/providers.d.ts +56 -0
- package/dist/providers.js +305 -0
- package/dist/quantization.d.ts +12 -0
- package/dist/quantization.js +54 -0
- package/dist/rag.d.ts +132 -7
- package/dist/rag.js +314 -38
- package/dist/repair.d.ts +1 -1
- package/dist/repair.js +53 -2
- package/dist/rerank.d.ts +51 -6
- package/dist/rerank.js +69 -10
- package/dist/search.d.ts +49 -3
- package/dist/search.js +387 -78
- package/dist/serve.d.ts +35 -0
- package/dist/serve.js +574 -58
- package/dist/tui.d.ts +14 -0
- package/dist/tui.js +132 -15
- package/dist/watcher.js +23 -12
- package/dist/webui.d.ts +3 -3
- package/dist/webui.js +225 -5
- package/dist/wikilinks.d.ts +31 -0
- package/dist/wikilinks.js +185 -1
- package/package.json +3 -3
- package/src/binary-format.ts +13 -0
- package/src/completions.ts +11 -1
- package/src/core.ts +249 -13
- package/src/federation.ts +51 -17
- package/src/filter.ts +14 -0
- package/src/hnsw.ts +414 -0
- package/src/index-format.ts +7 -1
- package/src/index.ts +7 -2
- package/src/indexer.ts +112 -19
- package/src/ivf.ts +157 -20
- package/src/lexical.ts +91 -13
- package/src/markdown-parser.ts +279 -37
- package/src/mcp.ts +820 -47
- package/src/models.ts +97 -35
- package/src/providers.ts +345 -0
- package/src/quantization.ts +61 -0
- package/src/rag.ts +451 -34
- package/src/repair.ts +51 -2
- package/src/rerank.ts +121 -14
- package/src/search.ts +424 -82
- package/src/serve.ts +589 -66
- package/src/tui.ts +142 -16
- package/src/watcher.ts +21 -10
- package/src/webui.ts +225 -5
- package/src/wikilinks.ts +220 -1
package/dist/markdown-parser.js
CHANGED
|
@@ -139,9 +139,63 @@ export function resolveChunkBudget(tokens, defaultMax = 1200) {
|
|
|
139
139
|
}
|
|
140
140
|
return defaultMax;
|
|
141
141
|
}
|
|
142
|
+
/**
|
|
143
|
+
* issue #202: script-calibrated token estimate for the sentencepiece E5/BGE
|
|
144
|
+
* vocab family — the fast-path alternative to running the real tokenizer per
|
|
145
|
+
* block. Ratios are calibrated so the estimate errs slightly HIGH: a budget
|
|
146
|
+
* checked against this estimate never overflows the model's real maxTokens.
|
|
147
|
+
* Latin/digits/punctuation ≈ 4 chars/token, Cyrillic ≈ 1.8 chars/token
|
|
148
|
+
* (RU systematically under-chunked by the old char budget — the point of
|
|
149
|
+
* this issue), CJK ≈ 1 token/char, whitespace mostly merges into the
|
|
150
|
+
* following subword so it counts a little.
|
|
151
|
+
*/
|
|
152
|
+
export function estimateTokens(text) {
|
|
153
|
+
let tokens = 0;
|
|
154
|
+
for (let i = 0; i < text.length; i++) {
|
|
155
|
+
const c = text.charCodeAt(i);
|
|
156
|
+
if (c >= 0x0400 && c <= 0x04ff)
|
|
157
|
+
tokens += 0.55; // Cyrillic (+ Cyrillic supplement edges)
|
|
158
|
+
else if ((c >= 0x3040 && c <= 0x30ff) || // hiragana / katakana
|
|
159
|
+
(c >= 0x3400 && c <= 0x4dbf) || // CJK ext A
|
|
160
|
+
(c >= 0x4e00 && c <= 0x9fff) || // CJK unified
|
|
161
|
+
(c >= 0xff00 && c <= 0xffef) // fullwidth forms
|
|
162
|
+
) {
|
|
163
|
+
tokens += 1.0;
|
|
164
|
+
}
|
|
165
|
+
else if (c === 0x20 || (c >= 0x09 && c <= 0x0d))
|
|
166
|
+
tokens += 0.08;
|
|
167
|
+
else if (c < 0x80)
|
|
168
|
+
tokens += 0.25; // ASCII (latin, digits, punctuation, markdown syntax)
|
|
169
|
+
else
|
|
170
|
+
tokens += 0.35; // other scripts / symbols — conservative middle ground
|
|
171
|
+
}
|
|
172
|
+
return Math.ceil(tokens);
|
|
173
|
+
}
|
|
174
|
+
/** Char budget (pre-#202 behavior) vs token budget (issue #202). */
|
|
175
|
+
export function resolveChunkingBudget(opts) {
|
|
176
|
+
if (opts.maxChunkTokens && Number.isFinite(opts.maxChunkTokens) && opts.maxChunkTokens > 0) {
|
|
177
|
+
const budget = Math.max(32, Math.round(opts.maxChunkTokens));
|
|
178
|
+
return { budget, measure: opts.countTokens ?? estimateTokens, unit: 'tokens' };
|
|
179
|
+
}
|
|
180
|
+
const budget = resolveChunkBudget(undefined, opts.maxChunk ?? 1200);
|
|
181
|
+
return { budget, measure: (t) => t.length, unit: 'chars' };
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* issue #202: the DEFAULT chunk budget in tokens, derived from the embedding
|
|
185
|
+
* adapter's context window. Capped at 512 — a passage much larger than that
|
|
186
|
+
* dilutes retrieval granularity even for models with 8k+ context (bge-m3).
|
|
187
|
+
* Returns undefined for adapters without a known maxTokens, which keeps the
|
|
188
|
+
* legacy char budget instead of guessing.
|
|
189
|
+
*/
|
|
190
|
+
export function defaultChunkTokenBudget(adapter) {
|
|
191
|
+
const mt = adapter?.maxTokens;
|
|
192
|
+
if (!mt || !Number.isFinite(mt) || mt <= 0)
|
|
193
|
+
return undefined;
|
|
194
|
+
return Math.min(Math.round(mt), 512);
|
|
195
|
+
}
|
|
142
196
|
export function chunkMarkdownStructural(input, opts) {
|
|
143
197
|
const options = typeof opts === 'number' ? { maxChunk: opts } : (opts || {});
|
|
144
|
-
const maxChunk =
|
|
198
|
+
const { budget: maxChunk, measure } = resolveChunkingBudget(options);
|
|
145
199
|
const blocks = typeof input === 'string' ? parseMarkdownBlocks(input) : input;
|
|
146
200
|
const chunks = [];
|
|
147
201
|
const headingStack = [];
|
|
@@ -149,6 +203,8 @@ export function chunkMarkdownStructural(input, opts) {
|
|
|
149
203
|
let currentLen = 0;
|
|
150
204
|
let groupStartLine = 0;
|
|
151
205
|
let groupEndLine = 0;
|
|
206
|
+
// measured size of the '\n\n' joiner — 2 in char mode, ~1 in token mode
|
|
207
|
+
const joinerSize = measure('\n\n');
|
|
152
208
|
const flushGroup = () => {
|
|
153
209
|
if (currentGroup.length === 0)
|
|
154
210
|
return;
|
|
@@ -180,26 +236,25 @@ export function chunkMarkdownStructural(input, opts) {
|
|
|
180
236
|
const blockText = block.text.trim();
|
|
181
237
|
if (!blockText)
|
|
182
238
|
continue;
|
|
183
|
-
if (blockText
|
|
239
|
+
if (measure(blockText) > maxChunk) {
|
|
184
240
|
flushGroup();
|
|
185
241
|
const headingPath = headingStack.map(h => h.heading);
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
242
|
+
// Issue #167: pieces carry the block-relative range of REAL source
|
|
243
|
+
// lines backing them — synthetic fence/header repeats must not advance
|
|
244
|
+
// the line cursor, or every later sub-chunk drifts down the file.
|
|
245
|
+
const subPieces = splitOversizedByMeasure(blockText, maxChunk, measure, block.type);
|
|
246
|
+
for (const piece of subPieces) {
|
|
191
247
|
chunks.push({
|
|
192
248
|
heading: headingPath.at(-1) ?? '',
|
|
193
249
|
headingPath: [...headingPath],
|
|
194
|
-
text: piece,
|
|
195
|
-
startLine:
|
|
196
|
-
endLine: Math.min(block.endLine,
|
|
250
|
+
text: piece.text,
|
|
251
|
+
startLine: block.startLine + piece.startLine,
|
|
252
|
+
endLine: Math.min(block.endLine, block.startLine + piece.endLine),
|
|
197
253
|
});
|
|
198
|
-
lineCursor += pieceLineCount;
|
|
199
254
|
}
|
|
200
255
|
continue;
|
|
201
256
|
}
|
|
202
|
-
const addedLen = currentLen === 0 ? blockText
|
|
257
|
+
const addedLen = currentLen === 0 ? measure(blockText) : currentLen + joinerSize + measure(blockText);
|
|
203
258
|
if (addedLen > maxChunk && currentGroup.length > 0) {
|
|
204
259
|
flushGroup();
|
|
205
260
|
}
|
|
@@ -207,7 +262,7 @@ export function chunkMarkdownStructural(input, opts) {
|
|
|
207
262
|
groupStartLine = block.startLine;
|
|
208
263
|
}
|
|
209
264
|
currentGroup.push(blockText);
|
|
210
|
-
currentLen += (currentGroup.length === 1 ? 0 :
|
|
265
|
+
currentLen += (currentGroup.length === 1 ? 0 : joinerSize) + measure(blockText);
|
|
211
266
|
groupEndLine = block.endLine;
|
|
212
267
|
}
|
|
213
268
|
flushGroup();
|
|
@@ -228,9 +283,121 @@ function splitLongLine(line, maxChunk) {
|
|
|
228
283
|
return pieces;
|
|
229
284
|
}
|
|
230
285
|
export function splitOversizedBlock(text, maxChunk, blockType) {
|
|
286
|
+
return splitOversizedBlockWithRanges(text, maxChunk, blockType).map((p) => p.text);
|
|
287
|
+
}
|
|
288
|
+
/**
|
|
289
|
+
* issue #202: oversized-block splitting under an arbitrary measure (chars or
|
|
290
|
+
* tokens). The heavy lifting (fence repeats, table header repeats, #167 line
|
|
291
|
+
* ranges) stays in the char-based splitOversizedBlockWithRanges; the token
|
|
292
|
+
* budget is converted to a char budget for THIS block via its own chars-per-
|
|
293
|
+
* measured-unit ratio, then every piece is verified — script-skew inside the
|
|
294
|
+
* block (ASCII header + Cyrillic body) is caught by a greedy re-split.
|
|
295
|
+
*/
|
|
296
|
+
export function splitOversizedByMeasure(text, budget, measure, blockType) {
|
|
297
|
+
if (measure(text) <= budget) {
|
|
298
|
+
return [{ text, startLine: 0, endLine: Number.MAX_SAFE_INTEGER }];
|
|
299
|
+
}
|
|
300
|
+
// char equivalent of the budget for this block's script mix; fall back to
|
|
301
|
+
// a plain char split with the raw budget when measuring is degenerate
|
|
302
|
+
const measured = Math.max(1, measure(text));
|
|
303
|
+
const charsPerUnit = text.length / measured;
|
|
304
|
+
const charBudget = Math.max(32, Math.floor(budget * charsPerUnit));
|
|
305
|
+
const pieces = splitOversizedBlockWithRanges(text, charBudget, blockType);
|
|
306
|
+
const out = [];
|
|
307
|
+
for (const piece of pieces) {
|
|
308
|
+
if (measure(piece.text) <= budget) {
|
|
309
|
+
out.push(piece);
|
|
310
|
+
continue;
|
|
311
|
+
}
|
|
312
|
+
// script-skew safety net: greedy line/word accumulation under measure;
|
|
313
|
+
// the greedy splitter tracks consumed source lines so #167 ranges stay real
|
|
314
|
+
const spanLines = piece.endLine - piece.startLine + 1;
|
|
315
|
+
const textLines = piece.text.split('\n');
|
|
316
|
+
// when synthetic lines (fence/table repeats) inflated the piece, the text
|
|
317
|
+
// has more lines than the source span — clamp keeps ranges inside the block
|
|
318
|
+
const lineStride = textLines.length > spanLines ? spanLines / textLines.length : 1;
|
|
319
|
+
for (const sub of greedySplitByMeasure(piece.text, budget, measure)) {
|
|
320
|
+
const startLine = piece.startLine + Math.min(spanLines - 1, Math.floor(sub.startLine * lineStride));
|
|
321
|
+
const endLine = piece.startLine + Math.min(spanLines - 1, Math.max(sub.startLine, Math.floor(sub.endLine * lineStride)));
|
|
322
|
+
out.push({ text: sub.text, startLine, endLine });
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
return out;
|
|
326
|
+
}
|
|
327
|
+
function greedySplitByMeasure(text, budget, measure) {
|
|
328
|
+
const out = [];
|
|
329
|
+
const lines = text.split('\n');
|
|
330
|
+
let bufStart = 0;
|
|
331
|
+
let bufEnd = 0;
|
|
332
|
+
let current = '';
|
|
333
|
+
const flush = (uptoLine) => {
|
|
334
|
+
if (current.trim())
|
|
335
|
+
out.push({ text: current.trim(), startLine: bufStart, endLine: uptoLine });
|
|
336
|
+
current = '';
|
|
337
|
+
};
|
|
338
|
+
for (let i = 0; i < lines.length; i++) {
|
|
339
|
+
const line = lines[i];
|
|
340
|
+
const candidate = current ? `${current}\n${line}` : line;
|
|
341
|
+
if (measure(candidate) <= budget) {
|
|
342
|
+
if (!current)
|
|
343
|
+
bufStart = i;
|
|
344
|
+
current = candidate;
|
|
345
|
+
bufEnd = i;
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
flush(i - 1);
|
|
349
|
+
if (measure(line) <= budget) {
|
|
350
|
+
bufStart = i;
|
|
351
|
+
current = line;
|
|
352
|
+
bufEnd = i;
|
|
353
|
+
continue;
|
|
354
|
+
}
|
|
355
|
+
// single line over budget: word-level accumulation, all pieces stay on
|
|
356
|
+
// this line's range
|
|
357
|
+
const words = line.split(/\s+/);
|
|
358
|
+
let wordBuf = '';
|
|
359
|
+
for (const word of words) {
|
|
360
|
+
const wc = wordBuf ? `${wordBuf} ${word}` : word;
|
|
361
|
+
if (measure(wc) <= budget) {
|
|
362
|
+
wordBuf = wc;
|
|
363
|
+
continue;
|
|
364
|
+
}
|
|
365
|
+
if (wordBuf.trim())
|
|
366
|
+
out.push({ text: wordBuf.trim(), startLine: i, endLine: i });
|
|
367
|
+
if (measure(word) <= budget) {
|
|
368
|
+
wordBuf = word;
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
371
|
+
// monster token (no spaces): hard-slice by chars proportional to budget
|
|
372
|
+
const ratio = line.length / Math.max(1, measure(line));
|
|
373
|
+
const sliceLen = Math.max(8, Math.floor(budget * ratio));
|
|
374
|
+
let rest = word;
|
|
375
|
+
while (measure(rest) > budget && rest.length > sliceLen) {
|
|
376
|
+
out.push({ text: rest.slice(0, sliceLen), startLine: i, endLine: i });
|
|
377
|
+
rest = rest.slice(sliceLen);
|
|
378
|
+
}
|
|
379
|
+
wordBuf = rest;
|
|
380
|
+
}
|
|
381
|
+
if (wordBuf.trim()) {
|
|
382
|
+
bufStart = i;
|
|
383
|
+
current = wordBuf;
|
|
384
|
+
bufEnd = i;
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
flush(bufEnd);
|
|
388
|
+
return out;
|
|
389
|
+
}
|
|
390
|
+
/**
|
|
391
|
+
* Issue #167: like splitOversizedBlock, but each piece also reports the
|
|
392
|
+
* block-relative range of source lines it was built from. Synthetic lines
|
|
393
|
+
* added to keep pieces self-contained (fence repeats, table header repeats)
|
|
394
|
+
* do NOT count — only the first piece's opening fence/header and a real
|
|
395
|
+
* closing fence belong to the source.
|
|
396
|
+
*/
|
|
397
|
+
export function splitOversizedBlockWithRanges(text, maxChunk, blockType) {
|
|
231
398
|
const lines = text.split('\n');
|
|
232
399
|
if (lines.length <= 1) {
|
|
233
|
-
return splitLongLine(text, maxChunk);
|
|
400
|
+
return splitLongLine(text, maxChunk).map((piece) => ({ text: piece, startLine: 0, endLine: 0 }));
|
|
234
401
|
}
|
|
235
402
|
// If code block with opening fence
|
|
236
403
|
if (blockType === 'code' && lines.length > 2 && /^(\s*)(`{3,}|~{3,})/.test(lines[0])) {
|
|
@@ -241,17 +408,30 @@ export function splitOversizedBlock(text, maxChunk, blockType) {
|
|
|
241
408
|
const pieces = [];
|
|
242
409
|
let currentCode = [];
|
|
243
410
|
let currentLen = openFence.length + closeFence.length + 2;
|
|
244
|
-
|
|
411
|
+
// The opening fence is a real source line only in the first piece; a
|
|
412
|
+
// real closing fence only in the final one.
|
|
413
|
+
const flush = (isFinal) => {
|
|
414
|
+
const firstReal = pieces.length === 0 ? 0 : currentCode[0].srcIdx;
|
|
415
|
+
const lastReal = isFinal && hasCloseFence ? lines.length - 1 : currentCode[currentCode.length - 1].srcIdx;
|
|
416
|
+
pieces.push({
|
|
417
|
+
text: [openFence, ...currentCode.map((c) => c.line), closeFence].join('\n'),
|
|
418
|
+
startLine: firstReal,
|
|
419
|
+
endLine: lastReal,
|
|
420
|
+
});
|
|
421
|
+
};
|
|
422
|
+
for (let offset = 0; offset < innerLines.length; offset++) {
|
|
423
|
+
const line = innerLines[offset];
|
|
424
|
+
const srcIdx = 1 + offset;
|
|
245
425
|
if (currentLen + line.length + 1 > maxChunk && currentCode.length > 0) {
|
|
246
|
-
|
|
426
|
+
flush(false);
|
|
247
427
|
currentCode = [];
|
|
248
428
|
currentLen = openFence.length + closeFence.length + 2;
|
|
249
429
|
}
|
|
250
|
-
currentCode.push(line);
|
|
430
|
+
currentCode.push({ line, srcIdx });
|
|
251
431
|
currentLen += line.length + 1;
|
|
252
432
|
}
|
|
253
433
|
if (currentCode.length > 0) {
|
|
254
|
-
|
|
434
|
+
flush(true);
|
|
255
435
|
}
|
|
256
436
|
if (pieces.length > 0)
|
|
257
437
|
return pieces;
|
|
@@ -264,17 +444,27 @@ export function splitOversizedBlock(text, maxChunk, blockType) {
|
|
|
264
444
|
const pieces = [];
|
|
265
445
|
let currentRows = [];
|
|
266
446
|
let currentLen = headerLen;
|
|
267
|
-
|
|
447
|
+
const flush = () => {
|
|
448
|
+
const firstReal = pieces.length === 0 ? 0 : currentRows[0].srcIdx;
|
|
449
|
+
pieces.push({
|
|
450
|
+
text: [...headerRows, ...currentRows.map((r) => r.row)].join('\n'),
|
|
451
|
+
startLine: firstReal,
|
|
452
|
+
endLine: currentRows[currentRows.length - 1].srcIdx,
|
|
453
|
+
});
|
|
454
|
+
};
|
|
455
|
+
for (let offset = 0; offset < dataRows.length; offset++) {
|
|
456
|
+
const row = dataRows[offset];
|
|
457
|
+
const srcIdx = 2 + offset;
|
|
268
458
|
if (currentLen + row.length + 1 > maxChunk && currentRows.length > 0) {
|
|
269
|
-
|
|
459
|
+
flush();
|
|
270
460
|
currentRows = [];
|
|
271
461
|
currentLen = headerLen;
|
|
272
462
|
}
|
|
273
|
-
currentRows.push(row);
|
|
463
|
+
currentRows.push({ row, srcIdx });
|
|
274
464
|
currentLen += row.length + 1;
|
|
275
465
|
}
|
|
276
466
|
if (currentRows.length > 0) {
|
|
277
|
-
|
|
467
|
+
flush();
|
|
278
468
|
}
|
|
279
469
|
if (pieces.length > 0)
|
|
280
470
|
return pieces;
|
|
@@ -283,29 +473,36 @@ export function splitOversizedBlock(text, maxChunk, blockType) {
|
|
|
283
473
|
const pieces = [];
|
|
284
474
|
let currentLines = [];
|
|
285
475
|
let currentLen = 0;
|
|
286
|
-
|
|
476
|
+
const flush = () => {
|
|
477
|
+
pieces.push({
|
|
478
|
+
text: currentLines.map((l) => l.line).join('\n').trim(),
|
|
479
|
+
startLine: currentLines[0].srcIdx,
|
|
480
|
+
endLine: currentLines[currentLines.length - 1].srcIdx,
|
|
481
|
+
});
|
|
482
|
+
};
|
|
483
|
+
for (let srcIdx = 0; srcIdx < lines.length; srcIdx++) {
|
|
484
|
+
const line = lines[srcIdx];
|
|
287
485
|
if (currentLen + line.length + 1 > maxChunk && currentLines.length > 0) {
|
|
288
|
-
|
|
486
|
+
flush();
|
|
289
487
|
currentLines = [];
|
|
290
488
|
currentLen = 0;
|
|
291
489
|
}
|
|
292
490
|
if (line.length > maxChunk) {
|
|
293
491
|
if (currentLines.length > 0) {
|
|
294
|
-
|
|
492
|
+
flush();
|
|
295
493
|
currentLines = [];
|
|
296
494
|
}
|
|
297
|
-
const
|
|
298
|
-
|
|
299
|
-
pieces.push(piece);
|
|
495
|
+
for (const piece of splitLongLine(line, maxChunk)) {
|
|
496
|
+
pieces.push({ text: piece, startLine: srcIdx, endLine: srcIdx });
|
|
300
497
|
}
|
|
301
498
|
currentLen = 0;
|
|
302
499
|
continue;
|
|
303
500
|
}
|
|
304
|
-
currentLines.push(line);
|
|
501
|
+
currentLines.push({ line, srcIdx });
|
|
305
502
|
currentLen += line.length + 1;
|
|
306
503
|
}
|
|
307
504
|
if (currentLines.length > 0) {
|
|
308
|
-
|
|
505
|
+
flush();
|
|
309
506
|
}
|
|
310
507
|
return pieces;
|
|
311
508
|
}
|
package/dist/mcp.d.ts
CHANGED
|
@@ -4,9 +4,71 @@
|
|
|
4
4
|
* via a zero-dependency JSON-RPC 2.0 stdio transport per the MCP specification.
|
|
5
5
|
*/
|
|
6
6
|
import readline from 'node:readline';
|
|
7
|
+
/**
|
|
8
|
+
* Protocol versions this server can speak (issue #197), newest first.
|
|
9
|
+
* `initialize` echoes the client's requested version when we support it,
|
|
10
|
+
* otherwise answers with MCP_SUPPORTED_VERSIONS[0].
|
|
11
|
+
*/
|
|
12
|
+
export declare const MCP_SUPPORTED_VERSIONS: string[];
|
|
13
|
+
/**
|
|
14
|
+
* Resolves the protocol version to answer an `initialize` with: the client's
|
|
15
|
+
* exact request when supported, else the newest version we speak (the spec's
|
|
16
|
+
* "respond with another protocol version you support" branch).
|
|
17
|
+
*/
|
|
18
|
+
export declare function negotiateProtocolVersion(requested?: unknown): string;
|
|
19
|
+
export interface McpClientChannelRequestOptions {
|
|
20
|
+
timeoutMs?: number;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Reverse channel for server→client JSON-RPC requests (issue #197) — today
|
|
24
|
+
* only `sampling/createMessage`. Both transports build one: stdio writes
|
|
25
|
+
* requests to its stdout and matches response lines by id; HTTP pushes them
|
|
26
|
+
* onto the GET /mcp SSE stream and matches POSTed responses by id.
|
|
27
|
+
*/
|
|
28
|
+
export interface McpClientChannel {
|
|
29
|
+
request(method: string, params?: any, opts?: McpClientChannelRequestOptions): Promise<any>;
|
|
30
|
+
/**
|
|
31
|
+
* Feeds a client message in. Returns true when it resolved a pending
|
|
32
|
+
* request; false means the message is a client request/notification and
|
|
33
|
+
* belongs to the normal handler.
|
|
34
|
+
*/
|
|
35
|
+
handleResponse(msg: any): boolean;
|
|
36
|
+
}
|
|
37
|
+
/** Server→client requests that outlive this wait degrade, not hang (issue #197). */
|
|
38
|
+
export declare const DEFAULT_SAMPLING_TIMEOUT_MS = 60000;
|
|
39
|
+
/** `sampling/createMessage` requires maxTokens — a grounded citation answer fits comfortably. */
|
|
40
|
+
export declare const DEFAULT_SAMPLING_MAX_TOKENS = 1024;
|
|
41
|
+
export declare function samplingTimeoutMs(envValue?: string): number;
|
|
42
|
+
/**
|
|
43
|
+
* Builds a client channel over a raw sink. Ids are prefixed `mdss-srv-` so
|
|
44
|
+
* they can never collide with the client's own request ids in either
|
|
45
|
+
* direction's logs or pending maps.
|
|
46
|
+
*/
|
|
47
|
+
export declare function createClientChannel(send: (msg: any) => void): McpClientChannel;
|
|
48
|
+
/**
|
|
49
|
+
* A sampling-backed synthesis hook for the RAG layer (issue #197): returns
|
|
50
|
+
* null unless the client both declared the `sampling` capability at
|
|
51
|
+
* initialize AND a reverse channel exists — otherwise ask_knowledge_base goes
|
|
52
|
+
* straight to the local extractive answer. Any in-flight failure (client
|
|
53
|
+
* refusal, timeout, non-text content) throws and degrades the same way.
|
|
54
|
+
*/
|
|
55
|
+
export declare function createSamplingSynthesizer(state: {
|
|
56
|
+
channel?: McpClientChannel;
|
|
57
|
+
clientCapabilities?: any;
|
|
58
|
+
}): {
|
|
59
|
+
fn: (req: {
|
|
60
|
+
system: string;
|
|
61
|
+
user: string;
|
|
62
|
+
}) => Promise<string>;
|
|
63
|
+
used: () => boolean;
|
|
64
|
+
} | null;
|
|
7
65
|
export declare const MCP_TOOLS: ({
|
|
8
66
|
name: string;
|
|
9
67
|
description: string;
|
|
68
|
+
annotations: {
|
|
69
|
+
readOnly: boolean;
|
|
70
|
+
idempotent: boolean;
|
|
71
|
+
};
|
|
10
72
|
inputSchema: {
|
|
11
73
|
type: string;
|
|
12
74
|
properties: {
|
|
@@ -44,14 +106,23 @@ export declare const MCP_TOOLS: ({
|
|
|
44
106
|
type: string;
|
|
45
107
|
description: string;
|
|
46
108
|
};
|
|
47
|
-
file?: undefined;
|
|
48
109
|
heading?: undefined;
|
|
110
|
+
fromLine?: undefined;
|
|
111
|
+
maxLines?: undefined;
|
|
112
|
+
file?: undefined;
|
|
113
|
+
direction?: undefined;
|
|
114
|
+
semantic?: undefined;
|
|
115
|
+
sessionId?: undefined;
|
|
49
116
|
};
|
|
50
117
|
required: string[];
|
|
51
118
|
};
|
|
52
119
|
} | {
|
|
53
120
|
name: string;
|
|
54
121
|
description: string;
|
|
122
|
+
annotations: {
|
|
123
|
+
readOnly: boolean;
|
|
124
|
+
idempotent: boolean;
|
|
125
|
+
};
|
|
55
126
|
inputSchema: {
|
|
56
127
|
type: string;
|
|
57
128
|
properties: {
|
|
@@ -67,15 +138,100 @@ export declare const MCP_TOOLS: ({
|
|
|
67
138
|
type: string;
|
|
68
139
|
description: string;
|
|
69
140
|
};
|
|
141
|
+
fromLine?: undefined;
|
|
142
|
+
maxLines?: undefined;
|
|
143
|
+
direction?: undefined;
|
|
144
|
+
semantic?: undefined;
|
|
70
145
|
path?: undefined;
|
|
71
146
|
query?: undefined;
|
|
72
147
|
k?: undefined;
|
|
148
|
+
sessionId?: undefined;
|
|
73
149
|
};
|
|
74
150
|
required: string[];
|
|
75
151
|
};
|
|
76
152
|
} | {
|
|
77
153
|
name: string;
|
|
78
154
|
description: string;
|
|
155
|
+
annotations: {
|
|
156
|
+
readOnly: boolean;
|
|
157
|
+
idempotent: boolean;
|
|
158
|
+
};
|
|
159
|
+
inputSchema: {
|
|
160
|
+
type: string;
|
|
161
|
+
properties: {
|
|
162
|
+
vaults?: undefined;
|
|
163
|
+
tag?: undefined;
|
|
164
|
+
filter?: undefined;
|
|
165
|
+
graphBoost?: undefined;
|
|
166
|
+
heading?: undefined;
|
|
167
|
+
file: {
|
|
168
|
+
type: string;
|
|
169
|
+
description: string;
|
|
170
|
+
};
|
|
171
|
+
fromLine: {
|
|
172
|
+
type: string;
|
|
173
|
+
description: string;
|
|
174
|
+
};
|
|
175
|
+
maxLines: {
|
|
176
|
+
type: string;
|
|
177
|
+
description: string;
|
|
178
|
+
};
|
|
179
|
+
direction?: undefined;
|
|
180
|
+
semantic?: undefined;
|
|
181
|
+
path?: undefined;
|
|
182
|
+
query?: undefined;
|
|
183
|
+
k?: undefined;
|
|
184
|
+
sessionId?: undefined;
|
|
185
|
+
};
|
|
186
|
+
required: string[];
|
|
187
|
+
};
|
|
188
|
+
} | {
|
|
189
|
+
name: string;
|
|
190
|
+
description: string;
|
|
191
|
+
annotations: {
|
|
192
|
+
readOnly: boolean;
|
|
193
|
+
idempotent: boolean;
|
|
194
|
+
};
|
|
195
|
+
inputSchema: {
|
|
196
|
+
type: string;
|
|
197
|
+
properties: {
|
|
198
|
+
vaults?: undefined;
|
|
199
|
+
tag?: undefined;
|
|
200
|
+
filter?: undefined;
|
|
201
|
+
graphBoost?: undefined;
|
|
202
|
+
heading?: undefined;
|
|
203
|
+
fromLine?: undefined;
|
|
204
|
+
maxLines?: undefined;
|
|
205
|
+
file: {
|
|
206
|
+
type: string;
|
|
207
|
+
description: string;
|
|
208
|
+
};
|
|
209
|
+
k: {
|
|
210
|
+
type: string;
|
|
211
|
+
description: string;
|
|
212
|
+
};
|
|
213
|
+
direction: {
|
|
214
|
+
type: string;
|
|
215
|
+
enum: string[];
|
|
216
|
+
description: string;
|
|
217
|
+
};
|
|
218
|
+
semantic: {
|
|
219
|
+
type: string;
|
|
220
|
+
description: string;
|
|
221
|
+
};
|
|
222
|
+
path?: undefined;
|
|
223
|
+
query?: undefined;
|
|
224
|
+
sessionId?: undefined;
|
|
225
|
+
};
|
|
226
|
+
required: string[];
|
|
227
|
+
};
|
|
228
|
+
} | {
|
|
229
|
+
name: string;
|
|
230
|
+
description: string;
|
|
231
|
+
annotations: {
|
|
232
|
+
readOnly: boolean;
|
|
233
|
+
idempotent: boolean;
|
|
234
|
+
};
|
|
79
235
|
inputSchema: {
|
|
80
236
|
type: string;
|
|
81
237
|
properties: {
|
|
@@ -83,20 +239,29 @@ export declare const MCP_TOOLS: ({
|
|
|
83
239
|
tag?: undefined;
|
|
84
240
|
filter?: undefined;
|
|
85
241
|
graphBoost?: undefined;
|
|
86
|
-
file?: undefined;
|
|
87
242
|
heading?: undefined;
|
|
243
|
+
fromLine?: undefined;
|
|
244
|
+
maxLines?: undefined;
|
|
245
|
+
file?: undefined;
|
|
246
|
+
direction?: undefined;
|
|
247
|
+
semantic?: undefined;
|
|
88
248
|
path: {
|
|
89
249
|
type: string;
|
|
90
250
|
description: string;
|
|
91
251
|
};
|
|
92
252
|
query?: undefined;
|
|
93
253
|
k?: undefined;
|
|
254
|
+
sessionId?: undefined;
|
|
94
255
|
};
|
|
95
256
|
required?: undefined;
|
|
96
257
|
};
|
|
97
258
|
} | {
|
|
98
259
|
name: string;
|
|
99
260
|
description: string;
|
|
261
|
+
annotations: {
|
|
262
|
+
readOnly: boolean;
|
|
263
|
+
idempotent: boolean;
|
|
264
|
+
};
|
|
100
265
|
inputSchema: {
|
|
101
266
|
type: string;
|
|
102
267
|
properties: {
|
|
@@ -104,17 +269,26 @@ export declare const MCP_TOOLS: ({
|
|
|
104
269
|
tag?: undefined;
|
|
105
270
|
filter?: undefined;
|
|
106
271
|
graphBoost?: undefined;
|
|
107
|
-
file?: undefined;
|
|
108
272
|
heading?: undefined;
|
|
273
|
+
fromLine?: undefined;
|
|
274
|
+
maxLines?: undefined;
|
|
275
|
+
file?: undefined;
|
|
276
|
+
direction?: undefined;
|
|
277
|
+
semantic?: undefined;
|
|
109
278
|
path?: undefined;
|
|
110
279
|
query?: undefined;
|
|
111
280
|
k?: undefined;
|
|
281
|
+
sessionId?: undefined;
|
|
112
282
|
};
|
|
113
283
|
required?: undefined;
|
|
114
284
|
};
|
|
115
285
|
} | {
|
|
116
286
|
name: string;
|
|
117
287
|
description: string;
|
|
288
|
+
annotations: {
|
|
289
|
+
readOnly: boolean;
|
|
290
|
+
idempotent: boolean;
|
|
291
|
+
};
|
|
118
292
|
inputSchema: {
|
|
119
293
|
type: string;
|
|
120
294
|
properties: {
|
|
@@ -122,8 +296,12 @@ export declare const MCP_TOOLS: ({
|
|
|
122
296
|
tag?: undefined;
|
|
123
297
|
filter?: undefined;
|
|
124
298
|
graphBoost?: undefined;
|
|
125
|
-
file?: undefined;
|
|
126
299
|
heading?: undefined;
|
|
300
|
+
fromLine?: undefined;
|
|
301
|
+
maxLines?: undefined;
|
|
302
|
+
file?: undefined;
|
|
303
|
+
direction?: undefined;
|
|
304
|
+
semantic?: undefined;
|
|
127
305
|
path?: undefined;
|
|
128
306
|
query: {
|
|
129
307
|
type: string;
|
|
@@ -133,16 +311,46 @@ export declare const MCP_TOOLS: ({
|
|
|
133
311
|
type: string;
|
|
134
312
|
description: string;
|
|
135
313
|
};
|
|
314
|
+
sessionId: {
|
|
315
|
+
type: string;
|
|
316
|
+
description: string;
|
|
317
|
+
};
|
|
136
318
|
};
|
|
137
319
|
required: string[];
|
|
138
320
|
};
|
|
139
321
|
})[];
|
|
140
|
-
export declare
|
|
322
|
+
export declare const MCP_RESOURCE_TEMPLATES: {
|
|
323
|
+
uriTemplate: string;
|
|
324
|
+
name: string;
|
|
325
|
+
description: string;
|
|
326
|
+
mimeType: string;
|
|
327
|
+
}[];
|
|
328
|
+
export declare const MCP_PROMPTS: {
|
|
329
|
+
name: string;
|
|
330
|
+
description: string;
|
|
331
|
+
arguments: {
|
|
332
|
+
name: string;
|
|
333
|
+
description: string;
|
|
334
|
+
required: boolean;
|
|
335
|
+
}[];
|
|
336
|
+
}[];
|
|
337
|
+
/**
|
|
338
|
+
* Shared state for both MCP transports (stdio + Streamable HTTP): the
|
|
339
|
+
* transports own the extras — `channel` (server→client requests, issue #197),
|
|
340
|
+
* `clientCapabilities` (recorded at initialize), `output` (stdio sink, test seam).
|
|
341
|
+
*/
|
|
342
|
+
export interface McpHandlerState {
|
|
141
343
|
loaded: any;
|
|
142
344
|
cacheDir: string;
|
|
143
345
|
offline: boolean;
|
|
144
346
|
embedFn?: any;
|
|
145
|
-
|
|
347
|
+
channel?: McpClientChannel;
|
|
348
|
+
clientCapabilities?: any;
|
|
349
|
+
output?: {
|
|
350
|
+
write(chunk: string): void;
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
export declare function handleMcpRequest(req: any, state: McpHandlerState): Promise<any | null>;
|
|
146
354
|
export interface StartMcpServerOptions {
|
|
147
355
|
indexDir: string;
|
|
148
356
|
cacheDir: string;
|
|
@@ -151,9 +359,23 @@ export interface StartMcpServerOptions {
|
|
|
151
359
|
ignore?: string[];
|
|
152
360
|
offline?: boolean;
|
|
153
361
|
embedFn?: any;
|
|
362
|
+
/** Stdio source (default process.stdin) — test/embedding seam (issue #197). */
|
|
363
|
+
input?: NodeJS.ReadableStream;
|
|
364
|
+
/** Stdio sink (default process.stdout) — test/embedding seam (issue #197). */
|
|
365
|
+
output?: {
|
|
366
|
+
write(chunk: string): void;
|
|
367
|
+
};
|
|
154
368
|
log?: (msg: string) => void;
|
|
155
369
|
}
|
|
156
370
|
export declare function startMcpServer(opts: StartMcpServerOptions): Promise<{
|
|
157
|
-
state:
|
|
371
|
+
state: McpHandlerState;
|
|
158
372
|
rl: readline.Interface;
|
|
159
373
|
}>;
|
|
374
|
+
/**
|
|
375
|
+
* One guarded, in-order stdio JSON-RPC exchange (issues #180/#185): caps the
|
|
376
|
+
* line length, turns JSON parse failures into `-32700`, and converts ANY
|
|
377
|
+
* handler escape into a `-32603` response instead of an unhandled rejection
|
|
378
|
+
* that would kill the daemon.
|
|
379
|
+
*/
|
|
380
|
+
export declare const MAX_STDIO_LINE_CHARS: number;
|
|
381
|
+
export declare function handleStdioLine(line: string, state: McpHandlerState): Promise<void>;
|