md-semantic-search 0.3.0 → 0.5.0

package/dist/lexical.js

@@ -0,0 +1,226 @@
+import crypto from 'node:crypto';
+const STOP = new Set([
+    'the', 'and', 'for', 'are', 'was', 'has', 'with', 'this', 'that', 'from',
+    'not', 'but', 'you', 'your', 'can', 'all', 'any', 'its',
+    'все', 'как', 'что', 'это', 'при', 'для', 'или', 'был', 'без', 'над',
+    'под', 'так', 'его', 'нет', 'есть',
+    'of', 'to', 'in', 'on', 'it', 'is', 'at', 'by', 'be', 'we', 'us', 'he',
+    'as', 'or', 'an', 'do', 'so', 'no', 'if', 'up', 'my', 'me', 'am',
+    'по', 'от', 'из', 'на', 'за', 'во', 'со', 'не', 'ни', 'же', 'ли', 'бы',
+    'уж', 'мы', 'вы', 'он', 'то', 'но', 'до', 'ко',
+]);
+export const _lexicalStats = { documentsAnalyzed: 0 };
+function normalize(text) {
+    return (text ?? '').replace(/\r\n?/g, '\n').trim();
+}
+function equivalent(left, right) {
+    return left.toLowerCase() === right.toLowerCase();
+}
+export function lexicalDocument(chunk) {
+    const title = normalize(chunk.title);
+    const heading = normalize(chunk.heading);
+    const ancestors = (chunk.headingPath ?? (heading ? [chunk.heading] : [])).map(normalize);
+    if (ancestors.length > 0 && equivalent(ancestors[0], title))
+        ancestors.shift();
+    if (ancestors.length > 0 && equivalent(ancestors[ancestors.length - 1], heading))
+        ancestors.pop();
+    return [title, ...ancestors, heading, normalize(chunk.text)].join('\n');
+}
+export function lexicalIdentity(chunk) {
+    return crypto.createHash('sha256').update(lexicalDocument(chunk)).digest('hex');
+}
+export function tokenize(text) {
+    const matches = text.toLowerCase().match(/[\p{L}\p{N}][\p{L}\p{N}#+-]*/gu);
+    if (!matches)
+        return [];
+    return matches.filter((term) => term.length > 1 && !STOP.has(term));
+}
+export const DEFAULT_FIELD_WEIGHTS = {
+    title: 3.0,
+    aliases: 3.0,
+    headingPath: 1.8,
+    body: 1.0,
+};
+export function analyzeLexicalDocument(chunk, weights = DEFAULT_FIELD_WEIGHTS) {
+    _lexicalStats.documentsAnalyzed++;
+    const frequencies = Object.create(null);
+    const titleText = chunk.title || '';
+    const aliasesText = (chunk.meta?.aliases || []).join(' ');
+    const headingText = (chunk.headingPath || [chunk.heading]).join(' ');
+    const bodyText = chunk.text || '';
+    const addTokens = (text, weight) => {
+        for (const term of tokenize(text)) {
+            frequencies[term] = (frequencies[term] ?? 0) + weight;
+        }
+    };
+    addTokens(titleText, weights.title);
+    if (aliasesText)
+        addTokens(aliasesText, weights.aliases);
+    if (headingText)
+        addTokens(headingText, weights.headingPath);
+    addTokens(bodyText, weights.body);
+    return frequencies;
+}
+export function buildLexicalIndex(records) {
+    const postings = Object.create(null);
+    const documentLengths = records.map((record, docId) => {
+        let length = 0;
+        for (const [term, tf] of Object.entries(record)) {
+            length += tf;
+            const posting = postings[term] ?? [];
+            posting.push([docId, tf]);
+            postings[term] = posting;
+        }
+        return Number(length.toFixed(3));
+    });
+    return { format: 'bm25-v2', documentLengths, postings };
+}
+export function editDistance(a, b) {
+    if (a === b)
+        return 0;
+    const alen = a.length;
+    const blen = b.length;
+    if (alen === 0)
+        return blen;
+    if (blen === 0)
+        return alen;
+    const matrix = Array.from({ length: alen + 1 }, () => new Array(blen + 1).fill(0));
+    for (let i = 0; i <= alen; i++)
+        matrix[i][0] = i;
+    for (let j = 0; j <= blen; j++)
+        matrix[0][j] = j;
+    for (let i = 1; i <= alen; i++) {
+        for (let j = 1; j <= blen; j++) {
+            const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+            matrix[i][j] = Math.min(matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + cost);
+            if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) {
+                matrix[i][j] = Math.min(matrix[i][j], matrix[i - 2][j - 2] + cost);
+            }
+        }
+    }
+    return matrix[alen][blen];
+}
+export function fuzzyTitleAliasScores(chunks, query) {
+    const scores = new Map();
+    const qTerms = tokenize(query);
+    if (qTerms.length === 0)
+        return scores;
+    for (let docId = 0; docId < chunks.length; docId++) {
+        const c = chunks[docId];
+        const targets = [c.title, ...(c.meta?.aliases || [])].filter(Boolean);
+        let maxScore = 0;
+        for (const target of targets) {
+            const tTerms = tokenize(target);
+            for (const qt of qTerms) {
+                if (qt.length < 3)
+                    continue;
+                for (const tt of tTerms) {
+                    if (tt.length < 3)
+                        continue;
+                    const maxDist = Math.min(2, Math.floor(qt.length / 3));
+                    const dist = editDistance(qt, tt);
+                    if (dist <= maxDist) {
+                        const similarity = 1 - dist / Math.max(qt.length, tt.length);
+                        if (similarity > maxScore)
+                            maxScore = similarity;
+                    }
+                }
+            }
+        }
+        if (maxScore > 0) {
+            scores.set(docId, maxScore * 2.0);
+        }
+    }
+    return scores;
+}
+export function validateLexicalIndex(value, chunkCount) {
+    if (value === null || typeof value !== 'object' || Array.isArray(value)) {
+        return 'lexical must be an object';
+    }
+    const lexical = value;
+    if (lexical.format !== 'bm25-v1' && lexical.format !== 'bm25-v2') {
+        return 'unknown lexical format';
+    }
+    if (!Array.isArray(lexical.documentLengths) ||
+        lexical.documentLengths.length !== chunkCount ||
+        !lexical.documentLengths.every((length) => Number.isFinite(length) && length >= 0)) {
+        return `documentLengths must contain ${chunkCount} non-negative numbers`;
+    }
+    if (lexical.postings === null || typeof lexical.postings !== 'object' || Array.isArray(lexical.postings)) {
+        return 'postings must be an object';
+    }
+    const sums = new Array(chunkCount).fill(0);
+    for (const [term, rawPosting] of Object.entries(lexical.postings)) {
+        if (!Array.isArray(rawPosting))
+            return `posting for ${term} must be an array`;
+        let previous = -1;
+        for (const entry of rawPosting) {
+            if (!Array.isArray(entry) || entry.length !== 2)
+                return `posting for ${term} has an invalid entry`;
+            const [docId, tf] = entry;
+            if (!Number.isSafeInteger(docId)) {
+                return `posting for ${term} must have a safe integer document ID`;
+            }
+            if (docId < 0 || docId >= chunkCount) {
+                return `posting for ${term} has a document ID out of range`;
+            }
+            if (docId <= previous)
+                return `posting for ${term} document IDs must be strictly increasing`;
+            if (!Number.isFinite(tf) || tf <= 0)
+                return `posting for ${term} must have positive TF`;
+            sums[docId] += tf;
+            previous = docId;
+        }
+    }
+    for (let docId = 0; docId < chunkCount; docId++) {
+        if (Math.abs(sums[docId] - lexical.documentLengths[docId]) > 1e-3) {
+            return `document ${docId} TF sum does not equal documentLengths`;
+        }
+    }
+    return null;
+}
+export function reverseLexicalIndex(lexical) {
+    const records = lexical.documentLengths.map(() => Object.create(null));
+    for (const [term, posting] of Object.entries(lexical.postings)) {
+        for (const [docId, tf] of posting)
+            records[docId][term] = tf;
+    }
+    return records;
+}
+export function bm25Scores(lexical, queryTerms, eligible) {
+    const scores = new Map();
+    const documentCount = lexical.documentLengths.length;
+    if (documentCount === 0)
+        return scores;
+    const averageLength = lexical.documentLengths.reduce((sum, length) => sum + length, 0) / documentCount || 1;
+    if (!Number.isFinite(averageLength))
+        throw new Error('non-finite BM25 average document length');
+    for (const term of new Set(queryTerms)) {
+        const posting = Object.prototype.hasOwnProperty.call(lexical.postings, term) ? lexical.postings[term] : undefined;
+        if (!posting)
+            continue;
+        const idf = Math.log(1 + (documentCount - posting.length + 0.5) / (posting.length + 0.5));
+        if (!Number.isFinite(idf))
+            throw new Error(`non-finite BM25 IDF for ${term}`);
+        for (const [docId, tf] of posting) {
+            if (eligible && !eligible.has(docId))
+                continue;
+            const lengthRatio = lexical.documentLengths[docId] / averageLength;
+            const score = (idf * (tf * 2.2)) / (tf + 1.2 * (0.25 + 0.75 * lengthRatio));
+            if (!Number.isFinite(lengthRatio) || !Number.isFinite(score)) {
+                throw new Error(`non-finite BM25 score for ${term}`);
+            }
+            scores.set(docId, (scores.get(docId) ?? 0) + score);
+        }
+    }
+    return scores;
+}
+export function matchingTerms(lexical, queryTerms, docId) {
+    const matches = [];
+    for (const term of new Set(queryTerms)) {
+        const posting = Object.prototype.hasOwnProperty.call(lexical.postings, term) ? lexical.postings[term] : undefined;
+        if (posting?.some(([candidate]) => candidate === docId))
+            matches.push(term);
+    }
+    return matches;
+}

package/dist/markdown-parser.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Structural Markdown parser and block chunker (issue #57).
+ * Parses Markdown source into structured blocks (headings, fenced code,
+ * tables, blockquotes, lists, paragraphs) and emits chunk records.
+ */
+export declare const PARSER_VERSION = "v1-ast";
+export interface MarkdownBlock {
+    type: 'atx_heading' | 'setext_heading' | 'code' | 'table' | 'blockquote' | 'list' | 'paragraph';
+    level?: number;
+    headingText?: string;
+    text: string;
+    startLine: number;
+    endLine: number;
+}
+export interface StructuralChunk {
+    heading: string;
+    headingPath: string[];
+    text: string;
+    startLine: number;
+    endLine: number;
+}
+export declare function parseMarkdownBlocks(body: string): MarkdownBlock[];
+export declare function resolveChunkBudget(tokens?: number, defaultMax?: number): number;
+export declare function chunkMarkdownStructural(input: string | MarkdownBlock[], opts?: number | {
+    maxChunk?: number;
+    targetTokens?: number;
+}): StructuralChunk[];

package/dist/markdown-parser.js

@@ -0,0 +1,252 @@
+/**
+ * Structural Markdown parser and block chunker (issue #57).
+ * Parses Markdown source into structured blocks (headings, fenced code,
+ * tables, blockquotes, lists, paragraphs) and emits chunk records.
+ */
+export const PARSER_VERSION = 'v1-ast';
+export function parseMarkdownBlocks(body) {
+    const lines = body.split(/\r?\n/);
+    const blocks = [];
+    let i = 0;
+    while (i < lines.length) {
+        const line = lines[i];
+        const lineNum = i + 1;
+        const fenceMatch = line.match(/^(\s*)(`{3,}|~{3,})(.*)$/);
+        if (fenceMatch) {
+            const fenceChar = fenceMatch[2][0];
+            const fenceLen = fenceMatch[2].length;
+            const startLine = lineNum;
+            const codeLines = [line];
+            i++;
+            while (i < lines.length) {
+                const curLine = lines[i];
+                codeLines.push(curLine);
+                const closeMatch = curLine.match(/^(\s*)(`{3,}|~{3,})\s*$/);
+                if (closeMatch && closeMatch[2][0] === fenceChar && closeMatch[2].length >= fenceLen) {
+                    i++;
+                    break;
+                }
+                i++;
+            }
+            blocks.push({
+                type: 'code',
+                text: codeLines.join('\n'),
+                startLine,
+                endLine: i,
+            });
+            continue;
+        }
+        const atxMatch = line.match(/^(\s{0,3})(#{1,6})\s+(.+?)\s*#*\s*$/);
+        if (atxMatch) {
+            const level = atxMatch[2].length;
+            const headingText = atxMatch[3].trim();
+            blocks.push({
+                type: 'atx_heading',
+                level,
+                headingText,
+                text: line,
+                startLine: lineNum,
+                endLine: lineNum,
+            });
+            i++;
+            continue;
+        }
+        if (i + 1 < lines.length &&
+            line.trim().length > 0 &&
+            !/^\s*(?:[#>]|```|~~~|[-*+]\s|\d+\.\s)/.test(line)) {
+            const nextLine = lines[i + 1];
+            const setextMatch = nextLine.match(/^(\s{0,3})(=+|-+)\s*$/);
+            if (setextMatch) {
+                const level = setextMatch[2][0] === '=' ? 1 : 2;
+                const headingText = line.trim();
+                blocks.push({
+                    type: 'setext_heading',
+                    level,
+                    headingText,
+                    text: `${line}\n${nextLine}`,
+                    startLine: lineNum,
+                    endLine: lineNum + 1,
+                });
+                i += 2;
+                continue;
+            }
+        }
+        if (line.trim() === '') {
+            i++;
+            continue;
+        }
+        if (/^\s*\|/.test(line) || (/\|/.test(line) && i + 1 < lines.length && /^\s*\|?\s*[-:]+[-|\s:]*$/.test(lines[i + 1]))) {
+            const startLine = lineNum;
+            const tableLines = [];
+            while (i < lines.length && (lines[i].includes('|') || /^\s*\|?\s*[-:]+[-|\s:]*$/.test(lines[i]))) {
+                tableLines.push(lines[i]);
+                i++;
+            }
+            blocks.push({
+                type: 'table',
+                text: tableLines.join('\n'),
+                startLine,
+                endLine: i,
+            });
+            continue;
+        }
+        if (/^\s*>/.test(line)) {
+            const startLine = lineNum;
+            const bqLines = [];
+            while (i < lines.length && (/^\s*>/.test(lines[i]) || (lines[i].trim() !== '' && !/^\s*(?:#{1,6}\s|```|~~~|[-*+]\s|\d+\.\s)/.test(lines[i])))) {
+                bqLines.push(lines[i]);
+                i++;
+            }
+            blocks.push({
+                type: 'blockquote',
+                text: bqLines.join('\n'),
+                startLine,
+                endLine: i,
+            });
+            continue;
+        }
+        const startLine = lineNum;
+        const paraLines = [line];
+        i++;
+        while (i < lines.length) {
+            const l = lines[i];
+            if (l.trim() === '' ||
+                /^\s{0,3}#{1,6}\s/.test(l) ||
+                /^\s*(?:`{3,}|~{3,})/.test(l) ||
+                /^\s*>/.test(l) ||
+                /^\s*\|/.test(l)) {
+                break;
+            }
+            if (i + 1 < lines.length &&
+                /^\s{0,3}(?:=+|-+)\s*$/.test(lines[i + 1])) {
+                break;
+            }
+            paraLines.push(l);
+            i++;
+        }
+        blocks.push({
+            type: 'paragraph',
+            text: paraLines.join('\n'),
+            startLine,
+            endLine: i,
+        });
+    }
+    return blocks;
+}
+export function resolveChunkBudget(tokens, defaultMax = 1200) {
+    if (tokens && Number.isFinite(tokens) && tokens > 0) {
+        return Math.max(100, Math.min(8192, Math.round(tokens * 4)));
+    }
+    return defaultMax;
+}
+export function chunkMarkdownStructural(input, opts) {
+    const options = typeof opts === 'number' ? { maxChunk: opts } : (opts || {});
+    const maxChunk = resolveChunkBudget(options.targetTokens, options.maxChunk ?? 1200);
+    const blocks = typeof input === 'string' ? parseMarkdownBlocks(input) : input;
+    const chunks = [];
+    const headingStack = [];
+    let currentGroup = [];
+    let currentLen = 0;
+    let groupStartLine = 0;
+    let groupEndLine = 0;
+    const flushGroup = () => {
+        if (currentGroup.length === 0)
+            return;
+        const text = currentGroup.join('\n\n').trim();
+        if (text) {
+            const headingPath = headingStack.map(h => h.heading);
+            chunks.push({
+                heading: headingPath.at(-1) ?? '',
+                headingPath,
+                text,
+                startLine: groupStartLine,
+                endLine: groupEndLine,
+            });
+        }
+        currentGroup = [];
+        currentLen = 0;
+    };
+    for (const block of blocks) {
+        if (block.type === 'atx_heading' || block.type === 'setext_heading') {
+            flushGroup();
+            const level = block.level || 1;
+            const heading = block.headingText || '';
+            while (headingStack.length > 0 && headingStack[headingStack.length - 1].level >= level) {
+                headingStack.pop();
+            }
+            headingStack.push({ level, heading });
+            continue;
+        }
+        const blockText = block.text.trim();
+        if (!blockText)
+            continue;
+        if (blockText.length > maxChunk) {
+            flushGroup();
+            const headingPath = headingStack.map(h => h.heading);
+            const subPieces = splitOversizedBlock(blockText, maxChunk);
+            let lineCursor = block.startLine;
+            for (let idx = 0; idx < subPieces.length; idx++) {
+                const piece = subPieces[idx];
+                const pieceLineCount = piece.split('\n').length;
+                chunks.push({
+                    heading: headingPath.at(-1) ?? '',
+                    headingPath: [...headingPath],
+                    text: piece,
+                    startLine: lineCursor,
+                    endLine: Math.min(block.endLine, lineCursor + pieceLineCount - 1),
+                });
+                lineCursor += pieceLineCount;
+            }
+            continue;
+        }
+        const addedLen = currentLen === 0 ? blockText.length : currentLen + 2 + blockText.length;
+        if (addedLen > maxChunk && currentGroup.length > 0) {
+            flushGroup();
+        }
+        if (currentGroup.length === 0) {
+            groupStartLine = block.startLine;
+        }
+        currentGroup.push(blockText);
+        currentLen += (currentGroup.length === 1 ? 0 : 2) + blockText.length;
+        groupEndLine = block.endLine;
+    }
+    flushGroup();
+    return chunks.filter(c => c.text.replace(/\s/g, '').length >= 24);
+}
+function splitOversizedBlock(text, maxChunk) {
+    const lines = text.split('\n');
+    const pieces = [];
+    let currentLines = [];
+    let currentLen = 0;
+    for (const line of lines) {
+        if (currentLen + line.length + 1 > maxChunk && currentLines.length > 0) {
+            pieces.push(currentLines.join('\n').trim());
+            currentLines = [];
+            currentLen = 0;
+        }
+        if (line.length > maxChunk) {
+            if (currentLines.length > 0) {
+                pieces.push(currentLines.join('\n').trim());
+                currentLines = [];
+            }
+            let rest = line;
+            while (rest.length > maxChunk) {
+                let cut = rest.lastIndexOf(' ', maxChunk);
+                if (cut <= 0)
+                    cut = maxChunk;
+                pieces.push(rest.slice(0, cut).trim());
+                rest = rest.slice(cut).trim();
+            }
+            if (rest)
+                currentLines.push(rest);
+            currentLen = rest.length;
+            continue;
+        }
+        currentLines.push(line);
+        currentLen += line.length + 1;
+    }
+    if (currentLines.length > 0) {
+        pieces.push(currentLines.join('\n').trim());
+    }
+    return pieces;
+}

package/dist/mcp.d.ts

@@ -0,0 +1,111 @@
+/**
+ * MCP (Model Context Protocol) server mode (issue #62).
+ * Exposes local semantic search to IDEs and LLM agents (Cursor, Claude Desktop, Copilot)
+ * via a zero-dependency JSON-RPC 2.0 stdio transport per the MCP specification.
+ */
+import readline from 'node:readline';
+export declare const MCP_TOOLS: ({
+    name: string;
+    description: string;
+    inputSchema: {
+        type: string;
+        properties: {
+            query: {
+                type: string;
+                description: string;
+            };
+            k: {
+                type: string;
+                description: string;
+            };
+            path: {
+                type: string;
+                description: string;
+            };
+            tag: {
+                type: string;
+                items: {
+                    type: string;
+                };
+                description: string;
+            };
+            file?: undefined;
+            heading?: undefined;
+        };
+        required: string[];
+    };
+} | {
+    name: string;
+    description: string;
+    inputSchema: {
+        type: string;
+        properties: {
+            query?: undefined;
+            k?: undefined;
+            tag?: undefined;
+            file: {
+                type: string;
+                description: string;
+            };
+            heading: {
+                type: string;
+                description: string;
+            };
+            path?: undefined;
+        };
+        required: string[];
+    };
+} | {
+    name: string;
+    description: string;
+    inputSchema: {
+        required?: undefined;
+        type: string;
+        properties: {
+            query?: undefined;
+            k?: undefined;
+            tag?: undefined;
+            file?: undefined;
+            heading?: undefined;
+            path: {
+                type: string;
+                description: string;
+            };
+        };
+    };
+} | {
+    name: string;
+    description: string;
+    inputSchema: {
+        required?: undefined;
+        type: string;
+        properties: {
+            query?: undefined;
+            k?: undefined;
+            tag?: undefined;
+            file?: undefined;
+            heading?: undefined;
+            path?: undefined;
+        };
+    };
+})[];
+export declare function handleMcpRequest(req: any, state: {
+    loaded: any;
+    cacheDir: string;
+    offline: boolean;
+    embedFn?: any;
+}): Promise<any | null>;
+export interface StartMcpServerOptions {
+    indexDir: string;
+    cacheDir: string;
+    db?: string;
+    modelName?: string;
+    ignore?: string[];
+    offline?: boolean;
+    embedFn?: any;
+    log?: (msg: string) => void;
+}
+export declare function startMcpServer(opts: StartMcpServerOptions): Promise<{
+    state: any;
+    rl: readline.Interface;
+}>;