md-semantic-search 0.3.0 → 0.5.0

package/dist/mcp.js

@@ -0,0 +1,216 @@
+/**
+ * MCP (Model Context Protocol) server mode (issue #62).
+ * Exposes local semantic search to IDEs and LLM agents (Cursor, Claude Desktop, Copilot)
+ * via a zero-dependency JSON-RPC 2.0 stdio transport per the MCP specification.
+ */
+import readline from 'node:readline';
+import fs from 'node:fs';
+import path from 'node:path';
+import { buildIndex } from './indexer.js';
+import { loadIndex, searchIndex } from './search.js';
+import { globToRegExp } from './core.js';
+export const MCP_TOOLS = [
+    {
+        name: 'search_markdown',
+        description: 'Search local Markdown knowledge base by meaning and hybrid BM25 lexical ranking',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                query: { type: 'string', description: 'Natural language search query' },
+                k: { type: 'number', description: 'Number of top results to return (default: 6)' },
+                path: { type: 'string', description: 'Optional path glob filter (e.g. "docs/**")' },
+                tag: { type: 'array', items: { type: 'string' }, description: 'Optional frontmatter tag filters' },
+            },
+            required: ['query'],
+        },
+    },
+    {
+        name: 'get_chunk',
+        description: 'Retrieve a specific chunk or section by file path and optional heading label',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                file: { type: 'string', description: 'Relative file path (e.g. "guides/api.md")' },
+                heading: { type: 'string', description: 'Heading title (e.g. "Authentication")' },
+            },
+            required: ['file'],
+        },
+    },
+    {
+        name: 'list_files',
+        description: 'List all indexed Markdown files and their chunk counts',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                path: { type: 'string', description: 'Optional glob filter to match file paths' },
+            },
+        },
+    },
+    {
+        name: 'index_status',
+        description: 'Get current index status, model metadata, chunk count, and build timestamp',
+        inputSchema: {
+            type: 'object',
+            properties: {},
+        },
+    },
+];
+export async function handleMcpRequest(req, state) {
+    if (!req || typeof req !== 'object')
+        return null;
+    const { id, method, params } = req;
+    if (id === undefined || id === null) {
+        return null;
+    }
+    if (method === 'initialize') {
+        return {
+            jsonrpc: '2.0',
+            id,
+            result: {
+                protocolVersion: '2024-11-05',
+                capabilities: {
+                    tools: {},
+                },
+                serverInfo: {
+                    name: 'md-semantic-search',
+                    version: '0.4.0',
+                },
+            },
+        };
+    }
+    if (method === 'tools/list') {
+        return {
+            jsonrpc: '2.0',
+            id,
+            result: {
+                tools: MCP_TOOLS,
+            },
+        };
+    }
+    if (method === 'tools/call') {
+        const name = params?.name;
+        const args = params?.arguments || {};
+        try {
+            let content = [];
+            if (name === 'search_markdown') {
+                const query = typeof args.query === 'string' ? args.query : '';
+                const k = Number.isInteger(args.k) && args.k > 0 ? args.k : 6;
+                const results = await searchIndex({
+                    loaded: state.loaded,
+                    cacheDir: state.cacheDir,
+                    query,
+                    k,
+                    path: args.path,
+                    tag: args.tag,
+                    offline: state.offline,
+                    embedFn: state.embedFn,
+                });
+                content = [{ type: 'text', text: JSON.stringify(results, null, 2) }];
+            }
+            else if (name === 'get_chunk') {
+                const file = args.file;
+                const heading = args.heading;
+                const chunks = state.loaded.index.chunks.filter((c) => {
+                    if (c.file !== file)
+                        return false;
+                    if (heading && c.heading.toLowerCase() !== heading.toLowerCase())
+                        return false;
+                    return true;
+                });
+                content = [{ type: 'text', text: JSON.stringify(chunks, null, 2) }];
+            }
+            else if (name === 'list_files') {
+                const fileCounts = new Map();
+                for (const c of state.loaded.index.chunks) {
+                    fileCounts.set(c.file, (fileCounts.get(c.file) || 0) + 1);
+                }
+                let files = [...fileCounts.entries()].map(([file, count]) => ({ file, chunksCount: count }));
+                if (args.path) {
+                    const re = globToRegExp(args.path);
+                    files = files.filter((f) => re.test(f.file));
+                }
+                content = [{ type: 'text', text: JSON.stringify(files, null, 2) }];
+            }
+            else if (name === 'index_status') {
+                const info = {
+                    ok: true,
+                    chunks: state.loaded.index.chunks.length,
+                    model: state.loaded.model.id,
+                    dim: state.loaded.index.dim || state.loaded.model.dim || 0,
+                    built: state.loaded.index.built || null,
+                };
+                content = [{ type: 'text', text: JSON.stringify(info, null, 2) }];
+            }
+            else {
+                return {
+                    jsonrpc: '2.0',
+                    id,
+                    error: { code: -32601, message: `Unknown tool: ${name}` },
+                };
+            }
+            return {
+                jsonrpc: '2.0',
+                id,
+                result: { content },
+            };
+        }
+        catch (e) {
+            return {
+                jsonrpc: '2.0',
+                id,
+                result: {
+                    isError: true,
+                    content: [{ type: 'text', text: `Error: ${e.message}` }],
+                },
+            };
+        }
+    }
+    return {
+        jsonrpc: '2.0',
+        id,
+        error: { code: -32601, message: `Method not found: ${method}` },
+    };
+}
+export async function startMcpServer(opts) {
+    const { indexDir, cacheDir, db, modelName = 'e5-base', ignore = [], offline = false, embedFn, log = () => { }, } = opts;
+    fs.mkdirSync(indexDir, { recursive: true });
+    if (!fs.existsSync(path.join(indexDir, 'vectors.json'))) {
+        if (!db)
+            throw new Error('mcp: no index found and no --db given to build one');
+        log(`Building index at ${indexDir} from ${db}…`);
+        await buildIndex({ db, indexDir, cacheDir, modelName, ignore, offline, log, embedFn });
+    }
+    const state = {
+        loaded: loadIndex(indexDir),
+        cacheDir,
+        offline,
+        embedFn,
+    };
+    const rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout,
+        terminal: false,
+    });
+    rl.on('line', async (line) => {
+        const trimmed = line.trim();
+        if (!trimmed)
+            return;
+        let req;
+        try {
+            req = JSON.parse(trimmed);
+        }
+        catch {
+            process.stdout.write(JSON.stringify({
+                jsonrpc: '2.0',
+                id: null,
+                error: { code: -32700, message: 'Parse error: invalid JSON' },
+            }) + '\n');
+            return;
+        }
+        const res = await handleMcpRequest(req, state);
+        if (res) {
+            process.stdout.write(JSON.stringify(res) + '\n');
+        }
+    });
+    return { state, rl };
+}

package/dist/metrics.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Retrieval evaluation metrics (issue #56).
+ * Pure, dependency-free ranking metrics used by the golden-set harness.
+ */
+export interface BenchHit {
+    file: string;
+}
+export interface GradeConf {
+    grades?: number[];
+    minRelevant?: number;
+}
+export interface MetricValues {
+    ndcg: number;
+    mrr: number;
+    hit: number;
+    recall: number;
+}
+export declare function ndcgAtK(ranked: string[], grade: (doc: string, rankIndex: number) => number, k: number): number;
+export declare function reciprocalRank(ranked: string[], isRelevant: (doc: string) => boolean): number;
+export declare function hitAtK(ranked: string[], isRelevant: (doc: string) => boolean, k: number): 0 | 1;
+export declare function recallAtK(ranked: string[], isRelevant: (doc: string) => boolean, k: number): number;
+export declare function gradeByDoc(docGrades: Record<string, number>): (doc: string) => number;
+export declare function queryMetrics(ranked: string[], docGrades: Record<string, number>, opts?: {
+    k?: number;
+    minRelevant?: number;
+    allRelevant?: string[];
+}): MetricValues;
+export declare function aggregateMetrics(qm: MetricValues[]): {
+    mean: MetricValues;
+    n: number;
+};
+export declare function winTieLoss(a: Array<{
+    ndcg: number;
+}>, b: Array<{
+    ndcg: number;
+}>, epsilon?: number): {
+    win: number;
+    tie: number;
+    loss: number;
+};
+export declare function redundancyAtK(ranked: Array<{
+    file: string;
+}>, k: number): number;

package/dist/metrics.js

@@ -0,0 +1,97 @@
+/**
+ * Retrieval evaluation metrics (issue #56).
+ * Pure, dependency-free ranking metrics used by the golden-set harness.
+ */
+export function ndcgAtK(ranked, grade, k) {
+    const top = ranked.slice(0, k);
+    if (top.length === 0)
+        return 0;
+    let dcg = 0;
+    for (let i = 0; i < top.length; i++) {
+        const g = grade(top[i], i);
+        if (g <= 0)
+            continue;
+        dcg += g / Math.log2(i + 2);
+    }
+    if (dcg === 0)
+        return 0;
+    const all = (ranked ?? []).map((doc, i) => grade(doc, i)).filter((g) => g > 0);
+    const ideal = [...all].sort((a, b) => b - a).slice(0, k);
+    let idcg = 0;
+    for (let i = 0; i < ideal.length; i++)
+        idcg += ideal[i] / Math.log2(i + 2);
+    return idcg === 0 ? 0 : dcg / idcg;
+}
+export function reciprocalRank(ranked, isRelevant) {
+    for (let i = 0; i < ranked.length; i++) {
+        if (isRelevant(ranked[i]))
+            return 1 / (i + 1);
+    }
+    return 0;
+}
+export function hitAtK(ranked, isRelevant, k) {
+    return ranked.slice(0, k).some(isRelevant) ? 1 : 0;
+}
+export function recallAtK(ranked, isRelevant, k) {
+    const relevantCount = (ranked ?? []).filter(isRelevant).length;
+    if (relevantCount === 0)
+        return 0;
+    const retrieved = ranked.slice(0, k).filter(isRelevant).length;
+    return retrieved / relevantCount;
+}
+export function gradeByDoc(docGrades) {
+    const table = Object.create(null);
+    for (const [doc, g] of Object.entries(docGrades)) {
+        table[doc] = g;
+    }
+    return (doc) => table[doc] ?? 0;
+}
+export function queryMetrics(ranked, docGrades, opts = {}) {
+    const { k = 10, minRelevant = 1 } = opts;
+    const grade = gradeByDoc(docGrades);
+    const isRelevant = (doc) => grade(doc) >= minRelevant;
+    const allRelevant = opts.allRelevant ?? Object.entries(docGrades).filter(([, g]) => g >= minRelevant).map(([d]) => d);
+    const fullRanked = ranked.length >= k ? ranked : [...ranked, ...allRelevant.filter((d) => !ranked.includes(d))];
+    return {
+        ndcg: ndcgAtK(ranked, grade, k),
+        mrr: reciprocalRank(ranked, isRelevant),
+        hit: hitAtK(ranked, isRelevant, k),
+        recall: recallAtK(fullRanked, isRelevant, k),
+    };
+}
+export function aggregateMetrics(qm) {
+    const n = qm.length;
+    if (n === 0)
+        return { mean: { ndcg: 0, mrr: 0, hit: 0, recall: 0 }, n: 0 };
+    const sum = (key) => qm.reduce((acc, m) => acc + m[key], 0) / n;
+    return {
+        n,
+        mean: { ndcg: sum('ndcg'), mrr: sum('mrr'), hit: sum('hit'), recall: sum('recall') },
+    };
+}
+export function winTieLoss(a, b, epsilon = 1e-6) {
+    let win = 0, tie = 0, loss = 0;
+    for (let i = 0; i < Math.min(a.length, b.length); i++) {
+        const d = a[i].ndcg - b[i].ndcg;
+        if (d > epsilon)
+            win++;
+        else if (d < -epsilon)
+            loss++;
+        else
+            tie++;
+    }
+    return { win, tie, loss };
+}
+export function redundancyAtK(ranked, k) {
+    const seen = new Set();
+    let redundant = 0;
+    const counted = ranked.slice(0, k);
+    for (let i = 0; i < counted.length; i++) {
+        const doc = counted[i].file;
+        if (seen.has(doc))
+            redundant++;
+        else
+            seen.add(doc);
+    }
+    return counted.length === 0 ? 0 : redundant / counted.length;
+}

package/dist/models.d.ts

@@ -0,0 +1,38 @@
+export type Pooling = 'mean' | 'last_token' | 'none';
+export type DType = 'q8' | 'q4' | 'fp16' | 'fp32' | 'int8' | 'uint8';
+export interface ModelAdapter {
+    id: string;
+    revision?: string;
+    nativeDim: number;
+    dim: number;
+    dimensions?: number[];
+    queryPrefix: string;
+    passagePrefix: string;
+    pooling: Pooling;
+    normalize?: boolean;
+    dtype?: DType | string;
+    maxTokens?: number;
+    sessionOptions?: Record<string, unknown>;
+    family?: string;
+    note?: string;
+    unknownAdapter?: boolean;
+}
+export declare const MODELS: Record<string, ModelAdapter>;
+export declare const DEFAULT_MODEL = "e5-base";
+export declare const POOLING: Pooling[];
+export declare function normalizeAdapter(raw: Partial<ModelAdapter> | ModelAdapter): ModelAdapter;
+export declare function resolveModel(name?: string | Partial<ModelAdapter> | ModelAdapter | null): ModelAdapter;
+export interface FingerprintInput {
+    queryPrefix?: string;
+    passagePrefix?: string;
+    pooling?: Pooling;
+    normalize?: boolean;
+    dim?: number;
+    nativeDim?: number;
+}
+export declare function embeddingAdapterFingerprint(model: FingerprintInput): string;
+export declare function legacyEmbeddingAdapterFingerprint(model: {
+    id: string;
+    dim: number;
+    nativeDim?: number;
+}): string;

package/dist/models.js

@@ -0,0 +1,181 @@
+/**
+ * Embedding model adapter registry (issue #60).
+ */
+import crypto from 'node:crypto';
+export const MODELS = {
+    'e5-small': {
+        id: 'Xenova/multilingual-e5-small',
+        nativeDim: 384,
+        dim: 384,
+        queryPrefix: 'query: ',
+        passagePrefix: 'passage: ',
+        pooling: 'mean',
+        normalize: true,
+        dtype: 'q8',
+        maxTokens: 512,
+        family: 'e5',
+        note: 'Fastest, ~120MB. Weak cross-lingual ranking — see RESEARCH.md.',
+    },
+    'e5-base': {
+        id: 'Xenova/multilingual-e5-base',
+        nativeDim: 768,
+        dim: 768,
+        queryPrefix: 'query: ',
+        passagePrefix: 'passage: ',
+        pooling: 'mean',
+        normalize: true,
+        dtype: 'q8',
+        maxTokens: 512,
+        family: 'e5',
+        note: 'Default. ~280MB. Solid multilingual + cross-lingual balance.',
+    },
+    'e5-large': {
+        id: 'Xenova/multilingual-e5-large',
+        nativeDim: 1024,
+        dim: 1024,
+        queryPrefix: 'query: ',
+        passagePrefix: 'passage: ',
+        pooling: 'mean',
+        normalize: true,
+        dtype: 'q8',
+        maxTokens: 512,
+        family: 'e5',
+        note: '~2.2GB. Higher quality, slower.',
+    },
+    'bge-m3': {
+        id: 'Xenova/bge-m3',
+        nativeDim: 1024,
+        dim: 1024,
+        queryPrefix: '',
+        passagePrefix: '',
+        pooling: 'mean',
+        normalize: true,
+        dtype: 'q8',
+        maxTokens: 512,
+        family: 'bge',
+        note: '~2.3GB. Best cross-lingual separation in our tests; no prefixes.',
+    },
+    'qwen3-embedding-0.6b': {
+        id: 'onnx-community/Qwen3-Embedding-0.6B-ONNX',
+        revision: 'c25a394dd583836952667c12f008335071b3f43d',
+        nativeDim: 1024,
+        dim: 1024,
+        queryPrefix: 'Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:',
+        passagePrefix: '',
+        pooling: 'last_token',
+        normalize: true,
+        dtype: 'q8',
+        maxTokens: 32768,
+        family: 'qwen3',
+        note: '~613 MB q8. Opt-in Qwen3 embedding model.',
+    },
+};
+export const DEFAULT_MODEL = 'e5-base';
+export const POOLING = ['mean', 'last_token', 'none'];
+const POOLING_SET = new Set(POOLING);
+const aliasByRepoId = new Map();
+for (const [alias, adapter] of Object.entries(MODELS)) {
+    if (!aliasByRepoId.has(adapter.id))
+        aliasByRepoId.set(adapter.id, alias);
+}
+function withDim(adapter) {
+    return { ...adapter, dim: adapter.nativeDim ?? adapter.dim };
+}
+export function normalizeAdapter(raw) {
+    if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) {
+        throw new Error('a custom model adapter must be an object with id, pooling, normalize, and dimension');
+    }
+    if (typeof raw.id !== 'string' || raw.id.trim().length === 0) {
+        throw new Error('a custom model adapter requires a nonempty `id`');
+    }
+    const pooling = raw.pooling ?? 'mean';
+    if (!POOLING_SET.has(pooling)) {
+        throw new Error(`unsupported pooling "${pooling}" — expected one of: ${POOLING.join(', ')}`);
+    }
+    const nativeDim = raw.nativeDim ?? raw.dim ?? 0;
+    if (!Number.isSafeInteger(nativeDim) || nativeDim < 0) {
+        throw new Error(`invalid adapter dimension ${nativeDim} — must be a non-negative safe integer`);
+    }
+    const dimensions = raw.dimensions !== undefined
+        ? Array.isArray(raw.dimensions) && raw.dimensions.every((d) => Number.isSafeInteger(d) && d > 0)
+            ? [...new Set(raw.dimensions)].sort((a, b) => a - b)
+            : []
+        : [];
+    const adapter = withDim({
+        id: raw.id,
+        ...(raw.revision ? { revision: raw.revision } : {}),
+        nativeDim,
+        dim: nativeDim,
+        ...(dimensions.length > 0 ? { dimensions } : {}),
+        queryPrefix: typeof raw.queryPrefix === 'string' ? raw.queryPrefix : '',
+        passagePrefix: typeof raw.passagePrefix === 'string' ? raw.passagePrefix : '',
+        pooling,
+        normalize: raw.normalize !== false,
+        dtype: typeof raw.dtype === 'string' && raw.dtype.length > 0 ? raw.dtype : 'q8',
+        ...(Number.isSafeInteger(raw.maxTokens) && raw.maxTokens && raw.maxTokens > 0 ? { maxTokens: raw.maxTokens } : {}),
+        ...(typeof raw.family === 'string' && raw.family.length > 0 ? { family: raw.family } : {}),
+        ...(typeof raw.note === 'string' ? { note: raw.note } : {}),
+    });
+    if (raw.unknownAdapter === true && typeof raw.dtype !== 'string') {
+        return { ...adapter, unknownAdapter: true };
+    }
+    return adapter;
+}
+export function resolveModel(name) {
+    if (name === undefined || name === null || name === '') {
+        return withDim(MODELS[DEFAULT_MODEL]);
+    }
+    if (typeof name === 'object' && !Array.isArray(name)) {
+        return normalizeAdapter(name);
+    }
+    if (typeof name !== 'string') {
+        throw new Error(`model must be an alias, id, or adapter object — got ${typeof name}`);
+    }
+    if (MODELS[name])
+        return withDim(MODELS[name]);
+    let id = name;
+    let revision = 'main';
+    const at = name.indexOf('@');
+    const hasExplicitRevision = at > 0;
+    if (hasExplicitRevision) {
+        id = name.slice(0, at);
+        const requested = name.slice(at + 1);
+        revision = requested === '' ? 'main' : requested;
+    }
+    const alias = aliasByRepoId.get(id);
+    if (alias !== undefined) {
+        const registered = MODELS[alias];
+        const resolved = hasExplicitRevision ? { ...registered, revision } : registered;
+        return withDim(resolved);
+    }
+    return normalizeAdapter({
+        id,
+        ...(revision !== 'main' ? { revision } : {}),
+        pooling: 'none',
+        normalize: false,
+        nativeDim: 0,
+        dim: 0,
+        unknownAdapter: true,
+    });
+}
+export function embeddingAdapterFingerprint(model) {
+    const semantics = [
+        'embedding-adapter-v1',
+        `query-prefix:${model.queryPrefix ?? ''}`,
+        `passage-prefix:${model.passagePrefix ?? ''}`,
+        `pooling:${model.pooling ?? 'mean'}`,
+        `normalize:${model.normalize !== false}`,
+        `dimension:${model.nativeDim ?? model.dim ?? 0}`,
+    ].join('\u0000');
+    return `adapter-v1:${crypto.createHash('sha256').update(semantics).digest('hex')}`;
+}
+export function legacyEmbeddingAdapterFingerprint(model) {
+    const isBge = /bge/i.test(model.id);
+    return embeddingAdapterFingerprint({
+        nativeDim: model.nativeDim ?? model.dim,
+        queryPrefix: isBge ? '' : 'query: ',
+        passagePrefix: isBge ? '' : 'passage: ',
+        pooling: 'mean',
+        normalize: true,
+    });
+}

package/dist/rerank.d.ts

@@ -0,0 +1,10 @@
+export declare const RERANK_MODEL = "Xenova/bge-reranker-base";
+export declare function getReranker(cacheDir: string, offline?: boolean, retryOpts?: {
+    maxRetries?: number;
+    delays?: number[];
+    log?: (msg: string) => void;
+}): Promise<{
+    tokenizer: any;
+    model: any;
+}>;
+export declare function rerankScores(query: string, passages: string[], cacheDir: string, offline?: boolean): Promise<number[]>;

package/dist/rerank.js

@@ -0,0 +1,41 @@
+/**
+ * Cross-encoder re-ranking (issue #15).
+ */
+import { resolveSessionOptions, retryWithBackoff } from './core.js';
+const _rerankers = new Map();
+export const RERANK_MODEL = 'Xenova/bge-reranker-base';
+export async function getReranker(cacheDir, offline = false, retryOpts = {}) {
+    const sessionOptions = resolveSessionOptions();
+    const sessionKey = sessionOptions ? JSON.stringify(sessionOptions) : 'default';
+    const key = `${RERANK_MODEL}|${offline ? 'off' : 'on'}|${sessionKey}`;
+    if (_rerankers.has(key))
+        return _rerankers.get(key);
+    const loadPipeline = async () => {
+        const { AutoModelForSequenceClassification, AutoTokenizer, env } = await import('@huggingface/transformers');
+        if (cacheDir)
+            env.cacheDir = cacheDir;
+        env.allowRemoteModels = !offline;
+        const tokenizer = await AutoTokenizer.from_pretrained(RERANK_MODEL);
+        const loadOpts = {
+            dtype: 'q8',
+            ...(sessionOptions ? { session_options: sessionOptions } : {}),
+        };
+        const model = await AutoModelForSequenceClassification.from_pretrained(RERANK_MODEL, loadOpts);
+        const r = { tokenizer, model };
+        _rerankers.set(key, r);
+        return r;
+    };
+    if (offline) {
+        return await loadPipeline();
+    }
+    return await retryWithBackoff(loadPipeline, retryOpts);
+}
+export async function rerankScores(query, passages, cacheDir, offline = false) {
+    if (passages.length === 0)
+        return [];
+    const { tokenizer, model } = await getReranker(cacheDir, offline);
+    const queries = new Array(passages.length).fill(query);
+    const inputs = tokenizer(queries, { text_pair: passages, padding: true, truncation: true });
+    const outputs = await model(inputs);
+    return outputs.logits.tolist().map((row) => row[0]);
+}