md-semantic-search 0.5.0 → 1.0.2

package/dist/markdown-parser.js

@@ -183,7 +183,7 @@ export function chunkMarkdownStructural(input, opts) {
         if (blockText.length > maxChunk) {
             flushGroup();
             const headingPath = headingStack.map(h => h.heading);
-            const subPieces = splitOversizedBlock(blockText, maxChunk);
+            const subPieces = splitOversizedBlock(blockText, maxChunk, block.type);
             let lineCursor = block.startLine;
             for (let idx = 0; idx < subPieces.length; idx++) {
                 const piece = subPieces[idx];
@@ -213,8 +213,73 @@ export function chunkMarkdownStructural(input, opts) {
     flushGroup();
     return chunks.filter(c => c.text.replace(/\s/g, '').length >= 24);
 }
-function splitOversizedBlock(text, maxChunk) {
+function splitLongLine(line, maxChunk) {
+    const pieces = [];
+    let rest = line;
+    while (rest.length > maxChunk) {
+        let cut = rest.lastIndexOf(' ', maxChunk);
+        if (cut <= 0)
+            cut = maxChunk;
+        pieces.push(rest.slice(0, cut).trim());
+        rest = rest.slice(cut).trim();
+    }
+    if (rest)
+        pieces.push(rest);
+    return pieces;
+}
+export function splitOversizedBlock(text, maxChunk, blockType) {
     const lines = text.split('\n');
+    if (lines.length <= 1) {
+        return splitLongLine(text, maxChunk);
+    }
+    // If code block with opening fence
+    if (blockType === 'code' && lines.length > 2 && /^(\s*)(`{3,}|~{3,})/.test(lines[0])) {
+        const openFence = lines[0];
+        const hasCloseFence = /^(\s*)(`{3,}|~{3,})/.test(lines[lines.length - 1]);
+        const closeFence = hasCloseFence ? lines[lines.length - 1] : '```';
+        const innerLines = lines.slice(1, hasCloseFence ? lines.length - 1 : lines.length);
+        const pieces = [];
+        let currentCode = [];
+        let currentLen = openFence.length + closeFence.length + 2;
+        for (const line of innerLines) {
+            if (currentLen + line.length + 1 > maxChunk && currentCode.length > 0) {
+                pieces.push([openFence, ...currentCode, closeFence].join('\n'));
+                currentCode = [];
+                currentLen = openFence.length + closeFence.length + 2;
+            }
+            currentCode.push(line);
+            currentLen += line.length + 1;
+        }
+        if (currentCode.length > 0) {
+            pieces.push([openFence, ...currentCode, closeFence].join('\n'));
+        }
+        if (pieces.length > 0)
+            return pieces;
+    }
+    // If table with header + separator rows
+    if (blockType === 'table' && lines.length > 2 && /^\s*\|?\s*[-:]+[-|\s:]*$/.test(lines[1])) {
+        const headerRows = [lines[0], lines[1]];
+        const dataRows = lines.slice(2);
+        const headerLen = headerRows[0].length + headerRows[1].length + 2;
+        const pieces = [];
+        let currentRows = [];
+        let currentLen = headerLen;
+        for (const row of dataRows) {
+            if (currentLen + row.length + 1 > maxChunk && currentRows.length > 0) {
+                pieces.push([...headerRows, ...currentRows].join('\n'));
+                currentRows = [];
+                currentLen = headerLen;
+            }
+            currentRows.push(row);
+            currentLen += row.length + 1;
+        }
+        if (currentRows.length > 0) {
+            pieces.push([...headerRows, ...currentRows].join('\n'));
+        }
+        if (pieces.length > 0)
+            return pieces;
+    }
+    // Standard paragraph/list splitting
     const pieces = [];
     let currentLines = [];
     let currentLen = 0;
@@ -229,17 +294,11 @@ function splitOversizedBlock(text, maxChunk) {
                 pieces.push(currentLines.join('\n').trim());
                 currentLines = [];
             }
-            let rest = line;
-            while (rest.length > maxChunk) {
-                let cut = rest.lastIndexOf(' ', maxChunk);
-                if (cut <= 0)
-                    cut = maxChunk;
-                pieces.push(rest.slice(0, cut).trim());
-                rest = rest.slice(cut).trim();
+            const splitLines = splitLongLine(line, maxChunk);
+            for (const piece of splitLines) {
+                pieces.push(piece);
             }
-            if (rest)
-                currentLines.push(rest);
-            currentLen = rest.length;
+            currentLen = 0;
             continue;
         }
         currentLines.push(line);

package/dist/mcp.d.ts

@@ -22,6 +22,13 @@ export declare const MCP_TOOLS: ({
                 type: string;
                 description: string;
             };
+            vaults: {
+                type: string;
+                items: {
+                    type: string;
+                };
+                description: string;
+            };
             tag: {
                 type: string;
                 items: {
@@ -29,6 +36,14 @@ export declare const MCP_TOOLS: ({
                 };
                 description: string;
             };
+            filter: {
+                type: string;
+                description: string;
+            };
+            graphBoost: {
+                type: string;
+                description: string;
+            };
             file?: undefined;
             heading?: undefined;
         };
@@ -40,9 +55,10 @@ export declare const MCP_TOOLS: ({
     inputSchema: {
         type: string;
         properties: {
-            query?: undefined;
-            k?: undefined;
+            vaults?: undefined;
             tag?: undefined;
+            filter?: undefined;
+            graphBoost?: undefined;
             file: {
                 type: string;
                 description: string;
@@ -52,6 +68,8 @@ export declare const MCP_TOOLS: ({
                 description: string;
             };
             path?: undefined;
+            query?: undefined;
+            k?: undefined;
         };
         required: string[];
     };
@@ -59,34 +77,64 @@ export declare const MCP_TOOLS: ({
     name: string;
     description: string;
     inputSchema: {
-        required?: undefined;
         type: string;
         properties: {
-            query?: undefined;
-            k?: undefined;
+            vaults?: undefined;
             tag?: undefined;
+            filter?: undefined;
+            graphBoost?: undefined;
             file?: undefined;
             heading?: undefined;
             path: {
                 type: string;
                 description: string;
             };
+            query?: undefined;
+            k?: undefined;
         };
+        required?: undefined;
     };
 } | {
     name: string;
     description: string;
     inputSchema: {
-        required?: undefined;
         type: string;
         properties: {
+            vaults?: undefined;
+            tag?: undefined;
+            filter?: undefined;
+            graphBoost?: undefined;
+            file?: undefined;
+            heading?: undefined;
+            path?: undefined;
             query?: undefined;
             k?: undefined;
+        };
+        required?: undefined;
+    };
+} | {
+    name: string;
+    description: string;
+    inputSchema: {
+        type: string;
+        properties: {
+            vaults?: undefined;
             tag?: undefined;
+            filter?: undefined;
+            graphBoost?: undefined;
             file?: undefined;
             heading?: undefined;
             path?: undefined;
+            query: {
+                type: string;
+                description: string;
+            };
+            k: {
+                type: string;
+                description: string;
+            };
         };
+        required: string[];
     };
 })[];
 export declare function handleMcpRequest(req: any, state: {

package/dist/mcp.js

@@ -8,6 +8,8 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { buildIndex } from './indexer.js';
 import { loadIndex, searchIndex } from './search.js';
+import { searchFederated } from './federation.js';
+import { askQuestion } from './rag.js';
 import { globToRegExp } from './core.js';
 export const MCP_TOOLS = [
     {
@@ -19,7 +21,10 @@ export const MCP_TOOLS = [
                 query: { type: 'string', description: 'Natural language search query' },
                 k: { type: 'number', description: 'Number of top results to return (default: 6)' },
                 path: { type: 'string', description: 'Optional path glob filter (e.g. "docs/**")' },
+                vaults: { type: 'array', items: { type: 'string' }, description: 'Optional list of vault directories for federated search' },
                 tag: { type: 'array', items: { type: 'string' }, description: 'Optional frontmatter tag filters' },
+                filter: { type: 'string', description: 'Optional rich boolean filter expression (e.g. "tag:engineering AND status != archived")' },
+                graphBoost: { type: 'number', description: 'Optional boost weight for graph PageRank / wikilinks (0.0 to 1.0)' },
             },
             required: ['query'],
         },
@@ -54,6 +59,18 @@ export const MCP_TOOLS = [
             properties: {},
         },
     },
+    {
+        name: 'ask_knowledge_base',
+        description: 'Synthesize a grounded direct answer with citations from the local Markdown knowledge base',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                query: { type: 'string', description: 'Question to ask the knowledge base' },
+                k: { type: 'number', description: 'Number of candidate passages to retrieve (default: 5)' },
+            },
+            required: ['query'],
+        },
+    },
 ];
 export async function handleMcpRequest(req, state) {
     if (!req || typeof req !== 'object')
@@ -73,7 +90,7 @@ export async function handleMcpRequest(req, state) {
                 },
                 serverInfo: {
                     name: 'md-semantic-search',
-                    version: '0.4.0',
+                    version: '1.0.0',
                 },
             },
         };
@@ -95,16 +112,35 @@ export async function handleMcpRequest(req, state) {
             if (name === 'search_markdown') {
                 const query = typeof args.query === 'string' ? args.query : '';
                 const k = Number.isInteger(args.k) && args.k > 0 ? args.k : 6;
-                const results = await searchIndex({
-                    loaded: state.loaded,
-                    cacheDir: state.cacheDir,
-                    query,
-                    k,
-                    path: args.path,
-                    tag: args.tag,
-                    offline: state.offline,
-                    embedFn: state.embedFn,
-                });
+                let results;
+                if (Array.isArray(args.vaults) && args.vaults.length > 0) {
+                    results = await searchFederated({
+                        vaults: args.vaults,
+                        cacheDir: state.cacheDir,
+                        query,
+                        k,
+                        path: args.path,
+                        tag: args.tag,
+                        filter: typeof args.filter === 'string' ? args.filter : undefined,
+                        graphBoost: typeof args.graphBoost === 'number' ? args.graphBoost : undefined,
+                        offline: state.offline,
+                        embedFn: state.embedFn,
+                    });
+                }
+                else {
+                    results = await searchIndex({
+                        loaded: state.loaded,
+                        cacheDir: state.cacheDir,
+                        query,
+                        k,
+                        path: args.path,
+                        tag: args.tag,
+                        filter: typeof args.filter === 'string' ? args.filter : undefined,
+                        graphBoost: typeof args.graphBoost === 'number' ? args.graphBoost : undefined,
+                        offline: state.offline,
+                        embedFn: state.embedFn,
+                    });
+                }
                 content = [{ type: 'text', text: JSON.stringify(results, null, 2) }];
             }
             else if (name === 'get_chunk') {
@@ -141,6 +177,18 @@ export async function handleMcpRequest(req, state) {
                 };
                 content = [{ type: 'text', text: JSON.stringify(info, null, 2) }];
             }
+            else if (name === 'ask_knowledge_base') {
+                const query = args.query;
+                const k = typeof args.k === 'number' ? args.k : 5;
+                const answerResult = await askQuestion({
+                    loaded: state.loaded,
+                    cacheDir: state.cacheDir,
+                    query,
+                    k,
+                    embedFn: state.embedFn,
+                });
+                content = [{ type: 'text', text: JSON.stringify(answerResult, null, 2) }];
+            }
             else {
                 return {
                     jsonrpc: '2.0',

package/dist/quantization.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Int8 Scalar Quantization Tier (issue #96).
+ *
+ * Maps normalized Float32 unit vectors in [-1.0, 1.0] to signed 8-bit integers in [-128, 127],
+ * providing 4x memory savings with <1% loss in ranking precision.
+ */
+export declare const INT8_SCALE = 127;
+export declare const INT8_INV_SCALE: number;
+/**
+ * Quantizes an L2-normalized float vector into a signed Int8Array.
+ */
+export declare function quantizeToInt8(vec: ArrayLike<number>): Int8Array;
+/**
+ * Dequantizes an Int8Array back into an approximate Float32Array.
+ */
+export declare function dequantizeFromInt8(vec: Int8Array): Float32Array;
+/**
+ * Computes asymmetric cosine similarity between a normalized Float32 query vector
+ * and a quantized Int8 chunk vector.
+ *
+ * Since query is unit length (||q|| = 1.0),
+ * cosine(q, c) = (q · c_fp32) / ||c_fp32|| = (sum(q_i * c_i) / 127) / (||c|| / 127) = sum(q_i * c_i) / ||c||.
+ */
+export declare function asymmetricCosineInt8(queryFp32: Float32Array, chunkInt8: Int8Array): number;

package/dist/quantization.js

@@ -0,0 +1,85 @@
+/**
+ * Int8 Scalar Quantization Tier (issue #96).
+ *
+ * Maps normalized Float32 unit vectors in [-1.0, 1.0] to signed 8-bit integers in [-128, 127],
+ * providing 4x memory savings with <1% loss in ranking precision.
+ */
+export const INT8_SCALE = 127.0;
+export const INT8_INV_SCALE = 1.0 / 127.0;
+/**
+ * Quantizes an L2-normalized float vector into a signed Int8Array.
+ */
+export function quantizeToInt8(vec) {
+    const len = vec.length;
+    const out = new Int8Array(len);
+    for (let i = 0; i < len; i++) {
+        const val = vec[i];
+        // Clamp to [-1.0, 1.0] and scale to [-128, 127]
+        const clamped = val < -1.0 ? -1.0 : val > 1.0 ? 1.0 : val;
+        const scaled = Math.round(clamped * INT8_SCALE);
+        out[i] = scaled < -128 ? -128 : scaled > 127 ? 127 : scaled;
+    }
+    return out;
+}
+/**
+ * Dequantizes an Int8Array back into an approximate Float32Array.
+ */
+export function dequantizeFromInt8(vec) {
+    const len = vec.length;
+    const out = new Float32Array(len);
+    for (let i = 0; i < len; i++) {
+        out[i] = vec[i] * INT8_INV_SCALE;
+    }
+    return out;
+}
+/**
+ * Computes asymmetric cosine similarity between a normalized Float32 query vector
+ * and a quantized Int8 chunk vector.
+ *
+ * Since query is unit length (||q|| = 1.0),
+ * cosine(q, c) = (q · c_fp32) / ||c_fp32|| = (sum(q_i * c_i) / 127) / (||c|| / 127) = sum(q_i * c_i) / ||c||.
+ */
+export function asymmetricCosineInt8(queryFp32, chunkInt8) {
+    const dim = queryFp32.length;
+    let dot = 0.0;
+    let normSq = 0.0;
+    // Unroll loop in steps of 8 for performance
+    let i = 0;
+    const limit = dim - (dim % 8);
+    for (; i < limit; i += 8) {
+        const c0 = chunkInt8[i];
+        const c1 = chunkInt8[i + 1];
+        const c2 = chunkInt8[i + 2];
+        const c3 = chunkInt8[i + 3];
+        const c4 = chunkInt8[i + 4];
+        const c5 = chunkInt8[i + 5];
+        const c6 = chunkInt8[i + 6];
+        const c7 = chunkInt8[i + 7];
+        dot +=
+            queryFp32[i] * c0 +
+                queryFp32[i + 1] * c1 +
+                queryFp32[i + 2] * c2 +
+                queryFp32[i + 3] * c3 +
+                queryFp32[i + 4] * c4 +
+                queryFp32[i + 5] * c5 +
+                queryFp32[i + 6] * c6 +
+                queryFp32[i + 7] * c7;
+        normSq +=
+            c0 * c0 +
+                c1 * c1 +
+                c2 * c2 +
+                c3 * c3 +
+                c4 * c4 +
+                c5 * c5 +
+                c6 * c6 +
+                c7 * c7;
+    }
+    for (; i < dim; i++) {
+        const c = chunkInt8[i];
+        dot += queryFp32[i] * c;
+        normSq += c * c;
+    }
+    if (normSq <= 0)
+        return 0.0;
+    return dot / Math.sqrt(normSq);
+}

package/dist/rag.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Offline QA & Local RAG Answer Synthesis (issue #101).
+ *
+ * Implements grounded question answering with strict provenance citations ([file#heading]),
+ * supporting local LLM bridges (Ollama / LocalAI / OpenAI-compatible endpoint) and a
+ * fast built-in zero-dependency extractive synthesis fallback.
+ */
+import { LoadedIndex, SearchResultHit } from './search.js';
+export interface Citation {
+    file: string;
+    heading?: string;
+    score: number;
+}
+export interface SynthesisResult {
+    answer: string;
+    citations: Citation[];
+    passages: SearchResultHit[];
+}
+export interface AskOptions {
+    query: string;
+    loaded?: LoadedIndex;
+    indexDir?: string;
+    cacheDir?: string;
+    db?: string;
+    k?: number;
+    llmEndpoint?: string;
+    llmModel?: string;
+    llmFn?: (prompt: string) => Promise<string>;
+    systemPrompt?: string;
+    embedFn?: any;
+}
+export declare const DEFAULT_RAG_SYSTEM_PROMPT: string;
+/**
+ * Built-in extractive answer synthesizer for fully offline, zero-dependency QA.
+ */
+export declare function extractAnswerFallback(query: string, passages: SearchResultHit[]): string;
+/**
+ * Synthesizes an answer from retrieved passages using an LLM or fallback.
+ */
+export declare function synthesizeAnswer(query: string, passages: SearchResultHit[], options?: {
+    llmFn?: (prompt: string) => Promise<string>;
+    llmEndpoint?: string;
+    llmModel?: string;
+    systemPrompt?: string;
+}): Promise<SynthesisResult>;
+/**
+ * End-to-end question answering against a Markdown index.
+ */
+export declare function askQuestion(opts: AskOptions): Promise<SynthesisResult>;

package/dist/rag.js

@@ -0,0 +1,128 @@
+/**
+ * Offline QA & Local RAG Answer Synthesis (issue #101).
+ *
+ * Implements grounded question answering with strict provenance citations ([file#heading]),
+ * supporting local LLM bridges (Ollama / LocalAI / OpenAI-compatible endpoint) and a
+ * fast built-in zero-dependency extractive synthesis fallback.
+ */
+import { searchIndex, loadIndex } from './search.js';
+import { tokenize } from './lexical.js';
+export const DEFAULT_RAG_SYSTEM_PROMPT = 'You are a helpful assistant answering questions using only the provided Markdown notes. ' +
+    'Always cite your sources using [file#heading] format. ' +
+    'If the context does not contain the answer, say that the notes do not have enough information.';
+/**
+ * Built-in extractive answer synthesizer for fully offline, zero-dependency QA.
+ */
+export function extractAnswerFallback(query, passages) {
+    if (passages.length === 0) {
+        return 'No relevant passages found in the notes to answer the question.';
+    }
+    const queryTokens = new Set(tokenize(query));
+    const candidateSentences = [];
+    for (const hit of passages) {
+        const text = hit.snippet || '';
+        const cite = hit.heading ? `[${hit.file} › ${hit.heading}]` : `[${hit.file}]`;
+        // Split into sentences
+        const sentences = text
+            .split(/(?<=[.?!])\s+|\n+/)
+            .map((s) => s.trim())
+            .filter((s) => s.length > 15);
+        for (const sent of sentences) {
+            const sentTokens = tokenize(sent);
+            let matchCount = 0;
+            for (const tok of sentTokens) {
+                if (queryTokens.has(tok))
+                    matchCount++;
+            }
+            if (matchCount > 0) {
+                const score = (matchCount / (sentTokens.length + 1)) * (hit.score || 1);
+                candidateSentences.push({ sentence: sent, score, citation: cite });
+            }
+        }
+    }
+    if (candidateSentences.length === 0) {
+        // Return top snippets directly
+        const top = passages.slice(0, 2).map((p) => {
+            const cite = p.heading ? `[${p.file} › ${p.heading}]` : `[${p.file}]`;
+            return `${cite}: ${p.snippet.slice(0, 200)}…`;
+        });
+        return `Relevant excerpts from notes:\n\n${top.join('\n\n')}`;
+    }
+    candidateSentences.sort((a, b) => b.score - a.score);
+    const selected = candidateSentences.slice(0, 3);
+    const body = selected.map((s) => `${s.sentence} ${s.citation}`).join('\n\n');
+    return `Based on your notes:\n\n${body}`;
+}
+/**
+ * Synthesizes an answer from retrieved passages using an LLM or fallback.
+ */
+export async function synthesizeAnswer(query, passages, options = {}) {
+    const citations = passages.map((p) => ({
+        file: p.file,
+        heading: p.heading,
+        score: p.score,
+    }));
+    if (options.llmFn) {
+        const contextBlocks = passages.map((p, i) => {
+            const header = p.heading ? `${p.file} › ${p.heading}` : p.file;
+            return `[Source ${i + 1}: ${header}]\n${p.snippet}`;
+        }).join('\n\n---\n\n');
+        const prompt = `${options.systemPrompt || DEFAULT_RAG_SYSTEM_PROMPT}\n\nContext:\n${contextBlocks}\n\nQuestion: ${query}\n\nAnswer:`;
+        const answer = await options.llmFn(prompt);
+        return { answer, citations, passages };
+    }
+    if (options.llmEndpoint) {
+        const contextBlocks = passages.map((p, i) => {
+            const header = p.heading ? `${p.file} › ${p.heading}` : p.file;
+            return `[Source ${i + 1}: ${header}]\n${p.snippet}`;
+        }).join('\n\n---\n\n');
+        const prompt = `${options.systemPrompt || DEFAULT_RAG_SYSTEM_PROMPT}\n\nContext:\n${contextBlocks}\n\nQuestion: ${query}\n\nAnswer:`;
+        try {
+            const resp = await fetch(`${options.llmEndpoint}/api/generate`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({
+                    model: options.llmModel || 'llama3',
+                    prompt,
+                    stream: false,
+                }),
+            });
+            if (resp.ok) {
+                const data = await resp.json();
+                const answer = data.response || data.text || '';
+                if (answer)
+                    return { answer, citations, passages };
+            }
+        }
+        catch {
+            // Fallback to extractive
+        }
+    }
+    const answer = extractAnswerFallback(query, passages);
+    return { answer, citations, passages };
+}
+/**
+ * End-to-end question answering against a Markdown index.
+ */
+export async function askQuestion(opts) {
+    const { query, loaded: inputLoaded, indexDir, cacheDir = '', k = 5, llmFn, llmEndpoint, llmModel, systemPrompt, embedFn, } = opts;
+    let loaded = inputLoaded;
+    if (!loaded) {
+        if (!indexDir)
+            throw new Error('askQuestion requires either loaded index or indexDir');
+        loaded = loadIndex(indexDir);
+    }
+    const passages = await searchIndex({
+        loaded,
+        cacheDir,
+        query,
+        k,
+        embedFn,
+    });
+    return synthesizeAnswer(query, passages, {
+        llmFn,
+        llmEndpoint,
+        llmModel,
+        systemPrompt,
+    });
+}