md-semantic-search 0.5.0 → 1.0.2

package/dist/tui.js

@@ -2,6 +2,7 @@ import readline from 'node:readline';
 import { spawn } from 'node:child_process';
 import path from 'node:path';
 import { loadIndex, searchIndex } from './search.js';
+import { extractAnswerFallback } from './rag.js';
 export async function runTui(opts) {
     if (!process.stdin.isTTY || !process.stdout.isTTY) {
         throw new Error('interactive TUI requires a TTY terminal stdin and stdout');
@@ -55,6 +56,7 @@ export async function runTui(opts) {
             performSearch();
         }, opts.debounceMs ?? 300);
     };
+    let showRag = Boolean(opts.rag);
     const render = () => {
         const rows = process.stdout.rows || 24;
         const cols = process.stdout.columns || 80;
@@ -63,10 +65,11 @@ export async function runTui(opts) {
         buf += '\x1b[H\x1b[J'; // clear screen from cursor down
         // Header
         const searchStatus = searching ? ' [searching...]' : '';
-        const headerStr = ` mdss interactive search | Query: ${query}${searchStatus}`;
+        const modeLabel = showRag ? 'RAG QA & Synthesis' : 'search';
+        const headerStr = ` mdss interactive ${modeLabel} | Query: ${query}${searchStatus}`;
         buf += `\x1b[7m${headerStr.padEnd(cols).slice(0, cols)}\x1b[0m\n`;
         // Calculate layout heights
-        const listHeight = Math.max(4, Math.floor((rows - 4) * 0.4));
+        const listHeight = Math.max(3, Math.floor((rows - 4) * 0.35));
         const previewHeight = Math.max(4, rows - 4 - listHeight);
         // Render list
         buf += `\x1b[1m--- Results (${results.length}) ---\x1b[0m\n`;
@@ -89,14 +92,14 @@ export async function runTui(opts) {
                 buf += '~ \n';
             }
         }
-        // Render Preview
-        buf += `\x1b[1m--- Passage Preview ---\x1b[0m\n`;
-        const selectedHit = results[selectedIndex];
-        if (selectedHit) {
-            const textLines = selectedHit.snippet.split('\n');
+        // Render Preview or RAG synthesis
+        if (showRag) {
+            buf += `\x1b[1m--- Grounded Answer Synthesis & Citations ---\x1b[0m\n`;
+            const answer = extractAnswerFallback(query, results);
+            const answerLines = answer.split('\n');
             for (let i = 0; i < previewHeight; i++) {
-                if (i < textLines.length) {
-                    buf += `  ${textLines[i].slice(0, cols - 3)}\n`;
+                if (i < answerLines.length) {
+                    buf += `  ${answerLines[i].slice(0, cols - 3)}\n`;
                 }
                 else {
                     buf += '\n';
@@ -104,12 +107,27 @@ export async function runTui(opts) {
             }
         }
         else {
-            buf += '  (no passage selected)\n';
-            for (let i = 1; i < previewHeight; i++)
-                buf += '\n';
+            buf += `\x1b[1m--- Passage Preview ---\x1b[0m\n`;
+            const selectedHit = results[selectedIndex];
+            if (selectedHit) {
+                const textLines = selectedHit.snippet.split('\n');
+                for (let i = 0; i < previewHeight; i++) {
+                    if (i < textLines.length) {
+                        buf += `  ${textLines[i].slice(0, cols - 3)}\n`;
+                    }
+                    else {
+                        buf += '\n';
+                    }
+                }
+            }
+            else {
+                buf += '  (no passage selected)\n';
+                for (let i = 1; i < previewHeight; i++)
+                    buf += '\n';
+            }
         }
         // Footer
-        const footerStr = ' [Up/Down/j/k]: Navigate | [Enter]: Open in Editor | [Esc/q/Ctrl+C]: Quit';
+        const footerStr = ' [Tab]: Toggle RAG/Passage | [Up/Down]: Select | [Enter]: Open | [Esc]: Quit';
         buf += `\x1b[7m${footerStr.padEnd(cols).slice(0, cols)}\x1b[0m`;
         process.stdout.write(buf);
     };
@@ -144,6 +162,11 @@ export async function runTui(opts) {
                 resolve(null);
                 return;
             }
+            if (key.name === 'tab') {
+                showRag = !showRag;
+                render();
+                return;
+            }
             if (key.name === 'up' || (key.name === 'k' && key.ctrl)) {
                 if (selectedIndex > 0) {
                     selectedIndex--;

package/dist/watcher.d.ts

@@ -0,0 +1,11 @@
+export interface WatcherOptions {
+    debounceMs?: number;
+    ignore?: string[];
+    log?: (msg: string) => void;
+    fallbackIntervalMs?: number;
+}
+export interface FileWatcher {
+    close: () => void;
+    isNative: boolean;
+}
+export declare function createFileWatcher(dbPath: string, onChange: () => void | Promise<void>, options?: WatcherOptions): FileWatcher;

package/dist/watcher.js

@@ -0,0 +1,138 @@
+/**
+ * OS-Native File Watcher with Zero-Polling Kernel Events (issue #100).
+ *
+ * Utilizes OS kernel events (ReadDirectoryChangesW on Windows, FSEvents on macOS, inotify on Linux)
+ * via recursive `fs.watch`, with automatic debounce coalescing and graceful fallback to interval polling.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { globToRegExp } from './core.js';
+export function createFileWatcher(dbPath, onChange, options = {}) {
+    const { debounceMs = 250, ignore = [], log = () => { }, fallbackIntervalMs = 50, } = options;
+    let closed = false;
+    let debounceTimer = null;
+    const ignoreRegexes = [
+        /\.mdss($|[\\/])/,
+        /\.git($|[\\/])/,
+        /\.cache($|[\\/])/,
+        ...ignore.map((g) => globToRegExp(g)),
+    ];
+    const shouldIgnore = (filename) => {
+        if (!filename)
+            return false;
+        const normalized = filename.split(path.sep).join('/');
+        for (const re of ignoreRegexes) {
+            if (re.test(normalized))
+                return true;
+        }
+        // Only care about markdown files or directory changes
+        if (path.extname(filename) && !/\.(md|markdown)$/i.test(filename)) {
+            return true;
+        }
+        return false;
+    };
+    const scheduleChange = () => {
+        if (closed)
+            return;
+        if (debounceTimer) {
+            clearTimeout(debounceTimer);
+        }
+        debounceTimer = setTimeout(async () => {
+            if (closed)
+                return;
+            try {
+                await onChange();
+            }
+            catch (err) {
+                log(`watcher onChange error: ${err.message}`);
+            }
+        }, debounceMs);
+        debounceTimer.unref?.();
+    };
+    // Attempt recursive native fs.watch first
+    try {
+        const fsWatcher = fs.watch(dbPath, { recursive: true }, (_eventType, filename) => {
+            if (closed)
+                return;
+            if (filename && shouldIgnore(filename)) {
+                return;
+            }
+            scheduleChange();
+        });
+        fsWatcher.on('error', (err) => {
+            log(`Native watcher error (${err.message}), continuing.`);
+        });
+        // unref() lets the event loop exit even if the watcher is still active.
+        // Without this, on Node 18/Linux the inotify handle keeps the process
+        // alive indefinitely after all tests finish (or after close() is called
+        // but the handle hasn't been fully released by the kernel yet).
+        fsWatcher.unref?.();
+        return {
+            isNative: true,
+            close: () => {
+                closed = true;
+                if (debounceTimer)
+                    clearTimeout(debounceTimer);
+                try {
+                    fsWatcher.close();
+                }
+                catch { }
+            },
+        };
+    }
+    catch (nativeErr) {
+        log(`Native recursive watch unavailable (${nativeErr.message}); falling back to interval polling.`);
+        // Polling fallback
+        let lastMtime = 0;
+        const getDirMtime = (dir) => {
+            try {
+                let max = 0;
+                const entries = fs.readdirSync(dir, { withFileTypes: true });
+                for (const entry of entries) {
+                    if (entry.name.startsWith('.') || entry.name === '.mdss')
+                        continue;
+                    const full = path.join(dir, entry.name);
+                    if (shouldIgnore(entry.name))
+                        continue;
+                    if (entry.isDirectory()) {
+                        const sub = getDirMtime(full);
+                        if (sub > max)
+                            max = sub;
+                    }
+                    else if (/\.(md|markdown)$/i.test(entry.name)) {
+                        try {
+                            const fileStat = fs.statSync(full);
+                            if (fileStat.mtimeMs > max)
+                                max = fileStat.mtimeMs;
+                        }
+                        catch { }
+                    }
+                }
+                return max;
+            }
+            catch {
+                return 0;
+            }
+        };
+        lastMtime = getDirMtime(dbPath);
+        const pollInterval = setInterval(() => {
+            if (closed)
+                return;
+            const currentMtime = getDirMtime(dbPath);
+            if (currentMtime > lastMtime) {
+                lastMtime = currentMtime;
+                scheduleChange();
+            }
+        }, fallbackIntervalMs);
+        pollInterval.unref?.();
+        return {
+            isNative: false,
+            close: () => {
+                closed = true;
+                if (debounceTimer)
+                    clearTimeout(debounceTimer);
+                clearInterval(pollInterval);
+            },
+        };
+    }
+}

package/dist/wikilinks.d.ts

@@ -45,3 +45,25 @@ export declare function getRelatedNotes(graph: RelationshipGraph, targetFile: st
     direction?: 'both' | 'outgoing' | 'backlinks';
     depth?: number;
 }): RelatedNote[];
+export interface PageRankOptions {
+    damping?: number;
+    maxIterations?: number;
+    tolerance?: number;
+}
+/**
+ * Computes in-memory PageRank centrality scores for all nodes in the RelationshipGraph.
+ * Handles dangling nodes (nodes with 0 out-degree) and normalizes scores.
+ */
+export declare function computePageRank(graph: RelationshipGraph, opts?: PageRankOptions): Map<string, number>;
+export interface GraphExpansionOptions {
+    maxDepth?: number;
+    decay?: number;
+    topSeeds?: number;
+}
+/**
+ * 2-hop neighborhood expansion: propagates relevance scores across graph links.
+ */
+export declare function expandGraphNeighborhood(graph: RelationshipGraph, seedFiles: Array<{
+    file: string;
+    score: number;
+}>, opts?: GraphExpansionOptions): Map<string, number>;

package/dist/wikilinks.js

@@ -169,3 +169,103 @@ export function getRelatedNotes(graph, targetFile, opts = {}) {
     }
     return results;
 }
+/**
+ * Computes in-memory PageRank centrality scores for all nodes in the RelationshipGraph.
+ * Handles dangling nodes (nodes with 0 out-degree) and normalizes scores.
+ */
+export function computePageRank(graph, opts = {}) {
+    const damping = opts.damping ?? 0.85;
+    const maxIterations = opts.maxIterations ?? 30;
+    const tolerance = opts.tolerance ?? 1e-6;
+    const nodes = new Set();
+    for (const node of graph.outgoing.keys())
+        nodes.add(node);
+    for (const node of graph.backlinks.keys())
+        nodes.add(node);
+    const nodeList = Array.from(nodes);
+    const n = nodeList.length;
+    const rankMap = new Map();
+    if (n === 0)
+        return rankMap;
+    if (n === 1) {
+        rankMap.set(nodeList[0], 1.0);
+        return rankMap;
+    }
+    const outNeighbors = new Map();
+    for (const node of nodeList) {
+        const edges = graph.outgoing.get(node) || [];
+        const targets = new Set();
+        for (const e of edges) {
+            if (nodes.has(e.file) && e.file !== node) {
+                targets.add(e.file);
+            }
+        }
+        outNeighbors.set(node, targets);
+    }
+    const inNeighbors = new Map();
+    for (const node of nodeList) {
+        inNeighbors.set(node, new Set());
+    }
+    for (const [src, targets] of outNeighbors.entries()) {
+        for (const tgt of targets) {
+            inNeighbors.get(tgt)?.add(src);
+        }
+    }
+    let ranks = new Map();
+    const initialRank = 1.0 / n;
+    for (const node of nodeList) {
+        ranks.set(node, initialRank);
+    }
+    for (let iter = 0; iter < maxIterations; iter++) {
+        const nextRanks = new Map();
+        let danglingSum = 0;
+        for (const node of nodeList) {
+            const outDeg = outNeighbors.get(node)?.size || 0;
+            if (outDeg === 0) {
+                danglingSum += ranks.get(node) || 0;
+            }
+        }
+        let maxDiff = 0;
+        const baseRank = (1.0 - damping) / n + (damping * danglingSum) / n;
+        for (const node of nodeList) {
+            let inSum = 0;
+            const inNodes = inNeighbors.get(node) || new Set();
+            for (const inNode of inNodes) {
+                const outDeg = outNeighbors.get(inNode)?.size || 1;
+                inSum += (ranks.get(inNode) || 0) / outDeg;
+            }
+            const newRank = baseRank + damping * inSum;
+            nextRanks.set(node, newRank);
+            const diff = Math.abs(newRank - (ranks.get(node) || 0));
+            if (diff > maxDiff)
+                maxDiff = diff;
+        }
+        ranks = nextRanks;
+        if (maxDiff < tolerance)
+            break;
+    }
+    return ranks;
+}
+/**
+ * 2-hop neighborhood expansion: propagates relevance scores across graph links.
+ */
+export function expandGraphNeighborhood(graph, seedFiles, opts = {}) {
+    const maxDepth = opts.maxDepth ?? 2;
+    const decay = opts.decay ?? 0.5;
+    const topSeeds = opts.topSeeds ?? 10;
+    const propagationScores = new Map();
+    const sortedSeeds = [...seedFiles].sort((a, b) => b.score - a.score).slice(0, topSeeds);
+    for (const seed of sortedSeeds) {
+        if (seed.score <= 0)
+            continue;
+        const related = getRelatedNotes(graph, seed.file, { direction: 'both', depth: maxDepth });
+        for (const rel of related) {
+            const weight = seed.score * Math.pow(decay, rel.distance);
+            const current = propagationScores.get(rel.file) || 0;
+            if (weight > current) {
+                propagationScores.set(rel.file, weight);
+            }
+        }
+    }
+    return propagationScores;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "md-semantic-search",
-  "version": "0.5.0",
+  "version": "1.0.2",
   "description": "Local, private semantic (vector) search over any folder of Markdown files. Finds passages by meaning — cross-lingual — using transformers.js. No API keys, no cloud, no vector database.",
   "type": "module",
   "main": "./dist/index.js",
@@ -63,6 +63,8 @@
   },
   "devDependencies": {
     "@eslint/js": "^10.0.1",
+    "@types/node": "^26.2.0",
+    "@types/vscode": "^1.125.0",
     "eslint": "^10.8.1",
     "fast-check": "^4.9.0",
     "globals": "^17.11.0",

package/src/binary-format.ts

@@ -0,0 +1,216 @@
+/**
+ * Zero-Copy Binary Index Storage (vectors.bin) format (issue #95).
+ *
+ * Binary Layout:
+ * ┌────────────────────────────────────────────────────────┐
+ * │ Header (64 bytes):                                     │
+ * │  0..7:   Magic 'MDSSBIN1'                              │
+ * │  8..11:  Version (uint32 = 1)                          │
+ * │  12..15: Flags (uint32: 0=FP32, 1=INT8)               │
+ * │  16..19: Dim (uint32)                                  │
+ * │  20..23: ChunkCount (uint32)                           │
+ * │  24..31: VectorsOffset (uint64)                        │
+ * │  32..39: VectorsBytes (uint64)                         │
+ * │  40..47: MetadataOffset (uint64)                       │
+ * │  48..55: MetadataBytes (uint64)                        │
+ * │  56..63: Reserved (8 bytes zeros)                      │
+ * ├────────────────────────────────────────────────────────┤
+ * │ Vectors Section:                                       │
+ * │  Contiguous float32 array: [chunkCount * dim * 4 bytes]│
+ * ├────────────────────────────────────────────────────────┤
+ * │ Metadata Section:                                      │
+ * │  UTF-8 JSON string (chunks metadata + lexical state)   │
+ * └────────────────────────────────────────────────────────┘
+ */
+import { decodeVec } from './core.js';
+import { PersistedIndex, IndexChunk } from './indexer.js';
+import { quantizeToInt8 } from './quantization.js';
+
+export const BINARY_MAGIC = 'MDSSBIN1';
+export const BINARY_HEADER_SIZE = 64;
+export const BINARY_VERSION = 1;
+
+export const BINARY_FLAG_FP32 = 0;
+export const BINARY_FLAG_INT8 = 1;
+
+export interface BinaryIndexHeader {
+  version: number;
+  flags: number;
+  dim: number;
+  chunkCount: number;
+  vectorsOffset: number;
+  vectorsBytes: number;
+  metadataOffset: number;
+  metadataBytes: number;
+}
+
+export interface SerializeBinaryOptions {
+  quantize?: 'fp32' | 'int8';
+}
+
+export function serializeBinaryIndex(
+  index: PersistedIndex,
+  dim: number,
+  options: SerializeBinaryOptions = {}
+): Buffer {
+  const isInt8 = options.quantize === 'int8';
+  const bytesPerElement = isInt8 ? 1 : 4;
+  const count = index.chunks.length;
+  const vectorsBytes = count * dim * bytesPerElement;
+  const vectorsOffset = BINARY_HEADER_SIZE;
+  const flags = isInt8 ? BINARY_FLAG_INT8 : BINARY_FLAG_FP32;
+
+  // Build contiguous typed array
+  const rawArray = isInt8 ? new Int8Array(count * dim) : new Float32Array(count * dim);
+
+  for (let i = 0; i < count; i++) {
+    const chunk = index.chunks[i];
+    let v: ArrayLike<number>;
+    if (typeof chunk.vec === 'string') {
+      v = decodeVec(chunk.vec, dim);
+    } else if (chunk.vec instanceof Float32Array || chunk.vec instanceof Int8Array || Array.isArray(chunk.vec)) {
+      v = chunk.vec;
+    } else {
+      throw new Error(`chunk ${i} is missing vector for binary serialization`);
+    }
+
+    if (isInt8) {
+      const q = v instanceof Int8Array ? v : quantizeToInt8(v);
+      (rawArray as Int8Array).set(q, i * dim);
+    } else {
+      (rawArray as Float32Array).set(v, i * dim);
+    }
+  }
+
+  // Build metadata JSON omitting per-chunk vec strings to save space
+  const metadataChunks: Array<Omit<IndexChunk, 'vec'>> = index.chunks.map((c) => ({
+    file: c.file,
+    title: c.title,
+    heading: c.heading,
+    headingPath: c.headingPath,
+    text: c.text,
+    chunkHash: c.chunkHash,
+    startLine: c.startLine,
+    endLine: c.endLine,
+    meta: c.meta,
+  }));
+
+  const metadataObj = {
+    schemaVersion: index.schemaVersion,
+    format: 'binary-v1',
+    model: index.model,
+    modelAlias: index.modelAlias,
+    adapterFingerprint: index.adapterFingerprint,
+    dim,
+    db: index.db,
+    built: index.built,
+    complete: index.complete,
+    chunkCount: count,
+    quantized: isInt8 ? 'int8' : 'fp32',
+    lexical: index.lexical,
+    lexicalFormat: index.lexicalFormat,
+    chunks: metadataChunks,
+  };
+
+  const metadataBuffer = Buffer.from(JSON.stringify(metadataObj), 'utf8');
+  const metadataOffset = vectorsOffset + vectorsBytes;
+  const metadataBytes = metadataBuffer.length;
+
+  const totalSize = metadataOffset + metadataBytes;
+  const out = Buffer.alloc(totalSize);
+
+  // Write 64-byte header
+  out.write(BINARY_MAGIC, 0, 8, 'ascii');
+  out.writeUInt32LE(BINARY_VERSION, 8);
+  out.writeUInt32LE(flags, 12);
+  out.writeUInt32LE(dim, 16);
+  out.writeUInt32LE(count, 20);
+  out.writeBigUInt64LE(BigInt(vectorsOffset), 24);
+  out.writeBigUInt64LE(BigInt(vectorsBytes), 32);
+  out.writeBigUInt64LE(BigInt(metadataOffset), 40);
+  out.writeBigUInt64LE(BigInt(metadataBytes), 48);
+
+  // Copy vectors
+  Buffer.from(rawArray.buffer, rawArray.byteOffset, rawArray.byteLength).copy(out, vectorsOffset);
+
+  // Copy metadata
+  metadataBuffer.copy(out, metadataOffset);
+
+  return out;
+}
+
+export function readBinaryHeader(buffer: Buffer): BinaryIndexHeader {
+  if (buffer.length < BINARY_HEADER_SIZE) {
+    throw new Error(`Invalid binary index: file too small (${buffer.length} bytes, expected at least ${BINARY_HEADER_SIZE})`);
+  }
+  const magic = buffer.toString('ascii', 0, 8);
+  if (magic !== BINARY_MAGIC) {
+    throw new Error(`Invalid binary index magic: expected "${BINARY_MAGIC}", got "${magic}"`);
+  }
+  const version = buffer.readUInt32LE(8);
+  const flags = buffer.readUInt32LE(12);
+  const dim = buffer.readUInt32LE(16);
+  const chunkCount = buffer.readUInt32LE(20);
+  const vectorsOffset = Number(buffer.readBigUInt64LE(24));
+  const vectorsBytes = Number(buffer.readBigUInt64LE(32));
+  const metadataOffset = Number(buffer.readBigUInt64LE(40));
+  const metadataBytes = Number(buffer.readBigUInt64LE(48));
+
+  return {
+    version,
+    flags,
+    dim,
+    chunkCount,
+    vectorsOffset,
+    vectorsBytes,
+    metadataOffset,
+    metadataBytes,
+  };
+}
+
+export function deserializeBinaryIndex(
+  buffer: Buffer
+): PersistedIndex & { rawVectorsBuffer: Float32Array | Int8Array; quantized?: 'fp32' | 'int8' } {
+  const header = readBinaryHeader(buffer);
+  const { flags, dim, chunkCount, vectorsOffset, vectorsBytes, metadataOffset, metadataBytes } = header;
+  const isInt8 = (flags & BINARY_FLAG_INT8) !== 0;
+  const expectedVectorsBytes = chunkCount * dim * (isInt8 ? 1 : 4);
+  if (buffer.length < vectorsOffset + expectedVectorsBytes || buffer.length < metadataOffset + metadataBytes) {
+    throw new Error('Corrupt binary index: unexpected truncated file');
+  }
+  const byteOffset = buffer.byteOffset + vectorsOffset;
+  let rawVectorsBuffer: Float32Array | Int8Array;
+
+  if (isInt8) {
+    rawVectorsBuffer = new Int8Array(buffer.buffer, byteOffset, chunkCount * dim);
+  } else {
+    // Aligned 4-byte Float32Array view
+    if (byteOffset % 4 === 0) {
+      rawVectorsBuffer = new Float32Array(buffer.buffer, byteOffset, chunkCount * dim);
+    } else {
+      const aligned = Buffer.alloc(vectorsBytes);
+      buffer.copy(aligned, 0, vectorsOffset, vectorsOffset + vectorsBytes);
+      rawVectorsBuffer = new Float32Array(aligned.buffer, aligned.byteOffset, chunkCount * dim);
+    }
+  }
+
+  // Deserialize metadata JSON
+  const metaJson = buffer.toString('utf8', metadataOffset, metadataOffset + metadataBytes);
+  const parsed = JSON.parse(metaJson);
+
+  // Re-attach subarray views to chunks
+  const chunks: IndexChunk[] = parsed.chunks.map((c: any, i: number) => {
+    const vecSlice = rawVectorsBuffer.subarray(i * dim, (i + 1) * dim);
+    return {
+      ...c,
+      vec: vecSlice,
+    };
+  });
+
+  return {
+    ...parsed,
+    chunks,
+    rawVectorsBuffer,
+    quantized: isInt8 ? 'int8' : 'fp32',
+  };
+}