md-semantic-search 0.5.0 → 1.0.2

package/dist/search.d.ts

@@ -2,6 +2,8 @@ import { ModelDescriptor } from './core.js';
 import { LexicalIndex } from './lexical.js';
 import { DocumentMetadata } from './frontmatter.js';
 import { IVFIndex } from './ivf.js';
+import { RelationshipGraph } from './wikilinks.js';
+import { FilterNode } from './filter.js';
 export { tokenize } from './lexical.js';
 export declare const _stats: {
     corpusTokenizedChars: number;
@@ -10,7 +12,7 @@ export declare function keywordScores(chunks: IndexChunk[], query: string): numb
 export declare function rrf(rankings: Array<Array<{
     idx: number;
     score: number;
-}>>, k?: number): Map<number, number>;
+}>>, k?: number, weights?: number[]): Map<number, number>;
 export interface IndexChunk {
     file: string;
     title: string;
@@ -61,7 +63,7 @@ export interface RuntimeChunk {
     startLine?: number;
     endLine?: number;
     meta?: DocumentMetadata;
-    vec: Float32Array;
+    vec: Float32Array | Int8Array;
 }
 export interface RuntimeIndexState {
     schema: number;
@@ -72,17 +74,24 @@ export interface RuntimeIndexState {
     model: ModelDescriptor;
     queryCache: QueryEmbeddingCache;
     ivf?: IVFIndex;
+    graph?: RelationshipGraph;
+    pageRank?: Map<string, number>;
 }
+export declare function getRuntimeGraph(runtime: RuntimeIndexState): {
+    graph: RelationshipGraph;
+    pageRank: Map<string, number>;
+};
 export declare function loadIndex(indexDir: string): {
     index: IndexFile;
     model: ModelDescriptor;
 };
 export declare const MAX_QUERY_LENGTH = 2048;
+export interface LoadedIndex {
+    index: IndexFile;
+    model: ModelDescriptor;
+}
 export interface SearchOptions {
-    loaded: {
-        index: IndexFile;
-        model: ModelDescriptor;
-    };
+    loaded: LoadedIndex;
     cacheDir: string;
     query: string;
     k?: number;
@@ -106,6 +115,8 @@ export interface SearchOptions {
     useQueryCache?: boolean;
     ann?: boolean;
     nprobe?: number;
+    graphBoost?: number;
+    filter?: string | FilterNode;
 }
 export interface SearchResultHit {
     file: string;
@@ -116,6 +127,7 @@ export interface SearchResultHit {
     matches: string[];
     snippet: string;
     meta?: DocumentMetadata;
+    graphScore?: number;
     explain?: {
         cosine: number;
         rrfScore: number;
@@ -125,6 +137,8 @@ export interface SearchResultHit {
             headingPath: number;
             body: number;
         };
+        graphScore?: number;
+        pageRank?: number;
     };
     rerankScore?: number;
 }

package/dist/search.js

@@ -7,6 +7,10 @@ import { rerankScores } from './rerank.js';
 import { bm25Scores, matchingTerms, tokenize, fuzzyTitleAliasScores } from './lexical.js';
 import { collapseResults } from './collapse.js';
 import { trainIVF, searchIVFCandidates, deserializeIVF, ANN_THRESHOLD, DEFAULT_NPROBE } from './ivf.js';
+import { buildRelationshipGraph, computePageRank, expandGraphNeighborhood } from './wikilinks.js';
+import { evaluateFilter } from './filter.js';
+import { deserializeBinaryIndex } from './binary-format.js';
+import { dequantizeFromInt8 } from './quantization.js';
 export { tokenize } from './lexical.js';
 const chunkTokens = new WeakMap();
 export const _stats = { corpusTokenizedChars: 0 };
@@ -33,9 +37,13 @@ export function keywordScores(chunks, query) {
         return s;
     });
 }
-export function rrf(rankings, k = 60) {
+export function rrf(rankings, k = 60, weights) {
     const fused = new Map();
-    for (const ranking of rankings) {
+    for (let rankingIdx = 0; rankingIdx < rankings.length; rankingIdx++) {
+        const ranking = rankings[rankingIdx];
+        const weight = weights ? (weights[rankingIdx] ?? 1) : 1;
+        if (weight <= 0)
+            continue;
         const sorted = [...ranking].sort((a, b) => b.score - a.score);
         let scoreRank = 0;
         sorted.forEach((item, rank) => {
@@ -43,7 +51,7 @@ export function rrf(rankings, k = 60) {
                 return;
             if (rank > 0 && item.score < sorted[rank - 1].score)
                 scoreRank = rank;
-            fused.set(item.idx, (fused.get(item.idx) || 0) + 1 / (k + scoreRank + 1));
+            fused.set(item.idx, (fused.get(item.idx) || 0) + weight / (k + scoreRank + 1));
         });
     }
     return fused;
@@ -116,22 +124,51 @@ export class QueryEmbeddingCache {
         return this.cache.size;
     }
 }
+export function getRuntimeGraph(runtime) {
+    if (runtime.graph && runtime.pageRank) {
+        return { graph: runtime.graph, pageRank: runtime.pageRank };
+    }
+    const docsMap = new Map();
+    for (const chunk of runtime.chunks) {
+        const existing = docsMap.get(chunk.file);
+        if (!existing) {
+            docsMap.set(chunk.file, {
+                file: chunk.file,
+                title: chunk.title,
+                meta: chunk.meta,
+                text: chunk.text,
+            });
+        }
+        else {
+            existing.text = (existing.text || '') + '\n' + chunk.text;
+        }
+    }
+    const docs = Array.from(docsMap.values());
+    const graph = buildRelationshipGraph(docs);
+    const pageRank = computePageRank(graph);
+    runtime.graph = graph;
+    runtime.pageRank = pageRank;
+    return { graph, pageRank };
+}
 const runtimeIndexStates = new WeakMap();
 function snapshotRuntimeIndex(index, validation) {
     const dim = validation.dim || (index.chunks[0]?.vec?.length ?? 0);
     const count = index.chunks.length;
-    const vectorsBuffer = dim > 0 ? new Float32Array(count * dim) : null;
+    const isZeroCopy = !!validation.rawVectorsBuffer;
+    const vectorsBuffer = validation.rawVectorsBuffer ?? (dim > 0 ? new Float32Array(count * dim) : null);
     const chunks = index.chunks.map((chunk, i) => {
         let vec;
         if (vectorsBuffer && dim > 0) {
             const offset = i * dim;
-            if (chunk.vec instanceof Float32Array) {
-                vectorsBuffer.set(chunk.vec, offset);
-            }
-            else {
-                const arr = chunk.vec;
-                for (let j = 0; j < dim; j++)
-                    vectorsBuffer[offset + j] = arr[j];
+            if (!isZeroCopy) {
+                if (chunk.vec instanceof Float32Array || chunk.vec instanceof Int8Array) {
+                    vectorsBuffer.set(chunk.vec, offset);
+                }
+                else {
+                    const arr = chunk.vec;
+                    for (let j = 0; j < dim; j++)
+                        vectorsBuffer[offset + j] = arr[j];
+                }
             }
             vec = vectorsBuffer.subarray(offset, offset + dim);
         }
@@ -224,7 +261,69 @@ function warnIfStale(index) {
     process.stderr.write(`warning: index is ${mins} min older than the newest change in ${index.db}; run \`mdss index\` to refresh.\n`);
 }
 export function loadIndex(indexDir) {
+    const vectorsBinPath = path.join(indexDir, 'vectors.bin');
     const vectorsPath = path.join(indexDir, 'vectors.json');
+    let useBinary = false;
+    if (fs.existsSync(vectorsBinPath)) {
+        if (!fs.existsSync(vectorsPath)) {
+            useBinary = true;
+        }
+        else {
+            try {
+                const binStat = fs.statSync(vectorsBinPath);
+                const jsonStat = fs.statSync(vectorsPath);
+                if (binStat.mtimeMs >= jsonStat.mtimeMs) {
+                    useBinary = true;
+                }
+            }
+            catch {
+                useBinary = false;
+            }
+        }
+    }
+    if (useBinary) {
+        try {
+            const binBuffer = fs.readFileSync(vectorsBinPath);
+            const shaBinPath = path.join(indexDir, 'vectors.bin.sha256');
+            if (fs.existsSync(shaBinPath)) {
+                const expectedHash = fs.readFileSync(shaBinPath, 'utf8').trim().split(/\s+/)[0];
+                const actualHash = crypto.createHash('sha256').update(binBuffer).digest('hex');
+                if (expectedHash && actualHash !== expectedHash) {
+                    throw new Error('vectors.bin integrity check failed (SHA-256 mismatch) — run `mdss index` to rebuild.');
+                }
+            }
+            const deserialized = deserializeBinaryIndex(binBuffer);
+            const validated = validateIndexEnvelope(deserialized, vectorsBinPath, { encoding: 'loaded', validateVectors: true });
+            const index = validated.index;
+            const { model, dim: validatedDim } = validated;
+            const expectedDim = validatedDim ?? deserialized.dim;
+            warnIfStale(index);
+            const ivfPath = path.join(indexDir, 'ivf.json');
+            let ivf;
+            if (fs.existsSync(ivfPath)) {
+                try {
+                    const raw = JSON.parse(fs.readFileSync(ivfPath, 'utf8'));
+                    ivf = deserializeIVF(raw);
+                }
+                catch { }
+            }
+            runtimeIndexStates.set(index, snapshotRuntimeIndex(index, {
+                schema: validated.schema,
+                dim: expectedDim,
+                model,
+                ivf,
+                rawVectorsBuffer: deserialized.rawVectorsBuffer,
+            }));
+            return { index, model };
+        }
+        catch (e) {
+            if (e.message?.includes('integrity check failed'))
+                throw e;
+            if (!fs.existsSync(vectorsPath))
+                throw e;
+            // If vectors.bin was corrupt or unreadable, fall through to vectors.json fallback
+        }
+    }
     if (!fs.existsSync(vectorsPath)) {
         throw new Error(`No index at ${vectorsPath}. Run \`mdss index\` first.`);
     }
@@ -297,7 +396,8 @@ export function loadIndex(indexDir) {
 }
 export const MAX_QUERY_LENGTH = 2048;
 export async function searchIndex(opts) {
-    const { loaded, cacheDir, query, k = 6, semanticOnly = false, offline = false, path: pathFilter, since, rerank = false, rerankPool, tag, project, type, status, canonicalOnly, custom, } = opts;
+    const { loaded, cacheDir, query, k = 6, semanticOnly = false, offline = false, path: pathFilter, since, rerank = false, rerankPool, tag, project, type, status, canonicalOnly, custom, filter: filterExpr, } = opts;
+    const graphBoost = typeof opts.graphBoost === 'number' && Number.isFinite(opts.graphBoost) ? opts.graphBoost : 0;
     if (typeof query === 'string' && query.length > MAX_QUERY_LENGTH) {
         throw new Error(`query exceeds maximum length of ${MAX_QUERY_LENGTH} characters`);
     }
@@ -319,7 +419,8 @@ export async function searchIndex(opts) {
         type !== undefined ||
         status !== undefined ||
         canonicalOnly !== undefined ||
-        custom !== undefined;
+        custom !== undefined ||
+        filterExpr !== undefined;
     if (pathRes.length > 0 || sinceMs !== undefined || hasMetaFilter) {
         if (sinceMs !== undefined && !db) {
             throw new Error('--since requires an index that knows its --db (index.db missing). Re-run `mdss index`.');
@@ -342,6 +443,10 @@ export async function searchIndex(opts) {
                 if (m < sinceMs)
                     return false;
             }
+            if (filterExpr !== undefined) {
+                if (!evaluateFilter(filterExpr, chunk.meta, { file: chunk.file, title: chunk.title }))
+                    return false;
+            }
             if (tag !== undefined) {
                 const reqTags = (Array.isArray(tag) ? tag : [tag]).map((t) => String(t).toLowerCase().replace(/^#/, ''));
                 const chunkTags = chunk.meta?.tags || [];
@@ -390,7 +495,7 @@ export async function searchIndex(opts) {
     const useAnn = opts.ann ?? (runtime.ivf !== undefined && candidates.length >= ANN_THRESHOLD);
     if (useAnn && candidates.length >= ANN_THRESHOLD) {
         if (!runtime.ivf) {
-            const rawVecs = chunks.map((c) => c.vec);
+            const rawVecs = chunks.map((c) => (c.vec instanceof Float32Array ? c.vec : dequantizeFromInt8(c.vec)));
             runtime.ivf = trainIVF(rawVecs);
         }
         const qVecF32 = qVec instanceof Float32Array ? qVec : Float32Array.from(qVec);
@@ -409,12 +514,45 @@ export async function searchIndex(opts) {
     }
     const cosByIdx = new Map(semantic.map((s) => [s.idx, s.score]));
     const pool = rerank ? rerankPool || Math.max(20, k * 3) : k;
+    let graphRanking = [];
+    let graphMap;
+    if (graphBoost > 0) {
+        const { graph, pageRank } = getRuntimeGraph(runtime);
+        const seedFilesMap = new Map();
+        for (const s of semantic) {
+            if (s.score > 0) {
+                const file = chunks[s.idx].file;
+                const curr = seedFilesMap.get(file) || 0;
+                if (s.score > curr)
+                    seedFilesMap.set(file, s.score);
+            }
+        }
+        const seedFiles = Array.from(seedFilesMap.entries()).map(([file, score]) => ({ file, score }));
+        const propagationScores = expandGraphNeighborhood(graph, seedFiles, { maxDepth: 2, decay: 0.5 });
+        graphMap = new Map();
+        graphRanking = candidates.map(({ chunk, idx }) => {
+            const pr = pageRank.get(chunk.file) || 0;
+            const prop = propagationScores.get(chunk.file) || 0;
+            const totalGraph = pr + prop;
+            graphMap.set(idx, totalGraph);
+            return { idx, score: totalGraph };
+        });
+    }
     let ranked;
     if (semanticOnly) {
-        ranked = [...semantic]
-            .sort((a, b) => b.score - a.score)
-            .slice(0, pool)
-            .map((s) => ({ idx: s.idx, fscore: s.score, cos: s.score }));
+        if (graphBoost > 0) {
+            const fused = rrf([semantic, graphRanking], 60, [1, graphBoost]);
+            ranked = [...fused.entries()]
+                .map(([idx, fscore]) => ({ idx, fscore, cos: cosByIdx.get(idx) }))
+                .sort((a, b) => b.fscore - a.fscore)
+                .slice(0, pool);
+        }
+        else {
+            ranked = [...semantic]
+                .sort((a, b) => b.score - a.score)
+                .slice(0, pool)
+                .map((s) => ({ idx: s.idx, fscore: s.score, cos: s.score }));
+        }
     }
     else {
         const kw = lexicalState.kind === 'persisted-bm25'
@@ -427,11 +565,20 @@ export async function searchIndex(opts) {
             idx: candidates[position].idx,
             score,
         }));
-        const fused = rrf([semantic, kw, fuzzy]);
-        ranked = [...fused.entries()]
-            .map(([idx, fscore]) => ({ idx, fscore, cos: cosByIdx.get(idx) }))
-            .sort((a, b) => b.fscore - a.fscore)
-            .slice(0, pool);
+        if (graphBoost > 0) {
+            const fused = rrf([semantic, kw, fuzzy, graphRanking], 60, [1, 1, 1, graphBoost]);
+            ranked = [...fused.entries()]
+                .map(([idx, fscore]) => ({ idx, fscore, cos: cosByIdx.get(idx) }))
+                .sort((a, b) => b.fscore - a.fscore)
+                .slice(0, pool);
+        }
+        else {
+            const fused = rrf([semantic, kw, fuzzy]);
+            ranked = [...fused.entries()]
+                .map(([idx, fscore]) => ({ idx, fscore, cos: cosByIdx.get(idx) }))
+                .sort((a, b) => b.fscore - a.fscore)
+                .slice(0, pool);
+        }
     }
     if (rerank && ranked.length > 0) {
         const texts = ranked.map((r) => chunks[r.idx].text);
@@ -458,6 +605,8 @@ export async function searchIndex(opts) {
         const matches = lexicalState.kind === 'persisted-bm25'
             ? matchingTerms(lexicalState.lexical, queryTerms, r.idx)
             : queryTerms.filter((term) => tokenSet(c).has(term));
+        const gScore = graphMap?.get(r.idx);
+        const prScore = runtime.pageRank?.get(c.file);
         return {
             file: c.file,
             title: c.title,
@@ -467,12 +616,15 @@ export async function searchIndex(opts) {
             matches,
             snippet: c.text.replace(/\s+/g, ' ').slice(0, 220),
             ...(hasMetaFields(c.meta) ? { meta: c.meta } : {}),
+            ...(gScore !== undefined ? { graphScore: +gScore.toFixed(4) } : {}),
             ...(explain
                 ? {
                     explain: {
                         cosine: +r.cos.toFixed(4),
                         rrfScore: +r.fscore.toFixed(4),
                         bm25Weights: { title: 3.0, aliases: 3.0, headingPath: 1.8, body: 1.0 },
+                        ...(gScore !== undefined ? { graphScore: +gScore.toFixed(4) } : {}),
+                        ...(prScore !== undefined ? { pageRank: +prScore.toFixed(4) } : {}),
                     },
                 }
                 : {}),

package/dist/serve.js

@@ -10,6 +10,7 @@ import path from 'node:path';
 import { buildIndex } from './indexer.js';
 import { loadIndex, searchIndex } from './search.js';
 import { walkMarkdown, assertSafePath } from './core.js';
+import { createFileWatcher } from './watcher.js';
 const DEFAULT_PORT = 8747;
 const DEFAULT_HOST = '127.0.0.1';
 const WATCH_INTERVAL_MS = 3000;
@@ -39,13 +40,19 @@ export async function createServe(opts) {
         apiKey,
         healthPublic,
     };
+    const sockets = new Set();
     const server = http.createServer((req, res) => {
         handleRequest(req, res, state).catch((e) => {
             json(res, 500, { error: e.message });
         });
     });
+    server.on('connection', (socket) => {
+        sockets.add(socket);
+        socket.on('close', () => sockets.delete(socket));
+    });
     let stopped = false;
     let timer = null;
+    let resolveTimer = null;
     const reload = async () => {
         if (!db)
             return false;
@@ -140,14 +147,21 @@ export async function createServe(opts) {
         }
         return out;
     };
+    let lastFingerprint = '';
     const watchLoop = async () => {
         let indexedHashes = readIndexedHashes();
-        let lastFingerprint = treeContentFingerprint(scanTree());
+        lastFingerprint = treeContentFingerprint(scanTree());
         let lastChangeAt = 0;
         let pending = false;
         while (!stopped) {
             await new Promise((r) => {
-                timer = setTimeout(r, watchInterval);
+                resolveTimer = r;
+                timer = setTimeout(() => {
+                    timer = null;
+                    resolveTimer = null;
+                    r();
+                }, watchInterval);
+                timer.unref?.();
             });
             if (stopped)
                 return;
@@ -182,15 +196,40 @@ export async function createServe(opts) {
             }
         }
     };
+    let nativeWatcher = null;
     if (watch) {
         if (!db)
             throw new Error('serve --watch requires --db');
+        nativeWatcher = createFileWatcher(db, () => {
+            // Prompt watchLoop settle trigger on native event
+            lastFingerprint = '';
+        }, {
+            debounceMs: watchDelay,
+            ignore,
+            log,
+        });
         watchLoop();
     }
     const close = async () => {
         stopped = true;
-        if (timer)
+        if (timer) {
             clearTimeout(timer);
+            timer = null;
+        }
+        if (resolveTimer) {
+            resolveTimer();
+            resolveTimer = null;
+        }
+        if (nativeWatcher)
+            nativeWatcher.close();
+        server.closeAllConnections?.();
+        for (const socket of sockets) {
+            try {
+                socket.destroy();
+            }
+            catch { }
+        }
+        sockets.clear();
         await new Promise((resolve, reject) => server.close((err) => (err ? reject(err) : resolve())));
     };
     return { server, state, close };
@@ -282,6 +321,8 @@ async function handleRequest(req, res, state) {
                 maxPerFile: payload.maxPerFile || payload.maxPerDoc,
                 ann: payload.ann,
                 nprobe: payload.nprobe,
+                graphBoost: payload.graphBoost,
+                filter: payload.filter,
             });
             json(res, 200, { query, k, count: results.length, results });
         }

package/dist/summarize.d.ts

@@ -0,0 +1,16 @@
+export interface NoteAnalysis {
+    tags: string[];
+    summary: string;
+}
+/**
+ * Extracts salient topical tags from Markdown text using term frequency and position weighting.
+ */
+export declare function extractKeyphraseTags(text: string, maxTags?: number): string[];
+/**
+ * Generates a concise extractive summary (1-2 sentences) representing the salient points of a note.
+ */
+export declare function summarizePassage(text: string, maxSentences?: number): string;
+/**
+ * Analyzes a Markdown note to produce automated semantic tags and summary.
+ */
+export declare function analyzeNote(text: string): NoteAnalysis;

package/dist/summarize.js

@@ -0,0 +1,82 @@
+/**
+ * Automated Note Summarization & Semantic Keyword Tagging Pipeline (issue #103).
+ *
+ * Implements zero-dependency, deterministic extractive summarization and keyphrase
+ * tag extraction to enrich note chunks with discoverable semantic metadata.
+ */
+import { tokenize } from './lexical.js';
+/**
+ * Extracts salient topical tags from Markdown text using term frequency and position weighting.
+ */
+export function extractKeyphraseTags(text, maxTags = 5) {
+    if (!text || !text.trim())
+        return [];
+    const lines = text.split('\n');
+    const termScores = new Map();
+    lines.forEach((line, lineIdx) => {
+        const isHeading = /^#{1,6}\s+/.test(line);
+        const weight = isHeading ? 3.0 : lineIdx < 3 ? 1.5 : 1.0;
+        const tokens = tokenize(line);
+        for (const tok of tokens) {
+            if (tok.length < 3)
+                continue;
+            // Skip purely numeric tokens
+            if (/^\d+$/.test(tok))
+                continue;
+            const current = termScores.get(tok) || 0;
+            termScores.set(tok, current + weight);
+        }
+    });
+    const sorted = Array.from(termScores.entries())
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, maxTags)
+        .map(([term]) => term);
+    return sorted;
+}
+/**
+ * Generates a concise extractive summary (1-2 sentences) representing the salient points of a note.
+ */
+export function summarizePassage(text, maxSentences = 2) {
+    if (!text || !text.trim())
+        return '';
+    // Clean Markdown formatting
+    const clean = text
+        .replace(/^#{1,6}\s+.*$/gm, '')
+        .replace(/```[\s\S]*?```/g, '')
+        .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
+        .replace(/[*_`~]/g, '')
+        .trim();
+    const sentences = clean
+        .split(/(?<=[.?!])\s+|\n\n+/)
+        .map((s) => s.trim())
+        .filter((s) => s.length >= 20 && s.length <= 300);
+    if (sentences.length <= maxSentences) {
+        return sentences.join(' ');
+    }
+    const salientTags = new Set(extractKeyphraseTags(clean, 8));
+    const scored = sentences.map((sent, index) => {
+        const tokens = tokenize(sent);
+        let tagMatches = 0;
+        for (const t of tokens) {
+            if (salientTags.has(t))
+                tagMatches++;
+        }
+        // Favor earlier sentences and sentences containing salient keywords
+        const posScore = 1.0 / (index + 1);
+        const salienceScore = tagMatches / (tokens.length + 1);
+        const score = posScore * 0.4 + salienceScore * 0.6;
+        return { sent, score, index };
+    });
+    scored.sort((a, b) => b.score - a.score);
+    const selected = scored.slice(0, maxSentences).sort((a, b) => a.index - b.index);
+    return selected.map((s) => s.sent).join(' ');
+}
+/**
+ * Analyzes a Markdown note to produce automated semantic tags and summary.
+ */
+export function analyzeNote(text) {
+    return {
+        tags: extractKeyphraseTags(text),
+        summary: summarizePassage(text),
+    };
+}

package/dist/tui.d.ts

@@ -10,6 +10,7 @@ export interface TuiOptions {
     path?: string | string[];
     rerank?: boolean;
     rerankPool?: number;
+    rag?: boolean;
     embedFn?: any;
     rerankFn?: any;
     debounceMs?: number;