md-semantic-search 1.0.3 → 2.0.0

package/dist/hnsw.d.ts

@@ -0,0 +1,96 @@
+/**
+ * Pure JS/TS HNSW approximate nearest neighbor index (issue #201).
+ *
+ * Hierarchical Navigable Small World graphs (Malkov & Yashunin 2016): no
+ * training phase, incremental inserts, ~99% recall with sublinear search —
+ * the counterpart of the k-means IVF in `ivf.ts`, sharing its contract:
+ * build → serialize → persist → search as a candidate pre-filter in front
+ * of the exact sweep. Cosine distance (1 − dot on normalized vectors) is
+ * computed through an injected distance function so int8 zero-copy vectors
+ * work unchanged. Deleted nodes are tombstones — searches skip them, which
+ * is what lets the incremental watch rebuild reuse a persisted graph.
+ */
+export interface HnswIndex {
+    dim: number;
+    /** Max links per node on layers > 0. */
+    M: number;
+    /** Random level seed — rebuilds from the same insertion order are identical. */
+    seed: number;
+    /** Node index of the graph entry (top layer). */
+    entryPoint: number;
+    /** Highest layer in the graph. */
+    maxLevel: number;
+    /** levels[i] = top layer of node i. */
+    levels: number[];
+    /** neighbors[i][l] = link list of node i on layer l (l ≤ levels[i]). */
+    neighbors: number[][][];
+    /** Tombstoned node indices — skipped by search, kept for graph connectivity. */
+    deleted: Set<number>;
+}
+export interface SerializedHnsw {
+    format: 'hnsw-v1';
+    dim: number;
+    M: number;
+    seed: number;
+    entryPoint: number;
+    maxLevel: number;
+    count: number;
+    levels: number[];
+    neighbors: number[][][];
+    deleted: number[];
+}
+export interface BuildHnswOptions {
+    M?: number;
+    seed?: number;
+    /**
+     * Distance between the query/node vector at `a` and the stored node `b` —
+     * LOWER is closer (cosine distance: `1 − dot(a, vectors[b])`). Injected so
+     * int8 zero-copy and norm-cached sweeps work unchanged.
+     */
+    distance: (a: Float32Array | Int8Array | number[], b: number) => number;
+    vectors: Array<Float32Array | Int8Array | number[]>;
+}
+export declare const DEFAULT_HNSW_M = 16;
+export declare const DEFAULT_HNSW_EF = 64;
+export declare function buildHnsw(vectors: Array<Float32Array | Int8Array | number[]>, options: BuildHnswOptions): HnswIndex;
+/**
+ * Insert node `idx` (vector index) into the graph. Incremental — new indices
+ * attach to the existing structure without a rebuild; vectors for nodes NOT
+ * yet inserted are read from `vectors` when needed, so the caller can insert
+ * lazily. Tombstoned indices can be re-inserted (undeletes).
+ */
+export declare function insertHnsw(hnsw: HnswIndex, idx: number, distance: (a: Float32Array | Int8Array | number[], b: number) => number, vectors: Array<Float32Array | Int8Array | number[]>): void;
+/** Tombstone a node — searches skip it; the graph keeps it for connectivity. */
+export declare function deleteHnsw(hnsw: HnswIndex, idx: number): void;
+/**
+ * Algorithm 4 of the HNSW paper (Malkov & Yashunin 2016): candidates sorted
+ * by distance to the query enter the result in order; a candidate is skipped
+ * when it is closer to some already-selected neighbor than to the query
+ * (redundant direction). `pairwise(a, b)` measures node-to-node distance.
+ * If fewer than `m` survive (disconnected candidate set), the closest
+ * leftovers backfill so the link count stays useful.
+ */
+export declare function selectNeighborsHeuristic(candidates: Array<{
+    node: number;
+    dist: number;
+}>, m: number, pairwise: (a: number, b: number) => number): Array<{
+    node: number;
+    dist: number;
+}>;
+/**
+ * ANN candidate pre-filter: ef-search from the entry point, returning LIVE
+ * candidate chunk indices (tombstones excluded). Mirrors
+ * `searchIVFCandidates` — the result feeds the same probed-candidate set.
+ */
+export declare function searchHnswCandidates(queryVec: Float32Array | Int8Array | number[], hnsw: HnswIndex, vectors: Array<Float32Array | Int8Array | number[]>, options?: {
+    ef?: number;
+    /** Lower-is-closer distance; defaults to cosine distance via `cosine`. */
+    distance?: (a: Float32Array | Int8Array | number[], b: number) => number;
+}): number[];
+export declare function serializeHnsw(hnsw: HnswIndex): SerializedHnsw;
+/**
+ * Hardened deserialization — every structural invariant is checked with the
+ * same discipline as `deserializeIVF`; any violation throws (the loader
+ * falls back to brute force).
+ */
+export declare function deserializeHnsw(raw: unknown, expectedChunksCount?: number): HnswIndex;

package/dist/hnsw.js

@@ -0,0 +1,326 @@
+/**
+ * Pure JS/TS HNSW approximate nearest neighbor index (issue #201).
+ *
+ * Hierarchical Navigable Small World graphs (Malkov & Yashunin 2016): no
+ * training phase, incremental inserts, ~99% recall with sublinear search —
+ * the counterpart of the k-means IVF in `ivf.ts`, sharing its contract:
+ * build → serialize → persist → search as a candidate pre-filter in front
+ * of the exact sweep. Cosine distance (1 − dot on normalized vectors) is
+ * computed through an injected distance function so int8 zero-copy vectors
+ * work unchanged. Deleted nodes are tombstones — searches skip them, which
+ * is what lets the incremental watch rebuild reuse a persisted graph.
+ */
+/** Deterministic PRNG for level assignment (same family as ivf.ts). */
+function mulberry32(seed) {
+    let s = seed >>> 0;
+    return () => {
+        s = (s + 0x6d2b79f5) >>> 0;
+        let t = Math.imul(s ^ (s >>> 15), 1 | s);
+        t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+export const DEFAULT_HNSW_M = 16;
+export const DEFAULT_HNSW_EF = 64;
+export function buildHnsw(vectors, options) {
+    const hnsw = {
+        dim: vectors[0]?.length ?? 0,
+        M: options.M ?? DEFAULT_HNSW_M,
+        seed: options.seed ?? 42,
+        entryPoint: -1,
+        maxLevel: -1,
+        levels: [],
+        neighbors: [],
+        deleted: new Set(),
+    };
+    for (let i = 0; i < vectors.length; i++)
+        insertHnsw(hnsw, i, options.distance, options.vectors);
+    return hnsw;
+}
+/**
+ * Insert node `idx` (vector index) into the graph. Incremental — new indices
+ * attach to the existing structure without a rebuild; vectors for nodes NOT
+ * yet inserted are read from `vectors` when needed, so the caller can insert
+ * lazily. Tombstoned indices can be re-inserted (undeletes).
+ */
+export function insertHnsw(hnsw, idx, distance, vectors) {
+    const vec = vectors[idx];
+    if (!vec)
+        throw new Error(`hnsw insert: no vector at index ${idx}`);
+    if (hnsw.levels[idx] !== undefined && !hnsw.deleted.has(idx)) {
+        throw new Error(`hnsw insert: node ${idx} already present`);
+    }
+    hnsw.deleted.delete(idx);
+    const M = hnsw.M;
+    const M0 = M * 2;
+    const mL = 1 / Math.log(M);
+    const rng = mulberry32(hnsw.seed + idx * 2654435761);
+    const u = Math.max(rng(), Number.MIN_VALUE);
+    const level = Math.min(Math.floor(-Math.log(u) * mL), 64);
+    hnsw.levels[idx] = level;
+    hnsw.neighbors[idx] = Array.from({ length: level + 1 }, () => []);
+    if (hnsw.entryPoint < 0) {
+        hnsw.entryPoint = idx;
+        hnsw.maxLevel = level;
+        return;
+    }
+    const qDist = (node) => distance(vec, node);
+    // 1. Greedy descent through layers above the node's level
+    let ep = hnsw.entryPoint;
+    let epDist = qDist(ep);
+    for (let l = hnsw.maxLevel; l > level; l--) {
+        let changed = true;
+        while (changed) {
+            changed = false;
+            for (const nb of hnsw.neighbors[ep][l] ?? []) {
+                const d = qDist(nb);
+                if (d < epDist) {
+                    epDist = d;
+                    ep = nb;
+                    changed = true;
+                }
+            }
+        }
+    }
+    // 2. Insert on each layer from min(level, maxLevel) down to 0
+    const pairwise = (a, b) => distance(vectors[a], b);
+    for (let l = Math.min(level, hnsw.maxLevel); l >= 0; l--) {
+        const efC = Math.max(M0, DEFAULT_HNSW_EF);
+        const candidates = searchLayer(hnsw, qDist, ep, epDist, l, efC);
+        const cap = l === 0 ? M0 : M;
+        // Algorithm 4: diversity-aware neighbor selection keeps the graph
+        // navigable — a candidate is skipped only when it is CLOSER to an
+        // already-selected neighbor than to the query
+        const selected = selectNeighborsHeuristic(candidates, cap, pairwise);
+        hnsw.neighbors[idx][l] = selected.map((c) => c.node);
+        for (const { node } of selected) {
+            const links = hnsw.neighbors[node][l];
+            if (!links.includes(idx))
+                links.push(idx);
+            if (links.length > cap) {
+                // re-select the neighbor's own links with the same heuristic — the
+                // diversity rule preserves connectivity instead of blind top-M,
+                // which used to sever the just-added edge and strand whole regions
+                const pruned = selectNeighborsHeuristic(links.map((n) => ({ node: n, dist: pairwise(node, n) })), cap, pairwise);
+                hnsw.neighbors[node][l] = pruned.map((c) => c.node);
+            }
+        }
+        if (candidates.length > 0) {
+            ep = candidates[0].node;
+            epDist = candidates[0].dist;
+        }
+    }
+    if (level > hnsw.maxLevel) {
+        hnsw.maxLevel = level;
+        hnsw.entryPoint = idx;
+    }
+}
+/** Tombstone a node — searches skip it; the graph keeps it for connectivity. */
+export function deleteHnsw(hnsw, idx) {
+    if (hnsw.levels[idx] !== undefined)
+        hnsw.deleted.add(idx);
+}
+/** ef-search one layer, returning the closest `ef` live candidates (closest first). */
+function searchLayer(hnsw, qDist, ep, epDist, layer, ef) {
+    const visited = new Set([ep]);
+    // max-heap by dist via simple sorted insertion (candidate lists are small)
+    const candidates = [{ node: ep, dist: epDist }];
+    const results = hnsw.deleted.has(ep) ? [] : [{ node: ep, dist: epDist }];
+    while (candidates.length > 0) {
+        const current = candidates.shift();
+        const worst = results.length >= ef ? results[results.length - 1].dist : Infinity;
+        if (current.dist > worst)
+            break;
+        for (const nb of hnsw.neighbors[current.node][layer] ?? []) {
+            if (visited.has(nb))
+                continue;
+            visited.add(nb);
+            const d = qDist(nb);
+            const worstNow = results.length >= ef ? results[results.length - 1].dist : Infinity;
+            if (d < worstNow || results.length < ef) {
+                candidates.push({ node: nb, dist: d });
+                candidates.sort((a, b) => a.dist - b.dist);
+                if (!hnsw.deleted.has(nb)) {
+                    results.push({ node: nb, dist: d });
+                    results.sort((a, b) => a.dist - b.dist);
+                    if (results.length > ef)
+                        results.pop();
+                }
+            }
+        }
+    }
+    return results;
+}
+/**
+ * Algorithm 4 of the HNSW paper (Malkov & Yashunin 2016): candidates sorted
+ * by distance to the query enter the result in order; a candidate is skipped
+ * when it is closer to some already-selected neighbor than to the query
+ * (redundant direction). `pairwise(a, b)` measures node-to-node distance.
+ * If fewer than `m` survive (disconnected candidate set), the closest
+ * leftovers backfill so the link count stays useful.
+ */
+export function selectNeighborsHeuristic(candidates, m, pairwise) {
+    const sorted = [...candidates].sort((a, b) => a.dist - b.dist);
+    const result = [];
+    const skipped = [];
+    for (const cand of sorted) {
+        if (result.length >= m)
+            break;
+        let dominated = false;
+        for (const chosen of result) {
+            if (pairwise(cand.node, chosen.node) < cand.dist) {
+                dominated = true;
+                break;
+            }
+        }
+        if (dominated)
+            skipped.push(cand);
+        else
+            result.push(cand);
+    }
+    for (const cand of skipped) {
+        if (result.length >= m)
+            break;
+        result.push(cand);
+    }
+    return result;
+}
+/**
+ * ANN candidate pre-filter: ef-search from the entry point, returning LIVE
+ * candidate chunk indices (tombstones excluded). Mirrors
+ * `searchIVFCandidates` — the result feeds the same probed-candidate set.
+ */
+export function searchHnswCandidates(queryVec, hnsw, vectors, options = {}) {
+    if (hnsw.entryPoint < 0 || hnsw.levels.length === 0)
+        return [];
+    const distance = options.distance ??
+        ((a, b) => 1 - dotSimilarity(a, vectors[b]));
+    const qDist = (node) => distance(queryVec, node);
+    let ep = hnsw.entryPoint;
+    let epDist = qDist(ep);
+    for (let l = hnsw.maxLevel; l > 0; l--) {
+        let changed = true;
+        while (changed) {
+            changed = false;
+            for (const nb of hnsw.neighbors[ep][l] ?? []) {
+                const d = qDist(nb);
+                if (d < epDist) {
+                    epDist = d;
+                    ep = nb;
+                    changed = true;
+                }
+            }
+        }
+    }
+    const ef = Math.max(options.ef ?? DEFAULT_HNSW_EF, 1);
+    const found = searchLayer(hnsw, qDist, ep, epDist, 0, ef);
+    return found.map((f) => f.node);
+}
+/** Dot product that tolerates int8 storage (norm handling mirrors the exact sweep). */
+function dotSimilarity(a, b) {
+    let s = 0;
+    const n = Math.min(a.length, b.length);
+    for (let i = 0; i < n; i++)
+        s += a[i] * b[i];
+    return s;
+}
+export function serializeHnsw(hnsw) {
+    return {
+        format: 'hnsw-v1',
+        dim: hnsw.dim,
+        M: hnsw.M,
+        seed: hnsw.seed,
+        entryPoint: hnsw.entryPoint,
+        maxLevel: hnsw.maxLevel,
+        count: hnsw.levels.length,
+        levels: hnsw.levels,
+        neighbors: hnsw.neighbors,
+        deleted: [...hnsw.deleted].sort((a, b) => a - b),
+    };
+}
+/**
+ * Hardened deserialization — every structural invariant is checked with the
+ * same discipline as `deserializeIVF`; any violation throws (the loader
+ * falls back to brute force).
+ */
+export function deserializeHnsw(raw, expectedChunksCount) {
+    if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
+        throw new Error('invalid HNSW payload: expected JSON object — run `mdss index` to rebuild');
+    }
+    const obj = raw;
+    if (obj.format !== 'hnsw-v1') {
+        throw new Error(`invalid HNSW payload: format must be "hnsw-v1", got ${JSON.stringify(obj.format)} — run \`mdss index\` to rebuild`);
+    }
+    const { dim, M, seed, entryPoint, maxLevel, count } = obj;
+    if (typeof dim !== 'number' || !Number.isSafeInteger(dim) || dim <= 0 ||
+        typeof M !== 'number' || !Number.isSafeInteger(M) || M <= 0 || M > 128 ||
+        typeof seed !== 'number' || !Number.isSafeInteger(seed) ||
+        typeof entryPoint !== 'number' || !Number.isSafeInteger(entryPoint) || entryPoint < 0 ||
+        typeof maxLevel !== 'number' || !Number.isSafeInteger(maxLevel) || maxLevel < 0 ||
+        typeof count !== 'number' || !Number.isSafeInteger(count) || count <= 0) {
+        throw new Error('invalid HNSW payload: invalid header — run `mdss index` to rebuild');
+    }
+    if (!Array.isArray(obj.levels) || obj.levels.length !== count) {
+        throw new Error(`invalid HNSW payload: levels must be an array of ${count} — run \`mdss index\` to rebuild`);
+    }
+    const levels = [];
+    for (let i = 0; i < count; i++) {
+        const l = obj.levels[i];
+        if (typeof l !== 'number' || !Number.isSafeInteger(l) || l < 0 || l > maxLevel) {
+            throw new Error(`invalid HNSW payload: level of node ${i} must be an integer in [0, ${maxLevel}] — run \`mdss index\` to rebuild`);
+        }
+        levels.push(l);
+    }
+    if (levels[entryPoint] !== maxLevel) {
+        throw new Error(`invalid HNSW payload: entry point ${entryPoint} must sit on the top layer ${maxLevel} — run \`mdss index\` to rebuild`);
+    }
+    if (!Array.isArray(obj.neighbors) || obj.neighbors.length !== count) {
+        throw new Error(`invalid HNSW payload: neighbors must be an array of ${count} — run \`mdss index\` to rebuild`);
+    }
+    const neighbors = new Array(count);
+    const m0 = M * 2;
+    for (let i = 0; i < count; i++) {
+        const perLevel = obj.neighbors[i];
+        if (!Array.isArray(perLevel) || perLevel.length !== levels[i] + 1) {
+            throw new Error(`invalid HNSW payload: node ${i} must have ${levels[i] + 1} link lists — run \`mdss index\` to rebuild`);
+        }
+        const node = new Array(levels[i] + 1);
+        for (let l = 0; l <= levels[i]; l++) {
+            const list = perLevel[l];
+            if (!Array.isArray(list)) {
+                throw new Error(`invalid HNSW payload: node ${i} layer ${l} must be an array — run \`mdss index\` to rebuild`);
+            }
+            if (list.length > (l === 0 ? m0 : M)) {
+                throw new Error(`invalid HNSW payload: node ${i} layer ${l} exceeds the link cap — run \`mdss index\` to rebuild`);
+            }
+            const seen = new Set();
+            for (const nb of list) {
+                if (typeof nb !== 'number' || !Number.isSafeInteger(nb) || nb < 0 || nb >= count || nb === i) {
+                    throw new Error(`invalid HNSW payload: node ${i} layer ${l} has invalid link ${JSON.stringify(nb)} — run \`mdss index\` to rebuild`);
+                }
+                if (seen.has(nb)) {
+                    throw new Error(`invalid HNSW payload: duplicate link ${nb} on node ${i} layer ${l} — run \`mdss index\` to rebuild`);
+                }
+                seen.add(nb);
+            }
+            node[l] = list;
+        }
+        neighbors[i] = node;
+    }
+    let deleted = new Set();
+    if (obj.deleted !== undefined) {
+        if (!Array.isArray(obj.deleted)) {
+            throw new Error('invalid HNSW payload: deleted must be an array — run `mdss index` to rebuild');
+        }
+        for (const idx of obj.deleted) {
+            if (typeof idx !== 'number' || !Number.isSafeInteger(idx) || idx < 0 || idx >= count) {
+                throw new Error(`invalid HNSW payload: deleted index ${JSON.stringify(idx)} out of range — run \`mdss index\` to rebuild`);
+            }
+            deleted.add(idx);
+        }
+    }
+    if (expectedChunksCount !== undefined && count > expectedChunksCount) {
+        throw new Error(`invalid HNSW payload: ${count} nodes exceed ${expectedChunksCount} chunks — run \`mdss index\` to rebuild`);
+    }
+    return { dim, M, seed, entryPoint, maxLevel, levels, neighbors, deleted };
+}

package/dist/index-format.js

@@ -90,7 +90,14 @@ export function validateIndexEnvelope(value, location, options = {}) {
     else if (modelName !== undefined && modelName !== null && typeof modelName !== 'string') {
         throw new Error(`${location} model metadata must be a string${REBUILD}`);
     }
-    const model = resolveModel(modelName);
+    let model;
+    try {
+        const candidateDim = typeof index.dim === 'number' && index.dim > 0 ? index.dim : undefined;
+        model = resolveModel(modelName, candidateDim);
+    }
+    catch {
+        model = resolveModel(modelName);
+    }
     const dim = resolveIndexDimension(index.dim, model.dim);
     if (schema < 3) {
         return { index, schema, model, dim };

package/dist/index.d.ts

@@ -23,16 +23,16 @@ export { buildIndex, chunkHash } from './indexer.js';
 export { search, searchIndex, loadIndex, tokenize, keywordScores, rrf, QueryEmbeddingCache, } from './search.js';
 export { resolveModel, MODELS, DEFAULT_MODEL, normalizeAdapter, embeddingAdapterFingerprint, legacyEmbeddingAdapterFingerprint, } from './models.js';
 export { getReranker, rerankScores, RERANK_MODEL, } from './rerank.js';
-export { embed, getExtractor, cosine, walkMarkdown, parseFile, chunkMarkdown, splitFrontmatter, extractTitle, globToRegExp, encodeVec, decodeVec, isBinaryIndex, } from './core.js';
-export { MCP_TOOLS, handleMcpRequest, startMcpServer, } from './mcp.js';
+export { embed, getExtractor, cosine, truncateAndNormalizeVector, walkMarkdown, parseFile, chunkMarkdown, splitFrontmatter, extractTitle, globToRegExp, encodeVec, decodeVec, isBinaryIndex, getDocLines, GetDocLinesOptions, GetDocLinesResult, } from './core.js';
+export { MCP_TOOLS, MCP_RESOURCE_TEMPLATES, MCP_PROMPTS, handleMcpRequest, startMcpServer, } from './mcp.js';
 export { trainIVF, searchIVFCandidates, serializeIVF, deserializeIVF, ANN_THRESHOLD, DEFAULT_NPROBE, } from './ivf.js';
-export { extractLinks, resolveLinks, buildRelationshipGraph, getRelatedNotes, computePageRank, expandGraphNeighborhood, } from './wikilinks.js';
+export { extractLinks, resolveLinks, buildRelationshipGraph, getRelatedNotes, computePageRank, expandGraphNeighborhood, findRelatedNotes, FindRelatedNotesOptions, RelatedNoteHit, RelatedNoteResult, } from './wikilinks.js';
 export { parseFilter, tokenizeFilter, evaluateFilter, FilterNode, } from './filter.js';
 export { serializeBinaryIndex, deserializeBinaryIndex, readBinaryHeader, BINARY_MAGIC, BINARY_HEADER_SIZE, } from './binary-format.js';
 export { quantizeToInt8, dequantizeFromInt8, asymmetricCosineInt8, INT8_SCALE, INT8_INV_SCALE, } from './quantization.js';
 export { searchFederated, FederatedSearchOptions, FederatedSearchResultHit, VaultConfig, } from './federation.js';
 export { createFileWatcher, FileWatcher, WatcherOptions, } from './watcher.js';
-export { askQuestion, synthesizeAnswer, extractAnswerFallback, AskOptions, SynthesisResult, Citation, DEFAULT_RAG_SYSTEM_PROMPT, } from './rag.js';
+export { askQuestion, synthesizeAnswer, extractAnswerFallback, chatTurn, loadChatSession, saveChatSession, listChatSessions, getSessionsDir, AskOptions, SynthesisResult, Citation, DEFAULT_RAG_SYSTEM_PROMPT, ChatSession, ChatTurn, ChatSourceManifestItem, ChatTurnOptions, } from './rag.js';
 export { extractKeyphraseTags, summarizePassage, analyzeNote, NoteAnalysis, } from './summarize.js';
 export { runTui, TuiOptions } from './tui.js';
 export { generateCompletion, FLAGS, COMMANDS, COMPLETION_SHELLS, } from './completions.js';

package/dist/index.js

@@ -23,16 +23,16 @@ export { buildIndex, chunkHash } from './indexer.js';
 export { search, searchIndex, loadIndex, tokenize, keywordScores, rrf, QueryEmbeddingCache, } from './search.js';
 export { resolveModel, MODELS, DEFAULT_MODEL, normalizeAdapter, embeddingAdapterFingerprint, legacyEmbeddingAdapterFingerprint, } from './models.js';
 export { getReranker, rerankScores, RERANK_MODEL, } from './rerank.js';
-export { embed, getExtractor, cosine, walkMarkdown, parseFile, chunkMarkdown, splitFrontmatter, extractTitle, globToRegExp, encodeVec, decodeVec, isBinaryIndex, } from './core.js';
-export { MCP_TOOLS, handleMcpRequest, startMcpServer, } from './mcp.js';
+export { embed, getExtractor, cosine, truncateAndNormalizeVector, walkMarkdown, parseFile, chunkMarkdown, splitFrontmatter, extractTitle, globToRegExp, encodeVec, decodeVec, isBinaryIndex, getDocLines, } from './core.js';
+export { MCP_TOOLS, MCP_RESOURCE_TEMPLATES, MCP_PROMPTS, handleMcpRequest, startMcpServer, } from './mcp.js';
 export { trainIVF, searchIVFCandidates, serializeIVF, deserializeIVF, ANN_THRESHOLD, DEFAULT_NPROBE, } from './ivf.js';
-export { extractLinks, resolveLinks, buildRelationshipGraph, getRelatedNotes, computePageRank, expandGraphNeighborhood, } from './wikilinks.js';
+export { extractLinks, resolveLinks, buildRelationshipGraph, getRelatedNotes, computePageRank, expandGraphNeighborhood, findRelatedNotes, } from './wikilinks.js';
 export { parseFilter, tokenizeFilter, evaluateFilter, } from './filter.js';
 export { serializeBinaryIndex, deserializeBinaryIndex, readBinaryHeader, BINARY_MAGIC, BINARY_HEADER_SIZE, } from './binary-format.js';
 export { quantizeToInt8, dequantizeFromInt8, asymmetricCosineInt8, INT8_SCALE, INT8_INV_SCALE, } from './quantization.js';
 export { searchFederated, } from './federation.js';
 export { createFileWatcher, } from './watcher.js';
-export { askQuestion, synthesizeAnswer, extractAnswerFallback, DEFAULT_RAG_SYSTEM_PROMPT, } from './rag.js';
+export { askQuestion, synthesizeAnswer, extractAnswerFallback, chatTurn, loadChatSession, saveChatSession, listChatSessions, getSessionsDir, DEFAULT_RAG_SYSTEM_PROMPT, } from './rag.js';
 export { extractKeyphraseTags, summarizePassage, analyzeNote, } from './summarize.js';
 export { runTui } from './tui.js';
 export { generateCompletion, FLAGS, COMMANDS, COMPLETION_SHELLS, } from './completions.js';

package/dist/indexer.d.ts

@@ -18,6 +18,14 @@ export interface PersistedIndex {
     model?: string | null;
     modelAlias?: string;
     adapterFingerprint?: string;
+    /**
+     * issue #202: the effective chunk token budget these chunks were built
+     * with (null = legacy char budget). A mismatch forces a full re-chunk —
+     * otherwise the file-level fast path would keep stale char-budget chunks
+     * for unchanged files and produce a mixed index. Identical chunk texts
+     * still reuse their cached vectors through chunkHash.
+     */
+    chunkBudget?: number | null;
     dim?: number;
     db?: string;
     built?: string;
@@ -57,6 +65,7 @@ export interface BuildIndexOptions {
     indexDir: string;
     cacheDir: string;
     modelName: string;
+    dimensions?: number;
     ignore?: string[];
     offline?: boolean;
     log?: (s: string) => void;
@@ -65,8 +74,16 @@ export interface BuildIndexOptions {
     maxRetries?: number;
     onProgress?: (done: number, total: number, chunksPerSec: number) => void;
     workers?: number;
-    ann?: boolean;
+    /** issue #201: ANN sidecar engine — `true`/`'ivf'` = ivf.json, `'hnsw'` = hnsw.json. */
+    ann?: boolean | 'ivf' | 'hnsw';
     quantize?: 'fp32' | 'int8';
+    /**
+     * issue #202: chunk budget in TOKENS (overrides the legacy char budget).
+     * Default: min(adapter.maxTokens, 512) — the point of the issue is that the
+     * char budget overflowed real maxTokens for Cyrillic. Explicit `0` restores
+     * the legacy char behavior.
+     */
+    maxChunkTokens?: number;
 }
 export interface BuildIndexResult {
     files: number;