md-semantic-search 1.1.0 → 2.0.0

package/dist/hnsw.js

@@ -0,0 +1,326 @@
+/**
+ * Pure JS/TS HNSW approximate nearest neighbor index (issue #201).
+ *
+ * Hierarchical Navigable Small World graphs (Malkov & Yashunin 2016): no
+ * training phase, incremental inserts, ~99% recall with sublinear search —
+ * the counterpart of the k-means IVF in `ivf.ts`, sharing its contract:
+ * build → serialize → persist → search as a candidate pre-filter in front
+ * of the exact sweep. Cosine distance (1 − dot on normalized vectors) is
+ * computed through an injected distance function so int8 zero-copy vectors
+ * work unchanged. Deleted nodes are tombstones — searches skip them, which
+ * is what lets the incremental watch rebuild reuse a persisted graph.
+ */
+/** Deterministic PRNG for level assignment (same family as ivf.ts). */
+function mulberry32(seed) {
+    let s = seed >>> 0;
+    return () => {
+        s = (s + 0x6d2b79f5) >>> 0;
+        let t = Math.imul(s ^ (s >>> 15), 1 | s);
+        t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+export const DEFAULT_HNSW_M = 16;
+export const DEFAULT_HNSW_EF = 64;
+export function buildHnsw(vectors, options) {
+    const hnsw = {
+        dim: vectors[0]?.length ?? 0,
+        M: options.M ?? DEFAULT_HNSW_M,
+        seed: options.seed ?? 42,
+        entryPoint: -1,
+        maxLevel: -1,
+        levels: [],
+        neighbors: [],
+        deleted: new Set(),
+    };
+    for (let i = 0; i < vectors.length; i++)
+        insertHnsw(hnsw, i, options.distance, options.vectors);
+    return hnsw;
+}
+/**
+ * Insert node `idx` (vector index) into the graph. Incremental — new indices
+ * attach to the existing structure without a rebuild; vectors for nodes NOT
+ * yet inserted are read from `vectors` when needed, so the caller can insert
+ * lazily. Tombstoned indices can be re-inserted (undeletes).
+ */
+export function insertHnsw(hnsw, idx, distance, vectors) {
+    const vec = vectors[idx];
+    if (!vec)
+        throw new Error(`hnsw insert: no vector at index ${idx}`);
+    if (hnsw.levels[idx] !== undefined && !hnsw.deleted.has(idx)) {
+        throw new Error(`hnsw insert: node ${idx} already present`);
+    }
+    hnsw.deleted.delete(idx);
+    const M = hnsw.M;
+    const M0 = M * 2;
+    const mL = 1 / Math.log(M);
+    const rng = mulberry32(hnsw.seed + idx * 2654435761);
+    const u = Math.max(rng(), Number.MIN_VALUE);
+    const level = Math.min(Math.floor(-Math.log(u) * mL), 64);
+    hnsw.levels[idx] = level;
+    hnsw.neighbors[idx] = Array.from({ length: level + 1 }, () => []);
+    if (hnsw.entryPoint < 0) {
+        hnsw.entryPoint = idx;
+        hnsw.maxLevel = level;
+        return;
+    }
+    const qDist = (node) => distance(vec, node);
+    // 1. Greedy descent through layers above the node's level
+    let ep = hnsw.entryPoint;
+    let epDist = qDist(ep);
+    for (let l = hnsw.maxLevel; l > level; l--) {
+        let changed = true;
+        while (changed) {
+            changed = false;
+            for (const nb of hnsw.neighbors[ep][l] ?? []) {
+                const d = qDist(nb);
+                if (d < epDist) {
+                    epDist = d;
+                    ep = nb;
+                    changed = true;
+                }
+            }
+        }
+    }
+    // 2. Insert on each layer from min(level, maxLevel) down to 0
+    const pairwise = (a, b) => distance(vectors[a], b);
+    for (let l = Math.min(level, hnsw.maxLevel); l >= 0; l--) {
+        const efC = Math.max(M0, DEFAULT_HNSW_EF);
+        const candidates = searchLayer(hnsw, qDist, ep, epDist, l, efC);
+        const cap = l === 0 ? M0 : M;
+        // Algorithm 4: diversity-aware neighbor selection keeps the graph
+        // navigable — a candidate is skipped only when it is CLOSER to an
+        // already-selected neighbor than to the query
+        const selected = selectNeighborsHeuristic(candidates, cap, pairwise);
+        hnsw.neighbors[idx][l] = selected.map((c) => c.node);
+        for (const { node } of selected) {
+            const links = hnsw.neighbors[node][l];
+            if (!links.includes(idx))
+                links.push(idx);
+            if (links.length > cap) {
+                // re-select the neighbor's own links with the same heuristic — the
+                // diversity rule preserves connectivity instead of blind top-M,
+                // which used to sever the just-added edge and strand whole regions
+                const pruned = selectNeighborsHeuristic(links.map((n) => ({ node: n, dist: pairwise(node, n) })), cap, pairwise);
+                hnsw.neighbors[node][l] = pruned.map((c) => c.node);
+            }
+        }
+        if (candidates.length > 0) {
+            ep = candidates[0].node;
+            epDist = candidates[0].dist;
+        }
+    }
+    if (level > hnsw.maxLevel) {
+        hnsw.maxLevel = level;
+        hnsw.entryPoint = idx;
+    }
+}
+/** Tombstone a node — searches skip it; the graph keeps it for connectivity. */
+export function deleteHnsw(hnsw, idx) {
+    if (hnsw.levels[idx] !== undefined)
+        hnsw.deleted.add(idx);
+}
+/** ef-search one layer, returning the closest `ef` live candidates (closest first). */
+function searchLayer(hnsw, qDist, ep, epDist, layer, ef) {
+    const visited = new Set([ep]);
+    // max-heap by dist via simple sorted insertion (candidate lists are small)
+    const candidates = [{ node: ep, dist: epDist }];
+    const results = hnsw.deleted.has(ep) ? [] : [{ node: ep, dist: epDist }];
+    while (candidates.length > 0) {
+        const current = candidates.shift();
+        const worst = results.length >= ef ? results[results.length - 1].dist : Infinity;
+        if (current.dist > worst)
+            break;
+        for (const nb of hnsw.neighbors[current.node][layer] ?? []) {
+            if (visited.has(nb))
+                continue;
+            visited.add(nb);
+            const d = qDist(nb);
+            const worstNow = results.length >= ef ? results[results.length - 1].dist : Infinity;
+            if (d < worstNow || results.length < ef) {
+                candidates.push({ node: nb, dist: d });
+                candidates.sort((a, b) => a.dist - b.dist);
+                if (!hnsw.deleted.has(nb)) {
+                    results.push({ node: nb, dist: d });
+                    results.sort((a, b) => a.dist - b.dist);
+                    if (results.length > ef)
+                        results.pop();
+                }
+            }
+        }
+    }
+    return results;
+}
+/**
+ * Algorithm 4 of the HNSW paper (Malkov & Yashunin 2016): candidates sorted
+ * by distance to the query enter the result in order; a candidate is skipped
+ * when it is closer to some already-selected neighbor than to the query
+ * (redundant direction). `pairwise(a, b)` measures node-to-node distance.
+ * If fewer than `m` survive (disconnected candidate set), the closest
+ * leftovers backfill so the link count stays useful.
+ */
+export function selectNeighborsHeuristic(candidates, m, pairwise) {
+    const sorted = [...candidates].sort((a, b) => a.dist - b.dist);
+    const result = [];
+    const skipped = [];
+    for (const cand of sorted) {
+        if (result.length >= m)
+            break;
+        let dominated = false;
+        for (const chosen of result) {
+            if (pairwise(cand.node, chosen.node) < cand.dist) {
+                dominated = true;
+                break;
+            }
+        }
+        if (dominated)
+            skipped.push(cand);
+        else
+            result.push(cand);
+    }
+    for (const cand of skipped) {
+        if (result.length >= m)
+            break;
+        result.push(cand);
+    }
+    return result;
+}
+/**
+ * ANN candidate pre-filter: ef-search from the entry point, returning LIVE
+ * candidate chunk indices (tombstones excluded). Mirrors
+ * `searchIVFCandidates` — the result feeds the same probed-candidate set.
+ */
+export function searchHnswCandidates(queryVec, hnsw, vectors, options = {}) {
+    if (hnsw.entryPoint < 0 || hnsw.levels.length === 0)
+        return [];
+    const distance = options.distance ??
+        ((a, b) => 1 - dotSimilarity(a, vectors[b]));
+    const qDist = (node) => distance(queryVec, node);
+    let ep = hnsw.entryPoint;
+    let epDist = qDist(ep);
+    for (let l = hnsw.maxLevel; l > 0; l--) {
+        let changed = true;
+        while (changed) {
+            changed = false;
+            for (const nb of hnsw.neighbors[ep][l] ?? []) {
+                const d = qDist(nb);
+                if (d < epDist) {
+                    epDist = d;
+                    ep = nb;
+                    changed = true;
+                }
+            }
+        }
+    }
+    const ef = Math.max(options.ef ?? DEFAULT_HNSW_EF, 1);
+    const found = searchLayer(hnsw, qDist, ep, epDist, 0, ef);
+    return found.map((f) => f.node);
+}
+/** Dot product that tolerates int8 storage (norm handling mirrors the exact sweep). */
+function dotSimilarity(a, b) {
+    let s = 0;
+    const n = Math.min(a.length, b.length);
+    for (let i = 0; i < n; i++)
+        s += a[i] * b[i];
+    return s;
+}
+export function serializeHnsw(hnsw) {
+    return {
+        format: 'hnsw-v1',
+        dim: hnsw.dim,
+        M: hnsw.M,
+        seed: hnsw.seed,
+        entryPoint: hnsw.entryPoint,
+        maxLevel: hnsw.maxLevel,
+        count: hnsw.levels.length,
+        levels: hnsw.levels,
+        neighbors: hnsw.neighbors,
+        deleted: [...hnsw.deleted].sort((a, b) => a - b),
+    };
+}
+/**
+ * Hardened deserialization — every structural invariant is checked with the
+ * same discipline as `deserializeIVF`; any violation throws (the loader
+ * falls back to brute force).
+ */
+export function deserializeHnsw(raw, expectedChunksCount) {
+    if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
+        throw new Error('invalid HNSW payload: expected JSON object — run `mdss index` to rebuild');
+    }
+    const obj = raw;
+    if (obj.format !== 'hnsw-v1') {
+        throw new Error(`invalid HNSW payload: format must be "hnsw-v1", got ${JSON.stringify(obj.format)} — run \`mdss index\` to rebuild`);
+    }
+    const { dim, M, seed, entryPoint, maxLevel, count } = obj;
+    if (typeof dim !== 'number' || !Number.isSafeInteger(dim) || dim <= 0 ||
+        typeof M !== 'number' || !Number.isSafeInteger(M) || M <= 0 || M > 128 ||
+        typeof seed !== 'number' || !Number.isSafeInteger(seed) ||
+        typeof entryPoint !== 'number' || !Number.isSafeInteger(entryPoint) || entryPoint < 0 ||
+        typeof maxLevel !== 'number' || !Number.isSafeInteger(maxLevel) || maxLevel < 0 ||
+        typeof count !== 'number' || !Number.isSafeInteger(count) || count <= 0) {
+        throw new Error('invalid HNSW payload: invalid header — run `mdss index` to rebuild');
+    }
+    if (!Array.isArray(obj.levels) || obj.levels.length !== count) {
+        throw new Error(`invalid HNSW payload: levels must be an array of ${count} — run \`mdss index\` to rebuild`);
+    }
+    const levels = [];
+    for (let i = 0; i < count; i++) {
+        const l = obj.levels[i];
+        if (typeof l !== 'number' || !Number.isSafeInteger(l) || l < 0 || l > maxLevel) {
+            throw new Error(`invalid HNSW payload: level of node ${i} must be an integer in [0, ${maxLevel}] — run \`mdss index\` to rebuild`);
+        }
+        levels.push(l);
+    }
+    if (levels[entryPoint] !== maxLevel) {
+        throw new Error(`invalid HNSW payload: entry point ${entryPoint} must sit on the top layer ${maxLevel} — run \`mdss index\` to rebuild`);
+    }
+    if (!Array.isArray(obj.neighbors) || obj.neighbors.length !== count) {
+        throw new Error(`invalid HNSW payload: neighbors must be an array of ${count} — run \`mdss index\` to rebuild`);
+    }
+    const neighbors = new Array(count);
+    const m0 = M * 2;
+    for (let i = 0; i < count; i++) {
+        const perLevel = obj.neighbors[i];
+        if (!Array.isArray(perLevel) || perLevel.length !== levels[i] + 1) {
+            throw new Error(`invalid HNSW payload: node ${i} must have ${levels[i] + 1} link lists — run \`mdss index\` to rebuild`);
+        }
+        const node = new Array(levels[i] + 1);
+        for (let l = 0; l <= levels[i]; l++) {
+            const list = perLevel[l];
+            if (!Array.isArray(list)) {
+                throw new Error(`invalid HNSW payload: node ${i} layer ${l} must be an array — run \`mdss index\` to rebuild`);
+            }
+            if (list.length > (l === 0 ? m0 : M)) {
+                throw new Error(`invalid HNSW payload: node ${i} layer ${l} exceeds the link cap — run \`mdss index\` to rebuild`);
+            }
+            const seen = new Set();
+            for (const nb of list) {
+                if (typeof nb !== 'number' || !Number.isSafeInteger(nb) || nb < 0 || nb >= count || nb === i) {
+                    throw new Error(`invalid HNSW payload: node ${i} layer ${l} has invalid link ${JSON.stringify(nb)} — run \`mdss index\` to rebuild`);
+                }
+                if (seen.has(nb)) {
+                    throw new Error(`invalid HNSW payload: duplicate link ${nb} on node ${i} layer ${l} — run \`mdss index\` to rebuild`);
+                }
+                seen.add(nb);
+            }
+            node[l] = list;
+        }
+        neighbors[i] = node;
+    }
+    let deleted = new Set();
+    if (obj.deleted !== undefined) {
+        if (!Array.isArray(obj.deleted)) {
+            throw new Error('invalid HNSW payload: deleted must be an array — run `mdss index` to rebuild');
+        }
+        for (const idx of obj.deleted) {
+            if (typeof idx !== 'number' || !Number.isSafeInteger(idx) || idx < 0 || idx >= count) {
+                throw new Error(`invalid HNSW payload: deleted index ${JSON.stringify(idx)} out of range — run \`mdss index\` to rebuild`);
+            }
+            deleted.add(idx);
+        }
+    }
+    if (expectedChunksCount !== undefined && count > expectedChunksCount) {
+        throw new Error(`invalid HNSW payload: ${count} nodes exceed ${expectedChunksCount} chunks — run \`mdss index\` to rebuild`);
+    }
+    return { dim, M, seed, entryPoint, maxLevel, levels, neighbors, deleted };
+}

package/dist/indexer.d.ts

@@ -18,6 +18,14 @@ export interface PersistedIndex {
     model?: string | null;
     modelAlias?: string;
     adapterFingerprint?: string;
+    /**
+     * issue #202: the effective chunk token budget these chunks were built
+     * with (null = legacy char budget). A mismatch forces a full re-chunk —
+     * otherwise the file-level fast path would keep stale char-budget chunks
+     * for unchanged files and produce a mixed index. Identical chunk texts
+     * still reuse their cached vectors through chunkHash.
+     */
+    chunkBudget?: number | null;
     dim?: number;
     db?: string;
     built?: string;
@@ -66,8 +74,16 @@ export interface BuildIndexOptions {
     maxRetries?: number;
     onProgress?: (done: number, total: number, chunksPerSec: number) => void;
     workers?: number;
-    ann?: boolean;
+    /** issue #201: ANN sidecar engine — `true`/`'ivf'` = ivf.json, `'hnsw'` = hnsw.json. */
+    ann?: boolean | 'ivf' | 'hnsw';
     quantize?: 'fp32' | 'int8';
+    /**
+     * issue #202: chunk budget in TOKENS (overrides the legacy char budget).
+     * Default: min(adapter.maxTokens, 512) — the point of the issue is that the
+     * char budget overflowed real maxTokens for Cyrillic. Explicit `0` restores
+     * the legacy char behavior.
+     */
+    maxChunkTokens?: number;
 }
 export interface BuildIndexResult {
     files: number;

package/dist/indexer.js

@@ -15,7 +15,9 @@ import { validateIndexEnvelope, validateNumericVector } from './index-format.js'
 import { analyzeLexicalDocument, buildLexicalIndex, lexicalIdentity, reverseLexicalIndex, validateLexicalIndex, } from './lexical.js';
 import { embeddingAdapterFingerprint, legacyEmbeddingAdapterFingerprint } from './models.js';
 import { trainIVF, serializeIVF, ANN_THRESHOLD } from './ivf.js';
+import { buildHnsw, serializeHnsw } from './hnsw.js';
 import { serializeBinaryIndex } from './binary-format.js';
+import { defaultChunkTokenBudget } from './markdown-parser.js';
 const md5 = (s) => crypto.createHash('md5').update(s).digest('hex');
 const sha256 = (s) => crypto.createHash('sha256').update(s).digest('hex');
 export const _stats = {
@@ -23,7 +25,17 @@ export const _stats = {
     fileHits: 0,
     embedCount: 0,
 };
-const BATCH = 32;
+// issue #194: batch size is tunable — 64-128 often feeds ONNX noticeably
+// better on short passages; overridable for experiments (clamped 1..512).
+const BATCH = (() => {
+    const raw = parseInt(process.env.MDSS_BATCH || '', 10);
+    return Number.isSafeInteger(raw) && raw > 0 ? Math.min(raw, 512) : 32;
+})();
+// issue #194: mid-run checkpoints serialize the WHOLE index so far — with
+// faster embedding that cost scales up. Time-budget them (~every 30 s) in
+// addition to the batch cadence, so at most ~30 s of embedding is lost on a
+// crash while serialization stays a small slice of wall time.
+const CHECKPOINT_MIN_INTERVAL_MS = 30_000;
 const CHECKPOINT_BATCHES = 8;
 const INDEX_FORMAT = 'binary-v1';
 function isStringRecord(value) {
@@ -79,13 +91,20 @@ export function chunkHash(model, chunk) {
     return sha256(input);
 }
 export async function buildIndex(opts) {
-    const { db, indexDir, cacheDir, modelName, dimensions, ignore = [], log = () => { }, offline = false, embedFn = embed, _lockHeld = false, maxRetries = 3, onProgress, workers = 1, ann = false, quantize = 'fp32', } = opts;
+    const { db, indexDir, cacheDir, modelName, dimensions, ignore = [], log = () => { }, offline = false, embedFn = embed, _lockHeld = false, maxRetries = 3, onProgress, workers = 1, ann = false, quantize = 'fp32', maxChunkTokens, } = opts;
     if (_lockHeld)
-        return _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions, ignore, log, offline, embedFn, maxRetries, onProgress, workers, ann, quantize });
-    return withIndexLock(indexDir, () => _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions, ignore, log, offline, embedFn, maxRetries, onProgress, workers, ann, quantize }));
+        return _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions, ignore, log, offline, embedFn, maxRetries, onProgress, workers, ann, quantize, maxChunkTokens });
+    return withIndexLock(indexDir, () => _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions, ignore, log, offline, embedFn, maxRetries, onProgress, workers, ann, quantize, maxChunkTokens }));
 }
-async function _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions, ignore = [], log = () => { }, offline = false, embedFn = embed, maxRetries = 3, onProgress, workers = 1, ann = false, quantize = 'fp32', }) {
+async function _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions, ignore = [], log = () => { }, offline = false, embedFn = embed, maxRetries = 3, onProgress, workers = 1, ann = false, quantize = 'fp32', maxChunkTokens = undefined, }) {
     const model = resolveModel(modelName, dimensions);
+    // issue #202: explicit maxChunkTokens wins; `0` restores the legacy char
+    // budget; otherwise min(adapter.maxTokens, 512), undefined → char mode
+    const effectiveChunkTokens = maxChunkTokens === 0
+        ? undefined
+        : maxChunkTokens && maxChunkTokens > 0
+            ? maxChunkTokens
+            : defaultChunkTokenBudget(model);
     const modelIdentity = `${model.id}@${model.revision || 'main'}`;
     const adapterFingerprint = embeddingAdapterFingerprint(model);
     const legacyAdapterFingerprint = legacyEmbeddingAdapterFingerprint(model);
@@ -94,8 +113,14 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions,
     const vectorsPath = path.join(indexDir, 'vectors.json');
     const hashesPath = path.join(indexDir, '.hashes.json');
     const checkpointPath = path.join(indexDir, '.checkpoint.json');
+    let lastCheckpointAt = 0;
     fs.mkdirSync(indexDir, { recursive: true });
-    const files = walkMarkdown(db, ignore);
+    // issue #189: unreadable directories shrink the corpus — collect and log
+    // them so a "shrinking" index is never published silently
+    const walkErrors = [];
+    const files = walkMarkdown(db, ignore, walkErrors);
+    for (const we of walkErrors)
+        log(we);
     const canonicalHashes = loadJSON(hashesPath, {}, log);
     let canonicalIndex = loadJSON(vectorsPath, { chunks: [] }, log);
     let canonicalSchema = 0;
@@ -194,6 +219,14 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions,
     }
     const modelChanged = !!oldIndex.model && oldIndex.model !== modelIdentity;
     const contextVectorsReusable = sourceSchema >= 2 && !modelChanged && adapterCompatible(oldIndex);
+    // issue #202: a different chunk budget invalidates the file-level fast path
+    // (unchanged files must be re-chunked), NOT the per-chunk vector cache —
+    // chunks whose text did not move across the budget boundary re-embed free.
+    const chunkBudgetMarker = effectiveChunkTokens ?? null;
+    const chunkerChanged = (oldIndex.chunkBudget ?? null) !== chunkBudgetMarker;
+    if (chunkerChanged && oldIndex.chunks.length > 0) {
+        log('chunk budget changed — re-chunking all files (per-chunk vector cache still applies).');
+    }
     let buildDim = model.dim > 0 ? model.dim : contextVectorsReusable && oldDimensionValid ? oldExpectedDim : undefined;
     const acceptBuildVector = (vector) => {
         const length = validateNumericVector(vector, buildDim, 'embedding vector');
@@ -250,6 +283,7 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions,
         model: modelIdentity,
         modelAlias: typeof modelName === 'string' ? modelName || 'e5-base' : model.id,
         adapterFingerprint,
+        chunkBudget: chunkBudgetMarker,
         ...(buildDim === undefined ? {} : { dim: buildDim }),
         db,
         built: new Date().toISOString(),
@@ -269,7 +303,7 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions,
             newHashes[rel] = h;
             if (oldHashes[rel] === h)
                 reusableLexicalRecords = lexicalByFile.get(rel);
-            if (sourceSchema === SCHEMA_VERSION && contextVectorsReusable && oldHashes[rel] === h && oldByFile.has(rel)) {
+            if (sourceSchema === SCHEMA_VERSION && contextVectorsReusable && !chunkerChanged && oldHashes[rel] === h && oldByFile.has(rel)) {
                 const old = oldByFile.get(rel);
                 for (const c of old) {
                     if (!c.chunkHash)
@@ -290,7 +324,7 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions,
                 continue;
             }
             changedFiles++;
-            parsed = parseFile(abs, db, undefined, raw);
+            parsed = parseFile(abs, db, undefined, raw, effectiveChunkTokens);
         }
         catch (e) {
             delete newHashes[rel];
@@ -347,7 +381,8 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions,
                     c.vec = vecs[j];
                 });
                 completedBatches++;
-                if (completedBatches % CHECKPOINT_BATCHES === 0) {
+                if (completedBatches % CHECKPOINT_BATCHES === 0 && Date.now() - lastCheckpointAt >= CHECKPOINT_MIN_INTERVAL_MS) {
+                    lastCheckpointAt = Date.now();
                     atomicWrite(checkpointPath, JSON.stringify(checkpointSnapshot(false)));
                 }
                 const done = Math.min(i + BATCH, toEmbed.length);
@@ -393,7 +428,8 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions,
                     });
                     completedBatches++;
                     embeddedCount += slice.length;
-                    if (completedBatches % CHECKPOINT_BATCHES === 0) {
+                    if (completedBatches % CHECKPOINT_BATCHES === 0 && Date.now() - lastCheckpointAt >= CHECKPOINT_MIN_INTERVAL_MS) {
+                        lastCheckpointAt = Date.now();
                         atomicWrite(checkpointPath, JSON.stringify(checkpointSnapshot(false)));
                     }
                     const elapsedSec = (Date.now() - embedStartTime) / 1000;
@@ -422,6 +458,7 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions,
         model: modelIdentity,
         modelAlias: typeof modelName === 'string' ? modelName || 'e5-base' : model.id,
         adapterFingerprint,
+        chunkBudget: chunkBudgetMarker,
         ...(dim === undefined ? {} : { dim }),
         db,
         built: new Date().toISOString(),
@@ -438,17 +475,54 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions,
         const binBuffer = serializeBinaryIndex(index, dim, { quantize });
         atomicWrite(vectorsBinPath, binBuffer);
     }
+    // issue #201: ANN sidecar engine — explicit 'hnsw' builds the graph
+    // instead of IVF; `true`/undefined/'ivf' keep the historical ivf.json path.
+    const annEngine = ann === 'hnsw' ? 'hnsw' : 'ivf';
+    const buildSidecar = Boolean(ann) || chunks.length >= ANN_THRESHOLD;
     const ivfPath = path.join(indexDir, 'ivf.json');
-    if (ann || chunks.length >= ANN_THRESHOLD) {
+    const hnswPath = path.join(indexDir, 'hnsw.json');
+    if (buildSidecar) {
         const rawVecs = chunks.map((c) => (c.vec instanceof Float32Array ? c.vec : Float32Array.from(c.vec)));
-        const ivfIndex = trainIVF(rawVecs);
-        atomicWrite(ivfPath, JSON.stringify(serializeIVF(ivfIndex)));
+        if (annEngine === 'hnsw') {
+            const hnswIndex = buildHnsw(rawVecs, {
+                vectors: rawVecs,
+                // cosine distance (lower = closer) over the flat normalized vectors
+                distance: (a, b) => {
+                    const bv = rawVecs[b];
+                    let dot = 0;
+                    for (let i = 0; i < a.length; i++)
+                        dot += a[i] * bv[i];
+                    return 1 - dot;
+                },
+            });
+            atomicWrite(hnswPath, JSON.stringify(serializeHnsw(hnswIndex)));
+            if (fs.existsSync(ivfPath)) {
+                try {
+                    fs.unlinkSync(ivfPath);
+                }
+                catch { }
+            }
+        }
+        else {
+            const ivfIndex = trainIVF(rawVecs);
+            atomicWrite(ivfPath, JSON.stringify(serializeIVF(ivfIndex)));
+            if (fs.existsSync(hnswPath)) {
+                try {
+                    fs.unlinkSync(hnswPath);
+                }
+                catch { }
+            }
+        }
     }
-    else if (fs.existsSync(ivfPath)) {
-        try {
-            fs.unlinkSync(ivfPath);
+    else {
+        for (const sidecar of [ivfPath, hnswPath]) {
+            if (fs.existsSync(sidecar)) {
+                try {
+                    fs.unlinkSync(sidecar);
+                }
+                catch { }
+            }
         }
-        catch { }
     }
     fs.unlinkSync(checkpointPath);
     return {

package/dist/lexical.d.ts

@@ -33,6 +33,16 @@ export declare function analyzeLexicalDocument(chunk: ChunkForLexical, weights?:
 }): TermFrequencies;
 export declare function buildLexicalIndex(records: TermFrequencies[]): LexicalIndex;
 export declare function editDistance(a: string, b: string): number;
+/**
+ * Damerau-Levenshtein distance with a threshold (issue #193): banded DP that
+ * never leaves the |i-j| <= maxDist diagonal and aborts as soon as the whole
+ * row exceeds the threshold — O(maxDist × min(alen, blen)) instead of the
+ * full O(alen × blen) matrix. Returns the EXACT distance when it is
+ * <= maxDist and null otherwise; decisions are identical to
+ * `editDistance(a, b) <= maxDist` (optimal paths within the threshold never
+ * leave the band).
+ */
+export declare function editDistanceAtMost(a: string, b: string, maxDist: number): number | null;
 export declare function fuzzyTitleAliasScores(chunks: Array<{
     file?: string;
     title: string;