md-semantic-search 1.0.3 → 2.0.0

package/dist/indexer.js

@@ -15,7 +15,9 @@ import { validateIndexEnvelope, validateNumericVector } from './index-format.js'
 import { analyzeLexicalDocument, buildLexicalIndex, lexicalIdentity, reverseLexicalIndex, validateLexicalIndex, } from './lexical.js';
 import { embeddingAdapterFingerprint, legacyEmbeddingAdapterFingerprint } from './models.js';
 import { trainIVF, serializeIVF, ANN_THRESHOLD } from './ivf.js';
+import { buildHnsw, serializeHnsw } from './hnsw.js';
 import { serializeBinaryIndex } from './binary-format.js';
+import { defaultChunkTokenBudget } from './markdown-parser.js';
 const md5 = (s) => crypto.createHash('md5').update(s).digest('hex');
 const sha256 = (s) => crypto.createHash('sha256').update(s).digest('hex');
 export const _stats = {
@@ -23,7 +25,17 @@ export const _stats = {
     fileHits: 0,
     embedCount: 0,
 };
-const BATCH = 32;
+// issue #194: batch size is tunable — 64-128 often feeds ONNX noticeably
+// better on short passages; overridable for experiments (clamped 1..512).
+const BATCH = (() => {
+    const raw = parseInt(process.env.MDSS_BATCH || '', 10);
+    return Number.isSafeInteger(raw) && raw > 0 ? Math.min(raw, 512) : 32;
+})();
+// issue #194: mid-run checkpoints serialize the WHOLE index so far — with
+// faster embedding that cost scales up. Time-budget them (~every 30 s) in
+// addition to the batch cadence, so at most ~30 s of embedding is lost on a
+// crash while serialization stays a small slice of wall time.
+const CHECKPOINT_MIN_INTERVAL_MS = 30_000;
 const CHECKPOINT_BATCHES = 8;
 const INDEX_FORMAT = 'binary-v1';
 function isStringRecord(value) {
@@ -79,13 +91,20 @@ export function chunkHash(model, chunk) {
     return sha256(input);
 }
 export async function buildIndex(opts) {
-    const { db, indexDir, cacheDir, modelName, ignore = [], log = () => { }, offline = false, embedFn = embed, _lockHeld = false, maxRetries = 3, onProgress, workers = 1, ann = false, quantize = 'fp32', } = opts;
+    const { db, indexDir, cacheDir, modelName, dimensions, ignore = [], log = () => { }, offline = false, embedFn = embed, _lockHeld = false, maxRetries = 3, onProgress, workers = 1, ann = false, quantize = 'fp32', maxChunkTokens, } = opts;
     if (_lockHeld)
-        return _buildIndexInner({ db, indexDir, cacheDir, modelName, ignore, log, offline, embedFn, maxRetries, onProgress, workers, ann, quantize });
-    return withIndexLock(indexDir, () => _buildIndexInner({ db, indexDir, cacheDir, modelName, ignore, log, offline, embedFn, maxRetries, onProgress, workers, ann, quantize }));
+        return _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions, ignore, log, offline, embedFn, maxRetries, onProgress, workers, ann, quantize, maxChunkTokens });
+    return withIndexLock(indexDir, () => _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions, ignore, log, offline, embedFn, maxRetries, onProgress, workers, ann, quantize, maxChunkTokens }));
 }
-async function _buildIndexInner({ db, indexDir, cacheDir, modelName, ignore = [], log = () => { }, offline = false, embedFn = embed, maxRetries = 3, onProgress, workers = 1, ann = false, quantize = 'fp32', }) {
-    const model = resolveModel(modelName);
+async function _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions, ignore = [], log = () => { }, offline = false, embedFn = embed, maxRetries = 3, onProgress, workers = 1, ann = false, quantize = 'fp32', maxChunkTokens = undefined, }) {
+    const model = resolveModel(modelName, dimensions);
+    // issue #202: explicit maxChunkTokens wins; `0` restores the legacy char
+    // budget; otherwise min(adapter.maxTokens, 512), undefined → char mode
+    const effectiveChunkTokens = maxChunkTokens === 0
+        ? undefined
+        : maxChunkTokens && maxChunkTokens > 0
+            ? maxChunkTokens
+            : defaultChunkTokenBudget(model);
     const modelIdentity = `${model.id}@${model.revision || 'main'}`;
     const adapterFingerprint = embeddingAdapterFingerprint(model);
     const legacyAdapterFingerprint = legacyEmbeddingAdapterFingerprint(model);
@@ -94,8 +113,14 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, ignore = []
     const vectorsPath = path.join(indexDir, 'vectors.json');
     const hashesPath = path.join(indexDir, '.hashes.json');
     const checkpointPath = path.join(indexDir, '.checkpoint.json');
+    let lastCheckpointAt = 0;
     fs.mkdirSync(indexDir, { recursive: true });
-    const files = walkMarkdown(db, ignore);
+    // issue #189: unreadable directories shrink the corpus — collect and log
+    // them so a "shrinking" index is never published silently
+    const walkErrors = [];
+    const files = walkMarkdown(db, ignore, walkErrors);
+    for (const we of walkErrors)
+        log(we);
     const canonicalHashes = loadJSON(hashesPath, {}, log);
     let canonicalIndex = loadJSON(vectorsPath, { chunks: [] }, log);
     let canonicalSchema = 0;
@@ -194,6 +219,14 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, ignore = []
     }
     const modelChanged = !!oldIndex.model && oldIndex.model !== modelIdentity;
     const contextVectorsReusable = sourceSchema >= 2 && !modelChanged && adapterCompatible(oldIndex);
+    // issue #202: a different chunk budget invalidates the file-level fast path
+    // (unchanged files must be re-chunked), NOT the per-chunk vector cache —
+    // chunks whose text did not move across the budget boundary re-embed free.
+    const chunkBudgetMarker = effectiveChunkTokens ?? null;
+    const chunkerChanged = (oldIndex.chunkBudget ?? null) !== chunkBudgetMarker;
+    if (chunkerChanged && oldIndex.chunks.length > 0) {
+        log('chunk budget changed — re-chunking all files (per-chunk vector cache still applies).');
+    }
     let buildDim = model.dim > 0 ? model.dim : contextVectorsReusable && oldDimensionValid ? oldExpectedDim : undefined;
     const acceptBuildVector = (vector) => {
         const length = validateNumericVector(vector, buildDim, 'embedding vector');
@@ -250,6 +283,7 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, ignore = []
         model: modelIdentity,
         modelAlias: typeof modelName === 'string' ? modelName || 'e5-base' : model.id,
         adapterFingerprint,
+        chunkBudget: chunkBudgetMarker,
         ...(buildDim === undefined ? {} : { dim: buildDim }),
         db,
         built: new Date().toISOString(),
@@ -269,7 +303,7 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, ignore = []
             newHashes[rel] = h;
             if (oldHashes[rel] === h)
                 reusableLexicalRecords = lexicalByFile.get(rel);
-            if (sourceSchema === SCHEMA_VERSION && contextVectorsReusable && oldHashes[rel] === h && oldByFile.has(rel)) {
+            if (sourceSchema === SCHEMA_VERSION && contextVectorsReusable && !chunkerChanged && oldHashes[rel] === h && oldByFile.has(rel)) {
                 const old = oldByFile.get(rel);
                 for (const c of old) {
                     if (!c.chunkHash)
@@ -290,7 +324,7 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, ignore = []
                 continue;
             }
             changedFiles++;
-            parsed = parseFile(abs, db, undefined, raw);
+            parsed = parseFile(abs, db, undefined, raw, effectiveChunkTokens);
         }
         catch (e) {
             delete newHashes[rel];
@@ -347,7 +381,8 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, ignore = []
                     c.vec = vecs[j];
                 });
                 completedBatches++;
-                if (completedBatches % CHECKPOINT_BATCHES === 0) {
+                if (completedBatches % CHECKPOINT_BATCHES === 0 && Date.now() - lastCheckpointAt >= CHECKPOINT_MIN_INTERVAL_MS) {
+                    lastCheckpointAt = Date.now();
                     atomicWrite(checkpointPath, JSON.stringify(checkpointSnapshot(false)));
                 }
                 const done = Math.min(i + BATCH, toEmbed.length);
@@ -393,7 +428,8 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, ignore = []
                     });
                     completedBatches++;
                     embeddedCount += slice.length;
-                    if (completedBatches % CHECKPOINT_BATCHES === 0) {
+                    if (completedBatches % CHECKPOINT_BATCHES === 0 && Date.now() - lastCheckpointAt >= CHECKPOINT_MIN_INTERVAL_MS) {
+                        lastCheckpointAt = Date.now();
                         atomicWrite(checkpointPath, JSON.stringify(checkpointSnapshot(false)));
                     }
                     const elapsedSec = (Date.now() - embedStartTime) / 1000;
@@ -422,6 +458,7 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, ignore = []
         model: modelIdentity,
         modelAlias: typeof modelName === 'string' ? modelName || 'e5-base' : model.id,
         adapterFingerprint,
+        chunkBudget: chunkBudgetMarker,
         ...(dim === undefined ? {} : { dim }),
         db,
         built: new Date().toISOString(),
@@ -438,17 +475,54 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, ignore = []
         const binBuffer = serializeBinaryIndex(index, dim, { quantize });
         atomicWrite(vectorsBinPath, binBuffer);
     }
+    // issue #201: ANN sidecar engine — explicit 'hnsw' builds the graph
+    // instead of IVF; `true`/undefined/'ivf' keep the historical ivf.json path.
+    const annEngine = ann === 'hnsw' ? 'hnsw' : 'ivf';
+    const buildSidecar = Boolean(ann) || chunks.length >= ANN_THRESHOLD;
     const ivfPath = path.join(indexDir, 'ivf.json');
-    if (ann || chunks.length >= ANN_THRESHOLD) {
+    const hnswPath = path.join(indexDir, 'hnsw.json');
+    if (buildSidecar) {
         const rawVecs = chunks.map((c) => (c.vec instanceof Float32Array ? c.vec : Float32Array.from(c.vec)));
-        const ivfIndex = trainIVF(rawVecs);
-        atomicWrite(ivfPath, JSON.stringify(serializeIVF(ivfIndex)));
+        if (annEngine === 'hnsw') {
+            const hnswIndex = buildHnsw(rawVecs, {
+                vectors: rawVecs,
+                // cosine distance (lower = closer) over the flat normalized vectors
+                distance: (a, b) => {
+                    const bv = rawVecs[b];
+                    let dot = 0;
+                    for (let i = 0; i < a.length; i++)
+                        dot += a[i] * bv[i];
+                    return 1 - dot;
+                },
+            });
+            atomicWrite(hnswPath, JSON.stringify(serializeHnsw(hnswIndex)));
+            if (fs.existsSync(ivfPath)) {
+                try {
+                    fs.unlinkSync(ivfPath);
+                }
+                catch { }
+            }
+        }
+        else {
+            const ivfIndex = trainIVF(rawVecs);
+            atomicWrite(ivfPath, JSON.stringify(serializeIVF(ivfIndex)));
+            if (fs.existsSync(hnswPath)) {
+                try {
+                    fs.unlinkSync(hnswPath);
+                }
+                catch { }
+            }
+        }
     }
-    else if (fs.existsSync(ivfPath)) {
-        try {
-            fs.unlinkSync(ivfPath);
+    else {
+        for (const sidecar of [ivfPath, hnswPath]) {
+            if (fs.existsSync(sidecar)) {
+                try {
+                    fs.unlinkSync(sidecar);
+                }
+                catch { }
+            }
         }
-        catch { }
     }
     fs.unlinkSync(checkpointPath);
     return {

package/dist/ivf.d.ts

@@ -10,15 +10,22 @@ export interface SerializedIVF {
     centroids: string;
     clusters: number[][];
 }
+export interface TrainIVFOptions {
+    k?: number;
+    maxIterations?: number;
+    seeding?: 'kmeans++' | 'spread';
+    seed?: number;
+    onMetric?: (metric: {
+        emptyClusters: number;
+        iterations: number;
+    }) => void;
+}
 export declare const ANN_THRESHOLD = 500;
 export declare const DEFAULT_NPROBE = 8;
 /**
- * Train a Spherical K-Means IVF index over normalized vectors.
+ * Train a Spherical K-Means IVF index over normalized vectors using k-means++ seeding.
  */
-export declare function trainIVF(vectors: Float32Array[], options?: {
-    k?: number;
-    maxIterations?: number;
-}): IVFIndex;
+export declare function trainIVF(vectors: Float32Array[], options?: TrainIVFOptions): IVFIndex;
 /**
  * Probe top-nprobe closest centroids and return candidate chunk indices.
  */
@@ -28,6 +35,6 @@ export declare function searchIVFCandidates(queryVec: Float32Array, ivf: IVFInde
  */
 export declare function serializeIVF(ivf: IVFIndex): SerializedIVF;
 /**
- * Deserialize JSON object back to IVFIndex.
+ * Deserialize JSON object back to IVFIndex with hardened validation.
  */
-export declare function deserializeIVF(raw: any): IVFIndex;
+export declare function deserializeIVF(raw: unknown, expectedChunksCount?: number): IVFIndex;

package/dist/ivf.js

@@ -2,7 +2,19 @@ import { cosine } from './core.js';
 export const ANN_THRESHOLD = 500;
 export const DEFAULT_NPROBE = 8;
 /**
- * Train a Spherical K-Means IVF index over normalized vectors.
+ * Fast, deterministic pseudo-random number generator (Mulberry32).
+ */
+function mulberry32(seed) {
+    let s = seed >>> 0;
+    return () => {
+        s = (s + 0x6d2b79f5) >>> 0;
+        let t = Math.imul(s ^ (s >>> 15), 1 | s);
+        t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+/**
+ * Train a Spherical K-Means IVF index over normalized vectors using k-means++ seeding.
  */
 export function trainIVF(vectors, options = {}) {
     const n = vectors.length;
@@ -13,16 +25,62 @@ export function trainIVF(vectors, options = {}) {
     const targetK = options.k ?? Math.min(256, Math.max(2, Math.floor(Math.sqrt(n))));
     const k = Math.min(targetK, n);
     const maxIterations = options.maxIterations ?? 20;
-    // Initialize centroids using deterministic spread or k-means++
+    const seeding = options.seeding ?? 'kmeans++';
     const centroids = new Float32Array(k * dim);
-    const step = Math.floor(n / k);
-    for (let c = 0; c < k; c++) {
-        const src = vectors[c * step];
-        centroids.set(src, c * dim);
+    if (seeding === 'spread' || n <= k) {
+        // Deterministic spread
+        const step = Math.floor(n / k);
+        for (let c = 0; c < k; c++) {
+            const src = vectors[c * step];
+            centroids.set(src, c * dim);
+        }
+    }
+    else {
+        // Deterministic k-means++ seeding with D^2 weighted probability
+        const rng = mulberry32(options.seed ?? 42);
+        const firstIdx = Math.floor(rng() * n);
+        centroids.set(vectors[firstIdx], 0);
+        const minDistSq = new Float64Array(n);
+        for (let i = 0; i < n; i++) {
+            const cos = cosine(vectors[i], centroids, 0);
+            minDistSq[i] = Math.max(0, 1 - cos);
+        }
+        for (let c = 1; c < k; c++) {
+            let sumDistSq = 0;
+            for (let i = 0; i < n; i++) {
+                sumDistSq += minDistSq[i];
+            }
+            let chosenIdx = -1;
+            if (sumDistSq > 1e-12) {
+                const target = rng() * sumDistSq;
+                let cumulative = 0;
+                for (let i = 0; i < n; i++) {
+                    cumulative += minDistSq[i];
+                    if (cumulative >= target) {
+                        chosenIdx = i;
+                        break;
+                    }
+                }
+            }
+            if (chosenIdx === -1 || chosenIdx >= n) {
+                chosenIdx = Math.floor(rng() * n);
+            }
+            centroids.set(vectors[chosenIdx], c * dim);
+            for (let i = 0; i < n; i++) {
+                const cos = cosine(vectors[i], centroids, c * dim);
+                const distSq = Math.max(0, 1 - cos);
+                if (distSq < minDistSq[i]) {
+                    minDistSq[i] = distSq;
+                }
+            }
+        }
     }
     let clusters = Array.from({ length: k }, () => []);
     const assignments = new Int32Array(n).fill(-1);
+    let totalEmptyClusters = 0;
+    let finalIterations = 0;
     for (let iter = 0; iter < maxIterations; iter++) {
+        finalIterations = iter + 1;
         let changed = 0;
         const newClusters = Array.from({ length: k }, () => []);
         // 1. Assign each vector to closest centroid (highest cosine / dot product)
@@ -44,11 +102,13 @@ export function trainIVF(vectors, options = {}) {
             newClusters[bestCluster].push(i);
         }
         clusters = newClusters;
-        // 2. Recompute centroids
+        // 2. Recompute centroids & rescue empty clusters
+        let emptyCount = 0;
         for (let c = 0; c < k; c++) {
             const members = clusters[c];
             if (members.length === 0) {
-                // If empty, reassign to a vector from the largest cluster
+                emptyCount++;
+                // Reassign to a vector from the largest cluster
                 let largest = 0;
                 for (let j = 1; j < k; j++) {
                     if (clusters[j].length > clusters[largest].length)
@@ -82,9 +142,11 @@ export function trainIVF(vectors, options = {}) {
                 centroids[cOffset + d] /= norm;
             }
         }
+        totalEmptyClusters = emptyCount;
         if (changed === 0)
             break;
     }
+    options.onMetric?.({ emptyClusters: totalEmptyClusters, iterations: finalIterations });
     return { dim, k, centroids, clusters };
 }
 /**
@@ -126,19 +188,65 @@ export function serializeIVF(ivf) {
     };
 }
 /**
- * Deserialize JSON object back to IVFIndex.
+ * Deserialize JSON object back to IVFIndex with hardened validation.
  */
-export function deserializeIVF(raw) {
-    if (!raw || typeof raw !== 'object') {
-        throw new Error('invalid IVF payload: expected object');
+export function deserializeIVF(raw, expectedChunksCount) {
+    if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
+        throw new Error('invalid IVF payload: expected JSON object — run `mdss index` to rebuild');
+    }
+    const obj = raw;
+    const dim = obj.dim;
+    const k = obj.k;
+    if (typeof dim !== 'number' ||
+        !Number.isSafeInteger(dim) ||
+        dim <= 0 ||
+        typeof k !== 'number' ||
+        !Number.isSafeInteger(k) ||
+        k <= 0) {
+        throw new Error('invalid IVF payload: invalid dim or k — run `mdss index` to rebuild');
+    }
+    if (typeof obj.centroids !== 'string' || obj.centroids.length === 0) {
+        throw new Error('invalid IVF payload: centroids must be a base64 string — run `mdss index` to rebuild');
     }
-    const dim = Number(raw.dim);
-    const k = Number(raw.k);
-    if (!Number.isInteger(dim) || dim <= 0 || !Number.isInteger(k) || k <= 0) {
-        throw new Error('invalid IVF payload: invalid dim or k');
+    const buf = Buffer.from(obj.centroids, 'base64');
+    if (buf.byteLength % 4 !== 0 || buf.byteLength !== k * dim * 4) {
+        throw new Error(`invalid IVF payload: centroids buffer has ${buf.byteLength} bytes, expected ${k * dim * 4} bytes — run \`mdss index\` to rebuild`);
     }
-    const buf = Buffer.from(raw.centroids, 'base64');
     const centroids = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
-    const clusters = Array.isArray(raw.clusters) ? raw.clusters : [];
+    for (let i = 0; i < centroids.length; i++) {
+        if (!Number.isFinite(centroids[i])) {
+            throw new Error(`invalid IVF payload: non-finite centroid value at index ${i} — run \`mdss index\` to rebuild`);
+        }
+    }
+    if (!Array.isArray(obj.clusters)) {
+        throw new Error('invalid IVF payload: clusters must be an array — run `mdss index` to rebuild');
+    }
+    if (obj.clusters.length !== k) {
+        throw new Error(`invalid IVF payload: clusters array has length ${obj.clusters.length}, expected ${k} — run \`mdss index\` to rebuild`);
+    }
+    const clusters = new Array(k);
+    for (let c = 0; c < k; c++) {
+        const list = obj.clusters[c];
+        if (!Array.isArray(list)) {
+            throw new Error(`invalid IVF payload: cluster ${c} must be an array — run \`mdss index\` to rebuild`);
+        }
+        const validatedCluster = new Array(list.length);
+        const seen = new Set();
+        for (let j = 0; j < list.length; j++) {
+            const idx = list[j];
+            if (typeof idx !== 'number' || !Number.isSafeInteger(idx) || idx < 0) {
+                throw new Error(`invalid IVF payload: cluster ${c} contains non-integer index "${idx}" — run \`mdss index\` to rebuild`);
+            }
+            if (expectedChunksCount !== undefined && idx >= expectedChunksCount) {
+                throw new Error(`invalid IVF payload: chunk index ${idx} out of range [0, ${expectedChunksCount}) — run \`mdss index\` to rebuild`);
+            }
+            if (seen.has(idx)) {
+                throw new Error(`invalid IVF payload: duplicate chunk index ${idx} in cluster ${c} — run \`mdss index\` to rebuild`);
+            }
+            seen.add(idx);
+            validatedCluster[j] = idx;
+        }
+        clusters[c] = validatedCluster;
+    }
     return { dim, k, centroids, clusters };
 }

package/dist/lexical.d.ts

@@ -33,6 +33,16 @@ export declare function analyzeLexicalDocument(chunk: ChunkForLexical, weights?:
 }): TermFrequencies;
 export declare function buildLexicalIndex(records: TermFrequencies[]): LexicalIndex;
 export declare function editDistance(a: string, b: string): number;
+/**
+ * Damerau-Levenshtein distance with a threshold (issue #193): banded DP that
+ * never leaves the |i-j| <= maxDist diagonal and aborts as soon as the whole
+ * row exceeds the threshold — O(maxDist × min(alen, blen)) instead of the
+ * full O(alen × blen) matrix. Returns the EXACT distance when it is
+ * <= maxDist and null otherwise; decisions are identical to
+ * `editDistance(a, b) <= maxDist` (optimal paths within the threshold never
+ * leave the band).
+ */
+export declare function editDistanceAtMost(a: string, b: string, maxDist: number): number | null;
 export declare function fuzzyTitleAliasScores(chunks: Array<{
     file?: string;
     title: string;

package/dist/lexical.js

@@ -100,6 +100,90 @@ export function editDistance(a, b) {
     }
     return matrix[alen][blen];
 }
+/**
+ * Damerau-Levenshtein distance with a threshold (issue #193): banded DP that
+ * never leaves the |i-j| <= maxDist diagonal and aborts as soon as the whole
+ * row exceeds the threshold — O(maxDist × min(alen, blen)) instead of the
+ * full O(alen × blen) matrix. Returns the EXACT distance when it is
+ * <= maxDist and null otherwise; decisions are identical to
+ * `editDistance(a, b) <= maxDist` (optimal paths within the threshold never
+ * leave the band).
+ */
+export function editDistanceAtMost(a, b, maxDist) {
+    if (a === b)
+        return 0;
+    const alen = a.length;
+    const blen = b.length;
+    if (Math.abs(alen - blen) > maxDist)
+        return null;
+    if (alen === 0)
+        return blen <= maxDist ? blen : null;
+    if (blen === 0)
+        return alen <= maxDist ? alen : null;
+    if (maxDist < 0)
+        return null;
+    const INF = maxDist + 1;
+    // rows are padded with an INF sentinel column on each side so band-edge
+    // lookups (j-1 / j+1) never read a real out-of-band value
+    const width = blen + 3;
+    const rowA = new Int32Array(width);
+    const rowB = new Int32Array(width);
+    const rowC = new Int32Array(width);
+    let prev2 = null; // row i-2 (for transpositions)
+    let prev = rowA;
+    let cur = rowB;
+    const scratch = rowC;
+    for (let j = 1; j <= blen + 1; j++)
+        prev[j] = j; // row 0: dp[0][j] = j (col 0 unused sentinel)
+    prev[0] = 0;
+    for (let i = 1; i <= alen; i++) {
+        const jLo = Math.max(1, i - maxDist);
+        const jHi = Math.min(blen, i + maxDist);
+        cur[0] = i; // dp[i][0] = i (column 0 inside the band only when i <= maxDist)
+        if (i > maxDist)
+            cur[0] = INF;
+        let rowMin = cur[0];
+        for (let j = jLo; j <= jHi; j++) {
+            const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+            let v = Math.min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + cost);
+            if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1] && prev2 !== null) {
+                v = Math.min(v, prev2[j - 2] + cost);
+            }
+            cur[j] = v;
+            if (v < rowMin)
+                rowMin = v;
+        }
+        // band-out cells stay INF for this row
+        for (let j = 1; j <= blen; j++) {
+            if (j < jLo || j > jHi)
+                cur[j] = INF;
+        }
+        if (rowMin > maxDist)
+            return null;
+        // three-row rotation: prev2 <- row(i-1), prev <- row(i), cur <- freed row
+        const free = prev2 ?? scratch;
+        prev2 = prev;
+        prev = cur;
+        cur = free;
+    }
+    const d = prev[blen];
+    return d <= maxDist ? d : null;
+}
+/**
+ * issue #193: target tokenization memoized per chunk object — the fuzzy scan
+ * runs on every query, but a chunk's title/aliases tokenize to the same tokens
+ * every time. WeakMap so runtime chunks (stable objects in serve) pay once.
+ */
+const fuzzyTargetTokens = new WeakMap();
+function targetTokens(chunk) {
+    let tokens = fuzzyTargetTokens.get(chunk);
+    if (!tokens) {
+        const targets = [chunk.title, ...(chunk.meta?.aliases || [])].filter(Boolean);
+        tokens = targets.flatMap((t) => tokenize(t));
+        fuzzyTargetTokens.set(chunk, tokens);
+    }
+    return tokens;
+}
 export function fuzzyTitleAliasScores(chunks, query) {
     const scores = new Map();
     const qTerms = tokenize(query);
@@ -107,23 +191,22 @@ export function fuzzyTitleAliasScores(chunks, query) {
         return scores;
     for (let docId = 0; docId < chunks.length; docId++) {
         const c = chunks[docId];
-        const targets = [c.title, ...(c.meta?.aliases || [])].filter(Boolean);
+        const tTerms = targetTokens(c);
         let maxScore = 0;
-        for (const target of targets) {
-            const tTerms = tokenize(target);
-            for (const qt of qTerms) {
-                if (qt.length < 3)
+        for (const qt of qTerms) {
+            if (qt.length < 3)
+                continue;
+            const maxDist = Math.min(2, Math.floor(qt.length / 3));
+            for (const tt of tTerms) {
+                if (tt.length < 3)
                     continue;
-                for (const tt of tTerms) {
-                    if (tt.length < 3)
-                        continue;
-                    const maxDist = Math.min(2, Math.floor(qt.length / 3));
-                    const dist = editDistance(qt, tt);
-                    if (dist <= maxDist) {
-                        const similarity = 1 - dist / Math.max(qt.length, tt.length);
-                        if (similarity > maxScore)
-                            maxScore = similarity;
-                    }
+                // exact same decision as editDistance(qt, tt) <= maxDist, without the
+                // full O(len²) matrix (issue #193)
+                const dist = editDistanceAtMost(qt, tt, maxDist);
+                if (dist !== null) {
+                    const similarity = 1 - dist / Math.max(qt.length, tt.length);
+                    if (similarity > maxScore)
+                        maxScore = similarity;
                 }
             }
         }

package/dist/markdown-parser.d.ts

@@ -21,8 +21,70 @@ export interface StructuralChunk {
 }
 export declare function parseMarkdownBlocks(body: string): MarkdownBlock[];
 export declare function resolveChunkBudget(tokens?: number, defaultMax?: number): number;
+/**
+ * issue #202: script-calibrated token estimate for the sentencepiece E5/BGE
+ * vocab family — the fast-path alternative to running the real tokenizer per
+ * block. Ratios are calibrated so the estimate errs slightly HIGH: a budget
+ * checked against this estimate never overflows the model's real maxTokens.
+ *   Latin/digits/punctuation ≈ 4 chars/token, Cyrillic ≈ 1.8 chars/token
+ *   (RU systematically under-chunked by the old char budget — the point of
+ *   this issue), CJK ≈ 1 token/char, whitespace mostly merges into the
+ *   following subword so it counts a little.
+ */
+export declare function estimateTokens(text: string): number;
+/** Token measure: injected exact counter when available, else the calibrated estimate. */
+export type TokenCounter = (text: string) => number;
+export interface ChunkBudget {
+    /** Max measured size a single chunk may reach before it is split/flushed. */
+    budget: number;
+    /** Measures a text in the same unit the budget is expressed in. */
+    measure: TokenCounter;
+    /** Unit name for diagnostics/tests. */
+    unit: 'chars' | 'tokens';
+}
+/** Char budget (pre-#202 behavior) vs token budget (issue #202). */
+export declare function resolveChunkingBudget(opts: {
+    maxChunk?: number;
+    maxChunkTokens?: number;
+    countTokens?: TokenCounter;
+}): ChunkBudget;
+/**
+ * issue #202: the DEFAULT chunk budget in tokens, derived from the embedding
+ * adapter's context window. Capped at 512 — a passage much larger than that
+ * dilutes retrieval granularity even for models with 8k+ context (bge-m3).
+ * Returns undefined for adapters without a known maxTokens, which keeps the
+ * legacy char budget instead of guessing.
+ */
+export declare function defaultChunkTokenBudget(adapter: {
+    maxTokens?: number;
+}): number | undefined;
 export declare function chunkMarkdownStructural(input: string | MarkdownBlock[], opts?: number | {
     maxChunk?: number;
     targetTokens?: number;
+    maxChunkTokens?: number;
+    countTokens?: TokenCounter;
 }): StructuralChunk[];
+export interface OversizedPiece {
+    text: string;
+    /** 0-based block-relative range of REAL source lines backing this piece. */
+    startLine: number;
+    endLine: number;
+}
 export declare function splitOversizedBlock(text: string, maxChunk: number, blockType?: MarkdownBlock['type']): string[];
+/**
+ * issue #202: oversized-block splitting under an arbitrary measure (chars or
+ * tokens). The heavy lifting (fence repeats, table header repeats, #167 line
+ * ranges) stays in the char-based splitOversizedBlockWithRanges; the token
+ * budget is converted to a char budget for THIS block via its own chars-per-
+ * measured-unit ratio, then every piece is verified — script-skew inside the
+ * block (ASCII header + Cyrillic body) is caught by a greedy re-split.
+ */
+export declare function splitOversizedByMeasure(text: string, budget: number, measure: TokenCounter, blockType?: MarkdownBlock['type']): OversizedPiece[];
+/**
+ * Issue #167: like splitOversizedBlock, but each piece also reports the
+ * block-relative range of source lines it was built from. Synthetic lines
+ * added to keep pieces self-contained (fence repeats, table header repeats)
+ * do NOT count — only the first piece's opening fence/header and a real
+ * closing fence belong to the source.
+ */
+export declare function splitOversizedBlockWithRanges(text: string, maxChunk: number, blockType?: MarkdownBlock['type']): OversizedPiece[];