md-semantic-search 1.0.3 → 2.0.0

package/src/indexer.ts

@@ -36,8 +36,10 @@ import {
 } from './lexical.js';
 import { embeddingAdapterFingerprint, legacyEmbeddingAdapterFingerprint } from './models.js';
 import { trainIVF, serializeIVF, ANN_THRESHOLD } from './ivf.js';
+import { buildHnsw, serializeHnsw, HnswIndex } from './hnsw.js';
 import { DocumentMetadata } from './frontmatter.js';
 import { serializeBinaryIndex } from './binary-format.js';
+import { defaultChunkTokenBudget } from './markdown-parser.js';
 
 export interface IndexChunk {
   file: string;
@@ -58,6 +60,14 @@ export interface PersistedIndex {
   model?: string | null;
   modelAlias?: string;
   adapterFingerprint?: string;
+  /**
+   * issue #202: the effective chunk token budget these chunks were built
+   * with (null = legacy char budget). A mismatch forces a full re-chunk —
+   * otherwise the file-level fast path would keep stale char-budget chunks
+   * for unchanged files and produce a mixed index. Identical chunk texts
+   * still reuse their cached vectors through chunkHash.
+   */
+  chunkBudget?: number | null;
   dim?: number;
   db?: string;
   built?: string;
@@ -78,7 +88,17 @@ export const _stats = {
   embedCount: 0,
 };
 
-const BATCH = 32;
+// issue #194: batch size is tunable — 64-128 often feeds ONNX noticeably
+// better on short passages; overridable for experiments (clamped 1..512).
+const BATCH = (() => {
+  const raw = parseInt(process.env.MDSS_BATCH || '', 10);
+  return Number.isSafeInteger(raw) && raw > 0 ? Math.min(raw, 512) : 32;
+})();
+// issue #194: mid-run checkpoints serialize the WHOLE index so far — with
+// faster embedding that cost scales up. Time-budget them (~every 30 s) in
+// addition to the batch cadence, so at most ~30 s of embedding is lost on a
+// crash while serialization stays a small slice of wall time.
+const CHECKPOINT_MIN_INTERVAL_MS = 30_000;
 const CHECKPOINT_BATCHES = 8;
 const INDEX_FORMAT = 'binary-v1';
 
@@ -156,6 +176,7 @@ export interface BuildIndexOptions {
   indexDir: string;
   cacheDir: string;
   modelName: string;
+  dimensions?: number;
   ignore?: string[];
   offline?: boolean;
   log?: (s: string) => void;
@@ -164,8 +185,16 @@ export interface BuildIndexOptions {
   maxRetries?: number;
   onProgress?: (done: number, total: number, chunksPerSec: number) => void;
   workers?: number;
-  ann?: boolean;
+  /** issue #201: ANN sidecar engine — `true`/`'ivf'` = ivf.json, `'hnsw'` = hnsw.json. */
+  ann?: boolean | 'ivf' | 'hnsw';
   quantize?: 'fp32' | 'int8';
+  /**
+   * issue #202: chunk budget in TOKENS (overrides the legacy char budget).
+   * Default: min(adapter.maxTokens, 512) — the point of the issue is that the
+   * char budget overflowed real maxTokens for Cyrillic. Explicit `0` restores
+   * the legacy char behavior.
+   */
+  maxChunkTokens?: number;
 }
 
 export interface BuildIndexResult {
@@ -187,6 +216,7 @@ export async function buildIndex(opts: BuildIndexOptions): Promise<BuildIndexRes
     indexDir,
     cacheDir,
     modelName,
+    dimensions,
     ignore = [],
     log = () => {},
     offline = false,
@@ -197,11 +227,12 @@ export async function buildIndex(opts: BuildIndexOptions): Promise<BuildIndexRes
     workers = 1,
     ann = false,
     quantize = 'fp32',
+    maxChunkTokens,
   } = opts;
   if (_lockHeld)
-    return _buildIndexInner({ db, indexDir, cacheDir, modelName, ignore, log, offline, embedFn, maxRetries, onProgress, workers, ann, quantize });
+    return _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions, ignore, log, offline, embedFn, maxRetries, onProgress, workers, ann, quantize, maxChunkTokens });
   return withIndexLock(indexDir, () =>
-    _buildIndexInner({ db, indexDir, cacheDir, modelName, ignore, log, offline, embedFn, maxRetries, onProgress, workers, ann, quantize })
+    _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions, ignore, log, offline, embedFn, maxRetries, onProgress, workers, ann, quantize, maxChunkTokens })
   );
 }
 
@@ -210,6 +241,7 @@ async function _buildIndexInner({
   indexDir,
   cacheDir,
   modelName,
+  dimensions,
   ignore = [],
   log = () => {},
   offline = false,
@@ -219,13 +251,24 @@ async function _buildIndexInner({
   workers = 1,
   ann = false,
   quantize = 'fp32',
-}: Required<Omit<BuildIndexOptions, '_lockHeld' | 'onProgress' | 'workers' | 'ann' | 'quantize'>> & {
+  maxChunkTokens = undefined,
+}: Required<Omit<BuildIndexOptions, '_lockHeld' | 'onProgress' | 'workers' | 'ann' | 'quantize' | 'dimensions' | 'maxChunkTokens'>> & {
+  dimensions?: number;
   onProgress?: (done: number, total: number, chunksPerSec: number) => void;
   workers?: number;
-  ann?: boolean;
+  ann?: boolean | 'ivf' | 'hnsw';
   quantize?: 'fp32' | 'int8';
+  maxChunkTokens?: number;
 }): Promise<BuildIndexResult> {
-  const model = resolveModel(modelName);
+  const model = resolveModel(modelName, dimensions);
+  // issue #202: explicit maxChunkTokens wins; `0` restores the legacy char
+  // budget; otherwise min(adapter.maxTokens, 512), undefined → char mode
+  const effectiveChunkTokens =
+    maxChunkTokens === 0
+      ? undefined
+      : maxChunkTokens && maxChunkTokens > 0
+        ? maxChunkTokens
+        : defaultChunkTokenBudget(model);
   const modelIdentity = `${model.id}@${model.revision || 'main'}`;
   const adapterFingerprint = embeddingAdapterFingerprint(model);
   const legacyAdapterFingerprint = legacyEmbeddingAdapterFingerprint(model);
@@ -235,10 +278,15 @@ async function _buildIndexInner({
   const vectorsPath = path.join(indexDir, 'vectors.json');
   const hashesPath = path.join(indexDir, '.hashes.json');
   const checkpointPath = path.join(indexDir, '.checkpoint.json');
+  let lastCheckpointAt = 0;
 
   fs.mkdirSync(indexDir, { recursive: true });
 
-  const files = walkMarkdown(db, ignore);
+  // issue #189: unreadable directories shrink the corpus — collect and log
+  // them so a "shrinking" index is never published silently
+  const walkErrors: string[] = [];
+  const files = walkMarkdown(db, ignore, walkErrors);
+  for (const we of walkErrors) log(we);
   const canonicalHashes = loadJSON<Record<string, string>>(hashesPath, {}, log);
   let canonicalIndex = loadJSON<PersistedIndex>(vectorsPath, { chunks: [] }, log);
   let canonicalSchema = 0;
@@ -337,6 +385,14 @@ async function _buildIndexInner({
 
   const modelChanged = !!oldIndex.model && oldIndex.model !== modelIdentity;
   const contextVectorsReusable = sourceSchema >= 2 && !modelChanged && adapterCompatible(oldIndex);
+  // issue #202: a different chunk budget invalidates the file-level fast path
+  // (unchanged files must be re-chunked), NOT the per-chunk vector cache —
+  // chunks whose text did not move across the budget boundary re-embed free.
+  const chunkBudgetMarker = effectiveChunkTokens ?? null;
+  const chunkerChanged = (oldIndex.chunkBudget ?? null) !== chunkBudgetMarker;
+  if (chunkerChanged && oldIndex.chunks.length > 0) {
+    log('chunk budget changed — re-chunking all files (per-chunk vector cache still applies).');
+  }
   let buildDim = model.dim > 0 ? model.dim : contextVectorsReusable && oldDimensionValid ? oldExpectedDim : undefined;
   const acceptBuildVector = (vector: unknown) => {
     const length = validateNumericVector(vector, buildDim, 'embedding vector');
@@ -398,6 +454,7 @@ async function _buildIndexInner({
     model: modelIdentity,
     modelAlias: typeof modelName === 'string' ? modelName || 'e5-base' : model.id,
     adapterFingerprint,
+    chunkBudget: chunkBudgetMarker,
     ...(buildDim === undefined ? {} : { dim: buildDim }),
     db,
     built: new Date().toISOString(),
@@ -418,7 +475,7 @@ async function _buildIndexInner({
       newHashes[rel] = h;
       if (oldHashes[rel] === h) reusableLexicalRecords = lexicalByFile.get(rel);
 
-      if (sourceSchema === SCHEMA_VERSION && contextVectorsReusable && oldHashes[rel] === h && oldByFile.has(rel)) {
+      if (sourceSchema === SCHEMA_VERSION && contextVectorsReusable && !chunkerChanged && oldHashes[rel] === h && oldByFile.has(rel)) {
         const old = oldByFile.get(rel)!;
         for (const c of old) {
           if (!c.chunkHash) c.chunkHash = chunkHash(model, c);
@@ -437,7 +494,7 @@ async function _buildIndexInner({
         continue;
       }
       changedFiles++;
-      parsed = parseFile(abs, db, undefined, raw);
+      parsed = parseFile(abs, db, undefined, raw, effectiveChunkTokens);
     } catch (e: any) {
       delete newHashes[rel];
       skipped++;
@@ -493,7 +550,8 @@ async function _buildIndexInner({
           c.vec = vecs[j];
         });
         completedBatches++;
-        if (completedBatches % CHECKPOINT_BATCHES === 0) {
+        if (completedBatches % CHECKPOINT_BATCHES === 0 && Date.now() - lastCheckpointAt >= CHECKPOINT_MIN_INTERVAL_MS) {
+          lastCheckpointAt = Date.now();
           atomicWrite(checkpointPath, JSON.stringify(checkpointSnapshot(false)));
         }
         const done = Math.min(i + BATCH, toEmbed.length);
@@ -539,7 +597,8 @@ async function _buildIndexInner({
           });
           completedBatches++;
           embeddedCount += slice.length;
-          if (completedBatches % CHECKPOINT_BATCHES === 0) {
+          if (completedBatches % CHECKPOINT_BATCHES === 0 && Date.now() - lastCheckpointAt >= CHECKPOINT_MIN_INTERVAL_MS) {
+            lastCheckpointAt = Date.now();
             atomicWrite(checkpointPath, JSON.stringify(checkpointSnapshot(false)));
           }
           const elapsedSec = (Date.now() - embedStartTime) / 1000;
@@ -569,6 +628,7 @@ async function _buildIndexInner({
     model: modelIdentity,
     modelAlias: typeof modelName === 'string' ? modelName || 'e5-base' : model.id,
     adapterFingerprint,
+    chunkBudget: chunkBudgetMarker,
     ...(dim === undefined ? {} : { dim }),
     db,
     built: new Date().toISOString(),
@@ -589,15 +649,48 @@ async function _buildIndexInner({
     atomicWrite(vectorsBinPath, binBuffer);
   }
 
+  // issue #201: ANN sidecar engine — explicit 'hnsw' builds the graph
+  // instead of IVF; `true`/undefined/'ivf' keep the historical ivf.json path.
+  const annEngine = ann === 'hnsw' ? 'hnsw' : 'ivf';
+  const buildSidecar = Boolean(ann) || chunks.length >= ANN_THRESHOLD;
   const ivfPath = path.join(indexDir, 'ivf.json');
-  if (ann || chunks.length >= ANN_THRESHOLD) {
+  const hnswPath = path.join(indexDir, 'hnsw.json');
+  if (buildSidecar) {
     const rawVecs = chunks.map((c) => (c.vec instanceof Float32Array ? c.vec : Float32Array.from(c.vec as number[])));
-    const ivfIndex = trainIVF(rawVecs);
-    atomicWrite(ivfPath, JSON.stringify(serializeIVF(ivfIndex)));
-  } else if (fs.existsSync(ivfPath)) {
-    try {
-      fs.unlinkSync(ivfPath);
-    } catch {}
+    if (annEngine === 'hnsw') {
+      const hnswIndex: HnswIndex = buildHnsw(rawVecs, {
+        vectors: rawVecs,
+        // cosine distance (lower = closer) over the flat normalized vectors
+        distance: (a, b) => {
+          const bv = rawVecs[b];
+          let dot = 0;
+          for (let i = 0; i < a.length; i++) dot += (a as ArrayLike<number>)[i] * (bv as ArrayLike<number>)[i];
+          return 1 - dot;
+        },
+      });
+      atomicWrite(hnswPath, JSON.stringify(serializeHnsw(hnswIndex)));
+      if (fs.existsSync(ivfPath)) {
+        try {
+          fs.unlinkSync(ivfPath);
+        } catch {}
+      }
+    } else {
+      const ivfIndex = trainIVF(rawVecs);
+      atomicWrite(ivfPath, JSON.stringify(serializeIVF(ivfIndex)));
+      if (fs.existsSync(hnswPath)) {
+        try {
+          fs.unlinkSync(hnswPath);
+        } catch {}
+      }
+    }
+  } else {
+    for (const sidecar of [ivfPath, hnswPath]) {
+      if (fs.existsSync(sidecar)) {
+        try {
+          fs.unlinkSync(sidecar);
+        } catch {}
+      }
+    }
   }
 
   fs.unlinkSync(checkpointPath);

package/src/ivf.ts

@@ -4,7 +4,7 @@ export interface IVFIndex {
   dim: number;
   k: number;
   centroids: Float32Array; // Flattened k * dim
-  clusters: number[][];    // clusters[clusterIndex] = array of chunk indices
+  clusters: number[][]; // clusters[clusterIndex] = array of chunk indices
 }
 
 export interface SerializedIVF {
@@ -14,15 +14,36 @@ export interface SerializedIVF {
   clusters: number[][];
 }
 
+export interface TrainIVFOptions {
+  k?: number;
+  maxIterations?: number;
+  seeding?: 'kmeans++' | 'spread';
+  seed?: number;
+  onMetric?: (metric: { emptyClusters: number; iterations: number }) => void;
+}
+
 export const ANN_THRESHOLD = 500;
 export const DEFAULT_NPROBE = 8;
 
 /**
- * Train a Spherical K-Means IVF index over normalized vectors.
+ * Fast, deterministic pseudo-random number generator (Mulberry32).
+ */
+function mulberry32(seed: number): () => number {
+  let s = seed >>> 0;
+  return () => {
+    s = (s + 0x6d2b79f5) >>> 0;
+    let t = Math.imul(s ^ (s >>> 15), 1 | s);
+    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+/**
+ * Train a Spherical K-Means IVF index over normalized vectors using k-means++ seeding.
  */
 export function trainIVF(
   vectors: Float32Array[],
-  options: { k?: number; maxIterations?: number } = {}
+  options: TrainIVFOptions = {}
 ): IVFIndex {
   const n = vectors.length;
   if (n === 0) {
@@ -32,19 +53,70 @@ export function trainIVF(
   const targetK = options.k ?? Math.min(256, Math.max(2, Math.floor(Math.sqrt(n))));
   const k = Math.min(targetK, n);
   const maxIterations = options.maxIterations ?? 20;
+  const seeding = options.seeding ?? 'kmeans++';
 
-  // Initialize centroids using deterministic spread or k-means++
   const centroids = new Float32Array(k * dim);
-  const step = Math.floor(n / k);
-  for (let c = 0; c < k; c++) {
-    const src = vectors[c * step];
-    centroids.set(src, c * dim);
+
+  if (seeding === 'spread' || n <= k) {
+    // Deterministic spread
+    const step = Math.floor(n / k);
+    for (let c = 0; c < k; c++) {
+      const src = vectors[c * step];
+      centroids.set(src, c * dim);
+    }
+  } else {
+    // Deterministic k-means++ seeding with D^2 weighted probability
+    const rng = mulberry32(options.seed ?? 42);
+    const firstIdx = Math.floor(rng() * n);
+    centroids.set(vectors[firstIdx], 0);
+
+    const minDistSq = new Float64Array(n);
+    for (let i = 0; i < n; i++) {
+      const cos = cosine(vectors[i], centroids, 0);
+      minDistSq[i] = Math.max(0, 1 - cos);
+    }
+
+    for (let c = 1; c < k; c++) {
+      let sumDistSq = 0;
+      for (let i = 0; i < n; i++) {
+        sumDistSq += minDistSq[i];
+      }
+
+      let chosenIdx = -1;
+      if (sumDistSq > 1e-12) {
+        const target = rng() * sumDistSq;
+        let cumulative = 0;
+        for (let i = 0; i < n; i++) {
+          cumulative += minDistSq[i];
+          if (cumulative >= target) {
+            chosenIdx = i;
+            break;
+          }
+        }
+      }
+      if (chosenIdx === -1 || chosenIdx >= n) {
+        chosenIdx = Math.floor(rng() * n);
+      }
+
+      centroids.set(vectors[chosenIdx], c * dim);
+
+      for (let i = 0; i < n; i++) {
+        const cos = cosine(vectors[i], centroids, c * dim);
+        const distSq = Math.max(0, 1 - cos);
+        if (distSq < minDistSq[i]) {
+          minDistSq[i] = distSq;
+        }
+      }
+    }
   }
 
   let clusters: number[][] = Array.from({ length: k }, () => []);
   const assignments = new Int32Array(n).fill(-1);
+  let totalEmptyClusters = 0;
+  let finalIterations = 0;
 
   for (let iter = 0; iter < maxIterations; iter++) {
+    finalIterations = iter + 1;
     let changed = 0;
     const newClusters: number[][] = Array.from({ length: k }, () => []);
 
@@ -71,11 +143,13 @@ export function trainIVF(
 
     clusters = newClusters;
 
-    // 2. Recompute centroids
+    // 2. Recompute centroids & rescue empty clusters
+    let emptyCount = 0;
     for (let c = 0; c < k; c++) {
       const members = clusters[c];
       if (members.length === 0) {
-        // If empty, reassign to a vector from the largest cluster
+        emptyCount++;
+        // Reassign to a vector from the largest cluster
         let largest = 0;
         for (let j = 1; j < k; j++) {
           if (clusters[j].length > clusters[largest].length) largest = j;
@@ -111,9 +185,12 @@ export function trainIVF(
       }
     }
 
+    totalEmptyClusters = emptyCount;
     if (changed === 0) break;
   }
 
+  options.onMetric?.({ emptyClusters: totalEmptyClusters, iterations: finalIterations });
+
   return { dim, k, centroids, clusters };
 }
 
@@ -171,20 +248,80 @@ export function serializeIVF(ivf: IVFIndex): SerializedIVF {
 }
 
 /**
- * Deserialize JSON object back to IVFIndex.
+ * Deserialize JSON object back to IVFIndex with hardened validation.
  */
-export function deserializeIVF(raw: any): IVFIndex {
-  if (!raw || typeof raw !== 'object') {
-    throw new Error('invalid IVF payload: expected object');
+export function deserializeIVF(raw: unknown, expectedChunksCount?: number): IVFIndex {
+  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
+    throw new Error('invalid IVF payload: expected JSON object — run `mdss index` to rebuild');
   }
-  const dim = Number(raw.dim);
-  const k = Number(raw.k);
-  if (!Number.isInteger(dim) || dim <= 0 || !Number.isInteger(k) || k <= 0) {
-    throw new Error('invalid IVF payload: invalid dim or k');
+  const obj = raw as Record<string, unknown>;
+  const dim = obj.dim;
+  const k = obj.k;
+  if (
+    typeof dim !== 'number' ||
+    !Number.isSafeInteger(dim) ||
+    dim <= 0 ||
+    typeof k !== 'number' ||
+    !Number.isSafeInteger(k) ||
+    k <= 0
+  ) {
+    throw new Error('invalid IVF payload: invalid dim or k — run `mdss index` to rebuild');
+  }
+  if (typeof obj.centroids !== 'string' || obj.centroids.length === 0) {
+    throw new Error('invalid IVF payload: centroids must be a base64 string — run `mdss index` to rebuild');
+  }
+  const buf = Buffer.from(obj.centroids, 'base64');
+  if (buf.byteLength % 4 !== 0 || buf.byteLength !== k * dim * 4) {
+    throw new Error(
+      `invalid IVF payload: centroids buffer has ${buf.byteLength} bytes, expected ${k * dim * 4} bytes — run \`mdss index\` to rebuild`
+    );
   }
-  const buf = Buffer.from(raw.centroids, 'base64');
   const centroids = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
-  const clusters = Array.isArray(raw.clusters) ? raw.clusters : [];
+  for (let i = 0; i < centroids.length; i++) {
+    if (!Number.isFinite(centroids[i])) {
+      throw new Error(`invalid IVF payload: non-finite centroid value at index ${i} — run \`mdss index\` to rebuild`);
+    }
+  }
+
+  if (!Array.isArray(obj.clusters)) {
+    throw new Error('invalid IVF payload: clusters must be an array — run `mdss index` to rebuild');
+  }
+  if (obj.clusters.length !== k) {
+    throw new Error(
+      `invalid IVF payload: clusters array has length ${obj.clusters.length}, expected ${k} — run \`mdss index\` to rebuild`
+    );
+  }
+
+  const clusters: number[][] = new Array(k);
+  for (let c = 0; c < k; c++) {
+    const list = obj.clusters[c];
+    if (!Array.isArray(list)) {
+      throw new Error(`invalid IVF payload: cluster ${c} must be an array — run \`mdss index\` to rebuild`);
+    }
+    const validatedCluster: number[] = new Array(list.length);
+    const seen = new Set<number>();
+    for (let j = 0; j < list.length; j++) {
+      const idx = list[j];
+      if (typeof idx !== 'number' || !Number.isSafeInteger(idx) || idx < 0) {
+        throw new Error(
+          `invalid IVF payload: cluster ${c} contains non-integer index "${idx}" — run \`mdss index\` to rebuild`
+        );
+      }
+      if (expectedChunksCount !== undefined && idx >= expectedChunksCount) {
+        throw new Error(
+          `invalid IVF payload: chunk index ${idx} out of range [0, ${expectedChunksCount}) — run \`mdss index\` to rebuild`
+        );
+      }
+      if (seen.has(idx)) {
+        throw new Error(
+          `invalid IVF payload: duplicate chunk index ${idx} in cluster ${c} — run \`mdss index\` to rebuild`
+        );
+      }
+      seen.add(idx);
+      validatedCluster[j] = idx;
+    }
+    clusters[c] = validatedCluster;
+  }
 
   return { dim, k, centroids, clusters };
 }

package/src/lexical.ts

@@ -129,6 +129,85 @@ export function editDistance(a: string, b: string): number {
   return matrix[alen][blen];
 }
 
+/**
+ * Damerau-Levenshtein distance with a threshold (issue #193): banded DP that
+ * never leaves the |i-j| <= maxDist diagonal and aborts as soon as the whole
+ * row exceeds the threshold — O(maxDist × min(alen, blen)) instead of the
+ * full O(alen × blen) matrix. Returns the EXACT distance when it is
+ * <= maxDist and null otherwise; decisions are identical to
+ * `editDistance(a, b) <= maxDist` (optimal paths within the threshold never
+ * leave the band).
+ */
+export function editDistanceAtMost(a: string, b: string, maxDist: number): number | null {
+  if (a === b) return 0;
+  const alen = a.length;
+  const blen = b.length;
+  if (Math.abs(alen - blen) > maxDist) return null;
+  if (alen === 0) return blen <= maxDist ? blen : null;
+  if (blen === 0) return alen <= maxDist ? alen : null;
+  if (maxDist < 0) return null;
+
+  const INF = maxDist + 1;
+  // rows are padded with an INF sentinel column on each side so band-edge
+  // lookups (j-1 / j+1) never read a real out-of-band value
+  const width = blen + 3;
+  const rowA = new Int32Array(width);
+  const rowB = new Int32Array(width);
+  const rowC = new Int32Array(width);
+  let prev2: Int32Array<ArrayBuffer> | null = null; // row i-2 (for transpositions)
+  let prev: Int32Array<ArrayBuffer> = rowA;
+  let cur: Int32Array<ArrayBuffer> = rowB;
+  const scratch: Int32Array<ArrayBuffer> = rowC;
+  for (let j = 1; j <= blen + 1; j++) prev[j] = j; // row 0: dp[0][j] = j (col 0 unused sentinel)
+  prev[0] = 0;
+
+  for (let i = 1; i <= alen; i++) {
+    const jLo = Math.max(1, i - maxDist);
+    const jHi = Math.min(blen, i + maxDist);
+    cur[0] = i; // dp[i][0] = i (column 0 inside the band only when i <= maxDist)
+    if (i > maxDist) cur[0] = INF;
+    let rowMin = cur[0];
+    for (let j = jLo; j <= jHi; j++) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+      let v = Math.min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + cost);
+      if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1] && prev2 !== null) {
+        v = Math.min(v, prev2[j - 2] + cost);
+      }
+      cur[j] = v;
+      if (v < rowMin) rowMin = v;
+    }
+    // band-out cells stay INF for this row
+    for (let j = 1; j <= blen; j++) {
+      if (j < jLo || j > jHi) cur[j] = INF;
+    }
+    if (rowMin > maxDist) return null;
+    // three-row rotation: prev2 <- row(i-1), prev <- row(i), cur <- freed row
+    const free = prev2 ?? scratch;
+    prev2 = prev;
+    prev = cur;
+    cur = free;
+  }
+  const d = prev[blen];
+  return d <= maxDist ? d : null;
+}
+
+/**
+ * issue #193: target tokenization memoized per chunk object — the fuzzy scan
+ * runs on every query, but a chunk's title/aliases tokenize to the same tokens
+ * every time. WeakMap so runtime chunks (stable objects in serve) pay once.
+ */
+const fuzzyTargetTokens = new WeakMap<object, string[]>();
+
+function targetTokens(chunk: { title: string; meta?: DocumentMetadata }): string[] {
+  let tokens = fuzzyTargetTokens.get(chunk);
+  if (!tokens) {
+    const targets = [chunk.title, ...(chunk.meta?.aliases || [])].filter(Boolean);
+    tokens = targets.flatMap((t) => tokenize(t));
+    fuzzyTargetTokens.set(chunk, tokens);
+  }
+  return tokens;
+}
+
 export function fuzzyTitleAliasScores(
   chunks: Array<{ file?: string; title: string; meta?: DocumentMetadata }>,
   query: string
@@ -139,21 +218,20 @@ export function fuzzyTitleAliasScores(
 
   for (let docId = 0; docId < chunks.length; docId++) {
     const c = chunks[docId];
-    const targets = [c.title, ...(c.meta?.aliases || [])].filter(Boolean);
+    const tTerms = targetTokens(c);
     let maxScore = 0;
 
-    for (const target of targets) {
-      const tTerms = tokenize(target);
-      for (const qt of qTerms) {
-        if (qt.length < 3) continue;
-        for (const tt of tTerms) {
-          if (tt.length < 3) continue;
-          const maxDist = Math.min(2, Math.floor(qt.length / 3));
-          const dist = editDistance(qt, tt);
-          if (dist <= maxDist) {
-            const similarity = 1 - dist / Math.max(qt.length, tt.length);
-            if (similarity > maxScore) maxScore = similarity;
-          }
+    for (const qt of qTerms) {
+      if (qt.length < 3) continue;
+      const maxDist = Math.min(2, Math.floor(qt.length / 3));
+      for (const tt of tTerms) {
+        if (tt.length < 3) continue;
+        // exact same decision as editDistance(qt, tt) <= maxDist, without the
+        // full O(len²) matrix (issue #193)
+        const dist = editDistanceAtMost(qt, tt, maxDist);
+        if (dist !== null) {
+          const similarity = 1 - dist / Math.max(qt.length, tt.length);
+          if (similarity > maxScore) maxScore = similarity;
         }
       }
     }