md-semantic-search 2.0.0 → 2.1.0

package/README.md

@@ -679,7 +679,7 @@ Add to your `claude_desktop_config.json`:
 | `--multi-query <n>` | Multi-query expansion (issue #203): N query variants (original + N−1 LLM paraphrases) fused by union-RRF; needs `--llm-endpoint`, degrades silently without it. |
 | `--graph-boost <n>` | Weight for Obsidian wikilink graph ranking and PageRank prior (search; default 0). Incorporates document centrality and 2-hop contextual propagation. |
 | `--filter <expr>` | Rich boolean filter across frontmatter tags and properties (e.g. `tag:engineering AND status != archived`). |
-| `--ann [ivf\|hnsw]` | Approximate nearest neighbor search (search; auto-enables for large corpora). Engines (issue #201): `ivf` (k-means, the default sidecar) or `hnsw` (graph — no training phase, 100% measured recall at equal-or-lower query latency, `hnsw.json` sidecar, corrupt → brute-force fallback). Bare `--ann` prefers an existing sidecar. |
+| `--ann [ivf\|hnsw]` | Approximate nearest neighbor search (search; auto-enables for large corpora). Engines (issue #201): `ivf` (k-means, the default sidecar) or `hnsw` (graph — no training phase, 100% measured recall at equal-or-lower query latency, `hnsw.json` sidecar, corrupt → brute-force fallback; watch rebuilds reuse the persisted graph via chunk-hash remapping instead of rebuilding it — issue #223). Bare `--ann` prefers an existing sidecar. |
 | `--nprobe <n>` | Number of nearest centroid clusters to probe during ANN search (default 8). |
 | `--port <n>` | HTTP port for `serve` (default 8747, or `MDSS_PORT`). |
 | `--host <ip>` | Bind address for `serve` (default `127.0.0.1` — loopback only; use `0.0.0.0` to expose on the LAN, or `MDSS_HOST`). |

package/RESEARCH.md

@@ -421,8 +421,12 @@ corpora keeps **IVF as the default engine**; the criterion's second branch
 2. Bare `--ann` and the automatic threshold path stay on IVF until either
    (a) the JS graph build gets an order of magnitude cheaper (worker
    parallelism, WASM SIMD — see §13), or (b) incremental watch rebuilds
-   start reusing the persisted graph (the `insertHnsw`/`deleteHnsw` API is
-   already in place for that).
+   start reusing the persisted graph. **(b) is now implemented (issue
+   #223):** a watch rebuild remaps the persisted graph onto the new chunk
+   numbering via `chunkHash` (`remapHnsw` — vector reuse is hash-keyed, so
+   every carried edge distance stays valid), tombstones the gone nodes and
+   inserts only the new ones; the build-side gap now applies to FULL builds
+   only (first run, engine/budget switch, entry-point-unanchorable remap).
 3. `--ef <n>` tunes the HNSW search breadth (default 64; recall saturates
    at 100% already at ef 16 on the bench corpora — the default leaves
    headroom for noisier real-world distributions).

package/dist/hnsw.d.ts

@@ -94,3 +94,12 @@ export declare function serializeHnsw(hnsw: HnswIndex): SerializedHnsw;
  * falls back to brute force).
  */
 export declare function deserializeHnsw(raw: unknown, expectedChunksCount?: number): HnswIndex;
+/**
+ * issue #223: carry a persisted graph across a watch rebuild. `mapping[oldIdx]`
+ * is the chunk's NEW index (undefined = the chunk is gone → tombstone); nodes
+ * carry over because vector reuse is keyed by chunkHash — a surviving chunk
+ * keeps its exact vector, so every stored edge distance stays valid.
+ * Returns null when the remap cannot produce a navigable graph (e.g. the
+ * entry point did not survive) — the caller falls back to a full rebuild.
+ */
+export declare function remapHnsw(hnsw: HnswIndex, mapping: Array<number | undefined>, newCount: number): HnswIndex | null;

package/dist/hnsw.js

@@ -324,3 +324,75 @@ export function deserializeHnsw(raw, expectedChunksCount) {
     }
     return { dim, M, seed, entryPoint, maxLevel, levels, neighbors, deleted };
 }
+/**
+ * issue #223: carry a persisted graph across a watch rebuild. `mapping[oldIdx]`
+ * is the chunk's NEW index (undefined = the chunk is gone → tombstone); nodes
+ * carry over because vector reuse is keyed by chunkHash — a surviving chunk
+ * keeps its exact vector, so every stored edge distance stays valid.
+ * Returns null when the remap cannot produce a navigable graph (e.g. the
+ * entry point did not survive) — the caller falls back to a full rebuild.
+ */
+export function remapHnsw(hnsw, mapping, newCount) {
+    if (newCount <= 0)
+        return null;
+    // every new index may receive at most one old node
+    const taken = new Set();
+    const remapOf = new Map();
+    for (let oldIdx = 0; oldIdx < mapping.length; oldIdx++) {
+        const newIdx = mapping[oldIdx];
+        if (newIdx === undefined || newIdx < 0 || newIdx >= newCount)
+            continue;
+        if (taken.has(newIdx))
+            continue;
+        taken.add(newIdx);
+        remapOf.set(oldIdx, newIdx);
+    }
+    // the entry point does NOT need to survive — it re-anchors below
+    const levels = new Array(newCount);
+    const neighbors = new Array(newCount);
+    const deleted = new Set();
+    for (const [oldIdx, newIdx] of remapOf) {
+        levels[newIdx] = hnsw.levels[oldIdx];
+        neighbors[newIdx] = (hnsw.neighbors[oldIdx] ?? []).map((links) => (links ?? [])
+            .map((nb) => remapOf.get(nb))
+            .filter((nb) => nb !== undefined));
+        if (hnsw.deleted.has(oldIdx))
+            deleted.add(newIdx);
+    }
+    // structural holes would break insertHnsw/serialize — unmapped new indices
+    // are placeholders the caller fills with insertHnsw immediately after
+    for (let i = 0; i < newCount; i++) {
+        if (levels[i] === undefined) {
+            levels[i] = 0;
+            neighbors[i] = [[]];
+            deleted.add(i); // invisible until a real node is inserted over it
+        }
+    }
+    // the entry point must sit on the TOP layer: after a partial remap the
+    // old entry may carry a lower level, or some survivor may carry a higher
+    // one — re-anchor deterministically to the lowest-index survivor at maxLevel
+    let maxLevel = -1;
+    for (const newIdx of remapOf.values())
+        maxLevel = Math.max(maxLevel, levels[newIdx]);
+    if (maxLevel < 0)
+        return null;
+    let entryPoint = -1;
+    for (let i = 0; i < newCount; i++) {
+        if (!deleted.has(i) && levels[i] === maxLevel) {
+            entryPoint = i;
+            break;
+        }
+    }
+    if (entryPoint < 0)
+        return null;
+    return {
+        dim: hnsw.dim,
+        M: hnsw.M,
+        seed: hnsw.seed,
+        entryPoint,
+        maxLevel,
+        levels,
+        neighbors,
+        deleted,
+    };
+}

package/dist/indexer.js

@@ -15,7 +15,7 @@ import { validateIndexEnvelope, validateNumericVector } from './index-format.js'
 import { analyzeLexicalDocument, buildLexicalIndex, lexicalIdentity, reverseLexicalIndex, validateLexicalIndex, } from './lexical.js';
 import { embeddingAdapterFingerprint, legacyEmbeddingAdapterFingerprint } from './models.js';
 import { trainIVF, serializeIVF, ANN_THRESHOLD } from './ivf.js';
-import { buildHnsw, serializeHnsw } from './hnsw.js';
+import { buildHnsw, insertHnsw, remapHnsw, serializeHnsw, deserializeHnsw } from './hnsw.js';
 import { serializeBinaryIndex } from './binary-format.js';
 import { defaultChunkTokenBudget } from './markdown-parser.js';
 const md5 = (s) => crypto.createHash('md5').update(s).digest('hex');
@@ -484,17 +484,48 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions,
     if (buildSidecar) {
         const rawVecs = chunks.map((c) => (c.vec instanceof Float32Array ? c.vec : Float32Array.from(c.vec)));
         if (annEngine === 'hnsw') {
-            const hnswIndex = buildHnsw(rawVecs, {
-                vectors: rawVecs,
-                // cosine distance (lower = closer) over the flat normalized vectors
-                distance: (a, b) => {
-                    const bv = rawVecs[b];
-                    let dot = 0;
-                    for (let i = 0; i < a.length; i++)
-                        dot += a[i] * bv[i];
-                    return 1 - dot;
-                },
-            });
+            const hnswDistance = (vectors) => (a, b) => {
+                const bv = vectors[b];
+                let dot = 0;
+                for (let i = 0; i < a.length; i++)
+                    dot += a[i] * bv[i];
+                return 1 - dot;
+            };
+            // issue #223: reuse the persisted graph across watch rebuilds — remap
+            // surviving nodes onto the new chunk numbering via chunkHash (vector
+            // reuse is hash-keyed, so every carried edge distance stays valid),
+            // tombstone the gone ones, insert the new ones. Full rebuild only when
+            // no valid previous graph exists (first run, engine/budget switch) or
+            // the remap cannot anchor an entry point.
+            let hnswIndex = null;
+            if (!chunkerChanged && oldIndex.chunks.length > 0) {
+                try {
+                    const prev = deserializeHnsw(JSON.parse(fs.readFileSync(hnswPath, 'utf8')), oldIndex.chunks.length);
+                    if (prev.levels.length === oldIndex.chunks.length) {
+                        const newIdxByHash = new Map();
+                        chunks.forEach((c, newIdx) => newIdxByHash.set(c.chunkHash || chunkHash(model, c), newIdx));
+                        const mapping = oldIndex.chunks.map((c) => newIdxByHash.get(c.chunkHash || chunkHash(model, c)));
+                        hnswIndex = remapHnsw(prev, mapping, chunks.length);
+                        if (hnswIndex) {
+                            const inserted = [];
+                            for (let newIdx = 0; newIdx < chunks.length; newIdx++) {
+                                if (hnswIndex.levels[newIdx] !== undefined && !hnswIndex.deleted.has(newIdx))
+                                    continue;
+                                insertHnsw(hnswIndex, newIdx, hnswDistance(rawVecs), rawVecs);
+                                inserted.push(newIdx);
+                            }
+                            log(`hnsw: reused persisted graph — ` +
+                                `${chunks.length - inserted.length} surviving node(s), ${inserted.length} inserted`);
+                        }
+                    }
+                }
+                catch {
+                    hnswIndex = null; // corrupt/foreign sidecar — full rebuild below
+                }
+            }
+            if (!hnswIndex) {
+                hnswIndex = buildHnsw(rawVecs, { vectors: rawVecs, distance: hnswDistance(rawVecs) });
+            }
             atomicWrite(hnswPath, JSON.stringify(serializeHnsw(hnswIndex)));
             if (fs.existsSync(ivfPath)) {
                 try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "md-semantic-search",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "Local, private semantic (vector) search over any folder of Markdown files. Finds passages by meaning — cross-lingual — using transformers.js. No API keys, no cloud, no vector database.",
   "type": "module",
   "main": "./dist/index.js",

package/src/hnsw.ts

@@ -412,3 +412,79 @@ export function deserializeHnsw(raw: unknown, expectedChunksCount?: number): Hns
   }
   return { dim, M, seed, entryPoint, maxLevel, levels, neighbors, deleted };
 }
+
+/**
+ * issue #223: carry a persisted graph across a watch rebuild. `mapping[oldIdx]`
+ * is the chunk's NEW index (undefined = the chunk is gone → tombstone); nodes
+ * carry over because vector reuse is keyed by chunkHash — a surviving chunk
+ * keeps its exact vector, so every stored edge distance stays valid.
+ * Returns null when the remap cannot produce a navigable graph (e.g. the
+ * entry point did not survive) — the caller falls back to a full rebuild.
+ */
+export function remapHnsw(
+  hnsw: HnswIndex,
+  mapping: Array<number | undefined>,
+  newCount: number,
+): HnswIndex | null {
+  if (newCount <= 0) return null;
+  // every new index may receive at most one old node
+  const taken = new Set<number>();
+  const remapOf = new Map<number, number>();
+  for (let oldIdx = 0; oldIdx < mapping.length; oldIdx++) {
+    const newIdx = mapping[oldIdx];
+    if (newIdx === undefined || newIdx < 0 || newIdx >= newCount) continue;
+    if (taken.has(newIdx)) continue;
+    taken.add(newIdx);
+    remapOf.set(oldIdx, newIdx);
+  }
+  // the entry point does NOT need to survive — it re-anchors below
+
+  const levels: number[] = new Array(newCount);
+  const neighbors: number[][][] = new Array(newCount);
+  const deleted = new Set<number>();
+
+  for (const [oldIdx, newIdx] of remapOf) {
+    levels[newIdx] = hnsw.levels[oldIdx];
+    neighbors[newIdx] = (hnsw.neighbors[oldIdx] ?? []).map((links) =>
+      (links ?? [])
+        .map((nb) => remapOf.get(nb))
+        .filter((nb): nb is number => nb !== undefined),
+    );
+    if (hnsw.deleted.has(oldIdx)) deleted.add(newIdx);
+  }
+  // structural holes would break insertHnsw/serialize — unmapped new indices
+  // are placeholders the caller fills with insertHnsw immediately after
+  for (let i = 0; i < newCount; i++) {
+    if (levels[i] === undefined) {
+      levels[i] = 0;
+      neighbors[i] = [[]];
+      deleted.add(i); // invisible until a real node is inserted over it
+    }
+  }
+
+  // the entry point must sit on the TOP layer: after a partial remap the
+  // old entry may carry a lower level, or some survivor may carry a higher
+  // one — re-anchor deterministically to the lowest-index survivor at maxLevel
+  let maxLevel = -1;
+  for (const newIdx of remapOf.values()) maxLevel = Math.max(maxLevel, levels[newIdx]);
+  if (maxLevel < 0) return null;
+  let entryPoint = -1;
+  for (let i = 0; i < newCount; i++) {
+    if (!deleted.has(i) && levels[i] === maxLevel) {
+      entryPoint = i;
+      break;
+    }
+  }
+  if (entryPoint < 0) return null;
+
+  return {
+    dim: hnsw.dim,
+    M: hnsw.M,
+    seed: hnsw.seed,
+    entryPoint,
+    maxLevel,
+    levels,
+    neighbors,
+    deleted,
+  };
+}

package/src/indexer.ts

@@ -36,7 +36,7 @@ import {
 } from './lexical.js';
 import { embeddingAdapterFingerprint, legacyEmbeddingAdapterFingerprint } from './models.js';
 import { trainIVF, serializeIVF, ANN_THRESHOLD } from './ivf.js';
-import { buildHnsw, serializeHnsw, HnswIndex } from './hnsw.js';
+import { buildHnsw, insertHnsw, remapHnsw, serializeHnsw, deserializeHnsw, HnswIndex } from './hnsw.js';
 import { DocumentMetadata } from './frontmatter.js';
 import { serializeBinaryIndex } from './binary-format.js';
 import { defaultChunkTokenBudget } from './markdown-parser.js';
@@ -658,16 +658,51 @@ async function _buildIndexInner({
   if (buildSidecar) {
     const rawVecs = chunks.map((c) => (c.vec instanceof Float32Array ? c.vec : Float32Array.from(c.vec as number[])));
     if (annEngine === 'hnsw') {
-      const hnswIndex: HnswIndex = buildHnsw(rawVecs, {
-        vectors: rawVecs,
-        // cosine distance (lower = closer) over the flat normalized vectors
-        distance: (a, b) => {
-          const bv = rawVecs[b];
+      const hnswDistance = (vectors: Array<Float32Array | number[]>) =>
+        (a: Float32Array | Int8Array | number[], b: number) => {
+          const bv = vectors[b];
           let dot = 0;
           for (let i = 0; i < a.length; i++) dot += (a as ArrayLike<number>)[i] * (bv as ArrayLike<number>)[i];
           return 1 - dot;
-        },
-      });
+        };
+
+      // issue #223: reuse the persisted graph across watch rebuilds — remap
+      // surviving nodes onto the new chunk numbering via chunkHash (vector
+      // reuse is hash-keyed, so every carried edge distance stays valid),
+      // tombstone the gone ones, insert the new ones. Full rebuild only when
+      // no valid previous graph exists (first run, engine/budget switch) or
+      // the remap cannot anchor an entry point.
+      let hnswIndex: HnswIndex | null = null;
+      if (!chunkerChanged && oldIndex.chunks.length > 0) {
+        try {
+          const prev = deserializeHnsw(JSON.parse(fs.readFileSync(hnswPath, 'utf8')), oldIndex.chunks.length);
+          if (prev.levels.length === oldIndex.chunks.length) {
+            const newIdxByHash = new Map<string, number>();
+            chunks.forEach((c, newIdx) => newIdxByHash.set(c.chunkHash || chunkHash(model, c), newIdx));
+            const mapping: Array<number | undefined> = oldIndex.chunks.map((c) =>
+              newIdxByHash.get(c.chunkHash || chunkHash(model, c)),
+            );
+            hnswIndex = remapHnsw(prev, mapping, chunks.length);
+            if (hnswIndex) {
+              const inserted: number[] = [];
+              for (let newIdx = 0; newIdx < chunks.length; newIdx++) {
+                if (hnswIndex.levels[newIdx] !== undefined && !hnswIndex.deleted.has(newIdx)) continue;
+                insertHnsw(hnswIndex, newIdx, hnswDistance(rawVecs), rawVecs);
+                inserted.push(newIdx);
+              }
+              log(
+                `hnsw: reused persisted graph — ` +
+                  `${chunks.length - inserted.length} surviving node(s), ${inserted.length} inserted`,
+              );
+            }
+          }
+        } catch {
+          hnswIndex = null; // corrupt/foreign sidecar — full rebuild below
+        }
+      }
+      if (!hnswIndex) {
+        hnswIndex = buildHnsw(rawVecs, { vectors: rawVecs, distance: hnswDistance(rawVecs) });
+      }
       atomicWrite(hnswPath, JSON.stringify(serializeHnsw(hnswIndex)));
       if (fs.existsSync(ivfPath)) {
         try {