md-semantic-search 0.1.1 → 0.3.0

package/README.md

@@ -36,7 +36,10 @@ measurements that shaped its defaults.
 - 🌍 **Cross-lingual.** Multilingual embeddings (default `multilingual-e5-base`).
 - 🧠 **Hybrid ranking.** Reciprocal Rank Fusion of vector similarity (meaning)
   and lexical overlap (exact names like `win32`, `TextIOWrapper`).
-- ⚡ **Incremental.** Per-file md5 — re-indexing only re-embeds changed files.
+- ⚡ **Two-level incremental.** Per-file md5 (unchanged files skip work) **plus**
+  per-chunk hash — inside a *changed* file, sections whose text is unchanged
+  reuse their stored vector. Editing one place in a long file re-embeds only the
+  affected section, not the whole file.
 - 🔒 **Private & offline.** Model downloads once, then no network. Nothing is
   uploaded anywhere.
 - 📦 **Zero infra.** One JSON index, brute-force cosine in memory. No Pinecone,
@@ -81,7 +84,14 @@ mdss index --db /path/to/your/markdown
 
 First run downloads the model (~280 MB). The index is written to `<db>/.mdss/`
 by default (override with `--index-dir`). Re-run after editing your notes — it's
-incremental, so only changed files are re-embedded.
+incremental at two levels: unchanged files are skipped via their md5, and inside
+a changed file only the sections whose text actually changed are re-embedded
+(chunks carry a `chunkHash` in `vectors.json`; an append to a long log re-embeds
+just the new entry). The index output reports the split, e.g.:
+
+```
+Indexed 64 file(s) → 725 chunks (725 reused [5 chunk-level, 720 file-level], 0 embedded), ... 0.6s
+```
 
 ### 2. Search
 
@@ -113,6 +123,7 @@ Top 3 for: "how do I add a new translation language"
 | `--k <n>` | Number of results (default 6). |
 | `--json` | Machine-readable output. |
 | `--semantic` | Pure vector ranking, skip lexical fusion. |
+| `--offline` | Never download the model — require a cached one (or `MDSS_OFFLINE=1`). |
 
 ### The base can live outside the project
 
@@ -147,7 +158,20 @@ mdss models
 | `bge-m3` | `Xenova/bge-m3` | 1024 | ~2.3 GB. Best cross-lingual separation in tests. |
 
 Switching models invalidates the stored vectors automatically — the next
-`index` run does a full rebuild. You can also pass any raw `Xenova/*` id.
+`index` run does a full rebuild.
+
+Weights are downloaded in quantized (q8) form when the model repo ships them
+(all `Xenova/*` repos do — e5-base is ~280 MB, not ~1.1 GB fp32).
+
+You can also pass any raw `Xenova/*` id, optionally pinning a revision:
+
+```bash
+mdss index --db ./docs --model "Xenova/multilingual-e5-small@abc123def"
+```
+
+Pinned ids invalidate the index too (the revision is part of the model key), so
+a `@revision` bump triggers a full rebuild. Custom ids should have quantized
+weights (a `model_quantized.onnx` file) in the repo.
 
 ## How it works
 
@@ -156,28 +180,16 @@ Switching models invalidates the stored vectors automatically — the next
 2. **Chunk** each file by Markdown headings; oversized sections split on blank
    lines (~1400 chars/chunk).
 3. **Embed** each chunk (`passage:` prefix for E5) → store `{file, heading,
-   text, vec}` in `vectors.json`, plus per-file md5 in `.hashes.json`.
+   text, vec}` in `vectors.json`, plus per-file md5 in `.hashes.json`. Each chunk
+   also stores `chunkHash` (SHA-256 of the exact passage input + model identity),
+   so on re-index an unchanged section inside a modified file reuses its vector
+   instead of being embedded again.
 4. **Search**: embed the query (`query:` prefix), score every chunk by cosine,
    score by lexical term-overlap, then **fuse with RRF**. Return top-k chunks.
 
 No external services, no database — the whole index is one JSON file and search
 is an in-memory dot-product sweep.
 
-## Releasing (maintainers)
-
-Publishing is automated by [`.github/workflows/publish.yml`](./.github/workflows/publish.yml):
-pushing a `vX.Y.Z` tag publishes that version to npm with provenance.
-
-One-time setup: add a repo secret `NPM_TOKEN` — an npm **automation / granular**
-token with *bypass 2FA* enabled (npmjs.com → Account → Access Tokens).
-
-```bash
-npm version patch     # bumps package.json + creates the git tag
-git push --follow-tags
-```
-
-The workflow checks that the tag matches `package.json` before publishing.
-
 ## License
 
 MIT © chelslava

package/bin/cli.mjs

@@ -32,6 +32,7 @@ function parseArgs(argv) {
     else if (a === '--ignore') opts.ignore.push(argv[++i]);
     else if (a === '--json') opts.json = true;
     else if (a === '--semantic') opts.semantic = true;
+    else if (a === '--offline') opts.offline = true;
     else if (a === '-h' || a === '--help') opts.help = true;
     else opts._.push(a);
   }
@@ -57,6 +58,9 @@ function resolveIndexDir(opts, db) {
 const resolveCache = (opts) =>
   path.resolve(opts.cacheDir || process.env.MDSS_CACHE_DIR || DEFAULT_CACHE);
 
+/** --offline flag or MDSS_OFFLINE=1 env → never touch the network. */
+const resolveOffline = (opts) => !!opts.offline || !!process.env.MDSS_OFFLINE;
+
 function die(msg) {
   process.stderr.write(`error: ${msg}\n`);
   process.exit(1);
@@ -78,6 +82,7 @@ Options:
   --k <n>             Number of results (search, default 6).
   --json              Machine-readable output (search).
   --semantic          Pure vector ranking, skip lexical/RRF fusion (search).
+  --offline           Never download the model; require a cached one (env MDSS_OFFLINE=1).
   -h, --help          Show this help.
 
 Examples:
@@ -96,12 +101,14 @@ async function cmdIndex(opts) {
     db, indexDir, cacheDir,
     modelName: opts.model || DEFAULT_MODEL,
     ignore: opts.ignore,
+    offline: resolveOffline(opts),
     log: s => process.stderr.write(s + '\n'),
   });
   const secs = ((Date.now() - t0) / 1000).toFixed(1);
   process.stderr.write(
     `\nIndexed ${r.files} file(s) → ${r.chunks} chunks ` +
-    `(${r.reused} reused, ${r.embedded} embedded), dim=${r.dim}, ` +
+    `(${r.reused} reused [${r.reusedChunks} chunk-level, ${r.reusedFiles} file-level], ` +
+    `${r.embedded} embedded), dim=${r.dim}, ` +
     `model=${r.model}, ${secs}s\n→ ${r.vectorsPath}\n`,
   );
 }
@@ -117,6 +124,7 @@ async function cmdSearch(opts) {
     indexDir, cacheDir, query,
     k: opts.k || 6,
     semanticOnly: !!opts.semantic,
+    offline: resolveOffline(opts),
   });
 
   if (opts.json) { process.stdout.write(JSON.stringify(results, null, 2) + '\n'); return; }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "md-semantic-search",
-  "version": "0.1.1",
+  "version": "0.3.0",
   "description": "Local, private semantic (vector) search over any folder of Markdown files. Finds passages by meaning — cross-lingual — using transformers.js. No API keys, no cloud, no vector database.",
   "type": "module",
   "bin": {
@@ -43,6 +43,6 @@
     "url": "https://github.com/chelslava/md-semantic-search/issues"
   },
   "dependencies": {
-    "@xenova/transformers": "^2.17.2"
+    "@huggingface/transformers": "^4.2.0"
   }
 }

package/src/indexer.mjs

@@ -1,6 +1,11 @@
 /**
  * Build / refresh the semantic index for a folder of markdown files.
- * Incremental: per-file md5; unchanged files reuse their stored embeddings.
+ * Incremental at TWO levels:
+ *   1. per-file md5  -> completely unchanged files reuse their chunks as-is;
+ *   2. per-chunk hash (chunkHash = SHA-256 of the exact passage input that
+ *      goes to embed()) -> inside a *changed* file, sections whose embedding
+ *      input is unchanged reuse their stored vector, so an append/edit in one
+ *      place of a long file no longer re-embeds all of its sections.
  */
 import fs from 'node:fs';
 import path from 'node:path';
@@ -8,10 +13,62 @@ import crypto from 'node:crypto';
 import { walkMarkdown, parseFile, embed, resolveModel } from './core.mjs';
 
 const md5 = s => crypto.createHash('md5').update(s).digest('hex');
-const loadJSON = (p, fb) => {
-  try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch { return fb; }
+const sha256 = s => crypto.createHash('sha256').update(s).digest('hex');
+
+/**
+ * Read JSON, falling back to `fb` when the file is missing or corrupt.
+ * A corrupt-but-present file is reported through `warn` (default: silent) so a
+ * torn write or manual edit doesn't silently wipe the previous index state.
+ */
+const loadJSON = (p, fb, warn = () => {}) => {
+  try { return JSON.parse(fs.readFileSync(p, 'utf8')); }
+  catch (e) {
+    if (e.code !== 'ENOENT') {
+      warn(`warning: ${path.basename(p)} is not valid JSON (${e.message}); ` +
+           'rebuilding it from scratch.');
+    }
+    return fb;
+  }
 };
 
+/**
+ * Atomic write: dump to a temp file in the SAME directory, then rename over the
+ * target. Rename is atomic on POSIX and on NTFS, so a crash mid-write can never
+ * leave a truncated vectors.json that the next run would parse as "corrupt".
+ */
+function atomicWrite(target, data) {
+  const tmp = `${target}.${process.pid}.tmp`;
+  fs.writeFileSync(tmp, data);
+  fs.renameSync(tmp, target);
+}
+
+/** Normalize text for stable hashing across runs: CRLF -> LF, trim edges. */
+function normalize(s) {
+  return (s ?? '').replace(/\r\n?/g, '\n').trim();
+}
+
+/**
+ * Stable per-chunk hash: SHA-256 over the exact passage input that is passed
+ * to embed() (`title\nheading\ntext`) plus the model identity and its passage
+ * prefix. Because the model participates in the key, switching models
+ * invalidates every chunk (full rebuild) - by design. CRLF/whitespace-only
+ * edits of a section do NOT change its hash, so its vector is reused.
+ * @param {{id:string, passagePrefix?:string}} model - resolved model descriptor
+ * @param {string} modelName - alias or id as passed on the CLI
+ * @param {{title:string, heading:string, text:string}} chunk
+ */
+export function chunkHash(model, modelName, chunk) {
+  const input = [
+    model.id,
+    modelName || '',
+    model.passagePrefix || '',
+    normalize(chunk.title),
+    normalize(chunk.heading),
+    normalize(chunk.text),
+  ].join('\u0000');
+  return sha256(input);
+}
+
 /**
  * @param {object} opts
  * @param {string} opts.db        - base dir of .md files (absolute)
@@ -19,10 +76,16 @@ const loadJSON = (p, fb) => {
  * @param {string} opts.cacheDir  - model cache dir
  * @param {string} opts.modelName - model alias or id
  * @param {string[]} opts.ignore  - glob patterns to skip
+ * @param {boolean} [opts.offline=false] - never download the model; require a cached one
  * @param {(s:string)=>void} [opts.log]
+ * @param {Function} [opts.embedFn] - embed override for tests/dependency injection;
+ *   signature (texts, kind, model, cacheDir, offline) => Promise<number[][]>
  */
 export async function buildIndex(opts) {
-  const { db, indexDir, cacheDir, modelName, ignore = [], log = () => {} } = opts;
+  const {
+    db, indexDir, cacheDir, modelName, ignore = [], log = () => {},
+    offline = false, embedFn = embed,
+  } = opts;
   const model = resolveModel(modelName);
   const vectorsPath = path.join(indexDir, 'vectors.json');
   const hashesPath = path.join(indexDir, '.hashes.json');
@@ -30,24 +93,31 @@ export async function buildIndex(opts) {
   fs.mkdirSync(indexDir, { recursive: true });
 
   const files = walkMarkdown(db, ignore);
-  const oldHashes = loadJSON(hashesPath, {});
-  const oldIndex = loadJSON(vectorsPath, { chunks: [], model: null });
+  const oldHashes = loadJSON(hashesPath, {}, log);
+  const oldIndex = loadJSON(vectorsPath, { chunks: [], model: null }, log);
 
   // If the model changed, all stored vectors are incompatible → full rebuild.
   const modelChanged = oldIndex.model && oldIndex.model !== model.id;
 
   const oldByFile = new Map();
+  // chunk-level cache: hash of the passage input -> stored vector. Built from
+  // the OLD index so changed files can reuse vectors of unchanged sections.
+  // Old chunks (from a pre-chunkHash index) get their hash computed the same
+  // way, so reuse works even on the first run after an upgrade.
+  const vecByChunkHash = new Map();
   if (!modelChanged) {
     for (const c of oldIndex.chunks) {
       if (!oldByFile.has(c.file)) oldByFile.set(c.file, []);
       oldByFile.get(c.file).push(c);
+      const key = c.chunkHash || chunkHash(model, modelName, c);
+      if (c.vec) vecByChunkHash.set(key, c.vec);
     }
   }
 
   const newHashes = {};
   const chunks = [];
   const toEmbed = [];
-  let reused = 0, changedFiles = 0;
+  let reused = 0, reusedChunks = 0, changedFiles = 0;
 
   for (const abs of files) {
     const raw = fs.readFileSync(abs, 'utf8');
@@ -57,13 +127,31 @@ export async function buildIndex(opts) {
 
     if (!modelChanged && oldHashes[rel] === h && oldByFile.has(rel)) {
       const old = oldByFile.get(rel);
+      // Backfill chunkHash for chunks that predate the chunk-level cache, so
+      // the persisted index is self-contained and no recompute is needed later.
+      for (const c of old) {
+        if (!c.chunkHash) c.chunkHash = chunkHash(model, modelName, c);
+      }
       chunks.push(...old);
       reused += old.length;
       continue;
     }
     changedFiles++;
     const parsed = parseFile(abs, db);
-    for (const c of parsed) { chunks.push(c); toEmbed.push(c); }
+    for (const c of parsed) {
+      const key = chunkHash(model, modelName, c);
+      const cached = vecByChunkHash.get(key);
+      if (cached) {
+        c.vec = cached;
+        c.chunkHash = key;
+        reused++;
+        reusedChunks++;
+      } else {
+        c.chunkHash = key;
+        toEmbed.push(c);
+      }
+      chunks.push(c);
+    }
   }
 
   if (toEmbed.length > 0) {
@@ -72,9 +160,9 @@ export async function buildIndex(opts) {
     const BATCH = 32;
     for (let i = 0; i < toEmbed.length; i += BATCH) {
       const slice = toEmbed.slice(i, i + BATCH);
-      const vecs = await embed(
+      const vecs = await embedFn(
         slice.map(c => `${c.title}\n${c.heading}\n${c.text}`),
-        'passage', model, cacheDir,
+        'passage', model, cacheDir, offline,
       );
       slice.forEach((c, j) => { c.vec = vecs[j]; });
       log(`  ${Math.min(i + BATCH, toEmbed.length)}/${toEmbed.length}`);
@@ -92,13 +180,15 @@ export async function buildIndex(opts) {
     chunks,
   };
 
-  fs.writeFileSync(vectorsPath, JSON.stringify(index));
-  fs.writeFileSync(hashesPath, JSON.stringify(newHashes, null, 2));
+  atomicWrite(vectorsPath, JSON.stringify(index));
+  atomicWrite(hashesPath, JSON.stringify(newHashes, null, 2));
 
   return {
     files: files.length,
     chunks: chunks.length,
     reused,
+    reusedChunks,
+    reusedFiles: reused - reusedChunks, // via the file-level fast path
     embedded: toEmbed.length,
     dim,
     model: model.id,

package/src/models.mjs

@@ -39,14 +39,26 @@ export const MODELS = {
 
 export const DEFAULT_MODEL = 'e5-base';
 
-/** Resolve a model alias OR a raw Xenova/... id into a model descriptor. */
+/**
+ * Resolve a model alias OR a raw HF id into a model descriptor.
+ * A raw id may pin a revision with the `id@revision` syntax (defaults to
+ * `main`), e.g. `--model "Xenova/multilingual-e5-small@a1b2c3d"`.
+ */
 export function resolveModel(name) {
   if (!name) return MODELS[DEFAULT_MODEL];
   if (MODELS[name]) return MODELS[name];
   // Allow passing a raw HF/Xenova id; assume E5-style prefixes unless it's bge.
   const isBge = /bge/i.test(name);
+  let id = name;
+  let revision = 'main';
+  const at = name.indexOf('@');
+  if (at > 0) {
+    id = name.slice(0, at);
+    revision = name.slice(at + 1) || 'main';
+  }
   return {
-    id: name,
+    id,
+    revision,
     dim: 0,
     queryPrefix: isBge ? '' : 'query: ',
     passagePrefix: isBge ? '' : 'passage: ',

package/src/search.mjs

@@ -16,12 +16,12 @@ const STOP = new Set([
   'под', 'так', 'его', 'нет', 'код', 'кода', 'есть',
 ]);
 
-function tokenize(text) {
+export function tokenize(text) {
   const m = text.toLowerCase().match(/[\p{L}\p{N}]{3,}/gu);
   return m ? m.filter(t => !STOP.has(t)) : [];
 }
 
-function keywordScores(chunks, query) {
+export function keywordScores(chunks, query) {
   const qTerms = new Set(tokenize(query));
   return chunks.map(c => {
     const hay = `${c.title} ${c.heading} ${c.text}`.toLowerCase();
@@ -32,7 +32,7 @@ function keywordScores(chunks, query) {
 }
 
 /** Reciprocal Rank Fusion. rankings: arrays of {idx, score}; higher = better. */
-function rrf(rankings, k = 60) {
+export function rrf(rankings, k = 60) {
   const fused = new Map();
   for (const ranking of rankings) {
     const sorted = [...ranking].sort((a, b) => b.score - a.score);
@@ -51,18 +51,31 @@ function rrf(rankings, k = 60) {
  * @param {string} opts.query
  * @param {number} [opts.k=6]
  * @param {boolean} [opts.semanticOnly=false]
+ * @param {boolean} [opts.offline=false] - never download the model; require a cached one
+ * @param {Function} [opts.embedFn] - embed override for tests/dependency injection;
+ *   signature (texts, kind, model, cacheDir, offline) => Promise<number[][]>
  * @returns {Promise<Array>} results with file, title, heading, cosine, score, snippet
  */
 export async function search(opts) {
-  const { indexDir, cacheDir, query, k = 6, semanticOnly = false } = opts;
+  const {
+    indexDir, cacheDir, query, k = 6, semanticOnly = false,
+    offline = false, embedFn = embed,
+  } = opts;
   const vectorsPath = path.join(indexDir, 'vectors.json');
   if (!fs.existsSync(vectorsPath)) {
     throw new Error(`No index at ${vectorsPath}. Run \`mdss index\` first.`);
   }
   const index = JSON.parse(fs.readFileSync(vectorsPath, 'utf8'));
+
+  // Validate the model the index was built with. Indexes written by v0.1.x have
+  // no "model" field at all — warn instead of silently assuming a default.
+  if (!index.model && !index.modelAlias) {
+    process.stderr.write('warning: index has no "model" field (built by an old ' +
+      'version); assuming the default model. Re-run `mdss index` to refresh.\n');
+  }
   const model = resolveModel(index.modelAlias || index.model);
 
-  const [qVec] = await embed([query], 'query', model, cacheDir);
+  const [qVec] = await embedFn([query], 'query', model, cacheDir, offline);
 
   const semantic = index.chunks.map((c, idx) => ({ idx, score: cosine(qVec, c.vec) }));
   const cosByIdx = new Map(semantic.map(s => [s.idx, s.score]));