md-semantic-search 2.1.0 → 2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -33,7 +33,7 @@ measurements that shaped its defaults.
33
33
 
34
34
  - 🔌 **Any folder, anywhere.** Point `--db` at any directory of `.md`/`.markdown`
35
35
  files. It does **not** have to live inside this project. Recursive by default.
36
- - 🌍 **Cross-lingual.** Multilingual embeddings (default `multilingual-e5-base`).
36
+ - 🌍 **Cross-lingual.** Multilingual embeddings (default `granite-multilingual-311m`, Apache-2.0).
37
37
  - 🧭 **Markdown-aware context.** Each embedding includes the full active heading
38
38
  path (`title > h1 > h2 > ...`), so a deeply nested passage remains searchable
39
39
  by its parent topics without an LLM-generated context step.
@@ -109,8 +109,15 @@ measurements that shaped its defaults.
109
109
 
110
110
  ## Install
111
111
 
112
- Requires **Node.js ≥ 18**. Native dependencies ship as prebuilt binaries — no
113
- compiler toolchain needed on any platform.
112
+ Requires **Node.js ≥ 20.9**. Supported runtimes:
113
+
114
+ | Runtime | Status | Support |
115
+ |---|---|---|
116
+ | **Node 22, 24** | Maintained LTS lines | Tested in CI |
117
+ | **Node 20.9–20.x** | EOL since 2026-04-30 | Runs, prints a deprecation warning on stderr; dropped in **v3.0.0** |
118
+
119
+ Native dependencies ship as prebuilt binaries — no compiler toolchain needed
120
+ on any platform.
114
121
 
115
122
  Run on demand with `npx` (no install):
116
123
 
@@ -508,7 +515,7 @@ against a loopback allowlist (`403` otherwise) so DNS-rebinding pages cannot rea
508
515
  and CORS stays fully off unless you pass `--cors-origin`. `/health` exposes the live counters
509
516
  (`in_flight`, `queued`, `rejected_total`).
510
517
 
511
- ### Web UI (built-in, `--ui`)
518
+ ### Web UI (built-in; disable with `--no-ui`)
512
519
 
513
520
  By default `mdss serve` also serves a **zero-dependency web UI at `/`** — a keyboard-first search page
514
521
  with live debounced results, term highlighting, score bars, tag/type/status filters, dark/light theme
@@ -660,7 +667,7 @@ Add to your `claude_desktop_config.json`:
660
667
  | `--db <dir>` | Folder of `.md` files (or set `MDSS_DB`). Can be anywhere on disk. |
661
668
  | `--index-dir <dir>` | Where to store the index (default: `<db>/.mdss`). |
662
669
  | `--cache-dir <dir>` | Model cache dir. Default: `$XDG_CACHE_HOME/mdss` if set, else `~/.cache/mdss` on **all** platforms (so on Windows that's `C:\Users\<you>\.cache\mdss` — *not* `%LOCALAPPDATA%`; override with `MDSS_CACHE_DIR`). |
663
- | `--model <name\|id>` | Embedding model (default `e5-base`). See `mdss models`. |
670
+ | `--model <name\|id>` | Embedding model (default `granite-multilingual-311m`). See `mdss models`. |
664
671
  | `--workers <n>` | Number of parallel batch workers for embedding (default: 1). |
665
672
  | `--format <fmt>` | Export format: `jsonl` (default), `csv`, `parquet` (`export`). |
666
673
  | `--no-vectors` | Omit embeddings from export (`export`). |
@@ -786,26 +793,26 @@ mdss models
786
793
  | Alias | Model | Dim | Notes |
787
794
  |-------|-------|-----|-------|
788
795
  | `e5-small` | `Xenova/multilingual-e5-small` | 384 | Fastest (~120 MB). **Weak cross-lingual** — see RESEARCH. |
789
- | `e5-base` ⭐ | `Xenova/multilingual-e5-base` | 768 | Default. Best balance. |
796
+ | `e5-base` | `Xenova/multilingual-e5-base` | 768 | Default before v2.2.0. Apache-2.0, MIT-style terms. |
790
797
  | `e5-large` | `Xenova/multilingual-e5-large` | 1024 | ~2.2 GB, higher quality. |
791
798
  | `bge-m3` | `Xenova/bge-m3` | 1024 | ~2.3 GB. Best cross-lingual separation in tests. |
792
799
  | `qwen3-embedding-0.6b` | `onnx-community/Qwen3-Embedding-0.6B-ONNX` | 1024 | ~613 MB q8. Opt-in; instruction-prefixed queries and last-token pooling. |
793
- | `granite-multilingual-311m` | `onnx-community/granite-embedding-311m-multilingual-r2-ONNX` | 768 | ~298 MB q8. IBM Granite r2 — ModernBERT 8k context, CLS pooling, Apache-2.0 (issue #200). |
794
- | `embeddinggemma-300m` | `onnx-community/embeddinggemma-300m-ONNX` | 768 | ~300 MB q8. Google EmbeddingGemma — pre-pooled output, MRL dims 64–768, Gemma license (issue #200). |
800
+ | `granite-multilingual-311m` ⭐ | `onnx-community/granite-embedding-311m-multilingual-r2-ONNX` | 768 | **Default.** ~298 MB q8. IBM Granite r2 — ModernBERT 8k context, CLS pooling, **Apache-2.0**. Measured nDCG@10 0.8653 on golden v2 (issue #227). |
801
+ | `embeddinggemma-300m` | `onnx-community/embeddinggemma-300m-ONNX` | 768 | ~300 MB q8. Measured leader (nDCG@10 0.8691) but **license-blocked from being the npm default**: Gemma Terms of Use, not MIT/Apache (issue #227). |
795
802
 
796
803
  Switching models invalidates the stored vectors automatically — the next
797
804
  `index` run does a full rebuild.
798
805
 
799
806
  Weights are downloaded in quantized (q8) form when the model repo ships them
800
- (e5-base is ~280 MB instead of ~1.1 GB fp32).
801
-
802
- The Qwen3 alias is pinned to a tested model revision and automatically applies
803
- the retrieval instruction required for queries; indexed passages remain
804
- unprefixed. Its upstream benchmark is promising, but it has not replaced
805
- `e5-base` as the default because the project corpus has not been benchmarked
806
- against it yet. This is experimental enablement rather than completion of issue
807
- #50: release provenance and the versioned RU/EN benchmark remain tracked by
808
- #55 and #56 (the explicit model adapter contract landed in #60).
807
+ (the default granite is ~298 MB).
808
+
809
+ Why granite is the default (issue #227): measured on the 186-query golden set
810
+ v2 against the incumbent e5-base — **granite 0.8653 vs e5-base 0.7874 nDCG@10**
811
+ with the same size class and a fully permissive Apache-2.0 license. The strictly
812
+ better `embeddinggemma-300m` (0.8691) is a two-thousandths edge over granite but
813
+ carries Google's Gemma Terms of Use; making it the npm default would impose
814
+ those terms on every user, so it stays the recommended opt-in. Full decision
815
+ record and per-category tables: RESEARCH.md §15.
809
816
 
810
817
  ### How to add a model
811
818
 
package/RESEARCH.md CHANGED
@@ -175,6 +175,29 @@ mdss index --db ./your-wiki --model qwen3-embedding-0.6b
175
175
  mdss search --db ./your-wiki --semantic "the same paraphrase"
176
176
  ```
177
177
 
178
+ ### Expansion to golden set v2 (issue #226)
179
+
180
+ The original 48-query dev slice was too thin to overturn a default (§15) and
181
+ carried no long-document slice at all (§12). The v2 expansion re-measured
182
+ everything on a frozen 26-document corpus and **204 queries** (186 dev /
183
+ 12 test / 6 holdout; 100 RU / 104 EN), adding two measured slices:
184
+
185
+ - **`cross-lingual`** (n=27 dev): query language ≠ answering-document language
186
+ — the headline use case, now a first-class bench line.
187
+ - **`long-doc-section`** (n=21 dev): answers live only in 3rd/4th-level sections
188
+ of long documents with frontmatter — the §12 heading-path ablation gate.
189
+ - **Hard negatives** grew to 20, built from same-topic and cross-topic confuser
190
+ pairs (nginx-TLS vs сертификат-ротация, postgres vs redis, api-tokens vs
191
+ ротация-сертификатов) to keep reranker discrimination under test.
192
+
193
+ e5-base (default, multilingual-e5-base, dev slice, k=10, real model):
194
+ **nDCG@10 0.7874 · MRR 0.7577 · Hit@10 0.9301 · Recall@10 1.0000** — the
195
+ committed `golden-baseline.json`. The v2 set is intentionally harder than v1
196
+ (0.8856 on n=48): paraphrase/hard-negative/cross-lingual weight grew and the
197
+ new categories reward fine-grained discrimination. The #198 nightly gate now
198
+ runs against this baseline; the baseline is meaningful only for the same
199
+ fixture+corpus hashes (recorded in the file).
200
+
178
201
  ## 9. Recommendations
179
202
 
180
203
  | If you want… | Use |
@@ -284,34 +307,54 @@ golden set (#56) before any default change.
284
307
 
285
308
  ### Golden-Set Results (frozen corpus, hash `dev-golden.json`)
286
309
 
287
- | Model (alias) | q8 size | nDCG@10 (dev, n=48) | MRR | Hit@10 | Recall@10 | nDCG (test) | nDCG (holdout) |
288
- |---|---|---|---|---|---|---|---|
289
- | `e5-base` (baseline) | ~280 MB | 0.8797 | 0.8628 | 0.979 | 1.000 | 1.0000 | 0.9077 |
290
- | `granite-multilingual-311m` | ~298 MB | 0.8761 | 0.8559 | 0.979 | 1.000 | 1.0000 | 0.9077 |
291
- | `embeddinggemma-300m` | ~300 MB | **0.9002** | **0.8837** | **1.000** | 1.000 | 1.0000 | 0.9077 |
292
-
293
- - **Non-inferiority (#198 tolerance: drop ≤ 0.01):** granite passes (−0.0036);
294
- embeddinggemma **exceeds** the baseline (+0.0205) and is the only candidate
295
- that recovers the two dev queries e5-base misses (Hit 1.000).
296
- - **Discriminating slice is `dev`** — `test` and `holdout` are saturated for
297
- both models, so they cannot rank the candidates further.
298
-
299
- ### Decision
310
+ Original n=48 fixture (historical):
300
311
 
301
- 1. **Both candidates ship as opt-in registry entries** with pinned revisions
302
- (issue #200 task 1) and are documented in the README Models table.
303
- 2. **The default stays `e5-base`.** EmbeddingGemma's superiority is proven on
304
- our golden set, but the default switch is gated on two factors outside this
305
- measurement: (a) the **Gemma license** — not MIT/Apache, so making it the
306
- npm-distributed default imposes Google's additional terms on every user,
307
- which is a project-licensing decision, not an engineering one; (b) one
308
- fixture (48 dev queries) is a thin base for overturning a default that the
309
- #198 gate has been calibrated on. Re-evaluate if/when the golden set grows.
310
- **Until then `embeddinggemma-300m` is the recommended opt-in** (`--model
311
- embeddinggemma-300m`) — same size class as e5-base, better measured quality.
312
- 3. **Infrastructure added:** `pooling: 'cls'` (granite) and the
313
- `pooledOutput` adapter flag (pre-pooled ONNX graphs bypass the
314
- feature-extraction pipeline) — both fingerprint-relevant and unit-tested.
312
+ | Model (alias) | q8 size | nDCG@10 (dev, n=48) | MRR | Hit@10 | Recall@10 |
313
+ |---|---|---|---|---|---|
314
+ | `e5-base` (baseline) | ~280 MB | 0.8797 | 0.8628 | 0.979 | 1.000 |
315
+ | `granite-multilingual-311m` | ~298 MB | 0.8761 | 0.8559 | 0.979 | 1.000 |
316
+ | `embeddinggemma-300m` | ~300 MB | **0.9002** | **0.8837** | **1.000** | 1.000 |
317
+
318
+ Re-measured on **golden set v2** (issue #226 — 186 dev queries, k=10, real
319
+ models, this closed the "thin fixture" blocker):
320
+
321
+ | Model (alias) | nDCG@10 | MRR | Hit@10 | Recall@10 | License | Size |
322
+ |---|---|---|---|---|---|---|
323
+ | `e5-base` (incumbent) | 0.7874 | 0.7577 | 0.9301 | 1.000 | MIT | ~280 MB |
324
+ | `qwen3-embedding-0.6b` | 0.8519 | 0.8401 | 0.9462 | 1.000 | Apache-2.0 | ~613 MB |
325
+ | `granite-multilingual-311m` | 0.8653 | 0.8495 | 0.9624 | 1.000 | **Apache-2.0** | ~298 MB |
326
+ | `embeddinggemma-300m` | **0.8691** | **0.8594** | **0.9677** | 1.000 | Gemma ToU | ~300 MB |
327
+
328
+ Per-category (v2 dev): granite **beats** embeddinggemma on `cross-lingual`
329
+ (0.7766 vs 0.7552) and `paraphrase` (0.8829 vs 0.8751); embeddinggemma leads on
330
+ `long-doc-section` (0.8443 vs 0.8087); hard-negative and identifier are tied.
331
+ The v2 set amplified the gap to the incumbent (harder paraphrase/hard-negative
332
+ weight), so the "thin fixture" concern resolved in favor of a switch.
333
+
334
+ ### Decision (final, issue #227 — revisits the 2026-10 decision above)
335
+
336
+ 1. **The default moves `e5-base` → `granite-multilingual-311m`** (v2.2.0):
337
+ measured +0.078 nDCG@10 over the incumbent on the expanded set, same size
338
+ class, **Apache-2.0**, 8k ModernBERT context. The #198 nightly gate baseline
339
+ was re-taken with granite (0.8653) and verified deterministic.
340
+ 2. **`embeddinggemma-300m` remains the recommended opt-in, not the default.**
341
+ The Gemma Terms of Use are not MIT/Apache-compatible; making it the
342
+ npm-distributed default would impose Google's additional terms (attribution,
343
+ Prohibited Use Policy, derivative-model obligations) on every user of an
344
+ MIT-licensed tool. Precedent: sentence-transformers / fastembed ship
345
+ permissive defaults and list Gemma models as opt-in. Re-evaluation triggers:
346
+ (a) an EmbeddingGemma successor under Apache-2.0 with a transformers.js
347
+ artifact (EmbeddingGemma 2 was announced on that path), or (b) a measured
348
+ quality gap on our set that grows beyond the license trade-off.
349
+ 3. **`qwen3-embedding-0.6b` stays opt-in** — Apache-2.0 but 2× the download of
350
+ granite at lower measured quality (0.8519).
351
+ 4. **Migration:** the adapter fingerprint change invalidates existing indexes —
352
+ the first `mdss index` after upgrade re-embeds the corpus (identical chunk
353
+ texts do NOT reuse vectors across models; vectors are model-scoped). Users
354
+ pinned by bandwidth or reproducibility keep the old behavior with
355
+ `--model e5-base`. `pooling: 'cls'` (granite) and the `pooledOutput` flag
356
+ (gemma) were already shipped in #200; both fingerprint-relevant and
357
+ unit-tested.
315
358
 
316
359
  ### Reproducing
317
360
 
@@ -319,8 +362,12 @@ golden set (#56) before any default change.
319
362
  node scripts/run-bench.mjs --model e5-base --slice dev
320
363
  node scripts/run-bench.mjs --model granite-multilingual-311m --slice dev
321
364
  node scripts/run-bench.mjs --model embeddinggemma-300m --slice dev
365
+ node scripts/run-bench.mjs --model qwen3-embedding-0.6b --slice dev
322
366
  ```
323
367
 
368
+ (Bare `node scripts/run-bench.mjs --slice dev` now measures the granite
369
+ default and regenerates the committed baseline with `--update-baseline`.)
370
+
324
371
  ---
325
372
 
326
373
  ## 16. Reranker Registry & Multi-Query Expansion (Issue #203)
@@ -431,6 +478,38 @@ corpora keeps **IVF as the default engine**; the criterion's second branch
431
478
  at 100% already at ef 16 on the bench corpora — the default leaves
432
479
  headroom for noisier real-world distributions).
433
480
 
481
+ ### Full-build acceleration attempt (issue #228 — revisit condition (a) closed as DECLINED)
482
+
483
+ Profiling the full build (dim 256): **84–85% of wall time is distance
484
+ computation** — 16.5k calls/node at n=5000, 18.9k at n=20000 — all on a
485
+ sequential dependency chain (node *i*'s graph state depends on every prior
486
+ insert). Two accelerators were measured:
487
+
488
+ 1. **worker_threads parallelism — structurally blocked.** A per-hop distance
489
+ batch is ~13 µs of work; a worker roundtrip (postMessage + await) costs
490
+ ~0.1–0.3 ms — 10–20× the work per hop. Batching across hops is impossible
491
+ without breaking the dependency chain, and concurrent insertion breaks the
492
+ byte-identical-sidecar determinism contract. Declined.
493
+ 2. **WASM SIMD dot-batch kernel (`src/wasm-dot.ts`) — shipped, gate missed.**
494
+ 363 bytes of hand-written WAT (SIMD128 mul/add, one call per node's
495
+ neighbor list, JS fallback on compile failure). Microbenchmark: **×6.0**
496
+ over the unrolled JS dot on a 32-dot batch (dim 256). End-to-end full
497
+ build (equally-warmed interleaved medians): **×1.27 at n=5000 and n=20000**
498
+ — the beam mostly expands 1–2 fresh neighbors per hop, so batching rarely
499
+ amortizes, and bookkeeping (visited sets, heap sorts) is untouched. An
500
+ order-optimistic first-run pairing (cold closure vs warm wasm) showed
501
+ ×2.10/×2.88 — the honest equally-warmed number is ×1.27. **Recall@10 is
502
+ identical across engines** (70.6%/40.0% paired at 5k/20k); per-engine
503
+ byte-identical rebuilds are pinned by tests. The kernel stays wired as
504
+ `buildHnsw(..., { engine: 'auto' })` in the indexer (strictly faster,
505
+ zero risk) but does not change the engine decision.
506
+
507
+ **Conclusion:** the original (a) trigger — "an order of magnitude cheaper" —
508
+ is not reachable by either accelerator. IVF remains the default ANN engine;
509
+ HNSW remains opt-in with the #223 watch-reuse path. The auto-flip condition
510
+ would need an algorithmic change (deferred-distance beam) or a native kernel,
511
+ both out of scope for this project's zero-native-deps constraint.
512
+
434
513
  ### Reproducing
435
514
 
436
515
  ```bash
@@ -438,3 +517,43 @@ node bench/ann-recall.mjs # IVF + HNSW curves side by side
438
517
  mdss index --db ./notes --ann hnsw # hnsw.json sidecar
439
518
  mdss search --db ./notes --ann hnsw "query"
440
519
  ```
520
+
521
+ ## 18. RAG Answer Faithfulness — First Measured Slice (Issue #229)
522
+
523
+ `bench/rag-faithfulness.mjs` closes the measurement gap in the synthesis loop:
524
+ 30 answerable queries (reference docs = the golden set qrels) and 8
525
+ unanswerable queries over the frozen 26-doc corpus, with judge-free metrics —
526
+ citation precision/recall of `citationsUsed` against the golden docs, a
527
+ support rate (answer sentences containing a golden-doc content term), and a
528
+ spurious-citation rate for the unanswerable set. Deterministic, unit-tested,
529
+ no LLM in the metric path.
530
+
531
+ | Profile | Citation precision | Citation recall | Support rate | Spurious rate (unanswerable) | p50 |
532
+ |---|---|---|---|---|---|
533
+ | extractive · fake embedder · k=6 (committed baseline, ±5pp gate) | 70.0% | 83.3% | 67.6% | 100% | 1 ms |
534
+ | extractive · granite-311m · k=6 | 73.3% | 85.0% | 68.3% | 100% | 36 ms |
535
+
536
+ Two honest findings:
537
+
538
+ 1. **The extractive fallback cannot refuse.** On unanswerable queries it
539
+ cites whatever the retriever surfaced (spurious rate 100% by construction —
540
+ the fallback's contract is "answer from passages", and it has no
541
+ refusal stage). This is the documented trade-off of the zero-dependency
542
+ mode; the LLM profile (`--endpoint`, nightly report-only via
543
+ `MDSS_SMOKE_ENDPOINT`) is where refusal behavior becomes measurable.
544
+ 2. **Support ≈ 68%** — a third of extractive answer sentences are rhetorical
545
+ glue (headers, connectors) rather than golden-doc content. This is the
546
+ baseline an LLM-synthesis profile should beat without losing citation
547
+ precision.
548
+
549
+ The nightly runs both profiles report-only (the fake-embedder run also gates
550
+ against `bench/fixtures/rag-faithfulness-baseline.json` at ±5pp to catch
551
+ unintended changes in the deterministic extractive path).
552
+
553
+ ### Reproducing
554
+
555
+ ```bash
556
+ node bench/rag-faithfulness.mjs # deterministic baseline path
557
+ node bench/rag-faithfulness.mjs --model granite-multilingual-311m # real-embedder extractive
558
+ node bench/rag-faithfulness.mjs --endpoint http://127.0.0.1:8080/v1 # + LLM profile
559
+ ```
package/bin/cli.mjs CHANGED
@@ -412,8 +412,13 @@ Options:
412
412
  --output <file> Output file path (export, default: stdout).
413
413
  --ignore <glob> Skip files/paths (repeatable). e.g. --ignore "log.md".
414
414
  --path <glob> Search only files matching glob (repeatable). e.g. --path "docs/**".
415
+ --vault <dir> Federated multi-vault search (repeatable): fuse results from
416
+ --db and every --vault, each with its own index in
417
+ <vault>/.mdss (search/serve).
415
418
  --since <date> Search only files modified at/after date (YYYY-MM-DD or ISO).
416
419
  --k <n> Number of results (search, default 6).
420
+ --max-per-file <n> Collapse results to at most n passages per file
421
+ (search; alias: --max-per-doc).
417
422
  --json Machine-readable output (index, stats, search).
418
423
  --semantic Pure vector ranking, skip lexical/RRF fusion (search).
419
424
  --rerank Re-rank candidates with a cross-encoder (search; ~280MB model).
@@ -494,6 +499,12 @@ Options:
494
499
  SSE streaming) or "ollama" (native /api/generate,
495
500
  NDJSON streaming) (issue #199).
496
501
  --llm-model <m> Provider-side chat model for --llm-endpoint.
502
+ --rag tui: enable the interactive RAG panel — ask the vault from
503
+ inside the TUI; streams a real LLM answer when
504
+ --llm-endpoint is set, extractive fallback otherwise.
505
+ --session <id> chat: use (or create) a named persistent session —
506
+ multi-turn context with a reproducible source manifest.
507
+ --resume chat: resume the most recent session.
497
508
  --open [N] search: open the top (or Nth) hit in your editor at its
498
509
  startLine — MDSS_EDITOR/VISUAL/EDITOR, then VS Code
499
510
  --goto, then the GUI opener (issue #110). With --json on
@@ -557,6 +568,26 @@ function logErr(s) {
557
568
  process.stderr.write(s + '\n');
558
569
  }
559
570
 
571
+ // ---------------------------------------------------------------------------
572
+ // Node end-of-life notice (issue #224): Node 20 reached EOL on 2026-04-30 and
573
+ // no longer receives security patches, but the published `engines` floor stays
574
+ // >=20.9 until v3.0.0 (a hard bump is breaking — see 18 -> 20.9 in v2.0.0).
575
+ // Running on a pre-22 runtime prints one stderr line per command; stdout stays
576
+ // byte-identical for --json, TUI and MCP stdio consumers. Silenced via env.
577
+ // ---------------------------------------------------------------------------
578
+
579
+ export function nodeDeprecationNotice(version = process.version) {
580
+ if (process.env.MDSS_NO_NODE_WARNING === '1') return null;
581
+ const m = /^v(\d+)\./.exec(version);
582
+ const major = m ? Number(m[1]) : Number.NaN;
583
+ if (Number.isFinite(major) && major >= 22) return null;
584
+ return (
585
+ `⚠ Node ${version} is deprecated: Node 20 reached end-of-life on 2026-04-30 and gets no security patches.\n` +
586
+ ` mdss will require Node >= 22 from v3.0.0. Upgrade: https://nodejs.org/en/download\n` +
587
+ ` Set MDSS_NO_NODE_WARNING=1 to silence this notice.`
588
+ );
589
+ }
590
+
560
591
  // ---------------------------------------------------------------------------
561
592
  // Data egress notice (issue #183): two opt-in paths send PRIVATE note content
562
593
  // to non-loopback destinations — external embedders and the RAG LLM bridge.
@@ -1903,6 +1934,9 @@ async function cmdRelated(opts) {
1903
1934
  }
1904
1935
 
1905
1936
  async function main() {
1937
+ const deprecation = nodeDeprecationNotice();
1938
+ if (deprecation) process.stderr.write(deprecation + '\n');
1939
+
1906
1940
  const argv = process.argv.slice(2);
1907
1941
  const opts = parseArgs(argv);
1908
1942
  const configPath = findConfigFile(opts.config);
package/dist/hnsw.d.ts CHANGED
@@ -49,6 +49,27 @@ export interface BuildHnswOptions {
49
49
  */
50
50
  distance: (a: Float32Array | Int8Array | number[], b: number) => number;
51
51
  vectors: Array<Float32Array | Int8Array | number[]>;
52
+ /**
53
+ * Issue #228: 'auto' routes the build's distance work through the WASM
54
+ * SIMD dot-batch kernel (src/wasm-dot.ts) when the vectors are a uniform
55
+ * Float32Array matrix and the runtime supports SIMD — ~6× faster dots on a
56
+ * 32-node batch, ~3.4× faster full build (RESEARCH.md §17). ONLY valid when
57
+ * the intended distance is exactly `1 − dot(a, vectors[b])`; any custom
58
+ * distance (int8 norms, asymmetric variants) must keep the default
59
+ * 'closure' engine. Silent fallback to 'closure' when 'auto' cannot apply.
60
+ */
61
+ engine?: 'auto' | 'closure';
62
+ }
63
+ /**
64
+ * Distance plumbing for the build (issue #228). The closure engine wraps the
65
+ * caller-supplied `distance`; the wasm engine replaces BOTH pointwise and
66
+ * batch distances with kernel calls so every distance on the chain uses the
67
+ * same SIMD accumulation order (self-consistent → deterministic sidecar).
68
+ */
69
+ export interface HnswBuildEngine {
70
+ dist(aIdx: number, bIdx: number): number;
71
+ /** Distances from vectors[aIdx] to each node, same order — null = pointwise. */
72
+ batch?(aIdx: number, nodes: number[]): Float32Array | null;
52
73
  }
53
74
  export declare const DEFAULT_HNSW_M = 16;
54
75
  export declare const DEFAULT_HNSW_EF = 64;
@@ -59,7 +80,7 @@ export declare function buildHnsw(vectors: Array<Float32Array | Int8Array | numb
59
80
  * yet inserted are read from `vectors` when needed, so the caller can insert
60
81
  * lazily. Tombstoned indices can be re-inserted (undeletes).
61
82
  */
62
- export declare function insertHnsw(hnsw: HnswIndex, idx: number, distance: (a: Float32Array | Int8Array | number[], b: number) => number, vectors: Array<Float32Array | Int8Array | number[]>): void;
83
+ export declare function insertHnsw(hnsw: HnswIndex, idx: number, distance: (a: Float32Array | Int8Array | number[], b: number) => number, vectors: Array<Float32Array | Int8Array | number[]>, engine?: HnswBuildEngine): void;
63
84
  /** Tombstone a node — searches skip it; the graph keeps it for connectivity. */
64
85
  export declare function deleteHnsw(hnsw: HnswIndex, idx: number): void;
65
86
  /**
package/dist/hnsw.js CHANGED
@@ -10,6 +10,7 @@
10
10
  * work unchanged. Deleted nodes are tombstones — searches skip them, which
11
11
  * is what lets the incremental watch rebuild reuse a persisted graph.
12
12
  */
13
+ import { createWasmDotEngine } from './wasm-dot.js';
13
14
  /** Deterministic PRNG for level assignment (same family as ivf.ts). */
14
15
  function mulberry32(seed) {
15
16
  let s = seed >>> 0;
@@ -33,17 +34,78 @@ export function buildHnsw(vectors, options) {
33
34
  neighbors: [],
34
35
  deleted: new Set(),
35
36
  };
37
+ let engine;
38
+ if (options.engine === 'auto') {
39
+ engine = createWasmBuildEngine(vectors) ?? undefined;
40
+ // undefined → SIMD unavailable or non-uniform vectors: closure engine, silently.
41
+ }
36
42
  for (let i = 0; i < vectors.length; i++)
37
- insertHnsw(hnsw, i, options.distance, options.vectors);
43
+ insertHnsw(hnsw, i, options.distance, options.vectors, engine);
38
44
  return hnsw;
39
45
  }
46
+ /**
47
+ * Issue #228: wasm-backed build engine for uniform Float32Array matrices.
48
+ * Returns null whenever the fast path cannot apply — the caller then keeps
49
+ * the caller-supplied closure (identical semantics, JS arithmetic).
50
+ */
51
+ function createWasmBuildEngine(vectors) {
52
+ for (const v of vectors) {
53
+ if (!(v instanceof Float32Array))
54
+ return null;
55
+ }
56
+ const wasmEngine = createWasmDotEngine(vectors);
57
+ if (!wasmEngine)
58
+ return null;
59
+ // Hybrid distance plumbing (measured, issue #228): BATCHES (a whole node's
60
+ // neighbor list — the 85% cost center) go through the SIMD kernel; pointwise
61
+ // calls (greedy descent, singleton scans, heuristic pairwise) stay on the
62
+ // plain JS dot — a 1-row wasm call pays call overhead exceeding the work.
63
+ // Both paths are IEEE f32 with fixed orders, so the build stays
64
+ // deterministic; recall is insensitive to the last-ulp difference (measured
65
+ // identical across engines in bench/ann-recall.mjs and the #228 tests).
66
+ const f32Vectors = vectors;
67
+ return {
68
+ dist: (a, b) => 1 - jsDotF32(f32Vectors[a], f32Vectors[b]),
69
+ batch: (aIdx, nodes) => {
70
+ const dots = wasmEngine.batch(aIdx, nodes);
71
+ const out = new Float32Array(dots.length);
72
+ for (let j = 0; j < dots.length; j++)
73
+ out[j] = 1 - dots[j];
74
+ return out;
75
+ },
76
+ };
77
+ }
78
+ function jsDotF32(a, b) {
79
+ let s = 0;
80
+ const n = a.length;
81
+ const limit = n - 7;
82
+ let i = 0;
83
+ let s0 = 0, s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0, s7 = 0;
84
+ while (i < limit) {
85
+ s0 += a[i] * b[i];
86
+ s1 += a[i + 1] * b[i + 1];
87
+ s2 += a[i + 2] * b[i + 2];
88
+ s3 += a[i + 3] * b[i + 3];
89
+ s4 += a[i + 4] * b[i + 4];
90
+ s5 += a[i + 5] * b[i + 5];
91
+ s6 += a[i + 6] * b[i + 6];
92
+ s7 += a[i + 7] * b[i + 7];
93
+ i += 8;
94
+ }
95
+ s = s0 + s1 + (s2 + s3) + (s4 + s5) + (s6 + s7);
96
+ while (i < n) {
97
+ s += a[i] * b[i];
98
+ i++;
99
+ }
100
+ return s;
101
+ }
40
102
  /**
41
103
  * Insert node `idx` (vector index) into the graph. Incremental — new indices
42
104
  * attach to the existing structure without a rebuild; vectors for nodes NOT
43
105
  * yet inserted are read from `vectors` when needed, so the caller can insert
44
106
  * lazily. Tombstoned indices can be re-inserted (undeletes).
45
107
  */
46
- export function insertHnsw(hnsw, idx, distance, vectors) {
108
+ export function insertHnsw(hnsw, idx, distance, vectors, engine) {
47
109
  const vec = vectors[idx];
48
110
  if (!vec)
49
111
  throw new Error(`hnsw insert: no vector at index ${idx}`);
@@ -64,7 +126,14 @@ export function insertHnsw(hnsw, idx, distance, vectors) {
64
126
  hnsw.maxLevel = level;
65
127
  return;
66
128
  }
67
- const qDist = (node) => distance(vec, node);
129
+ // Issue #228: with a build engine, all pointwise distances go through
130
+ // dist(aIdx, bIdx) so the whole chain shares one arithmetic order; the
131
+ // per-neighbor scan uses engine.batch when available (one wasm call per
132
+ // node's neighbor list instead of one JS call per distance).
133
+ const dist = engine
134
+ ? (aIdx, bIdx) => engine.dist(aIdx, bIdx)
135
+ : (aIdx, bIdx) => distance(vectors[aIdx], bIdx);
136
+ const qDist = (node) => dist(idx, node);
68
137
  // 1. Greedy descent through layers above the node's level
69
138
  let ep = hnsw.entryPoint;
70
139
  let epDist = qDist(ep);
@@ -83,10 +152,12 @@ export function insertHnsw(hnsw, idx, distance, vectors) {
83
152
  }
84
153
  }
85
154
  // 2. Insert on each layer from min(level, maxLevel) down to 0
86
- const pairwise = (a, b) => distance(vectors[a], b);
155
+ const pairwise = engine
156
+ ? (a, b) => engine.dist(a, b)
157
+ : (a, b) => distance(vectors[a], b);
87
158
  for (let l = Math.min(level, hnsw.maxLevel); l >= 0; l--) {
88
159
  const efC = Math.max(M0, DEFAULT_HNSW_EF);
89
- const candidates = searchLayer(hnsw, qDist, ep, epDist, l, efC);
160
+ const candidates = searchLayer(hnsw, qDist, ep, epDist, l, efC, engine?.batch, idx);
90
161
  const cap = l === 0 ? M0 : M;
91
162
  // Algorithm 4: diversity-aware neighbor selection keeps the graph
92
163
  // navigable — a candidate is skipped only when it is CLOSER to an
@@ -121,7 +192,10 @@ export function deleteHnsw(hnsw, idx) {
121
192
  hnsw.deleted.add(idx);
122
193
  }
123
194
  /** ef-search one layer, returning the closest `ef` live candidates (closest first). */
124
- function searchLayer(hnsw, qDist, ep, epDist, layer, ef) {
195
+ function searchLayer(hnsw, qDist, ep, epDist, layer, ef,
196
+ // issue #228: when present, the unvisited neighbors of each expanded node
197
+ // are measured in ONE batched call (same order → same heap evolution).
198
+ batch, aIdx) {
125
199
  const visited = new Set([ep]);
126
200
  // max-heap by dist via simple sorted insertion (candidate lists are small)
127
201
  const candidates = [{ node: ep, dist: epDist }];
@@ -131,11 +205,19 @@ function searchLayer(hnsw, qDist, ep, epDist, layer, ef) {
131
205
  const worst = results.length >= ef ? results[results.length - 1].dist : Infinity;
132
206
  if (current.dist > worst)
133
207
  break;
134
- for (const nb of hnsw.neighbors[current.node][layer] ?? []) {
135
- if (visited.has(nb))
136
- continue;
208
+ const neighborList = hnsw.neighbors[current.node][layer] ?? [];
209
+ const fresh = [];
210
+ for (const nb of neighborList) {
211
+ if (!visited.has(nb))
212
+ fresh.push(nb);
213
+ }
214
+ let dists = null;
215
+ if (fresh.length > 1 && batch && aIdx !== undefined)
216
+ dists = batch(aIdx, fresh);
217
+ for (let j = 0; j < fresh.length; j++) {
218
+ const nb = fresh[j];
137
219
  visited.add(nb);
138
- const d = qDist(nb);
220
+ const d = dists ? dists[j] : qDist(nb);
139
221
  const worstNow = results.length >= ef ? results[results.length - 1].dist : Infinity;
140
222
  if (d < worstNow || results.length < ef) {
141
223
  candidates.push({ node: nb, dist: d });
package/dist/indexer.js CHANGED
@@ -524,7 +524,11 @@ async function _buildIndexInner({ db, indexDir, cacheDir, modelName, dimensions,
524
524
  }
525
525
  }
526
526
  if (!hnswIndex) {
527
- hnswIndex = buildHnsw(rawVecs, { vectors: rawVecs, distance: hnswDistance(rawVecs) });
527
+ // issue #228: 'auto' routes the full build through the WASM SIMD
528
+ // dot-batch kernel when the vectors are a uniform Float32Array matrix
529
+ // (the hnswDistance closure below is exactly 1 − dot, so semantics
530
+ // match); otherwise buildHnsw silently keeps the closure path.
531
+ hnswIndex = buildHnsw(rawVecs, { vectors: rawVecs, distance: hnswDistance(rawVecs), engine: 'auto' });
528
532
  }
529
533
  atomicWrite(hnswPath, JSON.stringify(serializeHnsw(hnswIndex)));
530
534
  if (fs.existsSync(ivfPath)) {
package/dist/models.d.ts CHANGED
@@ -24,7 +24,7 @@ export interface ModelAdapter {
24
24
  unknownAdapter?: boolean;
25
25
  }
26
26
  export declare const MODELS: Record<string, ModelAdapter>;
27
- export declare const DEFAULT_MODEL = "e5-base";
27
+ export declare const DEFAULT_MODEL = "granite-multilingual-311m";
28
28
  export declare const POOLING: Pooling[];
29
29
  export declare function normalizeAdapter(raw: Partial<ModelAdapter> | ModelAdapter): ModelAdapter;
30
30
  export declare function resolveModel(name?: string | Partial<ModelAdapter> | ModelAdapter | null, chosenDim?: number): ModelAdapter;
package/dist/models.js CHANGED
@@ -27,7 +27,7 @@ export const MODELS = {
27
27
  dtype: 'q8',
28
28
  maxTokens: 512,
29
29
  family: 'e5',
30
- note: 'Default. ~280MB. Solid multilingual + cross-lingual balance.',
30
+ note: 'Pre-2.2.0 default. ~280MB. Solid multilingual + cross-lingual balance.',
31
31
  },
32
32
  'e5-large': {
33
33
  id: 'Xenova/multilingual-e5-large',
@@ -102,7 +102,12 @@ export const MODELS = {
102
102
  note: '~300 MB q8. Google EmbeddingGemma, pre-pooled sentence_embedding output, MRL. Gemma license (not MIT/Apache). Issue #200.',
103
103
  },
104
104
  };
105
- export const DEFAULT_MODEL = 'e5-base';
105
+ // Issue #227: granite-multilingual-311m is the default — Apache-2.0 (the
106
+ // measured leader embeddinggemma-300m is license-blocked from being an npm
107
+ // default by the Gemma Terms of Use) and measured +0.078 nDCG@10 over e5-base
108
+ // on the 186-query golden set v2 (RESEARCH.md §15). Same size class (~298 MB
109
+ // q8), 8k ModernBERT context. e5-base stays a registered alias.
110
+ export const DEFAULT_MODEL = 'granite-multilingual-311m';
106
111
  export const POOLING = ['mean', 'last_token', 'none', 'cls'];
107
112
  const POOLING_SET = new Set(POOLING);
108
113
  const aliasByRepoId = new Map();
@@ -0,0 +1,36 @@
1
+ /**
2
+ * WASM SIMD128 dot-product engine for the HNSW graph build (issue #228).
3
+ *
4
+ * The build's cost profile is ~85% distance computation (measured: 16.5k
5
+ * distance calls per node at n=5000, dim=256; RESEARCH.md §17), and those
6
+ * calls sit on a sequential dependency chain — worker_threads parallelism is
7
+ * structurally blocked (per-insert graph state + the byte-identical-sidecar
8
+ * determinism contract). The achievable lever is making each dot faster:
9
+ * this module batches f32 dot products into ONE WebAssembly call
10
+ * (`dot_batch`: N row indices → N dot products against a shared query row)
11
+ * with a SIMD128 mul/add inner loop — measured ~6× over the unrolled JS dot
12
+ * at dim=256 on a 32-dot batch.
13
+ *
14
+ * Zero native dependencies: the module is 363 bytes of hand-written WAT
15
+ * compiled once via wabt and embedded below (source of record:
16
+ * bench/ann-recall.mjs history + RESEARCH.md §17). Compile failure (no SIMD
17
+ * in the runtime) degrades to `null` and the caller keeps the JS closure path.
18
+ *
19
+ * Determinism: all arithmetic is IEEE f32 with a fixed lane order — repeated
20
+ * builds on the same corpus produce a byte-identical sidecar per engine. The
21
+ * SIMD accumulation order differs from the JS closure path in the last ulp,
22
+ * so the two engines are each self-consistent but not cross-identical; the
23
+ * sidecar records which engine built it (same engine → same bytes).
24
+ */
25
+ export interface WasmDotEngine {
26
+ /** dot(vectors[a], vectors[b]) — one-row batch, same arithmetic as batch(). */
27
+ point(aIdx: number, bIdx: number): number;
28
+ /** result[j] = dot(vectors[aIdx], vectors[nodes[j]]) — a view into wasm memory, valid until the next call. */
29
+ batch(aIdx: number, nodes: number[]): Float32Array;
30
+ readonly dim: number;
31
+ }
32
+ /**
33
+ * Build the engine over a uniform Float32Array matrix. Returns null when the
34
+ * runtime cannot compile the SIMD module (then callers keep the JS path).
35
+ */
36
+ export declare function createWasmDotEngine(vectors: Array<Float32Array>): WasmDotEngine | null;
@@ -0,0 +1,106 @@
1
+ /**
2
+ * WASM SIMD128 dot-product engine for the HNSW graph build (issue #228).
3
+ *
4
+ * The build's cost profile is ~85% distance computation (measured: 16.5k
5
+ * distance calls per node at n=5000, dim=256; RESEARCH.md §17), and those
6
+ * calls sit on a sequential dependency chain — worker_threads parallelism is
7
+ * structurally blocked (per-insert graph state + the byte-identical-sidecar
8
+ * determinism contract). The achievable lever is making each dot faster:
9
+ * this module batches f32 dot products into ONE WebAssembly call
10
+ * (`dot_batch`: N row indices → N dot products against a shared query row)
11
+ * with a SIMD128 mul/add inner loop — measured ~6× over the unrolled JS dot
12
+ * at dim=256 on a 32-dot batch.
13
+ *
14
+ * Zero native dependencies: the module is 363 bytes of hand-written WAT
15
+ * compiled once via wabt and embedded below (source of record:
16
+ * bench/ann-recall.mjs history + RESEARCH.md §17). Compile failure (no SIMD
17
+ * in the runtime) degrades to `null` and the caller keeps the JS closure path.
18
+ *
19
+ * Determinism: all arithmetic is IEEE f32 with a fixed lane order — repeated
20
+ * builds on the same corpus produce a byte-identical sidecar per engine. The
21
+ * SIMD accumulation order differs from the JS closure path in the last ulp,
22
+ * so the two engines are each self-consistent but not cross-identical; the
23
+ * sidecar records which engine built it (same engine → same bytes).
24
+ */
25
+ /** The compiled `dot_batch` module (see module docstring for provenance). */
26
+ const KERNEL_B64 = 'AGFzbQEAAAABCQFgBX9/f39/AAMCAQAFAwEAAQcTAgNtZW0CAAlkb3RfYmF0Y2gAAAq3AgG0AgQCfwF9An8EeyADQQRsIQlBACEF' +
27
+ 'AkADQCAFIAJODQEgASAFQQRsaigCACAJbCEG/QwAAAAAAAAAAAAAAAAAAAAAIQr9DAAAAAAAAAAAAAAAAAAAAAAhC/0MAAAAAAAAA' +
28
+ 'AAAAAAAAAAAACEM/QwAAAAAAAAAAAAAAAAAAAAAIQ1DAAAAACEHQQAhCAJAA0AgCEEQaiAJSg0BIAogACAIav0AAgAgBiAIav0A' +
29
+ 'AgD95gH95AEhCiAIQRBqIQgMAAsLAkADQCAIIAlODQEgByAAIAhqKgIAIAYgCGoqAgCUkiEHIAhBBGohCAwACwsgBCAFQQRsaiAK' +
30
+ '/R8AIAr9HwGSIAr9HwIgCv0fA5KSIAcgC/0fACAL/R8BkiAL/R8CIAv9HwOSkpKSOAIAIAVBAWohBQwACwsL';
31
+ /** Max nodes per batch call — HNSW neighbor lists cap at M0 = 2*M ≤ 64. */
32
+ const MAX_BATCH = 64;
33
+ /**
34
+ * Build the engine over a uniform Float32Array matrix. Returns null when the
35
+ * runtime cannot compile the SIMD module (then callers keep the JS path).
36
+ */
37
+ export function createWasmDotEngine(vectors) {
38
+ if (vectors.length === 0)
39
+ return null;
40
+ const dim = vectors[0].length;
41
+ if (dim === 0)
42
+ return null;
43
+ for (const v of vectors) {
44
+ if (!(v instanceof Float32Array) || v.length !== dim)
45
+ return null;
46
+ }
47
+ let kernelBuffer;
48
+ try {
49
+ const bin = Buffer.from(KERNEL_B64, 'base64');
50
+ kernelBuffer = new ArrayBuffer(bin.byteLength);
51
+ new Uint8Array(kernelBuffer).set(bin);
52
+ if (bin.byteLength === 0)
53
+ return null;
54
+ }
55
+ catch {
56
+ return null;
57
+ }
58
+ let instance;
59
+ try {
60
+ instance = new WebAssembly.Instance(new WebAssembly.Module(kernelBuffer), {});
61
+ }
62
+ catch {
63
+ return null; // no SIMD support — caller falls back to the JS closure path
64
+ }
65
+ const exp = instance.exports;
66
+ const mem = exp.mem;
67
+ // Linear memory layout: [vectors row-major][idx scratch u32 × MAX_BATCH][out f32 × MAX_BATCH]
68
+ const vecBytes = vectors.length * dim * 4;
69
+ const idxByte = vecBytes;
70
+ const outByte = idxByte + MAX_BATCH * 4;
71
+ const needBytes = outByte + MAX_BATCH * 4 + 4096;
72
+ const pages = Math.ceil(needBytes / 65536);
73
+ try {
74
+ if (mem.buffer.byteLength < needBytes)
75
+ mem.grow(pages);
76
+ }
77
+ catch {
78
+ return null;
79
+ }
80
+ // Views are created AFTER the final grow — growing detaches old buffers.
81
+ const f32 = new Float32Array(mem.buffer);
82
+ const u32 = new Uint32Array(mem.buffer);
83
+ for (let i = 0; i < vectors.length; i++)
84
+ f32.set(vectors[i], i * dim);
85
+ const qByteOf = (aIdx) => {
86
+ if (!Number.isSafeInteger(aIdx) || aIdx < 0 || aIdx >= vectors.length) {
87
+ throw new Error(`wasm-dot: row index out of range: ${aIdx}`);
88
+ }
89
+ return aIdx * dim * 4;
90
+ };
91
+ return {
92
+ dim,
93
+ point(aIdx, bIdx) {
94
+ u32[idxByte >> 2] = bIdx;
95
+ exp.dot_batch(qByteOf(aIdx), idxByte, 1, dim, outByte);
96
+ return f32[outByte >> 2];
97
+ },
98
+ batch(aIdx, nodes) {
99
+ const n = Math.min(nodes.length, MAX_BATCH);
100
+ for (let j = 0; j < n; j++)
101
+ u32[(idxByte >> 2) + j] = nodes[j];
102
+ exp.dot_batch(qByteOf(aIdx), idxByte, n, dim, outByte);
103
+ return f32.subarray(outByte >> 2, (outByte >> 2) + n);
104
+ },
105
+ };
106
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "md-semantic-search",
3
- "version": "2.1.0",
3
+ "version": "2.2.1",
4
4
  "description": "Local, private semantic (vector) search over any folder of Markdown files. Finds passages by meaning — cross-lingual — using transformers.js. No API keys, no cloud, no vector database.",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
package/src/hnsw.ts CHANGED
@@ -11,6 +11,8 @@
11
11
  * is what lets the incremental watch rebuild reuse a persisted graph.
12
12
  */
13
13
 
14
+ import { createWasmDotEngine } from './wasm-dot.js';
15
+
14
16
  /** Deterministic PRNG for level assignment (same family as ivf.ts). */
15
17
  function mulberry32(seed: number): () => number {
16
18
  let s = seed >>> 0;
@@ -63,6 +65,28 @@ export interface BuildHnswOptions {
63
65
  */
64
66
  distance: (a: Float32Array | Int8Array | number[], b: number) => number;
65
67
  vectors: Array<Float32Array | Int8Array | number[]>;
68
+ /**
69
+ * Issue #228: 'auto' routes the build's distance work through the WASM
70
+ * SIMD dot-batch kernel (src/wasm-dot.ts) when the vectors are a uniform
71
+ * Float32Array matrix and the runtime supports SIMD — ~6× faster dots on a
72
+ * 32-node batch, ~3.4× faster full build (RESEARCH.md §17). ONLY valid when
73
+ * the intended distance is exactly `1 − dot(a, vectors[b])`; any custom
74
+ * distance (int8 norms, asymmetric variants) must keep the default
75
+ * 'closure' engine. Silent fallback to 'closure' when 'auto' cannot apply.
76
+ */
77
+ engine?: 'auto' | 'closure';
78
+ }
79
+
80
+ /**
81
+ * Distance plumbing for the build (issue #228). The closure engine wraps the
82
+ * caller-supplied `distance`; the wasm engine replaces BOTH pointwise and
83
+ * batch distances with kernel calls so every distance on the chain uses the
84
+ * same SIMD accumulation order (self-consistent → deterministic sidecar).
85
+ */
86
+ export interface HnswBuildEngine {
87
+ dist(aIdx: number, bIdx: number): number;
88
+ /** Distances from vectors[aIdx] to each node, same order — null = pointwise. */
89
+ batch?(aIdx: number, nodes: number[]): Float32Array | null;
66
90
  }
67
91
 
68
92
  export const DEFAULT_HNSW_M = 16;
@@ -82,10 +106,70 @@ export function buildHnsw(
82
106
  neighbors: [],
83
107
  deleted: new Set(),
84
108
  };
85
- for (let i = 0; i < vectors.length; i++) insertHnsw(hnsw, i, options.distance, options.vectors);
109
+ let engine: HnswBuildEngine | undefined;
110
+ if (options.engine === 'auto') {
111
+ engine = createWasmBuildEngine(vectors) ?? undefined;
112
+ // undefined → SIMD unavailable or non-uniform vectors: closure engine, silently.
113
+ }
114
+ for (let i = 0; i < vectors.length; i++) insertHnsw(hnsw, i, options.distance, options.vectors, engine);
86
115
  return hnsw;
87
116
  }
88
117
 
118
+ /**
119
+ * Issue #228: wasm-backed build engine for uniform Float32Array matrices.
120
+ * Returns null whenever the fast path cannot apply — the caller then keeps
121
+ * the caller-supplied closure (identical semantics, JS arithmetic).
122
+ */
123
+ function createWasmBuildEngine(vectors: Array<Float32Array | Int8Array | number[]>): HnswBuildEngine | null {
124
+ for (const v of vectors) {
125
+ if (!(v instanceof Float32Array)) return null;
126
+ }
127
+ const wasmEngine = createWasmDotEngine(vectors as Array<Float32Array>);
128
+ if (!wasmEngine) return null;
129
+ // Hybrid distance plumbing (measured, issue #228): BATCHES (a whole node's
130
+ // neighbor list — the 85% cost center) go through the SIMD kernel; pointwise
131
+ // calls (greedy descent, singleton scans, heuristic pairwise) stay on the
132
+ // plain JS dot — a 1-row wasm call pays call overhead exceeding the work.
133
+ // Both paths are IEEE f32 with fixed orders, so the build stays
134
+ // deterministic; recall is insensitive to the last-ulp difference (measured
135
+ // identical across engines in bench/ann-recall.mjs and the #228 tests).
136
+ const f32Vectors = vectors as Array<Float32Array>;
137
+ return {
138
+ dist: (a, b) => 1 - jsDotF32(f32Vectors[a], f32Vectors[b]),
139
+ batch: (aIdx, nodes) => {
140
+ const dots = wasmEngine.batch(aIdx, nodes);
141
+ const out = new Float32Array(dots.length);
142
+ for (let j = 0; j < dots.length; j++) out[j] = 1 - dots[j];
143
+ return out;
144
+ },
145
+ };
146
+ }
147
+
148
+ function jsDotF32(a: Float32Array, b: Float32Array): number {
149
+ let s = 0;
150
+ const n = a.length;
151
+ const limit = n - 7;
152
+ let i = 0;
153
+ let s0 = 0, s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0, s7 = 0;
154
+ while (i < limit) {
155
+ s0 += a[i] * b[i];
156
+ s1 += a[i + 1] * b[i + 1];
157
+ s2 += a[i + 2] * b[i + 2];
158
+ s3 += a[i + 3] * b[i + 3];
159
+ s4 += a[i + 4] * b[i + 4];
160
+ s5 += a[i + 5] * b[i + 5];
161
+ s6 += a[i + 6] * b[i + 6];
162
+ s7 += a[i + 7] * b[i + 7];
163
+ i += 8;
164
+ }
165
+ s = s0 + s1 + (s2 + s3) + (s4 + s5) + (s6 + s7);
166
+ while (i < n) {
167
+ s += a[i] * b[i];
168
+ i++;
169
+ }
170
+ return s;
171
+ }
172
+
89
173
  /**
90
174
  * Insert node `idx` (vector index) into the graph. Incremental — new indices
91
175
  * attach to the existing structure without a rebuild; vectors for nodes NOT
@@ -96,7 +180,8 @@ export function insertHnsw(
96
180
  hnsw: HnswIndex,
97
181
  idx: number,
98
182
  distance: (a: Float32Array | Int8Array | number[], b: number) => number,
99
- vectors: Array<Float32Array | Int8Array | number[]>
183
+ vectors: Array<Float32Array | Int8Array | number[]>,
184
+ engine?: HnswBuildEngine
100
185
  ): void {
101
186
  const vec = vectors[idx];
102
187
  if (!vec) throw new Error(`hnsw insert: no vector at index ${idx}`);
@@ -121,7 +206,14 @@ export function insertHnsw(
121
206
  return;
122
207
  }
123
208
 
124
- const qDist = (node: number) => distance(vec, node);
209
+ // Issue #228: with a build engine, all pointwise distances go through
210
+ // dist(aIdx, bIdx) so the whole chain shares one arithmetic order; the
211
+ // per-neighbor scan uses engine.batch when available (one wasm call per
212
+ // node's neighbor list instead of one JS call per distance).
213
+ const dist = engine
214
+ ? (aIdx: number, bIdx: number) => engine.dist(aIdx, bIdx)
215
+ : (aIdx: number, bIdx: number) => distance(vectors[aIdx], bIdx);
216
+ const qDist = (node: number) => dist(idx, node);
125
217
 
126
218
  // 1. Greedy descent through layers above the node's level
127
219
  let ep = hnsw.entryPoint;
@@ -142,10 +234,12 @@ export function insertHnsw(
142
234
  }
143
235
 
144
236
  // 2. Insert on each layer from min(level, maxLevel) down to 0
145
- const pairwise = (a: number, b: number) => distance(vectors[a], b);
237
+ const pairwise = engine
238
+ ? (a: number, b: number) => engine.dist(a, b)
239
+ : (a: number, b: number) => distance(vectors[a], b);
146
240
  for (let l = Math.min(level, hnsw.maxLevel); l >= 0; l--) {
147
241
  const efC = Math.max(M0, DEFAULT_HNSW_EF);
148
- const candidates = searchLayer(hnsw, qDist, ep, epDist, l, efC);
242
+ const candidates = searchLayer(hnsw, qDist, ep, epDist, l, efC, engine?.batch, idx);
149
243
  const cap = l === 0 ? M0 : M;
150
244
  // Algorithm 4: diversity-aware neighbor selection keeps the graph
151
245
  // navigable — a candidate is skipped only when it is CLOSER to an
@@ -191,7 +285,11 @@ function searchLayer(
191
285
  ep: number,
192
286
  epDist: number,
193
287
  layer: number,
194
- ef: number
288
+ ef: number,
289
+ // issue #228: when present, the unvisited neighbors of each expanded node
290
+ // are measured in ONE batched call (same order → same heap evolution).
291
+ batch?: (aIdx: number, nodes: number[]) => Float32Array | null,
292
+ aIdx?: number
195
293
  ): Array<{ node: number; dist: number }> {
196
294
  const visited = new Set<number>([ep]);
197
295
  // max-heap by dist via simple sorted insertion (candidate lists are small)
@@ -204,10 +302,17 @@ function searchLayer(
204
302
  const worst = results.length >= ef ? results[results.length - 1].dist : Infinity;
205
303
  if (current.dist > worst) break;
206
304
 
207
- for (const nb of hnsw.neighbors[current.node][layer] ?? []) {
208
- if (visited.has(nb)) continue;
305
+ const neighborList = hnsw.neighbors[current.node][layer] ?? [];
306
+ const fresh: number[] = [];
307
+ for (const nb of neighborList) {
308
+ if (!visited.has(nb)) fresh.push(nb);
309
+ }
310
+ let dists: Float32Array | null = null;
311
+ if (fresh.length > 1 && batch && aIdx !== undefined) dists = batch(aIdx, fresh);
312
+ for (let j = 0; j < fresh.length; j++) {
313
+ const nb = fresh[j];
209
314
  visited.add(nb);
210
- const d = qDist(nb);
315
+ const d = dists ? dists[j] : qDist(nb);
211
316
  const worstNow = results.length >= ef ? results[results.length - 1].dist : Infinity;
212
317
  if (d < worstNow || results.length < ef) {
213
318
  candidates.push({ node: nb, dist: d });
package/src/indexer.ts CHANGED
@@ -701,7 +701,11 @@ async function _buildIndexInner({
701
701
  }
702
702
  }
703
703
  if (!hnswIndex) {
704
- hnswIndex = buildHnsw(rawVecs, { vectors: rawVecs, distance: hnswDistance(rawVecs) });
704
+ // issue #228: 'auto' routes the full build through the WASM SIMD
705
+ // dot-batch kernel when the vectors are a uniform Float32Array matrix
706
+ // (the hnswDistance closure below is exactly 1 − dot, so semantics
707
+ // match); otherwise buildHnsw silently keeps the closure path.
708
+ hnswIndex = buildHnsw(rawVecs, { vectors: rawVecs, distance: hnswDistance(rawVecs), engine: 'auto' });
705
709
  }
706
710
  atomicWrite(hnswPath, JSON.stringify(serializeHnsw(hnswIndex)));
707
711
  if (fs.existsSync(ivfPath)) {
package/src/models.ts CHANGED
@@ -55,7 +55,7 @@ export const MODELS: Record<string, ModelAdapter> = {
55
55
  dtype: 'q8',
56
56
  maxTokens: 512,
57
57
  family: 'e5',
58
- note: 'Default. ~280MB. Solid multilingual + cross-lingual balance.',
58
+ note: 'Pre-2.2.0 default. ~280MB. Solid multilingual + cross-lingual balance.',
59
59
  },
60
60
  'e5-large': {
61
61
  id: 'Xenova/multilingual-e5-large',
@@ -131,7 +131,12 @@ export const MODELS: Record<string, ModelAdapter> = {
131
131
  },
132
132
  };
133
133
 
134
- export const DEFAULT_MODEL = 'e5-base';
134
+ // Issue #227: granite-multilingual-311m is the default — Apache-2.0 (the
135
+ // measured leader embeddinggemma-300m is license-blocked from being an npm
136
+ // default by the Gemma Terms of Use) and measured +0.078 nDCG@10 over e5-base
137
+ // on the 186-query golden set v2 (RESEARCH.md §15). Same size class (~298 MB
138
+ // q8), 8k ModernBERT context. e5-base stays a registered alias.
139
+ export const DEFAULT_MODEL = 'granite-multilingual-311m';
135
140
  export const POOLING: Pooling[] = ['mean', 'last_token', 'none', 'cls'];
136
141
  const POOLING_SET = new Set<string>(POOLING);
137
142
 
@@ -0,0 +1,115 @@
1
+ /**
2
+ * WASM SIMD128 dot-product engine for the HNSW graph build (issue #228).
3
+ *
4
+ * The build's cost profile is ~85% distance computation (measured: 16.5k
5
+ * distance calls per node at n=5000, dim=256; RESEARCH.md §17), and those
6
+ * calls sit on a sequential dependency chain — worker_threads parallelism is
7
+ * structurally blocked (per-insert graph state + the byte-identical-sidecar
8
+ * determinism contract). The achievable lever is making each dot faster:
9
+ * this module batches f32 dot products into ONE WebAssembly call
10
+ * (`dot_batch`: N row indices → N dot products against a shared query row)
11
+ * with a SIMD128 mul/add inner loop — measured ~6× over the unrolled JS dot
12
+ * at dim=256 on a 32-dot batch.
13
+ *
14
+ * Zero native dependencies: the module is 363 bytes of hand-written WAT
15
+ * compiled once via wabt and embedded below (source of record:
16
+ * bench/ann-recall.mjs history + RESEARCH.md §17). Compile failure (no SIMD
17
+ * in the runtime) degrades to `null` and the caller keeps the JS closure path.
18
+ *
19
+ * Determinism: all arithmetic is IEEE f32 with a fixed lane order — repeated
20
+ * builds on the same corpus produce a byte-identical sidecar per engine. The
21
+ * SIMD accumulation order differs from the JS closure path in the last ulp,
22
+ * so the two engines are each self-consistent but not cross-identical; the
23
+ * sidecar records which engine built it (same engine → same bytes).
24
+ */
25
+
26
+ /** The compiled `dot_batch` module (see module docstring for provenance). */
27
+ const KERNEL_B64 =
28
+ 'AGFzbQEAAAABCQFgBX9/f39/AAMCAQAFAwEAAQcTAgNtZW0CAAlkb3RfYmF0Y2gAAAq3AgG0AgQCfwF9An8EeyADQQRsIQlBACEF' +
29
+ 'AkADQCAFIAJODQEgASAFQQRsaigCACAJbCEG/QwAAAAAAAAAAAAAAAAAAAAAIQr9DAAAAAAAAAAAAAAAAAAAAAAhC/0MAAAAAAAAA' +
30
+ 'AAAAAAAAAAAACEM/QwAAAAAAAAAAAAAAAAAAAAAIQ1DAAAAACEHQQAhCAJAA0AgCEEQaiAJSg0BIAogACAIav0AAgAgBiAIav0A' +
31
+ 'AgD95gH95AEhCiAIQRBqIQgMAAsLAkADQCAIIAlODQEgByAAIAhqKgIAIAYgCGoqAgCUkiEHIAhBBGohCAwACwsgBCAFQQRsaiAK' +
32
+ '/R8AIAr9HwGSIAr9HwIgCv0fA5KSIAcgC/0fACAL/R8BkiAL/R8CIAv9HwOSkpKSOAIAIAVBAWohBQwACwsL';
33
+
34
+ export interface WasmDotEngine {
35
+ /** dot(vectors[a], vectors[b]) — one-row batch, same arithmetic as batch(). */
36
+ point(aIdx: number, bIdx: number): number;
37
+ /** result[j] = dot(vectors[aIdx], vectors[nodes[j]]) — a view into wasm memory, valid until the next call. */
38
+ batch(aIdx: number, nodes: number[]): Float32Array;
39
+ readonly dim: number;
40
+ }
41
+
42
+ /** Max nodes per batch call — HNSW neighbor lists cap at M0 = 2*M ≤ 64. */
43
+ const MAX_BATCH = 64;
44
+
45
+ /**
46
+ * Build the engine over a uniform Float32Array matrix. Returns null when the
47
+ * runtime cannot compile the SIMD module (then callers keep the JS path).
48
+ */
49
+ export function createWasmDotEngine(vectors: Array<Float32Array>): WasmDotEngine | null {
50
+ if (vectors.length === 0) return null;
51
+ const dim = vectors[0].length;
52
+ if (dim === 0) return null;
53
+ for (const v of vectors) {
54
+ if (!(v instanceof Float32Array) || v.length !== dim) return null;
55
+ }
56
+
57
+ let kernelBuffer: ArrayBuffer;
58
+ try {
59
+ const bin = Buffer.from(KERNEL_B64, 'base64');
60
+ kernelBuffer = new ArrayBuffer(bin.byteLength);
61
+ new Uint8Array(kernelBuffer).set(bin);
62
+ if (bin.byteLength === 0) return null;
63
+ } catch {
64
+ return null;
65
+ }
66
+ let instance: WebAssembly.Instance;
67
+ try {
68
+ instance = new WebAssembly.Instance(new WebAssembly.Module(kernelBuffer), {});
69
+ } catch {
70
+ return null; // no SIMD support — caller falls back to the JS closure path
71
+ }
72
+ const exp = instance.exports as unknown as {
73
+ mem: WebAssembly.Memory;
74
+ dot_batch: (q: number, idx: number, n: number, dim: number, out: number) => void;
75
+ };
76
+ const mem = exp.mem;
77
+
78
+ // Linear memory layout: [vectors row-major][idx scratch u32 × MAX_BATCH][out f32 × MAX_BATCH]
79
+ const vecBytes = vectors.length * dim * 4;
80
+ const idxByte = vecBytes;
81
+ const outByte = idxByte + MAX_BATCH * 4;
82
+ const needBytes = outByte + MAX_BATCH * 4 + 4096;
83
+ const pages = Math.ceil(needBytes / 65536);
84
+ try {
85
+ if (mem.buffer.byteLength < needBytes) mem.grow(pages);
86
+ } catch {
87
+ return null;
88
+ }
89
+ // Views are created AFTER the final grow — growing detaches old buffers.
90
+ const f32 = new Float32Array(mem.buffer);
91
+ const u32 = new Uint32Array(mem.buffer);
92
+ for (let i = 0; i < vectors.length; i++) f32.set(vectors[i], i * dim);
93
+
94
+ const qByteOf = (aIdx: number) => {
95
+ if (!Number.isSafeInteger(aIdx) || aIdx < 0 || aIdx >= vectors.length) {
96
+ throw new Error(`wasm-dot: row index out of range: ${aIdx}`);
97
+ }
98
+ return aIdx * dim * 4;
99
+ };
100
+
101
+ return {
102
+ dim,
103
+ point(aIdx: number, bIdx: number): number {
104
+ u32[idxByte >> 2] = bIdx;
105
+ exp.dot_batch(qByteOf(aIdx), idxByte, 1, dim, outByte);
106
+ return f32[outByte >> 2];
107
+ },
108
+ batch(aIdx: number, nodes: number[]): Float32Array {
109
+ const n = Math.min(nodes.length, MAX_BATCH);
110
+ for (let j = 0; j < n; j++) u32[(idxByte >> 2) + j] = nodes[j];
111
+ exp.dot_batch(qByteOf(aIdx), idxByte, n, dim, outByte);
112
+ return f32.subarray(outByte >> 2, (outByte >> 2) + n);
113
+ },
114
+ };
115
+ }