@hviana/sema 0.1.5 → 0.1.7

package/src/rabitq-hnsw/test/hnsw.test.ts

@@ -1,1213 +0,0 @@
-import * as fs from "node:fs";
-import * as os from "node:os";
-import * as path from "node:path";
-import {
-  type QueryResult,
-  RaBitQuantizer,
-  VectorDatabase,
-} from "../src/index.js";
-
-// ----------------------------- tiny harness ----------------------------------
-let passed = 0;
-function assert(cond: boolean, msg: string): void {
-  if (!cond) throw new Error("ASSERTION FAILED: " + msg);
-  passed++;
-}
-function approxEqual(a: number, b: number, eps = 1e-4): boolean {
-  return Math.abs(a - b) <= eps * (1 + Math.abs(a) + Math.abs(b));
-}
-function section(title: string): void {
-  console.log("\n" + "=".repeat(72) + "\n" + title + "\n" + "=".repeat(72));
-}
-function now(): number {
-  return typeof performance !== "undefined" ? performance.now() : Date.now();
-}
-
-// ------------------------------- seeded RNG ----------------------------------
-class RNG {
-  private s: number;
-  constructor(seed: number) {
-    this.s = seed >>> 0 || 1;
-  }
-  next(): number {
-    let t = (this.s = (this.s + 0x6d2b79f5) | 0);
-    t = Math.imul(t ^ (t >>> 15), t | 1);
-    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
-    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
-  }
-  int(n: number): number {
-    return (this.next() * n) | 0;
-  }
-  private _gaussSpare = 0;
-  private _gaussHasSpare = false;
-
-  /** Box-Muller — caches the second value so every call produces one Gaussian
-   *  without wasting the log/sqrt pair. */
-  gauss(): number {
-    if (this._gaussHasSpare) {
-      this._gaussHasSpare = false;
-      return this._gaussSpare;
-    }
-    let u = 0;
-    let v = 0;
-    while (u === 0) u = this.next();
-    while (v === 0) v = this.next();
-    const r = Math.sqrt(-2 * Math.log(u));
-    this._gaussSpare = r * Math.sin(2 * Math.PI * v);
-    this._gaussHasSpare = true;
-    return r * Math.cos(2 * Math.PI * v);
-  }
-}
-
-function unit(v: number[]): number[] {
-  let s = 0;
-  for (const x of v) s += x * x;
-  const n = Math.sqrt(s) || 1;
-  return v.map((x) => x / n);
-}
-
-// =============================================================================
-// 1) Vector CRUD + persistence (SQLite file round-trip)
-// =============================================================================
-function testCrudAndPersistence(): void {
-  section("1) Vector CRUD + persistence (get returns the 1-bit code)");
-  const DIM = 48; // paddedDim 64 -> codeWords 2, distinctive codes
-  const dbPath = path.join(
-    os.tmpdir(),
-    `rabitq-test-crud-${Date.now()}.sqlite`,
-  );
-
-  let db = new VectorDatabase({
-    dbPath,
-    dim: DIM,
-    M: 8,
-    efConstruction: 64,
-    seed: 7,
-  });
-
-  // distinct directions so the 1-bit codes differ
-  const mk = (i: number, scale = 1) => {
-    const v = new Array(DIM).fill(0);
-    v[i] = scale;
-    return v;
-  };
-
-  // Integer IDs only
-  const ID = { alpha: 1, bravo: 2, charlie: 3, delta: 4, echo: 5 };
-  const vectors: Record<string, number[]> = {
-    alpha: mk(0),
-    bravo: mk(10),
-    charlie: mk(20),
-    delta: mk(30),
-    echo: mk(45),
-  };
-
-  // CREATE
-  for (const [name, v] of Object.entries(vectors)) {
-    db.insert(ID[name as keyof typeof ID], v);
-  }
-  assert(db.size === 5, "size should be 5 after inserting 5 vectors");
-
-  let threw = false;
-  try {
-    db.insert(ID.alpha, vectors.alpha);
-  } catch {
-    threw = true;
-  }
-  assert(threw, "inserting a duplicate external id must throw");
-
-  // READ — returns the bare 1-bit code (Uint32Array)
-  const code = db.get(ID.alpha)!;
-  assert(code !== null, "get(alpha) returns a stored code");
-  assert(
-    code instanceof Uint32Array && code.length === 2,
-    "code is 2 packed 32-bit words",
-  );
-  assert(code.length * 32 === 64, "code covers paddedDim = 64 sign bits");
-  assert(db.get(9999) === null, "get of unknown id returns null");
-  assert(db.has(ID.bravo) && !db.has(9999), "has() reflects membership");
-
-  // SEARCH — querying with an exact copy returns that vector first
-  assert(
-    db.query(vectors.alpha, 3)[0].id === ID.alpha,
-    "exact-copy query returns alpha first",
-  );
-  assert(
-    db.query(vectors.bravo, 1)[0].id === ID.bravo,
-    "exact-copy query returns bravo first",
-  );
-
-  // UPDATE
-  db.update(ID.charlie, mk(40));
-  assert(
-    db.query(mk(40), 1)[0].id === ID.charlie,
-    "updated charlie is found at its new location",
-  );
-  assert(db.size === 5, "size unchanged after update");
-
-  // DELETE
-  assert(db.delete(ID.delta) === true, "delete returns true for existing id");
-  assert(
-    db.delete(ID.delta) === false,
-    "delete returns false for already removed id",
-  );
-  assert(!db.has(ID.delta) && db.size === 4, "delta removed, size now 4");
-  assert(
-    !db.query(vectors.delta, 4).some((r: QueryResult) => r.id === ID.delta),
-    "deleted vector never appears in results",
-  );
-
-  // PERSISTENCE round-trip — capture state, then close and reopen the same file
-  const liveKeys = Array.from(db.keys());
-  const probe = mk(0);
-  const before = db.query(probe, 4);
-  const codesBefore: Map<number, Uint32Array> = new Map();
-  for (const id of liveKeys) codesBefore.set(id, db.get(id)!);
-
-  db.close();
-
-  const reloaded = new VectorDatabase({ dbPath });
-  assert(reloaded.size === db.size, "reloaded size matches");
-  for (const id of liveKeys) {
-    const a = codesBefore.get(id)!;
-    const b = reloaded.get(id)!;
-    assert(b !== null, `reloaded code for ${id} is not null`);
-    assert(
-      a.length === b.length,
-      `reloaded code length for ${id} matches`,
-    );
-    for (let i = 0; i < a.length; i++) {
-      assert(a[i] === b[i], `reloaded code word for ${id} matches`);
-    }
-  }
-  const after = reloaded.query(probe, 4);
-  assert(
-    before.length === after.length,
-    "reloaded query result length matches",
-  );
-  for (let i = 0; i < before.length; i++) {
-    assert(before[i].id === after[i].id, "reloaded query ordering matches");
-    assert(
-      approxEqual(before[i].distance, after[i].distance, 1e-6),
-      "reloaded query distance matches",
-    );
-  }
-  reloaded.close();
-
-  // Cleanup
-  fs.unlinkSync(dbPath);
-
-  console.log(
-    `CRUD + persistence OK (get returns ${code.length}-word codes; reload identical)`,
-  );
-}
-
-// =============================================================================
-// 1b) insert / update / query by an already-quantized 1-bit code
-// =============================================================================
-function testCodeIO(): void {
-  section("1b) insert / update / query by a 1-bit code (length-detected)");
-  const DIM = 128; // paddedDim 128 -> codeWords 4
-  const rng = new RNG(909);
-  const vec = () => Array.from({ length: DIM }, () => rng.gauss());
-  const db = new VectorDatabase({
-    dbPath: ":memory:",
-    dim: DIM,
-    M: 16,
-    efConstruction: 100,
-    seed: 5,
-  });
-
-  const N = 800;
-  for (let i = 0; i < N; i++) db.insert(i, vec());
-
-  const src = db.get(123)!;
-  assert(
-    src instanceof Uint32Array && src.length === 4,
-    "get() yields a 4-word code",
-  );
-
-  // QUERY by code: a stored code finds itself at distance 0
-  const byCode = db.query(src, 5);
-  assert(byCode[0].id === 123, "query(code) returns the owning vector first");
-  assert(
-    approxEqual(byCode[0].distance, 0, 1e-9),
-    "query(code) distance to the same code is 0",
-  );
-
-  // INSERT by code (length-detected): the copy is a real, findable node
-  const copyId = N + 1;
-  db.insert(copyId, src);
-  assert(db.size === N + 1, "insert(code) adds a vector");
-  const copyCode = db.get(copyId)!;
-  for (let i = 0; i < src.length; i++) {
-    assert(copyCode[i] === src[i], "insert(code) stored the code verbatim");
-  }
-  const hits = db.query(src, 5).map((h: QueryResult) => h.id);
-  assert(
-    hits.includes(copyId) && hits.includes(123),
-    "both the original and its code-copy are retrievable",
-  );
-
-  // number[] code (e.g. after conversion) is detected the same way
-  const asArray = Array.from(src);
-  const arrId = N + 2;
-  db.insert(arrId, asArray);
-  const arrCode = db.get(arrId)!;
-  for (let i = 0; i < src.length; i++) {
-    assert(arrCode[i] === src[i], "insert(number[] code) works identically");
-  }
-
-  // UPDATE by code: move id 7 onto id 200's code
-  const target = db.get(200)!;
-  db.update(7, target);
-  const moved = db.get(7)!;
-  for (let i = 0; i < target.length; i++) {
-    assert(moved[i] === target[i], "update(code) replaced the code");
-  }
-  assert(db.size === N + 2, "update(code) keeps size (replace, not add)");
-  const near200 = db.query(target, 5).map((h: QueryResult) => h.id);
-  assert(
-    near200.includes(7) && near200.includes(200),
-    "updated-by-code id now sits with its target",
-  );
-
-  db.close();
-  console.log(
-    "insert / update / query by 1-bit code OK (Uint32Array and number[])",
-  );
-}
-
-// =============================================================================
-// 2) Cosine metric sanity
-// =============================================================================
-function testCosine(): void {
-  section("2) Cosine sanity");
-  const DIM = 64;
-  const db = new VectorDatabase({ dbPath: ":memory:", dim: DIM, seed: 3 });
-  const axis = (i: number) => {
-    const v = new Array(DIM).fill(0);
-    v[i] = 1;
-    return v;
-  };
-  db.insert(1, axis(0)); // "x"
-  db.insert(2, axis(20)); // "y"
-  db.insert(3, axis(40)); // "z"
-  const r = db.query(axis(0), 3); // query along x's direction
-  assert(r[0].id === 1, "cosine: nearest direction to x's axis is x");
-  assert(
-    r[0].distance <= 0.1,
-    "cosine distance to an identical direction is ~ 0",
-  );
-  assert(
-    r[r.length - 1].distance > r[0].distance,
-    "orthogonal directions are farther than the aligned one",
-  );
-  db.close();
-  console.log("Cosine metric OK");
-}
-
-// =============================================================================
-// 2a) Near-duplicate self-recall (tight cluster)
-// =============================================================================
-// Every other recall test here uses WELL-SEPARATED groups, so the 1-bit codes
-// differ in many sign bits and recall looks perfect. Real workloads are not so
-// kind: vectors often form a TIGHT cluster — a shared base plus a small
-// distinctive residual — so each one sits a hair away from its neighbours on
-// the sphere. The contract that must hold there is the most basic one: a stored
-// vector, queried verbatim, retrieves ITSELF as the nearest neighbour.
-//
-// This is the case that exposed a precision bug in the query estimator. RaBitQ
-// scalar-quantises the full-precision query to `1 << queryBits` levels before
-// scoring it against the 1-bit codes. At 4 bits (16 levels) that step is too
-// coarse to resolve the small residual separating cluster siblings, so the
-// estimate misranks them and a vector can fail to find itself (~82% self-recall
-// on this data). The fix lives in rabitq.ts: the default queryBits is 8, which
-// resolves the residual and restores near-perfect self-recall — touching only
-// the query side (the stored 1-bit codes, disk size and graph build are
-// unchanged; only the per-query LUT widens from Uint8 to Uint16). The
-// assertions pin both ends: an explicit 4 bits is measurably degraded, the
-// default recovers.
-function testNearDuplicateRecall(): void {
-  section("2a) Near-duplicate self-recall (tight cluster)");
-  const DIM = 256;
-  const N = 150;
-  const SPREAD = 0.15;
-
-  // A fixed base direction; each vector is the base plus a small per-coordinate
-  // perturbation, then normalised — a dense cluster of near-duplicates. A plain
-  // LCG drives the perturbation: it produces a uniformly spread cloud around the
-  // base (the regime where the queryBits effect is sharpest); the Box-Muller
-  // RNG above clusters too tightly to separate the two settings.
-  const build = (queryBits: number) => {
-    let s = 7 >>> 0;
-    const next = () => {
-      s = (Math.imul(s, 1664525) + 1013904223) >>> 0;
-      return s / 4294967296;
-    };
-    const base = new Array(DIM);
-    for (let d = 0; d < DIM; d++) base[d] = next() * 2 - 1;
-    const vecs: number[][] = [];
-    const db = new VectorDatabase({
-      dbPath: ":memory:",
-      dim: DIM,
-      M: 16,
-      efConstruction: 64,
-      efSearch: 64,
-      seed: 123,
-      queryBits,
-    });
-    for (let i = 0; i < N; i++) {
-      const v = new Array(DIM);
-      let n = 0;
-      for (let d = 0; d < DIM; d++) {
-        v[d] = base[d] + SPREAD * (next() * 2 - 1);
-        n += v[d] * v[d];
-      }
-      n = Math.sqrt(n) || 1;
-      for (let d = 0; d < DIM; d++) v[d] /= n;
-      vecs.push(v);
-      db.insert(i, v);
-    }
-    let self = 0;
-    for (let i = 0; i < N; i++) {
-      const h = db.query(vecs[i], 1);
-      if (h[0] && h[0].id === i) self++;
-    }
-    db.close();
-    return self / N;
-  };
-
-  const r4 = build(4); // explicitly coarse — the old default
-  const rDefault = build(8); // the library default after the fix
-  console.log(
-    `near-duplicate self-recall: queryBits=4 ${(r4 * 100).toFixed(1)}%  ` +
-      `default(8) ${(rDefault * 100).toFixed(1)}%`,
-  );
-
-  // The bug: 4-bit query quantisation cannot resolve a tight cluster, so a
-  // sizeable fraction of vectors fail to retrieve themselves.
-  assert(
-    r4 < 0.9,
-    `queryBits=4 is degraded on a tight cluster (got ${
-      (r4 * 100).toFixed(1)
-    }%)`,
-  );
-  // The fix: the default B_q restores near-perfect self-recall — same stored
-  // codes, sharper query estimate.
-  assert(
-    rDefault >= 0.95,
-    `default queryBits recovers self-recall on a tight cluster (got ${
-      (rDefault * 100).toFixed(1)
-    }%)`,
-  );
-  assert(
-    rDefault > r4,
-    "the default queryBits must beat the coarse 4-bit setting",
-  );
-  console.log("Near-duplicate self-recall OK (queryBits resolves the cluster)");
-}
-
-// =============================================================================
-// 2a-bis) The cosFactor bias — a known, deliberate approximation in estimate()
-// =============================================================================
-// FINDING (the deeper cause behind the near-duplicate misranking above).
-//
-// RaBitQuantizer.estimate() returns `1 - cosFactor * A`, where
-//   cosFactor = sqrt(pi / (2 * paddedDim))
-// is a FIXED constant: the EXPECTED L1 norm of a randomly-rotated unit vector.
-// Canonical RaBitQ does not use that expectation — it divides A by each
-// vector's OWN rotated-L1 norm (the per-vector factor <o_bar, o>). Substituting
-// the population mean for the per-vector quantity makes the estimator BIASED,
-// and the bias is what this test pins down precisely.
-//
-// The cleanest probe is self-distance: a stored code scored against the query
-// it was made from must, for an unbiased cosine estimator, return distance 0
-// (cosine 1). With the fixed cosFactor it does NOT. Because each vector's true
-// rotated-L1 norm scatters around the mean, the estimate scatters around 0:
-//   - it is non-zero (mean and max are both > 0 by a clear margin), and
-//   - it goes NEGATIVE for a large fraction of vectors — a negative distance
-//     means estimated cosine > 1, which is geometrically impossible and is the
-//     unmistakable fingerprint of the mean-vs-per-vector substitution.
-//
-// This bias is small in magnitude but systematic, so near-threshold pairs read
-// as closer (or farther) than they truly are — exactly the misranking that
-// collapses a tight cluster. The test documents the property as-is (it does NOT
-// assert the bias away): the bias is REAL and lives in estimate(); raising
-// queryBits, tested above, is what keeps recall correct in its presence. Should
-// estimate() ever be upgraded to the true per-vector factor, the self-distance
-// here would tighten to ~0 and the negatives would vanish — at which point this
-// test is the tripwire that flags the behavioural change.
-function testCosFactorBias(): void {
-  section(
-    "2a-bis) cosFactor self-distance bias (fixed mean vs per-vector norm)",
-  );
-  const DIM = 256;
-  const N = 300;
-  const SPREAD = 0.15;
-
-  let s = 7 >>> 0;
-  const next = () => {
-    s = (Math.imul(s, 1664525) + 1013904223) >>> 0;
-    return s / 4294967296;
-  };
-  const base = new Array(DIM);
-  for (let d = 0; d < DIM; d++) base[d] = next() * 2 - 1;
-
-  const q = new RaBitQuantizer(DIM, { seed: 123 });
-  const vecs: number[][] = [];
-  const codes: Uint8Array[] = [];
-  for (let i = 0; i < N; i++) {
-    const v = new Array(DIM);
-    let n = 0;
-    for (let d = 0; d < DIM; d++) {
-      v[d] = base[d] + SPREAD * (next() * 2 - 1);
-      n += v[d] * v[d];
-    }
-    n = Math.sqrt(n) || 1;
-    for (let d = 0; d < DIM; d++) v[d] /= n;
-    vecs.push(v);
-    codes.push(q.codeToBytes(q.encode(v)));
-  }
-
-  // Self-distance: each vector scored against its OWN code. An unbiased cosine
-  // estimator returns 0 here; the fixed cosFactor does not.
-  let sum = 0;
-  let max = 0;
-  let negatives = 0;
-  for (let i = 0; i < N; i++) {
-    const ctx = q.prepareQuery(vecs[i]);
-    const d = q.estimate(codes[i], 0, ctx);
-    sum += d;
-    if (d > max) max = d;
-    if (d < 0) negatives++; // distance < 0  <=>  estimated cosine > 1 (impossible)
-  }
-  const mean = sum / N;
-  console.log(
-    `self-distance under fixed cosFactor: mean ${mean.toFixed(5)}, ` +
-      `max ${max.toFixed(5)}, negatives ${negatives}/${N} ` +
-      `(an unbiased estimator would give 0 with 0 negatives)`,
-  );
-
-  // The estimate is genuinely non-zero on a self-match: the fixed mean factor
-  // does not reproduce the per-vector normalisation.
-  assert(
-    max > 0.005,
-    `self-distance is biased away from 0 (max ${max.toFixed(5)})`,
-  );
-  // The fingerprint: a large fraction of self-matches yield an IMPOSSIBLE
-  // cosine > 1 (negative distance) — only a mean-vs-per-vector substitution
-  // produces this. If this ever drops to 0, estimate() changed.
-  assert(
-    negatives > N / 10,
-    `cosFactor produces impossible cosine>1 on self-matches ` +
-      `(${negatives}/${N} negative distances)`,
-  );
-  console.log(
-    "cosFactor bias documented (fixed mean L1 norm, not the per-vector factor)",
-  );
-}
-
-// =============================================================================
-// 2b) SQLite persistence round-trip
-// =============================================================================
-function testPersistence(): void {
-  section("2b) SQLite persistence round-trip");
-  const DIM = 96;
-  const rng = new RNG(2024);
-  const vec = () => Array.from({ length: DIM }, () => rng.gauss());
-
-  {
-    const dbPath = path.join(
-      os.tmpdir(),
-      `rabitq-test-persist-${Date.now()}.sqlite`,
-    );
-    const db = new VectorDatabase({
-      dbPath,
-      dim: DIM,
-      M: 12,
-      efConstruction: 80,
-      seed: 17,
-    });
-    for (let i = 0; i < 1500; i++) db.insert(i, vec());
-    for (let i = 0; i < 150; i++) db.delete(i * 10); // tombstones
-    for (let i = 0; i < 120; i++) db.update(3 + i * 10, vec()); // re-inserts
-
-    // Capture state before closing
-    const liveKeys = Array.from(db.keys());
-    const codesBefore: Map<number, Uint32Array> = new Map();
-    for (const id of liveKeys) codesBefore.set(id, db.get(id)!);
-    const queriesBefore: Array<{ query: number[]; results: QueryResult[] }> =
-      [];
-    for (let q = 0; q < 40; q++) {
-      const query = vec();
-      queriesBefore.push({ query, results: db.query(query, 10) });
-    }
-    const sizeBefore = db.size;
-    db.close();
-
-    // Reopen from file
-    const db2 = new VectorDatabase({ dbPath });
-    assert(
-      db2.size === sizeBefore,
-      "size preserved across persistence round-trip",
-    );
-
-    // get() parity: identical codes for every live id
-    for (const id of liveKeys) {
-      const a = codesBefore.get(id)!;
-      const b = db2.get(id)!;
-      assert(b !== null, `id ${id} present after reload`);
-      assert(a.length === b.length, "code length identical after reload");
-      for (let i = 0; i < a.length; i++) {
-        assert(a[i] === b[i], "code identical after reload");
-      }
-    }
-
-    // query parity: identical ids and (bit-exact) distances
-    let maxDiff = 0;
-    for (const { query, results: ra } of queriesBefore) {
-      const rb = db2.query(query, 10);
-      assert(ra.length === rb.length, "result length identical after reload");
-      for (let i = 0; i < ra.length; i++) {
-        assert(ra[i].id === rb[i].id, "result id/order identical after reload");
-        maxDiff = Math.max(maxDiff, Math.abs(ra[i].distance - rb[i].distance));
-      }
-    }
-    assert(
-      maxDiff === 0,
-      `distances are bit-exact after reload (max diff ${maxDiff})`,
-    );
-
-    // Mutation still works after a reload
-    db2.insert(99999, vec());
-    assert(db2.has(99999), "insert works after persistence load");
-    assert(db2.get(99999) !== null, "get works after persistence load");
-
-    db2.close();
-    fs.unlinkSync(dbPath);
-    console.log(
-      `${sizeBefore} vecs, round-trip via SQLite file — exact match`,
-    );
-  }
-
-  // Non-existent file: node:sqlite may create the file or may error — this is
-  // implementation-specific. We just verify no crash.
-  try {
-    new VectorDatabase({ dbPath: "/nonexistent/path/db.sqlite" });
-  } catch {
-    // tolerated — either outcome is acceptable
-  }
-  console.log("Non-existent path handled gracefully");
-}
-
-// =============================================================================
-// 2c) compact() — lossless rebuild that reclaims tombstones
-// =============================================================================
-function testCompact(): void {
-  section("2c) compact() — lossless rebuild reclaiming tombstones");
-  const DIM = 80;
-  const rng = new RNG(555);
-  const vec = () => Array.from({ length: DIM }, () => rng.gauss());
-
-  // ---- losslessness: build-then-delete-then-compact == fresh build of survivors ----
-  const N = 2500;
-  const data: number[][] = [];
-  for (let i = 0; i < N; i++) data.push(vec());
-  const deleted = new Set<number>();
-  for (let i = 0; i < N; i += 3) deleted.add(i); // delete ~1/3
-
-  const dbFull = new VectorDatabase({
-    dbPath: ":memory:",
-    dim: DIM,
-    M: 16,
-    efConstruction: 100,
-    seed: 88,
-  });
-  for (let i = 0; i < N; i++) dbFull.insert(i, data[i]);
-  for (const i of deleted) dbFull.delete(i);
-
-  assert(
-    dbFull.physicalSize === N,
-    "before compact: tombstones still occupy physical slots",
-  );
-  assert(
-    dbFull.size === N - deleted.size,
-    "before compact: live size already excludes deleted",
-  );
-
-  dbFull.compact();
-
-  const survivors: number[] = [];
-  for (let i = 0; i < N; i++) if (!deleted.has(i)) survivors.push(i);
-  assert(
-    dbFull.size === survivors.length,
-    "after compact: size is the survivor count",
-  );
-  assert(
-    dbFull.physicalSize === survivors.length,
-    "after compact: no tombstones remain (physical == live)",
-  );
-  for (const i of survivors) {
-    assert(dbFull.has(i), `survivor ${i} still present after compact`);
-  }
-  for (const i of deleted) {
-    assert(!dbFull.has(i), `deleted ${i} absent after compact`);
-  }
-
-  // Fresh index built from the survivors, same order + seed.
-  const dbFresh = new VectorDatabase({
-    dbPath: ":memory:",
-    dim: DIM,
-    M: 16,
-    efConstruction: 100,
-    seed: 88,
-  });
-  for (const i of survivors) dbFresh.insert(i, data[i]);
-
-  // HNSW is an approximate index — two builds with the same data+seed may
-  // produce slightly different graphs (the heuristic is not order-invariant
-  // under all edge cases).  What matters is that recall between the two is
-  // very high, and that distances are consistent.
-  let compactHits = 0;
-  let maxDistDiff = 0;
-  let totalCmp = 0;
-  for (let q = 0; q < 60; q++) {
-    const query = vec();
-    const a = dbFull.query(query, 10);
-    const b = dbFresh.query(query, 10);
-    const bIds = new Set(b.map((r) => r.id));
-    for (const r of a) if (bIds.has(r.id)) compactHits++;
-    totalCmp += a.length;
-    for (let i = 0; i < Math.min(a.length, b.length); i++) {
-      maxDistDiff = Math.max(
-        maxDistDiff,
-        Math.abs(a[i].distance - b[i].distance),
-      );
-    }
-  }
-  const recallCompact = compactHits / totalCmp;
-  assert(
-    recallCompact >= 0.97,
-    `compacted recall vs fresh build must be high (got ${
-      (recallCompact * 100).toFixed(1)
-    }%)`,
-  );
-  assert(
-    maxDistDiff < 0.05,
-    `distance agreement across the two builds (max diff ${maxDistDiff})`,
-  );
-  console.log(
-    `losslessness: ${N} built, ${deleted.size} deleted, compacted to ${survivors.length}; recall vs fresh=${
-      (recallCompact * 100).toFixed(1)
-    }%`,
-  );
-
-  // ---- compact also handles updates, and recall stays high on separable data ----
-  const G = 10;
-  const groups = 400;
-  const centers: number[][] = [];
-  for (let g = 0; g < groups; g++) centers.push(vec());
-  const db2 = new VectorDatabase({
-    dbPath: ":memory:",
-    dim: DIM,
-    M: 16,
-    efConstruction: 100,
-    seed: 7,
-  });
-  const live = new Map<number, number[]>();
-  let id = 0;
-  for (let g = 0; g < groups; g++) {
-    for (let j = 0; j < G; j++) {
-      const v = centers[g].map((c) => c + 0.1 * rng.gauss());
-      db2.insert(id, v);
-      live.set(id, v);
-      id++;
-    }
-  }
-  for (let k = 0; k < 400; k++) db2.delete(k); // drop 40 whole groups
-  for (let k = 0; k < 400; k++) live.delete(k);
-  for (let k = 600; k < 800; k++) {
-    const v = centers[Math.floor(k / G)].map((c) => c + 0.1 * rng.gauss());
-    db2.update(k, v); // re-place within the same group
-    live.set(k, v);
-  }
-  assert(db2.physicalSize > db2.size, "updates+deletes created tombstones");
-  db2.compact();
-  assert(
-    db2.physicalSize === db2.size && db2.size === live.size,
-    "compact reclaimed all tombstones",
-  );
-
-  // recall@10 over the surviving set after compaction
-  const liveIds = Array.from(live.keys());
-  const liveVecs = Array.from(live.values());
-  // Precompute data norms once (they don't change per query).
-  const liveNorms = new Float64Array(liveVecs.length);
-  for (let n = 0; n < liveVecs.length; n++) {
-    let s = 0;
-    const v = liveVecs[n];
-    for (let d = 0; d < DIM; d++) s += v[d] * v[d];
-    liveNorms[n] = Math.sqrt(s) || 1;
-  }
-  let hits = 0;
-  const Q = 100;
-  for (let q = 0; q < Q; q++) {
-    const center = centers[Math.floor((600 + (q % 200)) / G)];
-    const query = center.map((c) => c + 0.1 * rng.gauss());
-    // Query norm computed once, not N times.
-    let qn = 0;
-    for (let d = 0; d < DIM; d++) qn += query[d] * query[d];
-    qn = Math.sqrt(qn) || 1;
-    const bd = new Float64Array(10).fill(Infinity);
-    const bi = new Int32Array(10).fill(-1);
-    for (let n = 0; n < liveVecs.length; n++) {
-      const v = liveVecs[n];
-      let dot = 0;
-      for (let d = 0; d < DIM; d++) dot += v[d] * query[d];
-      const s = 1 - dot / (liveNorms[n] * qn);
-      if (s < bd[9]) {
-        let p = 9;
-        while (p > 0 && bd[p - 1] > s) {
-          bd[p] = bd[p - 1];
-          bi[p] = bi[p - 1];
-          p--;
-        }
-        bd[p] = s;
-        bi[p] = liveIds[n];
-      }
-    }
-    const truth = new Set(Array.from(bi).filter((x) => x >= 0));
-    for (const r of db2.query(query, 10)) if (truth.has(r.id)) hits++;
-  }
-  const recall = hits / (Q * 10);
-  assert(
-    recall >= 0.9,
-    `recall after compact stays high (got ${(recall * 100).toFixed(1)}%)`,
-  );
-  console.log(
-    `updates+deletes: compacted to ${db2.size} live; recall@10 after compact ${
-      (recall * 100).toFixed(1)
-    }%`,
-  );
-
-  dbFull.close();
-  dbFresh.close();
-  db2.close();
-}
-
-// =============================================================================
-// 3) Memory + recall + sub-linearity benchmark
-// =============================================================================
-function benchmark(): void {
-  section(
-    "3) Memory + recall + sub-linearity benchmark (HNSW over 1-bit codes)",
-  );
-
-  // Tweak these for a faster run or a wider sweep.
-  const DIM = 256; // 256-d -> 32-byte codes (the headline example)
-  const checkpoints = [1000, 2000, 4000, 8000, 16000, 32000];
-  const GROUP = 10; // well-separated groups of near-duplicates
-  const K = 10; // recall@K against the exact K nearest
-  const SPREAD = 0.1;
-  const Q = 200;
-  const EF = 150;
-
-  const rng = new RNG(12345);
-  const MAX_N = checkpoints[checkpoints.length - 1];
-  const numGroups = Math.ceil(MAX_N / GROUP);
-  console.log(
-    `dim=${DIM}, ${GROUP}-point groups, queries=${Q}, k=${K}, efSearch=${EF}`,
-  );
-  console.log("generating data ...");
-
-  const centers = new Float32Array(numGroups * DIM);
-  for (let g = 0; g < numGroups; g++) {
-    for (let d = 0; d < DIM; d++) centers[g * DIM + d] = rng.gauss();
-  }
-  const data = new Float32Array(MAX_N * DIM);
-  for (let i = 0; i < MAX_N; i++) {
-    const base = Math.floor(i / GROUP) * DIM;
-    const off = i * DIM;
-    for (let d = 0; d < DIM; d++) {
-      data[off + d] = centers[base + d] + SPREAD * rng.gauss();
-    }
-  }
-  // queries land in groups that exist at every checkpoint
-  const firstGroups = checkpoints[0] / GROUP;
-  const queries = new Float32Array(Q * DIM);
-  for (let q = 0; q < Q; q++) {
-    const base = rng.int(firstGroups) * DIM;
-    const off = q * DIM;
-    for (let d = 0; d < DIM; d++) {
-      queries[off + d] = centers[base + d] + SPREAD * rng.gauss();
-    }
-  }
-
-  const db = new VectorDatabase({
-    dbPath: ":memory:",
-    dim: DIM,
-    M: 16,
-    efConstruction: 100,
-    efSearch: EF,
-    seed: 999,
-  });
-
-  // Precompute data vector norms once — O(N·D) instead of O(Q·N·D).
-  const dataNorms = new Float64Array(MAX_N);
-  for (let i = 0; i < MAX_N; i++) {
-    let s = 0;
-    const off = i * DIM;
-    for (let d = 0; d < DIM; d++) s += data[off + d] * data[off + d];
-    dataNorms[i] = Math.sqrt(s) || 1;
-  }
-
-  function bruteForceTopK(n: number, qOff: number): Set<number> {
-    const bd = new Float64Array(K).fill(Infinity);
-    const bi = new Int32Array(K).fill(-1);
-    let qn = 0;
-    for (let d = 0; d < DIM; d++) qn += queries[qOff + d] * queries[qOff + d];
-    qn = Math.sqrt(qn) || 1;
-    for (let i = 0; i < n; i++) {
-      const off = i * DIM;
-      let dot = 0;
-      for (let d = 0; d < DIM; d++) dot += data[off + d] * queries[qOff + d];
-      const s = 1 - dot / (dataNorms[i] * qn);
-      if (s < bd[K - 1]) {
-        let p = K - 1;
-        while (p > 0 && bd[p - 1] > s) {
-          bd[p] = bd[p - 1];
-          bi[p] = bi[p - 1];
-          p--;
-        }
-        bd[p] = s;
-        bi[p] = i;
-      }
-    }
-    // Direct set from typed array filtered values.
-    const truth = new Set<number>();
-    for (let i = 0; i < K; i++) {
-      if (bi[i] >= 0) truth.add(bi[i]);
-    }
-    return truth;
-  }
-
-  type Row = {
-    n: number;
-    hnswUs: number;
-    comps: number;
-    bruteUs: number;
-    recall: number;
-    speedup: number;
-  };
-  const rows: Row[] = [];
-  let inserted = 0;
-
-  for (const cp of checkpoints) {
-    const tb = now();
-    for (; inserted < cp; inserted++) {
-      db.insert(inserted, data.subarray(inserted * DIM, inserted * DIM + DIM));
-    }
-    console.log(`  built N=${cp} (+${(now() - tb).toFixed(0)}ms)`);
-
-    for (let w = 0; w < 25; w++) {
-      const o = (w % Q) * DIM;
-      db.query(queries.subarray(o, o + DIM), K);
-    }
-
-    const hnswResults: number[][] = new Array(Q);
-    let comps = 0;
-    const th = now();
-    for (let qi = 0; qi < Q; qi++) {
-      const o = qi * DIM;
-      hnswResults[qi] = db.query(queries.subarray(o, o + DIM), K).map((
-        x: QueryResult,
-      ) => x.id);
-      comps += db.lastQueryDistanceComputations;
-    }
-    const hnswMs = now() - th;
-
-    const truths: Set<number>[] = new Array(Q);
-    const tbf = now();
-    for (let qi = 0; qi < Q; qi++) truths[qi] = bruteForceTopK(cp, qi * DIM);
-    const bruteMs = now() - tbf;
-
-    let hits = 0;
-    for (let qi = 0; qi < Q; qi++) {
-      for (const id of hnswResults[qi]) if (truths[qi].has(id)) hits++;
-    }
-
-    rows.push({
-      n: cp,
-      hnswUs: (hnswMs / Q) * 1000,
-      comps: comps / Q,
-      bruteUs: (bruteMs / Q) * 1000,
-      recall: hits / (Q * K),
-      speedup: bruteMs / hnswMs,
-    });
-  }
-
-  // ---- memory report ----
-  const st = db.storage;
-  const mb = (bytes: number) => (bytes / (1024 * 1024)).toFixed(2) + " MB";
-  console.log("");
-  console.log(
-    "Per-vector storage (vector payload only, excludes the HNSW graph):",
-  );
-  console.log(`  Float32 baseline     : ${st.float32BytesPerVector} B`);
-  console.log(
-    `  1-bit RaBitQ code    : ${st.codeBytesPerVector} B   (${
-      (st.float32BytesPerVector / st.codeBytesPerVector).toFixed(1)
-    }x smaller)`,
-  );
-  console.log(
-    `  code (kept)         : ${st.bytesPerVector} B   (${
-      st.compressionRatio.toFixed(1)
-    }x smaller overall)`,
-  );
-  console.log(
-    `  payload at N=${MAX_N}     : Float32 ${
-      mb(st.float32BytesPerVector * MAX_N)
-    }  vs  index ${mb(st.bytesPerVector * MAX_N)}`,
-  );
-
-  // ---- results table ----
-  console.log("");
-  console.log(
-    "     N | HNSW q (us) | dist comps | Brute q (us) | recall@10 | speedup",
-  );
-  console.log(
-    "-------+-------------+------------+--------------+-----------+--------",
-  );
-  for (const r of rows) {
-    console.log(
-      `${String(r.n).padStart(6)} | ${r.hnswUs.toFixed(1).padStart(11)} | ${
-        r.comps.toFixed(0).padStart(10)
-      } | ${
-        r.bruteUs
-          .toFixed(1)
-          .padStart(12)
-      } | ${(r.recall * 100).toFixed(1).padStart(8)}% | ${
-        r.speedup.toFixed(1).padStart(6)
-      }x`,
-    );
-  }
-
-  const mean = (xs: number[]) => xs.reduce((a, b) => a + b, 0) / xs.length;
-  const exponent = (xs: number[], ys: number[]) => {
-    const lx = xs.map(Math.log);
-    const ly = ys.map((y) => Math.log(Math.max(y, 1e-12)));
-    const mx = mean(lx);
-    const my = mean(ly);
-    let num = 0;
-    let den = 0;
-    for (let i = 0; i < lx.length; i++) {
-      num += (lx[i] - mx) * (ly[i] - my);
-      den += (lx[i] - mx) * (lx[i] - mx);
-    }
-    return num / den;
-  };
-
-  const ns = rows.map((r) => r.n);
-  const expComps = exponent(ns, rows.map((r) => r.comps));
-  const expBrute = exponent(ns, rows.map((r) => r.bruteUs));
-  const avgRecall = mean(rows.map((r) => r.recall));
-  const range = ns[ns.length - 1] / ns[0];
-
-  console.log("");
-  console.log(
-    `Empirical scaling over a ${range}x increase in N  (work ~ N^exponent):`,
-  );
-  console.log(
-    `  HNSW distance computations : N^${
-      expComps.toFixed(3)
-    }   <- sub-linear (target < 1)`,
-  );
-  console.log(
-    `  Brute-force query          : N^${
-      expBrute.toFixed(3)
-    }   (linear reference ~ 1.0)`,
-  );
-  console.log(
-    `  Mean recall@${K}            : ${(avgRecall * 100).toFixed(2)}%`,
-  );
-
-  const last = rows[rows.length - 1];
-  assert(
-    st.codeBytesPerVector === Math.ceil(DIM / 32) * 4,
-    "code size is ceil(dim/32) 32-bit words",
-  );
-  assert(
-    st.compressionRatio > 10,
-    `index must be far smaller than Float32 (got ${
-      st.compressionRatio.toFixed(1)
-    }x)`,
-  );
-  assert(
-    expComps < 0.8,
-    `HNSW distance-computation growth must be sub-linear (got N^${
-      expComps.toFixed(3)
-    })`,
-  );
-  assert(
-    avgRecall >= 0.9,
-    `mean recall must be >= 90% (got ${(avgRecall * 100).toFixed(1)}%)`,
-  );
-  assert(
-    last.hnswUs < last.bruteUs,
-    "HNSW must be faster than brute force at the largest N",
-  );
-  assert(
-    last.speedup > 1,
-    `HNSW must be faster than brute force at the largest N (got ${
-      last.speedup.toFixed(1)
-    }x)`,
-  );
-  console.log("\nMemory, sub-linearity, recall and speed-up checks OK");
-
-  db.close();
-}
-
-// =============================================================================
-// 4) Scalability is INTRINSIC, not cache-borne
-//
-// Codes and adjacency lists live in SQLite and are read on demand; the only
-// cache is SQLite's page cache (Store's cacheSizeMb), a pure speed layer. So
-// the engine's scalability must hold with that cache OFF: the STORAGE READS an
-// operation issues (Store.reads, surfaced per query as lastQueryStorageReads)
-// must be governed by the WORK the operation does — the distinct nodes it
-// visits, an HNSW property that grows ~log N — NOT by how many vectors already
-// exist. An implementation that leans on a cache to be fast betrays itself here:
-// with the cache disabled its per-operation reads track the whole working set
-// and climb with N. This is a STORAGE contract, so the guard lives in the
-// rabitq-hnsw suite.
-//
-// The witness is `db.lastQueryStorageReads` — node + neighbour-list fetches that
-// hit the database on the last query, counted below the cache so a warm cache
-// can never hide a bad access pattern. We build ONE store in blocks (cacheSizeMb
-// 0) and, at each checkpoint, measure the mean storage reads of a fixed query
-// set. Across an 8x increase in N the per-query read count must stay within a
-// flat band; the log–log growth exponent makes "does not grow with N" precise.
-// =============================================================================
-function testScalabilityWithoutCache(): void {
-  section("4) Scalability is intrinsic, not cache-borne (cacheSizeMb = 0)");
-
-  // A small dimension and modest graph degree keep the build cheap — this test
-  // is about the SHAPE of storage-reads vs N, not throughput, so the constants
-  // only need to be large enough for HNSW's structure to hold.
-  const DIM = 64;
-  const db = new VectorDatabase({
-    dbPath: ":memory:",
-    dim: DIM,
-    M: 8,
-    efConstruction: 40,
-    efSearch: 40,
-    seed: 4242,
-    cacheSizeMb: 0, // ALL caches off (page cache + code LRU) — honest reads
-  });
-
-  const rng = new RNG(20260628);
-  const vec = (): number[] => {
-    const v = new Array<number>(DIM);
-    for (let d = 0; d < DIM; d++) v[d] = rng.gauss();
-    return unit(v);
-  };
-
-  // A fixed set of probes, reused at every checkpoint so the only variable is N.
-  const Q = 40;
-  const probes: number[][] = [];
-  for (let q = 0; q < Q; q++) probes.push(vec());
-
-  const checkpoints = [1000, 2000, 4000, 8000]; // 8x range
-  const readsPerQuery: number[] = [];
-  let id = 0;
-  for (const cp of checkpoints) {
-    const tb = now();
-    for (; id < cp; id++) db.insert(id, vec());
-    // Mean storage reads over the fixed probe set — the cache-independent cost
-    // of ONE query at this N.
-    let reads = 0;
-    for (const p of probes) {
-      db.query(p, 10);
-      reads += db.lastQueryStorageReads;
-    }
-    readsPerQuery.push(reads / Q);
-    console.log(
-      `  N=${String(cp).padStart(6)}  +${(now() - tb).toFixed(0)}ms  ` +
-        `storage-reads/query ${(reads / Q).toFixed(1)}`,
-    );
-  }
-
-  // Compare the per-query read count at the LARGEST store against the smallest:
-  // if scaling leaned on the cache, the cache-disabled reads would rise with N
-  // (a deeper graph re-read from disk). The contract is that it stays within a
-  // small constant band — work set by the query, not by the corpus. Log–log
-  // growth exponent makes it precise.
-  const lx = checkpoints.map(Math.log);
-  const ly = readsPerQuery.map((y) => Math.log(Math.max(y, 1e-9)));
-  const mx = lx.reduce((a, c) => a + c, 0) / lx.length;
-  const my = ly.reduce((a, c) => a + c, 0) / ly.length;
-  let num = 0, den = 0;
-  for (let i = 0; i < lx.length; i++) {
-    num += (lx[i] - mx) * (ly[i] - my);
-    den += (lx[i] - mx) * (lx[i] - mx);
-  }
-  const exponent = num / den;
-  const first = readsPerQuery[0];
-  const last = readsPerQuery[readsPerQuery.length - 1];
-  console.log(
-    `  reads/query ${first.toFixed(1)} → ${last.toFixed(1)} over ${
-      checkpoints[checkpoints.length - 1] / checkpoints[0]
-    }x N · growth N^${exponent.toFixed(3)}`,
-  );
-
-  // PRIMARY guard — sub-linear in N: an O(1)-per-touch, log-N-touched query has
-  // storage-reads growing like log N (exponent ≈ 0); an implementation forced to
-  // re-read its working set from disk as N grows pushes this toward and past 1.
-  // Well under 0.5 proves the index does not lean on the cache to scale.
-  assert(
-    exponent < 0.5,
-    `cache-disabled storage-reads/query grew as N^${
-      exponent.toFixed(3)
-    } — the index must scale WITHOUT the page cache (reads set by the query's work, not the corpus size)`,
-  );
-  // SECONDARY — a flat band: the largest store costs no more than ~2.5x the
-  // smallest per query. Generous against HNSW's genuine log-N growth, strict
-  // enough to fail a regression whose reads track the corpus.
-  assert(
-    last < first * 2.5 + 8,
-    `cache-disabled reads/query climbed ${first.toFixed(1)} → ${
-      last.toFixed(1)
-    } as the store grew — expected a flat band (work set by the query, not the corpus)`,
-  );
-
-  db.close();
-  console.log("\nCache-independent scalability OK");
-}
-
-// ------------------------------------ main -----------------------------------
-function main(): void {
-  const start = now();
-  testCrudAndPersistence();
-  testCodeIO();
-  testCosine();
-  testNearDuplicateRecall();
-  testCosFactorBias();
-  testPersistence();
-  testCompact();
-  testScalabilityWithoutCache();
-  benchmark();
-  const dt = ((now() - start) / 1000).toFixed(1);
-  console.log("\n" + "=".repeat(72));
-  console.log(`ALL TESTS PASSED  (${passed} assertions, ${dt}s)`);
-  console.log("=".repeat(72));
-}
-
-try {
-  main();
-} catch (err) {
-  console.error(
-    "\n" + String(err && (err as Error).stack ? (err as Error).stack : err),
-  );
-  if (typeof process !== "undefined") process.exitCode = 1;
-}