@hviana/sema 0.1.4 → 0.1.6

package/test/35-ivf.test.mjs

@@ -0,0 +1,263 @@
+// 35-ivf.test.mjs — the partitioned (IVF) vector index, at its own layer.
+//
+// VectorDatabase is an adaptive IVF index over 1-bit RaBitQ codes in SQLite
+// (src/rabitq-ivf).  These tests verify the index's OWN contract, below the
+// Sema store:
+//
+//   1. CRUD          — insert/has/get/update/delete, id checking.
+//   2. Recall        — clustered data resonates to its own cluster; a stored
+//                      vector finds itself.
+//   3. Splits        — the index partitions past SPLIT_MAX and stays correct.
+//   4. Persistence   — close/reopen preserves entries, config, and results.
+//   5. Determinism   — same insert sequence ⇒ identical results (no RNG).
+//   6. Compaction    — tombstones are reclaimed; internal ids (the keysSince
+//                      watermark) survive.
+//   7. Scaling       — per-query STORAGE READS are bounded once the
+//                      collection is past its first splits: 2x the corpus
+//                      must not move the reads (the honest witness the old
+//                      graph index also swore to).
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { rmSync } from "node:fs";
+import { VectorDatabase } from "../dist/src/rabitq-ivf/src/index.js";
+
+// Deterministic PRNG for test vectors.
+function rng(seed) {
+  let s = seed >>> 0;
+  return () => {
+    s ^= s << 13;
+    s ^= s >>> 17;
+    s ^= s << 5;
+    s >>>= 0;
+    return s / 0x100000000;
+  };
+}
+
+/** A unit vector near `center` with per-coordinate jitter. */
+function near(center, jitter, r) {
+  const v = new Float32Array(center.length);
+  for (let i = 0; i < v.length; i++) {
+    v[i] = center[i] + (r() * 2 - 1) * jitter;
+  }
+  return v;
+}
+
+function randVec(dim, r) {
+  const v = new Float32Array(dim);
+  for (let i = 0; i < dim; i++) v[i] = r() * 2 - 1;
+  return v;
+}
+
+test("1: CRUD — insert, has, get, update, delete", () => {
+  const db = new VectorDatabase({ dbPath: ":memory:", dim: 64 });
+  const r = rng(1);
+  const v1 = randVec(64, r);
+  const v2 = randVec(64, r);
+
+  db.insert(10, v1);
+  assert.equal(db.size, 1);
+  assert.ok(db.has(10));
+  assert.ok(!db.has(11));
+  assert.throws(() => db.insert(10, v2), /already exists/);
+  assert.throws(() => db.insert(1.5, v2), /integer/);
+
+  const code = db.get(10);
+  assert.ok(code instanceof Uint32Array);
+
+  // update to a genuinely different vector: same ext still resolves
+  db.update(10, v2);
+  assert.equal(db.size, 1);
+  assert.ok(db.has(10));
+
+  // byte-identical update is a no-op (no new tombstone)
+  const phys = db.physicalSize;
+  db.update(10, v2);
+  assert.equal(db.physicalSize, phys);
+
+  assert.ok(db.delete(10));
+  assert.ok(!db.has(10));
+  assert.equal(db.size, 0);
+  assert.ok(!db.delete(10));
+  db.close();
+});
+
+test("2: recall — clustered data finds its own cluster; self-recall", () => {
+  const dim = 128;
+  const db = new VectorDatabase({ dbPath: ":memory:", dim });
+  const r = rng(2);
+  // 8 well-separated centers, 100 members each.
+  const centers = [];
+  for (let c = 0; c < 8; c++) centers.push(randVec(dim, r));
+  const vecs = new Map();
+  let id = 0;
+  const batch = [];
+  for (let c = 0; c < 8; c++) {
+    for (let m = 0; m < 100; m++) {
+      const v = near(centers[c], 0.15, r);
+      vecs.set(id, { v, c });
+      batch.push({ id, vector: v });
+      id++;
+    }
+  }
+  db.upsertMany(batch);
+  assert.equal(db.size, 800);
+
+  // Query each center: top-10 should come from that center's members.
+  for (let c = 0; c < 8; c++) {
+    const hits = db.query(centers[c], 10);
+    assert.equal(hits.length, 10);
+    let own = 0;
+    for (const h of hits) if (vecs.get(h.id).c === c) own++;
+    assert.ok(own >= 8, `center ${c}: only ${own}/10 hits from own cluster`);
+  }
+
+  // Self-recall: a stored vector should find its own ext in the top 5.
+  let self = 0;
+  for (let probe = 0; probe < 100; probe += 7) {
+    const hits = db.query(vecs.get(probe).v, 5);
+    if (hits.some((h) => h.id === probe)) self++;
+  }
+  assert.ok(self >= 12, `self-recall ${self}/15 too low`);
+  db.close();
+});
+
+test("3: splits — collections past SPLIT_MAX partition and stay correct", () => {
+  const dim = 32;
+  const db = new VectorDatabase({ dbPath: ":memory:", dim });
+  const r = rng(3);
+  const N = 10_000; // > 2 × SPLIT_MAX (4096) — forces several splits
+  const batch = [];
+  for (let i = 0; i < N; i++) batch.push({ id: i, vector: randVec(dim, r) });
+  db.upsertMany(batch);
+  assert.equal(db.size, N);
+  assert.ok(
+    db.clusterCount >= 3,
+    `expected several clusters after ${N} inserts, got ${db.clusterCount}`,
+  );
+  // Every ext still resolves and queries still return k results.
+  assert.ok(db.has(0) && db.has(N - 1) && db.has(N >> 1));
+  const hits = db.query(randVec(dim, r), 20);
+  assert.equal(hits.length, 20);
+  db.close();
+});
+
+test("4: persistence — close/reopen preserves entries and results", (t) => {
+  const path = `/tmp/ivf-test-${process.pid}.vec`;
+  t.after(() => {
+    for (const s of ["", "-wal", "-shm"]) {
+      try {
+        rmSync(path + s);
+      } catch {}
+    }
+  });
+  const dim = 64;
+  const r = rng(4);
+  const vecs = [];
+  for (let i = 0; i < 500; i++) vecs.push(randVec(dim, r));
+
+  let db = new VectorDatabase({ dbPath: path, dim });
+  db.upsertMany(vecs.map((v, i) => ({ id: i, vector: v })));
+  const probe = vecs[123];
+  const before = db.query(probe, 10);
+  db.close();
+
+  db = new VectorDatabase({ dbPath: path });
+  assert.equal(db.dim, dim);
+  assert.equal(db.size, 500);
+  const after = db.query(probe, 10);
+  assert.deepEqual(after, before);
+  db.close();
+});
+
+test("5: determinism — same insert sequence, identical results", () => {
+  const dim = 64;
+  const r1 = rng(5), r2 = rng(5);
+  const a = new VectorDatabase({ dbPath: ":memory:", dim });
+  const b = new VectorDatabase({ dbPath: ":memory:", dim });
+  const N = 6000; // crosses a split
+  const batchA = [], batchB = [];
+  for (let i = 0; i < N; i++) {
+    batchA.push({ id: i, vector: randVec(dim, r1) });
+    batchB.push({ id: i, vector: randVec(dim, r2) });
+  }
+  a.upsertMany(batchA);
+  b.upsertMany(batchB);
+  assert.equal(a.clusterCount, b.clusterCount);
+  const probe = randVec(dim, r1);
+  assert.deepEqual(a.query(probe, 20), b.query(probe, 20));
+  a.close();
+  b.close();
+});
+
+test("6: compaction — tombstones reclaimed, watermark ids preserved", () => {
+  const dim = 32;
+  const db = new VectorDatabase({ dbPath: ":memory:", dim });
+  const r = rng(6);
+  const N = 2000;
+  db.upsertMany(
+    Array.from({ length: N }, (_, i) => ({ id: i, vector: randVec(dim, r) })),
+  );
+
+  // Watermark before deletion.
+  let maxInternal = 0;
+  for (const { internal } of db.keysSince(0)) {
+    maxInternal = Math.max(maxInternal, internal);
+  }
+
+  // Delete every third entry.
+  const dead = [];
+  for (let i = 0; i < N; i += 3) dead.push(i);
+  assert.equal(db.deleteMany(dead), dead.length);
+  assert.equal(db.size, N - dead.length);
+  assert.ok(db.physicalSize > db.size);
+
+  db.compact();
+  assert.equal(db.physicalSize, db.size);
+  for (const i of dead) assert.ok(!db.has(i));
+  assert.ok(db.has(1) && db.has(2));
+
+  // Internal ids survive compaction: entries added AFTER the old watermark
+  // are exactly the new inserts.
+  db.upsertMany([{ id: 99999, vector: randVec(dim, r) }]);
+  const fresh = [...db.keysSince(maxInternal)];
+  assert.equal(fresh.length, 1);
+  assert.equal(fresh[0].ext, 99999);
+  db.close();
+});
+
+test("7: scaling — per-query storage reads are bounded past the first splits", () => {
+  const dim = 32;
+  // cacheSizeMb 0: no page cache to hide behind; `reads` counts blob rows.
+  const db = new VectorDatabase({ dbPath: ":memory:", dim, cacheSizeMb: 0 });
+  const r = rng(7);
+  const probe = randVec(dim, r);
+  const readsNow = () => {
+    db.query(probe, 10);
+    return db.lastQueryStorageReads;
+  };
+
+  const grow = (n) => {
+    const batch = [];
+    for (let i = 0; i < n; i++) {
+      batch.push({ id: db.size + i, vector: randVec(dim, r) });
+    }
+    db.upsertMany(batch);
+  };
+
+  // Past ~nprobe × SPLIT_MAX entries the probed set is a fixed number of
+  // bounded clusters — reads must plateau.
+  grow(80_000);
+  const small = readsNow();
+  grow(80_000); // 2x the corpus
+  const large = readsNow();
+  console.log(
+    `    ivf reads for one query: N=80k → ${small}, N=160k → ${large}`,
+  );
+  assert.ok(
+    large <= small * 1.5 + 8,
+    `reads grew ${small} → ${large} when the corpus doubled — probes are ` +
+      `not bounded`,
+  );
+  db.close();
+});

package/test/36-bloom.test.mjs

@@ -0,0 +1,123 @@
+// 36-bloom.test.mjs — the negative dedup filter (store-sqlite's Bloom layer).
+//
+// The filter answers "this content hash is definitely not in the node table"
+// in RAM, sparing the training hot path its dominant SQLite probe (profiled:
+// 38.7% of ingest wall at 2.9M nodes).  Its ONE hazard is a false negative,
+// which would mint a duplicate node and silently break hash-consing.  These
+// tests pin the invariant on the three lifecycle paths:
+//
+//   1. live session   — content minted then re-probed dedups (filter sees
+//                       every insert synchronously);
+//   2. clean close    — reopen loads the persisted filter (or rebuilds) and
+//                       dedup still holds;
+//   3. CRASH          — a process that exits without close() leaves no
+//                       persisted filter (or a stale one); reopen must
+//                       rebuild/top-up from the table and STILL dedup all
+//                       previously-minted content, with zero duplicates.
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { rmSync } from "node:fs";
+import { SQliteStore } from "../dist/src/store-sqlite.js";
+
+const cleanup = (path) => {
+  for (
+    const s of [
+      ".sqlite",
+      ".sqlite-wal",
+      ".sqlite-shm",
+      ".content.vec",
+      ".content.vec-wal",
+      ".content.vec-shm",
+      ".halo.vec",
+      ".halo.vec-wal",
+      ".halo.vec-shm",
+    ]
+  ) {
+    try {
+      rmSync(path + s);
+    } catch {}
+  }
+};
+
+test("live session: re-probing minted content dedups (no false negatives)", async () => {
+  const s = new SQliteStore({ path: ":memory:", D: 64 });
+  const enc = new TextEncoder();
+  const g = new Float32Array(64).fill(0.1);
+  const ids = [];
+  for (let i = 0; i < 300; i++) {
+    ids.push(await s.putLeaf(enc.encode("bloom-live-" + i), g));
+  }
+  for (let i = 0; i < 300; i++) {
+    assert.equal(await s.putLeaf(enc.encode("bloom-live-" + i), g), ids[i]);
+  }
+  // Novel probes are answered by the filter without SQLite (the point).
+  const skipsBefore = s.bloomSkips;
+  s.findLeaf(enc.encode("definitely-not-present-xyzzy"));
+  assert.ok(s.bloomSkips >= skipsBefore, "bloomSkips must be observable");
+  await s.close();
+});
+
+test("clean close: reopen dedups against the persisted/rebuilt filter", async (t) => {
+  const path = `/tmp/bloom-clean-${process.pid}`;
+  t.after(() => cleanup(path));
+  const enc = new TextEncoder();
+  const g = new Float32Array(64).fill(0.1);
+
+  let s = new SQliteStore({ path, D: 64 });
+  const ids = [];
+  for (let i = 0; i < 300; i++) {
+    ids.push(await s.putLeaf(enc.encode("bloom-clean-" + i), g));
+  }
+  await s.close();
+
+  s = new SQliteStore({ path, D: 64 });
+  const before = s.nodeCount();
+  for (let i = 0; i < 300; i++) {
+    const id = await s.putLeaf(enc.encode("bloom-clean-" + i), g);
+    assert.equal(id, ids[i], `content ${i} re-minted after clean reopen`);
+  }
+  assert.equal(s.nodeCount(), before, "no duplicate mints after clean reopen");
+  await s.close();
+});
+
+test("crash: reopen after an unclean exit still dedups everything", (t) => {
+  const path = `/tmp/bloom-crash-${process.pid}`;
+  t.after(() => cleanup(path));
+
+  // Phase 1 in a SUBPROCESS that exits WITHOUT close() — a real crash as far
+  // as the store is concerned (WAL holds the rows; no filter was persisted).
+  const phase1 = `
+    import("${process.cwd()}/dist/src/store-sqlite.js").then(async ({ SQliteStore }) => {
+      const s = new SQliteStore({ path: "${path}", D: 64 });
+      const enc = new TextEncoder();
+      const g = new Float32Array(64).fill(0.1);
+      for (let i = 0; i < 400; i++) {
+        await s.putLeaf(enc.encode("bloom-crash-" + i), g);
+      }
+      s.commit();
+      process.exit(0); // no close()
+    });`;
+  execFileSync(process.execPath, ["-e", phase1], { stdio: "pipe" });
+
+  // Phase 2 in a second subprocess: every previously-minted content must
+  // resolve to an existing id — zero duplicates.
+  const phase2 = `
+    import("${process.cwd()}/dist/src/store-sqlite.js").then(async ({ SQliteStore }) => {
+      const s = new SQliteStore({ path: "${path}", D: 64 });
+      await s.size();
+      const enc = new TextEncoder();
+      const g = new Float32Array(64).fill(0.1);
+      const before = s.nodeCount();
+      for (let i = 0; i < 400; i++) {
+        const id = await s.putLeaf(enc.encode("bloom-crash-" + i), g);
+        if (id >= before) { console.error("DUP " + i); process.exit(1); }
+      }
+      if (s.nodeCount() !== before) { console.error("GREW"); process.exit(1); }
+      await s.close();
+      process.exit(0);
+    });`;
+  execFileSync(process.execPath, ["-e", phase2], { stdio: "pipe" });
+  assert.ok(true);
+});

package/dist/src/rabitq-hnsw/src/database.d.ts

@@ -1,200 +0,0 @@
-/** External ids are integers only. */
-export type ExternalId = number;
-export interface DatabaseOptions {
-    /** Path to the SQLite database file (use ":memory:" for a transient store). */
-    dbPath: string;
-    /** Vector dimensionality. Required for a new database; ignored when reopening. */
-    dim?: number;
-    /** Max neighbours per node on layers >= 1 (layer 0 uses 2*M). Default 16. */
-    M?: number;
-    /** Candidate list size during construction. Default 200. */
-    efConstruction?: number;
-    /** Candidate list size during search. Default 100. Tunable at runtime. */
-    efSearch?: number;
-    /** Bits used to scalar-quantise the query for the RaBitQ estimator.
-     *  Query-side only (stored codes and the graph are independent of it), so it
-     *  is tunable at reopen, like efSearch.  Default 8: 4 bits cannot resolve a
-     *  tight cluster's residual and costs ~18% self-recall there (test 2a); 8
-     *  bits restores it for the price of a Uint16 per-query LUT. */
-    queryBits?: number;
-    /** Number of sign-flip + Hadamard rounds in the random rotation. Default 3. */
-    rotationRounds?: number;
-    /** Seed for the rotation and graph levels. Default fixed. */
-    seed?: number;
-    /** Optional centroid the vectors are centered by before quantisation. */
-    centroid?: ArrayLike<number>;
-    /**
-     * RAM cache size in MiB -- the single memory knob. It sizes SQLite's page
-     * cache AND the immutable-code LRU (whose entry-capacity is derived from this
-     * budget and the code size — there is no second knob). Both are purely speed
-     * enhancements: correctness and the per-operation storage-read count are
-     * identical with it off. Pass 0 to run with essentially no cache. Default 64.
-     * Not persisted; set per open.
-     */
-    cacheSizeMb?: number;
-}
-export interface QueryResult {
-    id: ExternalId;
-    /** Estimated cosine distance (1 - cosine). */
-    distance: number;
-}
-export interface StorageStats {
-    /** Bytes a single Float32 copy of the vector would take. */
-    float32BytesPerVector: number;
-    /** Bytes of 1-bit code kept per vector (on disk). */
-    codeBytesPerVector: number;
-    /** Bytes kept per vector (just the code; excludes the graph adjacency). */
-    bytesPerVector: number;
-    /** float32BytesPerVector / bytesPerVector. */
-    compressionRatio: number;
-}
-/**
- * A persistent vector database: an HNSW graph over 1-bit RaBitQ codes (cosine),
- * stored entirely in SQLite at `dbPath`. The graph IS the database -- there is no
- * load/save step and no in-RAM copy of the data, so resident memory stays flat as
- * the collection grows and the store survives process restarts. Reopening the
- * same path restores the exact configuration (the rotation is regenerated from
- * the persisted seed), so the codes already on disk remain valid.
- *
- * External ids are integers. The original float vectors are never retained --
- * only the sign codes -- so a 256-d vector costs 32 bytes instead of 1024. `get`
- * returns the stored code, which can be fed straight back into
- * `insert`/`update`/`query`.
- */
-export declare class VectorDatabase {
-    readonly dim: number;
-    private readonly store;
-    private readonly quantizer;
-    private readonly index;
-    private readonly codeWords;
-    constructor(options: DatabaseOptions);
-    /** Number of live (non-deleted) vectors. */
-    get size(): number;
-    /**
-     * Physical node count including tombstones from deletes/updates. When it grows
-     * well beyond `size`, call `compact()` to reclaim the space on disk.
-     */
-    get physicalSize(): number;
-    get efSearch(): number;
-    set efSearch(value: number);
-    /** Distance computations performed during the most recent query. */
-    get lastQueryDistanceComputations(): number;
-    /**
-     * Storage row reads issued by the most recent query. This is the honest,
-     * cache-independent scalability metric: it counts every node/neighbour fetch
-     * that hit the database, so disabling the cache cannot hide a bad access pattern.
-     */
-    get lastQueryStorageReads(): number;
-    /** Per-vector storage cost of the index versus a Float32 baseline. */
-    get storage(): StorageStats;
-    has(id: ExternalId): boolean;
-    /** Stream every live external id (bounded memory). */
-    keys(): IterableIterator<ExternalId>;
-    /**
-     * Stream live entries whose INTERNAL id is > `after`, as
-     * {ext, internal} pairs in internal-id order.  Internal ids are assigned
-     * monotonically at insert and preserved by {@link compact}, so the largest
-     * internal id a caller has seen is a durable incremental watermark: a later
-     * call with it yields exactly the entries added since.
-     */
-    keysSince(after: number): IterableIterator<{
-        ext: ExternalId;
-        internal: number;
-    }>;
-    private checkId;
-    /**
-     * Convert a value to code bytes, selecting by length:
-     *  - `codeWords` elements -> an existing 1-bit code (e.g. from `get()`)
-     *  - otherwise            -> a raw `dim`-vector, encoded first
-     * `codeWords < dim` for any dim >= 2, so the two never collide.
-     */
-    private toCodeBytes;
-    /**
-     * Create. Accepts a raw `dim`-vector or a 1-bit code (`codeWords` words),
-     * detected by length. Throws if the id already exists (use `update`/`upsert`).
-     */
-    insert(id: ExternalId, value: ArrayLike<number>): void;
-    /** Insert with the caller owning the transaction (used by {@link upsertMany}).
-     *  `ef` optionally narrows the construction beam for this one vector (a
-     *  caller-declared cheap entry, e.g. a reach-only interior); omitted means
-     *  the index's configured efConstruction. */
-    private insertCore;
-    upsert(id: ExternalId, value: ArrayLike<number>): void;
-    /**
-     * Upsert many vectors under ONE transaction. The HNSW build touches the store
-     * on every wired edge, so a transaction per vector is one WAL commit per vector
-     * — which dominates a bulk load on disk. Wrapping the whole batch in a single
-     * transaction coalesces those commits into one while leaving the graph and its
-     * result identical (reads on the connection still see the uncommitted rows).
-     *
-     * This is purely about commit batching; it changes nothing about the storage
-     * model. Codes still live only in SQLite and are read on demand — the
-     * cache-independent per-operation read count is unchanged — so a batched load
-     * scales exactly as the per-item path does, just with far fewer fsyncs. A throw
-     * rolls the whole batch back, so the caller treats it as the per-item path on
-     * failure.
-     */
-    upsertMany(entries: Array<{
-        id: ExternalId;
-        vector: ArrayLike<number>;
-        ef?: number;
-    }>): void;
-    /**
-     * Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. The
-     * original vector is not retained; the code round-trips into insert/update/query.
-     */
-    get(id: ExternalId): Uint32Array | null;
-    /**
-     * Update the vector bound to an id (raw vector or code, detected by length).
-     * The previous node is tombstoned and a fresh one inserted. Throws if absent.
-     */
-    update(id: ExternalId, value: ArrayLike<number>): void;
-    /** Update with the caller owning the transaction (used by {@link upsertMany}). */
-    private updateCore;
-    /** Update when the live internal node id is already known.
-     *
-     *  Skips the write entirely when the new code equals the stored one — the
-     *  update would tombstone the node and replay a full graph insert for a
-     *  byte-identical result. This is not a corner case: content-addressed
-     *  callers re-upsert unchanged vectors wholesale after a restart (the same
-     *  content always encodes to the same code), and each such no-op otherwise
-     *  costs a tombstone (permanent routing/disk overhead until compaction)
-     *  plus an O(ef·log N) reinsert. */
-    private updateAt;
-    /**
-     * Delete many ids under ONE transaction — same commit-coalescing rationale
-     * as {@link upsertMany}: a tombstone per implicit transaction is one WAL
-     * commit per id, which dominates a bulk prune. Absent ids are skipped.
-     * Returns the number of vectors actually removed.
-     */
-    deleteMany(ids: ExternalId[]): number;
-    /** Delete the vector bound to an id. Returns false if absent. */
-    delete(id: ExternalId): boolean;
-    /**
-     * Pre-fill the RAM caches (codes + neighbour lists) with sequential table
-     * scans, up to their budget-derived caps.  Optional and purely a latency
-     * optimisation: a cold session otherwise pays the same warming through
-     * random point reads over its first minutes.  Call once after open on a
-     * session that will do sustained inserts/queries.  Returns rows warmed;
-     * a 0-budget database returns 0 immediately.
-     */
-    warmCache(): number;
-    /**
-     * Rebuild the graph from the live codes only, dropping tombstones and returning
-     * the freed pages to the filesystem. Lossless -- equivalent to the original
-     * build -- and needs no original vectors.
-     */
-    compact(): void;
-    /**
-     * k-NN search. The argument's length selects the mode:
-     *  - `dim` elements       -> a raw vector (accurate 4-bit-query estimator)
-     *  - `codeWords` elements -> a 1-bit code (e.g. `get(id)`), by sign-bit Hamming
-     * Detection is by length, so a code is recognised whether it is a Uint32Array
-     * or a plain number[]. `codeWords < dim` for any dim >= 2, so they never collide.
-     */
-    query(query: ArrayLike<number>, k?: number, opts?: {
-        ef?: number;
-    }): QueryResult[];
-    /** Close the underlying database. The instance must not be used afterwards. */
-    close(): void;
-}