@hviana/sema 0.1.4 → 0.1.6

package/src/config.ts

@@ -39,18 +39,9 @@ export interface AlphabetConfig {
 
 export interface StoreConfig {
   minHaloMass: number;
-  m: number;
-  efConstruction: number;
-  /** Construction budget for REACH-ONLY interior gists — the ~90% of content
-   *  index inserts that exist so a partial query can resonate a sub-region
-   *  and climb, never as dedup targets.  Deposit roots and halo-bearing
-   *  targets always build with the full `efConstruction`.  A smaller budget
-   *  here is the one deliberate speed-for-quality trade in ingestion (an
-   *  interior's layer-0 wiring is built from a narrower candidate beam);
-   *  the recall suite (partial recall, multi-topic attention, counterfactual
-   *  anchoring) is the gate for its value.  Set equal to `efConstruction`
-   *  to disable the trade. */
-  efConstructionInterior: number;
+  /** Query breadth of the IVF vector indices: clusters probed per query =
+   *  ceil(efSearch / 4).  Inserts have no quality knob — the partitioned
+   *  index routes and appends, so ingestion cost is flat by construction. */
   efSearch: number;
   /** Compact the in-memory vector indices after this many vectors are written.
    *  Compaction rebuilds an index from its live codes to reclaim the slots left
@@ -58,7 +49,7 @@ export interface StoreConfig {
    *  (not on a flush count that goes quiet during repeat-heavy training) keeps
    *  the index dense and query cost bounded. */
   compactEveryNWrites: number;
-  /** Over-fetch factor for HNSW queries (ANN recall cushion). */
+  /** Over-fetch factor for vector-index queries (ANN recall cushion). */
   overfetch: number;
   /** Combined buffered-write ceiling before a flush of both vector indices
    *  (content + halo). Higher ⇒ fewer, larger flushes into the in-memory
@@ -91,14 +82,20 @@ export interface StoreConfig {
    *  round-trips through the quantizer. An eviction or a reopen reads the
    *  2-bit row — the fidelity every cross-session consumer already gets. */
   haloCacheBytes: number;
-  /** Size, in MiB, of each `rabitq-hnsw` `VectorDatabase`'s memory budget
-   *  (forwarded as its `cacheSizeMb`).  It is the index's SINGLE memory knob: it
-   *  sizes both SQLite's page cache and the derived immutable-code LRU.  A PURE
-   *  latency optimisation — the index reads codes from SQLite on demand, so its
-   *  correctness and its asymptotic per-operation storage-read count are identical
-   *  with the budget at 0.  Exposed so a scaling test can set it to 0 and measure
+  /** Size, in MiB, of each `rabitq-ivf` `VectorDatabase`'s memory budget
+   *  (forwarded as its `cacheSizeMb` — its SQLite page cache).  A PURE latency
+   *  optimisation — the index reads chunk blobs from SQLite on demand, so its
+   *  correctness and its per-operation storage-read count are identical with
+   *  the budget at 0.  Exposed so a scaling test can set it to 0 and measure
    *  the honest, cache-independent cost. */
   vectorCacheMb: number;
+  /** Size, in MiB, of the MAIN DAG database's SQLite page cache.  The node /
+   *  kid / edge / contain tables serve millions of point probes per training
+   *  session (content-addressed findLeaf/findBranch, parent probes, contain
+   *  appends); SQLite's default cache (~2 MiB) thrashes once the DB outgrows
+   *  it, so every probe pays a file read.  A PURE latency knob — correctness
+   *  and result identical at any value. */
+  sqliteCacheMb: number;
   /** Max entries in the skipped-interior LRU set.  Interiors that
    *  {@link Store.indexSubtree} has already visited (indexed or skipped) are
    *  remembered here so subsequent calls prune their subtrees.  Session-local
@@ -151,9 +148,6 @@ export const DEFAULT_CONFIG: MindConfig = {
   },
   store: {
     minHaloMass: 1,
-    m: 8,
-    efConstruction: 64,
-    efConstructionInterior: 16,
     efSearch: 64,
     compactEveryNWrites: 50_000,
     overfetch: 4,
@@ -165,6 +159,7 @@ export const DEFAULT_CONFIG: MindConfig = {
     pendingGistBytes: 16_000_000,
     haloCacheBytes: 16_000_000,
     vectorCacheMb: 64,
+    sqliteCacheMb: 64,
     coveredIdsMax: 100_000,
     chainCacheBytes: 16_000_000,
   },
@@ -194,11 +189,6 @@ export function resolveConfig(opts: Partial<MindConfig> = {}): MindConfig {
     },
     store: {
       minHaloMass: opts.store?.minHaloMass ?? DEFAULT_CONFIG.store.minHaloMass,
-      m: opts.store?.m ?? DEFAULT_CONFIG.store.m,
-      efConstruction: opts.store?.efConstruction ??
-        DEFAULT_CONFIG.store.efConstruction,
-      efConstructionInterior: opts.store?.efConstructionInterior ??
-        DEFAULT_CONFIG.store.efConstructionInterior,
       efSearch: opts.store?.efSearch ?? DEFAULT_CONFIG.store.efSearch,
       compactEveryNWrites: opts.store?.compactEveryNWrites ??
         DEFAULT_CONFIG.store.compactEveryNWrites,
@@ -218,6 +208,8 @@ export function resolveConfig(opts: Partial<MindConfig> = {}): MindConfig {
         DEFAULT_CONFIG.store.haloCacheBytes,
       vectorCacheMb: opts.store?.vectorCacheMb ??
         DEFAULT_CONFIG.store.vectorCacheMb,
+      sqliteCacheMb: opts.store?.sqliteCacheMb ??
+        DEFAULT_CONFIG.store.sqliteCacheMb,
       coveredIdsMax: opts.store?.coveredIdsMax ??
         DEFAULT_CONFIG.store.coveredIdsMax,
       chainCacheBytes: opts.store?.chainCacheBytes ??

package/src/index.ts

@@ -13,26 +13,21 @@ export * from "./store-sqlite.js";
 export * from "./config.js";
 export * from "./extension.js";
 export * from "./ingest-cache.js";
-// rabitq-hnsw is re-exported selectively: its `Store`, `NodeRec`, and
-// `StoreConfig` are internal names that collide with sema's own top-level
-// `store.js` / `config.js` exports, and sema code that needs the rabitq ones
-// imports them from the subpath directly. Re-export the public vector-DB
-// surface under the sema root, omitting the three colliding names.
+// rabitq-ivf: the partitioned (IVF) vector index over 1-bit RaBitQ codes.
 export {
-  Heap,
-  HnswIndex,
+  IvfIndex,
   Prng,
   RaBitQuantizer,
   VectorDatabase,
-} from "./rabitq-hnsw/src/index.js";
+} from "./rabitq-ivf/src/index.js";
 export type {
   DatabaseOptions,
   ExternalId,
-  HnswParams,
-  KnnHit,
+  IvfConfig,
+  IvfHit,
   QueryContext,
   QueryResult,
   RaBitQOptions,
   StorageStats,
-} from "./rabitq-hnsw/src/index.js";
+} from "./rabitq-ivf/src/index.js";
 export * from "./derive/src/index.js";

package/src/rabitq-ivf/README.md

@@ -0,0 +1,56 @@
+# rabitq-ivf
+
+A small TypeScript library for **approximate nearest-neighbour search** using an
+**adaptive partitioned (IVF) index** over **1-bit RaBitQ codes**, with
+**cosine** distance. The entire index lives in **SQLite** — the durable copy of
+every code is on disk, resident memory is bounded (the pivot table plus a capped
+page cache, never the whole collection), and the index survives process
+restarts.
+
+The vectors are stored **only** as 1-bit RaBitQ codes — the original vectors are
+never kept — so the index is both fast _and_ tiny on disk:
+
+```
+a 1024-d vector:
+  Float32             : 1024 × 4 bytes = 4096 bytes
+  RaBitQ 1-bit (code) : 1024 × 1 bit   = 128 bytes    (32× smaller)
+```
+
+The code _is_ the entire per-vector representation. `get` returns the bare code,
+and `insert`, `update`, and `query` all accept either a raw vector or a code —
+detected by length.
+
+## The index
+
+- **Clusters, not a graph.** The collection is partitioned into clusters, each
+  with a binary pivot code and its member codes packed in fixed-size chunk
+  blobs.
+- **Insert = route + append.** Find the nearest pivot (one linear Hamming scan
+  of the RAM-resident pivot table) and append to that cluster's tail chunk. No
+  beam search, no neighbour rewiring — per-insert cost is essentially flat in
+  collection size.
+- **Query = probe + scan.** Rank all pivots with the accurate RaBitQ estimator,
+  scan the `ceil(efSearch/4)` nearest clusters with the same estimator, keep the
+  top k. Per-query storage reads are bounded by nprobe × chunks-per-cluster —
+  constant once the collection has split.
+- **Adaptive, deterministic splits.** A cluster reaching 4096 entries is
+  median-split on the margin between two farthest-point seeds (two exact halves,
+  cascade-proof), and both halves get fresh majority-bit pivots. There is no
+  RNG: the index is a pure function of the insertion sequence.
+- **Same durability discipline as the rest of Sema**: WAL, batched caller-owned
+  transactions (`upsertMany`), 1 KiB pages, 64 MiB WAL autocheckpoint.
+
+## Usage
+
+```ts
+import { VectorDatabase } from "./src/index.js";
+
+const db = new VectorDatabase({ dbPath: "vectors.db", dim: 1024 });
+db.upsertMany([{ id: 1, vector: v1 }, { id: 2, vector: v2 }]);
+const hits = db.query(q, 10); // [{ id, distance }]
+db.close();
+```
+
+`lastQueryStorageReads` reports the cache-independent storage-read count of the
+most recent query — the honest scalability witness the test suite asserts on
+(`test/35-ivf.test.mjs` at the repo root).

package/src/rabitq-ivf/src/database.ts

@@ -0,0 +1,276 @@
+import { IvfHit, IvfIndex } from "./ivf.js";
+
+/** External ids are integers only. */
+export type ExternalId = number;
+
+export interface DatabaseOptions {
+  /** Path to the SQLite database file (use ":memory:" for a transient store). */
+  dbPath: string;
+  /** Vector dimensionality. Required for a new database; ignored when reopening. */
+  dim?: number;
+  /** Query breadth: clusters probed per query = ceil(efSearch / 4).  Default
+   *  64 (16 probes). Tunable at runtime and at reopen. */
+  efSearch?: number;
+  /** Bits used to scalar-quantise the query for the RaBitQ estimator.
+   *  Query-side only (stored codes are independent of it), so it is tunable
+   *  at reopen, like efSearch.  Default 8. */
+  queryBits?: number;
+  /** Number of sign-flip + Hadamard rounds in the random rotation. Default 3. */
+  rotationRounds?: number;
+  /** Seed for the rotation. Default fixed. */
+  seed?: number;
+  /** Optional centroid the vectors are centered by before quantisation. */
+  centroid?: ArrayLike<number>;
+  /** SQLite page-cache budget in MiB.  Purely a latency knob: correctness and
+   *  the per-operation storage-read count are identical with it at 0. */
+  cacheSizeMb?: number;
+}
+
+export interface QueryResult {
+  id: ExternalId;
+  /** Estimated cosine distance (1 - cosine). */
+  distance: number;
+}
+
+export interface StorageStats {
+  float32BytesPerVector: number;
+  codeBytesPerVector: number;
+  bytesPerVector: number;
+  compressionRatio: number;
+}
+
+/**
+ * A persistent vector database: an adaptive PARTITIONED (IVF) index over
+ * 1-bit RaBitQ codes (cosine), stored entirely in SQLite at `dbPath`.  See
+ * ivf.ts for the index design.  The original float vectors are never
+ * retained — only the sign codes — so a 1024-d vector costs 128 bytes.
+ *
+ * Inserting is route-and-append: cost is essentially FLAT in collection size
+ * (one RAM scan of the pivot table + one chunk append — no per-insert graph
+ * walk).  Query cost is bounded by nprobe × cluster size — constant in N
+ * once the collection is past its first splits.
+ */
+export class VectorDatabase {
+  private readonly index: IvfIndex;
+
+  constructor(options: DatabaseOptions) {
+    if (
+      !options || typeof options.dbPath !== "string" ||
+      options.dbPath.length === 0
+    ) {
+      throw new Error("DatabaseOptions.dbPath (string) is required");
+    }
+    this.index = new IvfIndex(options.dbPath, options);
+  }
+
+  get dim(): number {
+    return this.index.dim;
+  }
+
+  /** Number of live (non-deleted) vectors. */
+  get size(): number {
+    return this.index.size;
+  }
+
+  /** Physical slot count including tombstones from deletes/updates. */
+  get physicalSize(): number {
+    return this.index.physicalSize;
+  }
+
+  /** Number of clusters (partitions) currently in the index. */
+  get clusterCount(): number {
+    return this.index.clusterCount;
+  }
+
+  get efSearch(): number {
+    return this.index.efSearch;
+  }
+  set efSearch(value: number) {
+    this.index.efSearch = value;
+  }
+
+  /** Distance computations performed during the most recent query. */
+  get lastQueryDistanceComputations(): number {
+    return this.index.lastQueryDistComps;
+  }
+
+  /** Storage row reads issued by the most recent query — the honest,
+   *  cache-independent scalability metric. */
+  get lastQueryStorageReads(): number {
+    return this.index.lastQueryStorageReads;
+  }
+
+  get storage(): StorageStats {
+    const f32 = this.dim * 4;
+    const bpv = this.index.bytesPerVector;
+    return {
+      float32BytesPerVector: f32,
+      codeBytesPerVector: bpv,
+      bytesPerVector: bpv,
+      compressionRatio: f32 / bpv,
+    };
+  }
+
+  private checkId(id: ExternalId): number {
+    // 32-bit signed range: chunk blobs store exts as int32 — a wider id
+    // would silently truncate, so it is rejected at the door instead.
+    if (!Number.isInteger(id) || id > 0x7fffffff || id < -0x80000000) {
+      throw new Error(`External id must be a 32-bit integer, got ${id}`);
+    }
+    return id;
+  }
+
+  /** Convert a value to code bytes, selecting by length: `codeWords`
+   *  elements → an existing 1-bit code; otherwise a raw `dim`-vector. */
+  private toCodeBytes(value: ArrayLike<number>): Uint8Array {
+    if (value.length === this.index.codeWords) {
+      return this.index.codeToBytes(value);
+    }
+    if (value.length !== this.dim) {
+      throw new Error(
+        `Vector dimension mismatch: expected ${this.dim}, got ${value.length}`,
+      );
+    }
+    return this.index.encodeToBytes(value);
+  }
+
+  has(id: ExternalId): boolean {
+    return this.index.has(this.checkId(id));
+  }
+
+  /** Stream every live external id (bounded memory). */
+  *keys(): IterableIterator<ExternalId> {
+    yield* this.index.keys();
+  }
+
+  /** Stream live entries whose INTERNAL id is > `after` — a durable
+   *  incremental watermark (internal ids are monotone at insert and preserved
+   *  by update and compact). */
+  *keysSince(
+    after: number,
+  ): IterableIterator<{ ext: ExternalId; internal: number }> {
+    yield* this.index.keysSince(after);
+  }
+
+  /** Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. */
+  get(id: ExternalId): Uint32Array | null {
+    const bytes = this.index.codeOf(this.checkId(id));
+    return bytes ? this.index.bytesToCode(bytes) : null;
+  }
+
+  insert(id: ExternalId, value: ArrayLike<number>): void {
+    this.index.begin();
+    try {
+      this.index.insert(this.checkId(id), this.toCodeBytes(value));
+      this.index.commit();
+    } catch (e) {
+      this.index.rollback();
+      throw e;
+    }
+  }
+
+  update(id: ExternalId, value: ArrayLike<number>): void {
+    this.index.begin();
+    try {
+      this.index.update(this.checkId(id), this.toCodeBytes(value));
+      this.index.commit();
+    } catch (e) {
+      this.index.rollback();
+      throw e;
+    }
+  }
+
+  upsert(id: ExternalId, value: ArrayLike<number>): void {
+    this.upsertMany([{ id, vector: value }]);
+  }
+
+  /**
+   * Upsert many vectors under ONE transaction — one WAL commit for the whole
+   * batch instead of one per vector, with chunk appends to the same cluster
+   * coalesced in the index's write-back buffer.
+   */
+  upsertMany(
+    entries: Array<{ id: ExternalId; vector: ArrayLike<number> }>,
+  ): void {
+    if (entries.length === 0) return;
+    this.index.begin();
+    try {
+      for (const e of entries) {
+        // One shared vmap probe decides insert vs update — the per-row point
+        // query is the dominant bulk-load cost, so it is never duplicated.
+        this.index.upsert(this.checkId(e.id), this.toCodeBytes(e.vector));
+      }
+      this.index.commit();
+    } catch (e) {
+      this.index.rollback();
+      throw e;
+    }
+  }
+
+  /** Delete many ids under ONE transaction. Absent ids are skipped.
+   *  Returns the number of vectors actually removed. */
+  deleteMany(ids: ExternalId[]): number {
+    if (ids.length === 0) return 0;
+    let removed = 0;
+    this.index.begin();
+    try {
+      for (const id of ids) {
+        if (this.index.remove(this.checkId(id))) removed++;
+      }
+      this.index.commit();
+    } catch (e) {
+      this.index.rollback();
+      throw e;
+    }
+    return removed;
+  }
+
+  /** Delete the vector bound to an id. Returns false if absent. */
+  delete(id: ExternalId): boolean {
+    return this.deleteMany([id]) === 1;
+  }
+
+  /** Heat the SQLite page cache with sequential scans (latency only). */
+  warmCache(): number {
+    return this.index.warmCache();
+  }
+
+  /** Reclaim tombstoned slots by rewriting the clusters that carry any, then
+   *  VACUUM.  Internal ids and cluster assignment are preserved. */
+  compact(): void {
+    this.index.compact();
+  }
+
+  /**
+   * k-NN search. The argument's length selects the mode:
+   *  - `dim` elements       → a raw vector (accurate estimator)
+   *  - `codeWords` elements → a 1-bit code, by sign-bit Hamming
+   */
+  query(
+    query: ArrayLike<number>,
+    k = 10,
+    opts?: { ef?: number },
+  ): QueryResult[] {
+    let hits: IvfHit[];
+    if (query.length === this.index.codeWords) {
+      hits = this.index.searchKnnByCode(
+        this.index.codeToBytes(query),
+        k,
+        opts?.ef,
+      );
+    } else {
+      if (query.length !== this.dim) {
+        throw new Error(
+          `Vector dimension mismatch: expected ${this.dim}, got ${query.length}`,
+        );
+      }
+      hits = this.index.searchKnn(query, k, opts?.ef);
+    }
+    return hits;
+  }
+
+  /** Close the underlying database. The instance must not be used afterwards. */
+  close(): void {
+    this.index.close();
+  }
+}

package/src/{rabitq-hnsw → rabitq-ivf}/src/index.ts

@@ -5,11 +5,8 @@ export type {
   QueryResult,
   StorageStats,
 } from "./database.js";
-export { HnswIndex } from "./hnsw.js";
-export type { HnswParams, KnnHit } from "./hnsw.js";
+export { IvfIndex } from "./ivf.js";
+export type { IvfConfig, IvfHit } from "./ivf.js";
 export { RaBitQuantizer } from "./rabitq.js";
 export type { QueryContext, RaBitQOptions } from "./rabitq.js";
-export { Store } from "./store.js";
-export type { GlobalState, NodeRec, StoreConfig } from "./store.js";
-export { Heap } from "./heap.js";
 export { Prng } from "./prng.js";