@hviana/sema 0.1.5 → 0.1.6

package/dist/src/vec.d.ts

@@ -14,21 +14,17 @@ export declare function normalize(v: Vec): Vec;
 export declare function cosine(a: Vec, b: Vec): number;
 /** Set vector epsilon thresholds. Called once by Mind at construction. */
 export declare function setVecConfig(cfg: {
-  normalizeEpsilon?: number;
-  cosineEpsilon?: number;
+    normalizeEpsilon?: number;
+    cosineEpsilon?: number;
 }): void;
 export interface Permutation {
-  fwd: Uint32Array;
-  inv: Uint32Array;
+    fwd: Uint32Array;
+    inv: Uint32Array;
 }
 /** The keyring: one independent permutation per seat.
  *  Independent keys do not commute, so an address in a tree is the
  *  path itself — "seat 2 inside seat 1" ≠ "seat 1 inside seat 2". */
-export declare function makeKeyring(
-  D: number,
-  seats: number,
-  rand: () => number,
-): Permutation[];
+export declare function makeKeyring(D: number, seats: number, rand: () => number): Permutation[];
 /** Apply permutation: out[i] = v[table[i]]. */
 export declare function permute(v: Vec, table: Uint32Array): Vec;
 /** Permute into existing buffer — zero allocation. */

package/dist/src/vec.js

@@ -4,49 +4,44 @@
 // No weights. No gradients.
 /** Deterministic PRNG (32-bit mixer). */
 export function rng(seed) {
-  let s = seed >>> 0;
-  return () => {
-    s = (s + 0x6d2b79f5) >>> 0;
-    let t = s;
-    t = Math.imul(t ^ (t >>> 15), t | 1);
-    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
-    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
-  };
+    let s = seed >>> 0;
+    return () => {
+        s = (s + 0x6d2b79f5) >>> 0;
+        let t = s;
+        t = Math.imul(t ^ (t >>> 15), t | 1);
+        t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
 }
 /** Standard normal sample (Box–Muller). */
 function gaussian(rand) {
-  let u = 0, v = 0;
-  while (u === 0) {
-    u = rand();
-  }
-  while (v === 0) {
-    v = rand();
-  }
-  return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v);
+    let u = 0, v = 0;
+    while (u === 0)
+        u = rand();
+    while (v === 0)
+        v = rand();
+    return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v);
 }
 export const zeros = (D) => new Float32Array(D);
 export const copy = (v) => new Float32Array(v);
 /** Random point on the unit sphere. */
 export function randomUnit(D, rand) {
-  const v = zeros(D);
-  for (let i = 0; i < D; i++) {
-    v[i] = gaussian(rand);
-  }
-  return normalize(v);
+    const v = zeros(D);
+    for (let i = 0; i < D; i++)
+        v[i] = gaussian(rand);
+    return normalize(v);
 }
 /** target += src * scale (in place). */
 export function addInto(target, src, scale = 1) {
-  for (let i = 0; i < target.length; i++) {
-    target[i] += src[i] * scale;
-  }
-  return target;
+    for (let i = 0; i < target.length; i++)
+        target[i] += src[i] * scale;
+    return target;
 }
 export function dot(a, b) {
-  let s = 0;
-  for (let i = 0; i < a.length; i++) {
-    s += a[i] * b[i];
-  }
-  return s;
+    let s = 0;
+    for (let i = 0; i < a.length; i++)
+        s += a[i] * b[i];
+    return s;
 }
 const norm = (v) => Math.sqrt(dot(v, v));
 // Epsilon thresholds (settable once via setVecConfig).
@@ -54,68 +49,61 @@ let _normalizeEpsilon = 1e-12;
 let _cosineEpsilon = 1e-12;
 /** In-place normalization. */
 export function normalize(v) {
-  const n = norm(v);
-  if (n > _normalizeEpsilon) {
-    for (let i = 0; i < v.length; i++) {
-      v[i] /= n;
+    const n = norm(v);
+    if (n > _normalizeEpsilon) {
+        for (let i = 0; i < v.length; i++)
+            v[i] /= n;
     }
-  }
-  return v;
+    return v;
 }
 /** Resonance: 1 = same, 0 = unrelated. */
 export function cosine(a, b) {
-  const na = norm(a), nb = norm(b);
-  return na > _cosineEpsilon && nb > _cosineEpsilon ? dot(a, b) / (na * nb) : 0;
+    const na = norm(a), nb = norm(b);
+    return na > _cosineEpsilon && nb > _cosineEpsilon ? dot(a, b) / (na * nb) : 0;
 }
 /** Set vector epsilon thresholds. Called once by Mind at construction. */
 export function setVecConfig(cfg) {
-  if (cfg.normalizeEpsilon !== undefined) {
-    _normalizeEpsilon = cfg.normalizeEpsilon;
-  }
-  if (cfg.cosineEpsilon !== undefined) {
-    _cosineEpsilon = cfg.cosineEpsilon;
-  }
+    if (cfg.normalizeEpsilon !== undefined) {
+        _normalizeEpsilon = cfg.normalizeEpsilon;
+    }
+    if (cfg.cosineEpsilon !== undefined)
+        _cosineEpsilon = cfg.cosineEpsilon;
 }
 /** One fixed random permutation (Fisher–Yates). */
 function makePermutation(D, rand) {
-  const fwd = new Uint32Array(D);
-  for (let i = 0; i < D; i++) {
-    fwd[i] = i;
-  }
-  for (let i = D - 1; i > 0; i--) {
-    const j = Math.floor(rand() * (i + 1));
-    const t = fwd[i];
-    fwd[i] = fwd[j];
-    fwd[j] = t;
-  }
-  const inv = new Uint32Array(D);
-  for (let i = 0; i < D; i++) {
-    inv[fwd[i]] = i;
-  }
-  return { fwd, inv };
+    const fwd = new Uint32Array(D);
+    for (let i = 0; i < D; i++)
+        fwd[i] = i;
+    for (let i = D - 1; i > 0; i--) {
+        const j = Math.floor(rand() * (i + 1));
+        const t = fwd[i];
+        fwd[i] = fwd[j];
+        fwd[j] = t;
+    }
+    const inv = new Uint32Array(D);
+    for (let i = 0; i < D; i++)
+        inv[fwd[i]] = i;
+    return { fwd, inv };
 }
 /** The keyring: one independent permutation per seat.
  *  Independent keys do not commute, so an address in a tree is the
  *  path itself — "seat 2 inside seat 1" ≠ "seat 1 inside seat 2". */
 export function makeKeyring(D, seats, rand) {
-  const ring = [];
-  for (let s = 0; s < seats; s++) {
-    ring.push(makePermutation(D, rand));
-  }
-  return ring;
+    const ring = [];
+    for (let s = 0; s < seats; s++)
+        ring.push(makePermutation(D, rand));
+    return ring;
 }
 /** Apply permutation: out[i] = v[table[i]]. */
 export function permute(v, table) {
-  const out = new Float32Array(v.length);
-  for (let i = 0; i < v.length; i++) {
-    out[i] = v[table[i]];
-  }
-  return out;
+    const out = new Float32Array(v.length);
+    for (let i = 0; i < v.length; i++)
+        out[i] = v[table[i]];
+    return out;
 }
 /** Permute into existing buffer — zero allocation. */
 export function permuteInto(out, v, table) {
-  for (let i = 0; i < v.length; i++) {
-    out[i] = v[table[i]];
-  }
-  return out;
+    for (let i = 0; i < v.length; i++)
+        out[i] = v[table[i]];
+    return out;
 }

package/example/train_base.ts

@@ -1610,20 +1610,23 @@ async function saveProgress(store: Store, p: SavedProgress): Promise<void> {
 // ═══════════════════════════════════════════════════════════════════════
 
 async function main(): Promise<void> {
-  // The vector indices' single memory knob (MiB) — sizes both the SQLite page
-  // cache and the decoded-code LRU of each index.  Training is HNSW-insert
-  // bound at scale (profiled at >80% of ingest wall-clock on a 5.5M-vector
-  // store, nearly all of it cache-missing code reads), and inserts walk the
-  // graph, so a cache that covers the indexed codes cuts wall-clock directly
-  // (measured ~1.6× at 1 GB vs 64 MB on a 2M-node store).  1 GB caches ~5.8M
-  // D=1024 codes — the whole content index of a 350 MB-corpus store — and is
-  // the default; override with VECTOR_CACHE_MB (e.g. 256 on a small-RAM
-  // machine, 64 for the library default).
-  const VECTOR_CACHE_MB = Math.max(0, Number(env("VECTOR_CACHE_MB", "1024")));
+  // The vector indices' memory knob (MiB) — each index's SQLite page cache.
+  // The IVF index routes inserts through a RAM-resident pivot table and
+  // appends to chunk blobs, so this cache mostly serves query-time cluster
+  // scans; 256 MiB comfortably covers the probed working set of a trained
+  // store.  Override with VECTOR_CACHE_MB (64 is the library default).
+  const VECTOR_CACHE_MB = Math.max(0, Number(env("VECTOR_CACHE_MB", "256")));
+  // Page cache for the MAIN DAG database (node/kid/edge/contain tables).
+  // Training issues millions of content-addressed point probes per session
+  // against a GB-scale file; the library default (64 MiB) is sized for a
+  // small machine — a training box affords more.  Override with
+  // SQLITE_CACHE_MB.
+  const SQLITE_CACHE_MB = Math.max(0, Number(env("SQLITE_CACHE_MB", "256")));
   const store = new SQliteStore({
     path: DB_PATH,
     D,
     vectorCacheMb: VECTOR_CACHE_MB,
+    sqliteCacheMb: SQLITE_CACHE_MB,
   });
 
   // The store IS the model: memories, progress, and metadata all persist in

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hviana/sema",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "Sema: a non-parametric, instance-based reasoning system.",
   "type": "module",
   "main": "dist/src/index.js",

package/src/alu/README.md

@@ -15,7 +15,7 @@ search as authoritative axioms (at `STEP` cost, like a learned edge).
 
 It has no dependency on the rest of the codebase except the pure byte helpers in
 `../bytes.ts`, and is intended to be reused as a self-contained sublibrary in
-the spirit of `derive/` and `rabitq-hnsw/`.
+the spirit of `derive/` and `rabitq-ivf/`.
 
 ## The thesis: one tiny kernel, everything else is a rewrite
 

package/src/alu/src/index.ts

@@ -1,7 +1,7 @@
 // alu — the ALU sub-library: a tiny irreducible kernel from which arithmetic,
 // logic, and numerical computation are all DERIVED.
 //
-// Self-contained in the spirit of derive/ and rabitq-hnsw/: it imports only the
+// Self-contained in the spirit of derive/ and rabitq-ivf/: it imports only the
 // pure byte helpers (../../bytes.js) and nothing else from SEMA.  The host
 // reaches meaning (operator synonymy, the symbolic inverse) through the injected
 // AluResonance, pre-resolved into a synchronous snapshot before the search runs.

package/src/config.ts

@@ -39,18 +39,9 @@ export interface AlphabetConfig {
 
 export interface StoreConfig {
   minHaloMass: number;
-  m: number;
-  efConstruction: number;
-  /** Construction budget for REACH-ONLY interior gists — the ~90% of content
-   *  index inserts that exist so a partial query can resonate a sub-region
-   *  and climb, never as dedup targets.  Deposit roots and halo-bearing
-   *  targets always build with the full `efConstruction`.  A smaller budget
-   *  here is the one deliberate speed-for-quality trade in ingestion (an
-   *  interior's layer-0 wiring is built from a narrower candidate beam);
-   *  the recall suite (partial recall, multi-topic attention, counterfactual
-   *  anchoring) is the gate for its value.  Set equal to `efConstruction`
-   *  to disable the trade. */
-  efConstructionInterior: number;
+  /** Query breadth of the IVF vector indices: clusters probed per query =
+   *  ceil(efSearch / 4).  Inserts have no quality knob — the partitioned
+   *  index routes and appends, so ingestion cost is flat by construction. */
   efSearch: number;
   /** Compact the in-memory vector indices after this many vectors are written.
    *  Compaction rebuilds an index from its live codes to reclaim the slots left
@@ -58,7 +49,7 @@ export interface StoreConfig {
    *  (not on a flush count that goes quiet during repeat-heavy training) keeps
    *  the index dense and query cost bounded. */
   compactEveryNWrites: number;
-  /** Over-fetch factor for HNSW queries (ANN recall cushion). */
+  /** Over-fetch factor for vector-index queries (ANN recall cushion). */
   overfetch: number;
   /** Combined buffered-write ceiling before a flush of both vector indices
    *  (content + halo). Higher ⇒ fewer, larger flushes into the in-memory
@@ -91,14 +82,20 @@ export interface StoreConfig {
    *  round-trips through the quantizer. An eviction or a reopen reads the
    *  2-bit row — the fidelity every cross-session consumer already gets. */
   haloCacheBytes: number;
-  /** Size, in MiB, of each `rabitq-hnsw` `VectorDatabase`'s memory budget
-   *  (forwarded as its `cacheSizeMb`).  It is the index's SINGLE memory knob: it
-   *  sizes both SQLite's page cache and the derived immutable-code LRU.  A PURE
-   *  latency optimisation — the index reads codes from SQLite on demand, so its
-   *  correctness and its asymptotic per-operation storage-read count are identical
-   *  with the budget at 0.  Exposed so a scaling test can set it to 0 and measure
+  /** Size, in MiB, of each `rabitq-ivf` `VectorDatabase`'s memory budget
+   *  (forwarded as its `cacheSizeMb` — its SQLite page cache).  A PURE latency
+   *  optimisation — the index reads chunk blobs from SQLite on demand, so its
+   *  correctness and its per-operation storage-read count are identical with
+   *  the budget at 0.  Exposed so a scaling test can set it to 0 and measure
    *  the honest, cache-independent cost. */
   vectorCacheMb: number;
+  /** Size, in MiB, of the MAIN DAG database's SQLite page cache.  The node /
+   *  kid / edge / contain tables serve millions of point probes per training
+   *  session (content-addressed findLeaf/findBranch, parent probes, contain
+   *  appends); SQLite's default cache (~2 MiB) thrashes once the DB outgrows
+   *  it, so every probe pays a file read.  A PURE latency knob — correctness
+   *  and result identical at any value. */
+  sqliteCacheMb: number;
   /** Max entries in the skipped-interior LRU set.  Interiors that
    *  {@link Store.indexSubtree} has already visited (indexed or skipped) are
    *  remembered here so subsequent calls prune their subtrees.  Session-local
@@ -151,9 +148,6 @@ export const DEFAULT_CONFIG: MindConfig = {
   },
   store: {
     minHaloMass: 1,
-    m: 8,
-    efConstruction: 64,
-    efConstructionInterior: 16,
     efSearch: 64,
     compactEveryNWrites: 50_000,
     overfetch: 4,
@@ -165,6 +159,7 @@ export const DEFAULT_CONFIG: MindConfig = {
     pendingGistBytes: 16_000_000,
     haloCacheBytes: 16_000_000,
     vectorCacheMb: 64,
+    sqliteCacheMb: 64,
     coveredIdsMax: 100_000,
     chainCacheBytes: 16_000_000,
   },
@@ -194,11 +189,6 @@ export function resolveConfig(opts: Partial<MindConfig> = {}): MindConfig {
     },
     store: {
       minHaloMass: opts.store?.minHaloMass ?? DEFAULT_CONFIG.store.minHaloMass,
-      m: opts.store?.m ?? DEFAULT_CONFIG.store.m,
-      efConstruction: opts.store?.efConstruction ??
-        DEFAULT_CONFIG.store.efConstruction,
-      efConstructionInterior: opts.store?.efConstructionInterior ??
-        DEFAULT_CONFIG.store.efConstructionInterior,
       efSearch: opts.store?.efSearch ?? DEFAULT_CONFIG.store.efSearch,
       compactEveryNWrites: opts.store?.compactEveryNWrites ??
         DEFAULT_CONFIG.store.compactEveryNWrites,
@@ -218,6 +208,8 @@ export function resolveConfig(opts: Partial<MindConfig> = {}): MindConfig {
         DEFAULT_CONFIG.store.haloCacheBytes,
       vectorCacheMb: opts.store?.vectorCacheMb ??
         DEFAULT_CONFIG.store.vectorCacheMb,
+      sqliteCacheMb: opts.store?.sqliteCacheMb ??
+        DEFAULT_CONFIG.store.sqliteCacheMb,
       coveredIdsMax: opts.store?.coveredIdsMax ??
         DEFAULT_CONFIG.store.coveredIdsMax,
       chainCacheBytes: opts.store?.chainCacheBytes ??

package/src/index.ts

@@ -13,26 +13,21 @@ export * from "./store-sqlite.js";
 export * from "./config.js";
 export * from "./extension.js";
 export * from "./ingest-cache.js";
-// rabitq-hnsw is re-exported selectively: its `Store`, `NodeRec`, and
-// `StoreConfig` are internal names that collide with sema's own top-level
-// `store.js` / `config.js` exports, and sema code that needs the rabitq ones
-// imports them from the subpath directly. Re-export the public vector-DB
-// surface under the sema root, omitting the three colliding names.
+// rabitq-ivf: the partitioned (IVF) vector index over 1-bit RaBitQ codes.
 export {
-  Heap,
-  HnswIndex,
+  IvfIndex,
   Prng,
   RaBitQuantizer,
   VectorDatabase,
-} from "./rabitq-hnsw/src/index.js";
+} from "./rabitq-ivf/src/index.js";
 export type {
   DatabaseOptions,
   ExternalId,
-  HnswParams,
-  KnnHit,
+  IvfConfig,
+  IvfHit,
   QueryContext,
   QueryResult,
   RaBitQOptions,
   StorageStats,
-} from "./rabitq-hnsw/src/index.js";
+} from "./rabitq-ivf/src/index.js";
 export * from "./derive/src/index.js";

package/src/rabitq-ivf/README.md

@@ -0,0 +1,56 @@
+# rabitq-ivf
+
+A small TypeScript library for **approximate nearest-neighbour search** using an
+**adaptive partitioned (IVF) index** over **1-bit RaBitQ codes**, with
+**cosine** distance. The entire index lives in **SQLite** — the durable copy of
+every code is on disk, resident memory is bounded (the pivot table plus a capped
+page cache, never the whole collection), and the index survives process
+restarts.
+
+The vectors are stored **only** as 1-bit RaBitQ codes — the original vectors are
+never kept — so the index is both fast _and_ tiny on disk:
+
+```
+a 1024-d vector:
+  Float32             : 1024 × 4 bytes = 4096 bytes
+  RaBitQ 1-bit (code) : 1024 × 1 bit   = 128 bytes    (32× smaller)
+```
+
+The code _is_ the entire per-vector representation. `get` returns the bare code,
+and `insert`, `update`, and `query` all accept either a raw vector or a code —
+detected by length.
+
+## The index
+
+- **Clusters, not a graph.** The collection is partitioned into clusters, each
+  with a binary pivot code and its member codes packed in fixed-size chunk
+  blobs.
+- **Insert = route + append.** Find the nearest pivot (one linear Hamming scan
+  of the RAM-resident pivot table) and append to that cluster's tail chunk. No
+  beam search, no neighbour rewiring — per-insert cost is essentially flat in
+  collection size.
+- **Query = probe + scan.** Rank all pivots with the accurate RaBitQ estimator,
+  scan the `ceil(efSearch/4)` nearest clusters with the same estimator, keep the
+  top k. Per-query storage reads are bounded by nprobe × chunks-per-cluster —
+  constant once the collection has split.
+- **Adaptive, deterministic splits.** A cluster reaching 4096 entries is
+  median-split on the margin between two farthest-point seeds (two exact halves,
+  cascade-proof), and both halves get fresh majority-bit pivots. There is no
+  RNG: the index is a pure function of the insertion sequence.
+- **Same durability discipline as the rest of Sema**: WAL, batched caller-owned
+  transactions (`upsertMany`), 1 KiB pages, 64 MiB WAL autocheckpoint.
+
+## Usage
+
+```ts
+import { VectorDatabase } from "./src/index.js";
+
+const db = new VectorDatabase({ dbPath: "vectors.db", dim: 1024 });
+db.upsertMany([{ id: 1, vector: v1 }, { id: 2, vector: v2 }]);
+const hits = db.query(q, 10); // [{ id, distance }]
+db.close();
+```
+
+`lastQueryStorageReads` reports the cache-independent storage-read count of the
+most recent query — the honest scalability witness the test suite asserts on
+(`test/35-ivf.test.mjs` at the repo root).