@hviana/sema 0.1.5 → 0.1.6

package/dist/src/rabitq-hnsw/src/rabitq.js

@@ -1,299 +0,0 @@
-import { Prng } from "./prng.js";
-function nextPow2(n) {
-  let p = 1;
-  while (p < n) {
-    p <<= 1;
-  }
-  return p;
-}
-/** Set-bit count for every byte value, for counting code bits during the byte scan. */
-const POPCOUNT8 = new Uint8Array(256);
-for (let i = 1; i < 256; i++) {
-  POPCOUNT8[i] = POPCOUNT8[i >> 1] + (i & 1);
-}
-/**
- * 1-bit RaBitQ quantizer (cosine) -- the ONLY representation of a vector kept by
- * the index. A D-dimensional vector collapses to ceil(D/32) 32-bit words of sign
- * bits, e.g. a 256-d vector goes from 256*4 = 1024 bytes to 32 bytes of code.
- *
- * Each vector is centered by an optional centroid, normalised, rotated by a
- * fast random orthogonal transform (random sign flips + Walsh-Hadamard,
- * O(D log D)) and reduced to one sign bit per padded dimension. The random
- * rotation makes the quantisation error essentially uniform across vectors, so
- * the cosine estimate needs only a single fixed scale (`cosFactor`) rather than
- * any per-vector correction.
- *
- * Two estimators are provided:
- *   - `estimate`          : full-precision query vs stored code (accurate)
- *   - `codeDistanceBytes` : stored code vs stored code (Hamming based; used while
- *                           building the graph, where neither side is full precision)
- *
- * Reference: Gao & Long, "RaBitQ: Quantizing High-Dimensional Vectors with a
- * Theoretical Error Bound for Approximate Nearest Neighbor Search", SIGMOD 2024.
- */
-export class RaBitQuantizer {
-  dim;
-  paddedDim;
-  codeWords;
-  queryBits;
-  rounds;
-  seed;
-  centroid;
-  sqrtD;
-  maxQInt;
-  signs;
-  scratch;
-  /** Fixed cosine inner-product scale (= 1 / E[L1] of a rotated unit vector). */
-  cosFactor;
-  // byte-LUT machinery for the query/code estimator
-  nbytes;
-  /** coordinate index sitting at byte position p, bit k -> bitCoord[p*8 + k]. */
-  bitCoord;
-  /** true when the largest possible LUT entry overflows a Uint8. */
-  lutWide;
-  constructor(dim, opts = {}) {
-    this.dim = dim;
-    this.paddedDim = nextPow2(dim);
-    this.codeWords = Math.ceil(this.paddedDim / 32);
-    this.queryBits = opts.queryBits ?? 8;
-    this.rounds = opts.rounds ?? 3;
-    this.seed = (opts.seed ?? 0x1234abcd) >>> 0;
-    this.sqrtD = Math.sqrt(this.paddedDim);
-    this.cosFactor = Math.sqrt(Math.PI / (2 * this.paddedDim));
-    this.maxQInt = (1 << this.queryBits) - 1;
-    this.centroid = new Float64Array(dim);
-    if (opts.centroid) {
-      for (let i = 0; i < dim; i++) {
-        this.centroid[i] = opts.centroid[i] ?? 0;
-      }
-    }
-    const prng = new Prng(this.seed);
-    this.signs = [];
-    for (let r = 0; r < this.rounds; r++) {
-      const s = new Float64Array(this.paddedDim);
-      for (let i = 0; i < this.paddedDim; i++) {
-        s[i] = prng.next() < 0.5 ? -1 : 1;
-      }
-      this.signs.push(s);
-    }
-    this.scratch = new Float64Array(this.paddedDim);
-    // Map (code-byte position, bit-in-byte) -> coordinate, honouring the host
-    // byte order so the Uint8 view of the (Uint32) code buffer is interpreted
-    // correctly on both little- and big-endian platforms.
-    this.nbytes = this.paddedDim >>> 3;
-    const littleEndian = new Uint8Array(new Uint32Array([1]).buffer)[0] === 1;
-    this.bitCoord = new Int32Array(this.nbytes * 8);
-    for (let p = 0; p < this.nbytes; p++) {
-      const word = p >>> 2;
-      const localByte = littleEndian ? p & 3 : 3 - (p & 3);
-      const bitBase = word * 32 + localByte * 8;
-      for (let k = 0; k < 8; k++) {
-        this.bitCoord[p * 8 + k] = bitBase + k;
-      }
-    }
-    this.lutWide = 8 * this.maxQInt > 255;
-  }
-  /** In-place fast Walsh-Hadamard transform; `a.length` must be a power of two. */
-  fwht(a) {
-    const n = a.length;
-    for (let len = 1; len < n; len <<= 1) {
-      const span = len << 1;
-      for (let i = 0; i < n; i += span) {
-        for (let j = i; j < i + len; j++) {
-          const u = a[j];
-          const v = a[j + len];
-          a[j] = u + v;
-          a[j + len] = u - v;
-        }
-      }
-    }
-  }
-  /** Apply the orthogonal rotation in place (a.length === paddedDim). */
-  rotate(a) {
-    const n = this.paddedDim;
-    const inv = 1 / this.sqrtD;
-    for (let r = 0; r < this.rounds; r++) {
-      const s = this.signs[r];
-      for (let i = 0; i < n; i++) {
-        a[i] *= s[i];
-      }
-      this.fwht(a);
-      for (let i = 0; i < n; i++) {
-        a[i] *= inv;
-      }
-    }
-  }
-  /** Encode a raw vector into its 1-bit sign code (the whole representation). */
-  encode(vec) {
-    const dim = this.dim;
-    const pd = this.paddedDim;
-    const buf = this.scratch;
-    let sq = 0;
-    for (let i = 0; i < dim; i++) {
-      const v = vec[i] - this.centroid[i];
-      buf[i] = v;
-      sq += v * v;
-    }
-    for (let i = dim; i < pd; i++) {
-      buf[i] = 0;
-    }
-    const code = new Uint32Array(this.codeWords);
-    if (sq === 0) {
-      return code;
-    }
-    const invNorm = 1 / Math.sqrt(sq);
-    for (let i = 0; i < dim; i++) {
-      buf[i] *= invNorm; // unit residual; padded dims stay 0
-    }
-    this.rotate(buf);
-    for (let i = 0; i < pd; i++) {
-      if (buf[i] > 0) {
-        code[i >>> 5] |= 1 << (i & 31);
-      }
-    }
-    return code;
-  }
-  /** Pre-process a full-precision query into the structure consumed by `estimate`. */
-  prepareQuery(vec) {
-    const dim = this.dim;
-    const pd = this.paddedDim;
-    const nb = this.nbytes;
-    const buf = this.scratch;
-    let sq = 0;
-    for (let i = 0; i < dim; i++) {
-      const v = vec[i] - this.centroid[i];
-      buf[i] = v;
-      sq += v * v;
-    }
-    for (let i = dim; i < pd; i++) {
-      buf[i] = 0;
-    }
-    const qNorm = Math.sqrt(sq);
-    const qlut = this.lutWide
-      ? new Uint16Array(nb * 256)
-      : new Uint8Array(nb * 256);
-    if (qNorm === 0) {
-      return { vmin: 0, delta: 0, sumQInt: 0, qlut, nbytes: nb, zero: true };
-    }
-    const invNorm = 1 / qNorm;
-    for (let i = 0; i < dim; i++) {
-      buf[i] *= invNorm;
-    }
-    this.rotate(buf);
-    let vmin = Infinity;
-    let vmax = -Infinity;
-    for (let i = 0; i < pd; i++) {
-      const x = buf[i];
-      if (x < vmin) {
-        vmin = x;
-      }
-      if (x > vmax) {
-        vmax = x;
-      }
-    }
-    const range = vmax - vmin;
-    const delta = range > 0 ? range / this.maxQInt : 0;
-    const invDelta = delta > 0 ? 1 / delta : 0;
-    // Quantise each (rotated) query coordinate to queryBits bits.
-    const qint = buf; // reuse: write the integer code back over the float buffer
-    let sumQInt = 0;
-    for (let i = 0; i < pd; i++) {
-      let q = delta > 0 ? Math.round((buf[i] - vmin) * invDelta) : 0;
-      if (q < 0) {
-        q = 0;
-      } else if (q > this.maxQInt) {
-        q = this.maxQInt;
-      }
-      qint[i] = q;
-      sumQInt += q;
-    }
-    // Build the byte LUT: qlut[p*256 + v] = sum of qint at coords whose bit is
-    // set in v, grown incrementally as qlut[..(v with lowest set bit cleared)..]
-    // plus the contribution of that lowest set bit.
-    const bitCoord = this.bitCoord;
-    for (let p = 0; p < nb; p++) {
-      const base = p << 8;
-      const cb = p << 3;
-      for (let v = 1; v < 256; v++) {
-        const low = v & -v;
-        const k = 31 - Math.clz32(low);
-        qlut[base + v] = qlut[base + (v & (v - 1))] + qint[bitCoord[cb + k]];
-      }
-    }
-    return { vmin, delta, sumQInt, qlut, nbytes: nb, zero: false };
-  }
-  /**
-   * Estimate the cosine distance (1 - cosine) between a stored code and a
-   * full-precision query, reading the code's bytes against the query's byte LUT.
-   * The code's set-bit count is tallied in the same byte scan, so nothing beyond
-   * the code itself is needed.
-   *
-   * `codeBytes` is a Uint8 view of the packed code buffer and `byteOffset` is the
-   * code's start byte (id * paddedDim/8).
-   */
-  estimate(codeBytes, byteOffset, q) {
-    if (q.zero) {
-      return 1;
-    }
-    const nb = q.nbytes;
-    const lut = q.qlut;
-    let dot = 0;
-    let popcount = 0;
-    for (let p = 0; p < nb; p++) {
-      const b = codeBytes[byteOffset + p];
-      dot += lut[(p << 8) + b];
-      popcount += POPCOUNT8[b];
-    }
-    // A = sum_i sign_i * q_rot_i, recovered from the quantised query.
-    const A = q.vmin * (2 * popcount - this.paddedDim) +
-      q.delta * (2 * dot - q.sumQInt);
-    return 1 - this.cosFactor * A;
-  }
-  /**
-   * Cosine distance (1 - cosine, in [0, 2]) between two packed codes, computed
-   * directly from the BLOB bytes via their sign-bit Hamming distance — no word
-   * reinterpretation, so it is endianness-agnostic. Identical codes score 0.
-   *
-   * This is the distance used to build the graph (code vs code) and to answer a
-   * query given an already-quantized code. It is coarser than `estimate`, where
-   * one side is full precision.
-   */
-  codeDistanceBytes(a, b) {
-    const nb = this.nbytes;
-    let ham = 0;
-    // 32-bit-word Hamming: this is the hot arithmetic of graph construction
-    // (every candidate/prune comparison), so fold 4 bytes into one word and
-    // popcount it — ~4× fewer loop iterations than a byte-LUT scan, with no
-    // allocation and no dependence on the buffers' alignment or endianness
-    // (both sides are composed identically, so XOR is order-agnostic).
-    let p = 0;
-    for (const n4 = nb & ~3; p < n4; p += 4) {
-      let x = (a[p] ^ b[p]) |
-        ((a[p + 1] ^ b[p + 1]) << 8) |
-        ((a[p + 2] ^ b[p + 2]) << 16) |
-        ((a[p + 3] ^ b[p + 3]) << 24);
-      x -= (x >>> 1) & 0x55555555;
-      x = (x & 0x33333333) + ((x >>> 2) & 0x33333333);
-      x = (x + (x >>> 4)) & 0x0f0f0f0f;
-      ham += Math.imul(x, 0x01010101) >>> 24;
-    }
-    for (; p < nb; p++) {
-      ham += POPCOUNT8[a[p] ^ b[p]];
-    }
-    return (2 * ham) / this.paddedDim;
-  }
-  /** Pack a code (codeWords 32-bit words) into its little-endian BLOB bytes. */
-  codeToBytes(code) {
-    const u = new Uint32Array(this.codeWords);
-    for (let i = 0; i < this.codeWords; i++) {
-      u[i] = code[i];
-    }
-    return new Uint8Array(u.buffer, 0, u.byteLength);
-  }
-  /** Reinterpret a code BLOB as a copy of codeWords 32-bit words. */
-  bytesToCode(bytes) {
-    const u = new Uint32Array(this.codeWords);
-    new Uint8Array(u.buffer).set(bytes);
-    return u;
-  }
-}

package/dist/src/rabitq-hnsw/src/store.d.ts

@@ -1,162 +0,0 @@
-import { DatabaseSync } from "node:sqlite";
-/**
- * The on-disk structure of the index, in SQLite. This is NOT a serialization of
- * an in-memory graph: it IS the graph. Every code and every adjacency list lives
- * in these tables and is read/written on demand, so process memory stays flat as
- * the collection grows -- only the working set of the current operation, plus
- * SQLite's fixed-size page cache, is ever resident.
- *
- * Tables
- *   meta   one row of configuration + mutable global state (entry point, counts,
- *          level-RNG state). The quantizer's rotation is fully determined by
- *          {dim, rounds, seed, centroid}, so reopening a database reproduces it
- *          exactly without storing any matrix.
- *   nodes  one row per inserted vector: id (rowid), external id, top level,
- *          tombstone flag, and the 1-bit code as a BLOB. Indexed by rowid (code
- *          lookup) and by a partial unique index on the external id (live nodes).
- *   links  one row per (node, layer) holding that node's neighbour ids packed as
- *          a BLOB. WITHOUT ROWID makes (node, layer) the clustering key, so a
- *          neighbour-list fetch -- the hot path of search -- is a single B-tree
- *          descent landing on the inline BLOB.
- */
-export interface StoreConfig {
-  dim: number;
-  m: number;
-  efConstruction: number;
-  efSearch: number;
-  queryBits: number;
-  rotationRounds: number;
-  seed: number;
-  centroid: Float64Array | null;
-  codeWords: number;
-  paddedDim: number;
-}
-export interface GlobalState {
-  entryPoint: number;
-  maxLevel: number;
-  live: number;
-  total: number;
-  rng: number;
-}
-/** A node as the graph algorithms need it: code bytes, tombstone, external id. */
-export interface NodeRec {
-  code: Uint8Array;
-  deleted: number;
-  ext: number | null;
-}
-export declare class Store {
-  readonly db: DatabaseSync;
-  /**
-   * Number of row reads served from the database (node and neighbour-list
-   * fetches). This counts *storage* accesses only, so it is unaffected by any
-   * caching layer above it -- the honest measure of how the engine scales.
-   */
-  reads: number;
-  private sInsNode;
-  private sNode;
-  private sIdByExt;
-  private sTombstone;
-  private sNbrs;
-  private sSetNbrs;
-  private sState;
-  private readonly cacheBudgetBytes;
-  /** Whether the shared memory budget is non-zero — consumers that trade RAM
-   *  for speed (the graph's visited-tag array) key off this so the
-   *  `cacheSizeMb: 0` flat-memory mode stays exactly flat. */
-  get cacheEnabled(): boolean;
-  private slab;
-  private slabHot;
-  private nbrSlab;
-  private slabsDerived;
-  /**
-   * @param dbPath      path to the SQLite file (":memory:" for transient).
-   * @param cacheSizeMb the ONE memory knob (MiB). It sizes BOTH SQLite's page
-   *        cache AND the immutable-code LRU above (whose entry-capacity is derived
-   *        from this budget and the code size — no second knob). Both are pure
-   *        speed enhancements: correctness and the per-operation storage-read
-   *        count do not depend on them. Pass 0 to run with essentially no cache,
-   *        which is how the tests run so a poor access pattern can never hide
-   *        behind a warm cache.
-   */
-  constructor(dbPath: string, cacheSizeMb?: number);
-  private tableExists;
-  /** (Re)compile the hot statements against the current schema. */
-  private prepareAll;
-  /** Load the persisted configuration, or null for a fresh database. */
-  loadConfig(): StoreConfig | null;
-  /** Initialise the meta row for a new database. */
-  initConfig(c: StoreConfig): void;
-  loadState(): GlobalState;
-  saveState(s: GlobalState): void;
-  setEfSearch(ef: number): void;
-  setQueryBits(bits: number): void;
-  resetReads(): void;
-  /** Stream every live external id in id order, in bounded-memory batches. */
-  liveExts(batch?: number): IterableIterator<number>;
-  /** Size the two slabs from the shared memory budget and the actual code
-   *  size (known on the first record cached): total slots = budget / bytes
-   *  per slot, the upper-layer slab taking half.  A slot is the code bytes
-   *  plus 17 B of typed-array bookkeeping (key + ext + deleted). */
-  private deriveSlabs;
-  /** Record a node in the slab cache.  Upper-layer nodes (level ≥ 1) go to
-   *  the pinned section layer-0 traffic never touches; a colliding slot is
-   *  simply overwritten (direct-mapped eviction — no bookkeeping). */
-  private cacheRec;
-  /** Insert a node, returning its assigned id (rowid). */
-  addNode(ext: number | null, level: number, code: Uint8Array): number;
-  /** Fetch a node's code (copied), tombstone flag and external id, or null.
-   *  Served from the slab cache when present; a miss reads SQLite, counts
-   *  one `reads` (the cache-independent scalability witness), and caches the
-   *  result. The cache is a latency layer ONLY — `reads` does not count hits, so
-   *  the index's storage-read scaling is identical with the cache off. */
-  getNode(id: number): NodeRec | null;
-  idByExt(ext: number): number | null;
-  private readonly batchStmts;
-  private static readonly BATCH_MAX;
-  private batchStmt;
-  /** Fetch several nodes into `out` (skipping ids already present).  Serves
-   *  from the code LRU first; the misses are read with one IN query per
-   *  chunk.  Ids with no row are simply absent from `out`.  Each id that
-   *  reaches SQLite counts one `reads`, exactly like a getNode miss. */
-  getNodesInto(ids: number[], out: Map<number, NodeRec>, count?: number): void;
-  /** Pre-fill the code and neighbour-list caches with ONE sequential table
-   *  scan each, up to their existing budget-derived caps.  A cold session
-   *  otherwise warms the caches through hundreds of thousands of RANDOM point
-   *  reads spread across its first minutes of inserts/queries; a sequential
-   *  scan streams the same rows at C speed in seconds.  A pure latency
-   *  optimisation with the exact same caps and coherence rules as demand
-   *  filling — nothing about results, `reads` discipline, or memory ceilings
-   *  changes.  Two passes over `nodes` so the upper-layer pinned section is
-   *  filled before layer-0 rows compete for the LRU.  Returns rows warmed. */
-  warmCache(): number;
-  tombstone(id: number): void;
-  /** A node's neighbour ids at a layer, or null if it has no list there. */
-  getNeighbors(node: number, layer: number): Uint32Array | null;
-  setNeighbors(node: number, layer: number, ids: number[]): void;
-  begin(): void;
-  commit(): void;
-  rollback(): void;
-  /** Encode a neighbour id list to the delta-varint blob format. */
-  private encodeNbrs;
-  /** Decode a delta-varint neighbour blob into a number[]. */
-  private decodeNbrs;
-  /** Tombstone-splice compaction.  `M`/`Mmax0` cap a rewritten list on upper
-   *  layers / layer 0.  Returns the new global state scalars (internal ids are
-   *  preserved, so `entry` survives unless it was itself dead).  The caller
-   *  persists state and vacuums. */
-  spliceCompact(M: number, Mmax0: number, entry: number): {
-    entry: number;
-    maxLevel: number;
-    live: number;
-  };
-  /** Stream live external ids with internal id > `after`, in internal-id
-   *  order, in bounded batches.  Internal ids are assigned monotonically and
-   *  PRESERVED by {@link spliceCompact}, so a caller can use the largest
-   *  internal id it has seen as a durable incremental watermark. */
-  liveExtsSince(after: number, batch?: number): IterableIterator<{
-    ext: number;
-    internal: number;
-  }>;
-  vacuum(): void;
-  close(): void;
-}