@hviana/sema 0.1.5 → 0.1.6

package/dist/src/rabitq-hnsw/src/database.js

@@ -1,405 +0,0 @@
-import { HnswIndex } from "./hnsw.js";
-import { RaBitQuantizer } from "./rabitq.js";
-import { Store } from "./store.js";
-const DEFAULTS = {
-  M: 16,
-  efConstruction: 200,
-  efSearch: 100,
-  queryBits: 8,
-  rotationRounds: 3,
-  seed: 0x1234abcd,
-};
-/**
- * A persistent vector database: an HNSW graph over 1-bit RaBitQ codes (cosine),
- * stored entirely in SQLite at `dbPath`. The graph IS the database -- there is no
- * load/save step and no in-RAM copy of the data, so resident memory stays flat as
- * the collection grows and the store survives process restarts. Reopening the
- * same path restores the exact configuration (the rotation is regenerated from
- * the persisted seed), so the codes already on disk remain valid.
- *
- * External ids are integers. The original float vectors are never retained --
- * only the sign codes -- so a 256-d vector costs 32 bytes instead of 1024. `get`
- * returns the stored code, which can be fed straight back into
- * `insert`/`update`/`query`.
- */
-export class VectorDatabase {
-  dim;
-  store;
-  quantizer;
-  index;
-  codeWords;
-  constructor(options) {
-    if (
-      !options || typeof options.dbPath !== "string" ||
-      options.dbPath.length === 0
-    ) {
-      throw new Error("DatabaseOptions.dbPath (string) is required");
-    }
-    this.store = new Store(options.dbPath, options.cacheSizeMb ?? 64);
-    let cfg = this.store.loadConfig();
-    if (cfg === null) {
-      // Fresh database: derive the configuration from the options and persist it.
-      if (!Number.isInteger(options.dim) || options.dim <= 0) {
-        throw new Error(
-          "DatabaseOptions.dim must be a positive integer for a new database",
-        );
-      }
-      const dim = options.dim;
-      const centroid = options.centroid
-        ? Float64Array.from(options.centroid)
-        : null;
-      const probe = new RaBitQuantizer(dim, { rounds: 1, seed: 0 });
-      cfg = {
-        dim,
-        m: options.M ?? DEFAULTS.M,
-        efConstruction: options.efConstruction ?? DEFAULTS.efConstruction,
-        efSearch: options.efSearch ?? DEFAULTS.efSearch,
-        queryBits: options.queryBits ?? DEFAULTS.queryBits,
-        rotationRounds: options.rotationRounds ?? DEFAULTS.rotationRounds,
-        seed: (options.seed ?? DEFAULTS.seed) >>> 0,
-        centroid,
-        codeWords: probe.codeWords,
-        paddedDim: probe.paddedDim,
-      };
-      this.store.initConfig(cfg);
-    } else {
-      // Reopened: keep the stored STRUCTURAL config, but allow tuning the two
-      // query-side knobs — efSearch and queryBits shape only how a query is
-      // executed, never what is stored, so honouring an explicit option here
-      // is safe and lets an existing database adopt better query settings.
-      if (options.efSearch !== undefined) {
-        cfg.efSearch = options.efSearch;
-        this.store.setEfSearch(options.efSearch);
-      }
-      if (
-        options.queryBits !== undefined && options.queryBits !== cfg.queryBits
-      ) {
-        cfg.queryBits = options.queryBits;
-        this.store.setQueryBits(options.queryBits);
-      }
-    }
-    this.dim = cfg.dim;
-    this.quantizer = new RaBitQuantizer(cfg.dim, {
-      queryBits: cfg.queryBits,
-      rounds: cfg.rotationRounds,
-      seed: cfg.seed,
-      centroid: cfg.centroid ?? undefined,
-    });
-    if (this.quantizer.codeWords !== cfg.codeWords) {
-      throw new Error(
-        "Stored code geometry does not match the quantizer (corrupt database?)",
-      );
-    }
-    this.codeWords = this.quantizer.codeWords;
-    this.index = new HnswIndex(this.quantizer, this.store, {
-      M: cfg.m,
-      efConstruction: cfg.efConstruction,
-      efSearch: cfg.efSearch,
-      seed: cfg.seed,
-    });
-  }
-  /** Number of live (non-deleted) vectors. */
-  get size() {
-    return this.index.size;
-  }
-  /**
-   * Physical node count including tombstones from deletes/updates. When it grows
-   * well beyond `size`, call `compact()` to reclaim the space on disk.
-   */
-  get physicalSize() {
-    return this.index.physicalSize;
-  }
-  get efSearch() {
-    return this.index.efSearch;
-  }
-  set efSearch(value) {
-    this.index.efSearch = value;
-  }
-  /** Distance computations performed during the most recent query. */
-  get lastQueryDistanceComputations() {
-    return this.index.lastQueryDistComps;
-  }
-  /**
-   * Storage row reads issued by the most recent query. This is the honest,
-   * cache-independent scalability metric: it counts every node/neighbour fetch
-   * that hit the database, so disabling the cache cannot hide a bad access pattern.
-   */
-  get lastQueryStorageReads() {
-    return this.index.lastQueryStorageReads;
-  }
-  /** Per-vector storage cost of the index versus a Float32 baseline. */
-  get storage() {
-    const f32 = this.dim * 4;
-    const bpv = this.index.bytesPerVector;
-    return {
-      float32BytesPerVector: f32,
-      codeBytesPerVector: bpv,
-      bytesPerVector: bpv,
-      compressionRatio: f32 / bpv,
-    };
-  }
-  has(id) {
-    return this.store.idByExt(this.checkId(id)) !== null;
-  }
-  /** Stream every live external id (bounded memory). */
-  *keys() {
-    yield* this.store.liveExts();
-  }
-  /**
-   * Stream live entries whose INTERNAL id is > `after`, as
-   * {ext, internal} pairs in internal-id order.  Internal ids are assigned
-   * monotonically at insert and preserved by {@link compact}, so the largest
-   * internal id a caller has seen is a durable incremental watermark: a later
-   * call with it yields exactly the entries added since.
-   */
-  *keysSince(after) {
-    yield* this.store.liveExtsSince(after);
-  }
-  checkId(id) {
-    if (!Number.isInteger(id)) {
-      throw new Error(`External id must be an integer, got ${id}`);
-    }
-    return id;
-  }
-  /**
-   * Convert a value to code bytes, selecting by length:
-   *  - `codeWords` elements -> an existing 1-bit code (e.g. from `get()`)
-   *  - otherwise            -> a raw `dim`-vector, encoded first
-   * `codeWords < dim` for any dim >= 2, so the two never collide.
-   */
-  toCodeBytes(value) {
-    if (value.length === this.codeWords) {
-      return this.quantizer.codeToBytes(value);
-    }
-    if (value.length !== this.dim) {
-      throw new Error(
-        `Vector dimension mismatch: expected ${this.dim}, got ${value.length}`,
-      );
-    }
-    return this.quantizer.codeToBytes(this.quantizer.encode(value));
-  }
-  // ------------------------------- CRUD ------------------------------------
-  /**
-   * Create. Accepts a raw `dim`-vector or a 1-bit code (`codeWords` words),
-   * detected by length. Throws if the id already exists (use `update`/`upsert`).
-   */
-  insert(id, value) {
-    this.store.begin();
-    try {
-      this.insertCore(id, value);
-      this.store.commit();
-    } catch (e) {
-      this.store.rollback();
-      throw e;
-    }
-  }
-  /** Insert with the caller owning the transaction (used by {@link upsertMany}).
-   *  `ef` optionally narrows the construction beam for this one vector (a
-   *  caller-declared cheap entry, e.g. a reach-only interior); omitted means
-   *  the index's configured efConstruction. */
-  insertCore(id, value, ef) {
-    this.checkId(id);
-    if (this.store.idByExt(id) !== null) {
-      throw new Error(`External id already exists: ${id} (use update())`);
-    }
-    this.index.insert(id, this.toCodeBytes(value), ef);
-  }
-  upsert(id, value) {
-    const nodeId = this.store.idByExt(this.checkId(id));
-    if (nodeId === null) {
-      this.insert(id, value);
-      return;
-    }
-    this.store.begin();
-    try {
-      this.updateAt(nodeId, id, value);
-      this.store.commit();
-    } catch (e) {
-      this.store.rollback();
-      throw e;
-    }
-  }
-  /**
-   * Upsert many vectors under ONE transaction. The HNSW build touches the store
-   * on every wired edge, so a transaction per vector is one WAL commit per vector
-   * — which dominates a bulk load on disk. Wrapping the whole batch in a single
-   * transaction coalesces those commits into one while leaving the graph and its
-   * result identical (reads on the connection still see the uncommitted rows).
-   *
-   * This is purely about commit batching; it changes nothing about the storage
-   * model. Codes still live only in SQLite and are read on demand — the
-   * cache-independent per-operation read count is unchanged — so a batched load
-   * scales exactly as the per-item path does, just with far fewer fsyncs. A throw
-   * rolls the whole batch back, so the caller treats it as the per-item path on
-   * failure.
-   */
-  upsertMany(entries) {
-    if (entries.length === 0) {
-      return;
-    }
-    this.store.begin();
-    try {
-      for (const e of entries) {
-        const nodeId = this.store.idByExt(this.checkId(e.id));
-        if (nodeId !== null) {
-          this.updateAt(nodeId, e.id, e.vector, e.ef);
-        } else {
-          this.insertCore(e.id, e.vector, e.ef);
-        }
-      }
-      this.store.commit();
-    } catch (e) {
-      this.store.rollback();
-      throw e;
-    }
-  }
-  /**
-   * Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. The
-   * original vector is not retained; the code round-trips into insert/update/query.
-   */
-  get(id) {
-    const nodeId = this.store.idByExt(this.checkId(id));
-    if (nodeId === null) {
-      return null;
-    }
-    const rec = this.store.getNode(nodeId);
-    return rec ? this.quantizer.bytesToCode(rec.code) : null;
-  }
-  /**
-   * Update the vector bound to an id (raw vector or code, detected by length).
-   * The previous node is tombstoned and a fresh one inserted. Throws if absent.
-   */
-  update(id, value) {
-    this.store.begin();
-    try {
-      this.updateCore(id, value);
-      this.store.commit();
-    } catch (e) {
-      this.store.rollback();
-      throw e;
-    }
-  }
-  /** Update with the caller owning the transaction (used by {@link upsertMany}). */
-  updateCore(id, value) {
-    const oldId = this.store.idByExt(this.checkId(id));
-    if (oldId === null) {
-      throw new Error(`Unknown external id: ${id}`);
-    }
-    this.updateAt(oldId, id, value);
-  }
-  /** Update when the live internal node id is already known.
-   *
-   *  Skips the write entirely when the new code equals the stored one — the
-   *  update would tombstone the node and replay a full graph insert for a
-   *  byte-identical result. This is not a corner case: content-addressed
-   *  callers re-upsert unchanged vectors wholesale after a restart (the same
-   *  content always encodes to the same code), and each such no-op otherwise
-   *  costs a tombstone (permanent routing/disk overhead until compaction)
-   *  plus an O(ef·log N) reinsert. */
-  updateAt(nodeId, ext, value, ef) {
-    const bytes = this.toCodeBytes(value);
-    const rec = this.store.getNode(nodeId);
-    if (rec !== null && rec.code.length === bytes.length) {
-      let same = true;
-      for (let i = 0; i < bytes.length; i++) {
-        if (rec.code[i] !== bytes[i]) {
-          same = false;
-          break;
-        }
-      }
-      if (same) {
-        return; // identical code — the update is a no-op
-      }
-    }
-    this.index.remove(nodeId); // frees the ext id, then re-insert under it
-    this.index.insert(ext, bytes, ef);
-  }
-  /**
-   * Delete many ids under ONE transaction — same commit-coalescing rationale
-   * as {@link upsertMany}: a tombstone per implicit transaction is one WAL
-   * commit per id, which dominates a bulk prune. Absent ids are skipped.
-   * Returns the number of vectors actually removed.
-   */
-  deleteMany(ids) {
-    if (ids.length === 0) {
-      return 0;
-    }
-    let removed = 0;
-    this.store.begin();
-    try {
-      for (const id of ids) {
-        const nodeId = this.store.idByExt(this.checkId(id));
-        if (nodeId !== null && this.index.remove(nodeId)) {
-          removed++;
-        }
-      }
-      this.store.commit();
-    } catch (e) {
-      this.store.rollback();
-      throw e;
-    }
-    return removed;
-  }
-  /** Delete the vector bound to an id. Returns false if absent. */
-  delete(id) {
-    const nodeId = this.store.idByExt(this.checkId(id));
-    if (nodeId === null) {
-      return false;
-    }
-    this.store.begin();
-    try {
-      this.index.remove(nodeId);
-      this.store.commit();
-    } catch (e) {
-      this.store.rollback();
-      throw e;
-    }
-    return true;
-  }
-  /**
-   * Pre-fill the RAM caches (codes + neighbour lists) with sequential table
-   * scans, up to their budget-derived caps.  Optional and purely a latency
-   * optimisation: a cold session otherwise pays the same warming through
-   * random point reads over its first minutes.  Call once after open on a
-   * session that will do sustained inserts/queries.  Returns rows warmed;
-   * a 0-budget database returns 0 immediately.
-   */
-  warmCache() {
-    return this.store.warmCache();
-  }
-  /**
-   * Rebuild the graph from the live codes only, dropping tombstones and returning
-   * the freed pages to the filesystem. Lossless -- equivalent to the original
-   * build -- and needs no original vectors.
-   */
-  compact() {
-    this.index.compact();
-  }
-  // ------------------------------ search -----------------------------------
-  /**
-   * k-NN search. The argument's length selects the mode:
-   *  - `dim` elements       -> a raw vector (accurate 4-bit-query estimator)
-   *  - `codeWords` elements -> a 1-bit code (e.g. `get(id)`), by sign-bit Hamming
-   * Detection is by length, so a code is recognised whether it is a Uint32Array
-   * or a plain number[]. `codeWords < dim` for any dim >= 2, so they never collide.
-   */
-  query(query, k = 10, opts) {
-    if (query.length === this.codeWords) {
-      return this.index.searchKnnByCode(
-        this.quantizer.codeToBytes(query),
-        k,
-        opts?.ef,
-      );
-    }
-    if (query.length !== this.dim) {
-      throw new Error(
-        `Vector dimension mismatch: expected ${this.dim}, got ${query.length}`,
-      );
-    }
-    return this.index.searchKnn(query, k, opts?.ef);
-  }
-  /** Close the underlying database. The instance must not be used afterwards. */
-  close() {
-    this.store.close();
-  }
-}

package/dist/src/rabitq-hnsw/src/heap.d.ts

@@ -1,22 +0,0 @@
-/**
- * Binary heap over two parallel numeric arrays storing (key, value) pairs.
- *
- * When `minHeap` is true the smallest key sits at the root, otherwise the
- * largest. Keys are distances, values are integer node ids. Parallel plain
- * arrays are used (rather than an array of objects) to avoid per-element
- * allocation in the search hot path.
- */
-export declare class Heap {
-  readonly keys: number[];
-  readonly vals: number[];
-  private readonly minHeap;
-  constructor(minHeap: boolean);
-  get size(): number;
-  clear(): void;
-  topKey(): number;
-  topVal(): number;
-  /** true if `a` belongs closer to the root than `b`. */
-  private higher;
-  push(key: number, val: number): void;
-  pop(): void;
-}

package/dist/src/rabitq-hnsw/src/heap.js

@@ -1,94 +0,0 @@
-/**
- * Binary heap over two parallel numeric arrays storing (key, value) pairs.
- *
- * When `minHeap` is true the smallest key sits at the root, otherwise the
- * largest. Keys are distances, values are integer node ids. Parallel plain
- * arrays are used (rather than an array of objects) to avoid per-element
- * allocation in the search hot path.
- */
-export class Heap {
-  keys = [];
-  vals = [];
-  minHeap;
-  constructor(minHeap) {
-    this.minHeap = minHeap;
-  }
-  get size() {
-    return this.keys.length;
-  }
-  clear() {
-    this.keys.length = 0;
-    this.vals.length = 0;
-  }
-  topKey() {
-    return this.keys[0];
-  }
-  topVal() {
-    return this.vals[0];
-  }
-  /** true if `a` belongs closer to the root than `b`. */
-  higher(a, b) {
-    return this.minHeap ? a < b : a > b;
-  }
-  push(key, val) {
-    const keys = this.keys;
-    const vals = this.vals;
-    let i = keys.length;
-    keys.push(key);
-    vals.push(val);
-    while (i > 0) {
-      const parent = (i - 1) >> 1;
-      if (this.higher(keys[i], keys[parent])) {
-        const tk = keys[i];
-        keys[i] = keys[parent];
-        keys[parent] = tk;
-        const tv = vals[i];
-        vals[i] = vals[parent];
-        vals[parent] = tv;
-        i = parent;
-      } else {
-        break;
-      }
-    }
-  }
-  pop() {
-    const keys = this.keys;
-    const vals = this.vals;
-    const n = keys.length;
-    if (n === 0) {
-      return;
-    }
-    const lastKey = keys[n - 1];
-    const lastVal = vals[n - 1];
-    keys.pop();
-    vals.pop();
-    const m = keys.length;
-    if (m === 0) {
-      return;
-    }
-    keys[0] = lastKey;
-    vals[0] = lastVal;
-    let i = 0;
-    while (true) {
-      const left = 2 * i + 1;
-      const right = left + 1;
-      let best = i;
-      if (left < m && this.higher(keys[left], keys[best])) {
-        best = left;
-      }
-      if (right < m && this.higher(keys[right], keys[best])) {
-        best = right;
-      }
-      if (best === i) {
-        break;
-      }
-      const tk = keys[i];
-      keys[i] = keys[best];
-      keys[best] = tk;
-      const tv = vals[i];
-      vals[i] = vals[best];
-      vals[best] = tv;
-      i = best;
-    }
-  }
-}

package/dist/src/rabitq-hnsw/src/hnsw.d.ts

@@ -1,125 +0,0 @@
-import { RaBitQuantizer } from "./rabitq.js";
-import { Store } from "./store.js";
-export interface HnswParams {
-  M: number;
-  efConstruction: number;
-  efSearch: number;
-  seed: number;
-}
-export interface KnnHit {
-  /** external (user) id */
-  id: number;
-  /** estimated cosine distance (1 - cosine) */
-  distance: number;
-}
-/**
- * Hierarchical Navigable Small World graph (Malkov & Yashunin, 2018) backed by
- * 1-bit RaBitQ codes that live in SQLite (see Store) rather than in RAM. The
- * algorithm is the textbook one; the only difference is that codes and adjacency
- * lists are read and written through the store on demand, so the resident set is
- * the working set of the current operation -- never the whole graph.
- *
- * Distances use the codes throughout:
- *   - searching with a full-precision query uses RaBitQ's accurate estimator;
- *   - building the graph and searching by code compare two codes by sign-bit
- *     Hamming distance.
- * Building on codes alone is what lets `compact` rebuild an honest index with no
- * access to the original vectors.
- */
-export declare class HnswIndex {
-  readonly M: number;
-  readonly Mmax0: number;
-  readonly efConstruction: number;
-  private readonly mL;
-  private readonly quantizer;
-  private readonly store;
-  private readonly seed;
-  private readonly levelRng;
-  private entry;
-  private maxLevel;
-  private live;
-  private total;
-  private _efSearch;
-  private readonly working;
-  private readonly visited;
-  private visitedTag;
-  private visitedEpoch;
-  private seenTag;
-  private markVisited;
-  private readonly candHeap;
-  private readonly resHeap;
-  private readonly singleEp;
-  private readonly idxScratch;
-  private readonly distScratch;
-  private readonly freshScratch;
-  private qCtx;
-  private refBytes;
-  private building;
-  lastQueryDistComps: number;
-  /** Storage row reads issued by the most recent query (cache-independent). */
-  lastQueryStorageReads: number;
-  constructor(quantizer: RaBitQuantizer, store: Store, params: HnswParams);
-  get size(): number;
-  get physicalSize(): number;
-  get bytesPerVector(): number;
-  get efSearch(): number;
-  set efSearch(v: number);
-  private persistState;
-  private randomLevel;
-  /**
-   * Fetch a node record into the operation's working set. The first touch in an
-   * op is a storage read; later touches in the same op reuse it. The set is
-   * cleared per op and bounded by efConstruction / efSearch, so storage reads
-   * per op equal the number of *distinct* nodes the op visits -- minimal, and
-   * independent of any cache.
-   */
-  private node;
-  /** Prefetch several nodes into the working set with ONE batched storage
-   *  read for the misses.  Point queries per neighbour made statement
-   *  dispatch — not distance arithmetic — the dominant cost of a large
-   *  build; the batch keeps `reads` accounting identical per row. */
-  private fetchNodes;
-  /** Distance from the current source (query vector or reference code) to a node. */
-  private distOf;
-  /** Code-to-code distance between two stored nodes (graph wiring only). */
-  private codeDist;
-  /**
-   * SEARCH-LAYER (Algorithm 2). Frontier in `candHeap`, bounded result set in
-   * `resHeap` (non-deleted only). Deleted nodes are traversed for routing but
-   * never returned. `visited` is a per-call set sized by the nodes seen here.
-   */
-  private searchLayer;
-  /** Greedy single-best descent from `fromLayer` down to (but not into) `toLayer`. */
-  private greedyDescend;
-  /**
-   * SELECT-NEIGHBORS-HEURISTIC (Algorithm 4) on codes. Picks up to `M` diverse
-   * neighbours from `candIds`, preferring those closer to `base` than to any
-   * already-chosen neighbour, then fills remaining slots with the closest
-   * leftovers (keep-pruned connections). Appends to `out`.
-   */
-  private selectNeighbors;
-  /**
-   * Insert a vector's code under external id `ext`; returns the internal node id.
-   * All graph reads/writes go through the store; the caller owns the transaction.
-   */
-  insert(ext: number, code: Uint8Array, efC?: number): number;
-  /** k-NN with a full-precision query vector (accurate estimator). */
-  searchKnn(vec: ArrayLike<number>, k: number, ef?: number): KnnHit[];
-  /** k-NN with an already-quantized code (sign-bit Hamming / angular distance). */
-  searchKnnByCode(codeBytes: Uint8Array, k: number, ef?: number): KnnHit[];
-  private collectKnn;
-  /** Tombstone a live node by internal id. Caller owns the transaction. */
-  remove(id: number): boolean;
-  /**
-   * Reclaim tombstones with a graph-preserving splice (see
-   * {@link Store.spliceCompact}): live nodes keep their internal ids and their
-   * wiring; each dead neighbour is replaced by its own live neighbours.  Codes
-   * are untouched, so the index loses nothing beyond what deletion itself
-   * removes.  The previous implementation replayed every live code through a
-   * full HNSW insert — O(live · ef · log N) storage reads, HOURS on a trained
-   * multi-million-node store, inside one WAL-bloating transaction.  The splice
-   * is a streaming pass with batched commits, then a VACUUM to return the
-   * freed pages.
-   */
-  compact(): void;
-}