@hviana/sema 0.1.4 → 0.1.6

package/dist/src/rabitq-hnsw/src/database.js

@@ -1,388 +0,0 @@
-import { HnswIndex } from "./hnsw.js";
-import { RaBitQuantizer } from "./rabitq.js";
-import { Store } from "./store.js";
-const DEFAULTS = {
-    M: 16,
-    efConstruction: 200,
-    efSearch: 100,
-    queryBits: 8,
-    rotationRounds: 3,
-    seed: 0x1234abcd,
-};
-/**
- * A persistent vector database: an HNSW graph over 1-bit RaBitQ codes (cosine),
- * stored entirely in SQLite at `dbPath`. The graph IS the database -- there is no
- * load/save step and no in-RAM copy of the data, so resident memory stays flat as
- * the collection grows and the store survives process restarts. Reopening the
- * same path restores the exact configuration (the rotation is regenerated from
- * the persisted seed), so the codes already on disk remain valid.
- *
- * External ids are integers. The original float vectors are never retained --
- * only the sign codes -- so a 256-d vector costs 32 bytes instead of 1024. `get`
- * returns the stored code, which can be fed straight back into
- * `insert`/`update`/`query`.
- */
-export class VectorDatabase {
-    dim;
-    store;
-    quantizer;
-    index;
-    codeWords;
-    constructor(options) {
-        if (!options || typeof options.dbPath !== "string" ||
-            options.dbPath.length === 0) {
-            throw new Error("DatabaseOptions.dbPath (string) is required");
-        }
-        this.store = new Store(options.dbPath, options.cacheSizeMb ?? 64);
-        let cfg = this.store.loadConfig();
-        if (cfg === null) {
-            // Fresh database: derive the configuration from the options and persist it.
-            if (!Number.isInteger(options.dim) || options.dim <= 0) {
-                throw new Error("DatabaseOptions.dim must be a positive integer for a new database");
-            }
-            const dim = options.dim;
-            const centroid = options.centroid
-                ? Float64Array.from(options.centroid)
-                : null;
-            const probe = new RaBitQuantizer(dim, { rounds: 1, seed: 0 });
-            cfg = {
-                dim,
-                m: options.M ?? DEFAULTS.M,
-                efConstruction: options.efConstruction ?? DEFAULTS.efConstruction,
-                efSearch: options.efSearch ?? DEFAULTS.efSearch,
-                queryBits: options.queryBits ?? DEFAULTS.queryBits,
-                rotationRounds: options.rotationRounds ?? DEFAULTS.rotationRounds,
-                seed: (options.seed ?? DEFAULTS.seed) >>> 0,
-                centroid,
-                codeWords: probe.codeWords,
-                paddedDim: probe.paddedDim,
-            };
-            this.store.initConfig(cfg);
-        }
-        else {
-            // Reopened: keep the stored STRUCTURAL config, but allow tuning the two
-            // query-side knobs — efSearch and queryBits shape only how a query is
-            // executed, never what is stored, so honouring an explicit option here
-            // is safe and lets an existing database adopt better query settings.
-            if (options.efSearch !== undefined) {
-                cfg.efSearch = options.efSearch;
-                this.store.setEfSearch(options.efSearch);
-            }
-            if (options.queryBits !== undefined && options.queryBits !== cfg.queryBits) {
-                cfg.queryBits = options.queryBits;
-                this.store.setQueryBits(options.queryBits);
-            }
-        }
-        this.dim = cfg.dim;
-        this.quantizer = new RaBitQuantizer(cfg.dim, {
-            queryBits: cfg.queryBits,
-            rounds: cfg.rotationRounds,
-            seed: cfg.seed,
-            centroid: cfg.centroid ?? undefined,
-        });
-        if (this.quantizer.codeWords !== cfg.codeWords) {
-            throw new Error("Stored code geometry does not match the quantizer (corrupt database?)");
-        }
-        this.codeWords = this.quantizer.codeWords;
-        this.index = new HnswIndex(this.quantizer, this.store, {
-            M: cfg.m,
-            efConstruction: cfg.efConstruction,
-            efSearch: cfg.efSearch,
-            seed: cfg.seed,
-        });
-    }
-    /** Number of live (non-deleted) vectors. */
-    get size() {
-        return this.index.size;
-    }
-    /**
-     * Physical node count including tombstones from deletes/updates. When it grows
-     * well beyond `size`, call `compact()` to reclaim the space on disk.
-     */
-    get physicalSize() {
-        return this.index.physicalSize;
-    }
-    get efSearch() {
-        return this.index.efSearch;
-    }
-    set efSearch(value) {
-        this.index.efSearch = value;
-    }
-    /** Distance computations performed during the most recent query. */
-    get lastQueryDistanceComputations() {
-        return this.index.lastQueryDistComps;
-    }
-    /**
-     * Storage row reads issued by the most recent query. This is the honest,
-     * cache-independent scalability metric: it counts every node/neighbour fetch
-     * that hit the database, so disabling the cache cannot hide a bad access pattern.
-     */
-    get lastQueryStorageReads() {
-        return this.index.lastQueryStorageReads;
-    }
-    /** Per-vector storage cost of the index versus a Float32 baseline. */
-    get storage() {
-        const f32 = this.dim * 4;
-        const bpv = this.index.bytesPerVector;
-        return {
-            float32BytesPerVector: f32,
-            codeBytesPerVector: bpv,
-            bytesPerVector: bpv,
-            compressionRatio: f32 / bpv,
-        };
-    }
-    has(id) {
-        return this.store.idByExt(this.checkId(id)) !== null;
-    }
-    /** Stream every live external id (bounded memory). */
-    *keys() {
-        yield* this.store.liveExts();
-    }
-    /**
-     * Stream live entries whose INTERNAL id is > `after`, as
-     * {ext, internal} pairs in internal-id order.  Internal ids are assigned
-     * monotonically at insert and preserved by {@link compact}, so the largest
-     * internal id a caller has seen is a durable incremental watermark: a later
-     * call with it yields exactly the entries added since.
-     */
-    *keysSince(after) {
-        yield* this.store.liveExtsSince(after);
-    }
-    checkId(id) {
-        if (!Number.isInteger(id)) {
-            throw new Error(`External id must be an integer, got ${id}`);
-        }
-        return id;
-    }
-    /**
-     * Convert a value to code bytes, selecting by length:
-     *  - `codeWords` elements -> an existing 1-bit code (e.g. from `get()`)
-     *  - otherwise            -> a raw `dim`-vector, encoded first
-     * `codeWords < dim` for any dim >= 2, so the two never collide.
-     */
-    toCodeBytes(value) {
-        if (value.length === this.codeWords) {
-            return this.quantizer.codeToBytes(value);
-        }
-        if (value.length !== this.dim) {
-            throw new Error(`Vector dimension mismatch: expected ${this.dim}, got ${value.length}`);
-        }
-        return this.quantizer.codeToBytes(this.quantizer.encode(value));
-    }
-    // ------------------------------- CRUD ------------------------------------
-    /**
-     * Create. Accepts a raw `dim`-vector or a 1-bit code (`codeWords` words),
-     * detected by length. Throws if the id already exists (use `update`/`upsert`).
-     */
-    insert(id, value) {
-        this.store.begin();
-        try {
-            this.insertCore(id, value);
-            this.store.commit();
-        }
-        catch (e) {
-            this.store.rollback();
-            throw e;
-        }
-    }
-    /** Insert with the caller owning the transaction (used by {@link upsertMany}).
-     *  `ef` optionally narrows the construction beam for this one vector (a
-     *  caller-declared cheap entry, e.g. a reach-only interior); omitted means
-     *  the index's configured efConstruction. */
-    insertCore(id, value, ef) {
-        this.checkId(id);
-        if (this.store.idByExt(id) !== null) {
-            throw new Error(`External id already exists: ${id} (use update())`);
-        }
-        this.index.insert(id, this.toCodeBytes(value), ef);
-    }
-    upsert(id, value) {
-        const nodeId = this.store.idByExt(this.checkId(id));
-        if (nodeId === null) {
-            this.insert(id, value);
-            return;
-        }
-        this.store.begin();
-        try {
-            this.updateAt(nodeId, id, value);
-            this.store.commit();
-        }
-        catch (e) {
-            this.store.rollback();
-            throw e;
-        }
-    }
-    /**
-     * Upsert many vectors under ONE transaction. The HNSW build touches the store
-     * on every wired edge, so a transaction per vector is one WAL commit per vector
-     * — which dominates a bulk load on disk. Wrapping the whole batch in a single
-     * transaction coalesces those commits into one while leaving the graph and its
-     * result identical (reads on the connection still see the uncommitted rows).
-     *
-     * This is purely about commit batching; it changes nothing about the storage
-     * model. Codes still live only in SQLite and are read on demand — the
-     * cache-independent per-operation read count is unchanged — so a batched load
-     * scales exactly as the per-item path does, just with far fewer fsyncs. A throw
-     * rolls the whole batch back, so the caller treats it as the per-item path on
-     * failure.
-     */
-    upsertMany(entries) {
-        if (entries.length === 0)
-            return;
-        this.store.begin();
-        try {
-            for (const e of entries) {
-                const nodeId = this.store.idByExt(this.checkId(e.id));
-                if (nodeId !== null)
-                    this.updateAt(nodeId, e.id, e.vector, e.ef);
-                else
-                    this.insertCore(e.id, e.vector, e.ef);
-            }
-            this.store.commit();
-        }
-        catch (e) {
-            this.store.rollback();
-            throw e;
-        }
-    }
-    /**
-     * Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. The
-     * original vector is not retained; the code round-trips into insert/update/query.
-     */
-    get(id) {
-        const nodeId = this.store.idByExt(this.checkId(id));
-        if (nodeId === null)
-            return null;
-        const rec = this.store.getNode(nodeId);
-        return rec ? this.quantizer.bytesToCode(rec.code) : null;
-    }
-    /**
-     * Update the vector bound to an id (raw vector or code, detected by length).
-     * The previous node is tombstoned and a fresh one inserted. Throws if absent.
-     */
-    update(id, value) {
-        this.store.begin();
-        try {
-            this.updateCore(id, value);
-            this.store.commit();
-        }
-        catch (e) {
-            this.store.rollback();
-            throw e;
-        }
-    }
-    /** Update with the caller owning the transaction (used by {@link upsertMany}). */
-    updateCore(id, value) {
-        const oldId = this.store.idByExt(this.checkId(id));
-        if (oldId === null)
-            throw new Error(`Unknown external id: ${id}`);
-        this.updateAt(oldId, id, value);
-    }
-    /** Update when the live internal node id is already known.
-     *
-     *  Skips the write entirely when the new code equals the stored one — the
-     *  update would tombstone the node and replay a full graph insert for a
-     *  byte-identical result. This is not a corner case: content-addressed
-     *  callers re-upsert unchanged vectors wholesale after a restart (the same
-     *  content always encodes to the same code), and each such no-op otherwise
-     *  costs a tombstone (permanent routing/disk overhead until compaction)
-     *  plus an O(ef·log N) reinsert. */
-    updateAt(nodeId, ext, value, ef) {
-        const bytes = this.toCodeBytes(value);
-        const rec = this.store.getNode(nodeId);
-        if (rec !== null && rec.code.length === bytes.length) {
-            let same = true;
-            for (let i = 0; i < bytes.length; i++) {
-                if (rec.code[i] !== bytes[i]) {
-                    same = false;
-                    break;
-                }
-            }
-            if (same)
-                return; // identical code — the update is a no-op
-        }
-        this.index.remove(nodeId); // frees the ext id, then re-insert under it
-        this.index.insert(ext, bytes, ef);
-    }
-    /**
-     * Delete many ids under ONE transaction — same commit-coalescing rationale
-     * as {@link upsertMany}: a tombstone per implicit transaction is one WAL
-     * commit per id, which dominates a bulk prune. Absent ids are skipped.
-     * Returns the number of vectors actually removed.
-     */
-    deleteMany(ids) {
-        if (ids.length === 0)
-            return 0;
-        let removed = 0;
-        this.store.begin();
-        try {
-            for (const id of ids) {
-                const nodeId = this.store.idByExt(this.checkId(id));
-                if (nodeId !== null && this.index.remove(nodeId))
-                    removed++;
-            }
-            this.store.commit();
-        }
-        catch (e) {
-            this.store.rollback();
-            throw e;
-        }
-        return removed;
-    }
-    /** Delete the vector bound to an id. Returns false if absent. */
-    delete(id) {
-        const nodeId = this.store.idByExt(this.checkId(id));
-        if (nodeId === null)
-            return false;
-        this.store.begin();
-        try {
-            this.index.remove(nodeId);
-            this.store.commit();
-        }
-        catch (e) {
-            this.store.rollback();
-            throw e;
-        }
-        return true;
-    }
-    /**
-     * Pre-fill the RAM caches (codes + neighbour lists) with sequential table
-     * scans, up to their budget-derived caps.  Optional and purely a latency
-     * optimisation: a cold session otherwise pays the same warming through
-     * random point reads over its first minutes.  Call once after open on a
-     * session that will do sustained inserts/queries.  Returns rows warmed;
-     * a 0-budget database returns 0 immediately.
-     */
-    warmCache() {
-        return this.store.warmCache();
-    }
-    /**
-     * Rebuild the graph from the live codes only, dropping tombstones and returning
-     * the freed pages to the filesystem. Lossless -- equivalent to the original
-     * build -- and needs no original vectors.
-     */
-    compact() {
-        this.index.compact();
-    }
-    // ------------------------------ search -----------------------------------
-    /**
-     * k-NN search. The argument's length selects the mode:
-     *  - `dim` elements       -> a raw vector (accurate 4-bit-query estimator)
-     *  - `codeWords` elements -> a 1-bit code (e.g. `get(id)`), by sign-bit Hamming
-     * Detection is by length, so a code is recognised whether it is a Uint32Array
-     * or a plain number[]. `codeWords < dim` for any dim >= 2, so they never collide.
-     */
-    query(query, k = 10, opts) {
-        if (query.length === this.codeWords) {
-            return this.index.searchKnnByCode(this.quantizer.codeToBytes(query), k, opts?.ef);
-        }
-        if (query.length !== this.dim) {
-            throw new Error(`Vector dimension mismatch: expected ${this.dim}, got ${query.length}`);
-        }
-        return this.index.searchKnn(query, k, opts?.ef);
-    }
-    /** Close the underlying database. The instance must not be used afterwards. */
-    close() {
-        this.store.close();
-    }
-}

package/dist/src/rabitq-hnsw/src/heap.d.ts

@@ -1,22 +0,0 @@
-/**
- * Binary heap over two parallel numeric arrays storing (key, value) pairs.
- *
- * When `minHeap` is true the smallest key sits at the root, otherwise the
- * largest. Keys are distances, values are integer node ids. Parallel plain
- * arrays are used (rather than an array of objects) to avoid per-element
- * allocation in the search hot path.
- */
-export declare class Heap {
-    readonly keys: number[];
-    readonly vals: number[];
-    private readonly minHeap;
-    constructor(minHeap: boolean);
-    get size(): number;
-    clear(): void;
-    topKey(): number;
-    topVal(): number;
-    /** true if `a` belongs closer to the root than `b`. */
-    private higher;
-    push(key: number, val: number): void;
-    pop(): void;
-}

package/dist/src/rabitq-hnsw/src/heap.js

@@ -1,89 +0,0 @@
-/**
- * Binary heap over two parallel numeric arrays storing (key, value) pairs.
- *
- * When `minHeap` is true the smallest key sits at the root, otherwise the
- * largest. Keys are distances, values are integer node ids. Parallel plain
- * arrays are used (rather than an array of objects) to avoid per-element
- * allocation in the search hot path.
- */
-export class Heap {
-    keys = [];
-    vals = [];
-    minHeap;
-    constructor(minHeap) {
-        this.minHeap = minHeap;
-    }
-    get size() {
-        return this.keys.length;
-    }
-    clear() {
-        this.keys.length = 0;
-        this.vals.length = 0;
-    }
-    topKey() {
-        return this.keys[0];
-    }
-    topVal() {
-        return this.vals[0];
-    }
-    /** true if `a` belongs closer to the root than `b`. */
-    higher(a, b) {
-        return this.minHeap ? a < b : a > b;
-    }
-    push(key, val) {
-        const keys = this.keys;
-        const vals = this.vals;
-        let i = keys.length;
-        keys.push(key);
-        vals.push(val);
-        while (i > 0) {
-            const parent = (i - 1) >> 1;
-            if (this.higher(keys[i], keys[parent])) {
-                const tk = keys[i];
-                keys[i] = keys[parent];
-                keys[parent] = tk;
-                const tv = vals[i];
-                vals[i] = vals[parent];
-                vals[parent] = tv;
-                i = parent;
-            }
-            else
-                break;
-        }
-    }
-    pop() {
-        const keys = this.keys;
-        const vals = this.vals;
-        const n = keys.length;
-        if (n === 0)
-            return;
-        const lastKey = keys[n - 1];
-        const lastVal = vals[n - 1];
-        keys.pop();
-        vals.pop();
-        const m = keys.length;
-        if (m === 0)
-            return;
-        keys[0] = lastKey;
-        vals[0] = lastVal;
-        let i = 0;
-        while (true) {
-            const left = 2 * i + 1;
-            const right = left + 1;
-            let best = i;
-            if (left < m && this.higher(keys[left], keys[best]))
-                best = left;
-            if (right < m && this.higher(keys[right], keys[best]))
-                best = right;
-            if (best === i)
-                break;
-            const tk = keys[i];
-            keys[i] = keys[best];
-            keys[best] = tk;
-            const tv = vals[i];
-            vals[i] = vals[best];
-            vals[best] = tv;
-            i = best;
-        }
-    }
-}

package/dist/src/rabitq-hnsw/src/hnsw.d.ts

@@ -1,125 +0,0 @@
-import { RaBitQuantizer } from "./rabitq.js";
-import { Store } from "./store.js";
-export interface HnswParams {
-    M: number;
-    efConstruction: number;
-    efSearch: number;
-    seed: number;
-}
-export interface KnnHit {
-    /** external (user) id */
-    id: number;
-    /** estimated cosine distance (1 - cosine) */
-    distance: number;
-}
-/**
- * Hierarchical Navigable Small World graph (Malkov & Yashunin, 2018) backed by
- * 1-bit RaBitQ codes that live in SQLite (see Store) rather than in RAM. The
- * algorithm is the textbook one; the only difference is that codes and adjacency
- * lists are read and written through the store on demand, so the resident set is
- * the working set of the current operation -- never the whole graph.
- *
- * Distances use the codes throughout:
- *   - searching with a full-precision query uses RaBitQ's accurate estimator;
- *   - building the graph and searching by code compare two codes by sign-bit
- *     Hamming distance.
- * Building on codes alone is what lets `compact` rebuild an honest index with no
- * access to the original vectors.
- */
-export declare class HnswIndex {
-    readonly M: number;
-    readonly Mmax0: number;
-    readonly efConstruction: number;
-    private readonly mL;
-    private readonly quantizer;
-    private readonly store;
-    private readonly seed;
-    private readonly levelRng;
-    private entry;
-    private maxLevel;
-    private live;
-    private total;
-    private _efSearch;
-    private readonly working;
-    private readonly visited;
-    private visitedTag;
-    private visitedEpoch;
-    private seenTag;
-    private markVisited;
-    private readonly candHeap;
-    private readonly resHeap;
-    private readonly singleEp;
-    private readonly idxScratch;
-    private readonly distScratch;
-    private readonly freshScratch;
-    private qCtx;
-    private refBytes;
-    private building;
-    lastQueryDistComps: number;
-    /** Storage row reads issued by the most recent query (cache-independent). */
-    lastQueryStorageReads: number;
-    constructor(quantizer: RaBitQuantizer, store: Store, params: HnswParams);
-    get size(): number;
-    get physicalSize(): number;
-    get bytesPerVector(): number;
-    get efSearch(): number;
-    set efSearch(v: number);
-    private persistState;
-    private randomLevel;
-    /**
-     * Fetch a node record into the operation's working set. The first touch in an
-     * op is a storage read; later touches in the same op reuse it. The set is
-     * cleared per op and bounded by efConstruction / efSearch, so storage reads
-     * per op equal the number of *distinct* nodes the op visits -- minimal, and
-     * independent of any cache.
-     */
-    private node;
-    /** Prefetch several nodes into the working set with ONE batched storage
-     *  read for the misses.  Point queries per neighbour made statement
-     *  dispatch — not distance arithmetic — the dominant cost of a large
-     *  build; the batch keeps `reads` accounting identical per row. */
-    private fetchNodes;
-    /** Distance from the current source (query vector or reference code) to a node. */
-    private distOf;
-    /** Code-to-code distance between two stored nodes (graph wiring only). */
-    private codeDist;
-    /**
-     * SEARCH-LAYER (Algorithm 2). Frontier in `candHeap`, bounded result set in
-     * `resHeap` (non-deleted only). Deleted nodes are traversed for routing but
-     * never returned. `visited` is a per-call set sized by the nodes seen here.
-     */
-    private searchLayer;
-    /** Greedy single-best descent from `fromLayer` down to (but not into) `toLayer`. */
-    private greedyDescend;
-    /**
-     * SELECT-NEIGHBORS-HEURISTIC (Algorithm 4) on codes. Picks up to `M` diverse
-     * neighbours from `candIds`, preferring those closer to `base` than to any
-     * already-chosen neighbour, then fills remaining slots with the closest
-     * leftovers (keep-pruned connections). Appends to `out`.
-     */
-    private selectNeighbors;
-    /**
-     * Insert a vector's code under external id `ext`; returns the internal node id.
-     * All graph reads/writes go through the store; the caller owns the transaction.
-     */
-    insert(ext: number, code: Uint8Array, efC?: number): number;
-    /** k-NN with a full-precision query vector (accurate estimator). */
-    searchKnn(vec: ArrayLike<number>, k: number, ef?: number): KnnHit[];
-    /** k-NN with an already-quantized code (sign-bit Hamming / angular distance). */
-    searchKnnByCode(codeBytes: Uint8Array, k: number, ef?: number): KnnHit[];
-    private collectKnn;
-    /** Tombstone a live node by internal id. Caller owns the transaction. */
-    remove(id: number): boolean;
-    /**
-     * Reclaim tombstones with a graph-preserving splice (see
-     * {@link Store.spliceCompact}): live nodes keep their internal ids and their
-     * wiring; each dead neighbour is replaced by its own live neighbours.  Codes
-     * are untouched, so the index loses nothing beyond what deletion itself
-     * removes.  The previous implementation replayed every live code through a
-     * full HNSW insert — O(live · ef · log N) storage reads, HOURS on a trained
-     * multi-million-node store, inside one WAL-bloating transaction.  The splice
-     * is a streaming pass with batched commits, then a VACUUM to return the
-     * freed pages.
-     */
-    compact(): void;
-}