@hviana/sema 0.2.4 → 0.2.6

package/dist/src/rabitq-ivf/src/database.js

@@ -1,201 +0,0 @@
-import { IvfIndex } from "./ivf.js";
-/**
- * A persistent vector database: an adaptive PARTITIONED (IVF) index over
- * 1-bit RaBitQ codes (cosine), stored entirely in SQLite at `dbPath`.  See
- * ivf.ts for the index design.  The original float vectors are never
- * retained — only the sign codes — so a 1024-d vector costs 128 bytes.
- *
- * Inserting is route-and-append: cost is essentially FLAT in collection size
- * (one RAM scan of the pivot table + one chunk append — no per-insert graph
- * walk).  Query cost is bounded by nprobe × cluster size — constant in N
- * once the collection is past its first splits.
- */
-export class VectorDatabase {
-    index;
-    constructor(options) {
-        if (!options || typeof options.dbPath !== "string" ||
-            options.dbPath.length === 0) {
-            throw new Error("DatabaseOptions.dbPath (string) is required");
-        }
-        this.index = new IvfIndex(options.dbPath, options);
-    }
-    get dim() {
-        return this.index.dim;
-    }
-    /** Number of live (non-deleted) vectors. */
-    get size() {
-        return this.index.size;
-    }
-    /** Physical slot count including tombstones from deletes/updates. */
-    get physicalSize() {
-        return this.index.physicalSize;
-    }
-    /** Number of clusters (partitions) currently in the index. */
-    get clusterCount() {
-        return this.index.clusterCount;
-    }
-    get efSearch() {
-        return this.index.efSearch;
-    }
-    set efSearch(value) {
-        this.index.efSearch = value;
-    }
-    /** Distance computations performed during the most recent query. */
-    get lastQueryDistanceComputations() {
-        return this.index.lastQueryDistComps;
-    }
-    /** Storage row reads issued by the most recent query — the honest,
-     *  cache-independent scalability metric. */
-    get lastQueryStorageReads() {
-        return this.index.lastQueryStorageReads;
-    }
-    get storage() {
-        const f32 = this.dim * 4;
-        const bpv = this.index.bytesPerVector;
-        return {
-            float32BytesPerVector: f32,
-            codeBytesPerVector: bpv,
-            bytesPerVector: bpv,
-            compressionRatio: f32 / bpv,
-        };
-    }
-    checkId(id) {
-        // 32-bit signed range: chunk blobs store exts as int32 — a wider id
-        // would silently truncate, so it is rejected at the door instead.
-        if (!Number.isInteger(id) || id > 0x7fffffff || id < -0x80000000) {
-            throw new Error(`External id must be a 32-bit integer, got ${id}`);
-        }
-        return id;
-    }
-    /** Convert a value to code bytes, selecting by length: `codeWords`
-     *  elements → an existing 1-bit code; otherwise a raw `dim`-vector. */
-    toCodeBytes(value) {
-        if (value.length === this.index.codeWords) {
-            return this.index.codeToBytes(value);
-        }
-        if (value.length !== this.dim) {
-            throw new Error(`Vector dimension mismatch: expected ${this.dim}, got ${value.length}`);
-        }
-        return this.index.encodeToBytes(value);
-    }
-    has(id) {
-        return this.index.has(this.checkId(id));
-    }
-    /** Stream every live external id (bounded memory). */
-    *keys() {
-        yield* this.index.keys();
-    }
-    /** Stream live entries whose INTERNAL id is > `after` — a durable
-     *  incremental watermark (internal ids are monotone at insert and preserved
-     *  by update and compact). */
-    *keysSince(after) {
-        yield* this.index.keysSince(after);
-    }
-    /** Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. */
-    get(id) {
-        const bytes = this.index.codeOf(this.checkId(id));
-        return bytes ? this.index.bytesToCode(bytes) : null;
-    }
-    insert(id, value) {
-        this.index.begin();
-        try {
-            this.index.insert(this.checkId(id), this.toCodeBytes(value));
-            this.index.commit();
-        }
-        catch (e) {
-            this.index.rollback();
-            throw e;
-        }
-    }
-    update(id, value) {
-        this.index.begin();
-        try {
-            this.index.update(this.checkId(id), this.toCodeBytes(value));
-            this.index.commit();
-        }
-        catch (e) {
-            this.index.rollback();
-            throw e;
-        }
-    }
-    upsert(id, value) {
-        this.upsertMany([{ id, vector: value }]);
-    }
-    /**
-     * Upsert many vectors under ONE transaction — one WAL commit for the whole
-     * batch instead of one per vector, with chunk appends to the same cluster
-     * coalesced in the index's write-back buffer.
-     */
-    upsertMany(entries) {
-        if (entries.length === 0)
-            return;
-        this.index.begin();
-        try {
-            for (const e of entries) {
-                // One shared vmap probe decides insert vs update — the per-row point
-                // query is the dominant bulk-load cost, so it is never duplicated.
-                this.index.upsert(this.checkId(e.id), this.toCodeBytes(e.vector));
-            }
-            this.index.commit();
-        }
-        catch (e) {
-            this.index.rollback();
-            throw e;
-        }
-    }
-    /** Delete many ids under ONE transaction. Absent ids are skipped.
-     *  Returns the number of vectors actually removed. */
-    deleteMany(ids) {
-        if (ids.length === 0)
-            return 0;
-        let removed = 0;
-        this.index.begin();
-        try {
-            for (const id of ids) {
-                if (this.index.remove(this.checkId(id)))
-                    removed++;
-            }
-            this.index.commit();
-        }
-        catch (e) {
-            this.index.rollback();
-            throw e;
-        }
-        return removed;
-    }
-    /** Delete the vector bound to an id. Returns false if absent. */
-    delete(id) {
-        return this.deleteMany([id]) === 1;
-    }
-    /** Heat the SQLite page cache with sequential scans (latency only). */
-    warmCache() {
-        return this.index.warmCache();
-    }
-    /** Reclaim tombstoned slots by rewriting the clusters that carry any, then
-     *  VACUUM.  Internal ids and cluster assignment are preserved. */
-    compact() {
-        this.index.compact();
-    }
-    /**
-     * k-NN search. The argument's length selects the mode:
-     *  - `dim` elements       → a raw vector (accurate estimator)
-     *  - `codeWords` elements → a 1-bit code, by sign-bit Hamming
-     */
-    query(query, k = 10, opts) {
-        let hits;
-        if (query.length === this.index.codeWords) {
-            hits = this.index.searchKnnByCode(this.index.codeToBytes(query), k, opts?.ef);
-        }
-        else {
-            if (query.length !== this.dim) {
-                throw new Error(`Vector dimension mismatch: expected ${this.dim}, got ${query.length}`);
-            }
-            hits = this.index.searchKnn(query, k, opts?.ef);
-        }
-        return hits;
-    }
-    /** Close the underlying database. The instance must not be used afterwards. */
-    close() {
-        this.index.close();
-    }
-}

package/dist/src/rabitq-ivf/src/index.d.ts

@@ -1,7 +0,0 @@
-export { VectorDatabase } from "./database.js";
-export type { DatabaseOptions, ExternalId, QueryResult, StorageStats, } from "./database.js";
-export { IvfIndex } from "./ivf.js";
-export type { IvfConfig, IvfHit } from "./ivf.js";
-export { RaBitQuantizer } from "./rabitq.js";
-export type { QueryContext, RaBitQOptions } from "./rabitq.js";
-export { Prng } from "./prng.js";

package/dist/src/rabitq-ivf/src/index.js

@@ -1,4 +0,0 @@
-export { VectorDatabase } from "./database.js";
-export { IvfIndex } from "./ivf.js";
-export { RaBitQuantizer } from "./rabitq.js";
-export { Prng } from "./prng.js";

package/dist/src/rabitq-ivf/src/ivf.d.ts

@@ -1,200 +0,0 @@
-import { DatabaseSync } from "node:sqlite";
-export interface IvfConfig {
-    dim: number;
-    efSearch: number;
-    queryBits: number;
-    rotationRounds: number;
-    seed: number;
-    centroid: Float64Array | null;
-    codeWords: number;
-    paddedDim: number;
-}
-export interface IvfHit {
-    /** external (user) id */
-    id: number;
-    /** estimated cosine distance (1 - cosine) */
-    distance: number;
-}
-export declare class IvfIndex {
-    readonly db: DatabaseSync;
-    private readonly quantizer;
-    private readonly codeBytes;
-    private readonly chunkBytes;
-    private _efSearch;
-    /** Pivot codes, K × codeBytes, contiguous — the insert/query routing scan
-     *  is one linear pass over this array. */
-    private centCodes;
-    /** Aligned 32-bit word view of {@link centCodes} — the routing scan's
-     *  operand (4 bytes per XOR+popcount instead of byte re-composition). */
-    private centWords;
-    /** Per-cluster entry count (live + dead) — the split trigger. */
-    private centEntries;
-    /** Per-cluster chunk count (tail seq = count − 1). */
-    private centChunks;
-    private K;
-    private live;
-    private totalSlots;
-    private nextId;
-    /** Storage row reads (chunk blob fetches) — the cache-independent
-     *  scalability witness, same discipline as the graph index it replaces. */
-    reads: number;
-    lastQueryDistComps: number;
-    lastQueryStorageReads: number;
-    private readonly chunkCache;
-    private readonly dirtyChunks;
-    private readonly dirtyCents;
-    private metaDirty;
-    private readonly cacheEnabled;
-    private readonly chunkCacheMax;
-    private sSelChunk;
-    private sUpsChunk;
-    private sDelChunk;
-    private sSelVmap;
-    private sInsVmap;
-    private sUpdVmap;
-    private sDelVmap;
-    private sUpsCent;
-    private sMeta;
-    constructor(dbPath: string, options: {
-        dim?: number;
-        efSearch?: number;
-        queryBits?: number;
-        rotationRounds?: number;
-        seed?: number;
-        centroid?: ArrayLike<number>;
-        cacheSizeMb?: number;
-    });
-    readonly dim: number;
-    private prepareAll;
-    /** Load the pivot table into RAM — K rows, ~(codeBytes + 8) bytes each. */
-    private loadCents;
-    private growCents;
-    get size(): number;
-    get physicalSize(): number;
-    get clusterCount(): number;
-    get bytesPerVector(): number;
-    get efSearch(): number;
-    set efSearch(v: number);
-    resetReads(): void;
-    /** Fetch a chunk, serving dirty/cached copies first.  `retain` controls
-     *  whether a clean fetch enters the cache — read-only consumers pass the
-     *  cacheEnabled flag so the `cacheSizeMb: 0` mode stays honestly uncached;
-     *  writers always retain (the copy is about to become dirty). */
-    private chunkAt;
-    private cacheChunk;
-    private markDirty;
-    /** Write every dirty chunk row; drop the clean cache only when retention
-     *  is off (no budget) — under a budget the warm set survives the flush.
-     *  Runs inside the caller's transaction. */
-    private flushChunks;
-    private flushCents;
-    /** Persist all buffered state.  MUST be called before the enclosing
-     *  transaction commits (upsertMany/deleteMany do; single-op paths too). */
-    private flushAll;
-    private static readonly SUPER_G;
-    private static readonly SUPER_TOP;
-    /** Majority super-pivot codes, one per group of SUPER_G clusters. */
-    private superWords;
-    /** Group ids whose super-pivot is stale (member pivot changed). */
-    private readonly superStale;
-    /** Scratch word view of the code being routed (avoids per-call copies). */
-    private routeScratch;
-    private markSuperStale;
-    /** Recompute stale super-pivots (majority bit over member pivots). */
-    private freshenSupers;
-    /** Word-popcount Hamming between the routing scratch and a words-row. */
-    private hamAt;
-    /** Nearest cluster to a code — exact over member pivots of the SUPER_TOP
-     *  nearest super-groups (or over everything while K is small).  Pure RAM,
-     *  no allocation, no storage reads. */
-    private nearestCid;
-    /** The nprobe nearest clusters to a QUERY (accurate estimator on pivots),
-     *  as cids ordered nearest-first. */
-    private probeOrder;
-    private probeOrderByCode;
-    /** Whether an external id is present (live). */
-    has(ext: number): boolean;
-    /** The stored code for an ext (a copy), or null. */
-    codeOf(ext: number): Uint8Array | null;
-    /** Insert a code under `ext`.  Caller owns the transaction; buffered rows
-     *  are flushed by {@link commitFlush}.  Throws if ext is already live. */
-    insert(ext: number, code: Uint8Array): void;
-    /** Insert-or-update with ONE vmap probe.  The point probe is a real cost
-     *  at bulk-load rates (statement dispatch + a B-tree descent per row —
-     *  profiled as the single largest insert-path term), so the presence check
-     *  and the update lookup share it. */
-    upsert(ext: number, code: Uint8Array): void;
-    /** The insert core — presence already established by the caller. */
-    private insertNew;
-    private newCluster;
-    private appendToCluster;
-    /** Tombstone a live ext.  Returns false when absent. */
-    remove(ext: number): boolean;
-    /** Update the code bound to a live ext.  A byte-identical code is a no-op
-     *  (content-addressed callers re-upsert unchanged vectors wholesale after a
-     *  restart; each no-op otherwise costs a tombstone + reinsert). */
-    update(ext: number, code: Uint8Array): void;
-    private updateAt;
-    /** Flush buffered chunk/cent/meta rows.  Call before COMMIT. */
-    commitFlush(): void;
-    /** Split a full cluster in two, deterministically.  Dead slots are dropped
-     *  (splits double as incremental compaction).  Both halves get fresh
-     *  majority-bit pivots, so routing sharpens as the corpus grows. */
-    private split;
-    /** Bit-majority sample size per split side.  A pivot is a routing aid, not
-     *  a stored value: the majority bit of a 512-member deterministic sample
-     *  agrees with the full-population majority except on near-tied bits,
-     *  where either choice routes equally well — and counting every member of
-     *  a 4096-entry cluster was 25% of bulk-load CPU (profiled). */
-    private static readonly MAJORITY_SAMPLE;
-    /** Majority-bit code of the side's members (the binary centroid), or null
-     *  when the side is empty.  Members are sampled on a deterministic stride
-     *  when the side exceeds {@link MAJORITY_SAMPLE}. */
-    private majorityCode;
-    /** Rewrite a cluster's chunks from a member list (one side of a split or a
-     *  compaction survivor set), updating vmap rows.  `dropFrom` deletes any
-     *  leftover chunk rows beyond the new count. */
-    private rebuildCluster;
-    /** Multi-row vmap statements, prepared per row count (powers of a fixed
-     *  batch width) — REPLACE semantics on the ext primary key. */
-    private readonly vmapBatchStmts;
-    private static readonly VMAP_BATCH;
-    private vmapBatchStmt;
-    /** Write flat (ext, id, cid, seq, slot) tuples in wide batches. */
-    private flushVmapRows;
-    /** ef → clusters probed.  ef is the familiar "candidate breadth" knob; a
-     *  probe scans one whole cluster, so nprobe = ef/4 keeps the default
-     *  (ef 64 → 16 probes) both accurate on trained stores and bounded. */
-    private nprobeOf;
-    /** k-NN with a full-precision query (accurate RaBitQ estimator). */
-    searchKnn(vec: ArrayLike<number>, k: number, ef?: number): IvfHit[];
-    /** k-NN with an already-quantized code (sign-bit Hamming distance). */
-    searchKnnByCode(code: Uint8Array, k: number, ef?: number): IvfHit[];
-    private scanClusters;
-    /** Stream live entries whose internal id is > `after`, in id order.
-     *  Internal ids are monotone at insert and PRESERVED by update/compact, so
-     *  the largest id a caller has seen is a durable incremental watermark. */
-    keysSince(after: number, batch?: number): IterableIterator<{
-        ext: number;
-        internal: number;
-    }>;
-    /** Stream every live external id. */
-    keys(): IterableIterator<number>;
-    /** Drop tombstoned slots by rewriting each cluster that carries any, then
-     *  VACUUM.  Internal ids and cluster assignment are preserved — routing
-     *  quality is untouched, only dead space is reclaimed. */
-    compact(): void;
-    /** Heat SQLite's page cache with sequential scans of the chunk and vmap
-     *  tables — a cold session otherwise warms it through random point reads.
-     *  Purely a latency optimisation; returns rows touched. */
-    warmCache(): number;
-    begin(): void;
-    commit(): void;
-    rollback(): void;
-    /** Encode a raw vector to its 1-bit code bytes. */
-    encodeToBytes(vec: ArrayLike<number>): Uint8Array;
-    codeToBytes(code: ArrayLike<number>): Uint8Array;
-    bytesToCode(bytes: Uint8Array): Uint32Array;
-    get codeWords(): number;
-    close(): void;
-}