@hviana/sema 0.1.5 → 0.1.7

package/dist/src/rabitq-ivf/src/database.js

@@ -0,0 +1,201 @@
+import { IvfIndex } from "./ivf.js";
+/**
+ * A persistent vector database: an adaptive PARTITIONED (IVF) index over
+ * 1-bit RaBitQ codes (cosine), stored entirely in SQLite at `dbPath`.  See
+ * ivf.ts for the index design.  The original float vectors are never
+ * retained — only the sign codes — so a 1024-d vector costs 128 bytes.
+ *
+ * Inserting is route-and-append: cost is essentially FLAT in collection size
+ * (one RAM scan of the pivot table + one chunk append — no per-insert graph
+ * walk).  Query cost is bounded by nprobe × cluster size — constant in N
+ * once the collection is past its first splits.
+ */
+export class VectorDatabase {
+    index;
+    constructor(options) {
+        if (!options || typeof options.dbPath !== "string" ||
+            options.dbPath.length === 0) {
+            throw new Error("DatabaseOptions.dbPath (string) is required");
+        }
+        this.index = new IvfIndex(options.dbPath, options);
+    }
+    get dim() {
+        return this.index.dim;
+    }
+    /** Number of live (non-deleted) vectors. */
+    get size() {
+        return this.index.size;
+    }
+    /** Physical slot count including tombstones from deletes/updates. */
+    get physicalSize() {
+        return this.index.physicalSize;
+    }
+    /** Number of clusters (partitions) currently in the index. */
+    get clusterCount() {
+        return this.index.clusterCount;
+    }
+    get efSearch() {
+        return this.index.efSearch;
+    }
+    set efSearch(value) {
+        this.index.efSearch = value;
+    }
+    /** Distance computations performed during the most recent query. */
+    get lastQueryDistanceComputations() {
+        return this.index.lastQueryDistComps;
+    }
+    /** Storage row reads issued by the most recent query — the honest,
+     *  cache-independent scalability metric. */
+    get lastQueryStorageReads() {
+        return this.index.lastQueryStorageReads;
+    }
+    get storage() {
+        const f32 = this.dim * 4;
+        const bpv = this.index.bytesPerVector;
+        return {
+            float32BytesPerVector: f32,
+            codeBytesPerVector: bpv,
+            bytesPerVector: bpv,
+            compressionRatio: f32 / bpv,
+        };
+    }
+    checkId(id) {
+        // 32-bit signed range: chunk blobs store exts as int32 — a wider id
+        // would silently truncate, so it is rejected at the door instead.
+        if (!Number.isInteger(id) || id > 0x7fffffff || id < -0x80000000) {
+            throw new Error(`External id must be a 32-bit integer, got ${id}`);
+        }
+        return id;
+    }
+    /** Convert a value to code bytes, selecting by length: `codeWords`
+     *  elements → an existing 1-bit code; otherwise a raw `dim`-vector. */
+    toCodeBytes(value) {
+        if (value.length === this.index.codeWords) {
+            return this.index.codeToBytes(value);
+        }
+        if (value.length !== this.dim) {
+            throw new Error(`Vector dimension mismatch: expected ${this.dim}, got ${value.length}`);
+        }
+        return this.index.encodeToBytes(value);
+    }
+    has(id) {
+        return this.index.has(this.checkId(id));
+    }
+    /** Stream every live external id (bounded memory). */
+    *keys() {
+        yield* this.index.keys();
+    }
+    /** Stream live entries whose INTERNAL id is > `after` — a durable
+     *  incremental watermark (internal ids are monotone at insert and preserved
+     *  by update and compact). */
+    *keysSince(after) {
+        yield* this.index.keysSince(after);
+    }
+    /** Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. */
+    get(id) {
+        const bytes = this.index.codeOf(this.checkId(id));
+        return bytes ? this.index.bytesToCode(bytes) : null;
+    }
+    insert(id, value) {
+        this.index.begin();
+        try {
+            this.index.insert(this.checkId(id), this.toCodeBytes(value));
+            this.index.commit();
+        }
+        catch (e) {
+            this.index.rollback();
+            throw e;
+        }
+    }
+    update(id, value) {
+        this.index.begin();
+        try {
+            this.index.update(this.checkId(id), this.toCodeBytes(value));
+            this.index.commit();
+        }
+        catch (e) {
+            this.index.rollback();
+            throw e;
+        }
+    }
+    upsert(id, value) {
+        this.upsertMany([{ id, vector: value }]);
+    }
+    /**
+     * Upsert many vectors under ONE transaction — one WAL commit for the whole
+     * batch instead of one per vector, with chunk appends to the same cluster
+     * coalesced in the index's write-back buffer.
+     */
+    upsertMany(entries) {
+        if (entries.length === 0)
+            return;
+        this.index.begin();
+        try {
+            for (const e of entries) {
+                // One shared vmap probe decides insert vs update — the per-row point
+                // query is the dominant bulk-load cost, so it is never duplicated.
+                this.index.upsert(this.checkId(e.id), this.toCodeBytes(e.vector));
+            }
+            this.index.commit();
+        }
+        catch (e) {
+            this.index.rollback();
+            throw e;
+        }
+    }
+    /** Delete many ids under ONE transaction. Absent ids are skipped.
+     *  Returns the number of vectors actually removed. */
+    deleteMany(ids) {
+        if (ids.length === 0)
+            return 0;
+        let removed = 0;
+        this.index.begin();
+        try {
+            for (const id of ids) {
+                if (this.index.remove(this.checkId(id)))
+                    removed++;
+            }
+            this.index.commit();
+        }
+        catch (e) {
+            this.index.rollback();
+            throw e;
+        }
+        return removed;
+    }
+    /** Delete the vector bound to an id. Returns false if absent. */
+    delete(id) {
+        return this.deleteMany([id]) === 1;
+    }
+    /** Heat the SQLite page cache with sequential scans (latency only). */
+    warmCache() {
+        return this.index.warmCache();
+    }
+    /** Reclaim tombstoned slots by rewriting the clusters that carry any, then
+     *  VACUUM.  Internal ids and cluster assignment are preserved. */
+    compact() {
+        this.index.compact();
+    }
+    /**
+     * k-NN search. The argument's length selects the mode:
+     *  - `dim` elements       → a raw vector (accurate estimator)
+     *  - `codeWords` elements → a 1-bit code, by sign-bit Hamming
+     */
+    query(query, k = 10, opts) {
+        let hits;
+        if (query.length === this.index.codeWords) {
+            hits = this.index.searchKnnByCode(this.index.codeToBytes(query), k, opts?.ef);
+        }
+        else {
+            if (query.length !== this.dim) {
+                throw new Error(`Vector dimension mismatch: expected ${this.dim}, got ${query.length}`);
+            }
+            hits = this.index.searchKnn(query, k, opts?.ef);
+        }
+        return hits;
+    }
+    /** Close the underlying database. The instance must not be used afterwards. */
+    close() {
+        this.index.close();
+    }
+}

package/dist/src/rabitq-ivf/src/index.d.ts

@@ -0,0 +1,7 @@
+export { VectorDatabase } from "./database.js";
+export type { DatabaseOptions, ExternalId, QueryResult, StorageStats, } from "./database.js";
+export { IvfIndex } from "./ivf.js";
+export type { IvfConfig, IvfHit } from "./ivf.js";
+export { RaBitQuantizer } from "./rabitq.js";
+export type { QueryContext, RaBitQOptions } from "./rabitq.js";
+export { Prng } from "./prng.js";

package/dist/src/{rabitq-hnsw → rabitq-ivf}/src/index.js

@@ -1,6 +1,4 @@
 export { VectorDatabase } from "./database.js";
-export { HnswIndex } from "./hnsw.js";
+export { IvfIndex } from "./ivf.js";
 export { RaBitQuantizer } from "./rabitq.js";
-export { Store } from "./store.js";
-export { Heap } from "./heap.js";
 export { Prng } from "./prng.js";

package/dist/src/rabitq-ivf/src/ivf.d.ts

@@ -0,0 +1,200 @@
+import { DatabaseSync } from "node:sqlite";
+export interface IvfConfig {
+    dim: number;
+    efSearch: number;
+    queryBits: number;
+    rotationRounds: number;
+    seed: number;
+    centroid: Float64Array | null;
+    codeWords: number;
+    paddedDim: number;
+}
+export interface IvfHit {
+    /** external (user) id */
+    id: number;
+    /** estimated cosine distance (1 - cosine) */
+    distance: number;
+}
+export declare class IvfIndex {
+    readonly db: DatabaseSync;
+    private readonly quantizer;
+    private readonly codeBytes;
+    private readonly chunkBytes;
+    private _efSearch;
+    /** Pivot codes, K × codeBytes, contiguous — the insert/query routing scan
+     *  is one linear pass over this array. */
+    private centCodes;
+    /** Aligned 32-bit word view of {@link centCodes} — the routing scan's
+     *  operand (4 bytes per XOR+popcount instead of byte re-composition). */
+    private centWords;
+    /** Per-cluster entry count (live + dead) — the split trigger. */
+    private centEntries;
+    /** Per-cluster chunk count (tail seq = count − 1). */
+    private centChunks;
+    private K;
+    private live;
+    private totalSlots;
+    private nextId;
+    /** Storage row reads (chunk blob fetches) — the cache-independent
+     *  scalability witness, same discipline as the graph index it replaces. */
+    reads: number;
+    lastQueryDistComps: number;
+    lastQueryStorageReads: number;
+    private readonly chunkCache;
+    private readonly dirtyChunks;
+    private readonly dirtyCents;
+    private metaDirty;
+    private readonly cacheEnabled;
+    private readonly chunkCacheMax;
+    private sSelChunk;
+    private sUpsChunk;
+    private sDelChunk;
+    private sSelVmap;
+    private sInsVmap;
+    private sUpdVmap;
+    private sDelVmap;
+    private sUpsCent;
+    private sMeta;
+    constructor(dbPath: string, options: {
+        dim?: number;
+        efSearch?: number;
+        queryBits?: number;
+        rotationRounds?: number;
+        seed?: number;
+        centroid?: ArrayLike<number>;
+        cacheSizeMb?: number;
+    });
+    readonly dim: number;
+    private prepareAll;
+    /** Load the pivot table into RAM — K rows, ~(codeBytes + 8) bytes each. */
+    private loadCents;
+    private growCents;
+    get size(): number;
+    get physicalSize(): number;
+    get clusterCount(): number;
+    get bytesPerVector(): number;
+    get efSearch(): number;
+    set efSearch(v: number);
+    resetReads(): void;
+    /** Fetch a chunk, serving dirty/cached copies first.  `retain` controls
+     *  whether a clean fetch enters the cache — read-only consumers pass the
+     *  cacheEnabled flag so the `cacheSizeMb: 0` mode stays honestly uncached;
+     *  writers always retain (the copy is about to become dirty). */
+    private chunkAt;
+    private cacheChunk;
+    private markDirty;
+    /** Write every dirty chunk row; drop the clean cache only when retention
+     *  is off (no budget) — under a budget the warm set survives the flush.
+     *  Runs inside the caller's transaction. */
+    private flushChunks;
+    private flushCents;
+    /** Persist all buffered state.  MUST be called before the enclosing
+     *  transaction commits (upsertMany/deleteMany do; single-op paths too). */
+    private flushAll;
+    private static readonly SUPER_G;
+    private static readonly SUPER_TOP;
+    /** Majority super-pivot codes, one per group of SUPER_G clusters. */
+    private superWords;
+    /** Group ids whose super-pivot is stale (member pivot changed). */
+    private readonly superStale;
+    /** Scratch word view of the code being routed (avoids per-call copies). */
+    private routeScratch;
+    private markSuperStale;
+    /** Recompute stale super-pivots (majority bit over member pivots). */
+    private freshenSupers;
+    /** Word-popcount Hamming between the routing scratch and a words-row. */
+    private hamAt;
+    /** Nearest cluster to a code — exact over member pivots of the SUPER_TOP
+     *  nearest super-groups (or over everything while K is small).  Pure RAM,
+     *  no allocation, no storage reads. */
+    private nearestCid;
+    /** The nprobe nearest clusters to a QUERY (accurate estimator on pivots),
+     *  as cids ordered nearest-first. */
+    private probeOrder;
+    private probeOrderByCode;
+    /** Whether an external id is present (live). */
+    has(ext: number): boolean;
+    /** The stored code for an ext (a copy), or null. */
+    codeOf(ext: number): Uint8Array | null;
+    /** Insert a code under `ext`.  Caller owns the transaction; buffered rows
+     *  are flushed by {@link commitFlush}.  Throws if ext is already live. */
+    insert(ext: number, code: Uint8Array): void;
+    /** Insert-or-update with ONE vmap probe.  The point probe is a real cost
+     *  at bulk-load rates (statement dispatch + a B-tree descent per row —
+     *  profiled as the single largest insert-path term), so the presence check
+     *  and the update lookup share it. */
+    upsert(ext: number, code: Uint8Array): void;
+    /** The insert core — presence already established by the caller. */
+    private insertNew;
+    private newCluster;
+    private appendToCluster;
+    /** Tombstone a live ext.  Returns false when absent. */
+    remove(ext: number): boolean;
+    /** Update the code bound to a live ext.  A byte-identical code is a no-op
+     *  (content-addressed callers re-upsert unchanged vectors wholesale after a
+     *  restart; each no-op otherwise costs a tombstone + reinsert). */
+    update(ext: number, code: Uint8Array): void;
+    private updateAt;
+    /** Flush buffered chunk/cent/meta rows.  Call before COMMIT. */
+    commitFlush(): void;
+    /** Split a full cluster in two, deterministically.  Dead slots are dropped
+     *  (splits double as incremental compaction).  Both halves get fresh
+     *  majority-bit pivots, so routing sharpens as the corpus grows. */
+    private split;
+    /** Bit-majority sample size per split side.  A pivot is a routing aid, not
+     *  a stored value: the majority bit of a 512-member deterministic sample
+     *  agrees with the full-population majority except on near-tied bits,
+     *  where either choice routes equally well — and counting every member of
+     *  a 4096-entry cluster was 25% of bulk-load CPU (profiled). */
+    private static readonly MAJORITY_SAMPLE;
+    /** Majority-bit code of the side's members (the binary centroid), or null
+     *  when the side is empty.  Members are sampled on a deterministic stride
+     *  when the side exceeds {@link MAJORITY_SAMPLE}. */
+    private majorityCode;
+    /** Rewrite a cluster's chunks from a member list (one side of a split or a
+     *  compaction survivor set), updating vmap rows.  `dropFrom` deletes any
+     *  leftover chunk rows beyond the new count. */
+    private rebuildCluster;
+    /** Multi-row vmap statements, prepared per row count (powers of a fixed
+     *  batch width) — REPLACE semantics on the ext primary key. */
+    private readonly vmapBatchStmts;
+    private static readonly VMAP_BATCH;
+    private vmapBatchStmt;
+    /** Write flat (ext, id, cid, seq, slot) tuples in wide batches. */
+    private flushVmapRows;
+    /** ef → clusters probed.  ef is the familiar "candidate breadth" knob; a
+     *  probe scans one whole cluster, so nprobe = ef/4 keeps the default
+     *  (ef 64 → 16 probes) both accurate on trained stores and bounded. */
+    private nprobeOf;
+    /** k-NN with a full-precision query (accurate RaBitQ estimator). */
+    searchKnn(vec: ArrayLike<number>, k: number, ef?: number): IvfHit[];
+    /** k-NN with an already-quantized code (sign-bit Hamming distance). */
+    searchKnnByCode(code: Uint8Array, k: number, ef?: number): IvfHit[];
+    private scanClusters;
+    /** Stream live entries whose internal id is > `after`, in id order.
+     *  Internal ids are monotone at insert and PRESERVED by update/compact, so
+     *  the largest id a caller has seen is a durable incremental watermark. */
+    keysSince(after: number, batch?: number): IterableIterator<{
+        ext: number;
+        internal: number;
+    }>;
+    /** Stream every live external id. */
+    keys(): IterableIterator<number>;
+    /** Drop tombstoned slots by rewriting each cluster that carries any, then
+     *  VACUUM.  Internal ids and cluster assignment are preserved — routing
+     *  quality is untouched, only dead space is reclaimed. */
+    compact(): void;
+    /** Heat SQLite's page cache with sequential scans of the chunk and vmap
+     *  tables — a cold session otherwise warms it through random point reads.
+     *  Purely a latency optimisation; returns rows touched. */
+    warmCache(): number;
+    begin(): void;
+    commit(): void;
+    rollback(): void;
+    /** Encode a raw vector to its 1-bit code bytes. */
+    encodeToBytes(vec: ArrayLike<number>): Uint8Array;
+    codeToBytes(code: ArrayLike<number>): Uint8Array;
+    bytesToCode(bytes: Uint8Array): Uint32Array;
+    get codeWords(): number;
+    close(): void;
+}