eigen-db 4.3.0 → 4.4.0

package/dist/vector-db.d.ts

@@ -0,0 +1,131 @@
+/**
+ * VectorDB — Key-Value Vector Database
+ *
+ * Decoupled from embedding providers. Users pass pre-computed vectors
+ * as number arrays (or Float32Array) with string keys.
+ *
+ * Supports:
+ * - set/get/setMany/getMany for key-value CRUD
+ * - query for similarity search (dot product on normalized vectors)
+ * - flush to persist, close to flush+release, clear to wipe
+ * - Streaming export/import using Web Streams API
+ * - Last-write-wins semantics for duplicate keys (append-only storage)
+ */
+import type { ResultItem } from "./result-set";
+import type { OpenOptions, OpenOptionsInternal, QueryOptions, SetOptions, VectorInput } from "./types";
+export declare class VectorDB {
+    private readonly memoryManager;
+    private readonly storage;
+    private readonly _dimensions;
+    private readonly shouldNormalize;
+    private wasmExports;
+    /** Maps key to its slot index in the vector array */
+    private keyToSlot;
+    /** Maps slot index back to its key */
+    private slotToKey;
+    /** Whether this instance has been closed */
+    private closed;
+    private constructor();
+    /**
+     * Opens a VectorDB instance.
+     * Loads existing data from storage into WASM memory.
+     */
+    static open(options: OpenOptions): Promise<VectorDB>;
+    static open(options: OpenOptionsInternal): Promise<VectorDB>;
+    /** Total number of key-value pairs in the database */
+    get size(): number;
+    /** Number of dimensions per vector */
+    get dimensions(): number;
+    /**
+     * Check whether a key exists in the database.
+     * Uses the internal key-to-slot map for O(1) lookup.
+     */
+    has(key: string): boolean;
+    /**
+     * Delete an entry by key. Returns true if the key existed, false otherwise.
+     * Uses swap-and-pop to avoid gaps in the underlying vector array.
+     */
+    delete(key: string): boolean;
+    /**
+     * Returns an iterable of all keys in the database.
+     */
+    keys(): IterableIterator<string>;
+    /**
+     * Returns an iterable of [key, value] pairs.
+     * Values are returned as plain number array copies.
+     */
+    entries(): IterableIterator<[string, number[]]>;
+    /**
+     * Implements the iterable protocol. Same as entries().
+     */
+    [Symbol.iterator](): IterableIterator<[string, number[]]>;
+    /**
+     * Set a key-value pair. If the key already exists, its vector is overwritten (last-write-wins).
+     * The value is a number[] or Float32Array of length equal to the configured dimensions.
+     */
+    set(key: string, value: VectorInput, options?: SetOptions): void;
+    /**
+     * Get the stored vector for a key. Returns undefined if the key does not exist.
+     * Returns a copy of the stored vector as a plain number array.
+     */
+    get(key: string): number[] | undefined;
+    /**
+     * Set multiple key-value pairs at once. Last-write-wins applies within the batch.
+     */
+    setMany(entries: [string, VectorInput][]): void;
+    /**
+     * Get vectors for multiple keys. Returns undefined for keys that don't exist.
+     */
+    getMany(keys: string[]): (number[] | undefined)[];
+    /**
+     * Search for the most similar vectors to the given query vector.
+     *
+     * Default: returns a plain ResultItem[] sorted by descending similarity.
+     * With `{ iterable: true }`: returns a lazy Iterable<ResultItem> where keys
+     * are resolved only as each item is consumed.
+     *
+     * Similarity is the dot product of query and stored vectors. With
+     * normalization (default), this equals cosine similarity: 1 = identical,
+     * -1 = opposite.
+     */
+    query(value: VectorInput, options: QueryOptions & {
+        iterable: true;
+    }): Iterable<ResultItem>;
+    query(value: VectorInput, options?: QueryOptions): ResultItem[];
+    /**
+     * Persist the current in-memory state to storage.
+     */
+    flush(): Promise<void>;
+    /**
+     * Flush data to storage and release the instance.
+     * The instance cannot be used after close.
+     */
+    close(): Promise<void>;
+    /**
+     * Clear all data from the database and storage.
+     */
+    clear(): Promise<void>;
+    /**
+     * Export the entire database as a ReadableStream of binary chunks.
+     *
+     * Format: [Header 24 bytes][Vector data][Keys data]
+     * Header: magic(4) + version(4) + dimensions(4) + vectorCount(4) + vectorDataLen(4) + keysDataLen(4)
+     *
+     * Vectors are streamed in 64KB chunks from WASM memory to avoid large
+     * heap allocations.
+     */
+    export(): Promise<ReadableStream<Uint8Array>>;
+    /**
+     * Import data from a ReadableStream, replacing all existing data.
+     * Performs a dimension check against the configured dimensions.
+     *
+     * Vectors are streamed directly into WASM memory in chunks to avoid
+     * large heap allocations.
+     */
+    import(stream: ReadableStream<Uint8Array>): Promise<void>;
+    /**
+     * Normalize a vector using WASM (if available) or JS fallback.
+     */
+    private normalizeVector;
+    private assertOpen;
+}

package/dist/wasm-compute.d.ts

@@ -0,0 +1,13 @@
+/**
+ * WASM SIMD compute layer.
+ * Compiles the hand-written WAT module and provides typed wrappers
+ * that operate on shared WebAssembly.Memory.
+ */
+export interface WasmExports {
+    normalize(ptr: number, dimensions: number): void;
+    search_all(queryPtr: number, dbPtr: number, scoresPtr: number, dbSize: number, dimensions: number): void;
+}
+/**
+ * Instantiates a WASM module with the given memory and returns typed exports.
+ */
+export declare function instantiateWasm(wasmBinary: Uint8Array, memory: WebAssembly.Memory): Promise<WasmExports>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eigen-db",
-  "version": "4.3.0",
+  "version": "4.4.0",
   "type": "module",
   "files": [
     "dist",
@@ -12,16 +12,16 @@
   "module": "./dist/eigen-db.js",
   "exports": {
     ".": {
-      "types": "./src/lib/index.ts",
+      "types": "./dist/index.d.ts",
       "import": "./dist/eigen-db.js",
       "require": "./dist/eigen-db.umd.cjs"
     }
   },
-  "types": "./src/lib/index.ts",
+  "types": "./dist/index.d.ts",
   "scripts": {
     "dev": "vite",
     "compile-wat": "tsx scripts/compile-wat.ts",
-    "build": "npm run compile-wat && tsc && vite build",
+    "build": "npm run compile-wat && tsc && vite build && tsc -p tsconfig.build.json",
     "preview": "vite preview",
     "test": "vitest run",
     "test:watch": "vitest",

package/src/lib/__tests__/vector-db.test.ts

@@ -94,6 +94,16 @@ describe("VectorDB", () => {
       expect(db.size).toBe(0);
     });
 
+    it("exposes dimensions property", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      expect(db.dimensions).toBe(4);
+    });
+
     // --- set and get ---
     it("stores and retrieves a vector by key", async () => {
       const db = await VectorDB.open({
@@ -838,6 +848,284 @@ describe("VectorDB", () => {
       expect(db2.get("gamma")![2]).toBeCloseTo(1);
     });
 
+    // --- has ---
+    it("has returns true for existing key", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("a", [1, 0, 0, 0]);
+      expect(db.has("a")).toBe(true);
+    });
+
+    it("has returns false for non-existent key", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      expect(db.has("nonexistent")).toBe(false);
+    });
+
+    it("has throws on closed database", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      await db.close();
+      expect(() => db.has("a")).toThrow("closed");
+    });
+
+    // --- delete ---
+    it("delete removes an existing entry and returns true", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("a", [1, 0, 0, 0]);
+      db.set("b", [0, 1, 0, 0]);
+      expect(db.size).toBe(2);
+
+      const result = db.delete("a");
+      expect(result).toBe(true);
+      expect(db.size).toBe(1);
+      expect(db.get("a")).toBeUndefined();
+      expect(db.has("a")).toBe(false);
+      expect(db.get("b")).toBeDefined();
+    });
+
+    it("delete returns false for non-existent key", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      expect(db.delete("nonexistent")).toBe(false);
+    });
+
+    it("delete last entry leaves empty database", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("only", [1, 2, 3, 4]);
+      db.delete("only");
+      expect(db.size).toBe(0);
+      expect(db.get("only")).toBeUndefined();
+    });
+
+    it("delete preserves remaining entries and query works", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("x-axis", [1, 0, 0, 0]);
+      db.set("y-axis", [0, 1, 0, 0]);
+      db.set("z-axis", [0, 0, 1, 0]);
+
+      db.delete("y-axis");
+      expect(db.size).toBe(2);
+
+      const results = db.query([1, 0, 0, 0]);
+      expect(results.length).toBe(2);
+      expect(results[0].key).toBe("x-axis");
+      expect(results.find((r) => r.key === "y-axis")).toBeUndefined();
+    });
+
+    it("delete then set reuses the database correctly", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("a", [1, 0, 0, 0]);
+      db.set("b", [0, 1, 0, 0]);
+      db.delete("a");
+
+      db.set("c", [0, 0, 1, 0]);
+      expect(db.size).toBe(2);
+      expect(db.get("a")).toBeUndefined();
+      expect(db.get("b")![1]).toBeCloseTo(1);
+      expect(db.get("c")![2]).toBeCloseTo(1);
+    });
+
+    it("delete persists correctly after flush", async () => {
+      const db1 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db1.set("a", [1, 0, 0, 0]);
+      db1.set("b", [0, 1, 0, 0]);
+      db1.delete("a");
+      await db1.flush();
+
+      const db2 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      expect(db2.size).toBe(1);
+      expect(db2.get("a")).toBeUndefined();
+      expect(db2.get("b")![1]).toBeCloseTo(1);
+    });
+
+    it("delete throws on closed database", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      await db.close();
+      expect(() => db.delete("a")).toThrow("closed");
+    });
+
+    // --- keys ---
+    it("keys returns an iterable of all keys", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("a", [1, 0, 0, 0]);
+      db.set("b", [0, 1, 0, 0]);
+      db.set("c", [0, 0, 1, 0]);
+
+      const keys = [...db.keys()];
+      expect(keys).toHaveLength(3);
+      expect(keys).toContain("a");
+      expect(keys).toContain("b");
+      expect(keys).toContain("c");
+    });
+
+    it("keys returns empty iterable for empty database", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      expect([...db.keys()]).toEqual([]);
+    });
+
+    it("keys throws on closed database", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      await db.close();
+      expect(() => db.keys()).toThrow("closed");
+    });
+
+    // --- entries ---
+    it("entries returns an iterable of [key, value] pairs", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("a", [1, 0, 0, 0]);
+      db.set("b", [0, 1, 0, 0]);
+
+      const entries = [...db.entries()];
+      expect(entries).toHaveLength(2);
+
+      const aEntry = entries.find(([key]) => key === "a");
+      expect(aEntry).toBeDefined();
+      expect(aEntry![1][0]).toBeCloseTo(1);
+
+      const bEntry = entries.find(([key]) => key === "b");
+      expect(bEntry).toBeDefined();
+      expect(bEntry![1][1]).toBeCloseTo(1);
+    });
+
+    it("entries returns empty iterable for empty database", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      expect([...db.entries()]).toEqual([]);
+    });
+
+    it("entries throws on closed database", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      await db.close();
+      expect(() => db.entries()).toThrow("closed");
+    });
+
+    // --- Symbol.iterator ---
+    it("supports spread operator via Symbol.iterator", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("a", [1, 0, 0, 0]);
+      db.set("b", [0, 1, 0, 0]);
+
+      const spread = [...db];
+      expect(spread).toHaveLength(2);
+
+      // Same as entries()
+      const entries = [...db.entries()];
+      expect(spread).toEqual(entries);
+    });
+
+    it("supports for-of iteration", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("a", [1, 0, 0, 0]);
+      db.set("b", [0, 1, 0, 0]);
+
+      const collected: [string, number[]][] = [];
+      for (const entry of db) {
+        collected.push(entry);
+      }
+      expect(collected).toHaveLength(2);
+    });
+
     it("import works correctly with single-byte stream chunks", async () => {
       const db1 = await VectorDB.open({
         dimensions: 4,

package/src/lib/vector-db.ts

@@ -38,7 +38,7 @@ const STREAM_CHUNK_SIZE = 65536;
 export class VectorDB {
   private readonly memoryManager: MemoryManager;
   private readonly storage: StorageProvider;
-  private readonly dimensions: number;
+  private readonly _dimensions: number;
   private readonly shouldNormalize: boolean;
   private wasmExports: WasmExports | null;
 
@@ -62,7 +62,7 @@ export class VectorDB {
   ) {
     this.memoryManager = memoryManager;
     this.storage = storage;
-    this.dimensions = dimensions;
+    this._dimensions = dimensions;
     this.shouldNormalize = shouldNormalize;
     this.wasmExports = wasmExports;
     this.keyToSlot = keyToSlot;
@@ -122,6 +122,81 @@ export class VectorDB {
     return this.keyToSlot.size;
   }
 
+  /** Number of dimensions per vector */
+  get dimensions(): number {
+    return this._dimensions;
+  }
+
+  /**
+   * Check whether a key exists in the database.
+   * Uses the internal key-to-slot map for O(1) lookup.
+   */
+  has(key: string): boolean {
+    this.assertOpen();
+    return this.keyToSlot.has(key);
+  }
+
+  /**
+   * Delete an entry by key. Returns true if the key existed, false otherwise.
+   * Uses swap-and-pop to avoid gaps in the underlying vector array.
+   */
+  delete(key: string): boolean {
+    this.assertOpen();
+
+    const slot = this.keyToSlot.get(key);
+    if (slot === undefined) return false;
+
+    const lastSlot = this.memoryManager.vectorCount - 1;
+
+    if (slot !== lastSlot) {
+      // Move last vector into the deleted slot
+      const lastVector = new Float32Array(this.memoryManager.readVector(lastSlot));
+      this.memoryManager.writeVector(slot, lastVector);
+
+      // Update mappings for the moved key
+      const movedKey = this.slotToKey[lastSlot];
+      this.keyToSlot.set(movedKey, slot);
+      this.slotToKey[slot] = movedKey;
+    }
+
+    // Remove the deleted key and shrink
+    this.keyToSlot.delete(key);
+    this.slotToKey.length = lastSlot;
+    this.memoryManager.setVectorCount(lastSlot);
+
+    return true;
+  }
+
+  /**
+   * Returns an iterable of all keys in the database.
+   */
+  keys(): IterableIterator<string> {
+    this.assertOpen();
+    return this.keyToSlot.keys();
+  }
+
+  /**
+   * Returns an iterable of [key, value] pairs.
+   * Values are returned as plain number array copies.
+   */
+  entries(): IterableIterator<[string, number[]]> {
+    this.assertOpen();
+    const keyToSlot = this.keyToSlot;
+    const mm = this.memoryManager;
+    return (function* () {
+      for (const [key, slot] of keyToSlot) {
+        yield [key, Array.from(mm.readVector(slot))] as [string, number[]];
+      }
+    })();
+  }
+
+  /**
+   * Implements the iterable protocol. Same as entries().
+   */
+  [Symbol.iterator](): IterableIterator<[string, number[]]> {
+    return this.entries();
+  }
+
   /**
    * Set a key-value pair. If the key already exists, its vector is overwritten (last-write-wins).
    * The value is a number[] or Float32Array of length equal to the configured dimensions.
@@ -129,8 +204,8 @@ export class VectorDB {
   set(key: string, value: VectorInput, options?: SetOptions): void {
     this.assertOpen();
 
-    if (value.length !== this.dimensions) {
-      throw new Error(`Vector dimension mismatch: expected ${this.dimensions}, got ${value.length}`);
+    if (value.length !== this._dimensions) {
+      throw new Error(`Vector dimension mismatch: expected ${this._dimensions}, got ${value.length}`);
     }
 
     // Convert to Float32Array (also clones to avoid mutating caller's array)
@@ -214,8 +289,8 @@ export class VectorDB {
       return [];
     }
 
-    if (value.length !== this.dimensions) {
-      throw new Error(`Query vector dimension mismatch: expected ${this.dimensions}, got ${value.length}`);
+    if (value.length !== this._dimensions) {
+      throw new Error(`Query vector dimension mismatch: expected ${this._dimensions}, got ${value.length}`);
     }
 
     // Convert to Float32Array and optionally normalize the query vector
@@ -242,21 +317,21 @@ export class VectorDB {
         this.memoryManager.dbOffset,
         scoresOffset,
         totalVectors,
-        this.dimensions,
+        this._dimensions,
       );
     } else {
       const queryView = new Float32Array(
         this.memoryManager.memory.buffer,
         this.memoryManager.queryOffset,
-        this.dimensions,
+        this._dimensions,
       );
       const dbView = new Float32Array(
         this.memoryManager.memory.buffer,
         this.memoryManager.dbOffset,
-        totalVectors * this.dimensions,
+        totalVectors * this._dimensions,
       );
       const scoresView = new Float32Array(this.memoryManager.memory.buffer, scoresOffset, totalVectors);
-      searchAll(queryView, dbView, scoresView, totalVectors, this.dimensions);
+      searchAll(queryView, dbView, scoresView, totalVectors, this._dimensions);
     }
 
     // Read scores (make a copy so the buffer can be reused)
@@ -283,12 +358,12 @@ export class VectorDB {
     const totalVectors = this.memoryManager.vectorCount;
 
     // Serialize vectors from WASM memory
-    const vectorBytes = new Uint8Array(totalVectors * this.dimensions * 4);
+    const vectorBytes = new Uint8Array(totalVectors * this._dimensions * 4);
     if (totalVectors > 0) {
       const src = new Uint8Array(
         this.memoryManager.memory.buffer,
         this.memoryManager.dbOffset,
-        totalVectors * this.dimensions * 4,
+        totalVectors * this._dimensions * 4,
       );
       vectorBytes.set(src);
     }
@@ -335,7 +410,7 @@ export class VectorDB {
     this.assertOpen();
 
     const totalVectors = this.memoryManager.vectorCount;
-    const vectorDataLen = totalVectors * this.dimensions * 4;
+    const vectorDataLen = totalVectors * this._dimensions * 4;
 
     // Encode keys (typically much smaller than vectors)
     const keysBytes = encodeLexicon(this.slotToKey);
@@ -346,7 +421,7 @@ export class VectorDB {
     const headerView = new DataView(header);
     headerView.setUint32(0, EXPORT_MAGIC, true);
     headerView.setUint32(4, EXPORT_VERSION, true);
-    headerView.setUint32(8, this.dimensions, true);
+    headerView.setUint32(8, this._dimensions, true);
     headerView.setUint32(12, totalVectors, true);
     headerView.setUint32(16, vectorDataLen, true);
     headerView.setUint32(20, keysDataLen, true);
@@ -415,8 +490,8 @@ export class VectorDB {
     }
 
     const dimensions = headerView.getUint32(8, true);
-    if (dimensions !== this.dimensions) {
-      throw new Error(`Import dimension mismatch: expected ${this.dimensions}, got ${dimensions}`);
+    if (dimensions !== this._dimensions) {
+      throw new Error(`Import dimension mismatch: expected ${this._dimensions}, got ${dimensions}`);
     }
 
     const vectorCount = headerView.getUint32(12, true);