eigen-db 4.2.0 → 4.4.0

package/dist/vector-db.d.ts

@@ -0,0 +1,131 @@
+/**
+ * VectorDB — Key-Value Vector Database
+ *
+ * Decoupled from embedding providers. Users pass pre-computed vectors
+ * as number arrays (or Float32Array) with string keys.
+ *
+ * Supports:
+ * - set/get/setMany/getMany for key-value CRUD
+ * - query for similarity search (dot product on normalized vectors)
+ * - flush to persist, close to flush+release, clear to wipe
+ * - Streaming export/import using Web Streams API
+ * - Last-write-wins semantics for duplicate keys (append-only storage)
+ */
+import type { ResultItem } from "./result-set";
+import type { OpenOptions, OpenOptionsInternal, QueryOptions, SetOptions, VectorInput } from "./types";
+export declare class VectorDB {
+    private readonly memoryManager;
+    private readonly storage;
+    private readonly _dimensions;
+    private readonly shouldNormalize;
+    private wasmExports;
+    /** Maps key to its slot index in the vector array */
+    private keyToSlot;
+    /** Maps slot index back to its key */
+    private slotToKey;
+    /** Whether this instance has been closed */
+    private closed;
+    private constructor();
+    /**
+     * Opens a VectorDB instance.
+     * Loads existing data from storage into WASM memory.
+     */
+    static open(options: OpenOptions): Promise<VectorDB>;
+    static open(options: OpenOptionsInternal): Promise<VectorDB>;
+    /** Total number of key-value pairs in the database */
+    get size(): number;
+    /** Number of dimensions per vector */
+    get dimensions(): number;
+    /**
+     * Check whether a key exists in the database.
+     * Uses the internal key-to-slot map for O(1) lookup.
+     */
+    has(key: string): boolean;
+    /**
+     * Delete an entry by key. Returns true if the key existed, false otherwise.
+     * Uses swap-and-pop to avoid gaps in the underlying vector array.
+     */
+    delete(key: string): boolean;
+    /**
+     * Returns an iterable of all keys in the database.
+     */
+    keys(): IterableIterator<string>;
+    /**
+     * Returns an iterable of [key, value] pairs.
+     * Values are returned as plain number array copies.
+     */
+    entries(): IterableIterator<[string, number[]]>;
+    /**
+     * Implements the iterable protocol. Same as entries().
+     */
+    [Symbol.iterator](): IterableIterator<[string, number[]]>;
+    /**
+     * Set a key-value pair. If the key already exists, its vector is overwritten (last-write-wins).
+     * The value is a number[] or Float32Array of length equal to the configured dimensions.
+     */
+    set(key: string, value: VectorInput, options?: SetOptions): void;
+    /**
+     * Get the stored vector for a key. Returns undefined if the key does not exist.
+     * Returns a copy of the stored vector as a plain number array.
+     */
+    get(key: string): number[] | undefined;
+    /**
+     * Set multiple key-value pairs at once. Last-write-wins applies within the batch.
+     */
+    setMany(entries: [string, VectorInput][]): void;
+    /**
+     * Get vectors for multiple keys. Returns undefined for keys that don't exist.
+     */
+    getMany(keys: string[]): (number[] | undefined)[];
+    /**
+     * Search for the most similar vectors to the given query vector.
+     *
+     * Default: returns a plain ResultItem[] sorted by descending similarity.
+     * With `{ iterable: true }`: returns a lazy Iterable<ResultItem> where keys
+     * are resolved only as each item is consumed.
+     *
+     * Similarity is the dot product of query and stored vectors. With
+     * normalization (default), this equals cosine similarity: 1 = identical,
+     * -1 = opposite.
+     */
+    query(value: VectorInput, options: QueryOptions & {
+        iterable: true;
+    }): Iterable<ResultItem>;
+    query(value: VectorInput, options?: QueryOptions): ResultItem[];
+    /**
+     * Persist the current in-memory state to storage.
+     */
+    flush(): Promise<void>;
+    /**
+     * Flush data to storage and release the instance.
+     * The instance cannot be used after close.
+     */
+    close(): Promise<void>;
+    /**
+     * Clear all data from the database and storage.
+     */
+    clear(): Promise<void>;
+    /**
+     * Export the entire database as a ReadableStream of binary chunks.
+     *
+     * Format: [Header 24 bytes][Vector data][Keys data]
+     * Header: magic(4) + version(4) + dimensions(4) + vectorCount(4) + vectorDataLen(4) + keysDataLen(4)
+     *
+     * Vectors are streamed in 64KB chunks from WASM memory to avoid large
+     * heap allocations.
+     */
+    export(): Promise<ReadableStream<Uint8Array>>;
+    /**
+     * Import data from a ReadableStream, replacing all existing data.
+     * Performs a dimension check against the configured dimensions.
+     *
+     * Vectors are streamed directly into WASM memory in chunks to avoid
+     * large heap allocations.
+     */
+    import(stream: ReadableStream<Uint8Array>): Promise<void>;
+    /**
+     * Normalize a vector using WASM (if available) or JS fallback.
+     */
+    private normalizeVector;
+    private assertOpen;
+}

package/dist/wasm-compute.d.ts

@@ -0,0 +1,13 @@
+/**
+ * WASM SIMD compute layer.
+ * Compiles the hand-written WAT module and provides typed wrappers
+ * that operate on shared WebAssembly.Memory.
+ */
+export interface WasmExports {
+    normalize(ptr: number, dimensions: number): void;
+    search_all(queryPtr: number, dbPtr: number, scoresPtr: number, dbSize: number, dimensions: number): void;
+}
+/**
+ * Instantiates a WASM module with the given memory and returns typed exports.
+ */
+export declare function instantiateWasm(wasmBinary: Uint8Array, memory: WebAssembly.Memory): Promise<WasmExports>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eigen-db",
-  "version": "4.2.0",
+  "version": "4.4.0",
   "type": "module",
   "files": [
     "dist",
@@ -12,16 +12,16 @@
   "module": "./dist/eigen-db.js",
   "exports": {
     ".": {
-      "types": "./src/lib/index.ts",
+      "types": "./dist/index.d.ts",
       "import": "./dist/eigen-db.js",
       "require": "./dist/eigen-db.umd.cjs"
     }
   },
-  "types": "./src/lib/index.ts",
+  "types": "./dist/index.d.ts",
   "scripts": {
     "dev": "vite",
     "compile-wat": "tsx scripts/compile-wat.ts",
-    "build": "npm run compile-wat && tsc && vite build",
+    "build": "npm run compile-wat && tsc && vite build && tsc -p tsconfig.build.json",
     "preview": "vite preview",
     "test": "vitest run",
     "test:watch": "vitest",

package/src/lib/__tests__/vector-db.test.ts

@@ -94,6 +94,16 @@ describe("VectorDB", () => {
       expect(db.size).toBe(0);
     });
 
+    it("exposes dimensions property", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      expect(db.dimensions).toBe(4);
+    });
+
     // --- set and get ---
     it("stores and retrieves a vector by key", async () => {
       const db = await VectorDB.open({
@@ -838,6 +848,284 @@ describe("VectorDB", () => {
       expect(db2.get("gamma")![2]).toBeCloseTo(1);
     });
 
+    // --- has ---
+    it("has returns true for existing key", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("a", [1, 0, 0, 0]);
+      expect(db.has("a")).toBe(true);
+    });
+
+    it("has returns false for non-existent key", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      expect(db.has("nonexistent")).toBe(false);
+    });
+
+    it("has throws on closed database", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      await db.close();
+      expect(() => db.has("a")).toThrow("closed");
+    });
+
+    // --- delete ---
+    it("delete removes an existing entry and returns true", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("a", [1, 0, 0, 0]);
+      db.set("b", [0, 1, 0, 0]);
+      expect(db.size).toBe(2);
+
+      const result = db.delete("a");
+      expect(result).toBe(true);
+      expect(db.size).toBe(1);
+      expect(db.get("a")).toBeUndefined();
+      expect(db.has("a")).toBe(false);
+      expect(db.get("b")).toBeDefined();
+    });
+
+    it("delete returns false for non-existent key", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      expect(db.delete("nonexistent")).toBe(false);
+    });
+
+    it("delete last entry leaves empty database", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("only", [1, 2, 3, 4]);
+      db.delete("only");
+      expect(db.size).toBe(0);
+      expect(db.get("only")).toBeUndefined();
+    });
+
+    it("delete preserves remaining entries and query works", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("x-axis", [1, 0, 0, 0]);
+      db.set("y-axis", [0, 1, 0, 0]);
+      db.set("z-axis", [0, 0, 1, 0]);
+
+      db.delete("y-axis");
+      expect(db.size).toBe(2);
+
+      const results = db.query([1, 0, 0, 0]);
+      expect(results.length).toBe(2);
+      expect(results[0].key).toBe("x-axis");
+      expect(results.find((r) => r.key === "y-axis")).toBeUndefined();
+    });
+
+    it("delete then set reuses the database correctly", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("a", [1, 0, 0, 0]);
+      db.set("b", [0, 1, 0, 0]);
+      db.delete("a");
+
+      db.set("c", [0, 0, 1, 0]);
+      expect(db.size).toBe(2);
+      expect(db.get("a")).toBeUndefined();
+      expect(db.get("b")![1]).toBeCloseTo(1);
+      expect(db.get("c")![2]).toBeCloseTo(1);
+    });
+
+    it("delete persists correctly after flush", async () => {
+      const db1 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db1.set("a", [1, 0, 0, 0]);
+      db1.set("b", [0, 1, 0, 0]);
+      db1.delete("a");
+      await db1.flush();
+
+      const db2 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      expect(db2.size).toBe(1);
+      expect(db2.get("a")).toBeUndefined();
+      expect(db2.get("b")![1]).toBeCloseTo(1);
+    });
+
+    it("delete throws on closed database", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      await db.close();
+      expect(() => db.delete("a")).toThrow("closed");
+    });
+
+    // --- keys ---
+    it("keys returns an iterable of all keys", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("a", [1, 0, 0, 0]);
+      db.set("b", [0, 1, 0, 0]);
+      db.set("c", [0, 0, 1, 0]);
+
+      const keys = [...db.keys()];
+      expect(keys).toHaveLength(3);
+      expect(keys).toContain("a");
+      expect(keys).toContain("b");
+      expect(keys).toContain("c");
+    });
+
+    it("keys returns empty iterable for empty database", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      expect([...db.keys()]).toEqual([]);
+    });
+
+    it("keys throws on closed database", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      await db.close();
+      expect(() => db.keys()).toThrow("closed");
+    });
+
+    // --- entries ---
+    it("entries returns an iterable of [key, value] pairs", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("a", [1, 0, 0, 0]);
+      db.set("b", [0, 1, 0, 0]);
+
+      const entries = [...db.entries()];
+      expect(entries).toHaveLength(2);
+
+      const aEntry = entries.find(([key]) => key === "a");
+      expect(aEntry).toBeDefined();
+      expect(aEntry![1][0]).toBeCloseTo(1);
+
+      const bEntry = entries.find(([key]) => key === "b");
+      expect(bEntry).toBeDefined();
+      expect(bEntry![1][1]).toBeCloseTo(1);
+    });
+
+    it("entries returns empty iterable for empty database", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      expect([...db.entries()]).toEqual([]);
+    });
+
+    it("entries throws on closed database", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      await db.close();
+      expect(() => db.entries()).toThrow("closed");
+    });
+
+    // --- Symbol.iterator ---
+    it("supports spread operator via Symbol.iterator", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("a", [1, 0, 0, 0]);
+      db.set("b", [0, 1, 0, 0]);
+
+      const spread = [...db];
+      expect(spread).toHaveLength(2);
+
+      // Same as entries()
+      const entries = [...db.entries()];
+      expect(spread).toEqual(entries);
+    });
+
+    it("supports for-of iteration", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("a", [1, 0, 0, 0]);
+      db.set("b", [0, 1, 0, 0]);
+
+      const collected: [string, number[]][] = [];
+      for (const entry of db) {
+        collected.push(entry);
+      }
+      expect(collected).toHaveLength(2);
+    });
+
     it("import works correctly with single-byte stream chunks", async () => {
       const db1 = await VectorDB.open({
         dimensions: 4,
@@ -871,4 +1159,44 @@ describe("VectorDB", () => {
     expect(err).toBeInstanceOf(VectorCapacityExceededError);
     expect(err.message).toContain("100");
   });
+
+  // --- storage decoupling ---
+  it("defaults to in-memory storage when no storage is provided", async () => {
+    const db = await VectorDB.open({
+      dimensions: 4,
+      wasmBinary: null,
+    });
+
+    db.set("a", [1, 0, 0, 0]);
+    expect(db.size).toBe(1);
+    expect(db.get("a")).toBeDefined();
+
+    // Flush should succeed with default in-memory storage
+    await db.flush();
+    await db.close();
+  });
+
+  it("accepts an explicit storage provider via options", async () => {
+    const customStorage = new InMemoryStorageProvider();
+    const db = await VectorDB.open({
+      dimensions: 4,
+      normalize: false,
+      storage: customStorage,
+      wasmBinary: null,
+    });
+
+    db.set("key1", [1, 2, 3, 4]);
+    await db.flush();
+
+    // Reopen with the same storage to verify persistence
+    const db2 = await VectorDB.open({
+      dimensions: 4,
+      normalize: false,
+      storage: customStorage,
+      wasmBinary: null,
+    });
+
+    expect(db2.size).toBe(1);
+    expect(db2.get("key1")![0]).toBeCloseTo(1);
+  });
 });

package/src/lib/simd-binary.ts

@@ -1,5 +1,5 @@
 // AUTO-GENERATED - Do not edit. Run: npx tsx scripts/compile-wat.ts
-const SIMD_WASM_BASE64 = "AGFzbQEAAAABDgJgAn9/AGAFf39/f38AAg8BA2VudgZtZW1vcnkCAAEDAwIAAQcaAglub3JtYWxpemUAAApzZWFyY2hfYWxsAAEKgQ4C3wQFAX8EewN9AXsDf/0MAAAAAAAAAAAAAAAAAAAAACED/QwAAAAAAAAAAAAAAAAAAAAAIQT9DAAAAAAAAAAAAAAAAAAAAAAhBf0MAAAAAAAAAAAAAAAAAAAAACEGIAFBcHEhCyABQXxxIQxBACECAkADQCACIAtPDQEgACACQQJ0aiENIAMgDf0ABAAgDf0ABAD95gH95AEhAyAEIA39AAQQIA39AAQQ/eYB/eQBIQQgBSAN/QAEICAN/QAEIP3mAf3kASEFIAYgDf0ABDAgDf0ABDD95gH95AEhBiACQRBqIQIMAAsLIAMgBP3kASAFIAb95AH95AEhAwJAA0AgAiAMTw0BIAAgAkECdGohDSADIA39AAQAIA39AAQA/eYB/eQBIQMgAkEEaiECDAALCyAD/R8AIAP9HwGSIAP9HwIgA/0fA5KSIQcCQANAIAIgAU8NASAAIAJBAnRqIQ0gByANKgIAIA0qAgCUkiEHIAJBAWohAgwACwsgB5EhCCAIQwAAAABbBEAPC0MAAIA/IAiVIQkgCf0TIQpBACECAkADQCACIAtPDQEgACACQQJ0aiENIA0gDf0ABAAgCv3mAf0LBAAgDSAN/QAEECAK/eYB/QsEECANIA39AAQgIAr95gH9CwQgIA0gDf0ABDAgCv3mAf0LBDAgAkEQaiECDAALCwJAA0AgAiAMTw0BIAAgAkECdGohDSANIA39AAQAIAr95gH9CwQAIAJBBGohAgwACwsCQANAIAIgAU8NASAAIAJBAnRqIQ0gDSANKgIAIAmUOAIAIAJBAWohAgwACwsLnQkEAn8MewJ9CX8gBEFwcSEXIARBfHEhGCAEQQJ0IRwgA0F+cSEdQQAhBQJAA0AgBSAdTw0BIAEgBSAcbGohFSAVIBxqIRb9DAAAAAAAAAAAAAAAAAAAAAAhB/0MAAAAAAAAAAAAAAAAAAAAACEI/QwAAAAAAAAAAAAAAAAAAAAAIQn9DAAAAAAAAAAAAAAAAAAAAAAhCv0MAAAAAAAAAAAAAAAAAAAAACEL/QwAAAAAAAAAAAAAAAAAAAAAIQz9DAAAAAAAAAAAAAAAAAAAAAAhDf0MAAAAAAAAAAAAAAAAAAAAACEOQQAhBgJAA0AgBiAXTw0BIAAgBkECdGohGSAVIAZBAnRqIRogFiAGQQJ0aiEbIBn9AAQAIQ8gGf0ABBAhECAZ/QAEICERIBn9AAQwIRIgByAPIBr9AAQA/eYB/eQBIQcgCCAQIBr9AAQQ/eYB/eQBIQggCSARIBr9AAQg/eYB/eQBIQkgCiASIBr9AAQw/eYB/eQBIQogCyAPIBv9AAQA/eYB/eQBIQsgDCAQIBv9AAQQ/eYB/eQBIQwgDSARIBv9AAQg/eYB/eQBIQ0gDiASIBv9AAQw/eYB/eQBIQ4gBkEQaiEGDAALCyAHIAj95AEgCSAK/eQB/eQBIQcgCyAM/eQBIA0gDv3kAf3kASELAkADQCAGIBhPDQEgACAGQQJ0aiEZIBUgBkECdGohGiAWIAZBAnRqIRsgGf0ABAAhDyAHIA8gGv0ABAD95gH95AEhByALIA8gG/0ABAD95gH95AEhCyAGQQRqIQYMAAsLIAf9HwAgB/0fAZIgB/0fAiAH/R8DkpIhEyAL/R8AIAv9HwGSIAv9HwIgC/0fA5KSIRQCQANAIAYgBE8NASAAIAZBAnRqIRkgFSAGQQJ0aiEaIBYgBkECdGohGyATIBkqAgAgGioCAJSSIRMgFCAZKgIAIBsqAgCUkiEUIAZBAWohBgwACwsgAiAFQQJ0aiATOAIAIAIgBUEBakECdGogFDgCACAFQQJqIQUMAAsLIAUgA0kEQCABIAUgHGxqIRX9DAAAAAAAAAAAAAAAAAAAAAAhB/0MAAAAAAAAAAAAAAAAAAAAACEI/QwAAAAAAAAAAAAAAAAAAAAAIQn9DAAAAAAAAAAAAAAAAAAAAAAhCkEAIQYCQANAIAYgF08NASAAIAZBAnRqIRkgFSAGQQJ0aiEaIAcgGf0ABAAgGv0ABAD95gH95AEhByAIIBn9AAQQIBr9AAQQ/eYB/eQBIQggCSAZ/QAEICAa/QAEIP3mAf3kASEJIAogGf0ABDAgGv0ABDD95gH95AEhCiAGQRBqIQYMAAsLIAcgCP3kASAJIAr95AH95AEhBwJAA0AgBiAYTw0BIAAgBkECdGohGSAVIAZBAnRqIRogByAZ/QAEACAa/QAEAP3mAf3kASEHIAZBBGohBgwACwsgB/0fACAH/R8BkiAH/R8CIAf9HwOSkiETAkADQCAGIARPDQEgACAGQQJ0aiEZIBUgBkECdGohGiATIBkqAgAgGioCAJSSIRMgBkEBaiEGDAALCyACIAVBAnRqIBM4AgALCw==";
+const SIMD_WASM_BASE64 = "AGFzbQEAAAABDgJgAn9/AGAFf39/f38AAg8BA2VudgZtZW1vcnkCAAEDAwIAAQcaAglub3JtYWxpemUAAApzZWFyY2hfYWxsAAEKpgQCrQIFAX8BewN9AXsCf/0MAAAAAAAAAAAAAAAAAAAAACEDIAFBfHEhCEEAIQICQANAIAIgCE8NASAAIAJBAnRqIQkgAyAJ/QAEACAJ/QAEAP3mAf3kASEDIAJBBGohAgwACwsgA/0fACAD/R8BkiAD/R8CIAP9HwOSkiEEAkADQCACIAFPDQEgACACQQJ0aiEJIAQgCSoCACAJKgIAlJIhBCACQQFqIQIMAAsLIASRIQUgBUMAAAAAWwRADwtDAACAPyAFlSEGIAb9EyEHQQAhAgJAA0AgAiAITw0BIAAgAkECdGohCSAJIAn9AAQAIAf95gH9CwQAIAJBBGohAgwACwsCQANAIAIgAU8NASAAIAJBAnRqIQkgCSAJKgIAIAaUOAIAIAJBAWohAgwACwsL9AEEAn8BewF9BX8gBEF8cSEKIARBAnQhDUEAIQUCQANAIAUgA08NASABIAUgDWxqIQn9DAAAAAAAAAAAAAAAAAAAAAAhB0EAIQYCQANAIAYgCk8NASAAIAZBAnRqIQsgCSAGQQJ0aiEMIAcgC/0ABAAgDP0ABAD95gH95AEhByAGQQRqIQYMAAsLIAf9HwAgB/0fAZIgB/0fAiAH/R8DkpIhCAJAA0AgBiAETw0BIAAgBkECdGohCyAJIAZBAnRqIQwgCCALKgIAIAwqAgCUkiEIIAZBAWohBgwACwsgAiAFQQJ0aiAIOAIAIAVBAWohBQwACwsL";
 
 export function getSimdWasmBinary(): Uint8Array {
   const binaryString = atob(SIMD_WASM_BASE64);