eigen-db 4.3.0 → 5.0.0

package/dist/vector-db.d.ts

@@ -0,0 +1,131 @@
+/**
+ * VectorDB — Key-Value Vector Database
+ *
+ * Decoupled from embedding providers. Users pass pre-computed vectors
+ * as number arrays (or Float32Array) with string keys.
+ *
+ * Supports:
+ * - set/get/setMany/getMany for key-value CRUD
+ * - query for similarity search (dot product on normalized vectors)
+ * - flush to persist, close to flush+release, clear to wipe
+ * - Streaming export/import using Web Streams API
+ * - Last-write-wins semantics for duplicate keys (append-only storage)
+ */
+import type { ResultItem } from "./result-set";
+import type { OpenOptions, OpenOptionsInternal, QueryOptions, SetOptions, VectorInput } from "./types";
+export declare class VectorDB {
+    private readonly memoryManager;
+    private readonly storage;
+    private readonly _dimensions;
+    private readonly shouldNormalize;
+    private wasmExports;
+    /** Maps key to its slot index in the vector array */
+    private keyToSlot;
+    /** Maps slot index back to its key */
+    private slotToKey;
+    /** Whether this instance has been closed */
+    private closed;
+    private constructor();
+    /**
+     * Opens a VectorDB instance.
+     * Loads existing data from storage into WASM memory.
+     */
+    static open(options: OpenOptions): Promise<VectorDB>;
+    static open(options: OpenOptionsInternal): Promise<VectorDB>;
+    /** Total number of key-value pairs in the database */
+    get size(): number;
+    /** Number of dimensions per vector */
+    get dimensions(): number;
+    /**
+     * Check whether a key exists in the database.
+     * Uses the internal key-to-slot map for O(1) lookup.
+     */
+    has(key: string): boolean;
+    /**
+     * Delete an entry by key. Returns true if the key existed, false otherwise.
+     * Uses swap-and-pop to avoid gaps in the underlying vector array.
+     */
+    delete(key: string): boolean;
+    /**
+     * Returns an iterable of all keys in the database.
+     */
+    keys(): IterableIterator<string>;
+    /**
+     * Returns an iterable of [key, value] pairs.
+     * Values are returned as plain number array copies.
+     */
+    entries(): IterableIterator<[string, number[]]>;
+    /**
+     * Implements the iterable protocol. Same as entries().
+     */
+    [Symbol.iterator](): IterableIterator<[string, number[]]>;
+    /**
+     * Set a key-value pair. If the key already exists, its vector is overwritten (last-write-wins).
+     * The value is a number[] or Float32Array of length equal to the configured dimensions.
+     */
+    set(key: string, value: VectorInput, options?: SetOptions): void;
+    /**
+     * Get the stored vector for a key. Returns undefined if the key does not exist.
+     * Returns a copy of the stored vector as a plain number array.
+     */
+    get(key: string): number[] | undefined;
+    /**
+     * Set multiple key-value pairs at once. Last-write-wins applies within the batch.
+     */
+    setMany(entries: [string, VectorInput][]): void;
+    /**
+     * Get vectors for multiple keys. Returns undefined for keys that don't exist.
+     */
+    getMany(keys: string[]): (number[] | undefined)[];
+    /**
+     * Search for the most similar vectors to the given query vector.
+     *
+     * Default: returns a plain ResultItem[] sorted by descending similarity.
+     * With `{ iterable: true }`: returns a lazy Iterable<ResultItem> where keys
+     * are resolved only as each item is consumed.
+     *
+     * Similarity is the dot product of query and stored vectors. With
+     * normalization (default), this equals cosine similarity: 1 = identical,
+     * -1 = opposite.
+     */
+    query(value: VectorInput, options: QueryOptions & {
+        iterable: true;
+    }): Iterable<ResultItem>;
+    query(value: VectorInput, options?: QueryOptions): ResultItem[];
+    /**
+     * Persist the current in-memory state to storage.
+     */
+    flush(): Promise<void>;
+    /**
+     * Flush data to storage and release the instance.
+     * The instance cannot be used after close.
+     */
+    close(): Promise<void>;
+    /**
+     * Clear all data from the database and storage.
+     */
+    clear(): Promise<void>;
+    /**
+     * Export the entire database as a ReadableStream of binary chunks.
+     *
+     * Format: [Header 24 bytes][Vector data][Keys data]
+     * Header: magic(4) + version(4) + dimensions(4) + vectorCount(4) + vectorDataLen(4) + keysDataLen(4)
+     *
+     * Vectors are streamed in 64KB chunks from WASM memory to avoid large
+     * heap allocations.
+     */
+    export(): Promise<ReadableStream<Uint8Array>>;
+    /**
+     * Import data from a ReadableStream, replacing all existing data.
+     * Performs a dimension check against the configured dimensions.
+     *
+     * Vectors are streamed directly into WASM memory in chunks to avoid
+     * large heap allocations.
+     */
+    import(stream: ReadableStream<Uint8Array>): Promise<void>;
+    /**
+     * Normalize a vector using WASM (if available) or JS fallback.
+     */
+    private normalizeVector;
+    private assertOpen;
+}

package/dist/wasm-compute.d.ts

@@ -0,0 +1,13 @@
+/**
+ * WASM SIMD compute layer.
+ * Compiles the hand-written WAT module and provides typed wrappers
+ * that operate on shared WebAssembly.Memory.
+ */
+export interface WasmExports {
+    normalize(ptr: number, dimensions: number): void;
+    search_all(queryPtr: number, dbPtr: number, scoresPtr: number, dbSize: number, dimensions: number): void;
+}
+/**
+ * Instantiates a WASM module with the given memory and returns typed exports.
+ */
+export declare function instantiateWasm(wasmBinary: Uint8Array, memory: WebAssembly.Memory): Promise<WasmExports>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eigen-db",
-  "version": "4.3.0",
+  "version": "5.0.0",
   "type": "module",
   "files": [
     "dist",
@@ -12,16 +12,16 @@
   "module": "./dist/eigen-db.js",
   "exports": {
     ".": {
-      "types": "./src/lib/index.ts",
+      "types": "./dist/index.d.ts",
       "import": "./dist/eigen-db.js",
       "require": "./dist/eigen-db.umd.cjs"
     }
   },
-  "types": "./src/lib/index.ts",
+  "types": "./dist/index.d.ts",
   "scripts": {
     "dev": "vite",
     "compile-wat": "tsx scripts/compile-wat.ts",
-    "build": "npm run compile-wat && tsc && vite build",
+    "build": "npm run compile-wat && tsc && vite build && tsc -p tsconfig.build.json",
     "preview": "vite preview",
     "test": "vitest run",
     "test:watch": "vitest",

package/src/lib/__tests__/result-set.test.ts

@@ -1,13 +1,13 @@
 import { describe, expect, it } from "vitest";
-import { iterableResults, topKResults } from "../result-set";
+import { iterableResults, queryResults } from "../result-set";
 
-describe("topKResults", () => {
+describe("queryResults", () => {
   const keys = ["apple", "banana", "cherry", "date", "elderberry"];
   const resolveKey = (index: number) => keys[index];
 
-  it("sorts results by descending similarity", () => {
+  it("sorts results by descending similarity (default order)", () => {
     const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
-    const results = topKResults(scores, resolveKey, 5);
+    const results = queryResults(scores, resolveKey, { limit: Infinity, order: "descend" });
 
     expect(results).toHaveLength(5);
     expect(results[0].key).toBe("banana");
@@ -19,9 +19,22 @@ describe("topKResults", () => {
     expect(results[4].key).toBe("cherry");
   });
 
-  it("respects topK limit", () => {
+  it("sorts results by ascending similarity", () => {
+    const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
+    const results = queryResults(scores, resolveKey, { limit: Infinity, order: "ascend" });
+
+    expect(results).toHaveLength(5);
+    expect(results[0].key).toBe("cherry");
+    expect(results[0].similarity).toBeCloseTo(0.1, 4);
+    expect(results[1].key).toBe("apple");
+    expect(results[1].similarity).toBeCloseTo(0.3, 4);
+    expect(results[4].key).toBe("banana");
+    expect(results[4].similarity).toBeCloseTo(0.9, 4);
+  });
+
+  it("respects limit", () => {
     const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
-    const results = topKResults(scores, resolveKey, 3);
+    const results = queryResults(scores, resolveKey, { limit: 3, order: "descend" });
 
     expect(results).toHaveLength(3);
     expect(results[0].key).toBe("banana");
@@ -31,20 +44,19 @@ describe("topKResults", () => {
 
   it("handles empty scores", () => {
     const scores = new Float32Array(0);
-    const results = topKResults(scores, resolveKey, 10);
+    const results = queryResults(scores, resolveKey, { limit: 10, order: "descend" });
     expect(results).toEqual([]);
   });
 
-  it("handles topK larger than result count", () => {
+  it("handles limit larger than result count", () => {
     const scores = new Float32Array([0.5, 0.8]);
-    const results = topKResults(scores, resolveKey, 100);
+    const results = queryResults(scores, resolveKey, { limit: 100, order: "descend" });
     expect(results).toHaveLength(2);
   });
 
   it("filters results by minSimilarity", () => {
     const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
-    // similarities: apple=0.3, banana=0.9, cherry=0.1, date=0.7, elderberry=0.5
-    const results = topKResults(scores, resolveKey, 5, 0.5);
+    const results = queryResults(scores, resolveKey, { limit: Infinity, order: "descend", minSimilarity: 0.5 });
 
     expect(results).toHaveLength(3);
     expect(results[0].key).toBe("banana");
@@ -57,15 +69,81 @@ describe("topKResults", () => {
 
   it("minSimilarity is inclusive", () => {
     const scores = new Float32Array([0.5]);
-    // similarity = 0.5
-    const results = topKResults(scores, resolveKey, 10, 0.5);
+    const results = queryResults(scores, resolveKey, { limit: 10, order: "descend", minSimilarity: 0.5 });
     expect(results).toHaveLength(1);
   });
 
-  it("handles topK Infinity", () => {
+  it("filters results by maxSimilarity", () => {
     const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
-    const results = topKResults(scores, resolveKey, Infinity);
+    const results = queryResults(scores, resolveKey, { limit: Infinity, order: "descend", maxSimilarity: 0.7 });
+
+    expect(results).toHaveLength(4);
+    expect(results[0].key).toBe("date");
+    expect(results[0].similarity).toBeCloseTo(0.7, 4);
+    expect(results[1].key).toBe("elderberry");
+    expect(results[2].key).toBe("apple");
+    expect(results[3].key).toBe("cherry");
+  });
+
+  it("maxSimilarity is inclusive", () => {
+    const scores = new Float32Array([0.5]);
+    const results = queryResults(scores, resolveKey, { limit: 10, order: "descend", maxSimilarity: 0.5 });
+    expect(results).toHaveLength(1);
+  });
+
+  it("filters by both minSimilarity and maxSimilarity", () => {
+    const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
+    const results = queryResults(scores, resolveKey, { limit: Infinity, order: "descend", minSimilarity: 0.3, maxSimilarity: 0.7 });
+
+    expect(results).toHaveLength(3);
+    expect(results[0].key).toBe("date");
+    expect(results[1].key).toBe("elderberry");
+    expect(results[2].key).toBe("apple");
+  });
+
+  it("handles limit Infinity", () => {
+    const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
+    const results = queryResults(scores, resolveKey, { limit: Infinity, order: "descend" });
+    expect(results).toHaveLength(5);
+  });
+
+  it("supports full similarity range [-1, 1]", () => {
+    const scores = new Float32Array([-1.0, -0.5, 0.0, 0.5, 1.0]);
+    const results = queryResults(scores, resolveKey, { limit: Infinity, order: "descend" });
+
     expect(results).toHaveLength(5);
+    expect(results[0].similarity).toBeCloseTo(1.0, 5);
+    expect(results[4].similarity).toBeCloseTo(-1.0, 5);
+  });
+
+  it("handles tiny floating point values near boundaries", () => {
+    const epsilon = 1e-7;
+    const scores = new Float32Array([0.5 - epsilon, 0.5, 0.5 + epsilon]);
+    const results = queryResults(scores, resolveKey, { limit: Infinity, order: "descend", minSimilarity: 0.5 });
+
+    // 0.5 and 0.5 + epsilon should pass, 0.5 - epsilon should not
+    expect(results).toHaveLength(2);
+  });
+
+  it("ascending order with limit returns bottomK", () => {
+    const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
+    const results = queryResults(scores, resolveKey, { limit: 2, order: "ascend" });
+
+    expect(results).toHaveLength(2);
+    expect(results[0].key).toBe("cherry");
+    expect(results[0].similarity).toBeCloseTo(0.1, 4);
+    expect(results[1].key).toBe("apple");
+    expect(results[1].similarity).toBeCloseTo(0.3, 4);
+  });
+
+  it("ascending order with minSimilarity and maxSimilarity", () => {
+    const scores = new Float32Array([-1.0, -0.5, 0.0, 0.5, 1.0]);
+    const results = queryResults(scores, resolveKey, { limit: Infinity, order: "ascend", minSimilarity: -0.5, maxSimilarity: 0.5 });
+
+    expect(results).toHaveLength(3);
+    expect(results[0].similarity).toBeCloseTo(-0.5, 5);
+    expect(results[1].similarity).toBeCloseTo(0.0, 5);
+    expect(results[2].similarity).toBeCloseTo(0.5, 5);
   });
 });
 
@@ -75,7 +153,7 @@ describe("iterableResults", () => {
 
   it("sorts results by descending similarity", () => {
     const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
-    const results = [...iterableResults(scores, resolveKey, 5)];
+    const results = [...iterableResults(scores, resolveKey, { limit: Infinity, order: "descend" })];
 
     expect(results).toHaveLength(5);
     expect(results[0].key).toBe("banana");
@@ -84,16 +162,26 @@ describe("iterableResults", () => {
     expect(results[4].key).toBe("cherry");
   });
 
-  it("respects topK limit", () => {
+  it("sorts results by ascending similarity", () => {
     const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
-    const results = [...iterableResults(scores, resolveKey, 3)];
+    const results = [...iterableResults(scores, resolveKey, { limit: Infinity, order: "ascend" })];
+
+    expect(results).toHaveLength(5);
+    expect(results[0].key).toBe("cherry");
+    expect(results[0].similarity).toBeCloseTo(0.1, 4);
+    expect(results[4].key).toBe("banana");
+  });
+
+  it("respects limit", () => {
+    const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
+    const results = [...iterableResults(scores, resolveKey, { limit: 3, order: "descend" })];
 
     expect(results).toHaveLength(3);
     expect(results[0].key).toBe("banana");
   });
 
   it("handles empty scores", () => {
-    const results = [...iterableResults(new Float32Array(0), resolveKey, 10)];
+    const results = [...iterableResults(new Float32Array(0), resolveKey, { limit: 10, order: "descend" })];
     expect(results).toEqual([]);
   });
 
@@ -105,7 +193,7 @@ describe("iterableResults", () => {
     };
 
     const scores = new Float32Array([0.3, 0.9, 0.1]);
-    const iterable = iterableResults(scores, lazyResolver, 3);
+    const iterable = iterableResults(scores, lazyResolver, { limit: Infinity, order: "descend" });
 
     expect(callCount).toBe(0); // no key resolved yet
 
@@ -119,7 +207,7 @@ describe("iterableResults", () => {
 
   it("is re-iterable", () => {
     const scores = new Float32Array([0.3, 0.9, 0.1]);
-    const iterable = iterableResults(scores, resolveKey, 3);
+    const iterable = iterableResults(scores, resolveKey, { limit: Infinity, order: "descend" });
 
     const first = [...iterable];
     const second = [...iterable];
@@ -128,7 +216,7 @@ describe("iterableResults", () => {
 
   it("supports partial iteration (early break)", () => {
     const scores = new Float32Array([0.1, 0.2, 0.3, 0.4, 0.5]);
-    const iterable = iterableResults(scores, resolveKey, 5);
+    const iterable = iterableResults(scores, resolveKey, { limit: Infinity, order: "descend" });
 
     const partial: string[] = [];
     for (const item of iterable) {
@@ -140,10 +228,9 @@ describe("iterableResults", () => {
     expect(partial[1]).toBe("date"); // similarity 0.4
   });
 
-  it("early stops iteration by minSimilarity", () => {
+  it("filters by minSimilarity", () => {
     const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
-    // similarities: apple=0.3, banana=0.9, cherry=0.1, date=0.7, elderberry=0.5
-    const results = [...iterableResults(scores, resolveKey, 5, 0.5)];
+    const results = [...iterableResults(scores, resolveKey, { limit: Infinity, order: "descend", minSimilarity: 0.5 })];
 
     expect(results).toHaveLength(3);
     expect(results[0].key).toBe("banana");
@@ -153,8 +240,40 @@ describe("iterableResults", () => {
 
   it("minSimilarity is inclusive", () => {
     const scores = new Float32Array([0.5]);
-    // similarity = 0.5
-    const results = [...iterableResults(scores, resolveKey, 10, 0.5)];
+    const results = [...iterableResults(scores, resolveKey, { limit: Infinity, order: "descend", minSimilarity: 0.5 })];
     expect(results).toHaveLength(1);
   });
+
+  it("filters by maxSimilarity", () => {
+    const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
+    const results = [...iterableResults(scores, resolveKey, { limit: Infinity, order: "descend", maxSimilarity: 0.7 })];
+
+    expect(results).toHaveLength(4);
+    expect(results[0].key).toBe("date");
+    expect(results[3].key).toBe("cherry");
+  });
+
+  it("maxSimilarity is inclusive", () => {
+    const scores = new Float32Array([0.5]);
+    const results = [...iterableResults(scores, resolveKey, { limit: Infinity, order: "descend", maxSimilarity: 0.5 })];
+    expect(results).toHaveLength(1);
+  });
+
+  it("ascending order with limit returns bottomK", () => {
+    const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
+    const results = [...iterableResults(scores, resolveKey, { limit: 2, order: "ascend" })];
+
+    expect(results).toHaveLength(2);
+    expect(results[0].key).toBe("cherry");
+    expect(results[1].key).toBe("apple");
+  });
+
+  it("supports full similarity range [-1, 1]", () => {
+    const scores = new Float32Array([-1.0, -0.5, 0.0, 0.5, 1.0]);
+    const results = [...iterableResults(scores, resolveKey, { limit: Infinity, order: "ascend" })];
+
+    expect(results).toHaveLength(5);
+    expect(results[0].similarity).toBeCloseTo(-1.0, 5);
+    expect(results[4].similarity).toBeCloseTo(1.0, 5);
+  });
 });