eigen-db 4.1.0 → 4.3.0

package/src/lib/__tests__/result-set.test.ts

@@ -5,15 +5,15 @@ describe("topKResults", () => {
   const keys = ["apple", "banana", "cherry", "date", "elderberry"];
   const resolveKey = (index: number) => keys[index];
 
-  it("sorts results by ascending distance", () => {
+  it("sorts results by descending similarity", () => {
     const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
     const results = topKResults(scores, resolveKey, 5);
 
     expect(results).toHaveLength(5);
     expect(results[0].key).toBe("banana");
-    expect(results[0].distance).toBeCloseTo(0.1, 4);
+    expect(results[0].similarity).toBeCloseTo(0.9, 4);
     expect(results[1].key).toBe("date");
-    expect(results[1].distance).toBeCloseTo(0.3, 4);
+    expect(results[1].similarity).toBeCloseTo(0.7, 4);
     expect(results[2].key).toBe("elderberry");
     expect(results[3].key).toBe("apple");
     expect(results[4].key).toBe("cherry");
@@ -25,7 +25,7 @@ describe("topKResults", () => {
 
     expect(results).toHaveLength(3);
     expect(results[0].key).toBe("banana");
-    expect(results[0].distance).toBeCloseTo(0.1, 4);
+    expect(results[0].similarity).toBeCloseTo(0.9, 4);
     expect(results[2].key).toBe("elderberry");
   });
 
@@ -41,23 +41,23 @@ describe("topKResults", () => {
     expect(results).toHaveLength(2);
   });
 
-  it("filters results by maxDistance", () => {
+  it("filters results by minSimilarity", () => {
     const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
-    // distances: apple=0.7, banana=0.1, cherry=0.9, date=0.3, elderberry=0.5
+    // similarities: apple=0.3, banana=0.9, cherry=0.1, date=0.7, elderberry=0.5
     const results = topKResults(scores, resolveKey, 5, 0.5);
 
     expect(results).toHaveLength(3);
     expect(results[0].key).toBe("banana");
-    expect(results[0].distance).toBeCloseTo(0.1, 4);
+    expect(results[0].similarity).toBeCloseTo(0.9, 4);
     expect(results[1].key).toBe("date");
-    expect(results[1].distance).toBeCloseTo(0.3, 4);
+    expect(results[1].similarity).toBeCloseTo(0.7, 4);
     expect(results[2].key).toBe("elderberry");
-    expect(results[2].distance).toBeCloseTo(0.5, 4);
+    expect(results[2].similarity).toBeCloseTo(0.5, 4);
   });
 
-  it("maxDistance is inclusive", () => {
+  it("minSimilarity is inclusive", () => {
     const scores = new Float32Array([0.5]);
-    // distance = 1 - 0.5 = 0.5
+    // similarity = 0.5
     const results = topKResults(scores, resolveKey, 10, 0.5);
     expect(results).toHaveLength(1);
   });
@@ -73,13 +73,13 @@ describe("iterableResults", () => {
   const keys = ["apple", "banana", "cherry", "date", "elderberry"];
   const resolveKey = (index: number) => keys[index];
 
-  it("sorts results by ascending distance", () => {
+  it("sorts results by descending similarity", () => {
     const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
     const results = [...iterableResults(scores, resolveKey, 5)];
 
     expect(results).toHaveLength(5);
     expect(results[0].key).toBe("banana");
-    expect(results[0].distance).toBeCloseTo(0.1, 4);
+    expect(results[0].similarity).toBeCloseTo(0.9, 4);
     expect(results[1].key).toBe("date");
     expect(results[4].key).toBe("cherry");
   });
@@ -136,13 +136,13 @@ describe("iterableResults", () => {
       if (partial.length === 2) break;
     }
     expect(partial).toHaveLength(2);
-    expect(partial[0]).toBe("elderberry"); // distance 0.5 (lowest)
-    expect(partial[1]).toBe("date"); // distance 0.6
+    expect(partial[0]).toBe("elderberry"); // similarity 0.5 (highest)
+    expect(partial[1]).toBe("date"); // similarity 0.4
   });
 
-  it("early stops iteration by maxDistance", () => {
+  it("early stops iteration by minSimilarity", () => {
     const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
-    // distances: apple=0.7, banana=0.1, cherry=0.9, date=0.3, elderberry=0.5
+    // similarities: apple=0.3, banana=0.9, cherry=0.1, date=0.7, elderberry=0.5
     const results = [...iterableResults(scores, resolveKey, 5, 0.5)];
 
     expect(results).toHaveLength(3);
@@ -151,9 +151,9 @@ describe("iterableResults", () => {
     expect(results[2].key).toBe("elderberry");
   });
 
-  it("maxDistance is inclusive", () => {
+  it("minSimilarity is inclusive", () => {
     const scores = new Float32Array([0.5]);
-    // distance = 1 - 0.5 = 0.5
+    // similarity = 0.5
     const results = [...iterableResults(scores, resolveKey, 10, 0.5)];
     expect(results).toHaveLength(1);
   });

package/src/lib/__tests__/vector-db.test.ts

@@ -16,6 +16,51 @@ function getWasmBinary(): Promise<Uint8Array> {
   return wasmBinaryPromise;
 }
 
+/** Collect a ReadableStream into a single Uint8Array. */
+async function streamToBytes(stream: ReadableStream<Uint8Array>): Promise<Uint8Array> {
+  const chunks: Uint8Array[] = [];
+  const reader = stream.getReader();
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    chunks.push(value);
+  }
+  const totalSize = chunks.reduce((sum, c) => sum + c.byteLength, 0);
+  const result = new Uint8Array(totalSize);
+  let offset = 0;
+  for (const chunk of chunks) {
+    result.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return result;
+}
+
+/** Create a ReadableStream from a Uint8Array (single chunk). */
+function bytesToStream(data: Uint8Array): ReadableStream<Uint8Array> {
+  return new ReadableStream({
+    start(controller) {
+      controller.enqueue(data);
+      controller.close();
+    },
+  });
+}
+
+/** Create a ReadableStream from a Uint8Array, delivering it in small chunks. */
+function bytesToChunkedStream(data: Uint8Array, chunkSize: number): ReadableStream<Uint8Array> {
+  let offset = 0;
+  return new ReadableStream({
+    pull(controller) {
+      if (offset >= data.byteLength) {
+        controller.close();
+        return;
+      }
+      const end = Math.min(offset + chunkSize, data.byteLength);
+      controller.enqueue(data.subarray(offset, end));
+      offset = end;
+    },
+  });
+}
+
 describe("VectorDB", () => {
   let storage: InMemoryStorageProvider;
 
@@ -180,17 +225,17 @@ describe("VectorDB", () => {
       const results = db.query([1, 0, 0, 0]);
       expect(results.length).toBe(3);
 
-      // x-axis should be the best match (identical direction, distance ≈ 0)
+      // x-axis should be the best match (identical direction, similarity ≈ 1)
       expect(results[0].key).toBe("x-axis");
-      expect(results[0].distance).toBeCloseTo(0.0, 2);
+      expect(results[0].similarity).toBeCloseTo(1.0, 2);
 
       // xy-axis should be second (partially aligned)
       expect(results[1].key).toBe("xy-axis");
-      expect(results[1].distance).toBeLessThan(1);
+      expect(results[1].similarity).toBeGreaterThan(0);
 
-      // y-axis should be last (orthogonal, distance ≈ 1)
+      // y-axis should be last (orthogonal, similarity ≈ 0)
       expect(results[2].key).toBe("y-axis");
-      expect(results[2].distance).toBeCloseTo(1.0, 2);
+      expect(results[2].similarity).toBeCloseTo(0.0, 2);
     });
 
     it("query respects topK option", async () => {
@@ -251,7 +296,7 @@ describe("VectorDB", () => {
       expect(all).toHaveLength(5);
 
       // Partial iteration (simulate pagination)
-      const page: { key: string; distance: number }[] = [];
+      const page: { key: string; similarity: number }[] = [];
       for (const item of results) {
         page.push(item);
         if (page.length === 2) break;
@@ -273,13 +318,13 @@ describe("VectorDB", () => {
       db.set("point", [0, 1, 0, 0]);
 
       const results = db.query([0, 1, 0, 0]);
-      // Both 'point' and 'other' are now along y-axis, so both should have distance ≈ 0
-      expect(results[0].distance).toBeCloseTo(0.0, 2);
-      expect(results[1].distance).toBeCloseTo(0.0, 2);
+      // Both 'point' and 'other' are now along y-axis, so both should have similarity ≈ 1
+      expect(results[0].similarity).toBeCloseTo(1.0, 2);
+      expect(results[1].similarity).toBeCloseTo(1.0, 2);
       expect(db.size).toBe(2);
     });
 
-    it("query respects maxDistance option", async () => {
+    it("query respects minSimilarity option", async () => {
       const db = await VectorDB.open({
         dimensions: 4,
         storage,
@@ -290,16 +335,16 @@ describe("VectorDB", () => {
       db.set("y-axis", [0, 1, 0, 0]);
       db.set("xy-axis", [1, 1, 0, 0]);
 
-      // Only return results with distance ≤ 0.5 from the x-axis query
-      const results = db.query([1, 0, 0, 0], { maxDistance: 0.5 });
-      // x-axis: distance ≈ 0, xy-axis: distance ≈ 0.29
-      // y-axis: distance ≈ 1 (excluded)
+      // Only return results with similarity ≥ 0.5 from the x-axis query
+      const results = db.query([1, 0, 0, 0], { minSimilarity: 0.5 });
+      // x-axis: similarity ≈ 1, xy-axis: similarity ≈ 0.71
+      // y-axis: similarity ≈ 0 (excluded)
       expect(results.length).toBe(2);
       expect(results[0].key).toBe("x-axis");
       expect(results[1].key).toBe("xy-axis");
     });
 
-    it("query maxDistance works with iterable mode", async () => {
+    it("query minSimilarity works with iterable mode", async () => {
       const db = await VectorDB.open({
         dimensions: 4,
         storage,
@@ -310,7 +355,7 @@ describe("VectorDB", () => {
       db.set("y-axis", [0, 1, 0, 0]);
       db.set("xy-axis", [1, 1, 0, 0]);
 
-      const results = [...db.query([1, 0, 0, 0], { maxDistance: 0.5, iterable: true })];
+      const results = [...db.query([1, 0, 0, 0], { minSimilarity: 0.5, iterable: true })];
       expect(results.length).toBe(2);
       expect(results[0].key).toBe("x-axis");
       expect(results[1].key).toBe("xy-axis");
@@ -491,6 +536,334 @@ describe("VectorDB", () => {
       const result = db.get("a");
       expect(result![0]).toBeCloseTo(1.0);
     });
+
+    // --- export and import (streaming) ---
+    it("export returns a readable stream of the database", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db.set("a", [1, 2, 3, 4]);
+      db.set("b", [5, 6, 7, 8]);
+
+      const stream = await db.export();
+      expect(stream).toBeInstanceOf(ReadableStream);
+
+      const bytes = await streamToBytes(stream);
+      expect(bytes.byteLength).toBeGreaterThan(0);
+    });
+
+    it("export of empty database returns a valid stream", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      const stream = await db.export();
+      const bytes = await streamToBytes(stream);
+      expect(bytes.byteLength).toBeGreaterThan(0);
+    });
+
+    it("import restores data from an exported stream", async () => {
+      const db1 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db1.set("alpha", [1, 0, 0, 0]);
+      db1.set("beta", [0, 1, 0, 0]);
+
+      const stream = await db1.export();
+
+      // Import into a fresh database
+      const storage2 = new InMemoryStorageProvider();
+      const db2 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage: storage2,
+        wasmBinary,
+      });
+
+      await db2.import(stream);
+      expect(db2.size).toBe(2);
+      expect(db2.get("alpha")![0]).toBeCloseTo(1);
+      expect(db2.get("beta")![1]).toBeCloseTo(1);
+    });
+
+    it("import overrides existing data", async () => {
+      const db1 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db1.set("a", [1, 0, 0, 0]);
+      const stream = await db1.export();
+
+      // Create db2 with different data
+      const storage2 = new InMemoryStorageProvider();
+      const db2 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage: storage2,
+        wasmBinary,
+      });
+
+      db2.set("x", [0, 0, 0, 1]);
+      db2.set("y", [0, 0, 1, 0]);
+      expect(db2.size).toBe(2);
+
+      await db2.import(stream);
+      expect(db2.size).toBe(1);
+      expect(db2.get("a")![0]).toBeCloseTo(1);
+      expect(db2.get("x")).toBeUndefined();
+      expect(db2.get("y")).toBeUndefined();
+    });
+
+    it("import throws on dimension mismatch", async () => {
+      const db1 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db1.set("a", [1, 0, 0, 0]);
+      const bytes = await streamToBytes(await db1.export());
+
+      const storage2 = new InMemoryStorageProvider();
+      const db2 = await VectorDB.open({
+        dimensions: 8,
+        normalize: false,
+        storage: storage2,
+        wasmBinary,
+      });
+
+      await expect(db2.import(bytesToStream(bytes))).rejects.toThrow("dimension");
+    });
+
+    it("import of empty export clears the database", async () => {
+      const db1 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      const stream = await db1.export();
+
+      const storage2 = new InMemoryStorageProvider();
+      const db2 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage: storage2,
+        wasmBinary,
+      });
+
+      db2.set("existing", [1, 0, 0, 0]);
+      await db2.import(stream);
+      expect(db2.size).toBe(0);
+    });
+
+    it("imported data is queryable", async () => {
+      const db1 = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      db1.set("x-axis", [1, 0, 0, 0]);
+      db1.set("y-axis", [0, 1, 0, 0]);
+      db1.set("xy-axis", [1, 1, 0, 0]);
+
+      const stream = await db1.export();
+
+      const storage2 = new InMemoryStorageProvider();
+      const db2 = await VectorDB.open({
+        dimensions: 4,
+        storage: storage2,
+        wasmBinary,
+      });
+
+      await db2.import(stream);
+
+      const results = db2.query([1, 0, 0, 0]);
+      expect(results.length).toBe(3);
+      expect(results[0].key).toBe("x-axis");
+      expect(results[0].similarity).toBeCloseTo(1.0, 2);
+    });
+
+    it("export and import preserve data after set operations on imported db", async () => {
+      const db1 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db1.set("a", [1, 0, 0, 0]);
+      const stream = await db1.export();
+
+      const storage2 = new InMemoryStorageProvider();
+      const db2 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage: storage2,
+        wasmBinary,
+      });
+
+      await db2.import(stream);
+      db2.set("b", [0, 1, 0, 0]);
+
+      expect(db2.size).toBe(2);
+      expect(db2.get("a")![0]).toBeCloseTo(1);
+      expect(db2.get("b")![1]).toBeCloseTo(1);
+    });
+
+    it("export throws on closed database", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      await db.close();
+      await expect(db.export()).rejects.toThrow("closed");
+    });
+
+    it("import throws on closed database", async () => {
+      const db1 = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      const stream = await db1.export();
+
+      const storage2 = new InMemoryStorageProvider();
+      const db2 = await VectorDB.open({
+        dimensions: 4,
+        storage: storage2,
+        wasmBinary,
+      });
+
+      await db2.close();
+      await expect(db2.import(stream)).rejects.toThrow("closed");
+    });
+
+    it("import throws on invalid stream (bad magic)", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      const badBlob = new Uint8Array(24);
+      await expect(db.import(bytesToStream(badBlob))).rejects.toThrow();
+    });
+
+    it("import throws on stream too short for header", async () => {
+      const db = await VectorDB.open({
+        dimensions: 4,
+        storage,
+        wasmBinary,
+      });
+
+      const tooShort = new Uint8Array(10);
+      await expect(db.import(bytesToStream(tooShort))).rejects.toThrow();
+    });
+
+    it("import throws on truncated stream body", async () => {
+      const db1 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db1.set("a", [1, 0, 0, 0]);
+      const bytes = await streamToBytes(await db1.export());
+
+      // Truncate the blob to have valid header but incomplete body
+      const truncated = bytes.slice(0, 25);
+
+      const storage2 = new InMemoryStorageProvider();
+      const db2 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage: storage2,
+        wasmBinary,
+      });
+
+      await expect(db2.import(bytesToStream(truncated))).rejects.toThrow("unexpected end of stream");
+    });
+
+    it("import works correctly with chunked stream (small chunks)", async () => {
+      const db1 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db1.set("alpha", [1, 0, 0, 0]);
+      db1.set("beta", [0, 1, 0, 0]);
+      db1.set("gamma", [0, 0, 1, 0]);
+
+      const bytes = await streamToBytes(await db1.export());
+
+      // Feed it back as a stream with tiny 7-byte chunks (crosses header/vector/key boundaries)
+      const chunkedStream = bytesToChunkedStream(bytes, 7);
+
+      const storage2 = new InMemoryStorageProvider();
+      const db2 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage: storage2,
+        wasmBinary,
+      });
+
+      await db2.import(chunkedStream);
+      expect(db2.size).toBe(3);
+      expect(db2.get("alpha")![0]).toBeCloseTo(1);
+      expect(db2.get("beta")![1]).toBeCloseTo(1);
+      expect(db2.get("gamma")![2]).toBeCloseTo(1);
+    });
+
+    it("import works correctly with single-byte stream chunks", async () => {
+      const db1 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage,
+        wasmBinary,
+      });
+
+      db1.set("k", [1, 2, 3, 4]);
+
+      const bytes = await streamToBytes(await db1.export());
+      const chunkedStream = bytesToChunkedStream(bytes, 1);
+
+      const storage2 = new InMemoryStorageProvider();
+      const db2 = await VectorDB.open({
+        dimensions: 4,
+        normalize: false,
+        storage: storage2,
+        wasmBinary,
+      });
+
+      await db2.import(chunkedStream);
+      expect(db2.size).toBe(1);
+      expect(db2.get("k")![0]).toBeCloseTo(1);
+      expect(db2.get("k")![3]).toBeCloseTo(4);
+    });
   }
 
   it("throws VectorCapacityExceededError when full", async () => {
@@ -498,4 +871,44 @@ describe("VectorDB", () => {
     expect(err).toBeInstanceOf(VectorCapacityExceededError);
     expect(err.message).toContain("100");
   });
+
+  // --- storage decoupling ---
+  it("defaults to in-memory storage when no storage is provided", async () => {
+    const db = await VectorDB.open({
+      dimensions: 4,
+      wasmBinary: null,
+    });
+
+    db.set("a", [1, 0, 0, 0]);
+    expect(db.size).toBe(1);
+    expect(db.get("a")).toBeDefined();
+
+    // Flush should succeed with default in-memory storage
+    await db.flush();
+    await db.close();
+  });
+
+  it("accepts an explicit storage provider via options", async () => {
+    const customStorage = new InMemoryStorageProvider();
+    const db = await VectorDB.open({
+      dimensions: 4,
+      normalize: false,
+      storage: customStorage,
+      wasmBinary: null,
+    });
+
+    db.set("key1", [1, 2, 3, 4]);
+    await db.flush();
+
+    // Reopen with the same storage to verify persistence
+    const db2 = await VectorDB.open({
+      dimensions: 4,
+      normalize: false,
+      storage: customStorage,
+      wasmBinary: null,
+    });
+
+    expect(db2.size).toBe(1);
+    expect(db2.get("key1")![0]).toBeCloseTo(1);
+  });
 });

package/src/lib/memory-manager.ts

@@ -144,4 +144,12 @@ export class MemoryManager {
   reset(): void {
     this._vectorCount = 0;
   }
+
+  /**
+   * Set the vector count after bulk-loading data directly into the DB region.
+   * Use when writing raw bytes to the DB region externally (e.g., streaming import).
+   */
+  setVectorCount(count: number): void {
+    this._vectorCount = count;
+  }
 }