eigen-db 3.0.0 → 4.0.0

package/CHANGELOG.md

@@ -1,3 +1,7 @@
+# v4.0.0
+
+Changed: `query()` returns simple iterable
+
 # v3.0.0
 
 Changed: Input vectors should be of type `number[]`, not `Float32Array`.

package/README.md

@@ -50,15 +50,27 @@ Notes:
 ```ts
 const queryVector = embeddingQuery;
 
+// Returns a plain array of { key, score } sorted by similarity
 const results = db.query(queryVector, { topK: 10 });
 
-for (let i = 0; i < results.length; i++) {
-  const { key, score } = results.get(i);
-  console.log(i, key, score);
+for (const { key, score } of results) {
+  console.log(key, score);
+}
+```
+
+For lazy iteration (useful for pagination or early stopping):
+
+```ts
+const results = db.query(queryVector, { topK: 100, iterable: true });
+
+// Iterate and break early — keys are resolved on demand
+for (const { key, score } of results) {
+  if (score < 0.5) break;
+  console.log(key, score);
 }
 
-// Or paginate for UI rendering:
-const page0 = results.getPage(0, 10);
+// Or spread into an array when you need all results
+const all = [...results];
 ```
 
 ### 4) Persist and lifecycle
@@ -80,10 +92,9 @@ await db.clear();
 
 ```ts
 export { DB };
-export { ResultSet };
 export type { ResultItem };
 export { VectorCapacityExceededError };
-export type { OpenOptions, OpenOptionsInternal, SetOptions, QueryOptions, VectorInput };
+export type { OpenOptions, OpenOptionsInternal, SetOptions, QueryOptions, IterableQueryOptions, VectorInput };
 export { InMemoryStorageProvider, OPFSStorageProvider };
 export type { StorageProvider };
 ```
@@ -114,8 +125,12 @@ Opens (or creates) a database instance and loads persisted data.
   - Batch insert/update.
 - `getMany(keys: string[]): (number[] | undefined)[]`
   - Batch lookup.
-- `query(value: VectorInput, options?: QueryOptions): ResultSet`
-  - Returns similarity-ranked results.
+- `query(value: VectorInput, options?: QueryOptions): ResultItem[]`
+  - Returns similarity-ranked results as a plain array.
+  - Throws on dimension mismatch.
+- `query(value: VectorInput, options: IterableQueryOptions): Iterable<ResultItem>`
+  - With `{ iterable: true }`, returns a lazy iterable. Keys are resolved
+    only as each item is consumed, enabling early stopping and pagination.
   - Throws on dimension mismatch.
 - `flush(): Promise<void>`
   - Persists in-memory state to storage.
@@ -125,27 +140,6 @@ Opens (or creates) a database instance and loads persisted data.
 - `clear(): Promise<void>`
   - Clears in-memory state and destroys storage for this DB.
 
-### `ResultSet`
-
-Represents a lazily resolved, score-sorted search result collection.
-
-#### Properties
-
-- `length: number` — number of results available (bounded by `topK`)
-
-#### Methods
-
-- `get(rank: number): ResultItem`
-  - Returns the item at rank (`0` is best match).
-  - Throws `RangeError` when out of bounds.
-- `getPage(page: number, pageSize: number): ResultItem[]`
-  - Convenience pagination helper.
-
-#### Static
-
-- `fromScores(scores, resolveKey, topK): ResultSet`
-  - Constructs a sorted lazy result set from raw scores.
-
 ### `ResultItem`
 
 ```ts
@@ -201,6 +195,14 @@ interface QueryOptions {
 }
 ```
 
+#### `IterableQueryOptions`
+
+```ts
+interface IterableQueryOptions extends QueryOptions {
+  iterable: true; // returns Iterable<ResultItem> instead of ResultItem[]
+}
+```
+
 ### Storage
 
 #### `StorageProvider`
@@ -239,5 +241,5 @@ Thrown when memory growth would exceed WASM 32-bit memory limits for the configu
 ## Practical notes
 
 - Similarity is dot product; with normalization enabled (default), this behaves like cosine similarity.
-- Querying an empty database returns a `ResultSet` with `length === 0`.
+- Querying an empty database returns an empty array (`[]`).
 - `flush()` writes deduplicated state, and reopen preserves key-to-slot mapping.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eigen-db",
-  "version": "3.0.0",
+  "version": "4.0.0",
   "type": "module",
   "files": [
     "dist",

package/src/lib/__tests__/result-set.test.ts

@@ -1,69 +1,76 @@
-import { describe, it, expect } from "vitest";
-import { ResultSet } from "../result-set";
+import { describe, expect, it } from "vitest";
+import { iterableResults, topKResults } from "../result-set";
 
-describe("ResultSet", () => {
+describe("topKResults", () => {
   const keys = ["apple", "banana", "cherry", "date", "elderberry"];
   const resolveKey = (index: number) => keys[index];
 
   it("sorts results by descending score", () => {
     const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
-    const rs = ResultSet.fromScores(scores, resolveKey, 5);
-
-    expect(rs.length).toBe(5);
-    expect(rs.get(0).key).toBe("banana");
-    expect(rs.get(0).score).toBeCloseTo(0.9, 4);
-    expect(rs.get(1).key).toBe("date");
-    expect(rs.get(1).score).toBeCloseTo(0.7, 4);
-    expect(rs.get(2).key).toBe("elderberry");
-    expect(rs.get(3).key).toBe("apple");
-    expect(rs.get(4).key).toBe("cherry");
+    const results = topKResults(scores, resolveKey, 5);
+
+    expect(results).toHaveLength(5);
+    expect(results[0].key).toBe("banana");
+    expect(results[0].score).toBeCloseTo(0.9, 4);
+    expect(results[1].key).toBe("date");
+    expect(results[1].score).toBeCloseTo(0.7, 4);
+    expect(results[2].key).toBe("elderberry");
+    expect(results[3].key).toBe("apple");
+    expect(results[4].key).toBe("cherry");
   });
 
   it("respects topK limit", () => {
     const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
-    const rs = ResultSet.fromScores(scores, resolveKey, 3);
+    const results = topKResults(scores, resolveKey, 3);
 
-    expect(rs.length).toBe(3);
-    expect(rs.get(0).key).toBe("banana");
-    expect(rs.get(0).score).toBeCloseTo(0.9, 4);
-    expect(rs.get(2).key).toBe("elderberry");
+    expect(results).toHaveLength(3);
+    expect(results[0].key).toBe("banana");
+    expect(results[0].score).toBeCloseTo(0.9, 4);
+    expect(results[2].key).toBe("elderberry");
   });
 
-  it("throws on out-of-bounds rank", () => {
-    const scores = new Float32Array([0.5, 0.8]);
-    const rs = ResultSet.fromScores(scores, resolveKey, 2);
+  it("handles empty scores", () => {
+    const scores = new Float32Array(0);
+    const results = topKResults(scores, resolveKey, 10);
+    expect(results).toEqual([]);
+  });
 
-    expect(() => rs.get(-1)).toThrow(RangeError);
-    expect(() => rs.get(2)).toThrow(RangeError);
+  it("handles topK larger than result count", () => {
+    const scores = new Float32Array([0.5, 0.8]);
+    const results = topKResults(scores, resolveKey, 100);
+    expect(results).toHaveLength(2);
   });
+});
 
-  it("returns correct pages", () => {
-    const scores = new Float32Array([0.1, 0.2, 0.3, 0.4, 0.5]);
-    const rs = ResultSet.fromScores(scores, resolveKey, 5);
+describe("iterableResults", () => {
+  const keys = ["apple", "banana", "cherry", "date", "elderberry"];
+  const resolveKey = (index: number) => keys[index];
 
-    const page0 = rs.getPage(0, 2);
-    expect(page0).toHaveLength(2);
-    expect(page0[0].key).toBe("elderberry"); // score 0.5
-    expect(page0[1].key).toBe("date"); // score 0.4
+  it("sorts results by descending score", () => {
+    const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
+    const results = [...iterableResults(scores, resolveKey, 5)];
 
-    const page1 = rs.getPage(1, 2);
-    expect(page1).toHaveLength(2);
-    expect(page1[0].key).toBe("cherry"); // score 0.3
-    expect(page1[1].key).toBe("banana"); // score 0.2
+    expect(results).toHaveLength(5);
+    expect(results[0].key).toBe("banana");
+    expect(results[0].score).toBeCloseTo(0.9, 4);
+    expect(results[1].key).toBe("date");
+    expect(results[4].key).toBe("cherry");
+  });
 
-    const page2 = rs.getPage(2, 2);
-    expect(page2).toHaveLength(1); // only 1 remaining
-    expect(page2[0].key).toBe("apple"); // score 0.1
+  it("respects topK limit", () => {
+    const scores = new Float32Array([0.3, 0.9, 0.1, 0.7, 0.5]);
+    const results = [...iterableResults(scores, resolveKey, 3)];
+
+    expect(results).toHaveLength(3);
+    expect(results[0].key).toBe("banana");
   });
 
-  it("handles empty results", () => {
-    const scores = new Float32Array(0);
-    const rs = ResultSet.fromScores(scores, resolveKey, 10);
-    expect(rs.length).toBe(0);
-    expect(rs.getPage(0, 10)).toEqual([]);
+  it("handles empty scores", () => {
+    const results = [...iterableResults(new Float32Array(0), resolveKey, 10)];
+    expect(results).toEqual([]);
   });
 
-  it("only resolves keys lazily (on access)", () => {
+  it("only resolves keys lazily (on consumption)", () => {
     let callCount = 0;
     const lazyResolver = (index: number) => {
       callCount++;
@@ -71,20 +78,38 @@ describe("ResultSet", () => {
     };
 
     const scores = new Float32Array([0.3, 0.9, 0.1]);
-    const rs = ResultSet.fromScores(scores, lazyResolver, 3);
+    const iterable = iterableResults(scores, lazyResolver, 3);
 
     expect(callCount).toBe(0); // no key resolved yet
 
-    rs.get(0);
+    const iter = iterable[Symbol.iterator]();
+    iter.next();
     expect(callCount).toBe(1); // resolved only 1
 
-    rs.getPage(0, 2);
-    expect(callCount).toBe(3); // resolved 2 more
+    iter.next();
+    expect(callCount).toBe(2);
   });
 
-  it("handles topK larger than result count", () => {
-    const scores = new Float32Array([0.5, 0.8]);
-    const rs = ResultSet.fromScores(scores, resolveKey, 100);
-    expect(rs.length).toBe(2);
+  it("is re-iterable", () => {
+    const scores = new Float32Array([0.3, 0.9, 0.1]);
+    const iterable = iterableResults(scores, resolveKey, 3);
+
+    const first = [...iterable];
+    const second = [...iterable];
+    expect(first).toEqual(second);
+  });
+
+  it("supports partial iteration (early break)", () => {
+    const scores = new Float32Array([0.1, 0.2, 0.3, 0.4, 0.5]);
+    const iterable = iterableResults(scores, resolveKey, 5);
+
+    const partial: string[] = [];
+    for (const item of iterable) {
+      partial.push(item.key);
+      if (partial.length === 2) break;
+    }
+    expect(partial).toHaveLength(2);
+    expect(partial[0]).toBe("elderberry"); // score 0.5
+    expect(partial[1]).toBe("date"); // score 0.4
   });
 });

package/src/lib/__tests__/vector-db.test.ts

@@ -181,16 +181,16 @@ describe("VectorDB", () => {
       expect(results.length).toBe(3);
 
       // x-axis should be the best match (identical direction)
-      expect(results.get(0).key).toBe("x-axis");
-      expect(results.get(0).score).toBeCloseTo(1.0, 2);
+      expect(results[0].key).toBe("x-axis");
+      expect(results[0].score).toBeCloseTo(1.0, 2);
 
       // xy-axis should be second (partially aligned)
-      expect(results.get(1).key).toBe("xy-axis");
-      expect(results.get(1).score).toBeGreaterThan(0);
+      expect(results[1].key).toBe("xy-axis");
+      expect(results[1].score).toBeGreaterThan(0);
 
       // y-axis should be last (orthogonal)
-      expect(results.get(2).key).toBe("y-axis");
-      expect(results.get(2).score).toBeCloseTo(0.0, 2);
+      expect(results[2].key).toBe("y-axis");
+      expect(results[2].score).toBeCloseTo(0.0, 2);
     });
 
     it("query respects topK option", async () => {
@@ -208,7 +208,7 @@ describe("VectorDB", () => {
       expect(results.length).toBe(2);
     });
 
-    it("query on empty database returns empty result", async () => {
+    it("query on empty database returns empty array", async () => {
       const db = await VectorDB.open({
         dimensions: 4,
         storage,
@@ -216,7 +216,7 @@ describe("VectorDB", () => {
       });
 
       const results = db.query([1, 0, 0, 0]);
-      expect(results.length).toBe(0);
+      expect(results).toEqual([]);
     });
 
     it("query validates vector dimensions", async () => {
@@ -230,7 +230,7 @@ describe("VectorDB", () => {
       expect(() => db.query([1, 2, 3, 4, 5, 6, 7, 8])).toThrow("dimension mismatch");
     });
 
-    it("query results support pagination", async () => {
+    it("query results support iterable mode for pagination", async () => {
       const db = await VectorDB.open({
         dimensions: 4,
         normalize: false,
@@ -244,16 +244,19 @@ describe("VectorDB", () => {
         db.set(`t${i}`, vec);
       }
 
-      const results = db.query([1, 0, 0, 0], { normalize: false });
+      const results = db.query([1, 0, 0, 0], { normalize: false, iterable: true });
 
-      const page0 = results.getPage(0, 2);
-      expect(page0).toHaveLength(2);
+      // Spread into array
+      const all = [...results];
+      expect(all).toHaveLength(5);
 
-      const page1 = results.getPage(1, 2);
-      expect(page1).toHaveLength(2);
-
-      const page2 = results.getPage(2, 2);
-      expect(page2).toHaveLength(1);
+      // Partial iteration (simulate pagination)
+      const page: { key: string; score: number }[] = [];
+      for (const item of results) {
+        page.push(item);
+        if (page.length === 2) break;
+      }
+      expect(page).toHaveLength(2);
     });
 
     it("query after overwrite uses updated vector", async () => {
@@ -271,8 +274,8 @@ describe("VectorDB", () => {
 
       const results = db.query([0, 1, 0, 0]);
       // Both 'point' and 'other' are now along y-axis, so both should score high
-      expect(results.get(0).score).toBeCloseTo(1.0, 2);
-      expect(results.get(1).score).toBeCloseTo(1.0, 2);
+      expect(results[0].score).toBeCloseTo(1.0, 2);
+      expect(results[1].score).toBeCloseTo(1.0, 2);
       expect(db.size).toBe(2);
     });
 

package/src/lib/index.ts

@@ -1,14 +1,20 @@
 /**
- * web-vector-base: In-Browser Vector Compute Engine
+ * eigen-db: High-Performance In-Browser Vector Database
  *
- * High-performance vector search entirely on the client side,
+ * Stores and queries embedding vectors entirely on the client side,
  * utilizing OPFS for persistent storage and WASM SIMD for computation.
  */
 
 export { VectorCapacityExceededError } from "./errors";
-export { ResultSet } from "./result-set";
 export type { ResultItem } from "./result-set";
 export { InMemoryStorageProvider, OPFSStorageProvider } from "./storage";
 export type { StorageProvider } from "./storage";
-export type { OpenOptions, OpenOptionsInternal, QueryOptions, SetOptions, VectorInput } from "./types";
+export type {
+  IterableQueryOptions,
+  OpenOptions,
+  OpenOptionsInternal,
+  QueryOptions,
+  SetOptions,
+  VectorInput,
+} from "./types";
 export { VectorDB as DB } from "./vector-db";

package/src/lib/result-set.ts

@@ -1,8 +1,11 @@
 /**
- * LAZY RESULT SET
+ * RESULT HELPERS
  *
- * Holds pointers to sorted TypedArrays. Prevents JS heap overflow when K is massive.
- * Strings are only instantiated from the Lexicon when explicitly requested.
+ * Utility functions for sorting scores and producing query results.
+ * Two modes:
+ *   1. topKResults  — eagerly materializes a ResultItem[] (default query path)
+ *   2. iterableResults — returns a lazy Iterable<ResultItem> where keys are
+ *      resolved only as each item is consumed (for pagination / streaming)
  */
 
 export interface ResultItem {
@@ -12,78 +15,58 @@ export interface ResultItem {
 
 export type KeyResolver = (index: number) => string;
 
-export class ResultSet {
-  /** Total number of results */
-  readonly length: number;
-
-  /**
-   * Sorted indices into the original database (by descending score).
-   * sortedIndices[0] is the index of the best match.
-   */
-  private readonly sortedIndices: Uint32Array;
-
-  /** Raw scores array (not sorted, indexed by original DB position) */
-  private readonly scores: Float32Array;
+/**
+ * Sort scores descending and return the top K results as a plain array.
+ * All keys are resolved eagerly.
+ */
+export function topKResults(scores: Float32Array, resolveKey: KeyResolver, topK: number): ResultItem[] {
+  const n = scores.length;
+  if (n === 0) return [];
 
-  /** Function to lazily resolve key from the slot index */
-  private readonly resolveKey: KeyResolver;
+  const indices = new Uint32Array(n);
+  for (let i = 0; i < n; i++) indices[i] = i;
+  indices.sort((a, b) => scores[b] - scores[a]);
 
-  constructor(
-    scores: Float32Array,
-    sortedIndices: Uint32Array,
-    resolveKey: KeyResolver,
-    topK: number,
-  ) {
-    this.scores = scores;
-    this.sortedIndices = sortedIndices;
-    this.resolveKey = resolveKey;
-    this.length = Math.min(topK, sortedIndices.length);
+  const k = Math.min(topK, n);
+  const results: ResultItem[] = new Array(k);
+  for (let i = 0; i < k; i++) {
+    const idx = indices[i];
+    results[i] = { key: resolveKey(idx), score: scores[idx] };
   }
+  return results;
+}
 
-  /**
-   * Sort scores and return a ResultSet with lazy key resolution.
-   *
-   * @param scores - Float32Array of scores (one per DB vector)
-   * @param resolveKey - Function to resolve key by original index
-   * @param topK - Maximum number of results to include
-   */
-  static fromScores(
-    scores: Float32Array,
-    resolveKey: KeyResolver,
-    topK: number,
-  ): ResultSet {
-    const n = scores.length;
-
-    // Create index array for sorting
-    const indices = new Uint32Array(n);
-    for (let i = 0; i < n; i++) indices[i] = i;
-
-    // Sort indices by descending score
-    indices.sort((a, b) => scores[b] - scores[a]);
+/**
+ * Sort scores descending and return a lazy iterable over the top K results.
+ * Keys are resolved only when each item is consumed, saving allocations
+ * when the caller iterates partially (e.g., pagination).
+ *
+ * The returned iterable is re-iterable — each call to [Symbol.iterator]()
+ * produces a fresh cursor over the same pre-sorted data.
+ */
+export function iterableResults(scores: Float32Array, resolveKey: KeyResolver, topK: number): Iterable<ResultItem> {
+  const n = scores.length;
+  if (n === 0) return [];
 
-    return new ResultSet(scores, indices, resolveKey, topK);
-  }
+  const indices = new Uint32Array(n);
+  for (let i = 0; i < n; i++) indices[i] = i;
+  indices.sort((a, b) => scores[b] - scores[a]);
 
-  /** Fetch a single result by its rank (0 is best match) */
-  get(rank: number): ResultItem {
-    if (rank < 0 || rank >= this.length) {
-      throw new RangeError(`Rank ${rank} out of bounds [0, ${this.length})`);
-    }
-    const dbIndex = this.sortedIndices[rank];
-    return {
-      key: this.resolveKey(dbIndex),
-      score: this.scores[dbIndex],
-    };
-  }
+  const k = Math.min(topK, n);
 
-  /** Helper for UI pagination. Instantiates strings only for the requested page. */
-  getPage(page: number, pageSize: number): ResultItem[] {
-    const start = page * pageSize;
-    const end = Math.min(start + pageSize, this.length);
-    const results: ResultItem[] = [];
-    for (let i = start; i < end; i++) {
-      results.push(this.get(i));
-    }
-    return results;
-  }
+  return {
+    [Symbol.iterator](): Iterator<ResultItem> {
+      let i = 0;
+      return {
+        next(): IteratorResult<ResultItem> {
+          if (i >= k) return { done: true, value: undefined };
+          const idx = indices[i++];
+          return {
+            done: false,
+            value: { key: resolveKey(idx), score: scores[idx] },
+          };
+        },
+      };
+    },
+  };
 }

package/src/lib/types.ts

@@ -36,6 +36,7 @@ export interface SetOptions {
 
 /**
  * Options for query operations.
+ * Returns a plain ResultItem[] array by default.
  */
 export interface QueryOptions {
   /** Maximum number of results to return. Defaults to all. */
@@ -43,3 +44,13 @@ export interface QueryOptions {
   /** Override normalization for this call. */
   normalize?: boolean;
 }
+
+/**
+ * Query options requesting a lazy iterable instead of a materialized array.
+ * Keys are resolved only as each item is consumed, saving allocations
+ * when the caller doesn't need the full result set.
+ */
+export interface IterableQueryOptions extends QueryOptions {
+  /** When true, returns an Iterable<ResultItem> instead of ResultItem[]. */
+  iterable: true;
+}

package/src/lib/vector-db.ts

@@ -15,11 +15,19 @@ import { normalize, searchAll } from "./compute";
 import { VectorCapacityExceededError } from "./errors";
 import { decodeLexicon, encodeLexicon } from "./lexicon";
 import { MemoryManager } from "./memory-manager";
-import { ResultSet } from "./result-set";
+import type { ResultItem } from "./result-set";
+import { iterableResults, topKResults } from "./result-set";
 import { getSimdWasmBinary } from "./simd-binary";
 import type { StorageProvider } from "./storage";
 import { OPFSStorageProvider } from "./storage";
-import type { OpenOptions, OpenOptionsInternal, QueryOptions, SetOptions, VectorInput } from "./types";
+import type {
+  IterableQueryOptions,
+  OpenOptions,
+  OpenOptionsInternal,
+  QueryOptions,
+  SetOptions,
+  VectorInput,
+} from "./types";
 import { instantiateWasm, type WasmExports } from "./wasm-compute";
 
 const VECTORS_FILE = "vectors.bin";
@@ -183,15 +191,21 @@ export class VectorDB {
 
   /**
    * Search for the most similar vectors to the given query vector.
-   * Returns a ResultSet sorted by descending similarity score.
+   *
+   * Default: returns a plain ResultItem[] sorted by descending similarity.
+   * With `{ iterable: true }`: returns a lazy Iterable<ResultItem> where keys
+   * are resolved only as each item is consumed.
    */
-  query(value: VectorInput, options?: QueryOptions): ResultSet {
+  query(value: VectorInput, options: IterableQueryOptions): Iterable<ResultItem>;
+  query(value: VectorInput, options?: QueryOptions): ResultItem[];
+  query(value: VectorInput, options?: QueryOptions | IterableQueryOptions): ResultItem[] | Iterable<ResultItem> {
     this.assertOpen();
 
     const k = options?.topK ?? this.size;
+    const iterable = options && "iterable" in options && options.iterable;
 
     if (this.size === 0) {
-      return ResultSet.fromScores(new Float32Array(0), () => "", 0);
+      return [];
     }
 
     if (value.length !== this.dimensions) {
@@ -248,7 +262,10 @@ export class VectorDB {
       return slotToKey[slotIndex];
     };
 
-    return ResultSet.fromScores(scores, resolveKey, k);
+    if (iterable) {
+      return iterableResults(scores, resolveKey, k);
+    }
+    return topKResults(scores, resolveKey, k);
   }
 
   /**