@aztec/kv-store 0.0.1-commit.c31f2472 → 0.0.1-commit.c52d6e7

package/src/bench/shared_map_bench.ts

@@ -0,0 +1,111 @@
+import type { Logger } from '@aztec/foundation/log';
+import { Timer } from '@aztec/foundation/timer';
+
+import type { Key } from '../interfaces/common.js';
+import type { AztecAsyncMap } from '../interfaces/map.js';
+import type { AztecAsyncKVStore } from '../interfaces/store.js';
+
+/** One benchmark measurement. */
+export type BenchResult = {
+  /** Benchmark name (includes the backend prefix for disambiguation). */
+  name: string;
+  value: number;
+  unit: 'ms' | 'us';
+};
+
+export type BenchReporter = (results: BenchResult[]) => void | Promise<void>;
+
+/**
+ * Runs the standard Map benchmark suite against any `AztecAsyncKVStore` backend,
+ * populates `results`, and calls `reporter` in `afterAll`.
+ *
+ * Kept free of Node-only deps (`fs`, `path`) so the same runner works under
+ * vitest-browser for IndexedDB and SQLite-OPFS.
+ */
+export function describeAztecMapBench(
+  backendPrefix: string,
+  getStore: () => Promise<AztecAsyncKVStore>,
+  logger: Logger,
+  reporter: BenchReporter,
+) {
+  describe(`${backendPrefix} Map benchmarks`, () => {
+    let store: AztecAsyncKVStore;
+    let map: AztecAsyncMap<Key, string>;
+
+    const results: BenchResult[] = [];
+
+    const generateKeyValuePairs = (count: number, offset = 0) => {
+      const keys = Array.from({ length: count }, (_, i) => `key-${i + offset}`);
+      const values = Array.from({ length: count }, (_, i) => `value-${i + offset}`);
+      return keys.map((key, i) => ({ key, value: values[i] }));
+    };
+
+    const record = (name: string, value: number, unit: BenchResult['unit']) => {
+      results.push({ name: `${backendPrefix}/Map/${name}`, value, unit });
+    };
+
+    beforeEach(async () => {
+      store = await getStore();
+      map = store.openMap<Key, string>('test');
+    });
+
+    afterEach(async () => {
+      await store.delete();
+    });
+
+    afterAll(async () => {
+      const pretty = results.map(r => `${r.name}: ${r.value.toFixed(2)} ${r.unit}`).join('\n');
+      logger.info(`\n${pretty}\n`);
+      await reporter(results);
+    });
+
+    it('adds individual values', async () => {
+      const pairs = generateKeyValuePairs(1000);
+      const timer = new Timer();
+      for (const pair of pairs) {
+        await map.set(pair.key, pair.value);
+      }
+      record('Individual insertion', timer.ms() / pairs.length, 'ms');
+    });
+
+    it('adds batched values', async () => {
+      const batches = Array.from({ length: 100 }, (_, i) => generateKeyValuePairs(1000, i * 1000));
+      const timer = new Timer();
+      for (const batch of batches) {
+        await map.setMany(batch);
+      }
+      record(`Batch insertion of ${batches[0].length} items`, timer.ms() / batches.length, 'ms');
+    });
+
+    it('reads individual values', async () => {
+      const pairs = generateKeyValuePairs(10000);
+      await map.setMany(pairs);
+      const timer = new Timer();
+      for (const pair of pairs) {
+        await map.getAsync(pair.key);
+      }
+      record('Individual read', (timer.ms() * 1000) / pairs.length, 'us');
+    });
+
+    it('reads via a cursor', async () => {
+      const pairs = generateKeyValuePairs(10000);
+      await map.setMany(pairs);
+      const timer = new Timer();
+      for await (const _ of map.entriesAsync()) {
+        // consume
+      }
+      record(`Iterator per item read of ${pairs.length} items`, (timer.ms() * 1000) / pairs.length, 'us');
+    });
+
+    it('reads the size of the map', async () => {
+      const numIterations = 1000;
+      const pairs = generateKeyValuePairs(10000);
+      await map.setMany(pairs);
+      const timer = new Timer();
+      for (let i = 0; i < numIterations; i++) {
+        await map.sizeAsync();
+      }
+      record(`Read size of ${pairs.length} items`, (timer.ms() * 1000) / numIterations, 'us');
+    });
+  });
+}

package/src/{indexeddb → deprecated/indexeddb}/array.ts

@@ -1,8 +1,8 @@
 import type { IDBPDatabase, IDBPObjectStore } from 'idb';
 import { hash } from 'ohash';
 
-import type { AztecAsyncArray } from '../interfaces/array.js';
-import type { Value } from '../interfaces/common.js';
+import type { AztecAsyncArray } from '../../interfaces/array.js';
+import type { Value } from '../../interfaces/common.js';
 import type { AztecIDBSchema } from './store.js';
 
 /**
@@ -32,7 +32,7 @@ export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T>
     return (
       (await this.db
         .index('key')
-        .count(IDBKeyRange.bound([this.#container, this.#name], [this.#container, this.#name]))) ?? 0
+        .count(IDBKeyRange.bound([this.#container, [this.#name]], [this.#container, [this.#name]]))) ?? 0
     );
   }
 
@@ -43,7 +43,7 @@ export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T>
         value: val,
         hash: hash(val),
         container: this.#container,
-        key: this.#name,
+        key: [this.#name],
         keyCount: length + 1,
         slot: this.#slot(length),
       });
@@ -91,7 +91,7 @@ export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T>
       value: val,
       hash: hash(val),
       container: this.#container,
-      key: this.#name,
+      key: [this.#name],
       keyCount: index + 1,
       slot: this.#slot(index),
     });
@@ -100,7 +100,7 @@ export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T>
 
   async *entriesAsync(): AsyncIterableIterator<[number, T]> {
     const index = this.db.index('key');
-    const rangeQuery = IDBKeyRange.bound([this.#container, this.#name], [this.#container, this.#name]);
+    const rangeQuery = IDBKeyRange.bound([this.#container, [this.#name]], [this.#container, [this.#name]]);
     for await (const cursor of index.iterate(rangeQuery)) {
       yield [cursor.value.keyCount - 1, cursor.value.value] as [number, T];
     }

package/src/deprecated/indexeddb/index.ts

@@ -0,0 +1,12 @@
+import { createLogger } from '@aztec/foundation/log';
+
+import { AztecIndexedDBStore } from './store.js';
+
+export { AztecIndexedDBStore } from './store.js';
+
+/**
+ * @deprecated The IndexedDB backend is being retired. Use `@aztec/kv-store/sqlite-opfs` instead.
+ */
+export function openTmpStore(ephemeral: boolean = false): Promise<AztecIndexedDBStore> {
+  return AztecIndexedDBStore.open(createLogger('kv-store:indexeddb'), undefined, ephemeral);
+}

package/src/{indexeddb → deprecated/indexeddb}/map.ts

@@ -1,8 +1,8 @@
 import type { IDBPDatabase, IDBPObjectStore } from 'idb';
 import { hash } from 'ohash';
 
-import type { Key, Range, Value } from '../interfaces/common.js';
-import type { AztecAsyncMap } from '../interfaces/map.js';
+import type { Key, Range, Value } from '../../interfaces/common.js';
+import type { AztecAsyncMap } from '../../interfaces/map.js';
 import type { AztecIDBSchema } from './store.js';
 
 /**
@@ -31,7 +31,7 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
 
   async getAsync(key: K): Promise<V | undefined> {
     const data = await this.db.get(this.slot(key));
-    return data?.value as V;
+    return data ? this.restoreBuffers(data.value as V) : undefined;
   }
 
   async hasAsync(key: K): Promise<boolean> {
@@ -41,7 +41,7 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
 
   async sizeAsync(): Promise<number> {
     const index = this.db.index('key');
-    const rangeQuery = IDBKeyRange.bound([this.container, ''], [this.container, '\uffff']);
+    const rangeQuery = IDBKeyRange.bound([this.container], [this.container + '\uffff'], true, true);
     return await index.count(rangeQuery);
   }
 
@@ -80,9 +80,12 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
 
   async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, V]> {
     const index = this.db.index('key');
+    const startKey = range.start ? this.normalizeKey(range.start) : [];
+    const endKey = range.end ? this.normalizeKey(range.end) : ['\uffff'];
+
     const rangeQuery = IDBKeyRange.bound(
-      [this.container, range.start ? this.normalizeKey(range.start) : ''],
-      [this.container, range.end ? this.normalizeKey(range.end) : '\uffff'],
+      [this.container, startKey],
+      [this.container, endKey],
       !!range.reverse,
       !range.reverse,
     );
@@ -91,7 +94,7 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
       if (range.limit && count >= range.limit) {
         return;
       }
-      yield [this.#denormalizeKey(cursor.value.key), cursor.value.value] as [K, V];
+      yield [this.#denormalizeKey(cursor.value.key), this.restoreBuffers(cursor.value.value as V)] as [K, V];
       count++;
     }
   }
@@ -108,14 +111,24 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
     }
   }
 
-  #denormalizeKey(key: string): K {
-    const denormalizedKey = key.split(',').map(part => (part.startsWith('n_') ? Number(part.slice(2)) : part));
-    return (denormalizedKey.length > 1 ? denormalizedKey : denormalizedKey[0]) as K;
+  /**
+   * IndexedDB's structured clone downcasts Buffer (a Node.js subclass of Uint8Array) to plain Uint8Array.
+   * This breaks .toString() behavior: Buffer.toString() returns UTF-8, Uint8Array.toString() returns
+   * comma-separated decimal bytes. We restore Buffer identity on read so callers get consistent behavior.
+   */
+  protected restoreBuffers(val: V): V {
+    if (val instanceof Uint8Array && !Buffer.isBuffer(val)) {
+      return Buffer.from(val) as V;
+    }
+    return val;
+  }
+
+  #denormalizeKey(key: (string | number | Uint8Array)[]): K {
+    return (key.length > 1 ? key : key[0]) as K;
   }
 
-  protected normalizeKey(key: K): string {
-    const arrayKey = Array.isArray(key) ? key : [key];
-    return (arrayKey as K[]).map((element: K) => (typeof element === 'number' ? `n_${element}` : element)).join(',');
+  protected normalizeKey(key: K): (string | number | Uint8Array)[] {
+    return Array.isArray(key) ? key : [key];
   }
 
   protected slot(key: K, index: number = 0): string {

package/src/{indexeddb → deprecated/indexeddb}/multi_map.ts

@@ -1,7 +1,7 @@
 import { hash } from 'ohash';
 
-import type { Key, Value } from '../interfaces/common.js';
-import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
+import type { Key, Value } from '../../interfaces/common.js';
+import type { AztecAsyncMultiMap } from '../../interfaces/multi_map.js';
 import { IndexedDBAztecMap } from './map.js';
 
 /**
@@ -57,7 +57,7 @@ export class IndexedDBAztecMultiMap<K extends Key, V extends Value>
       false,
     );
     for await (const cursor of index.iterate(rangeQuery)) {
-      yield cursor.value.value as V;
+      yield this.restoreBuffers(cursor.value.value as V);
     }
   }
 
@@ -88,4 +88,18 @@ export class IndexedDBAztecMultiMap<K extends Key, V extends Value>
       await this.db.delete(fullKey);
     }
   }
+
+  override async delete(key: K): Promise<void> {
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.container, this.normalizeKey(key), 0],
+      [this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
+      false,
+      false,
+    );
+
+    for await (const cursor of index.iterate(rangeQuery)) {
+      await cursor.delete();
+    }
+  }
 }

package/src/{indexeddb → deprecated/indexeddb}/set.ts

@@ -1,7 +1,7 @@
 import type { IDBPDatabase, IDBPObjectStore } from 'idb';
 
-import type { Key, Range } from '../interfaces/common.js';
-import type { AztecAsyncSet } from '../interfaces/set.js';
+import type { Key, Range } from '../../interfaces/common.js';
+import type { AztecAsyncSet } from '../../interfaces/set.js';
 import { IndexedDBAztecMap } from './map.js';
 import type { AztecIDBSchema } from './store.js';
 

package/src/{indexeddb → deprecated/indexeddb}/singleton.ts

@@ -1,8 +1,8 @@
 import type { IDBPDatabase, IDBPObjectStore } from 'idb';
 import { hash } from 'ohash';
 
-import type { Value } from '../interfaces/common.js';
-import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
+import type { Value } from '../../interfaces/common.js';
+import type { AztecAsyncSingleton } from '../../interfaces/singleton.js';
 import type { AztecIDBSchema } from './store.js';
 
 /**
@@ -37,7 +37,7 @@ export class IndexedDBAztecSingleton<T extends Value> implements AztecAsyncSingl
     const result = await this.db.put({
       container: this.#container,
       slot: this.#slot,
-      key: this.#slot,
+      key: [this.#slot],
       keyCount: 1,
       value: val,
       hash: hash(val),

package/src/{indexeddb → deprecated/indexeddb}/store.ts

@@ -3,14 +3,14 @@ import { SerialQueue } from '@aztec/foundation/queue';
 
 import { type DBSchema, type IDBPDatabase, deleteDB, openDB } from 'idb';
 
-import type { AztecAsyncArray } from '../interfaces/array.js';
-import type { Key, StoreSize, Value } from '../interfaces/common.js';
-import type { AztecAsyncCounter } from '../interfaces/counter.js';
-import type { AztecAsyncMap } from '../interfaces/map.js';
-import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
-import type { AztecAsyncSet } from '../interfaces/set.js';
-import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
-import type { AztecAsyncKVStore } from '../interfaces/store.js';
+import type { AztecAsyncArray } from '../../interfaces/array.js';
+import type { Key, StoreSize, Value } from '../../interfaces/common.js';
+import type { AztecAsyncCounter } from '../../interfaces/counter.js';
+import type { AztecAsyncMap } from '../../interfaces/map.js';
+import type { AztecAsyncMultiMap } from '../../interfaces/multi_map.js';
+import type { AztecAsyncSet } from '../../interfaces/set.js';
+import type { AztecAsyncSingleton } from '../../interfaces/singleton.js';
+import type { AztecAsyncKVStore } from '../../interfaces/store.js';
 import { IndexedDBAztecArray } from './array.js';
 import { IndexedDBAztecMap } from './map.js';
 import { IndexedDBAztecMultiMap } from './multi_map.js';
@@ -20,7 +20,7 @@ import { IndexedDBAztecSingleton } from './singleton.js';
 export type StoredData<V extends Value> = {
   value: V;
   container: string;
-  key: string;
+  key: (string | number | Uint8Array)[];
   keyCount: number;
   slot: string;
   hash: string;
@@ -30,14 +30,20 @@ export interface AztecIDBSchema extends DBSchema {
   data: {
     value: StoredData<any>;
     key: string;
-    indexes: { container: string; key: string; keyCount: number; hash: string };
+    indexes: {
+      container: string;
+      key: any; // Runtime: (string | number | Uint8Array)[] - idb types don't support arrays
+      keyCount: number;
+      hash: string;
+    };
   };
 }
 
 /**
  * A key-value store backed by IndexedDB.
+ *
+ * @deprecated The IndexedDB backend is being retired. Use `@aztec/kv-store/sqlite-opfs` instead.
  */
-
 export class AztecIndexedDBStore implements AztecAsyncKVStore {
   #rootDB: IDBPDatabase<AztecIDBSchema>;
   #name: string;
@@ -187,18 +193,20 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
   }
 
   /** Deletes this store and removes the database */
-  delete() {
+  async delete() {
     this.#containers.clear();
+    await this.#txQueue.end();
     this.#rootDB.close();
-    return deleteDB(this.#name);
+    await deleteDB(this.#name);
   }
 
   estimateSize(): Promise<StoreSize> {
     return Promise.resolve({ mappingSize: 0, physicalFileSize: 0, actualSize: 0, numItems: 0 });
   }
 
-  close(): Promise<void> {
-    return Promise.resolve();
+  async close(): Promise<void> {
+    await this.#txQueue.end();
+    this.#rootDB.close();
   }
 
   backupTo(_dstPath: string, _compact?: boolean): Promise<void> {

package/src/interfaces/array_test_suite.ts

@@ -1,7 +1,5 @@
 import { toArray } from '@aztec/foundation/iterable';
 
-import { expect } from 'chai';
-
 import type { AztecArray, AztecAsyncArray } from './array.js';
 import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
 import { isSyncStore } from './utils.js';
@@ -53,12 +51,12 @@ export function describeAztecArray(
       await arr.push(2);
       await arr.push(3);
 
-      expect(await length()).to.equal(3);
+      expect(await length()).toBe(3);
 
-      expect(await arr.pop()).to.equal(3);
-      expect(await arr.pop()).to.equal(2);
-      expect(await arr.pop()).to.equal(1);
-      expect(await arr.pop()).to.equal(undefined);
+      expect(await arr.pop()).toBe(3);
+      expect(await arr.pop()).toBe(2);
+      expect(await arr.pop()).toBe(1);
+      expect(await arr.pop()).toBe(undefined);
     });
 
     it('should be able to get values by index', async () => {
@@ -66,14 +64,14 @@ export function describeAztecArray(
       await arr.push(2);
       await arr.push(3);
 
-      expect(await at(0)).to.equal(1);
-      expect(await at(1)).to.equal(2);
-      expect(await at(2)).to.equal(3);
-      expect(await at(3)).to.equal(undefined);
-      expect(await at(-1)).to.equal(3);
-      expect(await at(-2)).to.equal(2);
-      expect(await at(-3)).to.equal(1);
-      expect(await at(-4)).to.equal(undefined);
+      expect(await at(0)).toBe(1);
+      expect(await at(1)).toBe(2);
+      expect(await at(2)).toBe(3);
+      expect(await at(3)).toBe(undefined);
+      expect(await at(-1)).toBe(3);
+      expect(await at(-2)).toBe(2);
+      expect(await at(-3)).toBe(1);
+      expect(await at(-4)).toBe(undefined);
     });
 
     it('should be able to set values by index', async () => {
@@ -81,27 +79,27 @@ export function describeAztecArray(
       await arr.push(2);
       await arr.push(3);
 
-      expect(await arr.setAt(0, 4)).to.equal(true);
-      expect(await arr.setAt(1, 5)).to.equal(true);
-      expect(await arr.setAt(2, 6)).to.equal(true);
+      expect(await arr.setAt(0, 4)).toBe(true);
+      expect(await arr.setAt(1, 5)).toBe(true);
+      expect(await arr.setAt(2, 6)).toBe(true);
 
-      expect(await arr.setAt(3, 7)).to.equal(false);
+      expect(await arr.setAt(3, 7)).toBe(false);
 
-      expect(await at(0)).to.equal(4);
-      expect(await at(1)).to.equal(5);
-      expect(await at(2)).to.equal(6);
-      expect(await at(3)).to.equal(undefined);
+      expect(await at(0)).toBe(4);
+      expect(await at(1)).toBe(5);
+      expect(await at(2)).toBe(6);
+      expect(await at(3)).toBe(undefined);
 
-      expect(await arr.setAt(-1, 8)).to.equal(true);
-      expect(await arr.setAt(-2, 9)).to.equal(true);
-      expect(await arr.setAt(-3, 10)).to.equal(true);
+      expect(await arr.setAt(-1, 8)).toBe(true);
+      expect(await arr.setAt(-2, 9)).toBe(true);
+      expect(await arr.setAt(-3, 10)).toBe(true);
 
-      expect(await arr.setAt(-4, 11)).to.equal(false);
+      expect(await arr.setAt(-4, 11)).toBe(false);
 
-      expect(await at(-1)).to.equal(8);
-      expect(await at(-2)).to.equal(9);
-      expect(await at(-3)).to.equal(10);
-      expect(await at(-4)).to.equal(undefined);
+      expect(await at(-1)).toBe(8);
+      expect(await at(-2)).toBe(9);
+      expect(await at(-3)).toBe(10);
+      expect(await at(-4)).toBe(undefined);
     });
 
     it('should be able to iterate over values', async () => {
@@ -109,8 +107,8 @@ export function describeAztecArray(
       await arr.push(2);
       await arr.push(3);
 
-      expect(await values()).to.deep.equal([1, 2, 3]);
-      expect(await entries()).to.deep.equal([
+      expect(await values()).toEqual([1, 2, 3]);
+      expect(await entries()).toEqual([
         [0, 1],
         [1, 2],
         [2, 3],
@@ -123,8 +121,8 @@ export function describeAztecArray(
       await arr.push(3);
 
       const arr2 = store.openArray<number>('test');
-      expect(await length(arr2)).to.equal(3);
-      expect(await values(arr2)).to.deep.equal(await values());
+      expect(await length(arr2)).toBe(3);
+      expect(await values(arr2)).toEqual(await values());
     });
   });
 }

package/src/interfaces/index.ts

@@ -6,4 +6,4 @@ export * from './store.js';
 export * from './set.js';
 export * from './multi_map.js';
 export { mapRange } from './common.js';
-export type { CustomRange, Range, StoreSize } from './common.js';
+export type { CustomRange, Key, Range, StoreSize } from './common.js';