@aztec/kv-store 0.0.1-commit.fffb133c → 0.0.1-dev

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aztec/kv-store",
-  "version": "0.0.1-commit.fffb133c",
+  "version": "0.0.1-dev",
   "type": "module",
   "exports": {
     ".": "./dest/interfaces/index.js",
@@ -8,16 +8,28 @@
     "./lmdb": "./dest/lmdb/index.js",
     "./lmdb-v2": "./dest/lmdb-v2/index.js",
     "./indexeddb": "./dest/indexeddb/index.js",
+    "./sqlite-opfs": "./dest/sqlite-opfs/index.js",
     "./stores": "./dest/stores/index.js",
     "./config": "./dest/config.js"
   },
+  "imports": {
+    "#msgpackr": {
+      "browser": "msgpackr/index-no-eval",
+      "default": "msgpackr"
+    },
+    "#ordered-binary": {
+      "browser": "./dest/sqlite-opfs/internal/ordered-binary-browser.js",
+      "default": "ordered-binary"
+    }
+  },
   "scripts": {
     "build": "yarn clean && ../scripts/tsc.sh",
     "build:dev": "../scripts/tsc.sh --watch",
     "clean": "rm -rf ./dest .tsbuildinfo",
-    "test:node": "NODE_NO_WARNINGS=1 mocha --config ./.mocharc.json",
-    "test:browser": "wtr --config ./web-test-runner.config.mjs",
-    "test": "yarn test:node",
+    "test:node": "vitest run --project node",
+    "test:browser": "bash scripts/run-browser-tests.sh",
+    "bench:browser": "VITE_BENCH=1 vitest run --project browser src/bench",
+    "test": "vitest run",
     "test:jest": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
   },
   "inherits": [
@@ -25,11 +37,12 @@
     "./package.local.json"
   ],
   "dependencies": {
-    "@aztec/constants": "0.0.1-commit.fffb133c",
-    "@aztec/ethereum": "0.0.1-commit.fffb133c",
-    "@aztec/foundation": "0.0.1-commit.fffb133c",
-    "@aztec/native": "0.0.1-commit.fffb133c",
-    "@aztec/stdlib": "0.0.1-commit.fffb133c",
+    "@aztec/constants": "0.0.1-dev",
+    "@aztec/ethereum": "0.0.1-dev",
+    "@aztec/foundation": "0.0.1-dev",
+    "@aztec/native": "0.0.1-dev",
+    "@aztec/sqlite3mc-wasm": "0.0.1-dev",
+    "@aztec/stdlib": "0.0.1-dev",
     "idb": "^8.0.0",
     "lmdb": "^3.2.0",
     "msgpackr": "^1.11.2",
@@ -38,25 +51,17 @@
   },
   "devDependencies": {
     "@jest/globals": "^30.0.0",
-    "@types/chai": "^5.0.1",
-    "@types/chai-as-promised": "^8.0.1",
     "@types/jest": "^30.0.0",
-    "@types/mocha": "^10.0.10",
-    "@types/mocha-each": "^2.0.4",
     "@types/node": "^22.15.17",
-    "@types/sinon": "^17.0.3",
     "@typescript/native-preview": "7.0.0-dev.20260113.1",
-    "@web/dev-server-esbuild": "^1.0.3",
-    "@web/test-runner": "^0.19.0",
-    "@web/test-runner-playwright": "^0.11.0",
-    "chai": "^5.1.2",
-    "chai-as-promised": "^8.0.1",
+    "@vitest/browser-playwright": "^4.0.0",
+    "buffer": "^6.0.3",
     "jest": "^30.0.0",
-    "mocha": "^10.8.2",
-    "mocha-each": "^2.0.1",
-    "sinon": "^19.0.2",
+    "playwright": "1.49.0",
     "ts-node": "^10.9.1",
-    "typescript": "^5.3.3"
+    "typescript": "^5.3.3",
+    "util": "^0.12.5",
+    "vitest": "^4.0.0"
   },
   "files": [
     "dest",

package/src/bench/shared_map_bench.ts

@@ -0,0 +1,111 @@
+import type { Logger } from '@aztec/foundation/log';
+import { Timer } from '@aztec/foundation/timer';
+
+import type { Key } from '../interfaces/common.js';
+import type { AztecAsyncMap } from '../interfaces/map.js';
+import type { AztecAsyncKVStore } from '../interfaces/store.js';
+
+/** One benchmark measurement. */
+export type BenchResult = {
+  /** Benchmark name (includes the backend prefix for disambiguation). */
+  name: string;
+  value: number;
+  unit: 'ms' | 'us';
+};
+
+export type BenchReporter = (results: BenchResult[]) => void | Promise<void>;
+
+/**
+ * Runs the standard Map benchmark suite against any `AztecAsyncKVStore` backend,
+ * populates `results`, and calls `reporter` in `afterAll`.
+ *
+ * Kept free of Node-only deps (`fs`, `path`) so the same runner works under
+ * vitest-browser for IndexedDB and SQLite-OPFS.
+ */
+export function describeAztecMapBench(
+  backendPrefix: string,
+  getStore: () => Promise<AztecAsyncKVStore>,
+  logger: Logger,
+  reporter: BenchReporter,
+) {
+  describe(`${backendPrefix} Map benchmarks`, () => {
+    let store: AztecAsyncKVStore;
+    let map: AztecAsyncMap<Key, string>;
+
+    const results: BenchResult[] = [];
+
+    const generateKeyValuePairs = (count: number, offset = 0) => {
+      const keys = Array.from({ length: count }, (_, i) => `key-${i + offset}`);
+      const values = Array.from({ length: count }, (_, i) => `value-${i + offset}`);
+      return keys.map((key, i) => ({ key, value: values[i] }));
+    };
+
+    const record = (name: string, value: number, unit: BenchResult['unit']) => {
+      results.push({ name: `${backendPrefix}/Map/${name}`, value, unit });
+    };
+
+    beforeEach(async () => {
+      store = await getStore();
+      map = store.openMap<Key, string>('test');
+    });
+
+    afterEach(async () => {
+      await store.delete();
+    });
+
+    afterAll(async () => {
+      const pretty = results.map(r => `${r.name}: ${r.value.toFixed(2)} ${r.unit}`).join('\n');
+      logger.info(`\n${pretty}\n`);
+      await reporter(results);
+    });
+
+    it('adds individual values', async () => {
+      const pairs = generateKeyValuePairs(1000);
+      const timer = new Timer();
+      for (const pair of pairs) {
+        await map.set(pair.key, pair.value);
+      }
+      record('Individual insertion', timer.ms() / pairs.length, 'ms');
+    });
+
+    it('adds batched values', async () => {
+      const batches = Array.from({ length: 100 }, (_, i) => generateKeyValuePairs(1000, i * 1000));
+      const timer = new Timer();
+      for (const batch of batches) {
+        await map.setMany(batch);
+      }
+      record(`Batch insertion of ${batches[0].length} items`, timer.ms() / batches.length, 'ms');
+    });
+
+    it('reads individual values', async () => {
+      const pairs = generateKeyValuePairs(10000);
+      await map.setMany(pairs);
+      const timer = new Timer();
+      for (const pair of pairs) {
+        await map.getAsync(pair.key);
+      }
+      record('Individual read', (timer.ms() * 1000) / pairs.length, 'us');
+    });
+
+    it('reads via a cursor', async () => {
+      const pairs = generateKeyValuePairs(10000);
+      await map.setMany(pairs);
+      const timer = new Timer();
+      for await (const _ of map.entriesAsync()) {
+        // consume
+      }
+      record(`Iterator per item read of ${pairs.length} items`, (timer.ms() * 1000) / pairs.length, 'us');
+    });
+
+    it('reads the size of the map', async () => {
+      const numIterations = 1000;
+      const pairs = generateKeyValuePairs(10000);
+      await map.setMany(pairs);
+      const timer = new Timer();
+      for (let i = 0; i < numIterations; i++) {
+        await map.sizeAsync();
+      }
+      record(`Read size of ${pairs.length} items`, (timer.ms() * 1000) / numIterations, 'us');
+    });
+  });
+}

package/src/indexeddb/array.ts

@@ -32,7 +32,7 @@ export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T>
     return (
       (await this.db
         .index('key')
-        .count(IDBKeyRange.bound([this.#container, this.#name], [this.#container, this.#name]))) ?? 0
+        .count(IDBKeyRange.bound([this.#container, [this.#name]], [this.#container, [this.#name]]))) ?? 0
     );
   }
 
@@ -43,7 +43,7 @@ export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T>
         value: val,
         hash: hash(val),
         container: this.#container,
-        key: this.#name,
+        key: [this.#name],
         keyCount: length + 1,
         slot: this.#slot(length),
       });
@@ -91,7 +91,7 @@ export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T>
       value: val,
       hash: hash(val),
       container: this.#container,
-      key: this.#name,
+      key: [this.#name],
       keyCount: index + 1,
       slot: this.#slot(index),
     });
@@ -100,7 +100,7 @@ export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T>
 
   async *entriesAsync(): AsyncIterableIterator<[number, T]> {
     const index = this.db.index('key');
-    const rangeQuery = IDBKeyRange.bound([this.#container, this.#name], [this.#container, this.#name]);
+    const rangeQuery = IDBKeyRange.bound([this.#container, [this.#name]], [this.#container, [this.#name]]);
     for await (const cursor of index.iterate(rangeQuery)) {
       yield [cursor.value.keyCount - 1, cursor.value.value] as [number, T];
     }

package/src/indexeddb/index.ts

@@ -1,12 +1,17 @@
 import { type Logger, createLogger } from '@aztec/foundation/log';
 
 import type { DataStoreConfig } from '../config.js';
-import { initStoreForRollup } from '../utils.js';
+import { initStoreForRollupAndSchemaVersion } from '../utils.js';
 import { AztecIndexedDBStore } from './store.js';
 
 export { AztecIndexedDBStore } from './store.js';
 
-export async function createStore(name: string, config: DataStoreConfig, log: Logger = createLogger('kv-store')) {
+export async function createStore(
+  name: string,
+  config: DataStoreConfig,
+  schemaVersion: number | undefined = undefined,
+  log: Logger = createLogger('kv-store'),
+) {
   let { dataDirectory } = config;
   if (typeof dataDirectory !== 'undefined') {
     dataDirectory = `${dataDirectory}/${name}`;
@@ -18,10 +23,7 @@ export async function createStore(name: string, config: DataStoreConfig, log: Lo
       : `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKb} KB`,
   );
   const store = await AztecIndexedDBStore.open(createLogger('kv-store:indexeddb'), dataDirectory ?? '', false);
-  if (config.l1Contracts?.rollupAddress) {
-    return initStoreForRollup(store, config.l1Contracts.rollupAddress, log);
-  }
-  return store;
+  return initStoreForRollupAndSchemaVersion(store, schemaVersion, config.l1Contracts?.rollupAddress, log);
 }
 
 export function openTmpStore(ephemeral: boolean = false): Promise<AztecIndexedDBStore> {

package/src/indexeddb/map.ts

@@ -31,7 +31,7 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
 
   async getAsync(key: K): Promise<V | undefined> {
     const data = await this.db.get(this.slot(key));
-    return data?.value as V;
+    return data ? this.restoreBuffers(data.value as V) : undefined;
   }
 
   async hasAsync(key: K): Promise<boolean> {
@@ -41,7 +41,7 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
 
   async sizeAsync(): Promise<number> {
     const index = this.db.index('key');
-    const rangeQuery = IDBKeyRange.bound([this.container, ''], [this.container, '\uffff']);
+    const rangeQuery = IDBKeyRange.bound([this.container], [this.container + '\uffff'], true, true);
     return await index.count(rangeQuery);
   }
 
@@ -80,9 +80,12 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
 
   async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, V]> {
     const index = this.db.index('key');
+    const startKey = range.start ? this.normalizeKey(range.start) : [];
+    const endKey = range.end ? this.normalizeKey(range.end) : ['\uffff'];
+
     const rangeQuery = IDBKeyRange.bound(
-      [this.container, range.start ? this.normalizeKey(range.start) : ''],
-      [this.container, range.end ? this.normalizeKey(range.end) : '\uffff'],
+      [this.container, startKey],
+      [this.container, endKey],
       !!range.reverse,
       !range.reverse,
     );
@@ -91,7 +94,7 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
       if (range.limit && count >= range.limit) {
         return;
       }
-      yield [this.#denormalizeKey(cursor.value.key), cursor.value.value] as [K, V];
+      yield [this.#denormalizeKey(cursor.value.key), this.restoreBuffers(cursor.value.value as V)] as [K, V];
       count++;
     }
   }
@@ -108,14 +111,24 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
     }
   }
 
-  #denormalizeKey(key: string): K {
-    const denormalizedKey = key.split(',').map(part => (part.startsWith('n_') ? Number(part.slice(2)) : part));
-    return (denormalizedKey.length > 1 ? denormalizedKey : denormalizedKey[0]) as K;
+  /**
+   * IndexedDB's structured clone downcasts Buffer (a Node.js subclass of Uint8Array) to plain Uint8Array.
+   * This breaks .toString() behavior: Buffer.toString() returns UTF-8, Uint8Array.toString() returns
+   * comma-separated decimal bytes. We restore Buffer identity on read so callers get consistent behavior.
+   */
+  protected restoreBuffers(val: V): V {
+    if (val instanceof Uint8Array && !Buffer.isBuffer(val)) {
+      return Buffer.from(val) as V;
+    }
+    return val;
+  }
+
+  #denormalizeKey(key: (string | number | Uint8Array)[]): K {
+    return (key.length > 1 ? key : key[0]) as K;
   }
 
-  protected normalizeKey(key: K): string {
-    const arrayKey = Array.isArray(key) ? key : [key];
-    return (arrayKey as K[]).map((element: K) => (typeof element === 'number' ? `n_${element}` : element)).join(',');
+  protected normalizeKey(key: K): (string | number | Uint8Array)[] {
+    return Array.isArray(key) ? key : [key];
   }
 
   protected slot(key: K, index: number = 0): string {

package/src/indexeddb/multi_map.ts

@@ -57,7 +57,7 @@ export class IndexedDBAztecMultiMap<K extends Key, V extends Value>
       false,
     );
     for await (const cursor of index.iterate(rangeQuery)) {
-      yield cursor.value.value as V;
+      yield this.restoreBuffers(cursor.value.value as V);
     }
   }
 
@@ -88,4 +88,18 @@ export class IndexedDBAztecMultiMap<K extends Key, V extends Value>
       await this.db.delete(fullKey);
     }
   }
+
+  override async delete(key: K): Promise<void> {
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.container, this.normalizeKey(key), 0],
+      [this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
+      false,
+      false,
+    );
+
+    for await (const cursor of index.iterate(rangeQuery)) {
+      await cursor.delete();
+    }
+  }
 }

package/src/indexeddb/singleton.ts

@@ -37,7 +37,7 @@ export class IndexedDBAztecSingleton<T extends Value> implements AztecAsyncSingl
     const result = await this.db.put({
       container: this.#container,
       slot: this.#slot,
-      key: this.#slot,
+      key: [this.#slot],
       keyCount: 1,
       value: val,
       hash: hash(val),

package/src/indexeddb/store.ts

@@ -20,7 +20,7 @@ import { IndexedDBAztecSingleton } from './singleton.js';
 export type StoredData<V extends Value> = {
   value: V;
   container: string;
-  key: string;
+  key: (string | number | Uint8Array)[];
   keyCount: number;
   slot: string;
   hash: string;
@@ -30,7 +30,12 @@ export interface AztecIDBSchema extends DBSchema {
   data: {
     value: StoredData<any>;
     key: string;
-    indexes: { container: string; key: string; keyCount: number; hash: string };
+    indexes: {
+      container: string;
+      key: any; // Runtime: (string | number | Uint8Array)[] - idb types don't support arrays
+      keyCount: number;
+      hash: string;
+    };
   };
 }
 
@@ -187,18 +192,20 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
   }
 
   /** Deletes this store and removes the database */
-  delete() {
+  async delete() {
     this.#containers.clear();
+    await this.#txQueue.end();
     this.#rootDB.close();
-    return deleteDB(this.#name);
+    await deleteDB(this.#name);
   }
 
   estimateSize(): Promise<StoreSize> {
     return Promise.resolve({ mappingSize: 0, physicalFileSize: 0, actualSize: 0, numItems: 0 });
   }
 
-  close(): Promise<void> {
-    return Promise.resolve();
+  async close(): Promise<void> {
+    await this.#txQueue.end();
+    this.#rootDB.close();
   }
 
   backupTo(_dstPath: string, _compact?: boolean): Promise<void> {

package/src/interfaces/array_test_suite.ts

@@ -1,7 +1,5 @@
 import { toArray } from '@aztec/foundation/iterable';
 
-import { expect } from 'chai';
-
 import type { AztecArray, AztecAsyncArray } from './array.js';
 import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
 import { isSyncStore } from './utils.js';
@@ -53,12 +51,12 @@ export function describeAztecArray(
       await arr.push(2);
       await arr.push(3);
 
-      expect(await length()).to.equal(3);
+      expect(await length()).toBe(3);
 
-      expect(await arr.pop()).to.equal(3);
-      expect(await arr.pop()).to.equal(2);
-      expect(await arr.pop()).to.equal(1);
-      expect(await arr.pop()).to.equal(undefined);
+      expect(await arr.pop()).toBe(3);
+      expect(await arr.pop()).toBe(2);
+      expect(await arr.pop()).toBe(1);
+      expect(await arr.pop()).toBe(undefined);
     });
 
     it('should be able to get values by index', async () => {
@@ -66,14 +64,14 @@ export function describeAztecArray(
       await arr.push(2);
       await arr.push(3);
 
-      expect(await at(0)).to.equal(1);
-      expect(await at(1)).to.equal(2);
-      expect(await at(2)).to.equal(3);
-      expect(await at(3)).to.equal(undefined);
-      expect(await at(-1)).to.equal(3);
-      expect(await at(-2)).to.equal(2);
-      expect(await at(-3)).to.equal(1);
-      expect(await at(-4)).to.equal(undefined);
+      expect(await at(0)).toBe(1);
+      expect(await at(1)).toBe(2);
+      expect(await at(2)).toBe(3);
+      expect(await at(3)).toBe(undefined);
+      expect(await at(-1)).toBe(3);
+      expect(await at(-2)).toBe(2);
+      expect(await at(-3)).toBe(1);
+      expect(await at(-4)).toBe(undefined);
     });
 
     it('should be able to set values by index', async () => {
@@ -81,27 +79,27 @@ export function describeAztecArray(
       await arr.push(2);
       await arr.push(3);
 
-      expect(await arr.setAt(0, 4)).to.equal(true);
-      expect(await arr.setAt(1, 5)).to.equal(true);
-      expect(await arr.setAt(2, 6)).to.equal(true);
+      expect(await arr.setAt(0, 4)).toBe(true);
+      expect(await arr.setAt(1, 5)).toBe(true);
+      expect(await arr.setAt(2, 6)).toBe(true);
 
-      expect(await arr.setAt(3, 7)).to.equal(false);
+      expect(await arr.setAt(3, 7)).toBe(false);
 
-      expect(await at(0)).to.equal(4);
-      expect(await at(1)).to.equal(5);
-      expect(await at(2)).to.equal(6);
-      expect(await at(3)).to.equal(undefined);
+      expect(await at(0)).toBe(4);
+      expect(await at(1)).toBe(5);
+      expect(await at(2)).toBe(6);
+      expect(await at(3)).toBe(undefined);
 
-      expect(await arr.setAt(-1, 8)).to.equal(true);
-      expect(await arr.setAt(-2, 9)).to.equal(true);
-      expect(await arr.setAt(-3, 10)).to.equal(true);
+      expect(await arr.setAt(-1, 8)).toBe(true);
+      expect(await arr.setAt(-2, 9)).toBe(true);
+      expect(await arr.setAt(-3, 10)).toBe(true);
 
-      expect(await arr.setAt(-4, 11)).to.equal(false);
+      expect(await arr.setAt(-4, 11)).toBe(false);
 
-      expect(await at(-1)).to.equal(8);
-      expect(await at(-2)).to.equal(9);
-      expect(await at(-3)).to.equal(10);
-      expect(await at(-4)).to.equal(undefined);
+      expect(await at(-1)).toBe(8);
+      expect(await at(-2)).toBe(9);
+      expect(await at(-3)).toBe(10);
+      expect(await at(-4)).toBe(undefined);
     });
 
     it('should be able to iterate over values', async () => {
@@ -109,8 +107,8 @@ export function describeAztecArray(
       await arr.push(2);
       await arr.push(3);
 
-      expect(await values()).to.deep.equal([1, 2, 3]);
-      expect(await entries()).to.deep.equal([
+      expect(await values()).toEqual([1, 2, 3]);
+      expect(await entries()).toEqual([
         [0, 1],
         [1, 2],
         [2, 3],
@@ -123,8 +121,8 @@ export function describeAztecArray(
       await arr.push(3);
 
       const arr2 = store.openArray<number>('test');
-      expect(await length(arr2)).to.equal(3);
-      expect(await values(arr2)).to.deep.equal(await values());
+      expect(await length(arr2)).toBe(3);
+      expect(await values(arr2)).toEqual(await values());
     });
   });
 }

package/src/interfaces/index.ts

@@ -6,4 +6,4 @@ export * from './store.js';
 export * from './set.js';
 export * from './multi_map.js';
 export { mapRange } from './common.js';
-export type { CustomRange, Range, StoreSize } from './common.js';
+export type { CustomRange, Key, Range, StoreSize } from './common.js';