@aztec-labs/kv-store 6.0.0-nightly.20260829

package/src/bench/shared_map_bench.ts

@@ -0,0 +1,111 @@
+import type { Logger } from '@aztec-labs/foundation/log';
+import { Timer } from '@aztec-labs/foundation/timer';
+
+import type { Key } from '../interfaces/common.js';
+import type { AztecAsyncMap } from '../interfaces/map.js';
+import type { AztecAsyncKVStore } from '../interfaces/store.js';
+
+/** One benchmark measurement. */
+export type BenchResult = {
+  /** Benchmark name (includes the backend prefix for disambiguation). */
+  name: string;
+  value: number;
+  unit: 'ms' | 'us';
+};
+
+export type BenchReporter = (results: BenchResult[]) => void | Promise<void>;
+
+/**
+ * Runs the standard Map benchmark suite against any `AztecAsyncKVStore` backend,
+ * populates `results`, and calls `reporter` in `afterAll`.
+ *
+ * Kept free of Node-only deps (`fs`, `path`) so the same runner works under
+ * vitest-browser for IndexedDB and SQLite-OPFS.
+ */
+export function describeAztecMapBench(
+  backendPrefix: string,
+  getStore: () => Promise<AztecAsyncKVStore>,
+  logger: Logger,
+  reporter: BenchReporter,
+) {
+  describe(`${backendPrefix} Map benchmarks`, () => {
+    let store: AztecAsyncKVStore;
+    let map: AztecAsyncMap<Key, string>;
+
+    const results: BenchResult[] = [];
+
+    const generateKeyValuePairs = (count: number, offset = 0) => {
+      const keys = Array.from({ length: count }, (_, i) => `key-${i + offset}`);
+      const values = Array.from({ length: count }, (_, i) => `value-${i + offset}`);
+      return keys.map((key, i) => ({ key, value: values[i] }));
+    };
+
+    const record = (name: string, value: number, unit: BenchResult['unit']) => {
+      results.push({ name: `${backendPrefix}/Map/${name}`, value, unit });
+    };
+
+    beforeEach(async () => {
+      store = await getStore();
+      map = store.openMap<Key, string>('test');
+    });
+
+    afterEach(async () => {
+      await store.delete();
+    });
+
+    afterAll(async () => {
+      const pretty = results.map(r => `${r.name}: ${r.value.toFixed(2)} ${r.unit}`).join('\n');
+      logger.info(`\n${pretty}\n`);
+      await reporter(results);
+    });
+
+    it('adds individual values', async () => {
+      const pairs = generateKeyValuePairs(1000);
+      const timer = new Timer();
+      for (const pair of pairs) {
+        await map.set(pair.key, pair.value);
+      }
+      record('Individual insertion', timer.ms() / pairs.length, 'ms');
+    });
+
+    it('adds batched values', async () => {
+      const batches = Array.from({ length: 100 }, (_, i) => generateKeyValuePairs(1000, i * 1000));
+      const timer = new Timer();
+      for (const batch of batches) {
+        await map.setMany(batch);
+      }
+      record(`Batch insertion of ${batches[0].length} items`, timer.ms() / batches.length, 'ms');
+    });
+
+    it('reads individual values', async () => {
+      const pairs = generateKeyValuePairs(10000);
+      await map.setMany(pairs);
+      const timer = new Timer();
+      for (const pair of pairs) {
+        await map.getAsync(pair.key);
+      }
+      record('Individual read', (timer.ms() * 1000) / pairs.length, 'us');
+    });
+
+    it('reads via a cursor', async () => {
+      const pairs = generateKeyValuePairs(10000);
+      await map.setMany(pairs);
+      const timer = new Timer();
+      for await (const _ of map.entriesAsync()) {
+        // consume
+      }
+      record(`Iterator per item read of ${pairs.length} items`, (timer.ms() * 1000) / pairs.length, 'us');
+    });
+
+    it('reads the size of the map', async () => {
+      const numIterations = 1000;
+      const pairs = generateKeyValuePairs(10000);
+      await map.setMany(pairs);
+      const timer = new Timer();
+      for (let i = 0; i < numIterations; i++) {
+        await map.sizeAsync();
+      }
+      record(`Read size of ${pairs.length} items`, (timer.ms() * 1000) / numIterations, 'us');
+    });
+  });
+}

package/src/deprecated/indexeddb/array.ts

@@ -0,0 +1,122 @@
+import type { IDBPDatabase, IDBPObjectStore } from 'idb';
+import { hash } from 'ohash';
+
+import type { AztecAsyncArray } from '../../interfaces/array.js';
+import type { Value } from '../../interfaces/common.js';
+import type { AztecIDBSchema } from './store.js';
+
+/**
+ * A persistent array backed by IndexedDB.
+ */
+export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T> {
+  #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
+  #rootDB: IDBPDatabase<AztecIDBSchema>;
+  #container: string;
+  #name: string;
+
+  constructor(rootDB: IDBPDatabase<AztecIDBSchema>, name: string) {
+    this.#rootDB = rootDB;
+    this.#name = name;
+    this.#container = `array:${this.#name}`;
+  }
+
+  set db(db: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> | undefined) {
+    this.#_db = db;
+  }
+
+  get db(): IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> {
+    return this.#_db ? this.#_db : this.#rootDB.transaction('data', 'readwrite').store;
+  }
+
+  async lengthAsync(): Promise<number> {
+    return (
+      (await this.db
+        .index('key')
+        .count(IDBKeyRange.bound([this.#container, [this.#name]], [this.#container, [this.#name]]))) ?? 0
+    );
+  }
+
+  async push(...vals: T[]): Promise<number> {
+    let length = await this.lengthAsync();
+    for (const val of vals) {
+      await this.db.put({
+        value: val,
+        hash: hash(val),
+        container: this.#container,
+        key: [this.#name],
+        keyCount: length + 1,
+        slot: this.#slot(length),
+      });
+      length += 1;
+    }
+    return length;
+  }
+
+  async pop(): Promise<T | undefined> {
+    const length = await this.lengthAsync();
+    if (length === 0) {
+      return undefined;
+    }
+
+    const slot = this.#slot(length - 1);
+    const data = await this.db.get(slot);
+    await this.db.delete(slot);
+
+    return data?.value;
+  }
+
+  async atAsync(index: number): Promise<T | undefined> {
+    const length = await this.lengthAsync();
+
+    if (index < 0) {
+      index = length + index;
+    }
+
+    const data = await this.db.get(this.#slot(index));
+    return data?.value;
+  }
+
+  async setAt(index: number, val: T): Promise<boolean> {
+    const length = await this.lengthAsync();
+
+    if (index < 0) {
+      index = length + index;
+    }
+
+    if (index < 0 || index >= length) {
+      return Promise.resolve(false);
+    }
+
+    await this.db.put({
+      value: val,
+      hash: hash(val),
+      container: this.#container,
+      key: [this.#name],
+      keyCount: index + 1,
+      slot: this.#slot(index),
+    });
+    return true;
+  }
+
+  async *entriesAsync(): AsyncIterableIterator<[number, T]> {
+    const index = this.db.index('key');
+    const rangeQuery = IDBKeyRange.bound([this.#container, [this.#name]], [this.#container, [this.#name]]);
+    for await (const cursor of index.iterate(rangeQuery)) {
+      yield [cursor.value.keyCount - 1, cursor.value.value] as [number, T];
+    }
+  }
+
+  async *valuesAsync(): AsyncIterableIterator<T> {
+    for await (const [_, value] of this.entriesAsync()) {
+      yield value;
+    }
+  }
+
+  [Symbol.asyncIterator](): AsyncIterableIterator<T> {
+    return this.valuesAsync();
+  }
+
+  #slot(index: number): string {
+    return `array:${this.#name}:slot:${index}`;
+  }
+}

package/src/deprecated/indexeddb/index.ts

@@ -0,0 +1,12 @@
+import { createLogger } from '@aztec-labs/foundation/log';
+
+import { AztecIndexedDBStore } from './store.js';
+
+export { AztecIndexedDBStore } from './store.js';
+
+/**
+ * @deprecated The IndexedDB backend is being retired. Use `@aztec-labs/kv-store/sqlite-opfs` instead.
+ */
+export function openTmpStore(ephemeral: boolean = false): Promise<AztecIndexedDBStore> {
+  return AztecIndexedDBStore.open(createLogger('kv-store:indexeddb'), undefined, ephemeral);
+}

package/src/deprecated/indexeddb/map.ts

@@ -0,0 +1,137 @@
+import type { IDBPDatabase, IDBPObjectStore } from 'idb';
+import { hash } from 'ohash';
+
+import type { Key, Range, Value } from '../../interfaces/common.js';
+import type { AztecAsyncMap } from '../../interfaces/map.js';
+import type { AztecIDBSchema } from './store.js';
+
+/**
+ * A map backed by IndexedDB.
+ */
+export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecAsyncMap<K, V> {
+  protected name: string;
+  protected container: string;
+
+  #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
+  #rootDB: IDBPDatabase<AztecIDBSchema>;
+
+  constructor(rootDB: IDBPDatabase<AztecIDBSchema>, mapName: string) {
+    this.name = mapName;
+    this.container = `map:${mapName}`;
+    this.#rootDB = rootDB;
+  }
+
+  set db(db: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> | undefined) {
+    this.#_db = db;
+  }
+
+  get db(): IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> {
+    return this.#_db ? this.#_db : this.#rootDB.transaction('data', 'readwrite').store;
+  }
+
+  async getAsync(key: K): Promise<V | undefined> {
+    const data = await this.db.get(this.slot(key));
+    return data ? this.restoreBuffers(data.value as V) : undefined;
+  }
+
+  async hasAsync(key: K): Promise<boolean> {
+    const result = (await this.getAsync(key)) !== undefined;
+    return result;
+  }
+
+  async sizeAsync(): Promise<number> {
+    const index = this.db.index('key');
+    const rangeQuery = IDBKeyRange.bound([this.container], [this.container + '\uffff'], true, true);
+    return await index.count(rangeQuery);
+  }
+
+  async set(key: K, val: V): Promise<void> {
+    await this.db.put({
+      value: val,
+      hash: hash(val),
+      container: this.container,
+      key: this.normalizeKey(key),
+      keyCount: 1,
+      slot: this.slot(key),
+    });
+  }
+
+  async setMany(entries: { key: K; value: V }[]): Promise<void> {
+    for (const { key, value } of entries) {
+      await this.set(key, value);
+    }
+  }
+
+  swap(_key: K, _fn: (val: V | undefined) => V): Promise<void> {
+    throw new Error('Not implemented');
+  }
+
+  async setIfNotExists(key: K, val: V): Promise<boolean> {
+    if (!(await this.hasAsync(key))) {
+      await this.set(key, val);
+      return true;
+    }
+    return false;
+  }
+
+  async delete(key: K): Promise<void> {
+    await this.db.delete(this.slot(key));
+  }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, V]> {
+    const index = this.db.index('key');
+    const startKey = range.start ? this.normalizeKey(range.start) : [];
+    const endKey = range.end ? this.normalizeKey(range.end) : ['\uffff'];
+
+    const rangeQuery = IDBKeyRange.bound(
+      [this.container, startKey],
+      [this.container, endKey],
+      !!range.reverse,
+      !range.reverse,
+    );
+    let count = 0;
+    for await (const cursor of index.iterate(rangeQuery, range.reverse ? 'prev' : 'next')) {
+      if (range.limit && count >= range.limit) {
+        return;
+      }
+      yield [this.#denormalizeKey(cursor.value.key), this.restoreBuffers(cursor.value.value as V)] as [K, V];
+      count++;
+    }
+  }
+
+  async *valuesAsync(range: Range<K> = {}): AsyncIterableIterator<V> {
+    for await (const [_, value] of this.entriesAsync(range)) {
+      yield value;
+    }
+  }
+
+  async *keysAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    for await (const [key, _] of this.entriesAsync(range)) {
+      yield key;
+    }
+  }
+
+  /**
+   * IndexedDB's structured clone downcasts Buffer (a Node.js subclass of Uint8Array) to plain Uint8Array.
+   * This breaks .toString() behavior: Buffer.toString() returns UTF-8, Uint8Array.toString() returns
+   * comma-separated decimal bytes. We restore Buffer identity on read so callers get consistent behavior.
+   */
+  protected restoreBuffers(val: V): V {
+    if (val instanceof Uint8Array && !Buffer.isBuffer(val)) {
+      return Buffer.from(val) as V;
+    }
+    return val;
+  }
+
+  #denormalizeKey(key: (string | number | Uint8Array)[]): K {
+    return (key.length > 1 ? key : key[0]) as K;
+  }
+
+  protected normalizeKey(key: K): (string | number | Uint8Array)[] {
+    return Array.isArray(key) ? key : [key];
+  }
+
+  protected slot(key: K, index: number = 0): string {
+    return `map:${this.name}:slot:${this.normalizeKey(key)}:${index}`;
+  }
+}

package/src/deprecated/indexeddb/multi_map.ts

@@ -0,0 +1,106 @@
+import { first } from '@aztec-labs/foundation/iterable';
+import { hash } from 'ohash';
+
+import type { Key, Value } from '../../interfaces/common.js';
+import type { AztecAsyncMultiMap } from '../../interfaces/multi_map.js';
+import { IndexedDBAztecMap } from './map.js';
+
+/**
+ * A multi map backed by IndexedDB.
+ */
+export class IndexedDBAztecMultiMap<K extends Key, V extends Value>
+  extends IndexedDBAztecMap<K, V>
+  implements AztecAsyncMultiMap<K, V>
+{
+  override async set(key: K, val: V): Promise<void> {
+    // Inserting repeated values is a no-op
+    const exists = !!(await this.db
+      .index('hash')
+      .get(
+        IDBKeyRange.bound(
+          [this.container, this.normalizeKey(key), hash(val)],
+          [this.container, this.normalizeKey(key), hash(val)],
+        ),
+      ));
+    if (exists) {
+      return;
+    }
+    // Get the maximum keyCount for the given key
+    // In order to support sparse multimaps, we cannot rely
+    // on just counting the number of entries for the key, since we would repeat slots
+    // if we delete an entry
+    // set -> container:key:0 (keyCount = 1)
+    // set -> container:key:1 (keyCount = 2)
+    // delete -> container:key:0 (keyCount = 1)
+    // set -> container:key:1 <--- already exists!
+    // Instead, we iterate in reverse order to get the last inserted entry
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.upperBound([this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER]);
+    const maxEntry = await first(index.iterate(rangeQuery, 'prevunique'));
+    const count = maxEntry?.value?.keyCount ?? 0;
+    await this.db.put({
+      value: val,
+      hash: hash(val),
+      container: this.container,
+      key: this.normalizeKey(key),
+      keyCount: count + 1,
+      slot: this.slot(key, count),
+    });
+  }
+
+  async *getValuesAsync(key: K): AsyncIterableIterator<V> {
+    // Iterate over the whole range of keyCount for the given key
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.container, this.normalizeKey(key), 0],
+      [this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
+      false,
+      false,
+    );
+    for await (const cursor of index.iterate(rangeQuery)) {
+      yield this.restoreBuffers(cursor.value.value as V);
+    }
+  }
+
+  getValueCountAsync(key: K): Promise<number> {
+    // Count entries over the keyCount index range for this key
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.container, this.normalizeKey(key), 0],
+      [this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
+      false,
+      false,
+    );
+    return index.count(rangeQuery);
+  }
+
+  async deleteValue(key: K, val: V): Promise<void> {
+    // Since we know the value, we can hash it and directly query the "hash" index
+    // to avoid having to iterate over all the values
+    const fullKey = await this.db
+      .index('hash')
+      .getKey(
+        IDBKeyRange.bound(
+          [this.container, this.normalizeKey(key), hash(val)],
+          [this.container, this.normalizeKey(key), hash(val)],
+        ),
+      );
+    if (fullKey) {
+      await this.db.delete(fullKey);
+    }
+  }
+
+  override async delete(key: K): Promise<void> {
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.container, this.normalizeKey(key), 0],
+      [this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
+      false,
+      false,
+    );
+
+    for await (const cursor of index.iterate(rangeQuery)) {
+      await cursor.delete();
+    }
+  }
+}

package/src/deprecated/indexeddb/set.ts

@@ -0,0 +1,37 @@
+import type { IDBPDatabase, IDBPObjectStore } from 'idb';
+
+import type { Key, Range } from '../../interfaces/common.js';
+import type { AztecAsyncSet } from '../../interfaces/set.js';
+import { IndexedDBAztecMap } from './map.js';
+import type { AztecIDBSchema } from './store.js';
+
+/**
+ * A set backed by IndexedDB.
+ */
+export class IndexedDBAztecSet<K extends Key> implements AztecAsyncSet<K> {
+  private map: IndexedDBAztecMap<K, boolean>;
+
+  constructor(rootDb: IDBPDatabase<AztecIDBSchema>, mapName: string) {
+    this.map = new IndexedDBAztecMap(rootDb, mapName);
+  }
+
+  set db(db: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> | undefined) {
+    this.map.db = db;
+  }
+
+  hasAsync(key: K): Promise<boolean> {
+    return this.map.hasAsync(key);
+  }
+
+  add(key: K): Promise<void> {
+    return this.map.set(key, true);
+  }
+
+  delete(key: K): Promise<void> {
+    return this.map.delete(key);
+  }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    yield* this.map.keysAsync(range);
+  }
+}

package/src/deprecated/indexeddb/singleton.ts

@@ -0,0 +1,52 @@
+import type { IDBPDatabase, IDBPObjectStore } from 'idb';
+import { hash } from 'ohash';
+
+import type { Value } from '../../interfaces/common.js';
+import type { AztecAsyncSingleton } from '../../interfaces/singleton.js';
+import type { AztecIDBSchema } from './store.js';
+
+/**
+ * Stores a single value in IndexedDB.
+ */
+export class IndexedDBAztecSingleton<T extends Value> implements AztecAsyncSingleton<T> {
+  #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
+  #rootDB: IDBPDatabase<AztecIDBSchema>;
+  #container: string;
+  #slot: string;
+
+  constructor(rootDB: IDBPDatabase<AztecIDBSchema>, name: string) {
+    this.#rootDB = rootDB;
+    this.#container = `singleton:${name}`;
+    this.#slot = `singleton:${name}:value`;
+  }
+
+  set db(db: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> | undefined) {
+    this.#_db = db;
+  }
+
+  get db(): IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> {
+    return this.#_db ? this.#_db : this.#rootDB.transaction('data', 'readwrite').store;
+  }
+
+  async getAsync(): Promise<T | undefined> {
+    const data = await this.db.get(this.#slot);
+    return data?.value as T;
+  }
+
+  async set(val: T): Promise<boolean> {
+    const result = await this.db.put({
+      container: this.#container,
+      slot: this.#slot,
+      key: [this.#slot],
+      keyCount: 1,
+      value: val,
+      hash: hash(val),
+    });
+    return result !== undefined;
+  }
+
+  async delete(): Promise<boolean> {
+    await this.db.delete(this.#slot);
+    return true;
+  }
+}