@aztec/kv-store 0.66.0 → 0.67.1-devnet

package/src/indexeddb/array.ts

@@ -0,0 +1,118 @@
+import { type IDBPDatabase, type IDBPObjectStore } from 'idb';
+
+import { type AztecAsyncArray } from '../interfaces/array.js';
+import { type AztecIDBSchema } from './store.js';
+
+/**
+ * A persistent array backed by IndexedDB.
+ */
+export class IndexedDBAztecArray<T> implements AztecAsyncArray<T> {
+  #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
+  #rootDB: IDBPDatabase<AztecIDBSchema>;
+  #container: string;
+  #name: string;
+
+  constructor(rootDB: IDBPDatabase<AztecIDBSchema>, name: string) {
+    this.#rootDB = rootDB;
+    this.#name = name;
+    this.#container = `array:${this.#name}`;
+  }
+
+  set db(db: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> | undefined) {
+    this.#_db = db;
+  }
+
+  get db(): IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> {
+    return this.#_db ? this.#_db : this.#rootDB.transaction('data', 'readwrite').store;
+  }
+
+  async lengthAsync(): Promise<number> {
+    return (
+      (await this.db
+        .index('key')
+        .count(IDBKeyRange.bound([this.#container, this.#name], [this.#container, this.#name]))) ?? 0
+    );
+  }
+
+  async push(...vals: T[]): Promise<number> {
+    let length = await this.lengthAsync();
+    for (const val of vals) {
+      await this.db.put({
+        value: val,
+        container: this.#container,
+        key: this.#name,
+        keyCount: length + 1,
+        slot: this.#slot(length),
+      });
+      length += 1;
+    }
+    return length;
+  }
+
+  async pop(): Promise<T | undefined> {
+    const length = await this.lengthAsync();
+    if (length === 0) {
+      return undefined;
+    }
+
+    const slot = this.#slot(length - 1);
+    const data = await this.db.get(slot);
+    await this.db.delete(slot);
+
+    return data?.value;
+  }
+
+  async atAsync(index: number): Promise<T | undefined> {
+    const length = await this.lengthAsync();
+
+    if (index < 0) {
+      index = length + index;
+    }
+
+    const data = await this.db.get(this.#slot(index));
+    return data?.value;
+  }
+
+  async setAt(index: number, val: T): Promise<boolean> {
+    const length = await this.lengthAsync();
+
+    if (index < 0) {
+      index = length + index;
+    }
+
+    if (index < 0 || index >= length) {
+      return Promise.resolve(false);
+    }
+
+    await this.db.put({
+      value: val,
+      container: this.#container,
+      key: this.#name,
+      keyCount: index + 1,
+      slot: this.#slot(index),
+    });
+    return true;
+  }
+
+  async *entriesAsync(): AsyncIterableIterator<[number, T]> {
+    const index = this.db.index('key');
+    const rangeQuery = IDBKeyRange.bound([this.#container, this.#name], [this.#container, this.#name]);
+    for await (const cursor of index.iterate(rangeQuery)) {
+      yield [cursor.value.keyCount - 1, cursor.value.value] as [number, T];
+    }
+  }
+
+  async *valuesAsync(): AsyncIterableIterator<T> {
+    for await (const [_, value] of this.entriesAsync()) {
+      yield value;
+    }
+  }
+
+  [Symbol.asyncIterator](): AsyncIterableIterator<T> {
+    return this.valuesAsync();
+  }
+
+  #slot(index: number): string {
+    return `array:${this.#name}:slot:${index}`;
+  }
+}

package/src/indexeddb/index.ts

@@ -0,0 +1,29 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+
+import { type DataStoreConfig } from '../config.js';
+import { initStoreForRollup } from '../utils.js';
+import { AztecIndexedDBStore } from './store.js';
+
+export { AztecIndexedDBStore } from './store.js';
+
+export async function createStore(name: string, config: DataStoreConfig, log: Logger = createLogger('kv-store')) {
+  let { dataDirectory } = config;
+  if (typeof dataDirectory !== 'undefined') {
+    dataDirectory = `${dataDirectory}/${name}`;
+  }
+
+  log.info(
+    dataDirectory
+      ? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.dataStoreMapSizeKB} KB`
+      : `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKB} KB`,
+  );
+  const store = await AztecIndexedDBStore.open(createLogger('kv-store:indexeddb'), dataDirectory ?? '', false);
+  if (config.l1Contracts?.rollupAddress) {
+    return initStoreForRollup(store, config.l1Contracts.rollupAddress, log);
+  }
+  return store;
+}
+
+export function openTmpStore(ephemeral: boolean = false): Promise<AztecIndexedDBStore> {
+  return AztecIndexedDBStore.open(createLogger('kv-store:indexeddb'), undefined, ephemeral);
+}

package/src/indexeddb/map.ts

@@ -0,0 +1,142 @@
+import { type IDBPDatabase, type IDBPObjectStore } from 'idb';
+
+import { type Key, type Range } from '../interfaces/common.js';
+import { type AztecAsyncMultiMap } from '../interfaces/map.js';
+import { type AztecIDBSchema } from './store.js';
+
+/**
+ * A map backed by IndexedDB.
+ */
+export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K, V> {
+  protected name: string;
+  #container: string;
+
+  #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
+  #rootDB: IDBPDatabase<AztecIDBSchema>;
+
+  constructor(rootDB: IDBPDatabase<AztecIDBSchema>, mapName: string) {
+    this.name = mapName;
+    this.#container = `map:${mapName}`;
+    this.#rootDB = rootDB;
+  }
+
+  set db(db: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> | undefined) {
+    this.#_db = db;
+  }
+
+  get db(): IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> {
+    return this.#_db ? this.#_db : this.#rootDB.transaction('data', 'readwrite').store;
+  }
+
+  async getAsync(key: K): Promise<V | undefined> {
+    const data = await this.db.get(this.#slot(key));
+    return data?.value as V;
+  }
+
+  async *getValuesAsync(key: K): AsyncIterableIterator<V> {
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.#container, this.#normalizeKey(key), 0],
+      [this.#container, this.#normalizeKey(key), Number.MAX_SAFE_INTEGER],
+      false,
+      false,
+    );
+    for await (const cursor of index.iterate(rangeQuery)) {
+      yield cursor.value.value as V;
+    }
+  }
+
+  async hasAsync(key: K): Promise<boolean> {
+    const result = (await this.getAsync(key)) !== undefined;
+    return result;
+  }
+
+  async set(key: K, val: V): Promise<void> {
+    const count = await this.db
+      .index('key')
+      .count(IDBKeyRange.bound([this.#container, this.#normalizeKey(key)], [this.#container, this.#normalizeKey(key)]));
+    await this.db.put({
+      value: val,
+      container: this.#container,
+      key: this.#normalizeKey(key),
+      keyCount: count + 1,
+      slot: this.#slot(key, count),
+    });
+  }
+
+  swap(_key: K, _fn: (val: V | undefined) => V): Promise<void> {
+    throw new Error('Not implemented');
+  }
+
+  async setIfNotExists(key: K, val: V): Promise<boolean> {
+    if (!(await this.hasAsync(key))) {
+      await this.set(key, val);
+      return true;
+    }
+    return false;
+  }
+
+  async delete(key: K): Promise<void> {
+    await this.db.delete(this.#slot(key));
+  }
+
+  async deleteValue(key: K, val: V): Promise<void> {
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.#container, this.#normalizeKey(key), 0],
+      [this.#container, this.#normalizeKey(key), Number.MAX_SAFE_INTEGER],
+      false,
+      false,
+    );
+    for await (const cursor of index.iterate(rangeQuery)) {
+      if (JSON.stringify(cursor.value.value) === JSON.stringify(val)) {
+        await cursor.delete();
+        return;
+      }
+    }
+  }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, V]> {
+    const index = this.db.index('key');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.#container, range.start ?? ''],
+      [this.#container, range.end ?? '\uffff'],
+      !!range.reverse,
+      !range.reverse,
+    );
+    let count = 0;
+    for await (const cursor of index.iterate(rangeQuery, range.reverse ? 'prev' : 'next')) {
+      if (range.limit && count >= range.limit) {
+        return;
+      }
+      yield [cursor.value.key, cursor.value.value] as [K, V];
+      count++;
+    }
+  }
+
+  async *valuesAsync(range: Range<K> = {}): AsyncIterableIterator<V> {
+    for await (const [_, value] of this.entriesAsync(range)) {
+      yield value;
+    }
+  }
+
+  async *keysAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    for await (const [key, _] of this.entriesAsync(range)) {
+      yield this.#denormalizeKey(key as string);
+    }
+  }
+
+  #denormalizeKey(key: string): K {
+    const denormalizedKey = (key as string).split(',').map(part => (isNaN(parseInt(part)) ? part : parseInt(part)));
+    return (denormalizedKey.length > 1 ? denormalizedKey : key) as K;
+  }
+
+  #normalizeKey(key: K): string {
+    const arrayKey = Array.isArray(key) ? key : [key];
+    return arrayKey.join(',');
+  }
+
+  #slot(key: K, index: number = 0): string {
+    return `map:${this.name}:slot:${this.#normalizeKey(key)}:${index}`;
+  }
+}

package/src/indexeddb/set.ts

@@ -0,0 +1,37 @@
+import { type IDBPDatabase, type IDBPObjectStore } from 'idb';
+
+import { type Key, type Range } from '../interfaces/common.js';
+import { type AztecAsyncSet } from '../interfaces/set.js';
+import { IndexedDBAztecMap } from './map.js';
+import { type AztecIDBSchema } from './store.js';
+
+/**
+ * A set backed by IndexedDB.
+ */
+export class IndexedDBAztecSet<K extends Key> implements AztecAsyncSet<K> {
+  private map: IndexedDBAztecMap<K, boolean>;
+
+  constructor(rootDb: IDBPDatabase<AztecIDBSchema>, mapName: string) {
+    this.map = new IndexedDBAztecMap(rootDb, mapName);
+  }
+
+  set db(db: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> | undefined) {
+    this.map.db = db;
+  }
+
+  hasAsync(key: K): Promise<boolean> {
+    return this.map.hasAsync(key);
+  }
+
+  add(key: K): Promise<void> {
+    return this.map.set(key, true);
+  }
+
+  delete(key: K): Promise<void> {
+    return this.map.delete(key);
+  }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    yield* this.map.keysAsync(range);
+  }
+}

package/src/indexeddb/singleton.ts

@@ -0,0 +1,49 @@
+import { type IDBPDatabase, type IDBPObjectStore } from 'idb';
+
+import { type AztecAsyncSingleton } from '../interfaces/singleton.js';
+import { type AztecIDBSchema } from './store.js';
+
+/**
+ * Stores a single value in IndexedDB.
+ */
+export class IndexedDBAztecSingleton<T> implements AztecAsyncSingleton<T> {
+  #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
+  #rootDB: IDBPDatabase<AztecIDBSchema>;
+  #container: string;
+  #slot: string;
+
+  constructor(rootDB: IDBPDatabase<AztecIDBSchema>, name: string) {
+    this.#rootDB = rootDB;
+    this.#container = `singleton:${name}`;
+    this.#slot = `singleton:${name}:value`;
+  }
+
+  set db(db: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> | undefined) {
+    this.#_db = db;
+  }
+
+  get db(): IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> {
+    return this.#_db ? this.#_db : this.#rootDB.transaction('data', 'readwrite').store;
+  }
+
+  async getAsync(): Promise<T | undefined> {
+    const data = await this.db.get(this.#slot);
+    return data?.value as T;
+  }
+
+  async set(val: T): Promise<boolean> {
+    const result = await this.db.put({
+      container: this.#container,
+      slot: this.#slot,
+      key: this.#slot,
+      keyCount: 1,
+      value: val,
+    });
+    return result !== undefined;
+  }
+
+  async delete(): Promise<boolean> {
+    await this.db.delete(this.#slot);
+    return true;
+  }
+}

package/src/indexeddb/store.ts

@@ -0,0 +1,193 @@
+import { type Logger } from '@aztec/foundation/log';
+
+import { type DBSchema, type IDBPDatabase, deleteDB, openDB } from 'idb';
+
+import { type AztecAsyncArray } from '../interfaces/array.js';
+import { type Key } from '../interfaces/common.js';
+import { type AztecAsyncCounter } from '../interfaces/counter.js';
+import { type AztecAsyncMap, type AztecAsyncMultiMap } from '../interfaces/map.js';
+import { type AztecAsyncSet } from '../interfaces/set.js';
+import { type AztecAsyncSingleton } from '../interfaces/singleton.js';
+import { type AztecAsyncKVStore } from '../interfaces/store.js';
+import { IndexedDBAztecArray } from './array.js';
+import { IndexedDBAztecMap } from './map.js';
+import { IndexedDBAztecSet } from './set.js';
+import { IndexedDBAztecSingleton } from './singleton.js';
+
+export type StoredData<V> = { value: V; container: string; key: string; keyCount: number; slot: string };
+
+export interface AztecIDBSchema extends DBSchema {
+  data: {
+    value: StoredData<any>;
+    key: string;
+    indexes: { container: string; key: string; keyCount: number };
+  };
+}
+
+/**
+ * A key-value store backed by IndexedDB.
+ */
+
+export class AztecIndexedDBStore implements AztecAsyncKVStore {
+  #log: Logger;
+  #rootDB: IDBPDatabase<AztecIDBSchema>;
+  #name: string;
+
+  #containers = new Set<
+    IndexedDBAztecArray<any> | IndexedDBAztecMap<any, any> | IndexedDBAztecSet<any> | IndexedDBAztecSingleton<any>
+  >();
+
+  constructor(rootDB: IDBPDatabase<AztecIDBSchema>, public readonly isEphemeral: boolean, log: Logger, name: string) {
+    this.#rootDB = rootDB;
+    this.#log = log;
+    this.#name = name;
+  }
+  /**
+   * Creates a new AztecKVStore backed by IndexedDB. The path to the database is optional. If not provided,
+   * the database will be stored in a temporary location and be deleted when the process exists.
+   *
+   *
+   * @param path - A path on the disk to store the database. Optional
+   * @param ephemeral - true if the store should only exist in memory and not automatically be flushed to disk. Optional
+   * @param log - A logger to use. Optional
+   * @returns The store
+   */
+  static async open(log: Logger, name?: string, ephemeral: boolean = false): Promise<AztecIndexedDBStore> {
+    name = name && !ephemeral ? name : self.crypto.getRandomValues(new Uint8Array(16)).join('');
+    log.debug(`Opening IndexedDB ${ephemeral ? 'temp ' : ''}database with name ${name}`);
+    const rootDB = await openDB<AztecIDBSchema>(name, 1, {
+      upgrade(db) {
+        const objectStore = db.createObjectStore('data', { keyPath: 'slot' });
+
+        objectStore.createIndex('key', ['container', 'key'], { unique: false });
+        objectStore.createIndex('keyCount', ['container', 'key', 'keyCount'], { unique: false });
+      },
+    });
+
+    const kvStore = new AztecIndexedDBStore(rootDB, ephemeral, log, name);
+    return kvStore;
+  }
+
+  /**
+   * Forks the current DB into a new DB by backing it up to a temporary location and opening a new indexedb.
+   * @returns A new AztecIndexedDBStore.
+   */
+  async fork(): Promise<AztecAsyncKVStore> {
+    const forkedStore = await AztecIndexedDBStore.open(this.#log, undefined, true);
+    this.#log.verbose(`Forking store to ${forkedStore.#name}`);
+
+    // Copy old data to new store
+    const oldData = this.#rootDB.transaction('data').store;
+    const dataToWrite = [];
+    for await (const cursor of oldData.iterate()) {
+      dataToWrite.push(cursor.value);
+    }
+    const tx = forkedStore.#rootDB.transaction('data', 'readwrite').store;
+    for (const data of dataToWrite) {
+      await tx.add(data);
+    }
+
+    this.#log.debug(`Forked store at ${forkedStore.#name} opened successfully`);
+    return forkedStore;
+  }
+
+  /**
+   * Creates a new AztecMap in the store.
+   * @param name - Name of the map
+   * @returns A new AztecMap
+   */
+  openMap<K extends Key, V>(name: string): AztecAsyncMap<K, V> {
+    const map = new IndexedDBAztecMap<K, V>(this.#rootDB, name);
+    this.#containers.add(map);
+    return map;
+  }
+
+  /**
+   * Creates a new AztecSet in the store.
+   * @param name - Name of the set
+   * @returns A new AztecSet
+   */
+  openSet<K extends Key>(name: string): AztecAsyncSet<K> {
+    const set = new IndexedDBAztecSet<K>(this.#rootDB, name);
+    this.#containers.add(set);
+    return set;
+  }
+
+  /**
+   * Creates a new AztecMultiMap in the store. A multi-map stores multiple values for a single key automatically.
+   * @param name - Name of the map
+   * @returns A new AztecMultiMap
+   */
+  openMultiMap<K extends Key, V>(name: string): AztecAsyncMultiMap<K, V> {
+    const multimap = new IndexedDBAztecMap<K, V>(this.#rootDB, name);
+    this.#containers.add(multimap);
+    return multimap;
+  }
+
+  openCounter<K extends Key | Array<string | number>>(_name: string): AztecAsyncCounter<K> {
+    throw new Error('Method not implemented.');
+  }
+
+  /**
+   * Creates a new AztecArray in the store.
+   * @param name - Name of the array
+   * @returns A new AztecArray
+   */
+  openArray<T>(name: string): AztecAsyncArray<T> {
+    const array = new IndexedDBAztecArray<T>(this.#rootDB, name);
+    this.#containers.add(array);
+    return array;
+  }
+
+  /**
+   * Creates a new AztecSingleton in the store.
+   * @param name - Name of the singleton
+   * @returns A new AztecSingleton
+   */
+  openSingleton<T>(name: string): AztecAsyncSingleton<T> {
+    const singleton = new IndexedDBAztecSingleton<T>(this.#rootDB, name);
+    this.#containers.add(singleton);
+    return singleton;
+  }
+
+  /**
+   * Runs a callback in a transaction.
+   * @param callback - Function to execute in a transaction
+   * @returns A promise that resolves to the return value of the callback
+   */
+  async transactionAsync<T>(callback: () => Promise<T>): Promise<T> {
+    const tx = this.#rootDB.transaction('data', 'readwrite');
+    for (const container of this.#containers) {
+      container.db = tx.store;
+    }
+    // Avoid awaiting this promise so it doesn't get scheduled in the next microtask
+    // By then, the tx would be closed
+    const runningPromise = callback();
+    // Wait for the transaction to finish
+    await tx.done;
+    for (const container of this.#containers) {
+      container.db = undefined;
+    }
+    // Return the result of the callback.
+    // Tx is guaranteed to already be closed, so the await doesn't hurt anything here
+    return await runningPromise;
+  }
+
+  /**
+   * Clears all entries in the store & sub DBs.
+   */
+  async clear() {
+    await this.#rootDB.transaction('data', 'readwrite').store.clear();
+  }
+
+  /** Deletes this store and removes the database */
+  delete() {
+    this.#containers.clear();
+    this.#rootDB.close();
+    return deleteDB(this.#name);
+  }
+
+  estimateSize(): { mappingSize: number; actualSize: number; numItems: number } {
+    return { mappingSize: 0, actualSize: 0, numItems: 0 };
+  }
+}

package/src/interfaces/array.ts

@@ -1,12 +1,7 @@
 /**
  * An array backed by a persistent store. Can not have any holes in it.
  */
-export interface AztecArray<T> {
-  /**
-   * The size of the array
-   */
-  length: number;
-
+interface BaseAztecArray<T> {
   /**
    * Pushes values to the end of the array
    * @param vals - The values to push to the end of the array
@@ -20,6 +15,24 @@ export interface AztecArray<T> {
    */
   pop(): Promise<T | undefined>;
 
+  /**
+   * Updates the value at the given index. Index can be in the range [-length, length - 1).
+   * @param index - The index to set the value at
+   * @param val - The value to set
+   * @returns Whether the value was set
+   */
+  setAt(index: number, val: T): Promise<boolean>;
+}
+
+/**
+ * An array backed by a persistent store. Can not have any holes in it.
+ */
+export interface AztecAsyncArray<T> extends BaseAztecArray<T> {
+  /**
+   * The size of the array
+   */
+  lengthAsync(): Promise<number>;
+
   /**
    * Gets the value at the given index. Index can be in the range [-length, length - 1).
    * If the index is negative, it will be treated as an offset from the end of the array.
@@ -27,15 +40,38 @@ export interface AztecArray<T> {
    * @param index - The index to get the value from
    * @returns The value at the given index or undefined if the index is out of bounds
    */
-  at(index: number): T | undefined;
+  atAsync(index: number): Promise<T | undefined>;
 
   /**
-   * Updates the value at the given index. Index can be in the range [-length, length - 1).
-   * @param index - The index to set the value at
-   * @param val - The value to set
-   * @returns Whether the value was set
+   * Iterates over the array with indexes.
    */
-  setAt(index: number, val: T): Promise<boolean>;
+  entriesAsync(): AsyncIterableIterator<[number, T]>;
+
+  /**
+   * Iterates over the array.
+   */
+  valuesAsync(): AsyncIterableIterator<T>;
+
+  /**
+   * Iterates over the array.
+   */
+  [Symbol.asyncIterator](): AsyncIterableIterator<T>;
+}
+
+export interface AztecArray<T> extends BaseAztecArray<T> {
+  /**
+   * The size of the array
+   */
+  length: number;
+
+  /**
+   * Gets the value at the given index. Index can be in the range [-length, length - 1).
+   * If the index is negative, it will be treated as an offset from the end of the array.
+   *
+   * @param index - The index to get the value from
+   * @returns The value at the given index or undefined if the index is out of bounds
+   */
+  at(index: number): T | undefined;
 
   /**
    * Iterates over the array with indexes.