@aztec/kv-store 0.65.2 → 0.67.0

package/src/indexeddb/map.ts

@@ -0,0 +1,142 @@
+import { type IDBPDatabase, type IDBPObjectStore } from 'idb';
+
+import { type Key, type Range } from '../interfaces/common.js';
+import { type AztecAsyncMultiMap } from '../interfaces/map.js';
+import { type AztecIDBSchema } from './store.js';
+
+/**
+ * A map backed by IndexedDB.
+ */
+export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K, V> {
+  protected name: string;
+  #container: string;
+
+  #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
+  #rootDB: IDBPDatabase<AztecIDBSchema>;
+
+  constructor(rootDB: IDBPDatabase<AztecIDBSchema>, mapName: string) {
+    this.name = mapName;
+    this.#container = `map:${mapName}`;
+    this.#rootDB = rootDB;
+  }
+
+  set db(db: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> | undefined) {
+    this.#_db = db;
+  }
+
+  get db(): IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> {
+    return this.#_db ? this.#_db : this.#rootDB.transaction('data', 'readwrite').store;
+  }
+
+  async getAsync(key: K): Promise<V | undefined> {
+    const data = await this.db.get(this.#slot(key));
+    return data?.value as V;
+  }
+
+  async *getValuesAsync(key: K): AsyncIterableIterator<V> {
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.#container, this.#normalizeKey(key), 0],
+      [this.#container, this.#normalizeKey(key), Number.MAX_SAFE_INTEGER],
+      false,
+      false,
+    );
+    for await (const cursor of index.iterate(rangeQuery)) {
+      yield cursor.value.value as V;
+    }
+  }
+
+  async hasAsync(key: K): Promise<boolean> {
+    const result = (await this.getAsync(key)) !== undefined;
+    return result;
+  }
+
+  async set(key: K, val: V): Promise<void> {
+    const count = await this.db
+      .index('key')
+      .count(IDBKeyRange.bound([this.#container, this.#normalizeKey(key)], [this.#container, this.#normalizeKey(key)]));
+    await this.db.put({
+      value: val,
+      container: this.#container,
+      key: this.#normalizeKey(key),
+      keyCount: count + 1,
+      slot: this.#slot(key, count),
+    });
+  }
+
+  swap(_key: K, _fn: (val: V | undefined) => V): Promise<void> {
+    throw new Error('Not implemented');
+  }
+
+  async setIfNotExists(key: K, val: V): Promise<boolean> {
+    if (!(await this.hasAsync(key))) {
+      await this.set(key, val);
+      return true;
+    }
+    return false;
+  }
+
+  async delete(key: K): Promise<void> {
+    await this.db.delete(this.#slot(key));
+  }
+
+  async deleteValue(key: K, val: V): Promise<void> {
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.#container, this.#normalizeKey(key), 0],
+      [this.#container, this.#normalizeKey(key), Number.MAX_SAFE_INTEGER],
+      false,
+      false,
+    );
+    for await (const cursor of index.iterate(rangeQuery)) {
+      if (JSON.stringify(cursor.value.value) === JSON.stringify(val)) {
+        await cursor.delete();
+        return;
+      }
+    }
+  }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, V]> {
+    const index = this.db.index('key');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.#container, range.start ?? ''],
+      [this.#container, range.end ?? '\uffff'],
+      !!range.reverse,
+      !range.reverse,
+    );
+    let count = 0;
+    for await (const cursor of index.iterate(rangeQuery, range.reverse ? 'prev' : 'next')) {
+      if (range.limit && count >= range.limit) {
+        return;
+      }
+      yield [cursor.value.key, cursor.value.value] as [K, V];
+      count++;
+    }
+  }
+
+  async *valuesAsync(range: Range<K> = {}): AsyncIterableIterator<V> {
+    for await (const [_, value] of this.entriesAsync(range)) {
+      yield value;
+    }
+  }
+
+  async *keysAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    for await (const [key, _] of this.entriesAsync(range)) {
+      yield this.#denormalizeKey(key as string);
+    }
+  }
+
+  #denormalizeKey(key: string): K {
+    const denormalizedKey = (key as string).split(',').map(part => (isNaN(parseInt(part)) ? part : parseInt(part)));
+    return (denormalizedKey.length > 1 ? denormalizedKey : key) as K;
+  }
+
+  #normalizeKey(key: K): string {
+    const arrayKey = Array.isArray(key) ? key : [key];
+    return arrayKey.join(',');
+  }
+
+  #slot(key: K, index: number = 0): string {
+    return `map:${this.name}:slot:${this.#normalizeKey(key)}:${index}`;
+  }
+}

package/src/indexeddb/set.ts

@@ -0,0 +1,37 @@
+import { type IDBPDatabase, type IDBPObjectStore } from 'idb';
+
+import { type Key, type Range } from '../interfaces/common.js';
+import { type AztecAsyncSet } from '../interfaces/set.js';
+import { IndexedDBAztecMap } from './map.js';
+import { type AztecIDBSchema } from './store.js';
+
+/**
+ * A set backed by IndexedDB.
+ */
+export class IndexedDBAztecSet<K extends Key> implements AztecAsyncSet<K> {
+  private map: IndexedDBAztecMap<K, boolean>;
+
+  constructor(rootDb: IDBPDatabase<AztecIDBSchema>, mapName: string) {
+    this.map = new IndexedDBAztecMap(rootDb, mapName);
+  }
+
+  set db(db: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> | undefined) {
+    this.map.db = db;
+  }
+
+  hasAsync(key: K): Promise<boolean> {
+    return this.map.hasAsync(key);
+  }
+
+  add(key: K): Promise<void> {
+    return this.map.set(key, true);
+  }
+
+  delete(key: K): Promise<void> {
+    return this.map.delete(key);
+  }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    yield* this.map.keysAsync(range);
+  }
+}

package/src/indexeddb/singleton.ts

@@ -0,0 +1,49 @@
+import { type IDBPDatabase, type IDBPObjectStore } from 'idb';
+
+import { type AztecAsyncSingleton } from '../interfaces/singleton.js';
+import { type AztecIDBSchema } from './store.js';
+
+/**
+ * Stores a single value in IndexedDB.
+ */
+export class IndexedDBAztecSingleton<T> implements AztecAsyncSingleton<T> {
+  #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
+  #rootDB: IDBPDatabase<AztecIDBSchema>;
+  #container: string;
+  #slot: string;
+
+  constructor(rootDB: IDBPDatabase<AztecIDBSchema>, name: string) {
+    this.#rootDB = rootDB;
+    this.#container = `singleton:${name}`;
+    this.#slot = `singleton:${name}:value`;
+  }
+
+  set db(db: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> | undefined) {
+    this.#_db = db;
+  }
+
+  get db(): IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> {
+    return this.#_db ? this.#_db : this.#rootDB.transaction('data', 'readwrite').store;
+  }
+
+  async getAsync(): Promise<T | undefined> {
+    const data = await this.db.get(this.#slot);
+    return data?.value as T;
+  }
+
+  async set(val: T): Promise<boolean> {
+    const result = await this.db.put({
+      container: this.#container,
+      slot: this.#slot,
+      key: this.#slot,
+      keyCount: 1,
+      value: val,
+    });
+    return result !== undefined;
+  }
+
+  async delete(): Promise<boolean> {
+    await this.db.delete(this.#slot);
+    return true;
+  }
+}

package/src/indexeddb/store.ts

@@ -0,0 +1,192 @@
+import { type Logger } from '@aztec/foundation/log';
+
+import { type DBSchema, type IDBPDatabase, openDB } from 'idb';
+
+import { type AztecAsyncArray } from '../interfaces/array.js';
+import { type Key } from '../interfaces/common.js';
+import { type AztecAsyncCounter } from '../interfaces/counter.js';
+import { type AztecAsyncMap, type AztecAsyncMultiMap } from '../interfaces/map.js';
+import { type AztecAsyncSet } from '../interfaces/set.js';
+import { type AztecAsyncSingleton } from '../interfaces/singleton.js';
+import { type AztecAsyncKVStore } from '../interfaces/store.js';
+import { IndexedDBAztecArray } from './array.js';
+import { IndexedDBAztecMap } from './map.js';
+import { IndexedDBAztecSet } from './set.js';
+import { IndexedDBAztecSingleton } from './singleton.js';
+
+export type StoredData<V> = { value: V; container: string; key: string; keyCount: number; slot: string };
+
+export interface AztecIDBSchema extends DBSchema {
+  data: {
+    value: StoredData<any>;
+    key: string;
+    indexes: { container: string; key: string; keyCount: number };
+  };
+}
+
+/**
+ * A key-value store backed by IndexedDB.
+ */
+
+export class AztecIndexedDBStore implements AztecAsyncKVStore {
+  #log: Logger;
+  #rootDB: IDBPDatabase<AztecIDBSchema>;
+  #name: string;
+
+  #containers = new Set<
+    IndexedDBAztecArray<any> | IndexedDBAztecMap<any, any> | IndexedDBAztecSet<any> | IndexedDBAztecSingleton<any>
+  >();
+
+  constructor(rootDB: IDBPDatabase<AztecIDBSchema>, public readonly isEphemeral: boolean, log: Logger, name: string) {
+    this.#rootDB = rootDB;
+    this.#log = log;
+    this.#name = name;
+  }
+  /**
+   * Creates a new AztecKVStore backed by IndexedDB. The path to the database is optional. If not provided,
+   * the database will be stored in a temporary location and be deleted when the process exists.
+   *
+   *
+   * @param path - A path on the disk to store the database. Optional
+   * @param ephemeral - true if the store should only exist in memory and not automatically be flushed to disk. Optional
+   * @param log - A logger to use. Optional
+   * @returns The store
+   */
+  static async open(log: Logger, name?: string, ephemeral: boolean = false): Promise<AztecIndexedDBStore> {
+    name = name && !ephemeral ? name : self.crypto.getRandomValues(new Uint8Array(16)).join('');
+    log.debug(`Opening IndexedDB ${ephemeral ? 'temp ' : ''}database with name ${name}`);
+    const rootDB = await openDB<AztecIDBSchema>(name, 1, {
+      upgrade(db) {
+        const objectStore = db.createObjectStore('data', { keyPath: 'slot' });
+
+        objectStore.createIndex('key', ['container', 'key'], { unique: false });
+        objectStore.createIndex('keyCount', ['container', 'key', 'keyCount'], { unique: false });
+      },
+    });
+
+    const kvStore = new AztecIndexedDBStore(rootDB, ephemeral, log, name);
+    return kvStore;
+  }
+
+  /**
+   * Forks the current DB into a new DB by backing it up to a temporary location and opening a new indexedb.
+   * @returns A new AztecIndexedDBStore.
+   */
+  async fork(): Promise<AztecAsyncKVStore> {
+    const forkedStore = await AztecIndexedDBStore.open(this.#log, undefined, true);
+    this.#log.verbose(`Forking store to ${forkedStore.#name}`);
+
+    // Copy old data to new store
+    const oldData = this.#rootDB.transaction('data').store;
+    const dataToWrite = [];
+    for await (const cursor of oldData.iterate()) {
+      dataToWrite.push(cursor.value);
+    }
+    const tx = forkedStore.#rootDB.transaction('data', 'readwrite').store;
+    for (const data of dataToWrite) {
+      await tx.add(data);
+    }
+
+    this.#log.debug(`Forked store at ${forkedStore.#name} opened successfully`);
+    return forkedStore;
+  }
+
+  /**
+   * Creates a new AztecMap in the store.
+   * @param name - Name of the map
+   * @returns A new AztecMap
+   */
+  openMap<K extends Key, V>(name: string): AztecAsyncMap<K, V> {
+    const map = new IndexedDBAztecMap<K, V>(this.#rootDB, name);
+    this.#containers.add(map);
+    return map;
+  }
+
+  /**
+   * Creates a new AztecSet in the store.
+   * @param name - Name of the set
+   * @returns A new AztecSet
+   */
+  openSet<K extends Key>(name: string): AztecAsyncSet<K> {
+    const set = new IndexedDBAztecSet<K>(this.#rootDB, name);
+    this.#containers.add(set);
+    return set;
+  }
+
+  /**
+   * Creates a new AztecMultiMap in the store. A multi-map stores multiple values for a single key automatically.
+   * @param name - Name of the map
+   * @returns A new AztecMultiMap
+   */
+  openMultiMap<K extends Key, V>(name: string): AztecAsyncMultiMap<K, V> {
+    const multimap = new IndexedDBAztecMap<K, V>(this.#rootDB, name);
+    this.#containers.add(multimap);
+    return multimap;
+  }
+
+  openCounter<K extends Key | Array<string | number>>(_name: string): AztecAsyncCounter<K> {
+    throw new Error('Method not implemented.');
+  }
+
+  /**
+   * Creates a new AztecArray in the store.
+   * @param name - Name of the array
+   * @returns A new AztecArray
+   */
+  openArray<T>(name: string): AztecAsyncArray<T> {
+    const array = new IndexedDBAztecArray<T>(this.#rootDB, name);
+    this.#containers.add(array);
+    return array;
+  }
+
+  /**
+   * Creates a new AztecSingleton in the store.
+   * @param name - Name of the singleton
+   * @returns A new AztecSingleton
+   */
+  openSingleton<T>(name: string): AztecAsyncSingleton<T> {
+    const singleton = new IndexedDBAztecSingleton<T>(this.#rootDB, name);
+    this.#containers.add(singleton);
+    return singleton;
+  }
+
+  /**
+   * Runs a callback in a transaction.
+   * @param callback - Function to execute in a transaction
+   * @returns A promise that resolves to the return value of the callback
+   */
+  async transactionAsync<T>(callback: () => Promise<T>): Promise<T> {
+    const tx = this.#rootDB.transaction('data', 'readwrite');
+    for (const container of this.#containers) {
+      container.db = tx.store;
+    }
+    // Avoid awaiting this promise so it doesn't get scheduled in the next microtask
+    // By then, the tx would be closed
+    const runningPromise = callback();
+    // Wait for the transaction to finish
+    await tx.done;
+    for (const container of this.#containers) {
+      container.db = undefined;
+    }
+    // Return the result of the callback.
+    // Tx is guaranteed to already be closed, so the await doesn't hurt anything here
+    return await runningPromise;
+  }
+
+  /**
+   * Clears all entries in the store & sub DBs.
+   */
+  async clear() {
+    await this.#rootDB.transaction('data', 'readwrite').store.clear();
+  }
+
+  /** Deletes this store and removes the database */
+  delete() {
+    this.#containers.clear();
+    return Promise.resolve(this.#rootDB.deleteObjectStore('data'));
+  }
+
+  estimateSize(): { mappingSize: number; actualSize: number; numItems: number } {
+    return { mappingSize: 0, actualSize: 0, numItems: 0 };
+  }
+}

package/src/interfaces/array.ts

@@ -1,12 +1,7 @@
 /**
  * An array backed by a persistent store. Can not have any holes in it.
  */
-export interface AztecArray<T> {
-  /**
-   * The size of the array
-   */
-  length: number;
-
+interface BaseAztecArray<T> {
   /**
    * Pushes values to the end of the array
    * @param vals - The values to push to the end of the array
@@ -20,6 +15,24 @@ export interface AztecArray<T> {
    */
   pop(): Promise<T | undefined>;
 
+  /**
+   * Updates the value at the given index. Index can be in the range [-length, length - 1).
+   * @param index - The index to set the value at
+   * @param val - The value to set
+   * @returns Whether the value was set
+   */
+  setAt(index: number, val: T): Promise<boolean>;
+}
+
+/**
+ * An array backed by a persistent store. Can not have any holes in it.
+ */
+export interface AztecAsyncArray<T> extends BaseAztecArray<T> {
+  /**
+   * The size of the array
+   */
+  lengthAsync(): Promise<number>;
+
   /**
    * Gets the value at the given index. Index can be in the range [-length, length - 1).
    * If the index is negative, it will be treated as an offset from the end of the array.
@@ -27,15 +40,38 @@ export interface AztecArray<T> {
    * @param index - The index to get the value from
    * @returns The value at the given index or undefined if the index is out of bounds
    */
-  at(index: number): T | undefined;
+  atAsync(index: number): Promise<T | undefined>;
 
   /**
-   * Updates the value at the given index. Index can be in the range [-length, length - 1).
-   * @param index - The index to set the value at
-   * @param val - The value to set
-   * @returns Whether the value was set
+   * Iterates over the array with indexes.
    */
-  setAt(index: number, val: T): Promise<boolean>;
+  entriesAsync(): AsyncIterableIterator<[number, T]>;
+
+  /**
+   * Iterates over the array.
+   */
+  valuesAsync(): AsyncIterableIterator<T>;
+
+  /**
+   * Iterates over the array.
+   */
+  [Symbol.asyncIterator](): AsyncIterableIterator<T>;
+}
+
+export interface AztecArray<T> extends BaseAztecArray<T> {
+  /**
+   * The size of the array
+   */
+  length: number;
+
+  /**
+   * Gets the value at the given index. Index can be in the range [-length, length - 1).
+   * If the index is negative, it will be treated as an offset from the end of the array.
+   *
+   * @param index - The index to get the value from
+   * @returns The value at the given index or undefined if the index is out of bounds
+   */
+  at(index: number): T | undefined;
 
   /**
    * Iterates over the array with indexes.

package/src/interfaces/array_test_suite.ts

@@ -0,0 +1,126 @@
+import { toArray } from '@aztec/foundation/iterable';
+
+import { expect } from 'chai';
+
+import { type AztecArray, type AztecAsyncArray } from './array.js';
+import { type AztecAsyncKVStore, type AztecKVStore } from './store.js';
+import { isSyncStore } from './utils.js';
+
+export function describeAztecArray(
+  testName: string,
+  getStore: () => AztecKVStore | Promise<AztecAsyncKVStore>,
+  forceAsync: boolean = false,
+) {
+  describe(testName, () => {
+    let store: AztecKVStore | AztecAsyncKVStore;
+    let arr: AztecArray<number> | AztecAsyncArray<number>;
+
+    beforeEach(async () => {
+      store = await getStore();
+      arr = store.openArray<number>('test');
+    });
+
+    async function length(sut: AztecAsyncArray<number> | AztecArray<number> = arr) {
+      return isSyncStore(store) && !forceAsync
+        ? (sut as AztecArray<number>).length
+        : await (sut as AztecAsyncArray<number>).lengthAsync();
+    }
+
+    async function at(index: number) {
+      return isSyncStore(store) && !forceAsync
+        ? (arr as AztecArray<number>).at(index)
+        : await (arr as AztecAsyncArray<number>).atAsync(index);
+    }
+
+    async function entries() {
+      return isSyncStore(store) && !forceAsync
+        ? await toArray((arr as AztecArray<number>).entries())
+        : await toArray((arr as AztecAsyncArray<number>).entriesAsync());
+    }
+
+    async function values(sut: AztecAsyncArray<number> | AztecArray<number> = arr) {
+      return isSyncStore(store) && !forceAsync
+        ? await toArray((sut as AztecArray<number>).values())
+        : await toArray((sut as AztecAsyncArray<number>).valuesAsync());
+    }
+
+    it('should be able to push and pop values', async () => {
+      await arr.push(1);
+      await arr.push(2);
+      await arr.push(3);
+
+      expect(await length()).to.equal(3);
+
+      expect(await arr.pop()).to.equal(3);
+      expect(await arr.pop()).to.equal(2);
+      expect(await arr.pop()).to.equal(1);
+      expect(await arr.pop()).to.equal(undefined);
+    });
+
+    it('should be able to get values by index', async () => {
+      await arr.push(1);
+      await arr.push(2);
+      await arr.push(3);
+
+      expect(await at(0)).to.equal(1);
+      expect(await at(1)).to.equal(2);
+      expect(await at(2)).to.equal(3);
+      expect(await at(3)).to.equal(undefined);
+      expect(await at(-1)).to.equal(3);
+      expect(await at(-2)).to.equal(2);
+      expect(await at(-3)).to.equal(1);
+      expect(await at(-4)).to.equal(undefined);
+    });
+
+    it('should be able to set values by index', async () => {
+      await arr.push(1);
+      await arr.push(2);
+      await arr.push(3);
+
+      expect(await arr.setAt(0, 4)).to.equal(true);
+      expect(await arr.setAt(1, 5)).to.equal(true);
+      expect(await arr.setAt(2, 6)).to.equal(true);
+
+      expect(await arr.setAt(3, 7)).to.equal(false);
+
+      expect(await at(0)).to.equal(4);
+      expect(await at(1)).to.equal(5);
+      expect(await at(2)).to.equal(6);
+      expect(await at(3)).to.equal(undefined);
+
+      expect(await arr.setAt(-1, 8)).to.equal(true);
+      expect(await arr.setAt(-2, 9)).to.equal(true);
+      expect(await arr.setAt(-3, 10)).to.equal(true);
+
+      expect(await arr.setAt(-4, 11)).to.equal(false);
+
+      expect(await at(-1)).to.equal(8);
+      expect(await at(-2)).to.equal(9);
+      expect(await at(-3)).to.equal(10);
+      expect(await at(-4)).to.equal(undefined);
+    });
+
+    it('should be able to iterate over values', async () => {
+      await arr.push(1);
+      await arr.push(2);
+      await arr.push(3);
+
+      expect(await values()).to.deep.equal([1, 2, 3]);
+      expect(await entries()).to.deep.equal([
+        [0, 1],
+        [1, 2],
+        [2, 3],
+      ]);
+    });
+
+    it('should be able to restore state', async () => {
+      await arr.push(1);
+      await arr.push(2);
+      await arr.push(3);
+
+      const arr2 = store.openArray<number>('test');
+      expect(await length(arr2)).to.equal(3);
+      expect(await values(arr2)).to.deep.equal(await values());
+    });
+  });
+}

package/src/interfaces/counter.ts

@@ -6,7 +6,7 @@ import { type Key, type Range } from './common.js';
  *
  * Keys are stored in sorted order
  */
-export interface AztecCounter<K extends Key = Key> {
+interface AztecBaseCounter<K extends Key = Key> {
   /**
    * Resets the count of the given key to the given value.
    * @param key - The key to reset
@@ -22,7 +22,9 @@ export interface AztecCounter<K extends Key = Key> {
    * @param delta - The amount to modify the key by
    */
   update(key: K, delta: number): Promise<void>;
+}
 
+export interface AztecCounter<K extends Key = Key> extends AztecBaseCounter<K> {
   /**
    * Gets the current count.
    * @param key - The key to get the count of
@@ -41,3 +43,23 @@ export interface AztecCounter<K extends Key = Key> {
    */
   entries(range: Range<K>): IterableIterator<[K, number]>;
 }
+
+export interface AztecAsyncCounter<K extends Key = Key> extends AztecBaseCounter<K> {
+  /**
+   * Gets the current count.
+   * @param key - The key to get the count of
+   */
+  getAsync(key: K): Promise<number>;
+
+  /**
+   * Returns keys in the map in sorted order. Only returns keys that have been seen at least once.
+   * @param range - The range of keys to iterate over
+   */
+  keysAsync(range: Range<K>): AsyncIterableIterator<K>;
+
+  /**
+   * Returns keys and their counts in the map sorted by the key. Only returns keys that have been seen at least once.
+   * @param range - The range of keys to iterate over
+   */
+  entriesAsync(range: Range<K>): AsyncIterableIterator<[K, number]>;
+}