@aztec/kv-store 0.0.0-test.0

package/src/indexeddb/store.ts

@@ -0,0 +1,197 @@
+import type { Logger } from '@aztec/foundation/log';
+
+import { type DBSchema, type IDBPDatabase, deleteDB, openDB } from 'idb';
+
+import type { AztecAsyncArray } from '../interfaces/array.js';
+import type { Key, StoreSize } from '../interfaces/common.js';
+import type { AztecAsyncCounter } from '../interfaces/counter.js';
+import type { AztecAsyncMap, AztecAsyncMultiMap } from '../interfaces/map.js';
+import type { AztecAsyncSet } from '../interfaces/set.js';
+import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
+import type { AztecAsyncKVStore } from '../interfaces/store.js';
+import { IndexedDBAztecArray } from './array.js';
+import { IndexedDBAztecMap } from './map.js';
+import { IndexedDBAztecSet } from './set.js';
+import { IndexedDBAztecSingleton } from './singleton.js';
+
+export type StoredData<V> = { value: V; container: string; key: string; keyCount: number; slot: string };
+
+export interface AztecIDBSchema extends DBSchema {
+  data: {
+    value: StoredData<any>;
+    key: string;
+    indexes: { container: string; key: string; keyCount: number };
+  };
+}
+
+/**
+ * A key-value store backed by IndexedDB.
+ */
+
+export class AztecIndexedDBStore implements AztecAsyncKVStore {
+  #log: Logger;
+  #rootDB: IDBPDatabase<AztecIDBSchema>;
+  #name: string;
+
+  #containers = new Set<
+    IndexedDBAztecArray<any> | IndexedDBAztecMap<any, any> | IndexedDBAztecSet<any> | IndexedDBAztecSingleton<any>
+  >();
+
+  constructor(rootDB: IDBPDatabase<AztecIDBSchema>, public readonly isEphemeral: boolean, log: Logger, name: string) {
+    this.#rootDB = rootDB;
+    this.#log = log;
+    this.#name = name;
+  }
+  /**
+   * Creates a new AztecKVStore backed by IndexedDB. The path to the database is optional. If not provided,
+   * the database will be stored in a temporary location and be deleted when the process exists.
+   *
+   *
+   * @param path - A path on the disk to store the database. Optional
+   * @param ephemeral - true if the store should only exist in memory and not automatically be flushed to disk. Optional
+   * @param log - A logger to use. Optional
+   * @returns The store
+   */
+  static async open(log: Logger, name?: string, ephemeral: boolean = false): Promise<AztecIndexedDBStore> {
+    name = name && !ephemeral ? name : self.crypto.getRandomValues(new Uint8Array(16)).join('');
+    log.debug(`Opening IndexedDB ${ephemeral ? 'temp ' : ''}database with name ${name}`);
+    const rootDB = await openDB<AztecIDBSchema>(name, 1, {
+      upgrade(db) {
+        const objectStore = db.createObjectStore('data', { keyPath: 'slot' });
+
+        objectStore.createIndex('key', ['container', 'key'], { unique: false });
+        objectStore.createIndex('keyCount', ['container', 'key', 'keyCount'], { unique: false });
+      },
+    });
+
+    const kvStore = new AztecIndexedDBStore(rootDB, ephemeral, log, name);
+    return kvStore;
+  }
+
+  /**
+   * Forks the current DB into a new DB by backing it up to a temporary location and opening a new indexedb.
+   * @returns A new AztecIndexedDBStore.
+   */
+  async fork(): Promise<AztecAsyncKVStore> {
+    const forkedStore = await AztecIndexedDBStore.open(this.#log, undefined, true);
+    this.#log.verbose(`Forking store to ${forkedStore.#name}`);
+
+    // Copy old data to new store
+    const oldData = this.#rootDB.transaction('data').store;
+    const dataToWrite = [];
+    for await (const cursor of oldData.iterate()) {
+      dataToWrite.push(cursor.value);
+    }
+    const tx = forkedStore.#rootDB.transaction('data', 'readwrite').store;
+    for (const data of dataToWrite) {
+      await tx.add(data);
+    }
+
+    this.#log.debug(`Forked store at ${forkedStore.#name} opened successfully`);
+    return forkedStore;
+  }
+
+  /**
+   * Creates a new AztecMap in the store.
+   * @param name - Name of the map
+   * @returns A new AztecMap
+   */
+  openMap<K extends Key, V>(name: string): AztecAsyncMap<K, V> {
+    const map = new IndexedDBAztecMap<K, V>(this.#rootDB, name);
+    this.#containers.add(map);
+    return map;
+  }
+
+  /**
+   * Creates a new AztecSet in the store.
+   * @param name - Name of the set
+   * @returns A new AztecSet
+   */
+  openSet<K extends Key>(name: string): AztecAsyncSet<K> {
+    const set = new IndexedDBAztecSet<K>(this.#rootDB, name);
+    this.#containers.add(set);
+    return set;
+  }
+
+  /**
+   * Creates a new AztecMultiMap in the store. A multi-map stores multiple values for a single key automatically.
+   * @param name - Name of the map
+   * @returns A new AztecMultiMap
+   */
+  openMultiMap<K extends Key, V>(name: string): AztecAsyncMultiMap<K, V> {
+    const multimap = new IndexedDBAztecMap<K, V>(this.#rootDB, name);
+    this.#containers.add(multimap);
+    return multimap;
+  }
+
+  openCounter<K extends Key>(_name: string): AztecAsyncCounter<K> {
+    throw new Error('Method not implemented.');
+  }
+
+  /**
+   * Creates a new AztecArray in the store.
+   * @param name - Name of the array
+   * @returns A new AztecArray
+   */
+  openArray<T>(name: string): AztecAsyncArray<T> {
+    const array = new IndexedDBAztecArray<T>(this.#rootDB, name);
+    this.#containers.add(array);
+    return array;
+  }
+
+  /**
+   * Creates a new AztecSingleton in the store.
+   * @param name - Name of the singleton
+   * @returns A new AztecSingleton
+   */
+  openSingleton<T>(name: string): AztecAsyncSingleton<T> {
+    const singleton = new IndexedDBAztecSingleton<T>(this.#rootDB, name);
+    this.#containers.add(singleton);
+    return singleton;
+  }
+
+  /**
+   * Runs a callback in a transaction.
+   * @param callback - Function to execute in a transaction
+   * @returns A promise that resolves to the return value of the callback
+   */
+  async transactionAsync<T>(callback: () => Promise<T>): Promise<T> {
+    const tx = this.#rootDB.transaction('data', 'readwrite');
+    for (const container of this.#containers) {
+      container.db = tx.store;
+    }
+    // Avoid awaiting this promise so it doesn't get scheduled in the next microtask
+    // By then, the tx would be closed
+    const runningPromise = callback();
+    // Wait for the transaction to finish
+    await tx.done;
+    for (const container of this.#containers) {
+      container.db = undefined;
+    }
+    // Return the result of the callback.
+    // Tx is guaranteed to already be closed, so the await doesn't hurt anything here
+    return await runningPromise;
+  }
+
+  /**
+   * Clears all entries in the store & sub DBs.
+   */
+  async clear() {
+    await this.#rootDB.transaction('data', 'readwrite').store.clear();
+  }
+
+  /** Deletes this store and removes the database */
+  delete() {
+    this.#containers.clear();
+    this.#rootDB.close();
+    return deleteDB(this.#name);
+  }
+
+  estimateSize(): Promise<StoreSize> {
+    return Promise.resolve({ mappingSize: 0, actualSize: 0, numItems: 0 });
+  }
+
+  close(): Promise<void> {
+    return Promise.resolve();
+  }
+}

package/src/interfaces/array.ts

@@ -0,0 +1,90 @@
+/**
+ * An array backed by a persistent store. Can not have any holes in it.
+ */
+interface BaseAztecArray<T> {
+  /**
+   * Pushes values to the end of the array
+   * @param vals - The values to push to the end of the array
+   * @returns The new length of the array
+   */
+  push(...vals: T[]): Promise<number>;
+
+  /**
+   * Pops a value from the end of the array.
+   * @returns The value that was popped, or undefined if the array was empty
+   */
+  pop(): Promise<T | undefined>;
+
+  /**
+   * Updates the value at the given index. Index can be in the range [-length, length - 1).
+   * @param index - The index to set the value at
+   * @param val - The value to set
+   * @returns Whether the value was set
+   */
+  setAt(index: number, val: T): Promise<boolean>;
+}
+
+/**
+ * An array backed by a persistent store. Can not have any holes in it.
+ */
+export interface AztecAsyncArray<T> extends BaseAztecArray<T> {
+  /**
+   * The size of the array
+   */
+  lengthAsync(): Promise<number>;
+
+  /**
+   * Gets the value at the given index. Index can be in the range [-length, length - 1).
+   * If the index is negative, it will be treated as an offset from the end of the array.
+   *
+   * @param index - The index to get the value from
+   * @returns The value at the given index or undefined if the index is out of bounds
+   */
+  atAsync(index: number): Promise<T | undefined>;
+
+  /**
+   * Iterates over the array with indexes.
+   */
+  entriesAsync(): AsyncIterableIterator<[number, T]>;
+
+  /**
+   * Iterates over the array.
+   */
+  valuesAsync(): AsyncIterableIterator<T>;
+
+  /**
+   * Iterates over the array.
+   */
+  [Symbol.asyncIterator](): AsyncIterableIterator<T>;
+}
+
+export interface AztecArray<T> extends BaseAztecArray<T> {
+  /**
+   * The size of the array
+   */
+  length: number;
+
+  /**
+   * Gets the value at the given index. Index can be in the range [-length, length - 1).
+   * If the index is negative, it will be treated as an offset from the end of the array.
+   *
+   * @param index - The index to get the value from
+   * @returns The value at the given index or undefined if the index is out of bounds
+   */
+  at(index: number): T | undefined;
+
+  /**
+   * Iterates over the array with indexes.
+   */
+  entries(): IterableIterator<[number, T]>;
+
+  /**
+   * Iterates over the array.
+   */
+  values(): IterableIterator<T>;
+
+  /**
+   * Iterates over the array.
+   */
+  [Symbol.iterator](): IterableIterator<T>;
+}

package/src/interfaces/array_test_suite.ts

@@ -0,0 +1,130 @@
+import { toArray } from '@aztec/foundation/iterable';
+
+import { expect } from 'chai';
+
+import type { AztecArray, AztecAsyncArray } from './array.js';
+import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
+import { isSyncStore } from './utils.js';
+
+export function describeAztecArray(
+  testName: string,
+  getStore: () => AztecKVStore | Promise<AztecAsyncKVStore>,
+  forceAsync: boolean = false,
+) {
+  describe(testName, () => {
+    let store: AztecKVStore | AztecAsyncKVStore;
+    let arr: AztecArray<number> | AztecAsyncArray<number>;
+
+    beforeEach(async () => {
+      store = await getStore();
+      arr = store.openArray<number>('test');
+    });
+
+    afterEach(async () => {
+      await store.delete();
+    });
+
+    async function length(sut: AztecAsyncArray<number> | AztecArray<number> = arr) {
+      return isSyncStore(store) && !forceAsync
+        ? (sut as AztecArray<number>).length
+        : await (sut as AztecAsyncArray<number>).lengthAsync();
+    }
+
+    async function at(index: number) {
+      return isSyncStore(store) && !forceAsync
+        ? (arr as AztecArray<number>).at(index)
+        : await (arr as AztecAsyncArray<number>).atAsync(index);
+    }
+
+    async function entries() {
+      return isSyncStore(store) && !forceAsync
+        ? await toArray((arr as AztecArray<number>).entries())
+        : await toArray((arr as AztecAsyncArray<number>).entriesAsync());
+    }
+
+    async function values(sut: AztecAsyncArray<number> | AztecArray<number> = arr) {
+      return isSyncStore(store) && !forceAsync
+        ? await toArray((sut as AztecArray<number>).values())
+        : await toArray((sut as AztecAsyncArray<number>).valuesAsync());
+    }
+
+    it('should be able to push and pop values', async () => {
+      await arr.push(1);
+      await arr.push(2);
+      await arr.push(3);
+
+      expect(await length()).to.equal(3);
+
+      expect(await arr.pop()).to.equal(3);
+      expect(await arr.pop()).to.equal(2);
+      expect(await arr.pop()).to.equal(1);
+      expect(await arr.pop()).to.equal(undefined);
+    });
+
+    it('should be able to get values by index', async () => {
+      await arr.push(1);
+      await arr.push(2);
+      await arr.push(3);
+
+      expect(await at(0)).to.equal(1);
+      expect(await at(1)).to.equal(2);
+      expect(await at(2)).to.equal(3);
+      expect(await at(3)).to.equal(undefined);
+      expect(await at(-1)).to.equal(3);
+      expect(await at(-2)).to.equal(2);
+      expect(await at(-3)).to.equal(1);
+      expect(await at(-4)).to.equal(undefined);
+    });
+
+    it('should be able to set values by index', async () => {
+      await arr.push(1);
+      await arr.push(2);
+      await arr.push(3);
+
+      expect(await arr.setAt(0, 4)).to.equal(true);
+      expect(await arr.setAt(1, 5)).to.equal(true);
+      expect(await arr.setAt(2, 6)).to.equal(true);
+
+      expect(await arr.setAt(3, 7)).to.equal(false);
+
+      expect(await at(0)).to.equal(4);
+      expect(await at(1)).to.equal(5);
+      expect(await at(2)).to.equal(6);
+      expect(await at(3)).to.equal(undefined);
+
+      expect(await arr.setAt(-1, 8)).to.equal(true);
+      expect(await arr.setAt(-2, 9)).to.equal(true);
+      expect(await arr.setAt(-3, 10)).to.equal(true);
+
+      expect(await arr.setAt(-4, 11)).to.equal(false);
+
+      expect(await at(-1)).to.equal(8);
+      expect(await at(-2)).to.equal(9);
+      expect(await at(-3)).to.equal(10);
+      expect(await at(-4)).to.equal(undefined);
+    });
+
+    it('should be able to iterate over values', async () => {
+      await arr.push(1);
+      await arr.push(2);
+      await arr.push(3);
+
+      expect(await values()).to.deep.equal([1, 2, 3]);
+      expect(await entries()).to.deep.equal([
+        [0, 1],
+        [1, 2],
+        [2, 3],
+      ]);
+    });
+
+    it('should be able to restore state', async () => {
+      await arr.push(1);
+      await arr.push(2);
+      await arr.push(3);
+
+      const arr2 = store.openArray<number>('test');
+      expect(await length(arr2)).to.equal(3);
+      expect(await values(arr2)).to.deep.equal(await values());
+    });
+  });
+}

package/src/interfaces/common.ts

@@ -0,0 +1,20 @@
+/**
+ * The key type for use with the kv-store
+ */
+export type Key = string | number | Array<string | number>;
+
+/**
+ * A range of keys to iterate over.
+ */
+export type Range<K extends Key = Key> = {
+  /** The key of the first item to include */
+  start?: K;
+  /** The key of the last item to include */
+  end?: K;
+  /** Whether to iterate in reverse */
+  reverse?: boolean;
+  /** The maximum number of items to iterate over */
+  limit?: number;
+};
+
+export type StoreSize = { mappingSize: number; actualSize: number; numItems: number };

package/src/interfaces/counter.ts

@@ -0,0 +1,65 @@
+import type { Key, Range } from './common.js';
+
+/**
+ * A map that counts how many times it sees a key. Once 0 is reached,  that key is removed from the map.
+ * Iterating over the map will only return keys that have a count over 0.
+ *
+ * Keys are stored in sorted order
+ */
+interface AztecBaseCounter<K extends Key = Key> {
+  /**
+   * Resets the count of the given key to the given value.
+   * @param key - The key to reset
+   * @param value - The value to reset the key to
+   */
+  set(key: K, value: number): Promise<void>;
+
+  /**
+   * Updates the count of the given key by the given delta. This can be used to increment or decrement the count.
+   * Once a key's count reaches 0, it is removed from the map.
+   *
+   * @param key - The key to update
+   * @param delta - The amount to modify the key by
+   */
+  update(key: K, delta: number): Promise<void>;
+}
+
+export interface AztecCounter<K extends Key = Key> extends AztecBaseCounter<K> {
+  /**
+   * Gets the current count.
+   * @param key - The key to get the count of
+   */
+  get(key: K): number;
+
+  /**
+   * Returns keys in the map in sorted order. Only returns keys that have been seen at least once.
+   * @param range - The range of keys to iterate over
+   */
+  keys(range: Range<K>): IterableIterator<K>;
+
+  /**
+   * Returns keys and their counts in the map sorted by the key. Only returns keys that have been seen at least once.
+   * @param range - The range of keys to iterate over
+   */
+  entries(range: Range<K>): IterableIterator<[K, number]>;
+}
+
+export interface AztecAsyncCounter<K extends Key = Key> extends AztecBaseCounter<K> {
+  /**
+   * Gets the current count.
+   * @param key - The key to get the count of
+   */
+  getAsync(key: K): Promise<number>;
+
+  /**
+   * Returns keys in the map in sorted order. Only returns keys that have been seen at least once.
+   * @param range - The range of keys to iterate over
+   */
+  keysAsync(range: Range<K>): AsyncIterableIterator<K>;
+
+  /**
+   * Returns keys and their counts in the map sorted by the key. Only returns keys that have been seen at least once.
+   * @param range - The range of keys to iterate over
+   */
+  entriesAsync(range: Range<K>): AsyncIterableIterator<[K, number]>;
+}

package/src/interfaces/index.ts

@@ -0,0 +1,7 @@
+export * from './array.js';
+export * from './map.js';
+export * from './counter.js';
+export * from './singleton.js';
+export * from './store.js';
+export * from './set.js';
+export type { Range, StoreSize } from './common.js';

package/src/interfaces/map.ts

@@ -0,0 +1,151 @@
+import type { Key, Range } from './common.js';
+
+/**
+ * A map backed by a persistent store.
+ */
+interface AztecBaseMap<K extends Key, V> {
+  /**
+   * Sets the value at the given key.
+   * @param key - The key to set the value at
+   * @param val - The value to set
+   */
+  set(key: K, val: V): Promise<void>;
+
+  /**
+   * Sets the value at the given key if it does not already exist.
+   * @param key - The key to set the value at
+   * @param val - The value to set
+   */
+  setIfNotExists(key: K, val: V): Promise<boolean>;
+
+  /**
+   * Deletes the value at the given key.
+   * @param key - The key to delete the value at
+   */
+  delete(key: K): Promise<void>;
+}
+export interface AztecMap<K extends Key, V> extends AztecBaseMap<K, V> {
+  /**
+   * Gets the value at the given key.
+   * @param key - The key to get the value from
+   */
+  get(key: K): V | undefined;
+
+  /**
+   * Checks if a key exists in the map.
+   * @param key - The key to check
+   * @returns True if the key exists, false otherwise
+   */
+  has(key: K): boolean;
+
+  /**
+   * Iterates over the map's key-value entries in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  entries(range?: Range<K>): IterableIterator<[K, V]>;
+
+  /**
+   * Iterates over the map's values in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  values(range?: Range<K>): IterableIterator<V>;
+
+  /**
+   * Iterates over the map's keys in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  keys(range?: Range<K>): IterableIterator<K>;
+
+  /**
+   * Clears the map.
+   */
+  clear(): Promise<void>;
+}
+
+export interface AztecMapWithSize<K extends Key, V> extends AztecMap<K, V> {
+  /**
+   * Gets the size of the map.
+   * @returns The size of the map
+   */
+  size(): number;
+}
+
+/**
+ * A map backed by a persistent store that can have multiple values for a single key.
+ */
+export interface AztecMultiMap<K extends Key, V> extends AztecMap<K, V> {
+  /**
+   * Gets all the values at the given key.
+   * @param key - The key to get the values from
+   */
+  getValues(key: K): IterableIterator<V>;
+
+  /**
+   * Deletes a specific value at the given key.
+   * @param key - The key to delete the value at
+   * @param val - The value to delete
+   */
+  deleteValue(key: K, val: V): Promise<void>;
+}
+
+export interface AztecMultiMapWithSize<K extends Key, V> extends AztecMultiMap<K, V> {
+  /**
+   * Gets the size of the map.
+   * @returns The size of the map
+   */
+  size(): number;
+}
+
+/**
+ * A map backed by a persistent store.
+ */
+export interface AztecAsyncMap<K extends Key, V> extends AztecBaseMap<K, V> {
+  /**
+   * Gets the value at the given key.
+   * @param key - The key to get the value from
+   */
+  getAsync(key: K): Promise<V | undefined>;
+
+  /**
+   * Checks if a key exists in the map.
+   * @param key - The key to check
+   * @returns True if the key exists, false otherwise
+   */
+  hasAsync(key: K): Promise<boolean>;
+
+  /**
+   * Iterates over the map's key-value entries in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]>;
+
+  /**
+   * Iterates over the map's values in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  valuesAsync(range?: Range<K>): AsyncIterableIterator<V>;
+
+  /**
+   * Iterates over the map's keys in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  keysAsync(range?: Range<K>): AsyncIterableIterator<K>;
+}
+
+/**
+ * A map backed by a persistent store that can have multiple values for a single key.
+ */
+export interface AztecAsyncMultiMap<K extends Key, V> extends AztecAsyncMap<K, V> {
+  /**
+   * Gets all the values at the given key.
+   * @param key - The key to get the values from
+   */
+  getValuesAsync(key: K): AsyncIterableIterator<V>;
+
+  /**
+   * Deletes a specific value at the given key.
+   * @param key - The key to delete the value at
+   * @param val - The value to delete
+   */
+  deleteValue(key: K, val: V): Promise<void>;
+}