@aztec/kv-store 0.0.0-test.0 → 0.0.1-commit.0208eb9

package/src/indexeddb/multi_map.ts

@@ -0,0 +1,105 @@
+import { hash } from 'ohash';
+
+import type { Key, Value } from '../interfaces/common.js';
+import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
+import { IndexedDBAztecMap } from './map.js';
+
+/**
+ * A multi map backed by IndexedDB.
+ */
+export class IndexedDBAztecMultiMap<K extends Key, V extends Value>
+  extends IndexedDBAztecMap<K, V>
+  implements AztecAsyncMultiMap<K, V>
+{
+  override async set(key: K, val: V): Promise<void> {
+    // Inserting repeated values is a no-op
+    const exists = !!(await this.db
+      .index('hash')
+      .get(
+        IDBKeyRange.bound(
+          [this.container, this.normalizeKey(key), hash(val)],
+          [this.container, this.normalizeKey(key), hash(val)],
+        ),
+      ));
+    if (exists) {
+      return;
+    }
+    // Get the maximum keyCount for the given key
+    // In order to support sparse multimaps, we cannot rely
+    // on just counting the number of entries for the key, since we would repeat slots
+    // if we delete an entry
+    // set -> container:key:0 (keyCount = 1)
+    // set -> container:key:1 (keyCount = 2)
+    // delete -> container:key:0 (keyCount = 1)
+    // set -> container:key:1 <--- already exists!
+    // Instead, we iterate in reverse order to get the last inserted entry
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.upperBound([this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER]);
+    const maxEntry = (await index.iterate(rangeQuery, 'prevunique').next()).value;
+    const count = maxEntry?.value?.keyCount ?? 0;
+    await this.db.put({
+      value: val,
+      hash: hash(val),
+      container: this.container,
+      key: this.normalizeKey(key),
+      keyCount: count + 1,
+      slot: this.slot(key, count),
+    });
+  }
+
+  async *getValuesAsync(key: K): AsyncIterableIterator<V> {
+    // Iterate over the whole range of keyCount for the given key
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.container, this.normalizeKey(key), 0],
+      [this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
+      false,
+      false,
+    );
+    for await (const cursor of index.iterate(rangeQuery)) {
+      yield cursor.value.value as V;
+    }
+  }
+
+  getValueCountAsync(key: K): Promise<number> {
+    // Count entries over the keyCount index range for this key
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.container, this.normalizeKey(key), 0],
+      [this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
+      false,
+      false,
+    );
+    return index.count(rangeQuery);
+  }
+
+  async deleteValue(key: K, val: V): Promise<void> {
+    // Since we know the value, we can hash it and directly query the "hash" index
+    // to avoid having to iterate over all the values
+    const fullKey = await this.db
+      .index('hash')
+      .getKey(
+        IDBKeyRange.bound(
+          [this.container, this.normalizeKey(key), hash(val)],
+          [this.container, this.normalizeKey(key), hash(val)],
+        ),
+      );
+    if (fullKey) {
+      await this.db.delete(fullKey);
+    }
+  }
+
+  override async delete(key: K): Promise<void> {
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.container, this.normalizeKey(key), 0],
+      [this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
+      false,
+      false,
+    );
+
+    for await (const cursor of index.iterate(rangeQuery)) {
+      await cursor.delete();
+    }
+  }
+}

package/src/indexeddb/singleton.ts

@@ -1,12 +1,14 @@
 import type { IDBPDatabase, IDBPObjectStore } from 'idb';
+import { hash } from 'ohash';
 
+import type { Value } from '../interfaces/common.js';
 import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
 import type { AztecIDBSchema } from './store.js';
 
 /**
  * Stores a single value in IndexedDB.
  */
-export class IndexedDBAztecSingleton<T> implements AztecAsyncSingleton<T> {
+export class IndexedDBAztecSingleton<T extends Value> implements AztecAsyncSingleton<T> {
   #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
   #rootDB: IDBPDatabase<AztecIDBSchema>;
   #container: string;
@@ -35,9 +37,10 @@ export class IndexedDBAztecSingleton<T> implements AztecAsyncSingleton<T> {
     const result = await this.db.put({
       container: this.#container,
       slot: this.#slot,
-      key: this.#slot,
+      key: [this.#slot],
       keyCount: 1,
       value: val,
+      hash: hash(val),
     });
     return result !== undefined;
   }

package/src/indexeddb/store.ts

@@ -1,26 +1,41 @@
 import type { Logger } from '@aztec/foundation/log';
+import { SerialQueue } from '@aztec/foundation/queue';
 
 import { type DBSchema, type IDBPDatabase, deleteDB, openDB } from 'idb';
 
 import type { AztecAsyncArray } from '../interfaces/array.js';
-import type { Key, StoreSize } from '../interfaces/common.js';
+import type { Key, StoreSize, Value } from '../interfaces/common.js';
 import type { AztecAsyncCounter } from '../interfaces/counter.js';
-import type { AztecAsyncMap, AztecAsyncMultiMap } from '../interfaces/map.js';
+import type { AztecAsyncMap } from '../interfaces/map.js';
+import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
 import type { AztecAsyncSet } from '../interfaces/set.js';
 import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
 import type { AztecAsyncKVStore } from '../interfaces/store.js';
 import { IndexedDBAztecArray } from './array.js';
 import { IndexedDBAztecMap } from './map.js';
+import { IndexedDBAztecMultiMap } from './multi_map.js';
 import { IndexedDBAztecSet } from './set.js';
 import { IndexedDBAztecSingleton } from './singleton.js';
 
-export type StoredData<V> = { value: V; container: string; key: string; keyCount: number; slot: string };
+export type StoredData<V extends Value> = {
+  value: V;
+  container: string;
+  key: (string | number | Uint8Array)[];
+  keyCount: number;
+  slot: string;
+  hash: string;
+};
 
 export interface AztecIDBSchema extends DBSchema {
   data: {
     value: StoredData<any>;
     key: string;
-    indexes: { container: string; key: string; keyCount: number };
+    indexes: {
+      container: string;
+      key: any; // Runtime: (string | number | Uint8Array)[] - idb types don't support arrays
+      keyCount: number;
+      hash: string;
+    };
   };
 }
 
@@ -29,18 +44,27 @@ export interface AztecIDBSchema extends DBSchema {
  */
 
 export class AztecIndexedDBStore implements AztecAsyncKVStore {
-  #log: Logger;
   #rootDB: IDBPDatabase<AztecIDBSchema>;
   #name: string;
+  #txQueue: SerialQueue;
 
   #containers = new Set<
-    IndexedDBAztecArray<any> | IndexedDBAztecMap<any, any> | IndexedDBAztecSet<any> | IndexedDBAztecSingleton<any>
+    | IndexedDBAztecArray<any>
+    | IndexedDBAztecMap<any, any>
+    | IndexedDBAztecMultiMap<any, any>
+    | IndexedDBAztecSet<any>
+    | IndexedDBAztecSingleton<any>
   >();
 
-  constructor(rootDB: IDBPDatabase<AztecIDBSchema>, public readonly isEphemeral: boolean, log: Logger, name: string) {
+  constructor(
+    rootDB: IDBPDatabase<AztecIDBSchema>,
+    public readonly isEphemeral: boolean,
+    name: string,
+  ) {
     this.#rootDB = rootDB;
-    this.#log = log;
     this.#name = name;
+    this.#txQueue = new SerialQueue();
+    this.#txQueue.start();
   }
   /**
    * Creates a new AztecKVStore backed by IndexedDB. The path to the database is optional. If not provided,
@@ -53,50 +77,32 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
    * @returns The store
    */
   static async open(log: Logger, name?: string, ephemeral: boolean = false): Promise<AztecIndexedDBStore> {
-    name = name && !ephemeral ? name : self.crypto.getRandomValues(new Uint8Array(16)).join('');
+    name = name && !ephemeral ? name : globalThis.crypto.getRandomValues(new Uint8Array(16)).join('');
     log.debug(`Opening IndexedDB ${ephemeral ? 'temp ' : ''}database with name ${name}`);
     const rootDB = await openDB<AztecIDBSchema>(name, 1, {
       upgrade(db) {
         const objectStore = db.createObjectStore('data', { keyPath: 'slot' });
 
         objectStore.createIndex('key', ['container', 'key'], { unique: false });
-        objectStore.createIndex('keyCount', ['container', 'key', 'keyCount'], { unique: false });
+        // Keep count of the maximum number of keys ever inserted in the container
+        // This allows unique slots for repeated keys, which is useful for multi-maps
+        objectStore.createIndex('keyCount', ['container', 'key', 'keyCount'], { unique: true });
+        // Keep an index on the pair key-hash for a given container, allowing us to efficiently
+        // delete unique values from multi-maps
+        objectStore.createIndex('hash', ['container', 'key', 'hash'], { unique: true });
       },
     });
 
-    const kvStore = new AztecIndexedDBStore(rootDB, ephemeral, log, name);
+    const kvStore = new AztecIndexedDBStore(rootDB, ephemeral, name);
     return kvStore;
   }
 
-  /**
-   * Forks the current DB into a new DB by backing it up to a temporary location and opening a new indexedb.
-   * @returns A new AztecIndexedDBStore.
-   */
-  async fork(): Promise<AztecAsyncKVStore> {
-    const forkedStore = await AztecIndexedDBStore.open(this.#log, undefined, true);
-    this.#log.verbose(`Forking store to ${forkedStore.#name}`);
-
-    // Copy old data to new store
-    const oldData = this.#rootDB.transaction('data').store;
-    const dataToWrite = [];
-    for await (const cursor of oldData.iterate()) {
-      dataToWrite.push(cursor.value);
-    }
-    const tx = forkedStore.#rootDB.transaction('data', 'readwrite').store;
-    for (const data of dataToWrite) {
-      await tx.add(data);
-    }
-
-    this.#log.debug(`Forked store at ${forkedStore.#name} opened successfully`);
-    return forkedStore;
-  }
-
   /**
    * Creates a new AztecMap in the store.
    * @param name - Name of the map
    * @returns A new AztecMap
    */
-  openMap<K extends Key, V>(name: string): AztecAsyncMap<K, V> {
+  openMap<K extends Key, V extends Value>(name: string): AztecAsyncMap<K, V> {
     const map = new IndexedDBAztecMap<K, V>(this.#rootDB, name);
     this.#containers.add(map);
     return map;
@@ -118,8 +124,8 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
    * @param name - Name of the map
    * @returns A new AztecMultiMap
    */
-  openMultiMap<K extends Key, V>(name: string): AztecAsyncMultiMap<K, V> {
-    const multimap = new IndexedDBAztecMap<K, V>(this.#rootDB, name);
+  openMultiMap<K extends Key, V extends Value>(name: string): AztecAsyncMultiMap<K, V> {
+    const multimap = new IndexedDBAztecMultiMap<K, V>(this.#rootDB, name);
     this.#containers.add(multimap);
     return multimap;
   }
@@ -133,7 +139,7 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
    * @param name - Name of the array
    * @returns A new AztecArray
    */
-  openArray<T>(name: string): AztecAsyncArray<T> {
+  openArray<T extends Value>(name: string): AztecAsyncArray<T> {
     const array = new IndexedDBAztecArray<T>(this.#rootDB, name);
     this.#containers.add(array);
     return array;
@@ -144,7 +150,7 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
    * @param name - Name of the singleton
    * @returns A new AztecSingleton
    */
-  openSingleton<T>(name: string): AztecAsyncSingleton<T> {
+  openSingleton<T extends Value>(name: string): AztecAsyncSingleton<T> {
     const singleton = new IndexedDBAztecSingleton<T>(this.#rootDB, name);
     this.#containers.add(singleton);
     return singleton;
@@ -155,22 +161,27 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
    * @param callback - Function to execute in a transaction
    * @returns A promise that resolves to the return value of the callback
    */
-  async transactionAsync<T>(callback: () => Promise<T>): Promise<T> {
-    const tx = this.#rootDB.transaction('data', 'readwrite');
-    for (const container of this.#containers) {
-      container.db = tx.store;
-    }
-    // Avoid awaiting this promise so it doesn't get scheduled in the next microtask
-    // By then, the tx would be closed
-    const runningPromise = callback();
-    // Wait for the transaction to finish
-    await tx.done;
-    for (const container of this.#containers) {
-      container.db = undefined;
-    }
-    // Return the result of the callback.
-    // Tx is guaranteed to already be closed, so the await doesn't hurt anything here
-    return await runningPromise;
+  transactionAsync<T>(callback: () => Promise<T>): Promise<T> {
+    // We can only have one transaction at a time for the same store
+    // So we need to wait for the current one to finish
+    return this.#txQueue.put(async () => {
+      const tx = this.#rootDB.transaction('data', 'readwrite');
+      for (const container of this.#containers) {
+        container.db = tx.store;
+      }
+      // Avoid awaiting this promise so it doesn't get scheduled in the next microtask
+      // By then, the tx would be closed
+      const runningPromise = callback();
+      // Wait for the transaction to finish
+      await tx.done;
+      for (const container of this.#containers) {
+        container.db = undefined;
+      }
+
+      // Return the result of the callback.
+      // Tx is guaranteed to already be closed, so the await doesn't hurt anything here
+      return await runningPromise;
+    });
   }
 
   /**
@@ -188,10 +199,14 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
   }
 
   estimateSize(): Promise<StoreSize> {
-    return Promise.resolve({ mappingSize: 0, actualSize: 0, numItems: 0 });
+    return Promise.resolve({ mappingSize: 0, physicalFileSize: 0, actualSize: 0, numItems: 0 });
   }
 
   close(): Promise<void> {
     return Promise.resolve();
   }
+
+  backupTo(_dstPath: string, _compact?: boolean): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
 }

package/src/interfaces/array.ts

@@ -1,7 +1,9 @@
+import type { Value } from './common.js';
+
 /**
  * An array backed by a persistent store. Can not have any holes in it.
  */
-interface BaseAztecArray<T> {
+interface BaseAztecArray<T extends Value> {
   /**
    * Pushes values to the end of the array
    * @param vals - The values to push to the end of the array
@@ -27,7 +29,7 @@ interface BaseAztecArray<T> {
 /**
  * An array backed by a persistent store. Can not have any holes in it.
  */
-export interface AztecAsyncArray<T> extends BaseAztecArray<T> {
+export interface AztecAsyncArray<T extends Value> extends BaseAztecArray<T> {
   /**
    * The size of the array
    */
@@ -58,7 +60,7 @@ export interface AztecAsyncArray<T> extends BaseAztecArray<T> {
   [Symbol.asyncIterator](): AsyncIterableIterator<T>;
 }
 
-export interface AztecArray<T> extends BaseAztecArray<T> {
+export interface AztecArray<T extends Value> extends BaseAztecArray<T> {
   /**
    * The size of the array
    */

package/src/interfaces/common.ts

@@ -1,12 +1,10 @@
-/**
- * The key type for use with the kv-store
- */
-export type Key = string | number | Array<string | number>;
+/** The key type for use with the kv-store */
+export type Key = string | number | Uint8Array | Array<string | number>;
 
-/**
- * A range of keys to iterate over.
- */
-export type Range<K extends Key = Key> = {
+export type Value = NonNullable<any>;
+
+/** A range of keys of arbitrary type. */
+export type CustomRange<K> = {
   /** The key of the first item to include */
   start?: K;
   /** The key of the last item to include */
@@ -17,4 +15,17 @@ export type Range<K extends Key = Key> = {
   limit?: number;
 };
 
-export type StoreSize = { mappingSize: number; actualSize: number; numItems: number };
+/** Maps a custom range into a range of valid key types to iterate over. */
+export function mapRange<CK, K extends Key = Key>(range: CustomRange<CK>, mapFn: (key: CK) => K): Range<K> {
+  return {
+    start: range.start ? mapFn(range.start) : undefined,
+    end: range.end ? mapFn(range.end) : undefined,
+    reverse: range.reverse,
+    limit: range.limit,
+  };
+}
+
+/** A range of keys to iterate over. */
+export type Range<K extends Key = Key> = CustomRange<K>;
+
+export type StoreSize = { mappingSize: number; physicalFileSize: number; actualSize: number; numItems: number };

package/src/interfaces/index.ts

@@ -4,4 +4,6 @@ export * from './counter.js';
 export * from './singleton.js';
 export * from './store.js';
 export * from './set.js';
-export type { Range, StoreSize } from './common.js';
+export * from './multi_map.js';
+export { mapRange } from './common.js';
+export type { CustomRange, Range, StoreSize } from './common.js';

package/src/interfaces/map.ts

@@ -1,9 +1,9 @@
-import type { Key, Range } from './common.js';
+import type { Key, Range, Value } from './common.js';
 
 /**
  * A map backed by a persistent store.
  */
-interface AztecBaseMap<K extends Key, V> {
+interface AztecBaseMap<K extends Key, V extends Value> {
   /**
    * Sets the value at the given key.
    * @param key - The key to set the value at
@@ -11,6 +11,12 @@ interface AztecBaseMap<K extends Key, V> {
    */
   set(key: K, val: V): Promise<void>;
 
+  /**
+   * Sets the values at the given keys.
+   * @param entries - The entries to set
+   */
+  setMany(entries: { key: K; value: V }[]): Promise<void>;
+
   /**
    * Sets the value at the given key if it does not already exist.
    * @param key - The key to set the value at
@@ -24,13 +30,19 @@ interface AztecBaseMap<K extends Key, V> {
    */
   delete(key: K): Promise<void>;
 }
-export interface AztecMap<K extends Key, V> extends AztecBaseMap<K, V> {
+export interface AztecMap<K extends Key, V extends Value> extends AztecBaseMap<K, V> {
   /**
    * Gets the value at the given key.
    * @param key - The key to get the value from
    */
   get(key: K): V | undefined;
 
+  /**
+   * Gets the current size of the map.
+   * @returns The size of the map
+   */
+  size(): number;
+
   /**
    * Checks if a key exists in the map.
    * @param key - The key to check
@@ -62,44 +74,10 @@ export interface AztecMap<K extends Key, V> extends AztecBaseMap<K, V> {
   clear(): Promise<void>;
 }
 
-export interface AztecMapWithSize<K extends Key, V> extends AztecMap<K, V> {
-  /**
-   * Gets the size of the map.
-   * @returns The size of the map
-   */
-  size(): number;
-}
-
-/**
- * A map backed by a persistent store that can have multiple values for a single key.
- */
-export interface AztecMultiMap<K extends Key, V> extends AztecMap<K, V> {
-  /**
-   * Gets all the values at the given key.
-   * @param key - The key to get the values from
-   */
-  getValues(key: K): IterableIterator<V>;
-
-  /**
-   * Deletes a specific value at the given key.
-   * @param key - The key to delete the value at
-   * @param val - The value to delete
-   */
-  deleteValue(key: K, val: V): Promise<void>;
-}
-
-export interface AztecMultiMapWithSize<K extends Key, V> extends AztecMultiMap<K, V> {
-  /**
-   * Gets the size of the map.
-   * @returns The size of the map
-   */
-  size(): number;
-}
-
 /**
  * A map backed by a persistent store.
  */
-export interface AztecAsyncMap<K extends Key, V> extends AztecBaseMap<K, V> {
+export interface AztecAsyncMap<K extends Key, V extends Value> extends AztecBaseMap<K, V> {
   /**
    * Gets the value at the given key.
    * @param key - The key to get the value from
@@ -130,22 +108,10 @@ export interface AztecAsyncMap<K extends Key, V> extends AztecBaseMap<K, V> {
    * @param range - The range of keys to iterate over
    */
   keysAsync(range?: Range<K>): AsyncIterableIterator<K>;
-}
 
-/**
- * A map backed by a persistent store that can have multiple values for a single key.
- */
-export interface AztecAsyncMultiMap<K extends Key, V> extends AztecAsyncMap<K, V> {
   /**
-   * Gets all the values at the given key.
-   * @param key - The key to get the values from
-   */
-  getValuesAsync(key: K): AsyncIterableIterator<V>;
-
-  /**
-   * Deletes a specific value at the given key.
-   * @param key - The key to delete the value at
-   * @param val - The value to delete
+   * Gets the current size of the map.
+   * @returns The size of the map
    */
-  deleteValue(key: K, val: V): Promise<void>;
+  sizeAsync(): Promise<number>;
 }