@aztec/kv-store 0.66.0 → 0.67.1-devnet

package/src/interfaces/store.ts

@@ -1,32 +1,54 @@
-import { type AztecArray } from './array.js';
+import { type AztecArray, type AztecAsyncArray } from './array.js';
 import { type Key } from './common.js';
-import { type AztecCounter } from './counter.js';
-import { type AztecMap, type AztecMultiMap } from './map.js';
-import { type AztecSet } from './set.js';
-import { type AztecSingleton } from './singleton.js';
+import { type AztecAsyncCounter, type AztecCounter } from './counter.js';
+import {
+  type AztecAsyncMap,
+  type AztecAsyncMultiMap,
+  type AztecMap,
+  type AztecMapWithSize,
+  type AztecMultiMap,
+  type AztecMultiMapWithSize,
+} from './map.js';
+import { type AztecAsyncSet, type AztecSet } from './set.js';
+import { type AztecAsyncSingleton, type AztecSingleton } from './singleton.js';
 
 /** A key-value store */
 export interface AztecKVStore {
+  syncGetters: true;
   /**
    * Creates a new map.
    * @param name - The name of the map
    * @returns The map
    */
-  openMap<K extends string | number, V>(name: string): AztecMap<K, V>;
+  openMap<K extends Key, V>(name: string): AztecMap<K, V>;
 
   /**
    * Creates a new set.
    * @param name - The name of the set
    * @returns The set
    */
-  openSet<K extends string | number>(name: string): AztecSet<K>;
+  openSet<K extends Key>(name: string): AztecSet<K>;
 
   /**
    * Creates a new multi-map.
    * @param name - The name of the multi-map
    * @returns The multi-map
    */
-  openMultiMap<K extends string | number, V>(name: string): AztecMultiMap<K, V>;
+  openMultiMap<K extends Key, V>(name: string): AztecMultiMap<K, V>;
+
+  /**
+   * Creates a new multi-map with size.
+   * @param name - The name of the multi-map
+   * @returns The multi-map
+   */
+  openMultiMapWithSize<K extends Key, V>(name: string): AztecMultiMapWithSize<K, V>;
+
+  /**
+   * Creates a new map with size.
+   * @param name - The name of the map
+   * @returns The map
+   */
+  openMapWithSize<K extends Key, V>(name: string): AztecMapWithSize<K, V>;
 
   /**
    * Creates a new array.
@@ -74,3 +96,72 @@ export interface AztecKVStore {
    */
   estimateSize(): { mappingSize: number; actualSize: number; numItems: number };
 }
+
+export interface AztecAsyncKVStore {
+  /**
+   * Creates a new map.
+   * @param name - The name of the map
+   * @returns The map
+   */
+  openMap<K extends Key, V>(name: string): AztecAsyncMap<K, V>;
+
+  /**
+   * Creates a new set.
+   * @param name - The name of the set
+   * @returns The set
+   */
+  openSet<K extends Key>(name: string): AztecAsyncSet<K>;
+
+  /**
+   * Creates a new multi-map.
+   * @param name - The name of the multi-map
+   * @returns The multi-map
+   */
+  openMultiMap<K extends Key, V>(name: string): AztecAsyncMultiMap<K, V>;
+
+  /**
+   * Creates a new array.
+   * @param name - The name of the array
+   * @returns The array
+   */
+  openArray<T>(name: string): AztecAsyncArray<T>;
+
+  /**
+   * Creates a new singleton.
+   * @param name - The name of the singleton
+   * @returns The singleton
+   */
+  openSingleton<T>(name: string): AztecAsyncSingleton<T>;
+
+  /**
+   * Creates a new count map.
+   * @param name - name of the counter
+   */
+  openCounter<K extends Key>(name: string): AztecAsyncCounter<K>;
+
+  /**
+   * Starts a transaction. All calls to read/write data while in a transaction are queued and executed atomically.
+   * @param callback - The callback to execute in a transaction
+   */
+  transactionAsync<T extends Exclude<any, Promise<any>>>(callback: () => Promise<T>): Promise<T>;
+
+  /**
+   * Clears all entries in the store
+   */
+  clear(): Promise<void>;
+
+  /**
+   * Forks the store.
+   */
+  fork(): Promise<AztecAsyncKVStore>;
+
+  /**
+   * Deletes the store
+   */
+  delete(): Promise<void>;
+
+  /**
+   * Estimates the size of the store in bytes.
+   */
+  estimateSize(): { mappingSize: number; actualSize: number; numItems: number };
+}

package/src/interfaces/store_test_suite.ts

@@ -0,0 +1,56 @@
+import { expect } from 'chai';
+
+import { type AztecAsyncSingleton, type AztecSingleton } from './singleton.js';
+import { type AztecAsyncKVStore, type AztecKVStore } from './store.js';
+import { isSyncStore } from './utils.js';
+
+export function describeAztecStore(
+  testName: string,
+  getPersistentStore: () => Promise<AztecKVStore | AztecAsyncKVStore>,
+  getPersistentNoPathStore: () => Promise<AztecKVStore | AztecAsyncKVStore>,
+  getEphemeralStore: () => Promise<AztecKVStore | AztecAsyncKVStore>,
+) {
+  describe(testName, () => {
+    async function get(
+      store: AztecKVStore | AztecAsyncKVStore,
+      singleton: AztecSingleton<string> | AztecAsyncSingleton<string>,
+    ) {
+      return isSyncStore(store)
+        ? (singleton as AztecSingleton<string>).get()
+        : await (singleton as AztecAsyncSingleton<string>).getAsync();
+    }
+
+    const itForks = async (store: AztecKVStore | AztecAsyncKVStore) => {
+      const singleton = store.openSingleton<string>('singleton');
+      await singleton.set('foo');
+
+      const forkedStore = await store.fork();
+      const forkedSingleton = forkedStore.openSingleton<string>('singleton');
+      expect(await get(store, singleton)).to.equal('foo');
+      await forkedSingleton.set('bar');
+      expect(await get(store, singleton)).to.equal('foo');
+      expect(await get(forkedStore, forkedSingleton)).to.equal('bar');
+      await forkedSingleton.delete();
+      expect(await get(store, singleton)).to.equal('foo');
+      await forkedStore.delete();
+    };
+
+    it('forks a persistent store', async () => {
+      const store = await getPersistentStore();
+      await itForks(store);
+      await store.delete();
+    });
+
+    it('forks a persistent store with no path', async () => {
+      const store = await getPersistentNoPathStore();
+      await itForks(store);
+      await store.delete();
+    });
+
+    it('forks an ephemeral store', async () => {
+      const store = await getEphemeralStore();
+      await itForks(store);
+      await store.delete();
+    });
+  });
+}

package/src/interfaces/utils.ts

@@ -0,0 +1,21 @@
+import { type AztecAsyncKVStore, type AztecKVStore } from './store.js';
+
+/* eslint-disable no-console */
+export const mockLogger = {
+  debug: (msg: string, data: any) => console.log(msg, data),
+  info: (msg: string, data: any) => console.log(msg, data),
+  warn: (msg: string, data: any) => console.log(msg, data),
+  error: (msg: string, data: any) => console.error(msg, data),
+  fatal: (msg: string, data: any) => console.error(msg, data),
+  silent: (_msg: string, _data: any) => {},
+  verbose: (msg: string, data: any) => console.log(msg, data),
+  trace: (msg: string, data: any) => console.log(msg, data),
+  level: 'trace' as const,
+  isLevelEnabled: (_level: string) => true,
+  module: 'kv-store:mock-logger',
+};
+/* eslint-enable no-console */
+
+export function isSyncStore(store: AztecKVStore | AztecAsyncKVStore): store is AztecAsyncKVStore {
+  return (store as AztecKVStore).syncGetters === true;
+}

package/src/lmdb/array.ts

@@ -1,6 +1,6 @@
 import { type Database, type Key } from 'lmdb';
 
-import { type AztecArray } from '../interfaces/array.js';
+import { type AztecArray, type AztecAsyncArray } from '../interfaces/array.js';
 import { LmdbAztecSingleton } from './singleton.js';
 
 /** The shape of a key that stores a value in an array */
@@ -9,7 +9,7 @@ type ArrayIndexSlot = ['array', string, 'slot', number];
 /**
  * An persistent array backed by LMDB.
  */
-export class LmdbAztecArray<T> implements AztecArray<T> {
+export class LmdbAztecArray<T> implements AztecArray<T>, AztecAsyncArray<T> {
   #db: Database<T, ArrayIndexSlot>;
   #name: string;
   #length: LmdbAztecSingleton<number>;
@@ -24,6 +24,10 @@ export class LmdbAztecArray<T> implements AztecArray<T> {
     return this.#length.get() ?? 0;
   }
 
+  lengthAsync(): Promise<number> {
+    return Promise.resolve(this.length);
+  }
+
   push(...vals: T[]): Promise<number> {
     return this.#db.childTransaction(() => {
       let length = this.length;
@@ -69,6 +73,10 @@ export class LmdbAztecArray<T> implements AztecArray<T> {
     return this.#db.get(this.#slot(index));
   }
 
+  atAsync(index: number): Promise<T | undefined> {
+    return Promise.resolve(this.at(index));
+  }
+
   setAt(index: number, val: T): Promise<boolean> {
     if (index < 0) {
       index = this.length + index;
@@ -93,16 +101,32 @@ export class LmdbAztecArray<T> implements AztecArray<T> {
     }
   }
 
+  async *entriesAsync(): AsyncIterableIterator<[number, T]> {
+    for (const [key, value] of this.entries()) {
+      yield [key, value];
+    }
+  }
+
   *values(): IterableIterator<T> {
     for (const [_, value] of this.entries()) {
       yield value;
     }
   }
 
+  async *valuesAsync(): AsyncIterableIterator<T> {
+    for (const [_, value] of this.entries()) {
+      yield value;
+    }
+  }
+
   [Symbol.iterator](): IterableIterator<T> {
     return this.values();
   }
 
+  [Symbol.asyncIterator](): AsyncIterableIterator<T> {
+    return this.valuesAsync();
+  }
+
   #slot(index: number): ArrayIndexSlot {
     return ['array', this.#name, 'slot', index];
   }

package/src/lmdb/counter.ts

@@ -1,13 +1,13 @@
 import { type Key as BaseKey, type Database } from 'lmdb';
 
 import { type Key, type Range } from '../interfaces/common.js';
-import { type AztecCounter } from '../interfaces/counter.js';
+import { type AztecAsyncCounter, type AztecCounter } from '../interfaces/counter.js';
 import { LmdbAztecMap } from './map.js';
 
 /**
  * A counter implementation backed by LMDB
  */
-export class LmdbAztecCounter<K extends Key> implements AztecCounter<K> {
+export class LmdbAztecCounter<K extends Key> implements AztecCounter<K>, AztecAsyncCounter<K> {
   #db: Database;
   #name: string;
   #map: LmdbAztecMap<K, number>;
@@ -45,11 +45,23 @@ export class LmdbAztecCounter<K extends Key> implements AztecCounter<K> {
     return this.#map.get(key) ?? 0;
   }
 
+  getAsync(key: K): Promise<number> {
+    return Promise.resolve(this.get(key));
+  }
+
   entries(range: Range<K> = {}): IterableIterator<[K, number]> {
     return this.#map.entries(range);
   }
 
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, number]> {
+    yield* this.entries(range);
+  }
+
   keys(range: Range<K> = {}): IterableIterator<K> {
     return this.#map.keys(range);
   }
+
+  async *keysAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    yield* this.keys(range);
+  }
 }

package/src/lmdb/index.ts

@@ -1 +1,37 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+
+import { join } from 'path';
+
+import { type DataStoreConfig } from '../config.js';
+import { initStoreForRollup } from '../utils.js';
+import { AztecLmdbStore } from './store.js';
+
 export { AztecLmdbStore } from './store.js';
+
+export function createStore(name: string, config: DataStoreConfig, log: Logger = createLogger('kv-store')) {
+  let { dataDirectory } = config;
+  if (typeof dataDirectory !== 'undefined') {
+    dataDirectory = join(dataDirectory, name);
+  }
+
+  log.info(
+    dataDirectory
+      ? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.dataStoreMapSizeKB} KB`
+      : `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKB} KB`,
+  );
+
+  const store = AztecLmdbStore.open(dataDirectory, config.dataStoreMapSizeKB, false);
+  if (config.l1Contracts?.rollupAddress) {
+    return initStoreForRollup(store, config.l1Contracts.rollupAddress, log);
+  }
+  return store;
+}
+/**
+ * Opens a temporary store for testing purposes.
+ * @param ephemeral - true if the store should only exist in memory and not automatically be flushed to disk. Optional
+ * @returns A new store
+ */
+export function openTmpStore(ephemeral: boolean = false): AztecLmdbStore {
+  const mapSize = 1024 * 1024 * 10; // 10 GB map size
+  return AztecLmdbStore.open(undefined, mapSize, ephemeral);
+}

package/src/lmdb/map.ts

@@ -1,7 +1,7 @@
 import { type Database, type RangeOptions } from 'lmdb';
 
 import { type Key, type Range } from '../interfaces/common.js';
-import { type AztecMultiMap } from '../interfaces/map.js';
+import { type AztecAsyncMultiMap, type AztecMapWithSize, type AztecMultiMap } from '../interfaces/map.js';
 
 /** The slot where a key-value entry would be stored */
 type MapValueSlot<K extends Key | Buffer> = ['map', string, 'slot', K];
@@ -9,12 +9,12 @@ type MapValueSlot<K extends Key | Buffer> = ['map', string, 'slot', K];
 /**
  * A map backed by LMDB.
  */
-export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V> {
+export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, AztecAsyncMultiMap<K, V> {
   protected db: Database<[K, V], MapValueSlot<K>>;
   protected name: string;
 
-  #startSentinel: MapValueSlot<Buffer>;
-  #endSentinel: MapValueSlot<Buffer>;
+  protected startSentinel: MapValueSlot<Buffer>;
+  protected endSentinel: MapValueSlot<Buffer>;
 
   constructor(rootDb: Database, mapName: string) {
     this.name = mapName;
@@ -23,8 +23,8 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V> {
     // sentinels are used to define the start and end of the map
     // with LMDB's key encoding, no _primitive value_ can be "less than" an empty buffer or greater than Byte 255
     // these will be used later to answer range queries
-    this.#startSentinel = ['map', this.name, 'slot', Buffer.from([])];
-    this.#endSentinel = ['map', this.name, 'slot', Buffer.from([255])];
+    this.startSentinel = ['map', this.name, 'slot', Buffer.from([])];
+    this.endSentinel = ['map', this.name, 'slot', Buffer.from([255])];
   }
 
   close(): Promise<void> {
@@ -32,45 +32,59 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V> {
   }
 
   get(key: K): V | undefined {
-    return this.db.get(this.#slot(key))?.[1];
+    return this.db.get(this.slot(key))?.[1];
+  }
+
+  getAsync(key: K): Promise<V | undefined> {
+    return Promise.resolve(this.get(key));
   }
 
   *getValues(key: K): IterableIterator<V> {
-    const values = this.db.getValues(this.#slot(key));
+    const values = this.db.getValues(this.slot(key));
     for (const value of values) {
       yield value?.[1];
     }
   }
 
+  async *getValuesAsync(key: K): AsyncIterableIterator<V> {
+    for (const value of this.getValues(key)) {
+      yield value;
+    }
+  }
+
   has(key: K): boolean {
-    return this.db.doesExist(this.#slot(key));
+    return this.db.doesExist(this.slot(key));
+  }
+
+  hasAsync(key: K): Promise<boolean> {
+    return Promise.resolve(this.has(key));
   }
 
   async set(key: K, val: V): Promise<void> {
-    await this.db.put(this.#slot(key), [key, val]);
+    await this.db.put(this.slot(key), [key, val]);
   }
 
   swap(key: K, fn: (val: V | undefined) => V): Promise<void> {
     return this.db.childTransaction(() => {
-      const slot = this.#slot(key);
+      const slot = this.slot(key);
       const entry = this.db.get(slot);
       void this.db.put(slot, [key, fn(entry?.[1])]);
     });
   }
 
   setIfNotExists(key: K, val: V): Promise<boolean> {
-    const slot = this.#slot(key);
+    const slot = this.slot(key);
     return this.db.ifNoExists(slot, () => {
       void this.db.put(slot, [key, val]);
     });
   }
 
   async delete(key: K): Promise<void> {
-    await this.db.remove(this.#slot(key));
+    await this.db.remove(this.slot(key));
   }
 
   async deleteValue(key: K, val: V): Promise<void> {
-    await this.db.remove(this.#slot(key), [key, val]);
+    await this.db.remove(this.slot(key), [key, val]);
   }
 
   *entries(range: Range<K> = {}): IterableIterator<[K, V]> {
@@ -79,19 +93,19 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V> {
     // in that case, we need to swap the start and end sentinels
     const start = reverse
       ? range.end
-        ? this.#slot(range.end)
-        : this.#endSentinel
+        ? this.slot(range.end)
+        : this.endSentinel
       : range.start
-      ? this.#slot(range.start)
-      : this.#startSentinel;
+      ? this.slot(range.start)
+      : this.startSentinel;
 
     const end = reverse
       ? range.start
-        ? this.#slot(range.start)
-        : this.#startSentinel
+        ? this.slot(range.start)
+        : this.startSentinel
       : range.end
-      ? this.#slot(range.end)
-      : this.#endSentinel;
+      ? this.slot(range.end)
+      : this.endSentinel;
 
     const lmdbRange: RangeOptions = {
       start,
@@ -109,19 +123,112 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V> {
     }
   }
 
+  async *entriesAsync(range?: Range<K> | undefined): AsyncIterableIterator<[K, V]> {
+    for (const entry of this.entries(range)) {
+      yield entry;
+    }
+  }
+
   *values(range: Range<K> = {}): IterableIterator<V> {
     for (const [_, value] of this.entries(range)) {
       yield value;
     }
   }
 
+  async *valuesAsync(range: Range<K> = {}): AsyncIterableIterator<V> {
+    for await (const [_, value] of this.entriesAsync(range)) {
+      yield value;
+    }
+  }
+
   *keys(range: Range<K> = {}): IterableIterator<K> {
     for (const [key, _] of this.entries(range)) {
       yield key;
     }
   }
 
-  #slot(key: K): MapValueSlot<K> {
+  async *keysAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    for await (const [key, _] of this.entriesAsync(range)) {
+      yield key;
+    }
+  }
+
+  protected slot(key: K): MapValueSlot<K> {
     return ['map', this.name, 'slot', key];
   }
+
+  async clear(): Promise<void> {
+    const lmdbRange: RangeOptions = {
+      start: this.startSentinel,
+      end: this.endSentinel,
+    };
+
+    const iterator = this.db.getRange(lmdbRange);
+
+    for (const { key } of iterator) {
+      await this.db.remove(key);
+    }
+  }
+}
+
+export class LmdbAztecMapWithSize<K extends Key, V>
+  extends LmdbAztecMap<K, V>
+  implements AztecMapWithSize<K, V>, AztecAsyncMultiMap<K, V>
+{
+  #sizeCache?: number;
+
+  constructor(rootDb: Database, mapName: string) {
+    super(rootDb, mapName);
+  }
+
+  override async set(key: K, val: V): Promise<void> {
+    await this.db.childTransaction(() => {
+      const exists = this.db.doesExist(this.slot(key));
+      this.db.putSync(this.slot(key), [key, val], {
+        appendDup: true,
+      });
+      if (!exists) {
+        this.#sizeCache = undefined; // Invalidate cache
+      }
+    });
+  }
+
+  override async delete(key: K): Promise<void> {
+    await this.db.childTransaction(async () => {
+      const exists = this.db.doesExist(this.slot(key));
+      if (exists) {
+        await this.db.remove(this.slot(key));
+        this.#sizeCache = undefined; // Invalidate cache
+      }
+    });
+  }
+
+  override async deleteValue(key: K, val: V): Promise<void> {
+    await this.db.childTransaction(async () => {
+      const exists = this.db.doesExist(this.slot(key));
+      if (exists) {
+        await this.db.remove(this.slot(key), [key, val]);
+        this.#sizeCache = undefined; // Invalidate cache
+      }
+    });
+  }
+
+  /**
+   * Gets the size of the map by counting entries.
+   * @returns The number of entries in the map
+   */
+  size(): number {
+    if (this.#sizeCache === undefined) {
+      this.#sizeCache = this.db.getCount({
+        start: this.startSentinel,
+        end: this.endSentinel,
+      });
+    }
+    return this.#sizeCache;
+  }
+
+  // Reset cache on clear/drop operations
+  clearCache() {
+    this.#sizeCache = undefined;
+  }
 }

package/src/lmdb/set.ts

@@ -1,13 +1,13 @@
 import { type Database } from 'lmdb';
 
 import { type Key, type Range } from '../interfaces/common.js';
-import { type AztecSet } from '../interfaces/set.js';
+import { type AztecAsyncSet, type AztecSet } from '../interfaces/set.js';
 import { LmdbAztecMap } from './map.js';
 
 /**
  * A set backed by LMDB.
  */
-export class LmdbAztecSet<K extends Key> implements AztecSet<K> {
+export class LmdbAztecSet<K extends Key> implements AztecSet<K>, AztecAsyncSet<K> {
   private map: LmdbAztecMap<K, boolean>;
   constructor(rootDb: Database, mapName: string) {
     this.map = new LmdbAztecMap(rootDb, mapName);
@@ -21,6 +21,10 @@ export class LmdbAztecSet<K extends Key> implements AztecSet<K> {
     return this.map.has(key);
   }
 
+  hasAsync(key: K): Promise<boolean> {
+    return Promise.resolve(this.has(key));
+  }
+
   add(key: K): Promise<void> {
     return this.map.set(key, true);
   }
@@ -32,4 +36,10 @@ export class LmdbAztecSet<K extends Key> implements AztecSet<K> {
   entries(range: Range<K> = {}): IterableIterator<K> {
     return this.map.keys(range);
   }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    for await (const key of this.map.keysAsync(range)) {
+      yield key;
+    }
+  }
 }

package/src/lmdb/singleton.ts

@@ -1,6 +1,6 @@
 import { type Database, type Key } from 'lmdb';
 
-import { type AztecSingleton } from '../interfaces/singleton.js';
+import { type AztecAsyncSingleton, type AztecSingleton } from '../interfaces/singleton.js';
 
 /** The slot where this singleton will store its value */
 type ValueSlot = ['singleton', string, 'value'];
@@ -8,7 +8,7 @@ type ValueSlot = ['singleton', string, 'value'];
 /**
  * Stores a single value in LMDB.
  */
-export class LmdbAztecSingleton<T> implements AztecSingleton<T> {
+export class LmdbAztecSingleton<T> implements AztecSingleton<T>, AztecAsyncSingleton<T> {
   #db: Database<T, ValueSlot>;
   #slot: ValueSlot;
 
@@ -21,6 +21,10 @@ export class LmdbAztecSingleton<T> implements AztecSingleton<T> {
     return this.#db.get(this.#slot);
   }
 
+  getAsync(): Promise<T | undefined> {
+    return Promise.resolve(this.get());
+  }
+
   set(val: T): Promise<boolean> {
     return this.#db.put(this.#slot, val);
   }