@aztec-labs/kv-store 6.0.0-nightly.20260829

package/src/interfaces/set_test_suite.ts

@@ -0,0 +1,79 @@
+import { toArray } from '@aztec-labs/foundation/iterable';
+
+import type { Range } from './common.js';
+import type { AztecAsyncSet, AztecSet } from './set.js';
+import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
+import { isSyncStore } from './utils.js';
+
+export function describeAztecSet(
+  testName: string,
+  getStore: () => AztecKVStore | Promise<AztecAsyncKVStore>,
+  forceAsync: boolean = false,
+) {
+  describe(testName, () => {
+    let store: AztecKVStore | AztecAsyncKVStore;
+    let set: AztecSet<string> | AztecAsyncSet<string>;
+
+    beforeEach(async () => {
+      store = await getStore();
+      set = store.openSet<string>('test');
+    });
+
+    afterEach(async () => {
+      await store.delete();
+    });
+
+    async function has(key: string) {
+      return isSyncStore(store) && !forceAsync
+        ? (set as AztecSet<string>).has(key)
+        : await (set as AztecAsyncSet<string>).hasAsync(key);
+    }
+
+    async function entries(range?: Range<any>) {
+      return isSyncStore(store) && !forceAsync
+        ? await toArray((set as AztecSet<string>).entries(range))
+        : await toArray((set as AztecAsyncSet<string>).entriesAsync(range));
+    }
+
+    it('should be able to set and get values', async () => {
+      await set.add('foo');
+      await set.add('baz');
+
+      expect(await has('foo')).toBe(true);
+      expect(await has('baz')).toBe(true);
+      expect(await has('bar')).toBe(false);
+    });
+
+    it('should be able to delete values', async () => {
+      await set.add('foo');
+      await set.add('baz');
+
+      await set.delete('foo');
+
+      expect(await has('foo')).toBe(false);
+      expect(await has('baz')).toBe(true);
+    });
+
+    it('should be able to iterate over entries', async () => {
+      await set.add('baz');
+      await set.add('foo');
+
+      expect(await entries()).toEqual(['baz', 'foo']);
+    });
+
+    it('supports range queries', async () => {
+      await set.add('a');
+      await set.add('b');
+      await set.add('c');
+      await set.add('d');
+
+      expect(await entries({ start: 'b', end: 'c' })).toEqual(['b']);
+      expect(await entries({ start: 'b' })).toEqual(['b', 'c', 'd']);
+      expect(await entries({ end: 'c' })).toEqual(['a', 'b']);
+      expect(await entries({ start: 'b', end: 'c', reverse: true })).toEqual(['c']);
+      expect(await entries({ start: 'b', limit: 1 })).toEqual(['b']);
+      expect(await entries({ start: 'b', reverse: true })).toEqual(['d', 'c']);
+      expect(await entries({ end: 'b', reverse: true })).toEqual(['b', 'a']);
+    });
+  });
+}

package/src/interfaces/singleton.ts

@@ -0,0 +1,29 @@
+/**
+ * Represents a singleton value in the database.
+ * Note: The singleton loses type info so it's recommended to serialize to buffer when storing it.
+ */
+interface AztecBaseSingleton<T> {
+  /**
+   * Sets the value.
+   * @param val - The new value
+   */
+  set(val: T): Promise<boolean>;
+
+  /**
+   * Deletes the value.
+   */
+  delete(): Promise<boolean>;
+}
+export interface AztecSingleton<T> extends AztecBaseSingleton<T> {
+  /**
+   * Gets the value.
+   */
+  get(): T | undefined;
+}
+
+export interface AztecAsyncSingleton<T> extends AztecBaseSingleton<T> {
+  /**
+   * Gets the value.
+   */
+  getAsync(): Promise<T | undefined>;
+}

package/src/interfaces/singleton_test_suite.ts

@@ -0,0 +1,44 @@
+import type { AztecAsyncSingleton, AztecSingleton } from './singleton.js';
+import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
+import { isSyncStore } from './utils.js';
+
+export function describeAztecSingleton(
+  testName: string,
+  getStore: () => AztecKVStore | Promise<AztecAsyncKVStore>,
+  forceAsync: boolean = false,
+) {
+  describe(testName, () => {
+    let store: AztecKVStore | AztecAsyncKVStore;
+    let singleton: AztecSingleton<string> | AztecAsyncSingleton<string>;
+
+    beforeEach(async () => {
+      store = await getStore();
+      singleton = store.openSingleton<string>('test');
+    });
+
+    afterEach(async () => {
+      await store.delete();
+    });
+
+    async function get() {
+      return isSyncStore(store) && !forceAsync
+        ? (singleton as AztecSingleton<string>).get()
+        : await (singleton as AztecAsyncSingleton<string>).getAsync();
+    }
+
+    it('returns undefined if the value is not set', async () => {
+      expect(await get()).toBe(undefined);
+    });
+
+    it('should be able to set and get values', async () => {
+      expect(await singleton.set('foo')).toBe(true);
+      expect(await get()).toBe('foo');
+    });
+
+    it('overwrites the value if it is set again', async () => {
+      expect(await singleton.set('foo')).toBe(true);
+      expect(await singleton.set('bar')).toBe(true);
+      expect(await get()).toBe('bar');
+    });
+  });
+}

package/src/interfaces/store.ts

@@ -0,0 +1,142 @@
+import type { AztecArray, AztecAsyncArray } from './array.js';
+import type { Key, StoreSize, Value } from './common.js';
+import type { AztecAsyncCounter, AztecCounter } from './counter.js';
+import type { AztecAsyncMap, AztecMap } from './map.js';
+import type { AztecAsyncMultiMap, AztecMultiMap } from './multi_map.js';
+import type { AztecAsyncSet, AztecSet } from './set.js';
+import type { AztecAsyncSingleton, AztecSingleton } from './singleton.js';
+
+/** A key-value store */
+export interface AztecKVStore {
+  syncGetters: true;
+  /**
+   * Creates a new map.
+   * @param name - The name of the map
+   * @returns The map
+   */
+  openMap<K extends Key, V extends Value>(name: string): AztecMap<K, V>;
+
+  /**
+   * Creates a new set.
+   * @param name - The name of the set
+   * @returns The set
+   */
+  openSet<K extends Key>(name: string): AztecSet<K>;
+
+  /**
+   * Creates a new multi-map.
+   * @param name - The name of the multi-map
+   * @returns The multi-map
+   */
+  openMultiMap<K extends Key, V extends Value>(name: string): AztecMultiMap<K, V>;
+
+  /**
+   * Creates a new array.
+   * @param name - The name of the array
+   * @returns The array
+   */
+  openArray<T extends Value>(name: string): AztecArray<T>;
+
+  /**
+   * Creates a new singleton.
+   * @param name - The name of the singleton
+   * @returns The singleton
+   */
+  openSingleton<T extends Value>(name: string): AztecSingleton<T>;
+
+  /**
+   * Creates a new count map.
+   * @param name - name of the counter
+   */
+  openCounter<K extends Key>(name: string): AztecCounter<K>;
+
+  /**
+   * Starts a transaction. All calls to read/write data while in a transaction are queued and executed atomically.
+   * @param callback - The callback to execute in a transaction
+   */
+  transaction<T extends Exclude<any, Promise<any>>>(callback: () => T): Promise<T>;
+
+  /**
+   * Clears all entries in the store
+   */
+  clear(): Promise<void>;
+
+  /**
+   * Deletes the store
+   */
+  delete(): Promise<void>;
+
+  /**
+   * Estimates the size of the store in bytes.
+   */
+  estimateSize(): Promise<StoreSize>;
+
+  /**
+   * Closes the store
+   */
+  close(): Promise<void>;
+}
+
+export interface AztecAsyncKVStore {
+  /**
+   * Creates a new map.
+   * @param name - The name of the map
+   * @returns The map
+   */
+  openMap<K extends Key, V extends Value>(name: string): AztecAsyncMap<K, V>;
+
+  /**
+   * Creates a new set.
+   * @param name - The name of the set
+   * @returns The set
+   */
+  openSet<K extends Key>(name: string): AztecAsyncSet<K>;
+
+  /**
+   * Creates a new multi-map.
+   * @param name - The name of the multi-map
+   * @returns The multi-map
+   */
+  openMultiMap<K extends Key, V extends Value>(name: string): AztecAsyncMultiMap<K, V>;
+
+  /**
+   * Creates a new array.
+   * @param name - The name of the array
+   * @returns The array
+   */
+  openArray<T extends Value>(name: string): AztecAsyncArray<T>;
+
+  /**
+   * Creates a new singleton.
+   * @param name - The name of the singleton
+   * @returns The singleton
+   */
+  openSingleton<T extends Value>(name: string): AztecAsyncSingleton<T>;
+
+  /**
+   * Creates a new count map.
+   * @param name - name of the counter
+   */
+  openCounter<K extends Key>(name: string): AztecAsyncCounter<K>;
+
+  /**
+   * Starts a transaction. All calls to read/write data while in a transaction are queued and executed atomically.
+   * @param callback - The callback to execute in a transaction
+   */
+  transactionAsync<T extends Exclude<any, Promise<any>>>(callback: () => Promise<T>): Promise<T>;
+
+  /** Clears all entries in the store */
+  clear(): Promise<void>;
+
+  /** Deletes the store */
+  delete(): Promise<void>;
+
+  /** Estimates the size of the store in bytes. */
+  estimateSize(): Promise<StoreSize>;
+
+  /** Closes the store */
+  close(): Promise<void>;
+
+  /** Backups the store to the target folder.*/
+  backupTo(dstPath: string, compact?: boolean): Promise<void>;
+}

package/src/interfaces/utils.ts

@@ -0,0 +1,23 @@
+import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
+
+/* eslint-disable no-console */
+export const mockLogger = {
+  debug: (msg: string, data: any) => console.log(msg, data),
+  info: (msg: string, data: any) => console.log(msg, data),
+  warn: (msg: string, data: any) => console.log(msg, data),
+  error: (msg: string, data: any) => console.error(msg, data),
+  fatal: (msg: string, data: any) => console.error(msg, data),
+  silent: (_msg: string, _data: any) => {},
+  verbose: (msg: string, data: any) => console.log(msg, data),
+  trace: (msg: string, data: any) => console.log(msg, data),
+  level: 'trace' as const,
+  isLevelEnabled: (_level: string) => true,
+  module: 'kv-store:mock-logger',
+  createChild: () => mockLogger,
+  getBindings: () => ({}),
+};
+/* eslint-enable no-console */
+
+export function isSyncStore(store: AztecKVStore | AztecAsyncKVStore): store is AztecAsyncKVStore {
+  return (store as AztecKVStore).syncGetters === true;
+}

package/src/lmdb/array.ts

@@ -0,0 +1,140 @@
+import type { Database, Key } from 'lmdb';
+
+import type { AztecArray, AztecAsyncArray } from '../interfaces/array.js';
+import type { Value } from '../interfaces/common.js';
+import { LmdbAztecSingleton } from './singleton.js';
+
+/** The shape of a key that stores a value in an array */
+type ArrayIndexSlot = ['array', string, 'slot', number];
+
+/**
+ * An persistent array backed by LMDB.
+ */
+export class LmdbAztecArray<T extends Value> implements AztecArray<T>, AztecAsyncArray<T> {
+  #db: Database<T, ArrayIndexSlot>;
+  #name: string;
+  #length: LmdbAztecSingleton<number>;
+
+  constructor(db: Database<unknown, Key>, arrName: string) {
+    this.#name = arrName;
+    this.#length = new LmdbAztecSingleton(db, `${arrName}:meta:length`);
+    this.#db = db as Database<T, ArrayIndexSlot>;
+  }
+
+  get length(): number {
+    return this.#length.get() ?? 0;
+  }
+
+  lengthAsync(): Promise<number> {
+    return Promise.resolve(this.length);
+  }
+
+  push(...vals: T[]): Promise<number> {
+    return this.#db.childTransaction(() => {
+      let length = this.length;
+      for (const val of vals) {
+        void this.#db.put(this.#slot(length), val);
+        length += 1;
+      }
+
+      void this.#length.set(length);
+
+      return length;
+    });
+  }
+
+  pop(): Promise<T | undefined> {
+    return this.#db.childTransaction(() => {
+      const length = this.length;
+      if (length === 0) {
+        return undefined;
+      }
+
+      const slot = this.#slot(length - 1);
+      const val = this.#db.get(slot) as T;
+
+      void this.#db.remove(slot);
+      void this.#length.set(length - 1);
+
+      return val;
+    });
+  }
+
+  at(index: number): T | undefined {
+    if (index < 0) {
+      index = this.length + index;
+    }
+
+    // the Array API only accepts indexes in the range [-this.length, this.length)
+    // so if after normalizing the index is still out of range, return undefined
+    if (index < 0 || index >= this.length) {
+      return undefined;
+    }
+
+    return this.#db.get(this.#slot(index));
+  }
+
+  atAsync(index: number): Promise<T | undefined> {
+    return Promise.resolve(this.at(index));
+  }
+
+  setAt(index: number, val: T): Promise<boolean> {
+    if (index < 0) {
+      index = this.length + index;
+    }
+
+    if (index < 0 || index >= this.length) {
+      return Promise.resolve(false);
+    }
+
+    return this.#db.put(this.#slot(index), val);
+  }
+
+  *entries(): IterableIterator<[number, T]> {
+    const transaction = this.#db.useReadTransaction();
+    try {
+      const values = this.#db.getRange({
+        start: this.#slot(0),
+        limit: this.length,
+        transaction,
+      });
+
+      for (const { key, value } of values) {
+        const index = key[3];
+        yield [index, value];
+      }
+    } finally {
+      transaction.done();
+    }
+  }
+
+  async *entriesAsync(): AsyncIterableIterator<[number, T]> {
+    for (const [key, value] of this.entries()) {
+      yield [key, value];
+    }
+  }
+
+  *values(): IterableIterator<T> {
+    for (const [_, value] of this.entries()) {
+      yield value;
+    }
+  }
+
+  async *valuesAsync(): AsyncIterableIterator<T> {
+    for (const [_, value] of this.entries()) {
+      yield value;
+    }
+  }
+
+  [Symbol.iterator](): IterableIterator<T> {
+    return this.values();
+  }
+
+  [Symbol.asyncIterator](): AsyncIterableIterator<T> {
+    return this.valuesAsync();
+  }
+
+  #slot(index: number): ArrayIndexSlot {
+    return ['array', this.#name, 'slot', index];
+  }
+}

package/src/lmdb/counter.ts

@@ -0,0 +1,67 @@
+import type { Key as BaseKey, Database } from 'lmdb';
+
+import type { Key, Range } from '../interfaces/common.js';
+import type { AztecAsyncCounter, AztecCounter } from '../interfaces/counter.js';
+import { LmdbAztecMap } from './map.js';
+
+/**
+ * A counter implementation backed by LMDB
+ */
+export class LmdbAztecCounter<K extends Key> implements AztecCounter<K>, AztecAsyncCounter<K> {
+  #db: Database;
+  #name: string;
+  #map: LmdbAztecMap<K, number>;
+
+  constructor(db: Database<unknown, BaseKey>, name: string) {
+    this.#db = db;
+    this.#name = name;
+    this.#map = new LmdbAztecMap(db, name);
+  }
+
+  async set(key: K, value: number): Promise<void> {
+    await this.#map.set(key, value);
+  }
+
+  update(key: K, delta = 1): Promise<void> {
+    return this.#db.childTransaction(() => {
+      const current = this.#map.get(key) ?? 0;
+      const next = current + delta;
+
+      if (next < 0) {
+        throw new Error(`Cannot update ${key} in counter ${this.#name} below zero`);
+      }
+
+      if (next === 0) {
+        void this.#map.delete(key);
+      } else {
+        // store the key inside the entry because LMDB might return an internal representation
+        // of the key when iterating over the database
+        void this.#map.set(key, next);
+      }
+    });
+  }
+
+  get(key: K): number {
+    return this.#map.get(key) ?? 0;
+  }
+
+  getAsync(key: K): Promise<number> {
+    return Promise.resolve(this.get(key));
+  }
+
+  entries(range: Range<K> = {}): IterableIterator<[K, number]> {
+    return this.#map.entries(range);
+  }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, number]> {
+    yield* this.entries(range);
+  }
+
+  keys(range: Range<K> = {}): IterableIterator<K> {
+    return this.#map.keys(range);
+  }
+
+  async *keysAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    yield* this.keys(range);
+  }
+}

package/src/lmdb/index.ts

@@ -0,0 +1,41 @@
+import { type Logger, createLogger } from '@aztec-labs/foundation/log';
+import type { DataStoreConfig } from '@aztec-labs/stdlib/kv-store';
+import { join } from 'path';
+
+import { initStoreForRollupAndSchemaVersion } from '../utils.js';
+import { AztecLmdbStore } from './store.js';
+
+export { AztecLmdbStore } from './store.js';
+
+export function createStore(
+  name: string,
+  config: DataStoreConfig,
+  schemaVersion: number | undefined = undefined,
+  log: Logger = createLogger('kv-store'),
+) {
+  let { dataDirectory } = config;
+  if (typeof dataDirectory !== 'undefined') {
+    dataDirectory = join(dataDirectory, name);
+  }
+
+  log.info(
+    dataDirectory
+      ? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.dataStoreMapSizeKb} KB`
+      : `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKb} KB`,
+  );
+
+  const store = AztecLmdbStore.open(dataDirectory, config.dataStoreMapSizeKb, false);
+  if (config.rollupAddress) {
+    return initStoreForRollupAndSchemaVersion(store, schemaVersion, config.rollupAddress, log);
+  }
+  return store;
+}
+/**
+ * Opens a temporary store for testing purposes.
+ * @param ephemeral - true if the store should only exist in memory and not automatically be flushed to disk. Optional
+ * @returns A new store
+ */
+export function openTmpStore(ephemeral: boolean = false): AztecLmdbStore {
+  const mapSize = 1024 * 1024 * 10; // 10 GB map size
+  return AztecLmdbStore.open(undefined, mapSize, ephemeral);
+}