@aztec-labs/kv-store 6.0.0-nightly.20260829

package/src/lmdb-v2/factory.ts

@@ -0,0 +1,184 @@
+import { EthAddress } from '@aztec-labs/foundation/eth-address';
+import { type LoggerBindings, createLogger } from '@aztec-labs/foundation/log';
+import {
+  DatabaseVersionManager,
+  type SchemaVersionMismatchPolicy,
+  type VersionFileReadFailurePolicy,
+} from '@aztec-labs/stdlib/database-version/manager';
+import type { DataStoreConfig } from '@aztec-labs/stdlib/kv-store';
+import { copyFile, mkdir, mkdtemp, rm } from 'fs/promises';
+import { tmpdir } from 'os';
+import { join } from 'path';
+
+import { AztecLMDBStoreV2 } from './store.js';
+
+const MAX_READERS = 16;
+
+/** Optional versioning hooks for persistent LMDB stores. */
+export type CreateStoreOptions = {
+  onUpgrade?: (dataDir: string, currentVersion: number, latestVersion: number) => Promise<void>;
+  schemaVersionMismatchPolicy?: SchemaVersionMismatchPolicy;
+  versionFileReadFailurePolicy?: VersionFileReadFailurePolicy;
+};
+
+export async function createStore(
+  name: string,
+  schemaVersion: number,
+  config: DataStoreConfig,
+  bindings?: LoggerBindings,
+  options: CreateStoreOptions = {},
+): Promise<AztecLMDBStoreV2> {
+  const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
+  const { dataDirectory, rollupAddress: rollupFromConfig } = config;
+
+  let store: AztecLMDBStoreV2;
+  if (typeof dataDirectory !== 'undefined') {
+    // Get rollup address from contracts config, or use zero address
+    const subDir = join(dataDirectory, name);
+    await mkdir(subDir, { recursive: true });
+
+    const rollupAddress = rollupFromConfig ?? EthAddress.ZERO;
+
+    // Create a version manager
+    const versionManager = new DatabaseVersionManager({
+      schemaVersion,
+      rollupAddress,
+      dataDirectory: subDir,
+      onOpen: dbDirectory =>
+        AztecLMDBStoreV2.new(dbDirectory, config.dataStoreMapSizeKb, MAX_READERS, () => Promise.resolve(), bindings),
+      onUpgrade: options.onUpgrade,
+      schemaVersionMismatchPolicy: options.schemaVersionMismatchPolicy,
+      versionFileReadFailurePolicy: options.versionFileReadFailurePolicy,
+    });
+
+    log.info(
+      `Creating ${name} data store at directory ${subDir} with map size ${config.dataStoreMapSizeKb} KB (LMDB v2)`,
+    );
+    [store] = await versionManager.open();
+  } else {
+    store = await openTmpStore(name, true, config.dataStoreMapSizeKb, MAX_READERS, bindings);
+  }
+
+  return store;
+}
+
+/**
+ * Open a persistent on-disk store rooted in OS tmpdir. Caller chooses whether to
+ * auto-remove the directory on close. Uses standard durable LMDB flags (fsync on
+ * every commit) — for full in-memory / NOSYNC semantics use `openEphemeralStore`.
+ */
+export async function openTmpStore(
+  name: string,
+  cleanupTmpDir: boolean = true,
+  dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
+  maxReaders = MAX_READERS,
+  bindings?: LoggerBindings,
+): Promise<AztecLMDBStoreV2> {
+  const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
+  const dataDir = await mkdtemp(join(tmpdir(), name + '-'));
+  log.debug(`Created temporary data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
+
+  // pass a cleanup callback because process.on('beforeExit', cleanup) does not work under Jest
+  const cleanup = async () => {
+    if (cleanupTmpDir) {
+      try {
+        await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
+        log.debug(`Deleted temporary data store: ${dataDir}`);
+      } catch (err) {
+        log.warn(`Failed to delete temporary data directory (LMDB v2) ${dataDir}: ${err}`);
+      }
+    } else {
+      log.debug(`Leaving temporary data store: ${dataDir}`);
+    }
+  };
+
+  return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, bindings, false);
+}
+
+/**
+ * Open a fully-ephemeral store: a fresh tmpdir backing file, LMDB opened with
+ * MDB_NOSYNC | MDB_NOMETASYNC (no fsync, kernel-only writeback), and the directory
+ * unconditionally removed on close. Intended for tests and worker-local scratch
+ * stores where durability is explicitly not desired.
+ */
+export async function openEphemeralStore(
+  name: string,
+  dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
+  maxReaders = MAX_READERS,
+  bindings?: LoggerBindings,
+): Promise<AztecLMDBStoreV2> {
+  const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
+  const dataDir = await mkdtemp(join(tmpdir(), name + '-'));
+  log.debug(`Created ephemeral data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
+
+  const cleanup = async () => {
+    try {
+      await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
+      log.debug(`Deleted ephemeral data store: ${dataDir}`);
+    } catch (err) {
+      log.warn(`Failed to delete ephemeral data directory (LMDB v2) ${dataDir}: ${err}`);
+    }
+  };
+
+  return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, bindings, true);
+}
+
+export async function openStoreAt(
+  dataDir: string,
+  dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
+  maxReaders = MAX_READERS,
+  bindings?: LoggerBindings,
+): Promise<AztecLMDBStoreV2> {
+  const log = createLogger('kv-store:lmdb-v2', bindings);
+  log.debug(`Opening data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
+  return await AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, bindings);
+}
+
+/**
+ * Open a fully-ephemeral store seeded from an existing `data.mdb` file. Creates a fresh tmpdir,
+ * copies `srcDataMdbPath` into it as `data.mdb`, opens with MDB_NOSYNC | MDB_NOMETASYNC, and
+ * removes the tmpdir on close. Intended for worker clones of a main-thread-built store — the
+ * source file is never written to.
+ */
+export async function cloneEphemeralStoreFrom(
+  srcDataMdbPath: string,
+  name: string,
+  dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
+  maxReaders = MAX_READERS,
+  bindings?: LoggerBindings,
+): Promise<AztecLMDBStoreV2> {
+  const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
+  const dataDir = await mkdtemp(join(tmpdir(), name + '-'));
+  await copyFile(srcDataMdbPath, join(dataDir, 'data.mdb'));
+  log.debug(`Cloned ephemeral data store at: ${dataDir} from ${srcDataMdbPath} (LMDB v2)`);
+
+  const cleanup = async () => {
+    try {
+      await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
+      log.debug(`Deleted ephemeral data store: ${dataDir}`);
+    } catch (err) {
+      log.warn(`Failed to delete ephemeral data directory (LMDB v2) ${dataDir}: ${err}`);
+    }
+  };
+
+  return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, bindings, true);
+}
+
+export async function openVersionedStoreAt(
+  dataDirectory: string,
+  schemaVersion: number,
+  rollupAddress: EthAddress,
+  dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
+  maxReaders = MAX_READERS,
+  bindings?: LoggerBindings,
+): Promise<AztecLMDBStoreV2> {
+  const log = createLogger('kv-store:lmdb-v2', bindings);
+  log.debug(`Opening data store at: ${dataDirectory} with size: ${dbMapSizeKb} KB (LMDB v2)`);
+  const [store] = await new DatabaseVersionManager({
+    schemaVersion,
+    rollupAddress,
+    dataDirectory,
+    onOpen: dataDir => AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, bindings),
+  }).open();
+  return store;
+}

package/src/lmdb-v2/index.ts

@@ -0,0 +1,2 @@
+export * from './store.js';
+export * from './factory.js';

package/src/lmdb-v2/map.ts

@@ -0,0 +1,136 @@
+import { Encoder } from 'msgpackr';
+
+import type { Key, Range, Value } from '../interfaces/common.js';
+import type { AztecAsyncMap } from '../interfaces/map.js';
+import type { ReadTransaction } from './read_transaction.js';
+import type { AztecLMDBStoreV2 } from './store.js';
+import { execInReadTx, execInWriteTx } from './tx-helpers.js';
+import { deserializeKey, maxKey, minKey, serializeKey } from './utils.js';
+
+export class LMDBMap<K extends Key, V extends Value> implements AztecAsyncMap<K, V> {
+  private prefix: string;
+  private encoder = new Encoder();
+
+  constructor(
+    private store: AztecLMDBStoreV2,
+    name: string,
+  ) {
+    this.prefix = `map:${name}`;
+  }
+  /**
+   * Sets the value at the given key.
+   * @param key - The key to set the value at
+   * @param val - The value to set
+   */
+  set(key: K, val: V): Promise<void> {
+    return execInWriteTx(this.store, tx => tx.set(serializeKey(this.prefix, key), this.encoder.pack(val)));
+  }
+
+  /**
+   * Sets the values at the given keys.
+   * @param entries - The entries to set
+   */
+  async setMany(entries: { key: K; value: V }[]): Promise<void> {
+    await execInWriteTx(this.store, async tx => {
+      for (const { key, value } of entries) {
+        await tx.set(serializeKey(this.prefix, key), this.encoder.pack(value));
+      }
+    });
+  }
+
+  /**
+   * Sets the value at the given key if it does not already exist.
+   * @param key - The key to set the value at
+   * @param val - The value to set
+   */
+  setIfNotExists(key: K, val: V): Promise<boolean> {
+    return execInWriteTx(this.store, async tx => {
+      const strKey = serializeKey(this.prefix, key);
+      const exists = !!(await tx.get(strKey));
+      if (!exists) {
+        await tx.set(strKey, this.encoder.pack(val));
+        return true;
+      }
+      return false;
+    });
+  }
+
+  /**
+   * Deletes the value at the given key.
+   * @param key - The key to delete the value at
+   */
+  delete(key: K): Promise<void> {
+    return execInWriteTx(this.store, tx => tx.remove(serializeKey(this.prefix, key)));
+  }
+
+  getAsync(key: K): Promise<V | undefined> {
+    return execInReadTx(this.store, async tx => {
+      const val = await tx.get(serializeKey(this.prefix, key));
+      return val ? this.encoder.unpack(val) : undefined;
+    });
+  }
+
+  hasAsync(key: K): Promise<boolean> {
+    return execInReadTx(this.store, async tx => !!(await tx.get(serializeKey(this.prefix, key))));
+  }
+
+  sizeAsync(): Promise<number> {
+    return execInReadTx(this.store, tx => tx.countEntries(minKey(this.prefix), maxKey(this.prefix), false));
+  }
+
+  /**
+   * Iterates over the map's key-value entries in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
+    const reverse = range?.reverse ?? false;
+
+    const startKey = range?.start !== undefined ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
+
+    const endKey =
+      range?.end !== undefined ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined;
+
+    let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx();
+    const shouldClose = !tx;
+    tx ??= this.store.getReadTx();
+
+    try {
+      for await (const [key, val] of tx.iterate(
+        reverse ? endKey! : startKey,
+        reverse ? startKey : endKey,
+        reverse,
+        range?.limit,
+      )) {
+        const deserializedKey = deserializeKey<K>(this.prefix, key);
+        if (deserializedKey === false) {
+          break;
+        }
+        yield [deserializedKey, this.encoder.unpack(val)];
+      }
+    } finally {
+      if (shouldClose) {
+        tx.close();
+      }
+    }
+  }
+
+  /**
+   * Iterates over the map's values in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  async *valuesAsync(range?: Range<K>): AsyncIterableIterator<V> {
+    for await (const [_, value] of this.entriesAsync(range)) {
+      yield value;
+    }
+  }
+
+  /**
+   * Iterates over the map's keys in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  async *keysAsync(range?: Range<K>): AsyncIterableIterator<K> {
+    for await (const [key, _] of this.entriesAsync(range)) {
+      yield key;
+    }
+  }
+}

package/src/lmdb-v2/message.ts

@@ -0,0 +1,169 @@
+export enum Database {
+  DATA = 'data',
+  INDEX = 'index',
+}
+
+export const CURSOR_PAGE_SIZE = 10;
+
+export enum LMDBMessageType {
+  OPEN_DATABASE = 100,
+  GET,
+  HAS,
+
+  START_CURSOR,
+  ADVANCE_CURSOR,
+  ADVANCE_CURSOR_COUNT,
+  CLOSE_CURSOR,
+
+  BATCH,
+
+  STATS,
+
+  CLOSE,
+  COPY_STORE,
+}
+
+type Key = Uint8Array;
+type Value = Uint8Array;
+type OptionalValues = Array<Value[] | null>;
+type KeyOptionalValues = [Key, null | Array<Value>];
+type KeyValues = [Key, Value[]];
+
+interface OpenDatabaseRequest {
+  db: string;
+  uniqueKeys?: boolean;
+}
+
+interface GetRequest {
+  keys: Key[];
+  db: string;
+}
+
+interface GetResponse {
+  values: OptionalValues;
+}
+
+interface HasRequest {
+  entries: KeyOptionalValues[];
+  db: string;
+}
+
+interface StartCursorRequest {
+  key: Key;
+  reverse: boolean;
+  count: number | null;
+  onePage: boolean | null;
+  db: string;
+}
+
+interface AdvanceCursorRequest {
+  cursor: number;
+  count: number | null;
+}
+
+interface AdvanceCursorCountRequest {
+  cursor: number;
+  endKey: Key;
+}
+
+interface CloseCursorRequest {
+  cursor: number;
+}
+
+interface CopyStoreRequest {
+  dstPath: string;
+  compact: boolean;
+}
+
+export interface Batch {
+  addEntries: Array<KeyValues>;
+  removeEntries: Array<KeyOptionalValues>;
+}
+
+interface BatchRequest {
+  batches: Map<string, Batch>;
+}
+
+export type LMDBRequestBody = {
+  [LMDBMessageType.OPEN_DATABASE]: OpenDatabaseRequest;
+
+  [LMDBMessageType.GET]: GetRequest;
+  [LMDBMessageType.HAS]: HasRequest;
+
+  [LMDBMessageType.START_CURSOR]: StartCursorRequest;
+  [LMDBMessageType.ADVANCE_CURSOR]: AdvanceCursorRequest;
+  [LMDBMessageType.ADVANCE_CURSOR_COUNT]: AdvanceCursorCountRequest;
+  [LMDBMessageType.CLOSE_CURSOR]: CloseCursorRequest;
+
+  [LMDBMessageType.BATCH]: BatchRequest;
+
+  [LMDBMessageType.STATS]: void;
+
+  [LMDBMessageType.CLOSE]: void;
+  [LMDBMessageType.COPY_STORE]: CopyStoreRequest;
+};
+
+interface GetResponse {
+  values: OptionalValues;
+}
+
+interface HasResponse {
+  exists: boolean[];
+}
+
+interface StartCursorResponse {
+  cursor: number | null;
+  entries: Array<KeyValues>;
+}
+
+interface AdvanceCursorResponse {
+  entries: Array<KeyValues>;
+  done: boolean;
+}
+
+interface AdvanceCursorCountResponse {
+  count: number;
+  done: boolean;
+}
+
+interface BatchResponse {
+  durationNs: number;
+}
+
+interface BoolResponse {
+  ok: true;
+}
+
+interface StatsResponse {
+  stats: Array<{
+    name: string;
+    numDataItems: bigint | number;
+    totalUsedSize: bigint | number;
+  }>;
+  dbMapSizeBytes: bigint | number;
+  dbPhysicalFileSizeBytes: bigint | number;
+}
+
+export type LMDBResponseBody = {
+  [LMDBMessageType.OPEN_DATABASE]: BoolResponse;
+
+  [LMDBMessageType.GET]: GetResponse;
+  [LMDBMessageType.HAS]: HasResponse;
+
+  [LMDBMessageType.START_CURSOR]: StartCursorResponse;
+  [LMDBMessageType.ADVANCE_CURSOR]: AdvanceCursorResponse;
+  [LMDBMessageType.ADVANCE_CURSOR_COUNT]: AdvanceCursorCountResponse;
+  [LMDBMessageType.CLOSE_CURSOR]: BoolResponse;
+
+  [LMDBMessageType.BATCH]: BatchResponse;
+
+  [LMDBMessageType.STATS]: StatsResponse;
+
+  [LMDBMessageType.CLOSE]: BoolResponse;
+
+  [LMDBMessageType.COPY_STORE]: BoolResponse;
+};
+
+export interface LMDBMessageChannel {
+  sendMessage<T extends LMDBMessageType>(msgType: T, body: LMDBRequestBody[T]): Promise<LMDBResponseBody[T]>;
+}

package/src/lmdb-v2/multi_map.ts

@@ -0,0 +1,150 @@
+import { Encoder } from 'msgpackr/pack';
+import { MAXIMUM_KEY, toBufferKey } from 'ordered-binary';
+
+import type { Key, Range, Value } from '../interfaces/common.js';
+import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
+import type { ReadTransaction } from './read_transaction.js';
+import type { AztecLMDBStoreV2 } from './store.js';
+import { execInReadTx, execInWriteTx } from './tx-helpers.js';
+import { deserializeKey, maxKey, minKey, serializeKey } from './utils.js';
+
+export class LMDBMultiMap<K extends Key, V extends Value> implements AztecAsyncMultiMap<K, V> {
+  private prefix: string;
+  private encoder = new Encoder();
+  constructor(
+    private store: AztecLMDBStoreV2,
+    name: string,
+  ) {
+    this.prefix = `multimap:${name}`;
+  }
+
+  /**
+   * Sets the value at the given key.
+   * @param key - The key to set the value at
+   * @param val - The value to set
+   */
+  set(key: K, val: V): Promise<void> {
+    return execInWriteTx(this.store, tx => tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
+  }
+
+  async setMany(entries: { key: K; value: V }[]): Promise<void> {
+    await execInWriteTx(this.store, async tx => {
+      for (const { key, value } of entries) {
+        await tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(value));
+      }
+    });
+  }
+
+  /**
+   * Sets the value at the given key if it does not already exist.
+   * @param key - The key to set the value at
+   * @param val - The value to set
+   */
+  setIfNotExists(key: K, val: V): Promise<boolean> {
+    return execInWriteTx(this.store, async tx => {
+      const exists = !!(await this.getAsync(key));
+      if (!exists) {
+        await tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val));
+        return true;
+      }
+      return false;
+    });
+  }
+
+  /**
+   * Deletes the value at the given key.
+   * @param key - The key to delete the value at
+   */
+  delete(key: K): Promise<void> {
+    return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key)));
+  }
+
+  getAsync(key: K): Promise<V | undefined> {
+    return execInReadTx(this.store, async tx => {
+      const val = await tx.getIndex(serializeKey(this.prefix, key));
+      return val.length > 0 ? this.encoder.unpack(val[0]) : undefined;
+    });
+  }
+
+  hasAsync(key: K): Promise<boolean> {
+    return execInReadTx(this.store, async tx => (await tx.getIndex(serializeKey(this.prefix, key))).length > 0);
+  }
+
+  sizeAsync(): Promise<number> {
+    return execInReadTx(this.store, tx => tx.countEntriesIndex(minKey(this.prefix), maxKey(this.prefix), false));
+  }
+
+  /**
+   * Iterates over the map's key-value entries in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
+    const reverse = range?.reverse ?? false;
+    const startKey = range?.start ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
+    const endKey = range?.end ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined;
+
+    let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx();
+    const shouldClose = !tx;
+    tx ??= this.store.getReadTx();
+
+    try {
+      for await (const [key, vals] of tx.iterateIndex(
+        reverse ? endKey! : startKey,
+        reverse ? startKey : endKey,
+        reverse,
+        range?.limit,
+      )) {
+        const deserializedKey = deserializeKey<K>(this.prefix, key);
+        if (!deserializedKey) {
+          break;
+        }
+
+        for (const val of vals) {
+          yield [deserializedKey, this.encoder.unpack(val)];
+        }
+      }
+    } finally {
+      if (shouldClose) {
+        tx.close();
+      }
+    }
+  }
+
+  /**
+   * Iterates over the map's values in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  async *valuesAsync(range?: Range<K>): AsyncIterableIterator<V> {
+    for await (const [_, value] of this.entriesAsync(range)) {
+      yield value;
+    }
+  }
+
+  /**
+   * Iterates over the map's keys in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  async *keysAsync(range?: Range<K>): AsyncIterableIterator<K> {
+    for await (const [key, _] of this.entriesAsync(range)) {
+      yield key;
+    }
+  }
+
+  deleteValue(key: K, val: V | undefined): Promise<void> {
+    return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
+  }
+
+  async *getValuesAsync(key: K): AsyncIterableIterator<V> {
+    const values = await execInReadTx(this.store, tx => tx.getIndex(serializeKey(this.prefix, key)));
+    for (const value of values) {
+      yield this.encoder.unpack(value);
+    }
+  }
+
+  getValueCountAsync(key: K): Promise<number> {
+    const startKey = serializeKey(this.prefix, key);
+    const endKey = toBufferKey([this.prefix, key, MAXIMUM_KEY]);
+
+    return execInReadTx(this.store, tx => tx.countEntriesIndex(startKey, endKey, false));
+  }
+}