@aztec/kv-store 0.0.0-test.0

package/src/lmdb/array.ts

@@ -0,0 +1,139 @@
+import type { Database, Key } from 'lmdb';
+
+import type { AztecArray, AztecAsyncArray } from '../interfaces/array.js';
+import { LmdbAztecSingleton } from './singleton.js';
+
+/** The shape of a key that stores a value in an array */
+type ArrayIndexSlot = ['array', string, 'slot', number];
+
+/**
+ * An persistent array backed by LMDB.
+ */
+export class LmdbAztecArray<T> implements AztecArray<T>, AztecAsyncArray<T> {
+  #db: Database<T, ArrayIndexSlot>;
+  #name: string;
+  #length: LmdbAztecSingleton<number>;
+
+  constructor(db: Database<unknown, Key>, arrName: string) {
+    this.#name = arrName;
+    this.#length = new LmdbAztecSingleton(db, `${arrName}:meta:length`);
+    this.#db = db as Database<T, ArrayIndexSlot>;
+  }
+
+  get length(): number {
+    return this.#length.get() ?? 0;
+  }
+
+  lengthAsync(): Promise<number> {
+    return Promise.resolve(this.length);
+  }
+
+  push(...vals: T[]): Promise<number> {
+    return this.#db.childTransaction(() => {
+      let length = this.length;
+      for (const val of vals) {
+        void this.#db.put(this.#slot(length), val);
+        length += 1;
+      }
+
+      void this.#length.set(length);
+
+      return length;
+    });
+  }
+
+  pop(): Promise<T | undefined> {
+    return this.#db.childTransaction(() => {
+      const length = this.length;
+      if (length === 0) {
+        return undefined;
+      }
+
+      const slot = this.#slot(length - 1);
+      const val = this.#db.get(slot) as T;
+
+      void this.#db.remove(slot);
+      void this.#length.set(length - 1);
+
+      return val;
+    });
+  }
+
+  at(index: number): T | undefined {
+    if (index < 0) {
+      index = this.length + index;
+    }
+
+    // the Array API only accepts indexes in the range [-this.length, this.length)
+    // so if after normalizing the index is still out of range, return undefined
+    if (index < 0 || index >= this.length) {
+      return undefined;
+    }
+
+    return this.#db.get(this.#slot(index));
+  }
+
+  atAsync(index: number): Promise<T | undefined> {
+    return Promise.resolve(this.at(index));
+  }
+
+  setAt(index: number, val: T): Promise<boolean> {
+    if (index < 0) {
+      index = this.length + index;
+    }
+
+    if (index < 0 || index >= this.length) {
+      return Promise.resolve(false);
+    }
+
+    return this.#db.put(this.#slot(index), val);
+  }
+
+  *entries(): IterableIterator<[number, T]> {
+    const transaction = this.#db.useReadTransaction();
+    try {
+      const values = this.#db.getRange({
+        start: this.#slot(0),
+        limit: this.length,
+        transaction,
+      });
+
+      for (const { key, value } of values) {
+        const index = key[3];
+        yield [index, value];
+      }
+    } finally {
+      transaction.done();
+    }
+  }
+
+  async *entriesAsync(): AsyncIterableIterator<[number, T]> {
+    for (const [key, value] of this.entries()) {
+      yield [key, value];
+    }
+  }
+
+  *values(): IterableIterator<T> {
+    for (const [_, value] of this.entries()) {
+      yield value;
+    }
+  }
+
+  async *valuesAsync(): AsyncIterableIterator<T> {
+    for (const [_, value] of this.entries()) {
+      yield value;
+    }
+  }
+
+  [Symbol.iterator](): IterableIterator<T> {
+    return this.values();
+  }
+
+  [Symbol.asyncIterator](): AsyncIterableIterator<T> {
+    return this.valuesAsync();
+  }
+
+  #slot(index: number): ArrayIndexSlot {
+    return ['array', this.#name, 'slot', index];
+  }
+}

package/src/lmdb/counter.ts

@@ -0,0 +1,67 @@
+import type { Key as BaseKey, Database } from 'lmdb';
+
+import type { Key, Range } from '../interfaces/common.js';
+import type { AztecAsyncCounter, AztecCounter } from '../interfaces/counter.js';
+import { LmdbAztecMap } from './map.js';
+
+/**
+ * A counter implementation backed by LMDB
+ */
+export class LmdbAztecCounter<K extends Key> implements AztecCounter<K>, AztecAsyncCounter<K> {
+  #db: Database;
+  #name: string;
+  #map: LmdbAztecMap<K, number>;
+
+  constructor(db: Database<unknown, BaseKey>, name: string) {
+    this.#db = db;
+    this.#name = name;
+    this.#map = new LmdbAztecMap(db, name);
+  }
+
+  async set(key: K, value: number): Promise<void> {
+    await this.#map.set(key, value);
+  }
+
+  update(key: K, delta = 1): Promise<void> {
+    return this.#db.childTransaction(() => {
+      const current = this.#map.get(key) ?? 0;
+      const next = current + delta;
+
+      if (next < 0) {
+        throw new Error(`Cannot update ${key} in counter ${this.#name} below zero`);
+      }
+
+      if (next === 0) {
+        void this.#map.delete(key);
+      } else {
+        // store the key inside the entry because LMDB might return an internal representation
+        // of the key when iterating over the database
+        void this.#map.set(key, next);
+      }
+    });
+  }
+
+  get(key: K): number {
+    return this.#map.get(key) ?? 0;
+  }
+
+  getAsync(key: K): Promise<number> {
+    return Promise.resolve(this.get(key));
+  }
+
+  entries(range: Range<K> = {}): IterableIterator<[K, number]> {
+    return this.#map.entries(range);
+  }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, number]> {
+    yield* this.entries(range);
+  }
+
+  keys(range: Range<K> = {}): IterableIterator<K> {
+    return this.#map.keys(range);
+  }
+
+  async *keysAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    yield* this.keys(range);
+  }
+}

package/src/lmdb/index.ts

@@ -0,0 +1,37 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+
+import { join } from 'path';
+
+import type { DataStoreConfig } from '../config.js';
+import { initStoreForRollup } from '../utils.js';
+import { AztecLmdbStore } from './store.js';
+
+export { AztecLmdbStore } from './store.js';
+
+export function createStore(name: string, config: DataStoreConfig, log: Logger = createLogger('kv-store')) {
+  let { dataDirectory } = config;
+  if (typeof dataDirectory !== 'undefined') {
+    dataDirectory = join(dataDirectory, name);
+  }
+
+  log.info(
+    dataDirectory
+      ? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.dataStoreMapSizeKB} KB`
+      : `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKB} KB`,
+  );
+
+  const store = AztecLmdbStore.open(dataDirectory, config.dataStoreMapSizeKB, false);
+  if (config.l1Contracts?.rollupAddress) {
+    return initStoreForRollup(store, config.l1Contracts.rollupAddress, log);
+  }
+  return store;
+}
+/**
+ * Opens a temporary store for testing purposes.
+ * @param ephemeral - true if the store should only exist in memory and not automatically be flushed to disk. Optional
+ * @returns A new store
+ */
+export function openTmpStore(ephemeral: boolean = false): AztecLmdbStore {
+  const mapSize = 1024 * 1024 * 10; // 10 GB map size
+  return AztecLmdbStore.open(undefined, mapSize, ephemeral);
+}

package/src/lmdb/map.ts

@@ -0,0 +1,248 @@
+import type { Database, RangeOptions } from 'lmdb';
+
+import type { Key, Range } from '../interfaces/common.js';
+import type { AztecAsyncMultiMap, AztecMapWithSize, AztecMultiMap } from '../interfaces/map.js';
+
+/** The slot where a key-value entry would be stored */
+type MapValueSlot<K extends Key | Buffer> = ['map', string, 'slot', K];
+
+/**
+ * A map backed by LMDB.
+ */
+export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, AztecAsyncMultiMap<K, V> {
+  protected db: Database<[K, V], MapValueSlot<K>>;
+  protected name: string;
+
+  protected startSentinel: MapValueSlot<Buffer>;
+  protected endSentinel: MapValueSlot<Buffer>;
+
+  constructor(rootDb: Database, mapName: string) {
+    this.name = mapName;
+    this.db = rootDb as Database<[K, V], MapValueSlot<K>>;
+
+    // sentinels are used to define the start and end of the map
+    // with LMDB's key encoding, no _primitive value_ can be "less than" an empty buffer or greater than Byte 255
+    // these will be used later to answer range queries
+    this.startSentinel = ['map', this.name, 'slot', Buffer.from([])];
+    this.endSentinel = ['map', this.name, 'slot', Buffer.from([255])];
+  }
+
+  close(): Promise<void> {
+    return this.db.close();
+  }
+
+  get(key: K): V | undefined {
+    return this.db.get(this.slot(key))?.[1];
+  }
+
+  getAsync(key: K): Promise<V | undefined> {
+    return Promise.resolve(this.get(key));
+  }
+
+  *getValues(key: K): IterableIterator<V> {
+    const transaction = this.db.useReadTransaction();
+    try {
+      const values = this.db.getValues(this.slot(key), {
+        transaction,
+      });
+      for (const value of values) {
+        yield value?.[1];
+      }
+    } finally {
+      transaction.done();
+    }
+  }
+
+  async *getValuesAsync(key: K): AsyncIterableIterator<V> {
+    for (const value of this.getValues(key)) {
+      yield value;
+    }
+  }
+
+  has(key: K): boolean {
+    return this.db.doesExist(this.slot(key));
+  }
+
+  hasAsync(key: K): Promise<boolean> {
+    return Promise.resolve(this.has(key));
+  }
+
+  async set(key: K, val: V): Promise<void> {
+    await this.db.put(this.slot(key), [key, val]);
+  }
+
+  swap(key: K, fn: (val: V | undefined) => V): Promise<void> {
+    return this.db.childTransaction(() => {
+      const slot = this.slot(key);
+      const entry = this.db.get(slot);
+      void this.db.put(slot, [key, fn(entry?.[1])]);
+    });
+  }
+
+  setIfNotExists(key: K, val: V): Promise<boolean> {
+    const slot = this.slot(key);
+    return this.db.ifNoExists(slot, () => {
+      void this.db.put(slot, [key, val]);
+    });
+  }
+
+  async delete(key: K): Promise<void> {
+    await this.db.remove(this.slot(key));
+  }
+
+  async deleteValue(key: K, val: V): Promise<void> {
+    await this.db.remove(this.slot(key), [key, val]);
+  }
+
+  *entries(range: Range<K> = {}): IterableIterator<[K, V]> {
+    const transaction = this.db.useReadTransaction();
+
+    try {
+      const { reverse = false, limit } = range;
+      // LMDB has a quirk where it expects start > end when reverse=true
+      // in that case, we need to swap the start and end sentinels
+      const start = reverse
+        ? range.end
+          ? this.slot(range.end)
+          : this.endSentinel
+        : range.start
+        ? this.slot(range.start)
+        : this.startSentinel;
+
+      const end = reverse
+        ? range.start
+          ? this.slot(range.start)
+          : this.startSentinel
+        : range.end
+        ? this.slot(range.end)
+        : this.endSentinel;
+
+      const lmdbRange: RangeOptions = {
+        start,
+        end,
+        reverse,
+        limit,
+        transaction,
+      };
+
+      const iterator = this.db.getRange(lmdbRange);
+
+      for (const {
+        value: [key, value],
+      } of iterator) {
+        yield [key, value];
+      }
+    } finally {
+      transaction.done();
+    }
+  }
+
+  async *entriesAsync(range?: Range<K> | undefined): AsyncIterableIterator<[K, V]> {
+    for (const entry of this.entries(range)) {
+      yield entry;
+    }
+  }
+
+  *values(range: Range<K> = {}): IterableIterator<V> {
+    for (const [_, value] of this.entries(range)) {
+      yield value;
+    }
+  }
+
+  async *valuesAsync(range: Range<K> = {}): AsyncIterableIterator<V> {
+    for await (const [_, value] of this.entriesAsync(range)) {
+      yield value;
+    }
+  }
+
+  *keys(range: Range<K> = {}): IterableIterator<K> {
+    for (const [key, _] of this.entries(range)) {
+      yield key;
+    }
+  }
+
+  async *keysAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    for await (const [key, _] of this.entriesAsync(range)) {
+      yield key;
+    }
+  }
+
+  protected slot(key: K): MapValueSlot<K> {
+    return ['map', this.name, 'slot', key];
+  }
+
+  async clear(): Promise<void> {
+    const lmdbRange: RangeOptions = {
+      start: this.startSentinel,
+      end: this.endSentinel,
+    };
+
+    const iterator = this.db.getRange(lmdbRange);
+
+    for (const { key } of iterator) {
+      await this.db.remove(key);
+    }
+  }
+}
+
+export class LmdbAztecMapWithSize<K extends Key, V>
+  extends LmdbAztecMap<K, V>
+  implements AztecMapWithSize<K, V>, AztecAsyncMultiMap<K, V>
+{
+  #sizeCache?: number;
+
+  constructor(rootDb: Database, mapName: string) {
+    super(rootDb, mapName);
+  }
+
+  override async set(key: K, val: V): Promise<void> {
+    await this.db.childTransaction(() => {
+      const exists = this.db.doesExist(this.slot(key));
+      this.db.putSync(this.slot(key), [key, val], {
+        appendDup: true,
+      });
+      if (!exists) {
+        this.#sizeCache = undefined; // Invalidate cache
+      }
+    });
+  }
+
+  override async delete(key: K): Promise<void> {
+    await this.db.childTransaction(async () => {
+      const exists = this.db.doesExist(this.slot(key));
+      if (exists) {
+        await this.db.remove(this.slot(key));
+        this.#sizeCache = undefined; // Invalidate cache
+      }
+    });
+  }
+
+  override async deleteValue(key: K, val: V): Promise<void> {
+    await this.db.childTransaction(async () => {
+      const exists = this.db.doesExist(this.slot(key));
+      if (exists) {
+        await this.db.remove(this.slot(key), [key, val]);
+        this.#sizeCache = undefined; // Invalidate cache
+      }
+    });
+  }
+
+  /**
+   * Gets the size of the map by counting entries.
+   * @returns The number of entries in the map
+   */
+  size(): number {
+    if (this.#sizeCache === undefined) {
+      this.#sizeCache = this.db.getCount({
+        start: this.startSentinel,
+        end: this.endSentinel,
+      });
+    }
+    return this.#sizeCache;
+  }
+
+  // Reset cache on clear/drop operations
+  clearCache() {
+    this.#sizeCache = undefined;
+  }
+}

package/src/lmdb/set.ts

@@ -0,0 +1,45 @@
+import type { Database } from 'lmdb';
+
+import type { Key, Range } from '../interfaces/common.js';
+import type { AztecAsyncSet, AztecSet } from '../interfaces/set.js';
+import { LmdbAztecMap } from './map.js';
+
+/**
+ * A set backed by LMDB.
+ */
+export class LmdbAztecSet<K extends Key> implements AztecSet<K>, AztecAsyncSet<K> {
+  private map: LmdbAztecMap<K, boolean>;
+  constructor(rootDb: Database, mapName: string) {
+    this.map = new LmdbAztecMap(rootDb, mapName);
+  }
+
+  close(): Promise<void> {
+    return this.map.close();
+  }
+
+  has(key: K): boolean {
+    return this.map.has(key);
+  }
+
+  hasAsync(key: K): Promise<boolean> {
+    return Promise.resolve(this.has(key));
+  }
+
+  add(key: K): Promise<void> {
+    return this.map.set(key, true);
+  }
+
+  delete(key: K): Promise<void> {
+    return this.map.delete(key);
+  }
+
+  entries(range: Range<K> = {}): IterableIterator<K> {
+    return this.map.keys(range);
+  }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    for await (const key of this.map.keysAsync(range)) {
+      yield key;
+    }
+  }
+}

package/src/lmdb/singleton.ts

@@ -0,0 +1,35 @@
+import type { Database, Key } from 'lmdb';
+
+import type { AztecAsyncSingleton, AztecSingleton } from '../interfaces/singleton.js';
+
+/** The slot where this singleton will store its value */
+type ValueSlot = ['singleton', string, 'value'];
+
+/**
+ * Stores a single value in LMDB.
+ */
+export class LmdbAztecSingleton<T> implements AztecSingleton<T>, AztecAsyncSingleton<T> {
+  #db: Database<T, ValueSlot>;
+  #slot: ValueSlot;
+
+  constructor(db: Database<unknown, Key>, name: string) {
+    this.#db = db as Database<T, ValueSlot>;
+    this.#slot = ['singleton', name, 'value'];
+  }
+
+  get(): T | undefined {
+    return this.#db.get(this.#slot);
+  }
+
+  getAsync(): Promise<T | undefined> {
+    return Promise.resolve(this.get());
+  }
+
+  set(val: T): Promise<boolean> {
+    return this.#db.put(this.#slot, val);
+  }
+
+  delete(): Promise<boolean> {
+    return this.#db.remove(this.#slot);
+  }
+}