@aztec/kv-store 0.71.0 → 0.73.0

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "@aztec/kv-store",
-  "version": "0.71.0",
+  "version": "0.73.0",
   "type": "module",
   "exports": {
     ".": "./dest/interfaces/index.js",
     "./lmdb": "./dest/lmdb/index.js",
+    "./lmdb-v2": "./dest/lmdb-v2/index.js",
     "./indexeddb": "./dest/indexeddb/index.js",
     "./stores": "./dest/stores/index.js",
     "./config": "./dest/config.js"
@@ -12,23 +13,28 @@
   "scripts": {
     "build": "yarn clean && tsc -b",
     "build:dev": "tsc -b --watch",
+    "clean:cpp": "rm -rf $(git rev-parse --show-toplevel)/barretenberg/cpp/build-pic",
     "clean": "rm -rf ./dest .tsbuildinfo",
     "formatting": "run -T prettier --check ./src && run -T eslint ./src",
     "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
     "test:node": "NODE_NO_WARNINGS=1 mocha --config ./.mocharc.json --reporter dot",
     "test:browser": "wtr --config ./web-test-runner.config.mjs",
-    "test": "yarn test:node && yarn test:browser && true"
+    "test": "yarn test:node && yarn test:browser && true",
+    "generate": "mkdir -p build && cp -v ../../barretenberg/cpp/build-pic/lib/nodejs_module.node build"
   },
   "inherits": [
     "../package.common.json",
     "./package.local.json"
   ],
   "dependencies": {
-    "@aztec/circuit-types": "0.71.0",
-    "@aztec/ethereum": "0.71.0",
-    "@aztec/foundation": "0.71.0",
+    "@aztec/circuit-types": "0.73.0",
+    "@aztec/ethereum": "0.73.0",
+    "@aztec/foundation": "0.73.0",
+    "@aztec/native": "0.73.0",
     "idb": "^8.0.0",
-    "lmdb": "^3.2.0"
+    "lmdb": "^3.2.0",
+    "msgpackr": "^1.11.2",
+    "ordered-binary": "^1.5.3"
   },
   "devDependencies": {
     "@aztec/circuits.js": "workspace:^",
@@ -39,6 +45,7 @@
     "@types/mocha": "^10.0.10",
     "@types/mocha-each": "^2.0.4",
     "@types/node": "^18.7.23",
+    "@types/sinon": "^17.0.3",
     "@web/dev-server-esbuild": "^1.0.3",
     "@web/test-runner": "^0.19.0",
     "@web/test-runner-playwright": "^0.11.0",
@@ -47,6 +54,7 @@
     "jest": "^29.5.0",
     "mocha": "^10.8.2",
     "mocha-each": "^2.0.1",
+    "sinon": "^19.0.2",
     "ts-node": "^10.9.1",
     "typescript": "^5.0.4"
   },

package/src/config.ts

@@ -1,4 +1,4 @@
-import { l1ContractAddressesMapping } from '@aztec/ethereum';
+import { l1ContractAddressesMapping } from '@aztec/ethereum/l1-contract-addresses';
 import { type ConfigMappingsType, getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config';
 import { type EthAddress } from '@aztec/foundation/eth-address';
 
@@ -20,7 +20,7 @@ export const dataConfigMappings: ConfigMappingsType<DataStoreConfig> = {
   },
   l1Contracts: {
     description: 'The deployed L1 contract addresses',
-    defaultValue: l1ContractAddressesMapping,
+    nested: l1ContractAddressesMapping,
   },
 };
 

package/src/indexeddb/store.ts

@@ -3,7 +3,7 @@ import { type Logger } from '@aztec/foundation/log';
 import { type DBSchema, type IDBPDatabase, deleteDB, openDB } from 'idb';
 
 import { type AztecAsyncArray } from '../interfaces/array.js';
-import { type Key } from '../interfaces/common.js';
+import { type Key, type StoreSize } from '../interfaces/common.js';
 import { type AztecAsyncCounter } from '../interfaces/counter.js';
 import { type AztecAsyncMap, type AztecAsyncMultiMap } from '../interfaces/map.js';
 import { type AztecAsyncSet } from '../interfaces/set.js';
@@ -124,7 +124,7 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
     return multimap;
   }
 
-  openCounter<K extends Key | Array<string | number>>(_name: string): AztecAsyncCounter<K> {
+  openCounter<K extends Key>(_name: string): AztecAsyncCounter<K> {
     throw new Error('Method not implemented.');
   }
 
@@ -187,7 +187,11 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
     return deleteDB(this.#name);
   }
 
-  estimateSize(): { mappingSize: number; actualSize: number; numItems: number } {
-    return { mappingSize: 0, actualSize: 0, numItems: 0 };
+  estimateSize(): Promise<StoreSize> {
+    return Promise.resolve({ mappingSize: 0, actualSize: 0, numItems: 0 });
+  }
+
+  close(): Promise<void> {
+    return Promise.resolve();
   }
 }

package/src/interfaces/common.ts

@@ -1,7 +1,7 @@
 /**
  * The key type for use with the kv-store
  */
-export type Key = string | number | Array<string | number>;
+export type Key = string | number;
 
 /**
  * A range of keys to iterate over.
@@ -16,3 +16,5 @@ export type Range<K extends Key = Key> = {
   /** The maximum number of items to iterate over */
   limit?: number;
 };
+
+export type StoreSize = { mappingSize: number; actualSize: number; numItems: number };

package/src/interfaces/index.ts

@@ -4,4 +4,4 @@ export * from './counter.js';
 export * from './singleton.js';
 export * from './store.js';
 export * from './set.js';
-export { Range } from './common.js';
+export { Range, StoreSize } from './common.js';

package/src/interfaces/map.ts

@@ -11,13 +11,6 @@ interface AztecBaseMap<K extends Key, V> {
    */
   set(key: K, val: V): Promise<void>;
 
-  /**
-   * Atomically swap the value at the given key
-   * @param key - The key to swap the value at
-   * @param fn - The function to swap the value with
-   */
-  swap(key: K, fn: (val: V | undefined) => V): Promise<void>;
-
   /**
    * Sets the value at the given key if it does not already exist.
    * @param key - The key to set the value at

package/src/interfaces/map_test_suite.ts

@@ -18,7 +18,7 @@ export function describeAztecMap(
 
     beforeEach(async () => {
       store = await getStore();
-      map = store.openMultiMap<string | [number, string], string>('test');
+      map = store.openMultiMap<string, string>('test');
     });
 
     afterEach(async () => {
@@ -125,21 +125,6 @@ export function describeAztecMap(
       expect(await getValues('foo')).to.deep.equal(['baz']);
     });
 
-    it('supports tuple keys', async () => {
-      // Use a new map because key structure has changed
-      const tupleMap = store.openMap<[number, string], string>('test-tuple');
-
-      await tupleMap.set([5, 'bar'], 'val');
-      await tupleMap.set([0, 'foo'], 'val');
-
-      expect(await keys(undefined, tupleMap)).to.deep.equal([
-        [0, 'foo'],
-        [5, 'bar'],
-      ]);
-
-      expect(await get([5, 'bar'], tupleMap)).to.equal('val');
-    });
-
     it('supports range queries', async () => {
       await map.set('a', 'a');
       await map.set('b', 'b');

package/src/interfaces/store.ts

@@ -1,5 +1,5 @@
 import { type AztecArray, type AztecAsyncArray } from './array.js';
-import { type Key } from './common.js';
+import { type Key, type StoreSize } from './common.js';
 import { type AztecAsyncCounter, type AztecCounter } from './counter.js';
 import {
   type AztecAsyncMap,
@@ -94,7 +94,12 @@ export interface AztecKVStore {
   /**
    * Estimates the size of the store in bytes.
    */
-  estimateSize(): { mappingSize: number; actualSize: number; numItems: number };
+  estimateSize(): Promise<StoreSize>;
+
+  /**
+   * Closes the store
+   */
+  close(): Promise<void>;
 }
 
 export interface AztecAsyncKVStore {
@@ -163,5 +168,10 @@ export interface AztecAsyncKVStore {
   /**
    * Estimates the size of the store in bytes.
    */
-  estimateSize(): { mappingSize: number; actualSize: number; numItems: number };
+  estimateSize(): Promise<StoreSize>;
+
+  /**
+   * Closes the store
+   */
+  close(): Promise<void>;
 }

package/src/lmdb/store.ts

@@ -7,7 +7,7 @@ import { tmpdir } from 'os';
 import { join } from 'path';
 
 import { type AztecArray, type AztecAsyncArray } from '../interfaces/array.js';
-import { type Key } from '../interfaces/common.js';
+import { type Key, type StoreSize } from '../interfaces/common.js';
 import { type AztecAsyncCounter, type AztecCounter } from '../interfaces/counter.js';
 import {
   type AztecAsyncMap,
@@ -216,7 +216,7 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
     }
   }
 
-  estimateSize(): { mappingSize: number; actualSize: number; numItems: number } {
+  estimateSize(): Promise<StoreSize> {
     const stats = this.#rootDb.getStats();
     // The 'mapSize' is the total amount of virtual address space allocated to the DB (effectively the maximum possible size)
     // http://www.lmdb.tech/doc/group__mdb.html#a4bde3c8b676457342cba2fe27aed5fbd
@@ -226,11 +226,11 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
     }
     const dataResult = this.estimateSubDBSize(this.#data);
     const multiResult = this.estimateSubDBSize(this.#multiMapData);
-    return {
+    return Promise.resolve({
       mappingSize: mapSize,
       actualSize: dataResult.actualSize + multiResult.actualSize,
       numItems: dataResult.numItems + multiResult.numItems,
-    };
+    });
   }
 
   private estimateSubDBSize(db: Database<unknown, Key>): { actualSize: number; numItems: number } {

package/src/lmdb-v2/factory.ts

@@ -0,0 +1,79 @@
+import { EthAddress } from '@aztec/circuits.js';
+import { type Logger, createLogger } from '@aztec/foundation/log';
+
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'fs/promises';
+import { tmpdir } from 'os';
+import { join } from 'path';
+
+import { type DataStoreConfig } from '../config.js';
+import { AztecLMDBStoreV2 } from './store.js';
+
+const ROLLUP_ADDRESS_FILE = 'rollup_address';
+const MAX_READERS = 16;
+
+export async function createStore(
+  name: string,
+  config: DataStoreConfig,
+  log: Logger = createLogger('kv-store:lmdb-v2:' + name),
+): Promise<AztecLMDBStoreV2> {
+  const { dataDirectory, l1Contracts } = config;
+
+  let store: AztecLMDBStoreV2;
+  if (typeof dataDirectory !== 'undefined') {
+    const subDir = join(dataDirectory, name);
+    await mkdir(subDir, { recursive: true });
+
+    if (l1Contracts) {
+      const { rollupAddress } = l1Contracts;
+      const localRollupAddress = await readFile(join(subDir, ROLLUP_ADDRESS_FILE), 'utf-8')
+        .then(EthAddress.fromString)
+        .catch(() => EthAddress.ZERO);
+
+      if (!localRollupAddress.equals(rollupAddress)) {
+        if (!localRollupAddress.isZero()) {
+          log.warn(`Rollup address mismatch. Clearing entire database...`, {
+            expected: rollupAddress,
+            found: localRollupAddress,
+          });
+
+          await rm(subDir, { recursive: true, force: true });
+          await mkdir(subDir, { recursive: true });
+        }
+
+        await writeFile(join(subDir, ROLLUP_ADDRESS_FILE), rollupAddress.toString());
+      }
+    }
+
+    log.info(
+      `Creating ${name} data store at directory ${subDir} with map size ${config.dataStoreMapSizeKB} KB (LMDB v2)`,
+    );
+    store = await AztecLMDBStoreV2.new(subDir, config.dataStoreMapSizeKB, MAX_READERS, () => Promise.resolve(), log);
+  } else {
+    store = await openTmpStore(name, true, config.dataStoreMapSizeKB, MAX_READERS, log);
+  }
+
+  return store;
+}
+
+export async function openTmpStore(
+  name: string,
+  ephemeral: boolean = true,
+  dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
+  maxReaders = MAX_READERS,
+  log: Logger = createLogger('kv-store:lmdb-v2:' + name),
+): Promise<AztecLMDBStoreV2> {
+  const dataDir = await mkdtemp(join(tmpdir(), name + '-'));
+  log.debug(`Created temporary data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
+
+  // pass a cleanup callback because process.on('beforeExit', cleanup) does not work under Jest
+  const cleanup = async () => {
+    if (ephemeral) {
+      await rm(dataDir, { recursive: true, force: true });
+      log.debug(`Deleted temporary data store: ${dataDir}`);
+    } else {
+      log.debug(`Leaving temporary data store: ${dataDir}`);
+    }
+  };
+
+  return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, log);
+}

package/src/lmdb-v2/index.ts

@@ -0,0 +1,2 @@
+export * from './store.js';
+export * from './factory.js';

package/src/lmdb-v2/map.ts

@@ -0,0 +1,233 @@
+import { Encoder } from 'msgpackr';
+
+import type { Key, Range } from '../interfaces/common.js';
+import type { AztecAsyncMap, AztecAsyncMultiMap } from '../interfaces/map.js';
+import { type ReadTransaction } from './read_transaction.js';
+import { type AztecLMDBStoreV2, execInReadTx, execInWriteTx } from './store.js';
+import { deserializeKey, maxKey, minKey, serializeKey } from './utils.js';
+
+export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
+  private prefix: string;
+  private encoder = new Encoder();
+
+  constructor(private store: AztecLMDBStoreV2, name: string) {
+    this.prefix = `map:${name}`;
+  }
+  /**
+   * Sets the value at the given key.
+   * @param key - The key to set the value at
+   * @param val - The value to set
+   */
+  set(key: K, val: V): Promise<void> {
+    return execInWriteTx(this.store, tx => tx.set(serializeKey(this.prefix, key), this.encoder.pack(val)));
+  }
+
+  /**
+   * Sets the value at the given key if it does not already exist.
+   * @param key - The key to set the value at
+   * @param val - The value to set
+   */
+  setIfNotExists(key: K, val: V): Promise<boolean> {
+    return execInWriteTx(this.store, async tx => {
+      const strKey = serializeKey(this.prefix, key);
+      const exists = !!(await tx.get(strKey));
+      if (!exists) {
+        await tx.set(strKey, this.encoder.pack(val));
+        return true;
+      }
+      return false;
+    });
+  }
+
+  /**
+   * Deletes the value at the given key.
+   * @param key - The key to delete the value at
+   */
+  delete(key: K): Promise<void> {
+    return execInWriteTx(this.store, tx => tx.remove(serializeKey(this.prefix, key)));
+  }
+
+  getAsync(key: K): Promise<V | undefined> {
+    return execInReadTx(this.store, async tx => {
+      const val = await tx.get(serializeKey(this.prefix, key));
+      return val ? this.encoder.unpack(val) : undefined;
+    });
+  }
+
+  hasAsync(key: K): Promise<boolean> {
+    return execInReadTx(this.store, async tx => !!(await tx.get(serializeKey(this.prefix, key))));
+  }
+
+  /**
+   * Iterates over the map's key-value entries in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
+    const reverse = range?.reverse ?? false;
+    const startKey = range?.start ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
+
+    const endKey = range?.end ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined;
+
+    let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx();
+    const shouldClose = !tx;
+    tx ??= this.store.getReadTx();
+
+    try {
+      for await (const [key, val] of tx.iterate(
+        reverse ? endKey! : startKey,
+        reverse ? startKey : endKey,
+        reverse,
+        range?.limit,
+      )) {
+        const deserializedKey = deserializeKey<K>(this.prefix, key);
+        if (!deserializedKey) {
+          break;
+        }
+        yield [deserializedKey, this.encoder.unpack(val)];
+      }
+    } finally {
+      if (shouldClose) {
+        tx.close();
+      }
+    }
+  }
+
+  /**
+   * Iterates over the map's values in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  async *valuesAsync(range?: Range<K>): AsyncIterableIterator<V> {
+    for await (const [_, value] of this.entriesAsync(range)) {
+      yield value;
+    }
+  }
+
+  /**
+   * Iterates over the map's keys in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  async *keysAsync(range?: Range<K>): AsyncIterableIterator<K> {
+    for await (const [key, _] of this.entriesAsync(range)) {
+      yield key;
+    }
+  }
+}
+
+export class LMDBMultiMap<K extends Key, V> implements AztecAsyncMultiMap<K, V> {
+  private prefix: string;
+  private encoder = new Encoder();
+  constructor(private store: AztecLMDBStoreV2, name: string) {
+    this.prefix = `multimap:${name}`;
+  }
+
+  /**
+   * Sets the value at the given key.
+   * @param key - The key to set the value at
+   * @param val - The value to set
+   */
+  set(key: K, val: V): Promise<void> {
+    return execInWriteTx(this.store, tx => tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
+  }
+
+  /**
+   * Sets the value at the given key if it does not already exist.
+   * @param key - The key to set the value at
+   * @param val - The value to set
+   */
+  setIfNotExists(key: K, val: V): Promise<boolean> {
+    return execInWriteTx(this.store, async tx => {
+      const exists = !!(await this.getAsync(key));
+      if (!exists) {
+        await tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val));
+        return true;
+      }
+      return false;
+    });
+  }
+
+  /**
+   * Deletes the value at the given key.
+   * @param key - The key to delete the value at
+   */
+  delete(key: K): Promise<void> {
+    return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key)));
+  }
+
+  getAsync(key: K): Promise<V | undefined> {
+    return execInReadTx(this.store, async tx => {
+      const val = await tx.getIndex(serializeKey(this.prefix, key));
+      return val.length > 0 ? this.encoder.unpack(val[0]) : undefined;
+    });
+  }
+
+  hasAsync(key: K): Promise<boolean> {
+    return execInReadTx(this.store, async tx => (await tx.getIndex(serializeKey(this.prefix, key))).length > 0);
+  }
+
+  /**
+   * Iterates over the map's key-value entries in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
+    const reverse = range?.reverse ?? false;
+    const startKey = range?.start ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
+    const endKey = range?.end ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined;
+
+    let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx();
+    const shouldClose = !tx;
+    tx ??= this.store.getReadTx();
+
+    try {
+      for await (const [key, vals] of tx.iterateIndex(
+        reverse ? endKey! : startKey,
+        reverse ? startKey : endKey,
+        reverse,
+        range?.limit,
+      )) {
+        const deserializedKey = deserializeKey<K>(this.prefix, key);
+        if (!deserializedKey) {
+          break;
+        }
+
+        for (const val of vals) {
+          yield [deserializedKey, this.encoder.unpack(val)];
+        }
+      }
+    } finally {
+      if (shouldClose) {
+        tx.close();
+      }
+    }
+  }
+
+  /**
+   * Iterates over the map's values in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  async *valuesAsync(range?: Range<K>): AsyncIterableIterator<V> {
+    for await (const [_, value] of this.entriesAsync(range)) {
+      yield value;
+    }
+  }
+
+  /**
+   * Iterates over the map's keys in the key's natural order
+   * @param range - The range of keys to iterate over
+   */
+  async *keysAsync(range?: Range<K>): AsyncIterableIterator<K> {
+    for await (const [key, _] of this.entriesAsync(range)) {
+      yield key;
+    }
+  }
+
+  deleteValue(key: K, val: V | undefined): Promise<void> {
+    return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
+  }
+
+  async *getValuesAsync(key: K): AsyncIterableIterator<V> {
+    const values = await execInReadTx(this.store, tx => tx.getIndex(serializeKey(this.prefix, key)));
+    for (const value of values) {
+      yield this.encoder.unpack(value);
+    }
+  }
+}