@aztec/kv-store 0.0.0-test.1 → 0.0.1-commit.b655e406

package/dest/stores/l2_tips_store.js

@@ -1,4 +1,4 @@
-/** Stores currently synced L2 tips and unfinalized block hashes. */ export class L2TipsStore {
+/** Stores currently synced L2 tips and unfinalized block hashes. */ export class L2TipsKVStore {
     l2TipsStore;
     l2BlockHashesStore;
     constructor(store, namespace){
@@ -41,25 +41,34 @@
     async handleBlockStreamEvent(event) {
         switch(event.type){
             case 'blocks-added':
-                for (const block of event.blocks){
-                    await this.l2BlockHashesStore.set(block.number, (await block.header.hash()).toString());
+                {
+                    const blocks = event.blocks.map((b)=>b.block);
+                    for (const block of blocks){
+                        await this.l2BlockHashesStore.set(block.number, (await block.hash()).toString());
+                    }
+                    await this.l2TipsStore.set('latest', blocks.at(-1).number);
+                    break;
                 }
-                await this.l2TipsStore.set('latest', event.blocks.at(-1).number);
-                break;
             case 'chain-pruned':
-                await this.l2TipsStore.set('latest', event.blockNumber);
+                await this.saveTag('latest', event.block);
                 break;
             case 'chain-proven':
-                await this.l2TipsStore.set('proven', event.blockNumber);
+                await this.saveTag('proven', event.block);
                 break;
             case 'chain-finalized':
-                await this.l2TipsStore.set('finalized', event.blockNumber);
+                await this.saveTag('finalized', event.block);
                 for await (const key of this.l2BlockHashesStore.keysAsync({
-                    end: event.blockNumber
+                    end: event.block.number
                 })){
                     await this.l2BlockHashesStore.delete(key);
                 }
                 break;
         }
     }
+    async saveTag(name, block) {
+        await this.l2TipsStore.set(name, block.number);
+        if (block.hash) {
+            await this.l2BlockHashesStore.set(block.number, block.hash);
+        }
+    }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aztec/kv-store",
-  "version": "0.0.0-test.1",
+  "version": "0.0.1-commit.b655e406",
   "type": "module",
   "exports": {
     ".": "./dest/interfaces/index.js",
@@ -14,46 +14,46 @@
     "build": "yarn clean && tsc -b",
     "build:dev": "tsc -b --watch",
     "clean": "rm -rf ./dest .tsbuildinfo",
-    "formatting": "run -T prettier --check ./src && run -T eslint ./src",
-    "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
     "test:node": "NODE_NO_WARNINGS=1 mocha --config ./.mocharc.json",
     "test:browser": "wtr --config ./web-test-runner.config.mjs",
-    "test": "yarn test:node && yarn test:browser && true"
+    "test": "yarn test:node",
+    "test:jest": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
   },
   "inherits": [
     "../package.common.json",
     "./package.local.json"
   ],
   "dependencies": {
-    "@aztec/ethereum": "0.0.0-test.1",
-    "@aztec/foundation": "0.0.0-test.1",
-    "@aztec/native": "0.0.0-test.1",
-    "@aztec/stdlib": "0.0.0-test.1",
+    "@aztec/ethereum": "0.0.1-commit.b655e406",
+    "@aztec/foundation": "0.0.1-commit.b655e406",
+    "@aztec/native": "0.0.1-commit.b655e406",
+    "@aztec/stdlib": "0.0.1-commit.b655e406",
     "idb": "^8.0.0",
     "lmdb": "^3.2.0",
     "msgpackr": "^1.11.2",
+    "ohash": "^2.0.11",
     "ordered-binary": "^1.5.3"
   },
   "devDependencies": {
-    "@jest/globals": "^29.5.0",
+    "@jest/globals": "^30.0.0",
     "@types/chai": "^5.0.1",
     "@types/chai-as-promised": "^8.0.1",
-    "@types/jest": "^29.5.0",
+    "@types/jest": "^30.0.0",
     "@types/mocha": "^10.0.10",
     "@types/mocha-each": "^2.0.4",
-    "@types/node": "^18.7.23",
+    "@types/node": "^22.15.17",
     "@types/sinon": "^17.0.3",
     "@web/dev-server-esbuild": "^1.0.3",
     "@web/test-runner": "^0.19.0",
     "@web/test-runner-playwright": "^0.11.0",
     "chai": "^5.1.2",
     "chai-as-promised": "^8.0.1",
-    "jest": "^29.5.0",
+    "jest": "^30.0.0",
     "mocha": "^10.8.2",
     "mocha-each": "^2.0.1",
     "sinon": "^19.0.2",
     "ts-node": "^10.9.1",
-    "typescript": "^5.0.4"
+    "typescript": "^5.3.3"
   },
   "files": [
     "dest",
@@ -61,7 +61,7 @@
     "!*.test.*"
   ],
   "engines": {
-    "node": ">=18"
+    "node": ">=20.10"
   },
   "jest": {
     "extensionsToTreatAsEsm": [
@@ -94,6 +94,10 @@
     "testTimeout": 120000,
     "setupFiles": [
       "../../foundation/src/jest/setup.mjs"
+    ],
+    "testEnvironment": "../../foundation/src/jest/env.mjs",
+    "setupFilesAfterEnv": [
+      "../../foundation/src/jest/setupAfterEnv.mjs"
     ]
   }
 }

package/src/config.ts

@@ -4,7 +4,7 @@ import type { EthAddress } from '@aztec/foundation/eth-address';
 
 export type DataStoreConfig = {
   dataDirectory: string | undefined;
-  dataStoreMapSizeKB: number;
+  dataStoreMapSizeKb: number;
   l1Contracts?: { rollupAddress: EthAddress };
 };
 
@@ -13,14 +13,16 @@ export const dataConfigMappings: ConfigMappingsType<DataStoreConfig> = {
     env: 'DATA_DIRECTORY',
     description: 'Optional dir to store data. If omitted will store in memory.',
   },
-  dataStoreMapSizeKB: {
+  dataStoreMapSizeKb: {
     env: 'DATA_STORE_MAP_SIZE_KB',
-    description: 'DB mapping size to be applied to all key/value stores',
+    description: 'The maximum possible size of a data store DB in KB. Can be overridden by component-specific options.',
     ...numberConfigHelper(128 * 1_024 * 1_024), // Defaulted to 128 GB
   },
   l1Contracts: {
     description: 'The deployed L1 contract addresses',
-    nested: l1ContractAddressesMapping,
+    nested: {
+      rollupAddress: l1ContractAddressesMapping.rollupAddress,
+    },
   },
 };
 

package/src/indexeddb/array.ts

@@ -1,12 +1,14 @@
 import type { IDBPDatabase, IDBPObjectStore } from 'idb';
+import { hash } from 'ohash';
 
 import type { AztecAsyncArray } from '../interfaces/array.js';
+import type { Value } from '../interfaces/common.js';
 import type { AztecIDBSchema } from './store.js';
 
 /**
  * A persistent array backed by IndexedDB.
  */
-export class IndexedDBAztecArray<T> implements AztecAsyncArray<T> {
+export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T> {
   #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
   #rootDB: IDBPDatabase<AztecIDBSchema>;
   #container: string;
@@ -39,6 +41,7 @@ export class IndexedDBAztecArray<T> implements AztecAsyncArray<T> {
     for (const val of vals) {
       await this.db.put({
         value: val,
+        hash: hash(val),
         container: this.#container,
         key: this.#name,
         keyCount: length + 1,
@@ -86,6 +89,7 @@ export class IndexedDBAztecArray<T> implements AztecAsyncArray<T> {
 
     await this.db.put({
       value: val,
+      hash: hash(val),
       container: this.#container,
       key: this.#name,
       keyCount: index + 1,

package/src/indexeddb/index.ts

@@ -14,8 +14,8 @@ export async function createStore(name: string, config: DataStoreConfig, log: Lo
 
   log.info(
     dataDirectory
-      ? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.dataStoreMapSizeKB} KB`
-      : `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKB} KB`,
+      ? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.dataStoreMapSizeKb} KB`
+      : `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKb} KB`,
   );
   const store = await AztecIndexedDBStore.open(createLogger('kv-store:indexeddb'), dataDirectory ?? '', false);
   if (config.l1Contracts?.rollupAddress) {

package/src/indexeddb/map.ts

@@ -1,22 +1,23 @@
 import type { IDBPDatabase, IDBPObjectStore } from 'idb';
+import { hash } from 'ohash';
 
-import type { Key, Range } from '../interfaces/common.js';
-import type { AztecAsyncMultiMap } from '../interfaces/map.js';
+import type { Key, Range, Value } from '../interfaces/common.js';
+import type { AztecAsyncMap } from '../interfaces/map.js';
 import type { AztecIDBSchema } from './store.js';
 
 /**
  * A map backed by IndexedDB.
  */
-export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K, V> {
+export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecAsyncMap<K, V> {
   protected name: string;
-  #container: string;
+  protected container: string;
 
   #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
   #rootDB: IDBPDatabase<AztecIDBSchema>;
 
   constructor(rootDB: IDBPDatabase<AztecIDBSchema>, mapName: string) {
     this.name = mapName;
-    this.#container = `map:${mapName}`;
+    this.container = `map:${mapName}`;
     this.#rootDB = rootDB;
   }
 
@@ -29,41 +30,38 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
   }
 
   async getAsync(key: K): Promise<V | undefined> {
-    const data = await this.db.get(this.#slot(key));
+    const data = await this.db.get(this.slot(key));
     return data?.value as V;
   }
 
-  async *getValuesAsync(key: K): AsyncIterableIterator<V> {
-    const index = this.db.index('keyCount');
-    const rangeQuery = IDBKeyRange.bound(
-      [this.#container, this.#normalizeKey(key), 0],
-      [this.#container, this.#normalizeKey(key), Number.MAX_SAFE_INTEGER],
-      false,
-      false,
-    );
-    for await (const cursor of index.iterate(rangeQuery)) {
-      yield cursor.value.value as V;
-    }
-  }
-
   async hasAsync(key: K): Promise<boolean> {
     const result = (await this.getAsync(key)) !== undefined;
     return result;
   }
 
+  async sizeAsync(): Promise<number> {
+    const index = this.db.index('key');
+    const rangeQuery = IDBKeyRange.bound([this.container, ''], [this.container, '\uffff']);
+    return await index.count(rangeQuery);
+  }
+
   async set(key: K, val: V): Promise<void> {
-    const count = await this.db
-      .index('key')
-      .count(IDBKeyRange.bound([this.#container, this.#normalizeKey(key)], [this.#container, this.#normalizeKey(key)]));
     await this.db.put({
       value: val,
-      container: this.#container,
-      key: this.#normalizeKey(key),
-      keyCount: count + 1,
-      slot: this.#slot(key, count),
+      hash: hash(val),
+      container: this.container,
+      key: this.normalizeKey(key),
+      keyCount: 1,
+      slot: this.slot(key),
     });
   }
 
+  async setMany(entries: { key: K; value: V }[]): Promise<void> {
+    for (const { key, value } of entries) {
+      await this.set(key, value);
+    }
+  }
+
   swap(_key: K, _fn: (val: V | undefined) => V): Promise<void> {
     throw new Error('Not implemented');
   }
@@ -77,30 +75,14 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
   }
 
   async delete(key: K): Promise<void> {
-    await this.db.delete(this.#slot(key));
-  }
-
-  async deleteValue(key: K, val: V): Promise<void> {
-    const index = this.db.index('keyCount');
-    const rangeQuery = IDBKeyRange.bound(
-      [this.#container, this.#normalizeKey(key), 0],
-      [this.#container, this.#normalizeKey(key), Number.MAX_SAFE_INTEGER],
-      false,
-      false,
-    );
-    for await (const cursor of index.iterate(rangeQuery)) {
-      if (JSON.stringify(cursor.value.value) === JSON.stringify(val)) {
-        await cursor.delete();
-        return;
-      }
-    }
+    await this.db.delete(this.slot(key));
   }
 
   async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, V]> {
     const index = this.db.index('key');
     const rangeQuery = IDBKeyRange.bound(
-      [this.#container, range.start ?? ''],
-      [this.#container, range.end ?? '\uffff'],
+      [this.container, range.start ? this.normalizeKey(range.start) : ''],
+      [this.container, range.end ? this.normalizeKey(range.end) : '\uffff'],
       !!range.reverse,
       !range.reverse,
     );
@@ -109,7 +91,7 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
       if (range.limit && count >= range.limit) {
         return;
       }
-      yield [cursor.value.key, cursor.value.value] as [K, V];
+      yield [this.#denormalizeKey(cursor.value.key), cursor.value.value] as [K, V];
       count++;
     }
   }
@@ -122,21 +104,21 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
 
   async *keysAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
     for await (const [key, _] of this.entriesAsync(range)) {
-      yield this.#denormalizeKey(key as string);
+      yield key;
     }
   }
 
   #denormalizeKey(key: string): K {
-    const denormalizedKey = (key as string).split(',').map(part => (isNaN(parseInt(part)) ? part : parseInt(part)));
-    return (denormalizedKey.length > 1 ? denormalizedKey : key) as K;
+    const denormalizedKey = key.split(',').map(part => (part.startsWith('n_') ? Number(part.slice(2)) : part));
+    return (denormalizedKey.length > 1 ? denormalizedKey : denormalizedKey[0]) as K;
   }
 
-  #normalizeKey(key: K): string {
+  protected normalizeKey(key: K): string {
     const arrayKey = Array.isArray(key) ? key : [key];
-    return arrayKey.join(',');
+    return (arrayKey as K[]).map((element: K) => (typeof element === 'number' ? `n_${element}` : element)).join(',');
   }
 
-  #slot(key: K, index: number = 0): string {
-    return `map:${this.name}:slot:${this.#normalizeKey(key)}:${index}`;
+  protected slot(key: K, index: number = 0): string {
+    return `map:${this.name}:slot:${this.normalizeKey(key)}:${index}`;
   }
 }

package/src/indexeddb/multi_map.ts

@@ -0,0 +1,79 @@
+import { hash } from 'ohash';
+
+import type { Key, Value } from '../interfaces/common.js';
+import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
+import { IndexedDBAztecMap } from './map.js';
+
+/**
+ * A multi map backed by IndexedDB.
+ */
+export class IndexedDBAztecMultiMap<K extends Key, V extends Value>
+  extends IndexedDBAztecMap<K, V>
+  implements AztecAsyncMultiMap<K, V>
+{
+  override async set(key: K, val: V): Promise<void> {
+    // Inserting repeated values is a no-op
+    const exists = !!(await this.db
+      .index('hash')
+      .get(
+        IDBKeyRange.bound(
+          [this.container, this.normalizeKey(key), hash(val)],
+          [this.container, this.normalizeKey(key), hash(val)],
+        ),
+      ));
+    if (exists) {
+      return;
+    }
+    // Get the maximum keyCount for the given key
+    // In order to support sparse multimaps, we cannot rely
+    // on just counting the number of entries for the key, since we would repeat slots
+    // if we delete an entry
+    // set -> container:key:0 (keyCount = 1)
+    // set -> container:key:1 (keyCount = 2)
+    // delete -> container:key:0 (keyCount = 1)
+    // set -> container:key:1 <--- already exists!
+    // Instead, we iterate in reverse order to get the last inserted entry
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.upperBound([this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER]);
+    const maxEntry = (await index.iterate(rangeQuery, 'prevunique').next()).value;
+    const count = maxEntry?.value?.keyCount ?? 0;
+    await this.db.put({
+      value: val,
+      hash: hash(val),
+      container: this.container,
+      key: this.normalizeKey(key),
+      keyCount: count + 1,
+      slot: this.slot(key, count),
+    });
+  }
+
+  async *getValuesAsync(key: K): AsyncIterableIterator<V> {
+    // Iterate over the whole range of keyCount for the given key
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.container, this.normalizeKey(key), 0],
+      [this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
+      false,
+      false,
+    );
+    for await (const cursor of index.iterate(rangeQuery)) {
+      yield cursor.value.value as V;
+    }
+  }
+
+  async deleteValue(key: K, val: V): Promise<void> {
+    // Since we know the value, we can hash it and directly query the "hash" index
+    // to avoid having to iterate over all the values
+    const fullKey = await this.db
+      .index('hash')
+      .getKey(
+        IDBKeyRange.bound(
+          [this.container, this.normalizeKey(key), hash(val)],
+          [this.container, this.normalizeKey(key), hash(val)],
+        ),
+      );
+    if (fullKey) {
+      await this.db.delete(fullKey);
+    }
+  }
+}

package/src/indexeddb/singleton.ts

@@ -1,12 +1,14 @@
 import type { IDBPDatabase, IDBPObjectStore } from 'idb';
+import { hash } from 'ohash';
 
+import type { Value } from '../interfaces/common.js';
 import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
 import type { AztecIDBSchema } from './store.js';
 
 /**
  * Stores a single value in IndexedDB.
  */
-export class IndexedDBAztecSingleton<T> implements AztecAsyncSingleton<T> {
+export class IndexedDBAztecSingleton<T extends Value> implements AztecAsyncSingleton<T> {
   #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
   #rootDB: IDBPDatabase<AztecIDBSchema>;
   #container: string;
@@ -38,6 +40,7 @@ export class IndexedDBAztecSingleton<T> implements AztecAsyncSingleton<T> {
       key: this.#slot,
       keyCount: 1,
       value: val,
+      hash: hash(val),
     });
     return result !== undefined;
   }