@aztec/kv-store 0.0.0-test.0

package/dest/stores/l2_tips_store.js

@@ -0,0 +1,65 @@
+/** Stores currently synced L2 tips and unfinalized block hashes. */ export class L2TipsStore {
+    l2TipsStore;
+    l2BlockHashesStore;
+    constructor(store, namespace){
+        this.l2TipsStore = store.openMap([
+            namespace,
+            'l2_tips'
+        ].join('_'));
+        this.l2BlockHashesStore = store.openMap([
+            namespace,
+            'l2_block_hashes'
+        ].join('_'));
+    }
+    getL2BlockHash(number) {
+        return this.l2BlockHashesStore.getAsync(number);
+    }
+    async getL2Tips() {
+        return {
+            latest: await this.getL2Tip('latest'),
+            finalized: await this.getL2Tip('finalized'),
+            proven: await this.getL2Tip('proven')
+        };
+    }
+    async getL2Tip(tag) {
+        const blockNumber = await this.l2TipsStore.getAsync(tag);
+        if (blockNumber === undefined || blockNumber === 0) {
+            return {
+                number: 0,
+                hash: undefined
+            };
+        }
+        const blockHash = await this.l2BlockHashesStore.getAsync(blockNumber);
+        if (!blockHash) {
+            throw new Error(`Block hash not found for block number ${blockNumber}`);
+        }
+        return {
+            number: blockNumber,
+            hash: blockHash
+        };
+    }
+    async handleBlockStreamEvent(event) {
+        switch(event.type){
+            case 'blocks-added':
+                for (const block of event.blocks){
+                    await this.l2BlockHashesStore.set(block.number, (await block.header.hash()).toString());
+                }
+                await this.l2TipsStore.set('latest', event.blocks.at(-1).number);
+                break;
+            case 'chain-pruned':
+                await this.l2TipsStore.set('latest', event.blockNumber);
+                break;
+            case 'chain-proven':
+                await this.l2TipsStore.set('proven', event.blockNumber);
+                break;
+            case 'chain-finalized':
+                await this.l2TipsStore.set('finalized', event.blockNumber);
+                for await (const key of this.l2BlockHashesStore.keysAsync({
+                    end: event.blockNumber
+                })){
+                    await this.l2BlockHashesStore.delete(key);
+                }
+                break;
+        }
+    }
+}

package/dest/utils.d.ts

@@ -0,0 +1,12 @@
+import type { EthAddress } from '@aztec/foundation/eth-address';
+import type { Logger } from '@aztec/foundation/log';
+import type { AztecAsyncKVStore, AztecKVStore } from './interfaces/store.js';
+/**
+ * Clears the store if the rollup address does not match the one stored in the database.
+ * This is to prevent data from being accidentally shared between different rollup instances.
+ * @param store - The store to check
+ * @param rollupAddress - The ETH address of the rollup contract
+ * @returns A promise that resolves when the store is cleared, or rejects if the rollup address does not match
+ */
+export declare function initStoreForRollup<T extends AztecKVStore | AztecAsyncKVStore>(store: T, rollupAddress: EthAddress, log?: Logger): Promise<T>;
+//# sourceMappingURL=utils.d.ts.map

package/dest/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAGpD,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAG7E;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CAAC,CAAC,SAAS,YAAY,GAAG,iBAAiB,EACjF,KAAK,EAAE,CAAC,EACR,aAAa,EAAE,UAAU,EACzB,GAAG,CAAC,EAAE,MAAM,GACX,OAAO,CAAC,CAAC,CAAC,CAqBZ"}

package/dest/utils.js

@@ -0,0 +1,24 @@
+import { isSyncStore } from './interfaces/utils.js';
+/**
+ * Clears the store if the rollup address does not match the one stored in the database.
+ * This is to prevent data from being accidentally shared between different rollup instances.
+ * @param store - The store to check
+ * @param rollupAddress - The ETH address of the rollup contract
+ * @returns A promise that resolves when the store is cleared, or rejects if the rollup address does not match
+ */ export async function initStoreForRollup(store, rollupAddress, log) {
+    if (!rollupAddress) {
+        throw new Error('Rollup address is required');
+    }
+    const rollupAddressValue = store.openSingleton('rollupAddress');
+    const rollupAddressString = rollupAddress.toString();
+    const storedRollupAddressString = isSyncStore(store) ? rollupAddressValue.get() : await rollupAddressValue.getAsync();
+    if (typeof storedRollupAddressString !== 'undefined' && storedRollupAddressString !== rollupAddressString) {
+        log?.warn(`Rollup address mismatch. Clearing entire database...`, {
+            expected: rollupAddressString,
+            found: storedRollupAddressString
+        });
+        await store.clear();
+    }
+    await rollupAddressValue.set(rollupAddressString);
+    return store;
+}

package/package.json

@@ -0,0 +1,99 @@
+{
+  "name": "@aztec/kv-store",
+  "version": "0.0.0-test.0",
+  "type": "module",
+  "exports": {
+    ".": "./dest/interfaces/index.js",
+    "./lmdb": "./dest/lmdb/index.js",
+    "./lmdb-v2": "./dest/lmdb-v2/index.js",
+    "./indexeddb": "./dest/indexeddb/index.js",
+    "./stores": "./dest/stores/index.js",
+    "./config": "./dest/config.js"
+  },
+  "scripts": {
+    "build": "yarn clean && tsc -b",
+    "build:dev": "tsc -b --watch",
+    "clean": "rm -rf ./dest .tsbuildinfo",
+    "formatting": "run -T prettier --check ./src && run -T eslint ./src",
+    "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
+    "test:node": "NODE_NO_WARNINGS=1 mocha --config ./.mocharc.json",
+    "test:browser": "wtr --config ./web-test-runner.config.mjs",
+    "test": "yarn test:node && yarn test:browser && true"
+  },
+  "inherits": [
+    "../package.common.json",
+    "./package.local.json"
+  ],
+  "dependencies": {
+    "@aztec/ethereum": "0.0.0-test.0",
+    "@aztec/foundation": "0.0.0-test.0",
+    "@aztec/native": "0.0.0-test.0",
+    "@aztec/stdlib": "0.0.0-test.0",
+    "idb": "^8.0.0",
+    "lmdb": "^3.2.0",
+    "msgpackr": "^1.11.2",
+    "ordered-binary": "^1.5.3"
+  },
+  "devDependencies": {
+    "@jest/globals": "^29.5.0",
+    "@types/chai": "^5.0.1",
+    "@types/chai-as-promised": "^8.0.1",
+    "@types/jest": "^29.5.0",
+    "@types/mocha": "^10.0.10",
+    "@types/mocha-each": "^2.0.4",
+    "@types/node": "^18.7.23",
+    "@types/sinon": "^17.0.3",
+    "@web/dev-server-esbuild": "^1.0.3",
+    "@web/test-runner": "^0.19.0",
+    "@web/test-runner-playwright": "^0.11.0",
+    "chai": "^5.1.2",
+    "chai-as-promised": "^8.0.1",
+    "jest": "^29.5.0",
+    "mocha": "^10.8.2",
+    "mocha-each": "^2.0.1",
+    "sinon": "^19.0.2",
+    "ts-node": "^10.9.1",
+    "typescript": "^5.0.4"
+  },
+  "files": [
+    "dest",
+    "src",
+    "!*.test.*"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "jest": {
+    "extensionsToTreatAsEsm": [
+      ".ts"
+    ],
+    "transform": {
+      "^.+\\.tsx?$": [
+        "@swc/jest",
+        {
+          "jsc": {
+            "parser": {
+              "syntax": "typescript",
+              "decorators": true
+            },
+            "transform": {
+              "decoratorVersion": "2022-03"
+            }
+          }
+        }
+      ]
+    },
+    "moduleNameMapper": {
+      "^(\\.{1,2}/.*)\\.[cm]?js$": "$1"
+    },
+    "reporters": [
+      "default"
+    ],
+    "testRegex": "./src/.*\\.test\\.(js|mjs|ts)$",
+    "rootDir": "./src",
+    "testTimeout": 120000,
+    "setupFiles": [
+      "../../foundation/src/jest/setup.mjs"
+    ]
+  }
+}

package/src/config.ts

@@ -0,0 +1,34 @@
+import { l1ContractAddressesMapping } from '@aztec/ethereum/l1-contract-addresses';
+import { type ConfigMappingsType, getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config';
+import type { EthAddress } from '@aztec/foundation/eth-address';
+
+export type DataStoreConfig = {
+  dataDirectory: string | undefined;
+  dataStoreMapSizeKB: number;
+  l1Contracts?: { rollupAddress: EthAddress };
+};
+
+export const dataConfigMappings: ConfigMappingsType<DataStoreConfig> = {
+  dataDirectory: {
+    env: 'DATA_DIRECTORY',
+    description: 'Optional dir to store data. If omitted will store in memory.',
+  },
+  dataStoreMapSizeKB: {
+    env: 'DATA_STORE_MAP_SIZE_KB',
+    description: 'DB mapping size to be applied to all key/value stores',
+    ...numberConfigHelper(128 * 1_024 * 1_024), // Defaulted to 128 GB
+  },
+  l1Contracts: {
+    description: 'The deployed L1 contract addresses',
+    nested: l1ContractAddressesMapping,
+  },
+};
+
+/**
+ * Returns the archiver configuration from the environment variables.
+ * Note: If an environment variable is not set, the default value is used.
+ * @returns The archiver configuration.
+ */
+export function getDataConfigFromEnv(): DataStoreConfig {
+  return getConfigFromMappings<DataStoreConfig>(dataConfigMappings);
+}

package/src/indexeddb/array.ts

@@ -0,0 +1,118 @@
+import type { IDBPDatabase, IDBPObjectStore } from 'idb';
+
+import type { AztecAsyncArray } from '../interfaces/array.js';
+import type { AztecIDBSchema } from './store.js';
+
+/**
+ * A persistent array backed by IndexedDB.
+ */
+export class IndexedDBAztecArray<T> implements AztecAsyncArray<T> {
+  #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
+  #rootDB: IDBPDatabase<AztecIDBSchema>;
+  #container: string;
+  #name: string;
+
+  constructor(rootDB: IDBPDatabase<AztecIDBSchema>, name: string) {
+    this.#rootDB = rootDB;
+    this.#name = name;
+    this.#container = `array:${this.#name}`;
+  }
+
+  set db(db: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> | undefined) {
+    this.#_db = db;
+  }
+
+  get db(): IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> {
+    return this.#_db ? this.#_db : this.#rootDB.transaction('data', 'readwrite').store;
+  }
+
+  async lengthAsync(): Promise<number> {
+    return (
+      (await this.db
+        .index('key')
+        .count(IDBKeyRange.bound([this.#container, this.#name], [this.#container, this.#name]))) ?? 0
+    );
+  }
+
+  async push(...vals: T[]): Promise<number> {
+    let length = await this.lengthAsync();
+    for (const val of vals) {
+      await this.db.put({
+        value: val,
+        container: this.#container,
+        key: this.#name,
+        keyCount: length + 1,
+        slot: this.#slot(length),
+      });
+      length += 1;
+    }
+    return length;
+  }
+
+  async pop(): Promise<T | undefined> {
+    const length = await this.lengthAsync();
+    if (length === 0) {
+      return undefined;
+    }
+
+    const slot = this.#slot(length - 1);
+    const data = await this.db.get(slot);
+    await this.db.delete(slot);
+
+    return data?.value;
+  }
+
+  async atAsync(index: number): Promise<T | undefined> {
+    const length = await this.lengthAsync();
+
+    if (index < 0) {
+      index = length + index;
+    }
+
+    const data = await this.db.get(this.#slot(index));
+    return data?.value;
+  }
+
+  async setAt(index: number, val: T): Promise<boolean> {
+    const length = await this.lengthAsync();
+
+    if (index < 0) {
+      index = length + index;
+    }
+
+    if (index < 0 || index >= length) {
+      return Promise.resolve(false);
+    }
+
+    await this.db.put({
+      value: val,
+      container: this.#container,
+      key: this.#name,
+      keyCount: index + 1,
+      slot: this.#slot(index),
+    });
+    return true;
+  }
+
+  async *entriesAsync(): AsyncIterableIterator<[number, T]> {
+    const index = this.db.index('key');
+    const rangeQuery = IDBKeyRange.bound([this.#container, this.#name], [this.#container, this.#name]);
+    for await (const cursor of index.iterate(rangeQuery)) {
+      yield [cursor.value.keyCount - 1, cursor.value.value] as [number, T];
+    }
+  }
+
+  async *valuesAsync(): AsyncIterableIterator<T> {
+    for await (const [_, value] of this.entriesAsync()) {
+      yield value;
+    }
+  }
+
+  [Symbol.asyncIterator](): AsyncIterableIterator<T> {
+    return this.valuesAsync();
+  }
+
+  #slot(index: number): string {
+    return `array:${this.#name}:slot:${index}`;
+  }
+}

package/src/indexeddb/index.ts

@@ -0,0 +1,29 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+
+import type { DataStoreConfig } from '../config.js';
+import { initStoreForRollup } from '../utils.js';
+import { AztecIndexedDBStore } from './store.js';
+
+export { AztecIndexedDBStore } from './store.js';
+
+export async function createStore(name: string, config: DataStoreConfig, log: Logger = createLogger('kv-store')) {
+  let { dataDirectory } = config;
+  if (typeof dataDirectory !== 'undefined') {
+    dataDirectory = `${dataDirectory}/${name}`;
+  }
+
+  log.info(
+    dataDirectory
+      ? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.dataStoreMapSizeKB} KB`
+      : `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKB} KB`,
+  );
+  const store = await AztecIndexedDBStore.open(createLogger('kv-store:indexeddb'), dataDirectory ?? '', false);
+  if (config.l1Contracts?.rollupAddress) {
+    return initStoreForRollup(store, config.l1Contracts.rollupAddress, log);
+  }
+  return store;
+}
+
+export function openTmpStore(ephemeral: boolean = false): Promise<AztecIndexedDBStore> {
+  return AztecIndexedDBStore.open(createLogger('kv-store:indexeddb'), undefined, ephemeral);
+}

package/src/indexeddb/map.ts

@@ -0,0 +1,142 @@
+import type { IDBPDatabase, IDBPObjectStore } from 'idb';
+
+import type { Key, Range } from '../interfaces/common.js';
+import type { AztecAsyncMultiMap } from '../interfaces/map.js';
+import type { AztecIDBSchema } from './store.js';
+
+/**
+ * A map backed by IndexedDB.
+ */
+export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K, V> {
+  protected name: string;
+  #container: string;
+
+  #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
+  #rootDB: IDBPDatabase<AztecIDBSchema>;
+
+  constructor(rootDB: IDBPDatabase<AztecIDBSchema>, mapName: string) {
+    this.name = mapName;
+    this.#container = `map:${mapName}`;
+    this.#rootDB = rootDB;
+  }
+
+  set db(db: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> | undefined) {
+    this.#_db = db;
+  }
+
+  get db(): IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> {
+    return this.#_db ? this.#_db : this.#rootDB.transaction('data', 'readwrite').store;
+  }
+
+  async getAsync(key: K): Promise<V | undefined> {
+    const data = await this.db.get(this.#slot(key));
+    return data?.value as V;
+  }
+
+  async *getValuesAsync(key: K): AsyncIterableIterator<V> {
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.#container, this.#normalizeKey(key), 0],
+      [this.#container, this.#normalizeKey(key), Number.MAX_SAFE_INTEGER],
+      false,
+      false,
+    );
+    for await (const cursor of index.iterate(rangeQuery)) {
+      yield cursor.value.value as V;
+    }
+  }
+
+  async hasAsync(key: K): Promise<boolean> {
+    const result = (await this.getAsync(key)) !== undefined;
+    return result;
+  }
+
+  async set(key: K, val: V): Promise<void> {
+    const count = await this.db
+      .index('key')
+      .count(IDBKeyRange.bound([this.#container, this.#normalizeKey(key)], [this.#container, this.#normalizeKey(key)]));
+    await this.db.put({
+      value: val,
+      container: this.#container,
+      key: this.#normalizeKey(key),
+      keyCount: count + 1,
+      slot: this.#slot(key, count),
+    });
+  }
+
+  swap(_key: K, _fn: (val: V | undefined) => V): Promise<void> {
+    throw new Error('Not implemented');
+  }
+
+  async setIfNotExists(key: K, val: V): Promise<boolean> {
+    if (!(await this.hasAsync(key))) {
+      await this.set(key, val);
+      return true;
+    }
+    return false;
+  }
+
+  async delete(key: K): Promise<void> {
+    await this.db.delete(this.#slot(key));
+  }
+
+  async deleteValue(key: K, val: V): Promise<void> {
+    const index = this.db.index('keyCount');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.#container, this.#normalizeKey(key), 0],
+      [this.#container, this.#normalizeKey(key), Number.MAX_SAFE_INTEGER],
+      false,
+      false,
+    );
+    for await (const cursor of index.iterate(rangeQuery)) {
+      if (JSON.stringify(cursor.value.value) === JSON.stringify(val)) {
+        await cursor.delete();
+        return;
+      }
+    }
+  }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, V]> {
+    const index = this.db.index('key');
+    const rangeQuery = IDBKeyRange.bound(
+      [this.#container, range.start ?? ''],
+      [this.#container, range.end ?? '\uffff'],
+      !!range.reverse,
+      !range.reverse,
+    );
+    let count = 0;
+    for await (const cursor of index.iterate(rangeQuery, range.reverse ? 'prev' : 'next')) {
+      if (range.limit && count >= range.limit) {
+        return;
+      }
+      yield [cursor.value.key, cursor.value.value] as [K, V];
+      count++;
+    }
+  }
+
+  async *valuesAsync(range: Range<K> = {}): AsyncIterableIterator<V> {
+    for await (const [_, value] of this.entriesAsync(range)) {
+      yield value;
+    }
+  }
+
+  async *keysAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    for await (const [key, _] of this.entriesAsync(range)) {
+      yield this.#denormalizeKey(key as string);
+    }
+  }
+
+  #denormalizeKey(key: string): K {
+    const denormalizedKey = (key as string).split(',').map(part => (isNaN(parseInt(part)) ? part : parseInt(part)));
+    return (denormalizedKey.length > 1 ? denormalizedKey : key) as K;
+  }
+
+  #normalizeKey(key: K): string {
+    const arrayKey = Array.isArray(key) ? key : [key];
+    return arrayKey.join(',');
+  }
+
+  #slot(key: K, index: number = 0): string {
+    return `map:${this.name}:slot:${this.#normalizeKey(key)}:${index}`;
+  }
+}

package/src/indexeddb/set.ts

@@ -0,0 +1,37 @@
+import type { IDBPDatabase, IDBPObjectStore } from 'idb';
+
+import type { Key, Range } from '../interfaces/common.js';
+import type { AztecAsyncSet } from '../interfaces/set.js';
+import { IndexedDBAztecMap } from './map.js';
+import type { AztecIDBSchema } from './store.js';
+
+/**
+ * A set backed by IndexedDB.
+ */
+export class IndexedDBAztecSet<K extends Key> implements AztecAsyncSet<K> {
+  private map: IndexedDBAztecMap<K, boolean>;
+
+  constructor(rootDb: IDBPDatabase<AztecIDBSchema>, mapName: string) {
+    this.map = new IndexedDBAztecMap(rootDb, mapName);
+  }
+
+  set db(db: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> | undefined) {
+    this.map.db = db;
+  }
+
+  hasAsync(key: K): Promise<boolean> {
+    return this.map.hasAsync(key);
+  }
+
+  add(key: K): Promise<void> {
+    return this.map.set(key, true);
+  }
+
+  delete(key: K): Promise<void> {
+    return this.map.delete(key);
+  }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    yield* this.map.keysAsync(range);
+  }
+}

package/src/indexeddb/singleton.ts

@@ -0,0 +1,49 @@
+import type { IDBPDatabase, IDBPObjectStore } from 'idb';
+
+import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
+import type { AztecIDBSchema } from './store.js';
+
+/**
+ * Stores a single value in IndexedDB.
+ */
+export class IndexedDBAztecSingleton<T> implements AztecAsyncSingleton<T> {
+  #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
+  #rootDB: IDBPDatabase<AztecIDBSchema>;
+  #container: string;
+  #slot: string;
+
+  constructor(rootDB: IDBPDatabase<AztecIDBSchema>, name: string) {
+    this.#rootDB = rootDB;
+    this.#container = `singleton:${name}`;
+    this.#slot = `singleton:${name}:value`;
+  }
+
+  set db(db: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> | undefined) {
+    this.#_db = db;
+  }
+
+  get db(): IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'> {
+    return this.#_db ? this.#_db : this.#rootDB.transaction('data', 'readwrite').store;
+  }
+
+  async getAsync(): Promise<T | undefined> {
+    const data = await this.db.get(this.#slot);
+    return data?.value as T;
+  }
+
+  async set(val: T): Promise<boolean> {
+    const result = await this.db.put({
+      container: this.#container,
+      slot: this.#slot,
+      key: this.#slot,
+      keyCount: 1,
+      value: val,
+    });
+    return result !== undefined;
+  }
+
+  async delete(): Promise<boolean> {
+    await this.db.delete(this.#slot);
+    return true;
+  }
+}