@aztec/world-state 0.65.2 → 0.67.0

package/src/synchronizer/instrumentation.ts

@@ -1,119 +1,91 @@
 import { MerkleTreeId } from '@aztec/circuit-types';
-import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log';
-import { type Gauge, type Meter, type TelemetryClient, ValueType } from '@aztec/telemetry-client';
+import { createLogger } from '@aztec/foundation/log';
+import { Attributes, type Gauge, type TelemetryClient, ValueType } from '@aztec/telemetry-client';
 
 import { type DBStats, type TreeDBStats, type TreeMeta, type WorldStateStatusFull } from '../native/message.js';
 
-type TreeTypeString = 'nullifier' | 'note_hash' | 'archive' | 'message' | 'public_data';
-type DBTypeString = 'leaf_preimage' | 'leaf_indices' | 'nodes' | 'blocks';
+type DBTypeString = 'leaf_preimage' | 'leaf_indices' | 'nodes' | 'blocks' | 'block_indices';
 
-class TreeDBInstrumentation {
+export class WorldStateInstrumentation {
+  private dbMapSize: Gauge;
+  private treeSize: Gauge;
+  private unfinalisedHeight: Gauge;
+  private finalisedHeight: Gauge;
+  private oldestBlock: Gauge;
   private dbNumItems: Gauge;
   private dbUsedSize: Gauge;
 
-  constructor(meter: Meter, treeName: TreeTypeString, dbName: DBTypeString) {
-    this.dbUsedSize = meter.createGauge(`aztec.world_state.db_used_size.${dbName}.${treeName}`, {
-      description: `The current used database size for the ${treeName} tree ${dbName} database`,
+  constructor(telemetry: TelemetryClient, private log = createLogger('world-state:instrumentation')) {
+    const meter = telemetry.getMeter('World State');
+    this.dbMapSize = meter.createGauge(`aztec.world_state.db_map_size`, {
+      description: `The current configured map size for each merkle tree`,
       valueType: ValueType.INT,
     });
 
-    this.dbNumItems = meter.createGauge(`aztec.world_state.db_num_items.${dbName}.${treeName}`, {
-      description: `The current number of items in the ${treeName} tree ${dbName} database`,
+    this.treeSize = meter.createGauge(`aztec.world_state.tree_size`, {
+      description: `The current number of leaves in each merkle tree`,
       valueType: ValueType.INT,
     });
-  }
-
-  public updateMetrics(treeDbStats: DBStats) {
-    this.dbNumItems.record(Number(treeDbStats.numDataItems));
-    this.dbUsedSize.record(Number(treeDbStats.totalUsedSize));
-  }
-}
-
-class TreeInstrumentation {
-  private treeDbInstrumentation: Map<DBTypeString, TreeDBInstrumentation> = new Map<
-    DBTypeString,
-    TreeDBInstrumentation
-  >();
-  private dbMapSize: Gauge;
-  private treeSize: Gauge;
-  private unfinalisedHeight: Gauge;
-  private finalisedHeight: Gauge;
-  private oldestBlock: Gauge;
 
-  constructor(meter: Meter, treeName: TreeTypeString, private log: DebugLogger) {
-    this.dbMapSize = meter.createGauge(`aztec.world_state.db_map_size.${treeName}`, {
-      description: `The current configured map size for the ${treeName} tree`,
+    this.unfinalisedHeight = meter.createGauge(`aztec.world_state.unfinalised_height`, {
+      description: `The unfinalised block height of each merkle tree`,
       valueType: ValueType.INT,
     });
 
-    this.treeSize = meter.createGauge(`aztec.world_state.tree_size.${treeName}`, {
-      description: `The current number of leaves in the ${treeName} tree`,
+    this.finalisedHeight = meter.createGauge(`aztec.world_state.finalised_height`, {
+      description: `The finalised block height of each merkle tree`,
       valueType: ValueType.INT,
     });
 
-    this.unfinalisedHeight = meter.createGauge(`aztec.world_state.unfinalised_height.${treeName}`, {
-      description: `The unfinalised block height of the ${treeName} tree`,
+    this.oldestBlock = meter.createGauge(`aztec.world_state.oldest_block`, {
+      description: `The oldest historical block of each merkle tree`,
       valueType: ValueType.INT,
     });
 
-    this.finalisedHeight = meter.createGauge(`aztec.world_state.finalised_height.${treeName}`, {
-      description: `The finalised block height of the ${treeName} tree`,
+    this.dbUsedSize = meter.createGauge(`aztec.world_state.db_used_size`, {
+      description: `The current used database size for each db of each merkle tree`,
       valueType: ValueType.INT,
     });
 
-    this.oldestBlock = meter.createGauge(`aztec.world_state.oldest_block.${treeName}`, {
-      description: `The oldest historical block of the ${treeName} tree`,
+    this.dbNumItems = meter.createGauge(`aztec.world_state.db_num_items`, {
+      description: `The current number of items in each database of each merkle tree`,
       valueType: ValueType.INT,
     });
-
-    this.treeDbInstrumentation.set('blocks', new TreeDBInstrumentation(meter, treeName, 'blocks'));
-    this.treeDbInstrumentation.set('nodes', new TreeDBInstrumentation(meter, treeName, 'nodes'));
-    this.treeDbInstrumentation.set('leaf_preimage', new TreeDBInstrumentation(meter, treeName, 'leaf_preimage'));
-    this.treeDbInstrumentation.set('leaf_indices', new TreeDBInstrumentation(meter, treeName, 'leaf_indices'));
-  }
-
-  private updateDBMetrics(dbName: DBTypeString, dbStats: DBStats) {
-    const inst = this.treeDbInstrumentation.get(dbName);
-    if (!inst) {
-      this.log.error(`Failed to find instrumentation for ${dbName}`);
-      return;
-    }
-    inst.updateMetrics(dbStats);
   }
 
-  public updateMetrics(treeDbStats: TreeDBStats, treeMeta: TreeMeta) {
-    this.dbMapSize.record(Number(treeDbStats.mapSize));
-    this.treeSize.record(Number(treeMeta.committedSize));
-    this.finalisedHeight.record(Number(treeMeta.finalisedBlockHeight));
-    this.unfinalisedHeight.record(Number(treeMeta.unfinalisedBlockHeight));
-    this.oldestBlock.record(Number(treeMeta.oldestHistoricBlock));
-
-    this.updateDBMetrics('leaf_indices', treeDbStats.leafIndicesDBStats);
-    this.updateDBMetrics('leaf_preimage', treeDbStats.leafPreimagesDBStats);
-    this.updateDBMetrics('blocks', treeDbStats.blocksDBStats);
-    this.updateDBMetrics('nodes', treeDbStats.nodesDBStats);
-  }
-}
-
-export class WorldStateInstrumentation {
-  private treeInstrumentation: Map<MerkleTreeId, TreeInstrumentation> = new Map<MerkleTreeId, TreeInstrumentation>();
+  private updateTreeStats(treeDbStats: TreeDBStats, treeMeta: TreeMeta, tree: MerkleTreeId) {
+    this.dbMapSize.record(Number(treeDbStats.mapSize), {
+      [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree],
+    });
+    this.treeSize.record(Number(treeMeta.size), {
+      [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree],
+    });
+    this.unfinalisedHeight.record(Number(treeMeta.unfinalisedBlockHeight), {
+      [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree],
+    });
+    this.finalisedHeight.record(Number(treeMeta.finalisedBlockHeight), {
+      [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree],
+    });
+    this.oldestBlock.record(Number(treeMeta.oldestHistoricBlock), {
+      [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree],
+    });
 
-  constructor(telemetry: TelemetryClient, private log = createDebugLogger('aztec:world-state:instrumentation')) {
-    const meter = telemetry.getMeter('World State');
-    this.treeInstrumentation.set(MerkleTreeId.ARCHIVE, new TreeInstrumentation(meter, 'archive', log));
-    this.treeInstrumentation.set(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, new TreeInstrumentation(meter, 'message', log));
-    this.treeInstrumentation.set(MerkleTreeId.NOTE_HASH_TREE, new TreeInstrumentation(meter, 'note_hash', log));
-    this.treeInstrumentation.set(MerkleTreeId.NULLIFIER_TREE, new TreeInstrumentation(meter, 'nullifier', log));
-    this.treeInstrumentation.set(MerkleTreeId.PUBLIC_DATA_TREE, new TreeInstrumentation(meter, 'public_data', log));
+    this.updateTreeDBStats(treeDbStats.blockIndicesDBStats, 'block_indices', tree);
+    this.updateTreeDBStats(treeDbStats.blocksDBStats, 'blocks', tree);
+    this.updateTreeDBStats(treeDbStats.leafIndicesDBStats, 'leaf_indices', tree);
+    this.updateTreeDBStats(treeDbStats.leafPreimagesDBStats, 'leaf_preimage', tree);
+    this.updateTreeDBStats(treeDbStats.nodesDBStats, 'nodes', tree);
   }
 
-  private updateTreeStats(treeDbStats: TreeDBStats, treeMeta: TreeMeta, tree: MerkleTreeId) {
-    const instrumentation = this.treeInstrumentation.get(tree);
-    if (!instrumentation) {
-      this.log.error(`Failed to retrieve instrumentation for tree ${MerkleTreeId[tree]}`);
-      return;
-    }
-    instrumentation.updateMetrics(treeDbStats, treeMeta);
+  private updateTreeDBStats(dbStats: DBStats, dbType: DBTypeString, tree: MerkleTreeId) {
+    this.dbNumItems.record(Number(dbStats.numDataItems), {
+      [Attributes.WS_DB_DATA_TYPE]: dbType,
+      [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree],
+    });
+    this.dbUsedSize.record(Number(dbStats.totalUsedSize), {
+      [Attributes.WS_DB_DATA_TYPE]: dbType,
+      [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree],
+    });
   }
 
   public updateWorldStateMetrics(worldStateStatus: WorldStateStatusFull) {

package/src/synchronizer/server_world_state_synchronizer.ts

@@ -19,7 +19,7 @@ import { type L2BlockHandledStats } from '@aztec/circuit-types/stats';
 import { MerkleTreeCalculator } from '@aztec/circuits.js';
 import { L1_TO_L2_MSG_SUBTREE_HEIGHT } from '@aztec/circuits.js/constants';
 import { type Fr } from '@aztec/foundation/fields';
-import { createDebugLogger } from '@aztec/foundation/log';
+import { createLogger } from '@aztec/foundation/log';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { elapsed } from '@aztec/foundation/timer';
 import { SHA256Trunc } from '@aztec/merkle-tree';
@@ -54,7 +54,7 @@ export class ServerWorldStateSynchronizer
     private readonly l2BlockSource: L2BlockSource & L1ToL2MessageSource,
     private readonly config: WorldStateConfig,
     telemetry: TelemetryClient,
-    private readonly log = createDebugLogger('aztec:world_state'),
+    private readonly log = createLogger('world_state'),
   ) {
     this.instrumentation = new WorldStateInstrumentation(telemetry);
     this.merkleTreeCommitted = this.merkleTreeDb.getCommitted();
@@ -111,7 +111,7 @@ export class ServerWorldStateSynchronizer
   }
 
   protected createBlockStream() {
-    return new L2BlockStream(this.l2BlockSource, this, this, {
+    return new L2BlockStream(this.l2BlockSource, this, this, createLogger('world_state:block_stream'), {
       proven: this.config.worldStateProvenBlocksOnly,
       pollIntervalMS: this.config.worldStateBlockCheckIntervalMS,
       batchSize: this.config.worldStateBlockRequestBatchSize,
@@ -224,14 +224,13 @@ export class ServerWorldStateSynchronizer
    * @returns Whether the block handled was produced by this same node.
    */
   private async handleL2Blocks(l2Blocks: L2Block[]) {
-    this.log.verbose(`Handling new L2 blocks from ${l2Blocks[0].number} to ${l2Blocks[l2Blocks.length - 1].number}`);
     const messagePromises = l2Blocks.map(block => this.l2BlockSource.getL1ToL2Messages(BigInt(block.number)));
     const l1ToL2Messages: Fr[][] = await Promise.all(messagePromises);
     let updateStatus: WorldStateStatusFull | undefined = undefined;
 
     for (let i = 0; i < l2Blocks.length; i++) {
       const [duration, result] = await elapsed(() => this.handleL2Block(l2Blocks[i], l1ToL2Messages[i]));
-      this.log.verbose(`Handled new L2 block`, {
+      this.log.verbose(`World state updated with L2 block ${l2Blocks[i].number}`, {
         eventName: 'l2-block-handled',
         duration,
         unfinalisedBlockNumber: result.summary.unfinalisedBlockNumber,
@@ -272,7 +271,7 @@ export class ServerWorldStateSynchronizer
   }
 
   private async handleChainFinalized(blockNumber: number) {
-    this.log.verbose(`Chain finalized at block ${blockNumber}`);
+    this.log.verbose(`Finalized chain is now at block ${blockNumber}`);
     const summary = await this.merkleTreeDb.setFinalised(BigInt(blockNumber));
     if (this.historyToKeep === undefined) {
       return;
@@ -286,12 +285,12 @@ export class ServerWorldStateSynchronizer
   }
 
   private handleChainProven(blockNumber: number) {
-    this.log.verbose(`Chain proven at block ${blockNumber}`);
+    this.log.debug(`Proven chain is now at block ${blockNumber}`);
     return Promise.resolve();
   }
 
   private async handleChainPruned(blockNumber: number) {
-    this.log.info(`Chain pruned to block ${blockNumber}`);
+    this.log.warn(`Chain pruned to block ${blockNumber}`);
     const status = await this.merkleTreeDb.unwindBlocks(BigInt(blockNumber));
     this.latestBlockHashQuery = undefined;
     this.instrumentation.updateWorldStateMetrics(status);

package/src/world-state-db/merkle_tree_db.ts

@@ -1,5 +1,5 @@
 import { type L2Block, type MerkleTreeId } from '@aztec/circuit-types';
-import { type MerkleTreeReadOperations, type MerkleTreeWriteOperations } from '@aztec/circuit-types/interfaces';
+import { type ForkMerkleTreeOperations, type MerkleTreeReadOperations } from '@aztec/circuit-types/interfaces';
 import { type Fr, MAX_NULLIFIERS_PER_TX, MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX } from '@aztec/circuits.js';
 import { type IndexedTreeSnapshot, type TreeSnapshot } from '@aztec/merkle-tree';
 
@@ -32,7 +32,7 @@ export type TreeSnapshots = {
   [MerkleTreeId.ARCHIVE]: TreeSnapshot<Fr>;
 };
 
-export interface MerkleTreeAdminDatabase {
+export interface MerkleTreeAdminDatabase extends ForkMerkleTreeOperations {
   /**
    * Handles a single L2 block (i.e. Inserts the new note hashes into the merkle tree).
    * @param block - The L2 block to handle.
@@ -45,18 +45,6 @@ export interface MerkleTreeAdminDatabase {
    */
   getCommitted(): MerkleTreeReadOperations;
 
-  /**
-   * Gets a handle that allows reading the state as it was at the given block number
-   * @param blockNumber - The block number to get the snapshot for
-   */
-  getSnapshot(blockNumber: number): MerkleTreeReadOperations;
-
-  /**
-   * Forks the database at its current state.
-   * @param blockNumber - The block number to fork at. If not provided, the current block number is used.
-   */
-  fork(blockNumber?: number): Promise<MerkleTreeWriteOperations>;
-
   /**
    * Removes all historical snapshots up to but not including the given block number
    * @param toBlockNumber The block number of the new oldest historical block

package/src/world-state-db/merkle_tree_operations_facade.ts

@@ -6,7 +6,7 @@ import {
   type SequentialInsertionResult,
   type TreeInfo,
 } from '@aztec/circuit-types/interfaces';
-import { type Header, type StateReference } from '@aztec/circuits.js';
+import { type BlockHeader, type StateReference } from '@aztec/circuits.js';
 import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees';
 
 import { type MerkleTrees } from './merkle_trees.js';
@@ -39,7 +39,7 @@ export class MerkleTreeReadOperationsFacade implements MerkleTreeWriteOperations
    * Returns the initial header for the chain before the first block.
    * @returns The initial header.
    */
-  getInitialHeader(): Header {
+  getInitialHeader(): BlockHeader {
     return this.trees.getInitialHeader();
   }
 
@@ -149,7 +149,7 @@ export class MerkleTreeReadOperationsFacade implements MerkleTreeWriteOperations
    * This includes all of the current roots of all of the data trees and the current blocks global vars.
    * @param header - The header to insert into the archive.
    */
-  public updateArchive(header: Header): Promise<void> {
+  public updateArchive(header: BlockHeader): Promise<void> {
     return this.trees.updateArchive(header);
   }
 
@@ -181,6 +181,13 @@ export class MerkleTreeReadOperationsFacade implements MerkleTreeWriteOperations
     throw new Error('Method not implemented in legacy merkle tree');
   }
 
+  getBlockNumbersForLeafIndices<ID extends MerkleTreeId>(
+    _treeId: ID,
+    _leafIndices: bigint[],
+  ): Promise<(bigint | undefined)[]> {
+    throw new Error('Method not implemented in legacy merkle tree');
+  }
+
   close(): Promise<void> {
     return Promise.resolve();
   }

package/src/world-state-db/merkle_tree_snapshot_operations_facade.ts

@@ -5,7 +5,13 @@ import {
   type MerkleTreeReadOperations,
   type TreeInfo,
 } from '@aztec/circuit-types/interfaces';
-import { AppendOnlyTreeSnapshot, Fr, type Header, PartialStateReference, StateReference } from '@aztec/circuits.js';
+import {
+  AppendOnlyTreeSnapshot,
+  type BlockHeader,
+  Fr,
+  PartialStateReference,
+  StateReference,
+} from '@aztec/circuits.js';
 import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees';
 import { type IndexedTreeSnapshot } from '@aztec/merkle-tree';
 
@@ -101,6 +107,10 @@ export class MerkleTreeSnapshotOperationsFacade implements MerkleTreeReadOperati
     };
   }
 
+  getBlockNumbersForLeafIndices<ID extends MerkleTreeId>(_a: ID, _b: bigint[]): Promise<(bigint | undefined)[]> {
+    throw new Error('Not implemented');
+  }
+
   async getStateReference(): Promise<StateReference> {
     const snapshots = await Promise.all([
       this.#getTreeSnapshot(MerkleTreeId.NULLIFIER_TREE),
@@ -132,7 +142,7 @@ export class MerkleTreeSnapshotOperationsFacade implements MerkleTreeReadOperati
     );
   }
 
-  getInitialHeader(): Header {
+  getInitialHeader(): BlockHeader {
     throw new Error('Getting initial header not supported on snapshot.');
   }
 }

package/src/world-state-db/merkle_trees.ts

@@ -10,8 +10,8 @@ import {
 import {
   ARCHIVE_HEIGHT,
   AppendOnlyTreeSnapshot,
+  BlockHeader,
   Fr,
-  Header,
   L1_TO_L2_MSG_TREE_HEIGHT,
   MAX_NOTE_HASHES_PER_TX,
   MAX_NULLIFIERS_PER_TX,
@@ -31,12 +31,12 @@ import {
   StateReference,
 } from '@aztec/circuits.js';
 import { padArrayEnd } from '@aztec/foundation/collection';
-import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log';
+import { type Logger, createLogger } from '@aztec/foundation/log';
 import { SerialQueue } from '@aztec/foundation/queue';
 import { Timer, elapsed } from '@aztec/foundation/timer';
 import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees';
 import { type AztecKVStore, type AztecSingleton } from '@aztec/kv-store';
-import { openTmpStore } from '@aztec/kv-store/utils';
+import { openTmpStore } from '@aztec/kv-store/lmdb';
 import {
   type AppendOnlyTree,
   type IndexedTree,
@@ -111,7 +111,7 @@ export class MerkleTrees implements MerkleTreeAdminDatabase {
   private initialStateReference: AztecSingleton<Buffer>;
   private metrics: WorldStateMetrics;
 
-  private constructor(private store: AztecKVStore, private telemetryClient: TelemetryClient, private log: DebugLogger) {
+  private constructor(private store: AztecKVStore, private telemetryClient: TelemetryClient, private log: Logger) {
     this.initialStateReference = store.openSingleton('merkle_trees_initial_state_reference');
     this.metrics = new WorldStateMetrics(telemetryClient);
   }
@@ -121,7 +121,11 @@ export class MerkleTrees implements MerkleTreeAdminDatabase {
    * @param store - The db instance to use for data persistance.
    * @returns - A fully initialized MerkleTrees instance.
    */
-  public static async new(store: AztecKVStore, client: TelemetryClient, log = createDebugLogger('aztec:merkle_trees')) {
+  public static async new(
+    store: AztecKVStore,
+    client: TelemetryClient,
+    log = createLogger('world-state:merkle_trees'),
+  ) {
     const merkleTrees = new MerkleTrees(store, client, log);
     await merkleTrees.#init();
     return merkleTrees;
@@ -239,7 +243,7 @@ export class MerkleTrees implements MerkleTreeAdminDatabase {
     const forked = new MerkleTrees(
       this.store,
       this.telemetryClient,
-      createDebugLogger('aztec:merkle_trees:ephemeral_fork'),
+      createLogger('world-state:merkle_trees:ephemeral_fork'),
     );
     await forked.#init(true);
     return new MerkleTreeReadOperationsFacade(forked, true);
@@ -249,8 +253,8 @@ export class MerkleTrees implements MerkleTreeAdminDatabase {
     await this.store.delete();
   }
 
-  public getInitialHeader(): Header {
-    return Header.empty({ state: this.#loadInitialStateReference() });
+  public getInitialHeader(): BlockHeader {
+    return BlockHeader.empty({ state: this.#loadInitialStateReference() });
   }
 
   /**
@@ -285,7 +289,7 @@ export class MerkleTrees implements MerkleTreeAdminDatabase {
    * @param header - The header whose hash to insert into the archive.
    * @param includeUncommitted - Indicates whether to include uncommitted data.
    */
-  public async updateArchive(header: Header) {
+  public async updateArchive(header: BlockHeader) {
     await this.synchronize(() => this.#updateArchive(header));
   }
 
@@ -519,7 +523,7 @@ export class MerkleTrees implements MerkleTreeAdminDatabase {
     return StateReference.fromBuffer(serialized);
   }
 
-  async #updateArchive(header: Header) {
+  async #updateArchive(header: BlockHeader) {
     const state = await this.getStateReference(true);
 
     // This method should be called only when the block builder already updated the state so we sanity check that it's