@aztec/world-state 0.65.0 → 0.65.2

package/src/native/native_world_state.ts

@@ -32,6 +32,7 @@ import { MerkleTreesFacade, MerkleTreesForkFacade, serializeLeaf } from './merkl
 import {
   WorldStateMessageType,
   type WorldStateStatusFull,
+  type WorldStateStatusSummary,
   blockStateReference,
   sanitiseFullStatus,
   sanitiseSummary,
@@ -52,6 +53,8 @@ export const WORLD_STATE_DB_VERSION = 1; // The initial version
 
 export class NativeWorldStateService implements MerkleTreeDatabase {
   protected initialHeader: Header | undefined;
+  // This is read heavily and only changes when data is persisted, so we cache it
+  private cachedStatusSummary: WorldStateStatusSummary | undefined;
 
   protected constructor(
     protected readonly instance: NativeWorldState,
@@ -175,29 +178,29 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
       .flatMap(txEffect => padArrayEnd(txEffect.nullifiers, Fr.ZERO, MAX_NULLIFIERS_PER_TX))
       .map(nullifier => new NullifierLeaf(nullifier));
 
-    // We insert the public data tree leaves with one batch per tx to avoid updating the same key twice
-    const batchesOfPublicDataWrites: PublicDataTreeLeaf[][] = [];
-    for (const txEffect of paddedTxEffects) {
-      batchesOfPublicDataWrites.push(
-        txEffect.publicDataWrites.map(write => {
-          if (write.isEmpty()) {
-            throw new Error('Public data write must not be empty when syncing');
-          }
-          return new PublicDataTreeLeaf(write.leafSlot, write.value);
-        }),
-      );
-    }
-
-    const response = await this.instance.call(WorldStateMessageType.SYNC_BLOCK, {
-      blockNumber: l2Block.number,
-      blockHeaderHash: l2Block.header.hash(),
-      paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf),
-      paddedNoteHashes: paddedNoteHashes.map(serializeLeaf),
-      paddedNullifiers: paddedNullifiers.map(serializeLeaf),
-      batchesOfPublicDataWrites: batchesOfPublicDataWrites.map(batch => batch.map(serializeLeaf)),
-      blockStateRef: blockStateReference(l2Block.header.state),
+    const publicDataWrites: PublicDataTreeLeaf[] = paddedTxEffects.flatMap(txEffect => {
+      return txEffect.publicDataWrites.map(write => {
+        if (write.isEmpty()) {
+          throw new Error('Public data write must not be empty when syncing');
+        }
+        return new PublicDataTreeLeaf(write.leafSlot, write.value);
+      });
     });
-    return sanitiseFullStatus(response);
+
+    return await this.instance.call(
+      WorldStateMessageType.SYNC_BLOCK,
+      {
+        blockNumber: l2Block.number,
+        blockHeaderHash: l2Block.header.hash(),
+        paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf),
+        paddedNoteHashes: paddedNoteHashes.map(serializeLeaf),
+        paddedNullifiers: paddedNullifiers.map(serializeLeaf),
+        publicDataWrites: publicDataWrites.map(serializeLeaf),
+        blockStateRef: blockStateReference(l2Block.header.state),
+      },
+      this.sanitiseAndCacheSummaryFromFull.bind(this),
+      this.deleteCachedSummary.bind(this),
+    );
   }
 
   public async close(): Promise<void> {
@@ -210,16 +213,37 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
     return Header.empty({ state });
   }
 
+  private sanitiseAndCacheSummaryFromFull(response: WorldStateStatusFull) {
+    const sanitised = sanitiseFullStatus(response);
+    this.cachedStatusSummary = { ...sanitised.summary };
+    return sanitised;
+  }
+
+  private sanitiseAndCacheSummary(response: WorldStateStatusSummary) {
+    const sanitised = sanitiseSummary(response);
+    this.cachedStatusSummary = { ...sanitised };
+    return sanitised;
+  }
+
+  private deleteCachedSummary(_: string) {
+    this.cachedStatusSummary = undefined;
+  }
+
   /**
    * Advances the finalised block number to be the number provided
    * @param toBlockNumber The block number that is now the tip of the finalised chain
    * @returns The new WorldStateStatus
    */
   public async setFinalised(toBlockNumber: bigint) {
-    const response = await this.instance.call(WorldStateMessageType.FINALISE_BLOCKS, {
-      toBlockNumber,
-    });
-    return sanitiseSummary(response);
+    await this.instance.call(
+      WorldStateMessageType.FINALISE_BLOCKS,
+      {
+        toBlockNumber,
+      },
+      this.sanitiseAndCacheSummary.bind(this),
+      this.deleteCachedSummary.bind(this),
+    );
+    return this.getStatusSummary();
   }
 
   /**
@@ -228,10 +252,14 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
    * @returns The new WorldStateStatus
    */
   public async removeHistoricalBlocks(toBlockNumber: bigint) {
-    const response = await this.instance.call(WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS, {
-      toBlockNumber,
-    });
-    return sanitiseFullStatus(response);
+    return await this.instance.call(
+      WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS,
+      {
+        toBlockNumber,
+      },
+      this.sanitiseAndCacheSummaryFromFull.bind(this),
+      this.deleteCachedSummary.bind(this),
+    );
   }
 
   /**
@@ -240,15 +268,21 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
    * @returns The new WorldStateStatus
    */
   public async unwindBlocks(toBlockNumber: bigint) {
-    const response = await this.instance.call(WorldStateMessageType.UNWIND_BLOCKS, {
-      toBlockNumber,
-    });
-    return sanitiseFullStatus(response);
+    return await this.instance.call(
+      WorldStateMessageType.UNWIND_BLOCKS,
+      {
+        toBlockNumber,
+      },
+      this.sanitiseAndCacheSummaryFromFull.bind(this),
+      this.deleteCachedSummary.bind(this),
+    );
   }
 
   public async getStatusSummary() {
-    const response = await this.instance.call(WorldStateMessageType.GET_STATUS, void 0);
-    return sanitiseSummary(response);
+    if (this.cachedStatusSummary !== undefined) {
+      return { ...this.cachedStatusSummary };
+    }
+    return await this.instance.call(WorldStateMessageType.GET_STATUS, void 0, this.sanitiseAndCacheSummary.bind(this));
   }
 
   updateLeaf<ID extends IndexedTreeId>(

package/src/native/native_world_state_instance.ts

@@ -108,16 +108,28 @@ export class NativeWorldState implements NativeWorldStateInstance {
    * Sends a message to the native instance and returns the response.
    * @param messageType - The type of message to send
    * @param body - The message body
+   * @param responseHandler - A callback accepting the response, executed on the job queue
+   * @param errorHandler - A callback called on request error, executed on the job queue
    * @returns The response to the message
    */
   public call<T extends WorldStateMessageType>(
     messageType: T,
     body: WorldStateRequest[T],
+    // allows for the pre-processing of responses on the job queue before being passed back
+    responseHandler = (response: WorldStateResponse[T]): WorldStateResponse[T] => response,
+    errorHandler = (_: string) => {},
   ): Promise<WorldStateResponse[T]> {
-    return this.queue.put(() => {
+    return this.queue.put(async () => {
       assert.notEqual(messageType, WorldStateMessageType.CLOSE, 'Use close() to close the native instance');
       assert.equal(this.open, true, 'Native instance is closed');
-      return this._sendMessage(messageType, body);
+      let response: WorldStateResponse[T];
+      try {
+        response = await this._sendMessage(messageType, body);
+      } catch (error: any) {
+        errorHandler(error.message);
+        throw error;
+      }
+      return responseHandler(response);
     });
   }
 
@@ -188,7 +200,7 @@ export class NativeWorldState implements NativeWorldStateInstance {
         data['notesCount'] = body.paddedNoteHashes.length;
         data['nullifiersCount'] = body.paddedNullifiers.length;
         data['l1ToL2MessagesCount'] = body.paddedL1ToL2Messages.length;
-        data['publicDataWritesCount'] = body.batchesOfPublicDataWrites.reduce((acc, batch) => acc + batch.length, 0);
+        data['publicDataWritesCount'] = body.publicDataWrites.length;
       }
 
       this.log.debug(`Calling messageId=${messageId} ${WorldStateMessageType[messageType]} with ${fmtLogData(data)}`);

package/src/native/world_state_version.ts

@@ -1,38 +1,29 @@
 import { EthAddress } from '@aztec/circuits.js';
+import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
 
 import { readFile, writeFile } from 'fs/promises';
+import { z } from 'zod';
 
 export class WorldStateVersion {
-  constructor(readonly version: number, readonly rollupAddress: EthAddress) {}
+  constructor(public readonly version: number, public readonly rollupAddress: EthAddress) {}
 
   static async readVersion(filename: string) {
     const versionData = await readFile(filename, 'utf-8').catch(() => undefined);
-    if (versionData === undefined) {
-      return undefined;
-    }
-    const versionJSON = JSON.parse(versionData);
-    if (versionJSON.version === undefined || versionJSON.rollupAddress === undefined) {
-      return undefined;
-    }
-    return WorldStateVersion.fromJSON(versionJSON);
+    return versionData === undefined ? undefined : jsonParseWithSchema(versionData, WorldStateVersion.schema);
   }
 
   public async writeVersionFile(filename: string) {
-    const data = JSON.stringify(this.toJSON());
+    const data = jsonStringify(this);
     await writeFile(filename, data, 'utf-8');
   }
 
-  toJSON() {
-    return {
-      version: this.version,
-      rollupAddress: this.rollupAddress.toChecksumString(),
-    };
-  }
-
-  static fromJSON(obj: any): WorldStateVersion {
-    const version = obj.version;
-    const rollupAddress = EthAddress.fromString(obj.rollupAddress);
-    return new WorldStateVersion(version, rollupAddress);
+  static get schema() {
+    return z
+      .object({
+        version: z.number(),
+        rollupAddress: EthAddress.schema,
+      })
+      .transform(({ version, rollupAddress }) => new WorldStateVersion(version, rollupAddress));
   }
 
   static empty() {

package/src/synchronizer/config.ts

@@ -1,4 +1,9 @@
-import { type ConfigMappingsType, booleanConfigHelper, getConfigFromMappings } from '@aztec/foundation/config';
+import {
+  type ConfigMappingsType,
+  booleanConfigHelper,
+  getConfigFromMappings,
+  numberConfigHelper,
+} from '@aztec/foundation/config';
 
 /** World State synchronizer configuration values. */
 export interface WorldStateConfig {
@@ -16,6 +21,9 @@ export interface WorldStateConfig {
 
   /** Optional directory for the world state DB, if unspecified will default to the general data directory */
   worldStateDataDirectory?: string;
+
+  /** The number of historic blocks to maintain */
+  worldStateBlockHistory: number;
 }
 
 export const worldStateConfigMappings: ConfigMappingsType<WorldStateConfig> = {
@@ -44,6 +52,11 @@ export const worldStateConfigMappings: ConfigMappingsType<WorldStateConfig> = {
     env: 'WS_DATA_DIRECTORY',
     description: 'Optional directory for the world state database',
   },
+  worldStateBlockHistory: {
+    env: 'WS_NUM_HISTORIC_BLOCKS',
+    description: 'The number of historic blocks to maintain. Values less than 1 mean all history is maintained',
+    ...numberConfigHelper(64),
+  },
 };
 
 /**

package/src/synchronizer/server_world_state_synchronizer.ts

@@ -41,7 +41,9 @@ export class ServerWorldStateSynchronizer
   private readonly merkleTreeCommitted: MerkleTreeReadOperations;
 
   private latestBlockNumberAtStart = 0;
+  private historyToKeep: number | undefined;
   private currentState: WorldStateRunningState = WorldStateRunningState.IDLE;
+  private latestBlockHashQuery: { blockNumber: number; hash: string | undefined } | undefined = undefined;
 
   private syncPromise = promiseWithResolvers<void>();
   protected blockStream: L2BlockStream | undefined;
@@ -56,6 +58,12 @@ export class ServerWorldStateSynchronizer
   ) {
     this.instrumentation = new WorldStateInstrumentation(telemetry);
     this.merkleTreeCommitted = this.merkleTreeDb.getCommitted();
+    this.historyToKeep = config.worldStateBlockHistory < 1 ? undefined : config.worldStateBlockHistory;
+    this.log.info(
+      `Created world state synchroniser with block history of ${
+        this.historyToKeep === undefined ? 'infinity' : this.historyToKeep
+      }`,
+    );
   }
 
   public getCommitted(): MerkleTreeReadOperations {
@@ -160,10 +168,19 @@ export class ServerWorldStateSynchronizer
   }
 
   /** Returns the L2 block hash for a given number. Used by the L2BlockStream for detecting reorgs. */
-  public getL2BlockHash(number: number): Promise<string | undefined> {
-    return number === 0
-      ? Promise.resolve(this.merkleTreeCommitted.getInitialHeader().hash().toString())
-      : this.merkleTreeCommitted.getLeafValue(MerkleTreeId.ARCHIVE, BigInt(number)).then(leaf => leaf?.toString());
+  public async getL2BlockHash(number: number): Promise<string | undefined> {
+    if (number === 0) {
+      return Promise.resolve(this.merkleTreeCommitted.getInitialHeader().hash().toString());
+    }
+    if (this.latestBlockHashQuery?.hash === undefined || number !== this.latestBlockHashQuery.blockNumber) {
+      this.latestBlockHashQuery = {
+        hash: await this.merkleTreeCommitted
+          .getLeafValue(MerkleTreeId.ARCHIVE, BigInt(number))
+          .then(leaf => leaf?.toString()),
+        blockNumber: number,
+      };
+    }
+    return this.latestBlockHashQuery.hash;
   }
 
   /** Returns the latest L2 block number for each tip of the chain (latest, proven, finalized). */
@@ -256,7 +273,16 @@ export class ServerWorldStateSynchronizer
 
   private async handleChainFinalized(blockNumber: number) {
     this.log.verbose(`Chain finalized at block ${blockNumber}`);
-    await this.merkleTreeDb.setFinalised(BigInt(blockNumber));
+    const summary = await this.merkleTreeDb.setFinalised(BigInt(blockNumber));
+    if (this.historyToKeep === undefined) {
+      return;
+    }
+    const newHistoricBlock = summary.finalisedBlockNumber - BigInt(this.historyToKeep) + 1n;
+    if (newHistoricBlock <= 1) {
+      return;
+    }
+    this.log.verbose(`Pruning historic blocks to ${newHistoricBlock}`);
+    await this.merkleTreeDb.removeHistoricalBlocks(newHistoricBlock);
   }
 
   private handleChainProven(blockNumber: number) {
@@ -267,6 +293,7 @@ export class ServerWorldStateSynchronizer
   private async handleChainPruned(blockNumber: number) {
     this.log.info(`Chain pruned to block ${blockNumber}`);
     const status = await this.merkleTreeDb.unwindBlocks(BigInt(blockNumber));
+    this.latestBlockHashQuery = undefined;
     this.instrumentation.updateWorldStateMetrics(status);
   }
 

package/src/world-state-db/merkle_tree_operations_facade.ts

@@ -3,6 +3,7 @@ import {
   type IndexedTreeId,
   type MerkleTreeLeafType,
   type MerkleTreeWriteOperations,
+  type SequentialInsertionResult,
   type TreeInfo,
 } from '@aztec/circuit-types/interfaces';
 import { type Header, type StateReference } from '@aztec/circuits.js';
@@ -167,6 +168,19 @@ export class MerkleTreeReadOperationsFacade implements MerkleTreeWriteOperations
     return this.trees.batchInsert(treeId, leaves, subtreeHeight);
   }
 
+  /**
+   * Sequentially inserts multiple leaves into the tree.
+   * @param treeId - The ID of the tree.
+   * @param leaves - Leaves to insert into the tree.
+   * @returns Witnesses for the operations performed.
+   */
+  public sequentialInsert<TreeHeight extends number>(
+    _treeId: IndexedTreeId,
+    _leaves: Buffer[],
+  ): Promise<SequentialInsertionResult<TreeHeight>> {
+    throw new Error('Method not implemented in legacy merkle tree');
+  }
+
   close(): Promise<void> {
     return Promise.resolve();
   }