@aztec/world-state 0.1.0-alpha43 → 0.1.0-alpha44

package/src/synchroniser/server_world_state_synchroniser.ts

@@ -1,10 +1,10 @@
-import { Fr } from '@aztec/foundation/fields';
+import { SerialQueue } from '@aztec/foundation/fifo';
 import { createDebugLogger } from '@aztec/foundation/log';
 import { L2Block, L2BlockDownloader, L2BlockSource } from '@aztec/types';
 
-import { MerkleTreeDb, MerkleTreeOperations, computeGlobalVariablesHash } from '../index.js';
+import { MerkleTreeOperations, MerkleTrees } from '../index.js';
 import { MerkleTreeOperationsFacade } from '../merkle-tree/merkle_tree_operations_facade.js';
-import { getConfigEnvVars } from './config.js';
+import { WorldStateConfig } from './config.js';
 import { WorldStateRunningState, WorldStateStatus, WorldStateSynchroniser } from './world_state_synchroniser.js';
 
 /**
@@ -19,19 +19,17 @@ export class ServerWorldStateSynchroniser implements WorldStateSynchroniser {
   private l2BlockDownloader: L2BlockDownloader;
   private syncPromise: Promise<void> = Promise.resolve();
   private syncResolve?: () => void = undefined;
+  private jobQueue = new SerialQueue();
   private stopping = false;
   private runningPromise: Promise<void> = Promise.resolve();
   private currentState: WorldStateRunningState = WorldStateRunningState.IDLE;
 
-  /** The latest Global Variables hash for the HEAD of the chain. */
-  public latestGlobalVariablesHash: Fr = Fr.ZERO;
-
   constructor(
-    private merkleTreeDb: MerkleTreeDb,
+    private merkleTreeDb: MerkleTrees,
     private l2BlockSource: L2BlockSource,
+    config: WorldStateConfig,
     private log = createDebugLogger('aztec:world_state'),
   ) {
-    const config = getConfigEnvVars();
     this.l2BlockDownloader = new L2BlockDownloader(
       l2BlockSource,
       config.l2QueueSize,
@@ -57,7 +55,6 @@ export class ServerWorldStateSynchroniser implements WorldStateSynchroniser {
 
     // get the current latest block number
     this.latestBlockNumberAtStart = await this.l2BlockSource.getBlockHeight();
-    this.latestGlobalVariablesHash = await computeGlobalVariablesHash();
 
     const blockToDownloadFrom = this.currentL2BlockNum + 1;
 
@@ -78,10 +75,10 @@ export class ServerWorldStateSynchroniser implements WorldStateSynchroniser {
     // start looking for further blocks
     const blockProcess = async () => {
       while (!this.stopping) {
-        const blocks = await this.l2BlockDownloader.getL2Blocks();
-        await this.handleL2Blocks(blocks);
+        await this.jobQueue.put(() => this.collectAndProcessBlocks());
       }
     };
+    this.jobQueue.start();
     this.runningPromise = blockProcess();
     this.l2BlockDownloader.start(blockToDownloadFrom);
     this.log(`Started block downloader from block ${blockToDownloadFrom}`);
@@ -92,6 +89,8 @@ export class ServerWorldStateSynchroniser implements WorldStateSynchroniser {
     this.log('Stopping world state...');
     this.stopping = true;
     await this.l2BlockDownloader.stop();
+    await this.jobQueue.cancel();
+    await this.merkleTreeDb.stop();
     await this.runningPromise;
     this.setCurrentState(WorldStateRunningState.STOPPED);
   }
@@ -104,6 +103,56 @@ export class ServerWorldStateSynchroniser implements WorldStateSynchroniser {
     return Promise.resolve(status);
   }
 
+  /**
+   * Forces an immediate sync
+   * @param blockHeight - The minimum block height that we must sync to
+   * @returns A promise that resolves once the sync has completed.
+   */
+  public async syncImmediate(blockHeight?: number): Promise<void> {
+    if (this.currentState !== WorldStateRunningState.RUNNING) {
+      throw new Error(`World State is not running, unable to perform sync`);
+    }
+    // If we have been given a block height to sync to and we have reached that height
+    // then return.
+    if (blockHeight !== undefined && blockHeight <= this.currentL2BlockNum) {
+      return;
+    }
+    const blockToSyncTo = blockHeight === undefined ? 'latest' : `${blockHeight}`;
+    this.log(`World State at block ${this.currentL2BlockNum}, told to sync to block ${blockToSyncTo}...`);
+    // ensure any outstanding block updates are completed first.
+    await this.jobQueue.syncPoint();
+    while (true) {
+      // Check the block height again
+      if (blockHeight !== undefined && blockHeight <= this.currentL2BlockNum) {
+        return;
+      }
+      // Poll for more blocks
+      const numBlocks = await this.l2BlockDownloader.pollImmediate();
+      this.log(`Block download immediate poll yielded ${numBlocks} blocks`);
+      if (numBlocks) {
+        // More blocks were received, process them and go round again
+        await this.jobQueue.put(() => this.collectAndProcessBlocks());
+        continue;
+      }
+      // No blocks are available, if we have been given a block height then we can't achieve it
+      if (blockHeight !== undefined) {
+        throw new Error(
+          `Unable to sync to block height ${blockHeight}, currently synced to block ${this.currentL2BlockNum}`,
+        );
+      }
+      return;
+    }
+  }
+
+  /**
+   * Checks for the availability of new blocks and processes them.
+   */
+  private async collectAndProcessBlocks() {
+    // This request for blocks will timeout after 1 second if no blocks are received
+    const blocks = await this.l2BlockDownloader.getL2Blocks(1);
+    await this.handleL2Blocks(blocks);
+  }
+
   /**
    * Handles a list of L2 blocks (i.e. Inserts the new commitments into the merkle tree).
    * @param l2Blocks - The L2 blocks to handle.
@@ -121,8 +170,6 @@ export class ServerWorldStateSynchroniser implements WorldStateSynchroniser {
   private async handleL2Block(l2Block: L2Block) {
     await this.merkleTreeDb.handleL2Block(l2Block);
     this.currentL2BlockNum = l2Block.number;
-    this.latestGlobalVariablesHash = await computeGlobalVariablesHash(l2Block.globalVariables);
-    this.log(`Synced global variables with hash ${this.latestGlobalVariablesHash.toString()}`);
     if (
       this.currentState === WorldStateRunningState.SYNCHING &&
       this.currentL2BlockNum >= this.latestBlockNumberAtStart

package/src/synchroniser/world_state_synchroniser.ts

@@ -1,5 +1,3 @@
-import { Fr } from '@aztec/circuits.js';
-
 import { MerkleTreeOperations } from '../index.js';
 
 /**
@@ -47,6 +45,13 @@ export interface WorldStateSynchroniser {
    */
   stop(): Promise<void>;
 
+  /**
+   * Forces an immediate sync to an optionally provided minimum block height
+   * @param blockHeight - The minimum block height that we must sync to
+   * @returns A promise that resolves once the sync has completed.
+   */
+  syncImmediate(blockHeight?: number): Promise<void>;
+
   /**
    * Returns an instance of MerkleTreeOperations that will include uncommitted data.
    * @returns An instance of MerkleTreeOperations that will include uncommitted data.
@@ -58,9 +63,4 @@ export interface WorldStateSynchroniser {
    * @returns An instance of MerkleTreeOperations that will not include uncommitted data.
    */
   getCommitted(): MerkleTreeOperations;
-
-  /**
-   * The latest Global Variables hash for the HEAD of the chain.
-   */
-  latestGlobalVariablesHash: Fr;
 }

package/src/world-state-db/index.ts

@@ -1,4 +1,5 @@
-import { GlobalVariables, MAX_NEW_NULLIFIERS_PER_TX } from '@aztec/circuits.js';
+import { MAX_NEW_NULLIFIERS_PER_TX } from '@aztec/circuits.js';
+import { Fr } from '@aztec/foundation/fields';
 import { createDebugLogger } from '@aztec/foundation/log';
 import { LeafData, LowLeafWitnessData } from '@aztec/merkle-tree';
 import { L2Block, MerkleTreeId, SiblingPath } from '@aztec/types';
@@ -115,14 +116,14 @@ export interface MerkleTreeOperations {
   /**
    * Gets the current roots of the commitment trees.
    */
-  getTreeRoots(): CurrentTreeRoots;
+  getTreeRoots(): Promise<CurrentTreeRoots>;
 
   /**
    * Gets sibling path for a leaf.
    * @param treeId - The tree to be queried for a sibling path.
    * @param index - The index of the leaf for which a sibling path should be returned.
    */
-  getSiblingPath(treeId: MerkleTreeId, index: bigint): Promise<SiblingPath<number>>;
+  getSiblingPath<N extends number>(treeId: MerkleTreeId, index: bigint): Promise<SiblingPath<N>>;
 
   /**
    * Returns the previous index for a given value in an indexed tree.
@@ -175,9 +176,20 @@ export interface MerkleTreeOperations {
   /**
    * Inserts the new block hash into the new block hashes tree.
    * This includes all of the current roots of all of the data trees and the current blocks global vars.
-   * @param globalVariables - The global variables to insert into the block hash.
+   * @param globalVariablesHash - The global variables hash to insert into the block hash.
    */
-  updateHistoricBlocksTree(globalVariables: GlobalVariables): Promise<void>;
+  updateHistoricBlocksTree(globalVariablesHash: Fr): Promise<void>;
+
+  /**
+   * Updates the latest global variables hash
+   * @param globalVariablesHash - The latest global variables hash
+   */
+  updateLatestGlobalVariablesHash(globalVariablesHash: Fr): Promise<void>;
+
+  /**
+   * Gets the global variables hash from the previous block
+   */
+  getLatestGlobalVariablesHash(): Promise<Fr>;
 
   /**
    * Batch insert multiple leaves into the tree.

package/src/world-state-db/merkle_trees.ts

@@ -10,7 +10,8 @@ import {
   PRIVATE_DATA_TREE_HEIGHT,
   PUBLIC_DATA_TREE_HEIGHT,
 } from '@aztec/circuits.js';
-import { computeBlockHashWithGlobals } from '@aztec/circuits.js/abis';
+import { computeBlockHash } from '@aztec/circuits.js/abis';
+import { Committable } from '@aztec/foundation/committable';
 import { SerialQueue } from '@aztec/foundation/fifo';
 import { createDebugLogger } from '@aztec/foundation/log';
 import { IWasmModule } from '@aztec/foundation/wasm';
@@ -31,6 +32,7 @@ import { L2Block, MerkleTreeId, SiblingPath, merkleTreeIds } from '@aztec/types'
 import { default as levelup } from 'levelup';
 
 import { MerkleTreeOperationsFacade } from '../merkle-tree/merkle_tree_operations_facade.js';
+import { computeGlobalVariablesHash } from '../utils.js';
 import {
   CurrentTreeRoots,
   INITIAL_NULLIFIER_TREE_SIZE,
@@ -46,9 +48,12 @@ import {
  */
 export class MerkleTrees implements MerkleTreeDb {
   private trees: (AppendOnlyTree | UpdateOnlyTree)[] = [];
+  private latestGlobalVariablesHash: Committable<Fr>;
   private jobQueue = new SerialQueue();
 
-  constructor(private db: levelup.LevelUp, private log = createDebugLogger('aztec:merkle_trees')) {}
+  constructor(private db: levelup.LevelUp, private log = createDebugLogger('aztec:merkle_trees')) {
+    this.latestGlobalVariablesHash = new Committable(Fr.zero());
+  }
 
   /**
    * Initialises the collection of Merkle Trees.
@@ -105,8 +110,10 @@ export class MerkleTrees implements MerkleTreeDb {
     this.jobQueue.start();
 
     // The first leaf in the blocks tree contains the empty roots of the other trees and empty global variables.
-    await this.updateHistoricBlocksTree(GlobalVariables.empty(), true);
-    await historicBlocksTree.commit();
+    const initialGlobalVariablesHash = await computeGlobalVariablesHash(GlobalVariables.empty());
+    await this._updateLatestGlobalVariablesHash(initialGlobalVariablesHash);
+    await this._updateHistoricBlocksTree(initialGlobalVariablesHash, true);
+    await this._commit();
   }
 
   /**
@@ -147,12 +154,27 @@ export class MerkleTrees implements MerkleTreeDb {
   /**
    * Inserts into the roots trees (CONTRACT_TREE_ROOTS_TREE, PRIVATE_DATA_TREE_ROOTS_TREE, L1_TO_L2_MESSAGES_TREE_ROOTS_TREE)
    * the current roots of the corresponding trees (CONTRACT_TREE, PRIVATE_DATA_TREE, L1_TO_L2_MESSAGES_TREE).
-   * @param globals - The global variables to use for hashing.
+   * @param globalsHash - The current global variables hash.
+   * @param includeUncommitted - Indicates whether to include uncommitted data.
+   */
+  public async updateHistoricBlocksTree(globalsHash: Fr, includeUncommitted: boolean) {
+    await this.synchronise(() => this._updateHistoricBlocksTree(globalsHash, includeUncommitted));
+  }
+
+  /**
+   * Updates the latest global variables hash
+   * @param globalVariablesHash - The latest global variables hash
+   */
+  public async updateLatestGlobalVariablesHash(globalVariablesHash: Fr) {
+    return await this.synchronise(() => this._updateLatestGlobalVariablesHash(globalVariablesHash));
+  }
+
+  /**
+   * Gets the global variables hash from the previous block
    * @param includeUncommitted - Indicates whether to include uncommitted data.
    */
-  public async updateHistoricBlocksTree(globals: GlobalVariables, includeUncommitted: boolean) {
-    const blockHash = await this.getCurrentBlockHash(globals, includeUncommitted);
-    await this.appendLeaves(MerkleTreeId.BLOCKS_TREE, [blockHash.toBuffer()]);
+  public async getLatestGlobalVariablesHash(includeUncommitted: boolean): Promise<Fr> {
+    return await this.synchronise(() => this._getGlobalVariablesHash(includeUncommitted));
   }
 
   /**
@@ -170,8 +192,8 @@ export class MerkleTrees implements MerkleTreeDb {
    * @param includeUncommitted - Indicates whether to include uncommitted data.
    * @returns The current roots of the trees.
    */
-  public getTreeRoots(includeUncommitted: boolean): CurrentTreeRoots {
-    const roots = this.getAllTreeRoots(includeUncommitted);
+  public async getTreeRoots(includeUncommitted: boolean): Promise<CurrentTreeRoots> {
+    const roots = await this.synchronise(() => Promise.resolve(this._getAllTreeRoots(includeUncommitted)));
 
     return {
       privateDataTreeRoot: roots[0],
@@ -183,14 +205,14 @@ export class MerkleTrees implements MerkleTreeDb {
     };
   }
 
-  async getCurrentBlockHash(globals: GlobalVariables, includeUncommitted: boolean): Promise<Fr> {
-    const roots = this.getAllTreeRoots(includeUncommitted).map(root => Fr.fromBuffer(root));
+  private async _getCurrentBlockHash(globalsHash: Fr, includeUncommitted: boolean): Promise<Fr> {
+    const roots = (await this._getAllTreeRoots(includeUncommitted)).map(root => Fr.fromBuffer(root));
     const wasm = await CircuitsWasm.get();
-    return computeBlockHashWithGlobals(wasm, globals, roots[0], roots[1], roots[2], roots[3], roots[4]);
+    return computeBlockHash(wasm, globalsHash, roots[0], roots[1], roots[2], roots[3], roots[4]);
   }
 
-  getAllTreeRoots(includeUncommitted: boolean): Buffer[] {
-    return [
+  private _getAllTreeRoots(includeUncommitted: boolean): Promise<Buffer[]> {
+    const roots = [
       MerkleTreeId.PRIVATE_DATA_TREE,
       MerkleTreeId.NULLIFIER_TREE,
       MerkleTreeId.CONTRACT_TREE,
@@ -198,6 +220,8 @@ export class MerkleTrees implements MerkleTreeDb {
       MerkleTreeId.PUBLIC_DATA_TREE,
       MerkleTreeId.BLOCKS_TREE,
     ].map(tree => this.trees[tree].getRoot(includeUncommitted));
+
+    return Promise.resolve(roots);
   }
 
   /**
@@ -377,6 +401,20 @@ export class MerkleTrees implements MerkleTreeDb {
     return await this.jobQueue.put(fn);
   }
 
+  private _updateLatestGlobalVariablesHash(globalVariablesHash: Fr): Promise<void> {
+    this.latestGlobalVariablesHash.set(globalVariablesHash);
+    return Promise.resolve();
+  }
+
+  private _getGlobalVariablesHash(includeUncommitted: boolean): Promise<Fr> {
+    return Promise.resolve(this.latestGlobalVariablesHash.get(includeUncommitted));
+  }
+
+  private async _updateHistoricBlocksTree(globalsHash: Fr, includeUncommitted: boolean) {
+    const blockHash = await this._getCurrentBlockHash(globalsHash, includeUncommitted);
+    await this._appendLeaves(MerkleTreeId.BLOCKS_TREE, [blockHash.toBuffer()]);
+  }
+
   /**
    * Returns the tree info for the specified tree id.
    * @param treeId - Id of the tree to get information from.
@@ -451,6 +489,7 @@ export class MerkleTrees implements MerkleTreeDb {
     for (const tree of this.trees) {
       await tree.commit();
     }
+    this.latestGlobalVariablesHash.commit();
   }
 
   /**
@@ -461,6 +500,7 @@ export class MerkleTrees implements MerkleTreeDb {
     for (const tree of this.trees) {
       await tree.rollback();
     }
+    this.latestGlobalVariablesHash.rollback();
   }
 
   /**
@@ -514,7 +554,11 @@ export class MerkleTrees implements MerkleTreeDb {
       }
 
       // Sync and add the block to the historic blocks tree
-      const blockHash = await this.getCurrentBlockHash(l2Block.globalVariables, true);
+      const globalVariablesHash = await computeGlobalVariablesHash(l2Block.globalVariables);
+      await this._updateLatestGlobalVariablesHash(globalVariablesHash);
+      this.log(`Synced global variables with hash ${this.latestGlobalVariablesHash.toString()}`);
+
+      const blockHash = await this._getCurrentBlockHash(globalVariablesHash, true);
       await this._appendLeaves(MerkleTreeId.BLOCKS_TREE, [blockHash.toBuffer()]);
 
       await this._commit();