@aztec/world-state 0.22.0 → 0.24.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aztec/world-state",
-  "version": "0.22.0",
+  "version": "0.24.0",
   "type": "module",
   "exports": "./dest/index.js",
   "typedocOptions": {
@@ -30,12 +30,12 @@
     "rootDir": "./src"
   },
   "dependencies": {
-    "@aztec/circuit-types": "0.22.0",
-    "@aztec/circuits.js": "0.22.0",
-    "@aztec/foundation": "0.22.0",
-    "@aztec/kv-store": "0.22.0",
-    "@aztec/merkle-tree": "0.22.0",
-    "@aztec/types": "0.22.0",
+    "@aztec/circuit-types": "0.24.0",
+    "@aztec/circuits.js": "0.24.0",
+    "@aztec/foundation": "0.24.0",
+    "@aztec/kv-store": "0.24.0",
+    "@aztec/merkle-tree": "0.24.0",
+    "@aztec/types": "0.24.0",
     "tslib": "^2.4.0"
   },
   "devDependencies": {

package/src/index.ts

@@ -0,0 +1,3 @@
+export * from './synchronizer/index.js';
+export * from './world-state-db/index.js';
+export * from './synchronizer/config.js';

package/src/synchronizer/config.ts

@@ -0,0 +1,27 @@
+/**
+ * World State synchronizer configuration values.
+ */
+export interface WorldStateConfig {
+  /**
+   * The frequency in which to check.
+   */
+  worldStateBlockCheckIntervalMS: number;
+
+  /**
+   * Size of queue of L2 blocks to store.
+   */
+  l2QueueSize: number;
+}
+
+/**
+ * Returns the configuration values for the world state synchronizer.
+ * @returns The configuration values for the world state synchronizer.
+ */
+export function getConfigEnvVars(): WorldStateConfig {
+  const { WS_BLOCK_CHECK_INTERVAL_MS, WS_L2_BLOCK_QUEUE_SIZE } = process.env;
+  const envVars: WorldStateConfig = {
+    worldStateBlockCheckIntervalMS: WS_BLOCK_CHECK_INTERVAL_MS ? +WS_BLOCK_CHECK_INTERVAL_MS : 100,
+    l2QueueSize: WS_L2_BLOCK_QUEUE_SIZE ? +WS_L2_BLOCK_QUEUE_SIZE : 1000,
+  };
+  return envVars;
+}

package/src/synchronizer/index.ts

@@ -0,0 +1,2 @@
+export * from './server_world_state_synchronizer.js';
+export * from './world_state_synchronizer.js';

package/src/synchronizer/server_world_state_synchronizer.ts

@@ -0,0 +1,215 @@
+import { L2Block, L2BlockDownloader, L2BlockSource } from '@aztec/circuit-types';
+import { L2BlockHandledStats } from '@aztec/circuit-types/stats';
+import { SerialQueue } from '@aztec/foundation/fifo';
+import { createDebugLogger } from '@aztec/foundation/log';
+import { elapsed } from '@aztec/foundation/timer';
+import { AztecKVStore, AztecSingleton } from '@aztec/kv-store';
+
+import { HandleL2BlockResult, MerkleTreeOperations, MerkleTrees } from '../world-state-db/index.js';
+import { MerkleTreeOperationsFacade } from '../world-state-db/merkle_tree_operations_facade.js';
+import { MerkleTreeSnapshotOperationsFacade } from '../world-state-db/merkle_tree_snapshot_operations_facade.js';
+import { WorldStateConfig } from './config.js';
+import { WorldStateRunningState, WorldStateStatus, WorldStateSynchronizer } from './world_state_synchronizer.js';
+
+/**
+ * Synchronizes the world state with the L2 blocks from a L2BlockSource.
+ * The synchronizer will download the L2 blocks from the L2BlockSource and insert the new commitments into the merkle
+ * tree.
+ */
+export class ServerWorldStateSynchronizer implements WorldStateSynchronizer {
+  private latestBlockNumberAtStart = 0;
+
+  private l2BlockDownloader: L2BlockDownloader;
+  private syncPromise: Promise<void> = Promise.resolve();
+  private syncResolve?: () => void = undefined;
+  private jobQueue = new SerialQueue();
+  private stopping = false;
+  private runningPromise: Promise<void> = Promise.resolve();
+  private currentState: WorldStateRunningState = WorldStateRunningState.IDLE;
+  private blockNumber: AztecSingleton<number>;
+
+  constructor(
+    store: AztecKVStore,
+    private merkleTreeDb: MerkleTrees,
+    private l2BlockSource: L2BlockSource,
+    config: WorldStateConfig,
+    private log = createDebugLogger('aztec:world_state'),
+  ) {
+    this.blockNumber = store.openSingleton('world_state_synch_last_block_number');
+    this.l2BlockDownloader = new L2BlockDownloader(
+      l2BlockSource,
+      config.l2QueueSize,
+      config.worldStateBlockCheckIntervalMS,
+    );
+  }
+
+  public getLatest(): MerkleTreeOperations {
+    return new MerkleTreeOperationsFacade(this.merkleTreeDb, true);
+  }
+
+  public getCommitted(): MerkleTreeOperations {
+    return new MerkleTreeOperationsFacade(this.merkleTreeDb, false);
+  }
+
+  public getSnapshot(blockNumber: number): MerkleTreeOperations {
+    return new MerkleTreeSnapshotOperationsFacade(this.merkleTreeDb, blockNumber);
+  }
+
+  public async start() {
+    if (this.currentState === WorldStateRunningState.STOPPED) {
+      throw new Error('Synchronizer already stopped');
+    }
+    if (this.currentState !== WorldStateRunningState.IDLE) {
+      return this.syncPromise;
+    }
+
+    // get the current latest block number
+    this.latestBlockNumberAtStart = await this.l2BlockSource.getBlockNumber();
+
+    const blockToDownloadFrom = this.currentL2BlockNum + 1;
+
+    // if there are blocks to be retrieved, go to a synching state
+    if (blockToDownloadFrom <= this.latestBlockNumberAtStart) {
+      this.setCurrentState(WorldStateRunningState.SYNCHING);
+      this.syncPromise = new Promise(resolve => {
+        this.syncResolve = resolve;
+      });
+      this.log(`Starting sync from ${blockToDownloadFrom}, latest block ${this.latestBlockNumberAtStart}`);
+    } else {
+      // if no blocks to be retrieved, go straight to running
+      this.setCurrentState(WorldStateRunningState.RUNNING);
+      this.syncPromise = Promise.resolve();
+      this.log(`Next block ${blockToDownloadFrom} already beyond latest block at ${this.latestBlockNumberAtStart}`);
+    }
+
+    // start looking for further blocks
+    const blockProcess = async () => {
+      while (!this.stopping) {
+        await this.jobQueue.put(() => this.collectAndProcessBlocks());
+      }
+    };
+    this.jobQueue.start();
+    this.runningPromise = blockProcess();
+    this.l2BlockDownloader.start(blockToDownloadFrom);
+    this.log(`Started block downloader from block ${blockToDownloadFrom}`);
+    return this.syncPromise;
+  }
+
+  public async stop() {
+    this.log('Stopping world state...');
+    this.stopping = true;
+    await this.l2BlockDownloader.stop();
+    this.log('Cancelling job queue...');
+    await this.jobQueue.cancel();
+    this.log('Stopping Merkle trees');
+    await this.merkleTreeDb.stop();
+    this.log('Awaiting promise');
+    await this.runningPromise;
+    this.setCurrentState(WorldStateRunningState.STOPPED);
+  }
+
+  private get currentL2BlockNum(): number {
+    return this.blockNumber.get() ?? 0;
+  }
+
+  public status(): Promise<WorldStateStatus> {
+    const status = {
+      syncedToL2Block: this.currentL2BlockNum,
+      state: this.currentState,
+    } as WorldStateStatus;
+    return Promise.resolve(status);
+  }
+
+  /**
+   * Forces an immediate sync
+   * @param minBlockNumber - The minimum block number that we must sync to
+   * @returns A promise that resolves with the block number the world state was synced to
+   */
+  public async syncImmediate(minBlockNumber?: number): Promise<number> {
+    if (this.currentState !== WorldStateRunningState.RUNNING) {
+      throw new Error(`World State is not running, unable to perform sync`);
+    }
+    // If we have been given a block number to sync to and we have reached that number
+    // then return.
+    if (minBlockNumber !== undefined && minBlockNumber <= this.currentL2BlockNum) {
+      return this.currentL2BlockNum;
+    }
+    const blockToSyncTo = minBlockNumber === undefined ? 'latest' : `${minBlockNumber}`;
+    this.log(`World State at block ${this.currentL2BlockNum}, told to sync to block ${blockToSyncTo}...`);
+    // ensure any outstanding block updates are completed first.
+    await this.jobQueue.syncPoint();
+    while (true) {
+      // Check the block number again
+      if (minBlockNumber !== undefined && minBlockNumber <= this.currentL2BlockNum) {
+        return this.currentL2BlockNum;
+      }
+      // Poll for more blocks
+      const numBlocks = await this.l2BlockDownloader.pollImmediate();
+      this.log(`Block download immediate poll yielded ${numBlocks} blocks`);
+      if (numBlocks) {
+        // More blocks were received, process them and go round again
+        await this.jobQueue.put(() => this.collectAndProcessBlocks());
+        continue;
+      }
+      // No blocks are available, if we have been given a block number then we can't achieve it
+      if (minBlockNumber !== undefined) {
+        throw new Error(
+          `Unable to sync to block number ${minBlockNumber}, currently synced to block ${this.currentL2BlockNum}`,
+        );
+      }
+      return this.currentL2BlockNum;
+    }
+  }
+
+  /**
+   * Checks for the availability of new blocks and processes them.
+   */
+  private async collectAndProcessBlocks() {
+    // This request for blocks will timeout after 1 second if no blocks are received
+    const blocks = await this.l2BlockDownloader.getBlocks(1);
+    await this.handleL2Blocks(blocks);
+  }
+
+  /**
+   * Handles a list of L2 blocks (i.e. Inserts the new commitments into the merkle tree).
+   * @param l2Blocks - The L2 blocks to handle.
+   * @returns Whether the block handled was produced by this same node.
+   */
+  private async handleL2Blocks(l2Blocks: L2Block[]) {
+    for (const l2Block of l2Blocks) {
+      const [duration, result] = await elapsed(() => this.handleL2Block(l2Block));
+      this.log(`Handled new L2 block`, {
+        eventName: 'l2-block-handled',
+        duration,
+        isBlockOurs: result.isBlockOurs,
+        ...l2Block.getStats(),
+      } satisfies L2BlockHandledStats);
+    }
+  }
+
+  /**
+   * Handles a single L2 block (i.e. Inserts the new commitments into the merkle tree).
+   * @param l2Block - The L2 block to handle.
+   */
+  private async handleL2Block(l2Block: L2Block): Promise<HandleL2BlockResult> {
+    const result = await this.merkleTreeDb.handleL2Block(l2Block);
+    await this.blockNumber.set(l2Block.number);
+
+    if (this.currentState === WorldStateRunningState.SYNCHING && l2Block.number >= this.latestBlockNumberAtStart) {
+      this.setCurrentState(WorldStateRunningState.RUNNING);
+      if (this.syncResolve !== undefined) {
+        this.syncResolve();
+      }
+    }
+    return result;
+  }
+
+  /**
+   * Method to set the value of the current state.
+   * @param newState - New state value.
+   */
+  private setCurrentState(newState: WorldStateRunningState) {
+    this.currentState = newState;
+    this.log(`Moved to state ${WorldStateRunningState[this.currentState]}`);
+  }
+}

package/src/synchronizer/world_state_synchronizer.ts

@@ -0,0 +1,73 @@
+import { MerkleTreeOperations } from '../world-state-db/index.js';
+
+/**
+ * Defines the possible states of the world state synchronizer.
+ */
+export enum WorldStateRunningState {
+  IDLE,
+  SYNCHING,
+  RUNNING,
+  STOPPED,
+}
+
+/**
+ * Defines the status of the world state synchronizer.
+ */
+export interface WorldStateStatus {
+  /**
+   * The current state of the world state synchronizer.
+   */
+  state: WorldStateRunningState;
+  /**
+   * The block number that the world state synchronizer is synced to.
+   */
+  syncedToL2Block: number;
+}
+
+/**
+ * Defines the interface for a world state synchronizer.
+ */
+export interface WorldStateSynchronizer {
+  /**
+   * Starts the synchronizer.
+   * @returns A promise that resolves once the initial sync is completed.
+   */
+  start(): void;
+
+  /**
+   * Returns the current status of the synchronizer.
+   * @returns The current status of the synchronizer.
+   */
+  status(): Promise<WorldStateStatus>;
+
+  /**
+   * Stops the synchronizer.
+   */
+  stop(): Promise<void>;
+
+  /**
+   * Forces an immediate sync to an optionally provided minimum block number
+   * @param minBlockNumber - The minimum block number that we must sync to
+   * @returns A promise that resolves with the block number the world state was synced to
+   */
+  syncImmediate(minBlockNumber?: number): Promise<number>;
+
+  /**
+   * Returns an instance of MerkleTreeOperations that will include uncommitted data.
+   * @returns An instance of MerkleTreeOperations that will include uncommitted data.
+   */
+  getLatest(): MerkleTreeOperations;
+
+  /**
+   * Returns an instance of MerkleTreeOperations that will not include uncommitted data.
+   * @returns An instance of MerkleTreeOperations that will not include uncommitted data.
+   */
+  getCommitted(): MerkleTreeOperations;
+
+  /**
+   * Returns a readonly instance of MerkleTreeOperations where the state is as it was at the given block number
+   * @param block - The block number to look at
+   * @returns An instance of MerkleTreeOperations
+   */
+  getSnapshot(block: number): MerkleTreeOperations;
+}

package/src/world-state-db/index.ts

@@ -0,0 +1,3 @@
+export * from './merkle_trees.js';
+export * from './merkle_tree_db.js';
+export * from './merkle_tree_operations.js';

package/src/world-state-db/merkle_tree_db.ts

@@ -0,0 +1,50 @@
+import { MAX_NEW_NULLIFIERS_PER_TX, MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX } from '@aztec/circuits.js';
+import { IndexedTreeSnapshot, TreeSnapshot } from '@aztec/merkle-tree';
+
+import { MerkleTreeOperations } from './merkle_tree_operations.js';
+
+/**
+ *
+ * @remarks Short explanation:
+ *    The nullifier tree must be initially padded as the pre-populated 0 index prevents efficient subtree insertion.
+ *    Padding with some values solves this issue.
+ *
+ * @remarks Thorough explanation:
+ *    There needs to be an initial (0,0,0) leaf in the tree, so that when we insert the first 'proper' leaf, we can
+ *    prove that any value greater than 0 doesn't exist in the tree yet. We prefill/pad the tree with "the number of
+ *    leaves that are added by one block" so that the first 'proper' block can insert a full subtree.
+ *
+ *    Without this padding, there would be a leaf (0,0,0) at leaf index 0, making it really difficult to insert e.g.
+ *    1024 leaves for the first block, because there's only neat space for 1023 leaves after 0. By padding with 1023
+ *    more leaves, we can then insert the first block of 1024 leaves into indices 1024:2047.
+ */
+export const INITIAL_NULLIFIER_TREE_SIZE = 2 * MAX_NEW_NULLIFIERS_PER_TX;
+
+export const INITIAL_PUBLIC_DATA_TREE_SIZE = 2 * MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX;
+
+/**
+ * Adds a last boolean flag in each function on the type.
+ */
+type WithIncludeUncommitted<F> = F extends (...args: [...infer Rest]) => infer Return
+  ? (...args: [...Rest, boolean]) => Return
+  : F;
+
+/**
+ * Defines the names of the setters on Merkle Trees.
+ */
+type MerkleTreeSetters = 'appendLeaves' | 'updateLeaf' | 'commit' | 'rollback' | 'handleL2Block' | 'batchInsert';
+
+/**
+ * Defines the interface for operations on a set of Merkle Trees configuring whether to return committed or uncommitted data.
+ */
+export type MerkleTreeDb = {
+  [Property in keyof MerkleTreeOperations as Exclude<Property, MerkleTreeSetters>]: WithIncludeUncommitted<
+    MerkleTreeOperations[Property]
+  >;
+} & Pick<MerkleTreeOperations, MerkleTreeSetters> & {
+    /**
+     * Returns a snapshot of the current state of the trees.
+     * @param block - The block number to take the snapshot at.
+     */
+    getSnapshot(block: number): Promise<ReadonlyArray<TreeSnapshot | IndexedTreeSnapshot>>;
+  };

package/src/world-state-db/merkle_tree_operations.ts

@@ -0,0 +1,178 @@
+import { L2Block, MerkleTreeId, SiblingPath } from '@aztec/circuit-types';
+import { Header, NullifierLeafPreimage, StateReference } from '@aztec/circuits.js';
+import { createDebugLogger } from '@aztec/foundation/log';
+import { IndexedTreeLeafPreimage } from '@aztec/foundation/trees';
+import { BatchInsertionResult } from '@aztec/merkle-tree';
+
+/**
+ * Type alias for the nullifier tree ID.
+ */
+export type IndexedTreeId = MerkleTreeId.NULLIFIER_TREE | MerkleTreeId.PUBLIC_DATA_TREE;
+
+/**
+ *  Defines tree information.
+ */
+export interface TreeInfo {
+  /**
+   * The tree ID.
+   */
+  treeId: MerkleTreeId;
+  /**
+   * The tree root.
+   */
+  root: Buffer;
+  /**
+   * The number of leaves in the tree.
+   */
+  size: bigint;
+
+  /**
+   * The depth of the tree.
+   */
+  depth: number;
+}
+
+/**
+ * Defines the interface for operations on a set of Merkle Trees.
+ */
+export interface MerkleTreeOperations {
+  /**
+   * Appends leaves to a given tree.
+   * @param treeId - The tree to be updated.
+   * @param leaves - The set of leaves to be appended.
+   */
+  appendLeaves(treeId: MerkleTreeId, leaves: Buffer[]): Promise<void>;
+
+  /**
+   * Returns information about the given tree.
+   * @param treeId - The tree to be queried.
+   */
+  getTreeInfo(treeId: MerkleTreeId): Promise<TreeInfo>;
+
+  /**
+   * Gets the current state reference.
+   */
+  getStateReference(): Promise<StateReference>;
+
+  /**
+   * Builds the initial header.
+   */
+  buildInitialHeader(): Promise<Header>;
+
+  /**
+   * Gets sibling path for a leaf.
+   * @param treeId - The tree to be queried for a sibling path.
+   * @param index - The index of the leaf for which a sibling path should be returned.
+   */
+  getSiblingPath<N extends number>(treeId: MerkleTreeId, index: bigint): Promise<SiblingPath<N>>;
+
+  /**
+   * Returns the previous index for a given value in an indexed tree.
+   * @param treeId - The tree for which the previous value index is required.
+   * @param value - The value to be queried.
+   */
+  getPreviousValueIndex(
+    treeId: IndexedTreeId,
+    value: bigint,
+  ): Promise<
+    | {
+        /**
+         * The index of the found leaf.
+         */
+        index: bigint;
+        /**
+         * A flag indicating if the corresponding leaf's value is equal to `newValue`.
+         */
+        alreadyPresent: boolean;
+      }
+    | undefined
+  >;
+
+  /**
+   * Returns the data at a specific leaf.
+   * @param treeId - The tree for which leaf data should be returned.
+   * @param index - The index of the leaf required.
+   */
+  getLeafPreimage(treeId: IndexedTreeId, index: bigint): Promise<IndexedTreeLeafPreimage | undefined>;
+
+  /**
+   * Update the leaf data at the given index.
+   * @param treeId - The tree for which leaf data should be edited.
+   * @param leaf - The updated leaf value.
+   * @param index - The index of the leaf to be updated.
+   */
+  updateLeaf(treeId: IndexedTreeId, leaf: NullifierLeafPreimage | Buffer, index: bigint): Promise<void>;
+
+  /**
+   * Returns the index containing a leaf value.
+   * @param treeId - The tree for which the index should be returned.
+   * @param value - The value to search for in the tree.
+   */
+  findLeafIndex(treeId: MerkleTreeId, value: Buffer): Promise<bigint | undefined>;
+
+  /**
+   * Gets the value for a leaf in the tree.
+   * @param treeId - The tree for which the index should be returned.
+   * @param index - The index of the leaf.
+   */
+  getLeafValue(treeId: MerkleTreeId, index: bigint): Promise<Buffer | undefined>;
+
+  /**
+   * Inserts the block hash into the archive.
+   * This includes all of the current roots of all of the data trees and the current blocks global vars.
+   * @param header - The header to insert into the archive.
+   */
+  updateArchive(header: Header): Promise<void>;
+
+  /**
+   * Batch insert multiple leaves into the tree.
+   * @param leaves - Leaves to insert into the tree.
+   * @param treeId - The tree on which to insert.
+   * @param subtreeHeight - Height of the subtree.
+   * @returns The witness data for the leaves to be updated when inserting the new ones.
+   */
+  batchInsert<TreeHeight extends number, SubtreeSiblingPathHeight extends number>(
+    treeId: MerkleTreeId,
+    leaves: Buffer[],
+    subtreeHeight: number,
+  ): Promise<BatchInsertionResult<TreeHeight, SubtreeSiblingPathHeight>>;
+
+  /**
+   * Handles a single L2 block (i.e. Inserts the new commitments into the merkle tree).
+   * @param block - The L2 block to handle.
+   */
+  handleL2Block(block: L2Block): Promise<HandleL2BlockResult>;
+
+  /**
+   * Commits pending changes to the underlying store.
+   */
+  commit(): Promise<void>;
+
+  /**
+   * Rolls back pending changes.
+   */
+  rollback(): Promise<void>;
+}
+
+/** Return type for handleL2Block */
+export type HandleL2BlockResult = {
+  /** Whether the block processed was emitted by our sequencer */ isBlockOurs: boolean;
+};
+
+/**
+ * Outputs a tree leaves using for debugging purposes.
+ */
+export async function inspectTree(
+  db: MerkleTreeOperations,
+  treeId: MerkleTreeId,
+  log = createDebugLogger('aztec:inspect-tree'),
+) {
+  const info = await db.getTreeInfo(treeId);
+  const output = [`Tree id=${treeId} size=${info.size} root=0x${info.root.toString('hex')}`];
+  for (let i = 0; i < info.size; i++) {
+    output.push(
+      ` Leaf ${i}: ${await db.getLeafValue(treeId, BigInt(i)).then(x => x?.toString('hex') ?? '[undefined]')}`,
+    );
+  }
+  log(output.join('\n'));
+}