@aztec/world-state 0.1.0-alpha11

package/src/synchroniser/server_world_state_synchroniser.test.ts

@@ -0,0 +1,281 @@
+import {
+  AppendOnlyTreeSnapshot,
+  CircuitsWasm,
+  GlobalVariables,
+  KERNEL_NEW_COMMITMENTS_LENGTH,
+  KERNEL_NEW_CONTRACTS_LENGTH,
+  KERNEL_NEW_NULLIFIERS_LENGTH,
+  KERNEL_PUBLIC_DATA_UPDATE_REQUESTS_LENGTH,
+  NEW_L2_TO_L1_MSGS_LENGTH,
+  NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP,
+} from '@aztec/circuits.js';
+import { INITIAL_LEAF, Pedersen, SiblingPath } from '@aztec/merkle-tree';
+import { ContractData, L2Block, L2BlockL2Logs, L2BlockSource, MerkleTreeId, PublicDataWrite } from '@aztec/types';
+import { jest } from '@jest/globals';
+import { MerkleTreeDb } from '../index.js';
+import { ServerWorldStateSynchroniser } from './server_world_state_synchroniser.js';
+import { WorldStateRunningState } from './world_state_synchroniser.js';
+import { Fr } from '@aztec/foundation/fields';
+import { sleep } from '@aztec/foundation/sleep';
+import { createLogger } from '@aztec/foundation/log';
+import times from 'lodash.times';
+
+/**
+ * Generic mock implementation.
+ */
+type Mockify<T> = {
+  [P in keyof T]: jest.Mock;
+};
+
+const LATEST_BLOCK_NUMBER = 5;
+const getLatestBlockNumber = () => LATEST_BLOCK_NUMBER;
+let nextBlocks: L2Block[] = [];
+const consumeNextBlocks = () => {
+  const blocks = nextBlocks;
+  nextBlocks = [];
+  return Promise.resolve(blocks);
+};
+
+const getMockTreeSnapshot = () => {
+  return new AppendOnlyTreeSnapshot(Fr.random(), 16);
+};
+
+const getMockContractData = () => {
+  return ContractData.random();
+};
+
+const getMockGlobalVariables = () => {
+  return GlobalVariables.from({
+    chainId: Fr.random(),
+    version: Fr.random(),
+    blockNumber: Fr.random(),
+    timestamp: Fr.random(),
+  });
+};
+
+const getMockL1ToL2MessagesData = () => {
+  return new Array(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP).map(() => Fr.random());
+};
+
+const getMockBlock = (blockNumber: number, newContractsCommitments?: Buffer[]) => {
+  const newEncryptedLogs = L2BlockL2Logs.random(1, 2, 3);
+  const block = L2Block.fromFields({
+    number: blockNumber,
+    globalVariables: getMockGlobalVariables(),
+    startPrivateDataTreeSnapshot: getMockTreeSnapshot(),
+    startNullifierTreeSnapshot: getMockTreeSnapshot(),
+    startContractTreeSnapshot: getMockTreeSnapshot(),
+    startTreeOfHistoricPrivateDataTreeRootsSnapshot: getMockTreeSnapshot(),
+    startTreeOfHistoricContractTreeRootsSnapshot: getMockTreeSnapshot(),
+    startPublicDataTreeRoot: Fr.random(),
+    startL1ToL2MessageTreeSnapshot: getMockTreeSnapshot(),
+    startTreeOfHistoricL1ToL2MessageTreeRootsSnapshot: getMockTreeSnapshot(),
+    endPrivateDataTreeSnapshot: getMockTreeSnapshot(),
+    endNullifierTreeSnapshot: getMockTreeSnapshot(),
+    endContractTreeSnapshot: getMockTreeSnapshot(),
+    endTreeOfHistoricPrivateDataTreeRootsSnapshot: getMockTreeSnapshot(),
+    endTreeOfHistoricContractTreeRootsSnapshot: getMockTreeSnapshot(),
+    endPublicDataTreeRoot: Fr.random(),
+    endL1ToL2MessageTreeSnapshot: getMockTreeSnapshot(),
+    endTreeOfHistoricL1ToL2MessageTreeRootsSnapshot: getMockTreeSnapshot(),
+    newCommitments: times(KERNEL_NEW_COMMITMENTS_LENGTH, Fr.random),
+    newNullifiers: times(KERNEL_NEW_NULLIFIERS_LENGTH, Fr.random),
+    newContracts: newContractsCommitments?.map(x => Fr.fromBuffer(x)) ?? [Fr.random()],
+    newContractData: times(KERNEL_NEW_CONTRACTS_LENGTH, getMockContractData),
+    newPublicDataWrites: times(KERNEL_PUBLIC_DATA_UPDATE_REQUESTS_LENGTH, PublicDataWrite.random),
+    newL1ToL2Messages: getMockL1ToL2MessagesData(),
+    newL2ToL1Msgs: times(NEW_L2_TO_L1_MSGS_LENGTH, Fr.random),
+    newEncryptedLogs,
+  });
+  return block;
+};
+
+const createSynchroniser = (merkleTreeDb: any, rollupSource: any) =>
+  new ServerWorldStateSynchroniser(merkleTreeDb as MerkleTreeDb, rollupSource as L2BlockSource);
+
+const log = createLogger('aztec:server_world_state_synchroniser_test');
+
+describe('server_world_state_synchroniser', () => {
+  const rollupSource: Mockify<Pick<L2BlockSource, 'getBlockHeight' | 'getL2Blocks'>> = {
+    getBlockHeight: jest.fn().mockImplementation(getLatestBlockNumber),
+    getL2Blocks: jest.fn().mockImplementation(consumeNextBlocks),
+  };
+
+  const merkleTreeDb: Mockify<MerkleTreeDb> = {
+    getTreeInfo: jest
+      .fn()
+      .mockImplementation(() =>
+        Promise.resolve({ treeId: MerkleTreeId.CONTRACT_TREE, root: Buffer.alloc(32, 0), size: 0n }),
+      ),
+    appendLeaves: jest.fn().mockImplementation(() => Promise.resolve()),
+    updateLeaf: jest.fn().mockImplementation(() => Promise.resolve()),
+    getSiblingPath: jest.fn().mockImplementation(() => {
+      return async () => {
+        const wasm = await CircuitsWasm.get();
+        const pedersen: Pedersen = new Pedersen(wasm);
+        SiblingPath.ZERO(32, INITIAL_LEAF, pedersen);
+      }; //Promise.resolve();
+    }),
+    updateHistoricRootsTrees: jest.fn().mockImplementation(() => Promise.resolve()),
+    commit: jest.fn().mockImplementation(() => Promise.resolve()),
+    rollback: jest.fn().mockImplementation(() => Promise.resolve()),
+    handleL2Block: jest.fn().mockImplementation(() => Promise.resolve()),
+  } as any;
+
+  it('can be constructed', () => {
+    expect(() => createSynchroniser(merkleTreeDb, rollupSource)).not.toThrow();
+  });
+
+  it('updates sync progress', async () => {
+    const server = createSynchroniser(merkleTreeDb, rollupSource);
+
+    // test initial state
+    let status = await server.status();
+    expect(status.syncedToL2Block).toEqual(0);
+    expect(status.state).toEqual(WorldStateRunningState.IDLE);
+
+    // create an initial block
+    let currentBlockNumber = 0;
+    nextBlocks = [getMockBlock(currentBlockNumber + 1)];
+
+    // start the sync process but don't await
+    server.start().catch(err => log('Sync not completed: ', err));
+
+    // now setup a loop to monitor the sync progress and push new blocks in
+    while (currentBlockNumber <= LATEST_BLOCK_NUMBER) {
+      status = await server.status();
+      expect(
+        status.syncedToL2Block >= currentBlockNumber || status.syncedToL2Block <= currentBlockNumber + 1,
+      ).toBeTruthy();
+      if (status.syncedToL2Block === LATEST_BLOCK_NUMBER) {
+        break;
+      }
+      expect(
+        status.state >= WorldStateRunningState.IDLE || status.state <= WorldStateRunningState.SYNCHING,
+      ).toBeTruthy();
+      if (status.syncedToL2Block === currentBlockNumber) {
+        await sleep(100);
+        continue;
+      }
+      currentBlockNumber++;
+      nextBlocks = [getMockBlock(currentBlockNumber + 1)];
+    }
+
+    // check the status agian, should be fully synced
+    status = await server.status();
+    expect(status.state).toEqual(WorldStateRunningState.RUNNING);
+    expect(status.syncedToL2Block).toEqual(LATEST_BLOCK_NUMBER);
+
+    // stop the synchroniser
+    await server.stop();
+
+    // check the final status
+    status = await server.status();
+    expect(status.state).toEqual(WorldStateRunningState.STOPPED);
+    expect(status.syncedToL2Block).toEqual(LATEST_BLOCK_NUMBER);
+  });
+
+  it('enables blocking until synced', async () => {
+    const server = createSynchroniser(merkleTreeDb, rollupSource);
+    let currentBlockNumber = 0;
+
+    const newBlocks = async () => {
+      while (currentBlockNumber <= LATEST_BLOCK_NUMBER) {
+        await sleep(100);
+        nextBlocks = [...nextBlocks, getMockBlock(++currentBlockNumber)];
+      }
+    };
+
+    // kick off the background queueing of blocks
+    const newBlockPromise = newBlocks();
+
+    // kick off the synching
+    const syncPromise = server.start();
+
+    // await the synching
+    await syncPromise;
+
+    await newBlockPromise;
+
+    let status = await server.status();
+    expect(status.state).toEqual(WorldStateRunningState.RUNNING);
+    expect(status.syncedToL2Block).toEqual(LATEST_BLOCK_NUMBER);
+    await server.stop();
+    status = await server.status();
+    expect(status.state).toEqual(WorldStateRunningState.STOPPED);
+    expect(status.syncedToL2Block).toEqual(LATEST_BLOCK_NUMBER);
+  });
+
+  it('handles multiple calls to start', async () => {
+    const server = createSynchroniser(merkleTreeDb, rollupSource);
+    let currentBlockNumber = 0;
+
+    const newBlocks = async () => {
+      while (currentBlockNumber < LATEST_BLOCK_NUMBER) {
+        await sleep(100);
+        const newBlock = getMockBlock(++currentBlockNumber);
+        nextBlocks = [...nextBlocks, newBlock];
+      }
+    };
+
+    // kick off the background queueing of blocks
+    const newBlockPromise = newBlocks();
+
+    // kick off the synching
+    await server.start();
+
+    // call start again, should get back the same promise
+    await server.start();
+
+    // wait until the block production has finished
+    await newBlockPromise;
+
+    await server.stop();
+  });
+
+  it('immediately syncs if no new blocks', async () => {
+    const server = createSynchroniser(merkleTreeDb, rollupSource);
+    rollupSource.getBlockHeight.mockImplementationOnce(() => {
+      return Promise.resolve(0);
+    });
+
+    // kick off the synching
+    const syncPromise = server.start();
+
+    // it should already be synced, no need to push new blocks
+    await syncPromise;
+
+    const status = await server.status();
+    expect(status.state).toBe(WorldStateRunningState.RUNNING);
+    expect(status.syncedToL2Block).toBe(0);
+    await server.stop();
+  });
+
+  it("can't be started if already stopped", async () => {
+    const server = createSynchroniser(merkleTreeDb, rollupSource);
+    rollupSource.getBlockHeight.mockImplementationOnce(() => {
+      return Promise.resolve(0);
+    });
+
+    // kick off the synching
+    const syncPromise = server.start();
+    await syncPromise;
+    await server.stop();
+
+    await expect(server.start()).rejects.toThrow();
+  });
+
+  it('adds the received L2 blocks', async () => {
+    merkleTreeDb.handleL2Block.mockReset();
+    const server = createSynchroniser(merkleTreeDb, rollupSource);
+    const totalBlocks = LATEST_BLOCK_NUMBER + 1;
+    nextBlocks = Array(totalBlocks)
+      .fill(0)
+      .map((_, index) => getMockBlock(index, [Buffer.alloc(32, index)]));
+    // sync the server
+    await server.start();
+
+    expect(merkleTreeDb.handleL2Block).toHaveBeenCalledTimes(totalBlocks);
+    await server.stop();
+  });
+});

package/src/synchroniser/server_world_state_synchroniser.ts

@@ -0,0 +1,151 @@
+import { L2Block, L2BlockDownloader, L2BlockSource } from '@aztec/types';
+import { MerkleTreeDb, MerkleTreeOperations } from '../index.js';
+import { MerkleTreeOperationsFacade } from '../merkle-tree/merkle_tree_operations_facade.js';
+import { WorldStateRunningState, WorldStateStatus, WorldStateSynchroniser } from './world_state_synchroniser.js';
+import { getConfigEnvVars } from './config.js';
+import { createDebugLogger } from '@aztec/foundation/log';
+
+/**
+ * Synchronises the world state with the L2 blocks from a L2BlockSource.
+ * The synchroniser will download the L2 blocks from the L2BlockSource and insert the new commitments into the merkle
+ * tree.
+ */
+export class ServerWorldStateSynchroniser implements WorldStateSynchroniser {
+  private currentL2BlockNum = 0;
+  private latestBlockNumberAtStart = 0;
+  private l2BlockDownloader: L2BlockDownloader;
+  private syncPromise: Promise<void> = Promise.resolve();
+  private syncResolve?: () => void = undefined;
+  private stopping = false;
+  private runningPromise: Promise<void> = Promise.resolve();
+  private currentState: WorldStateRunningState = WorldStateRunningState.IDLE;
+
+  constructor(
+    private merkleTreeDb: MerkleTreeDb,
+    private l2BlockSource: L2BlockSource,
+    private log = createDebugLogger('aztec:world_state'),
+  ) {
+    const config = getConfigEnvVars();
+    this.l2BlockDownloader = new L2BlockDownloader(l2BlockSource, config.l2QueueSize, config.checkInterval);
+  }
+
+  /**
+   * Returns an instance of MerkleTreeOperations that will include uncommitted data.
+   * @returns An instance of MerkleTreeOperations that will include uncommitted data.
+   */
+  public getLatest(): MerkleTreeOperations {
+    return new MerkleTreeOperationsFacade(this.merkleTreeDb, true);
+  }
+
+  /**
+   * Returns an instance of MerkleTreeOperations that will not include uncommitted data.
+   * @returns An instance of MerkleTreeOperations that will not include uncommitted data.
+   */
+  public getCommitted(): MerkleTreeOperations {
+    return new MerkleTreeOperationsFacade(this.merkleTreeDb, false);
+  }
+
+  /**
+   * Starts the synchroniser.
+   * @returns A promise that resolves once the initial sync is completed.
+   */
+  public async start() {
+    if (this.currentState === WorldStateRunningState.STOPPED) {
+      throw new Error('Synchroniser already stopped');
+    }
+    if (this.currentState !== WorldStateRunningState.IDLE) {
+      return this.syncPromise;
+    }
+
+    // get the current latest block number
+    this.latestBlockNumberAtStart = await this.l2BlockSource.getBlockHeight();
+
+    const blockToDownloadFrom = this.currentL2BlockNum + 1;
+
+    // if there are blocks to be retrieved, go to a synching state
+    if (blockToDownloadFrom <= this.latestBlockNumberAtStart) {
+      this.setCurrentState(WorldStateRunningState.SYNCHING);
+      this.syncPromise = new Promise(resolve => {
+        this.syncResolve = resolve;
+      });
+      this.log(`Starting sync from ${blockToDownloadFrom}, latest block ${this.latestBlockNumberAtStart}`);
+    } else {
+      // if no blocks to be retrieved, go straight to running
+      this.setCurrentState(WorldStateRunningState.RUNNING);
+      this.syncPromise = Promise.resolve();
+      this.log(`Next block ${blockToDownloadFrom} already beyond latest block at ${this.latestBlockNumberAtStart}`);
+    }
+
+    // start looking for further blocks
+    const blockProcess = async () => {
+      while (!this.stopping) {
+        const blocks = await this.l2BlockDownloader.getL2Blocks();
+        await this.handleL2Blocks(blocks);
+      }
+    };
+    this.runningPromise = blockProcess();
+    this.l2BlockDownloader.start(blockToDownloadFrom);
+    this.log(`Started block downloader from block ${blockToDownloadFrom}`);
+    return this.syncPromise;
+  }
+
+  /**
+   * Stops the synchroniser.
+   */
+  public async stop() {
+    this.log('Stopping world state...');
+    this.stopping = true;
+    await this.l2BlockDownloader.stop();
+    await this.runningPromise;
+    this.setCurrentState(WorldStateRunningState.STOPPED);
+  }
+
+  /**
+   * Returns the current status of the synchroniser.
+   * @returns The current status of the synchroniser.
+   */
+  public status(): Promise<WorldStateStatus> {
+    const status = {
+      syncedToL2Block: this.currentL2BlockNum,
+      state: this.currentState,
+    } as WorldStateStatus;
+    return Promise.resolve(status);
+  }
+
+  /**
+   * Handles a list of L2 blocks (i.e. Inserts the new commitments into the merkle tree).
+   * @param l2Blocks - The L2 blocks to handle.
+   */
+  private async handleL2Blocks(l2Blocks: L2Block[]) {
+    for (const l2Block of l2Blocks) {
+      await this.handleL2Block(l2Block);
+    }
+  }
+
+  /**
+   * Handles a single L2 block (i.e. Inserts the new commitments into the merkle tree).
+   * @param l2Block - The L2 block to handle.
+   */
+  private async handleL2Block(l2Block: L2Block) {
+    await this.merkleTreeDb.handleL2Block(l2Block);
+    this.currentL2BlockNum = l2Block.number;
+    if (
+      this.currentState === WorldStateRunningState.SYNCHING &&
+      this.currentL2BlockNum >= this.latestBlockNumberAtStart
+    ) {
+      this.setCurrentState(WorldStateRunningState.RUNNING);
+      if (this.syncResolve !== undefined) {
+        this.syncResolve();
+      }
+    }
+  }
+
+  /**
+   * Method to set the value of the current state.
+   * @param newState - New state value.
+   */
+  private setCurrentState(newState: WorldStateRunningState) {
+    this.currentState = newState;
+    this.log(`Moved to state ${WorldStateRunningState[this.currentState]}`);
+  }
+}

package/src/synchroniser/world_state_synchroniser.ts

@@ -0,0 +1,36 @@
+import { MerkleTreeOperations } from '../index.js';
+
+/**
+ * Defines the possible states of the world state synchroniser.
+ */
+export enum WorldStateRunningState {
+  IDLE,
+  SYNCHING,
+  RUNNING,
+  STOPPED,
+}
+
+/**
+ * Defines the status of the world state synchroniser.
+ */
+export interface WorldStateStatus {
+  /**
+   * The current state of the world state synchroniser.
+   */
+  state: WorldStateRunningState;
+  /**
+   * The block number that the world state synchroniser is synced to.
+   */
+  syncedToL2Block: number;
+}
+
+/**
+ * Defines the interface for a world state synchroniser.
+ */
+export interface WorldStateSynchroniser {
+  start(): void;
+  status(): Promise<WorldStateStatus>;
+  stop(): Promise<void>;
+  getLatest(): MerkleTreeOperations;
+  getCommitted(): MerkleTreeOperations;
+}

package/src/utils.ts

@@ -0,0 +1,24 @@
+import { GeneratorIndex } from '@aztec/circuits.js';
+import { pedersenCompressWithHashIndex } from '@aztec/circuits.js/barretenberg';
+import { AztecAddress } from '@aztec/foundation/aztec-address';
+import { Fr } from '@aztec/foundation/fields';
+import { toBigInt } from '@aztec/foundation/serialize';
+
+import { IWasmModule } from '@aztec/foundation/wasm';
+
+/**
+ * Computes the index in the public data tree for a given contract and storage slot.
+ * @param contract - Address of the contract who owns the storage.
+ * @param slot - Slot within the contract storage.
+ * @param bbWasm - Wasm module for computing the hash.
+ * @returns The leaf index of the public data tree that maps to this storage slot.
+ */
+export function computePublicDataTreeLeafIndex(contract: AztecAddress, slot: Fr, wasm: IWasmModule): bigint {
+  return toBigInt(
+    pedersenCompressWithHashIndex(
+      wasm,
+      [contract, slot].map(f => f.toBuffer()),
+      GeneratorIndex.PUBLIC_LEAF_INDEX,
+    ),
+  );
+}