@aztec/world-state 0.0.1-commit.fffb133c → 0.0.1-dev

package/src/native/native_world_state.ts

@@ -4,9 +4,9 @@ import { fromEntries, padArrayEnd } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { tryRmDir } from '@aztec/foundation/fs';
-import { type Logger, createLogger } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import type { L2Block } from '@aztec/stdlib/block';
-import { DatabaseVersionManager } from '@aztec/stdlib/database-version';
+import { DatabaseVersionManager } from '@aztec/stdlib/database-version/manager';
 import type {
   IndexedTreeId,
   MerkleTreeReadOperations,
@@ -52,7 +52,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
   protected constructor(
     protected instance: NativeWorldState,
     protected readonly worldStateInstrumentation: WorldStateInstrumentation,
-    protected readonly log: Logger = createLogger('world-state:database'),
+    protected readonly log: Logger,
     private readonly cleanup = () => Promise.resolve(),
   ) {}
 
@@ -62,9 +62,10 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
     wsTreeMapSizes: WorldStateTreeMapSizes,
     prefilledPublicData: PublicDataTreeLeaf[] = [],
     instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
-    log = createLogger('world-state:database'),
+    bindings?: LoggerBindings,
     cleanup = () => Promise.resolve(),
   ): Promise<NativeWorldStateService> {
+    const log = createLogger('world-state:database', bindings);
     const worldStateDirectory = join(dataDir, WORLD_STATE_DIR);
     // Create a version manager to handle versioning
     const versionManager = new DatabaseVersionManager({
@@ -72,7 +73,9 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
       rollupAddress,
       dataDirectory: worldStateDirectory,
       onOpen: (dir: string) => {
-        return Promise.resolve(new NativeWorldState(dir, wsTreeMapSizes, prefilledPublicData, instrumentation));
+        return Promise.resolve(
+          new NativeWorldState(dir, wsTreeMapSizes, prefilledPublicData, instrumentation, bindings),
+        );
       },
     });
 
@@ -93,8 +96,9 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
     cleanupTmpDir = true,
     prefilledPublicData: PublicDataTreeLeaf[] = [],
     instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
+    bindings?: LoggerBindings,
   ): Promise<NativeWorldStateService> {
-    const log = createLogger('world-state:database');
+    const log = createLogger('world-state:database', bindings);
     const dataDir = await mkdtemp(join(tmpdir(), 'aztec-world-state-'));
     const dbMapSizeKb = 10 * 1024 * 1024;
     const worldStateTreeMapSizes: WorldStateTreeMapSizes = {
@@ -116,7 +120,15 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
       }
     };
 
-    return this.new(rollupAddress, dataDir, worldStateTreeMapSizes, prefilledPublicData, instrumentation, log, cleanup);
+    return this.new(
+      rollupAddress,
+      dataDir,
+      worldStateTreeMapSizes,
+      prefilledPublicData,
+      instrumentation,
+      bindings,
+      cleanup,
+    );
   }
 
   protected async init() {
@@ -135,9 +147,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
 
     // the initial header _must_ be the first element in the archive tree
     // if this assertion fails, check that the hashing done in Header in yarn-project matches the initial header hash done in world_state.cpp
-    const indices = await committed.findLeafIndices(MerkleTreeId.ARCHIVE, [
-      (await this.initialHeader.hash()).toField(),
-    ]);
+    const indices = await committed.findLeafIndices(MerkleTreeId.ARCHIVE, [(await this.initialHeader.hash()).toFr()]);
     const initialHeaderIndex = indices[0];
     assert.strictEqual(initialHeaderIndex, 0n, 'Invalid initial archive state');
   }
@@ -222,7 +232,10 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
         WorldStateMessageType.SYNC_BLOCK,
         {
           blockNumber: l2Block.number,
-          blockHeaderHash: await l2Block.hash(),
+          blockHeaderHash: (await l2Block.hash()).toBuffer(),
+          // Forwarded so the native sync verifies the archive root against canonical and rejects a divergent tree.
+          expectedArchiveRoot: l2Block.archive.root.toBuffer(),
+          expectedPreviousArchiveRoot: l2Block.header.lastArchive.root.toBuffer(),
           paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf),
           paddedNoteHashes: paddedNoteHashes.map(serializeLeaf),
           paddedNullifiers: paddedNullifiers.map(serializeLeaf),

package/src/native/native_world_state_instance.ts

@@ -1,6 +1,6 @@
 import {
   ARCHIVE_HEIGHT,
-  GeneratorIndex,
+  DomainSeparator,
   L1_TO_L2_MSG_TREE_HEIGHT,
   MAX_NULLIFIERS_PER_TX,
   MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
@@ -8,7 +8,7 @@ import {
   NULLIFIER_TREE_HEIGHT,
   PUBLIC_DATA_TREE_HEIGHT,
 } from '@aztec/constants';
-import { type Logger, createLogger } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import { NativeWorldState as BaseNativeWorldState, MsgpackChannel } from '@aztec/native';
 import { MerkleTreeId } from '@aztec/stdlib/trees';
 import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees';
@@ -57,7 +57,8 @@ export class NativeWorldState implements NativeWorldStateInstance {
     private readonly wsTreeMapSizes: WorldStateTreeMapSizes,
     private readonly prefilledPublicData: PublicDataTreeLeaf[] = [],
     private readonly instrumentation: WorldStateInstrumentation,
-    private readonly log: Logger = createLogger('world-state:database'),
+    bindings?: LoggerBindings,
+    private readonly log: Logger = createLogger('world-state:database', bindings),
   ) {
     const threads = Math.min(cpus().length, MAX_WORLD_STATE_THREADS);
     log.info(
@@ -80,7 +81,7 @@ export class NativeWorldState implements NativeWorldStateInstance {
         [MerkleTreeId.PUBLIC_DATA_TREE]: 2 * MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
       },
       prefilledPublicDataBufferArray,
-      GeneratorIndex.BLOCK_HASH,
+      DomainSeparator.BLOCK_HEADER_HASH,
       {
         [MerkleTreeId.NULLIFIER_TREE]: wsTreeMapSizes.nullifierTreeMapSizeKb,
         [MerkleTreeId.NOTE_HASH_TREE]: wsTreeMapSizes.noteHashTreeMapSizeKb,
@@ -105,6 +106,7 @@ export class NativeWorldState implements NativeWorldStateInstance {
       this.wsTreeMapSizes,
       this.prefilledPublicData,
       this.instrumentation,
+      this.log.getBindings(),
       this.log,
     );
   }

package/src/synchronizer/config.ts

@@ -29,8 +29,8 @@ export interface WorldStateConfig {
   /** Optional directory for the world state DB, if unspecified will default to the general data directory */
   worldStateDataDirectory?: string;
 
-  /** The number of historic blocks to maintain */
-  worldStateBlockHistory: number;
+  /** The number of historic checkpoints worth of blocks to maintain */
+  worldStateCheckpointHistory: number;
 }
 
 export const worldStateConfigMappings: ConfigMappingsType<WorldStateConfig> = {
@@ -84,9 +84,11 @@ export const worldStateConfigMappings: ConfigMappingsType<WorldStateConfig> = {
     env: 'WS_DATA_DIRECTORY',
     description: 'Optional directory for the world state database',
   },
-  worldStateBlockHistory: {
-    env: 'WS_NUM_HISTORIC_BLOCKS',
-    description: 'The number of historic blocks to maintain. Values less than 1 mean all history is maintained',
+  worldStateCheckpointHistory: {
+    env: 'WS_NUM_HISTORIC_CHECKPOINTS',
+    description:
+      'The number of historic checkpoints worth of blocks to maintain. Values less than 1 mean all history is maintained',
+    fallback: ['WS_NUM_HISTORIC_BLOCKS'],
     ...numberConfigHelper(64),
   },
 };

package/src/synchronizer/factory.ts

@@ -1,3 +1,4 @@
+import type { LoggerBindings } from '@aztec/foundation/log';
 import type { DataStoreConfig } from '@aztec/kv-store/config';
 import type { L2BlockSource } from '@aztec/stdlib/block';
 import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
@@ -22,9 +23,10 @@ export async function createWorldStateSynchronizer(
   l2BlockSource: L2BlockSource & L1ToL2MessageSource,
   prefilledPublicData: PublicDataTreeLeaf[] = [],
   client: TelemetryClient = getTelemetryClient(),
+  bindings?: LoggerBindings,
 ) {
   const instrumentation = new WorldStateInstrumentation(client);
-  const merkleTrees = await createWorldState(config, prefilledPublicData, instrumentation);
+  const merkleTrees = await createWorldState(config, prefilledPublicData, instrumentation, bindings);
   return new ServerWorldStateSynchronizer(merkleTrees, l2BlockSource, config, instrumentation);
 }
 
@@ -42,6 +44,7 @@ export async function createWorldState(
     Pick<DataStoreConfig, 'dataDirectory' | 'dataStoreMapSizeKb' | 'l1Contracts'>,
   prefilledPublicData: PublicDataTreeLeaf[] = [],
   instrumentation: WorldStateInstrumentation = new WorldStateInstrumentation(getTelemetryClient()),
+  bindings?: LoggerBindings,
 ) {
   const dataDirectory = config.worldStateDataDirectory ?? config.dataDirectory;
   const dataStoreMapSizeKb = config.worldStateDbMapSizeKb ?? config.dataStoreMapSizeKb;
@@ -65,11 +68,14 @@ export async function createWorldState(
         wsTreeMapSizes,
         prefilledPublicData,
         instrumentation,
+        bindings,
       )
     : await NativeWorldStateService.tmp(
         config.l1Contracts.rollupAddress,
         !['true', '1'].includes(process.env.DEBUG_WORLD_STATE!),
         prefilledPublicData,
+        instrumentation,
+        bindings,
       );
 
   return merkleTrees;

package/src/synchronizer/server_world_state_synchronizer.ts

@@ -1,10 +1,11 @@
-import { GENESIS_BLOCK_HEADER_HASH, INITIAL_L2_BLOCK_NUM, INITIAL_L2_CHECKPOINT_NUM } from '@aztec/constants';
-import { BlockNumber } from '@aztec/foundation/branded-types';
+import { GENESIS_BLOCK_HEADER_HASH, INITIAL_CHECKPOINT_NUMBER, INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
+import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
 import type { Fr } from '@aztec/foundation/curves/bn254';
 import { type Logger, createLogger } from '@aztec/foundation/log';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { elapsed } from '@aztec/foundation/timer';
 import {
+  type BlockHash,
   GENESIS_CHECKPOINT_HEADER_HASH,
   type L2Block,
   type L2BlockId,
@@ -64,7 +65,7 @@ export class ServerWorldStateSynchronizer
     private readonly log: Logger = createLogger('world_state'),
   ) {
     this.merkleTreeCommitted = this.merkleTreeDb.getCommitted();
-    this.historyToKeep = config.worldStateBlockHistory < 1 ? undefined : config.worldStateBlockHistory;
+    this.historyToKeep = config.worldStateCheckpointHistory < 1 ? undefined : config.worldStateCheckpointHistory;
     this.log.info(
       `Created world state synchroniser with block history of ${
         this.historyToKeep === undefined ? 'infinity' : this.historyToKeep
@@ -177,13 +178,10 @@ export class ServerWorldStateSynchronizer
   /**
    * Forces an immediate sync.
    * @param targetBlockNumber - The target block number that we must sync to. Will download unproven blocks if needed to reach it.
-   * @param skipThrowIfTargetNotReached - Whether to skip throwing if the target block number is not reached.
+   * @param blockHash - If provided, verifies the block at targetBlockNumber matches this hash. On mismatch, triggers a resync (reorg detection).
    * @returns A promise that resolves with the block number the world state was synced to
    */
-  public async syncImmediate(
-    targetBlockNumber?: BlockNumber,
-    skipThrowIfTargetNotReached?: boolean,
-  ): Promise<BlockNumber> {
+  public async syncImmediate(targetBlockNumber?: BlockNumber, blockHash?: BlockHash): Promise<BlockNumber> {
     if (this.currentState !== WorldStateRunningState.RUNNING) {
       throw new Error(`World State is not running. Unable to perform sync.`);
     }
@@ -195,7 +193,19 @@ export class ServerWorldStateSynchronizer
     // If we have been given a block number to sync to and we have reached that number then return
     const currentBlockNumber = await this.getLatestBlockNumber();
     if (targetBlockNumber !== undefined && targetBlockNumber <= currentBlockNumber) {
-      return currentBlockNumber;
+      if (blockHash === undefined) {
+        return currentBlockNumber;
+      }
+
+      // If a block hash was provided, verify we're on the expected fork
+      const currentHash = await this.getL2BlockHash(targetBlockNumber);
+      if (currentHash === blockHash.toString()) {
+        return currentBlockNumber;
+      }
+      // Hash mismatch: a reorg may have occurred, fall through to trigger sync
+      this.log.debug(
+        `World state block hash mismatch at ${targetBlockNumber} (expected ${blockHash}, got ${currentHash}). Triggering resync.`,
+      );
     }
     this.log.debug(`World State at ${currentBlockNumber} told to sync to ${targetBlockNumber ?? 'latest'}`);
 
@@ -213,7 +223,7 @@ export class ServerWorldStateSynchronizer
 
     // If we have been given a block number to sync to and we have not reached that number then fail
     const updatedBlockNumber = await this.getLatestBlockNumber();
-    if (!skipThrowIfTargetNotReached && targetBlockNumber !== undefined && targetBlockNumber > updatedBlockNumber) {
+    if (targetBlockNumber !== undefined && targetBlockNumber > updatedBlockNumber) {
       throw new WorldStateSynchronizerError(
         `Unable to sync to block number ${targetBlockNumber} (last synced is ${updatedBlockNumber})`,
         {
@@ -227,6 +237,24 @@ export class ServerWorldStateSynchronizer
       );
     }
 
+    // If a block hash was provided, verify we're on the expected fork after syncing, throw otherwise
+    if (blockHash !== undefined && targetBlockNumber !== undefined) {
+      const updatedHash = await this.getL2BlockHash(targetBlockNumber);
+      if (updatedHash !== blockHash.toString()) {
+        throw new WorldStateSynchronizerError(
+          `Block hash mismatch at block ${targetBlockNumber} (expected ${blockHash} but got ${updatedHash})`,
+          {
+            cause: {
+              reason: 'block_hash_mismatch',
+              targetBlockNumber,
+              expectedHash: blockHash.toString(),
+              actualHash: updatedHash,
+            },
+          },
+        );
+      }
+    }
+
     return updatedBlockNumber;
   }
 
@@ -263,15 +291,15 @@ export class ServerWorldStateSynchronizer
       proposed: latestBlockId,
       checkpointed: {
         block: { number: INITIAL_L2_BLOCK_NUM, hash: GENESIS_BLOCK_HEADER_HASH.toString() },
-        checkpoint: { number: INITIAL_L2_CHECKPOINT_NUM, hash: genesisCheckpointHeaderHash },
+        checkpoint: { number: INITIAL_CHECKPOINT_NUMBER, hash: genesisCheckpointHeaderHash },
       },
       finalized: {
         block: { number: status.finalizedBlockNumber, hash: finalizedBlockHash ?? '' },
-        checkpoint: { number: INITIAL_L2_CHECKPOINT_NUM, hash: genesisCheckpointHeaderHash },
+        checkpoint: { number: INITIAL_CHECKPOINT_NUMBER, hash: genesisCheckpointHeaderHash },
       },
       proven: {
         block: { number: provenBlockNumber, hash: provenBlockHash ?? '' },
-        checkpoint: { number: INITIAL_L2_CHECKPOINT_NUM, hash: genesisCheckpointHeaderHash },
+        checkpoint: { number: INITIAL_CHECKPOINT_NUMBER, hash: genesisCheckpointHeaderHash },
       },
     };
   }
@@ -300,7 +328,7 @@ export class ServerWorldStateSynchronizer
    * @returns Whether the block handled was produced by this same node.
    */
   private async handleL2Blocks(l2Blocks: L2Block[]) {
-    this.log.trace(`Handling L2 blocks ${l2Blocks[0].number} to ${l2Blocks.at(-1)!.number}`);
+    this.log.debug(`Handling L2 blocks ${l2Blocks[0].number} to ${l2Blocks.at(-1)!.number}`);
 
     // Fetch the L1->L2 messages for the first block in a checkpoint.
     const messagesForBlocks = new Map<BlockNumber, Fr[]>();
@@ -341,10 +369,12 @@ export class ServerWorldStateSynchronizer
    * @returns Whether the block handled was produced by this same node.
    */
   private async handleL2Block(l2Block: L2Block, l1ToL2Messages: Fr[]): Promise<WorldStateStatusFull> {
-    this.log.trace(`Pushing L2 block ${l2Block.number} to merkle tree db `, {
+    this.log.debug(`Pushing L2 block ${l2Block.number} to merkle tree db `, {
       blockNumber: l2Block.number,
       blockHash: await l2Block.hash().then(h => h.toString()),
       l1ToL2Messages: l1ToL2Messages.map(msg => msg.toString()),
+      blockHeader: l2Block.header.toInspect(),
+      blockStats: l2Block.getStats(),
     });
     const result = await this.merkleTreeDb.handleL2BlockAndMessages(l2Block, l1ToL2Messages);
 
@@ -357,17 +387,65 @@ export class ServerWorldStateSynchronizer
   }
 
   private async handleChainFinalized(blockNumber: BlockNumber) {
-    this.log.verbose(`Finalized chain is now at block ${blockNumber}`);
+    this.log.verbose(`Updating world state finalized chain to block ${blockNumber}`);
+    // If the finalized block number is older than the oldest available block in world state,
+    // skip entirely. The finalized block number can jump backwards (e.g. when the finalization
+    // heuristic changes) and try to read block data that has already been pruned. When this
+    // happens, there is nothing useful to do — the native world state is already finalized
+    // past this point and pruning has already happened.
+    const currentSummary = await this.merkleTreeDb.getStatusSummary();
+    if (blockNumber < currentSummary.oldestHistoricalBlock || blockNumber < 1) {
+      this.log.trace(
+        `Finalized block ${blockNumber} is older than the oldest available block ${currentSummary.oldestHistoricalBlock}. Skipping.`,
+      );
+      return;
+    }
     const summary = await this.merkleTreeDb.setFinalized(blockNumber);
+    this.log.info(`World state finalized chain updated`, {
+      finalizedBlockNumber: summary.finalizedBlockNumber,
+      unfinalizedBlockNumber: summary.unfinalizedBlockNumber,
+      oldestHistoricalBlock: summary.oldestHistoricalBlock,
+    });
     if (this.historyToKeep === undefined) {
       return;
     }
-    const newHistoricBlock = summary.finalizedBlockNumber - this.historyToKeep + 1;
-    if (newHistoricBlock <= 1) {
+    // Get the checkpointed block for the finalized block number
+    const finalisedCheckpoint = await this.l2BlockSource.getCheckpointedBlock(summary.finalizedBlockNumber);
+    if (finalisedCheckpoint === undefined) {
+      this.log.warn(
+        `Failed to retrieve checkpointed block for finalized block number: ${summary.finalizedBlockNumber}`,
+      );
+      return;
+    }
+    // Compute the required historic checkpoint number
+    const newHistoricCheckpointNumber = finalisedCheckpoint.checkpointNumber - this.historyToKeep + 1;
+    if (newHistoricCheckpointNumber <= 1) {
+      return;
+    }
+    // Retrieve the historic checkpoint
+    const historicCheckpoints = await this.l2BlockSource.getCheckpoints(
+      CheckpointNumber(newHistoricCheckpointNumber),
+      1,
+    );
+    if (historicCheckpoints.length === 0 || historicCheckpoints[0] === undefined) {
+      this.log.warn(`Failed to retrieve checkpoint number ${newHistoricCheckpointNumber} from Archiver`);
+      return;
+    }
+    const historicCheckpoint = historicCheckpoints[0];
+    if (historicCheckpoint.checkpoint.blocks.length === 0 || historicCheckpoint.checkpoint.blocks[0] === undefined) {
+      this.log.warn(`Retrieved checkpoint number ${newHistoricCheckpointNumber} has no blocks!`);
+      return;
+    }
+    // Find the block at the start of the checkpoint and remove blocks up to this one
+    const newHistoricBlock = historicCheckpoint.checkpoint.blocks[0];
+    if (newHistoricBlock.number <= currentSummary.oldestHistoricalBlock) {
+      this.log.debug(
+        `Historic block ${newHistoricBlock.number} is not newer than oldest available ${currentSummary.oldestHistoricalBlock}. Skipping prune.`,
+      );
       return;
     }
-    this.log.verbose(`Pruning historic blocks to ${newHistoricBlock}`);
-    const status = await this.merkleTreeDb.removeHistoricalBlocks(BlockNumber(newHistoricBlock));
+    this.log.verbose(`Pruning historic blocks to ${newHistoricBlock.number}`);
+    const status = await this.merkleTreeDb.removeHistoricalBlocks(BlockNumber(newHistoricBlock.number));
     this.log.debug(`World state summary `, status.summary);
   }
 
@@ -378,7 +456,7 @@ export class ServerWorldStateSynchronizer
   }
 
   private async handleChainPruned(blockNumber: BlockNumber) {
-    this.log.warn(`Chain pruned to block ${blockNumber}`);
+    this.log.info(`Chain pruned to block ${blockNumber}`);
     const status = await this.merkleTreeDb.unwindBlocks(blockNumber);
     this.provenBlockNumber = undefined;
     this.instrumentation.updateWorldStateMetrics(status);

package/src/test/utils.ts

@@ -60,7 +60,16 @@ export async function updateBlockState(block: L2Block, l1ToL2Messages: Fr[], for
   await Promise.all([publicDataInsert, nullifierInsert, noteHashInsert, messageInsert]);
 
   const state = await fork.getStateReference();
-  block.header = BlockHeader.from({ ...block.header, state });
+
+  // Capture the archive root *before* appending this block so the header's lastArchive chains off the previous block.
+  // sync_block now verifies lastArchive against the committed archive root, so a block carrying an unchained value
+  // (e.g. the random lastArchive from L2Block.random) would be rejected as a divergence from the canonical chain.
+  const previousArchive = await fork.getTreeInfo(MerkleTreeId.ARCHIVE);
+  block.header = BlockHeader.from({
+    ...block.header,
+    state,
+    lastArchive: new AppendOnlyTreeSnapshot(Fr.fromBuffer(previousArchive.root), Number(previousArchive.size)),
+  });
   await fork.updateArchive(block.header);
 
   const archiveState = await fork.getTreeInfo(MerkleTreeId.ARCHIVE);