@aztec/prover-node 0.0.1-commit.9b94fc1 → 0.0.1-commit.9ee6fcc6

package/src/job/epoch-proving-job-data.ts

@@ -1,40 +1,45 @@
-import { EpochNumber } from '@aztec/foundation/branded-types';
-import { Fr } from '@aztec/foundation/fields';
+import { CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
+import { Fr } from '@aztec/foundation/curves/bn254';
 import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize';
-import { CommitteeAttestation, L2Block } from '@aztec/stdlib/block';
+import { CommitteeAttestation } from '@aztec/stdlib/block';
+import { Checkpoint } from '@aztec/stdlib/checkpoint';
 import { BlockHeader, Tx } from '@aztec/stdlib/tx';
 
 /** All data from an epoch used in proving. */
 export type EpochProvingJobData = {
   epochNumber: EpochNumber;
-  blocks: L2Block[];
+  checkpoints: Checkpoint[];
   txs: Map<string, Tx>;
-  l1ToL2Messages: Record<number, Fr[]>;
+  l1ToL2Messages: Record<CheckpointNumber, Fr[]>;
   previousBlockHeader: BlockHeader;
   attestations: CommitteeAttestation[];
 };
 
 export function validateEpochProvingJobData(data: EpochProvingJobData) {
-  if (data.blocks.length > 0 && data.previousBlockHeader.getBlockNumber() + 1 !== data.blocks[0].number) {
+  if (data.checkpoints.length === 0) {
+    throw new Error('No checkpoints to prove');
+  }
+
+  const firstBlockNumber = data.checkpoints[0].blocks[0].number;
+  const previousBlockNumber = data.previousBlockHeader.getBlockNumber();
+  if (previousBlockNumber + 1 !== firstBlockNumber) {
     throw new Error(
-      `Initial block number ${
-        data.blocks[0].number
-      } does not match previous block header ${data.previousBlockHeader.getBlockNumber()}`,
+      `Initial block number ${firstBlockNumber} does not match previous block header ${previousBlockNumber}`,
     );
   }
 
-  for (const blockNumber of data.blocks.map(block => block.number)) {
-    if (!(blockNumber in data.l1ToL2Messages)) {
-      throw new Error(`Missing L1 to L2 messages for block number ${blockNumber}`);
+  for (const checkpoint of data.checkpoints) {
+    if (!(checkpoint.number in data.l1ToL2Messages)) {
+      throw new Error(`Missing L1 to L2 messages for checkpoint number ${checkpoint.number}`);
     }
   }
 }
 
 export function serializeEpochProvingJobData(data: EpochProvingJobData): Buffer {
-  const blocks = data.blocks.map(block => block.toBuffer());
+  const checkpoints = data.checkpoints.map(checkpoint => checkpoint.toBuffer());
   const txs = Array.from(data.txs.values()).map(tx => tx.toBuffer());
-  const l1ToL2Messages = Object.entries(data.l1ToL2Messages).map(([blockNumber, messages]) => [
-    Number(blockNumber),
+  const l1ToL2Messages = Object.entries(data.l1ToL2Messages).map(([checkpointNumber, messages]) => [
+    Number(checkpointNumber),
     messages.length,
     ...messages,
   ]);
@@ -43,8 +48,8 @@ export function serializeEpochProvingJobData(data: EpochProvingJobData): Buffer
   return serializeToBuffer(
     data.epochNumber,
     data.previousBlockHeader,
-    blocks.length,
-    ...blocks,
+    checkpoints.length,
+    ...checkpoints,
     txs.length,
     ...txs,
     l1ToL2Messages.length,
@@ -58,20 +63,20 @@ export function deserializeEpochProvingJobData(buf: Buffer): EpochProvingJobData
   const reader = BufferReader.asReader(buf);
   const epochNumber = EpochNumber(reader.readNumber());
   const previousBlockHeader = reader.readObject(BlockHeader);
-  const blocks = reader.readVector(L2Block);
+  const checkpoints = reader.readVector(Checkpoint);
   const txArray = reader.readVector(Tx);
 
-  const l1ToL2MessageBlockCount = reader.readNumber();
+  const l1ToL2MessageCheckpointCount = reader.readNumber();
   const l1ToL2Messages: Record<number, Fr[]> = {};
-  for (let i = 0; i < l1ToL2MessageBlockCount; i++) {
-    const blockNumber = reader.readNumber();
+  for (let i = 0; i < l1ToL2MessageCheckpointCount; i++) {
+    const checkpointNumber = CheckpointNumber(reader.readNumber());
     const messages = reader.readVector(Fr);
-    l1ToL2Messages[blockNumber] = messages;
+    l1ToL2Messages[checkpointNumber] = messages;
   }
 
   const attestations = reader.readVector(CommitteeAttestation);
 
   const txs = new Map<string, Tx>(txArray.map(tx => [tx.getTxHash().toString(), tx]));
 
-  return { epochNumber, previousBlockHeader, blocks, txs, l1ToL2Messages, attestations };
+  return { epochNumber, previousBlockHeader, checkpoints, txs, l1ToL2Messages, attestations };
 }

package/src/job/epoch-proving-job.ts

@@ -1,17 +1,19 @@
 import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
 import { asyncPool } from '@aztec/foundation/async-pool';
-import { EpochNumber } from '@aztec/foundation/branded-types';
+import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
 import { padArrayEnd } from '@aztec/foundation/collection';
-import { Fr } from '@aztec/foundation/fields';
-import { createLogger } from '@aztec/foundation/log';
+import { Fr } from '@aztec/foundation/curves/bn254';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 import { Timer } from '@aztec/foundation/timer';
+import { AVM_MAX_CONCURRENT_SIMULATIONS } from '@aztec/native';
 import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
 import { protocolContractsHash } from '@aztec/protocol-contracts';
 import { buildFinalBlobChallenges } from '@aztec/prover-client/helpers';
 import type { PublicProcessor, PublicProcessorFactory } from '@aztec/simulator/server';
 import { PublicSimulatorConfig } from '@aztec/stdlib/avm';
 import type { L2Block, L2BlockSource } from '@aztec/stdlib/block';
+import type { Checkpoint } from '@aztec/stdlib/checkpoint';
 import {
   type EpochProver,
   type EpochProvingJobState,
@@ -42,7 +44,7 @@ export type EpochProvingJobOptions = {
  */
 export class EpochProvingJob implements Traceable {
   private state: EpochProvingJobState = 'initialized';
-  private log = createLogger('prover-node:epoch-proving-job');
+  private log: Logger;
   private uuid: string;
 
   private runPromise: Promise<void> | undefined;
@@ -61,9 +63,14 @@ export class EpochProvingJob implements Traceable {
     private metrics: ProverNodeJobMetrics,
     private deadline: Date | undefined,
     private config: EpochProvingJobOptions,
+    bindings?: LoggerBindings,
   ) {
     validateEpochProvingJobData(data);
     this.uuid = crypto.randomUUID();
+    this.log = createLogger('prover-node:epoch-proving-job', {
+      ...bindings,
+      instanceId: `epoch-${data.epochNumber}`,
+    });
     this.tracer = metrics.tracer;
   }
 
@@ -91,8 +98,8 @@ export class EpochProvingJob implements Traceable {
     return this.data.epochNumber;
   }
 
-  private get blocks() {
-    return this.data.blocks;
+  private get checkpoints() {
+    return this.data.checkpoints;
   }
 
   private get txs() {
@@ -117,13 +124,21 @@ export class EpochProvingJob implements Traceable {
 
     const attestations = this.attestations.map(attestation => attestation.toViem());
     const epochNumber = this.epochNumber;
-    const epochSizeBlocks = this.blocks.length;
-    const epochSizeTxs = this.blocks.reduce((total, current) => total + current.body.txEffects.length, 0);
-    const [fromBlock, toBlock] = [this.blocks[0].number, this.blocks.at(-1)!.number];
-    this.log.info(`Starting epoch ${epochNumber} proving job with blocks ${fromBlock} to ${toBlock}`, {
+    const epochSizeCheckpoints = this.checkpoints.length;
+    const epochSizeBlocks = this.checkpoints.reduce((accum, checkpoint) => accum + checkpoint.blocks.length, 0);
+    const epochSizeTxs = this.checkpoints.reduce(
+      (accum, checkpoint) =>
+        accum + checkpoint.blocks.reduce((accumC, block) => accumC + block.body.txEffects.length, 0),
+      0,
+    );
+    const fromCheckpoint = this.checkpoints[0].number;
+    const toCheckpoint = this.checkpoints.at(-1)!.number;
+    const fromBlock = this.checkpoints[0].blocks[0].number;
+    const toBlock = this.checkpoints.at(-1)!.blocks.at(-1)!.number;
+    this.log.info(`Starting epoch ${epochNumber} proving job with checkpoints ${fromCheckpoint} to ${toCheckpoint}`, {
       fromBlock,
       toBlock,
-      epochSizeBlocks,
+      epochSizeTxs,
       epochNumber,
       uuid: this.uuid,
     });
@@ -134,87 +149,116 @@ export class EpochProvingJob implements Traceable {
     this.runPromise = promise;
 
     try {
-      const blobFieldsPerCheckpoint = this.blocks.map(block => block.getCheckpointBlobFields());
+      const blobTimer = new Timer();
+      const blobFieldsPerCheckpoint = this.checkpoints.map(checkpoint => checkpoint.toBlobFields());
       const finalBlobBatchingChallenges = await buildFinalBlobChallenges(blobFieldsPerCheckpoint);
+      this.metrics.recordBlobProcessing(blobTimer.ms());
 
-      // TODO(#17027): Enable multiple blocks per checkpoint.
-      // Total number of checkpoints equals number of blocks because we currently build a checkpoint with only one block.
-      const totalNumCheckpoints = epochSizeBlocks;
-
-      this.prover.startNewEpoch(epochNumber, totalNumCheckpoints, finalBlobBatchingChallenges);
+      this.prover.startNewEpoch(epochNumber, epochSizeCheckpoints, finalBlobBatchingChallenges);
+      const chonkTimer = new Timer();
       await this.prover.startChonkVerifierCircuits(Array.from(this.txs.values()));
+      this.metrics.recordChonkVerifier(chonkTimer.ms());
 
-      await asyncPool(this.config.parallelBlockLimit ?? 32, this.blocks, async block => {
-        this.checkState();
+      // Everything in the epoch should have the same chainId and version.
+      const { chainId, version } = this.checkpoints[0].blocks[0].header.globalVariables;
 
-        const globalVariables = block.header.globalVariables;
-        const txs = this.getTxs(block);
-        const l1ToL2Messages = this.getL1ToL2Messages(block);
-        const previousHeader = this.getBlockHeader(block.number - 1)!;
-
-        this.log.verbose(`Starting processing block ${block.number}`, {
-          number: block.number,
-          blockHash: (await block.hash()).toString(),
-          lastArchive: block.header.lastArchive.root,
-          noteHashTreeRoot: block.header.state.partial.noteHashTree.root,
-          nullifierTreeRoot: block.header.state.partial.nullifierTree.root,
-          publicDataTreeRoot: block.header.state.partial.publicDataTree.root,
-          previousHeader: previousHeader.hash(),
-          uuid: this.uuid,
-          ...globalVariables,
-        });
+      const previousBlockHeaders = this.gatherPreviousBlockHeaders();
+
+      const allCheckpointsTimer = new Timer();
+
+      const parallelism = this.config.parallelBlockLimit
+        ? this.config.parallelBlockLimit
+        : AVM_MAX_CONCURRENT_SIMULATIONS > 0
+          ? AVM_MAX_CONCURRENT_SIMULATIONS
+          : this.checkpoints.length;
 
+      await asyncPool(parallelism, this.checkpoints, async checkpoint => {
+        this.checkState();
+        const checkpointTimer = new Timer();
+
+        const checkpointIndex = checkpoint.number - fromCheckpoint;
         const checkpointConstants = CheckpointConstantData.from({
-          chainId: globalVariables.chainId,
-          version: globalVariables.version,
+          chainId,
+          version,
           vkTreeRoot: getVKTreeRoot(),
           protocolContractsHash: protocolContractsHash,
           proverId: this.prover.getProverId().toField(),
-          slotNumber: globalVariables.slotNumber,
-          coinbase: globalVariables.coinbase,
-          feeRecipient: globalVariables.feeRecipient,
-          gasFees: globalVariables.gasFees,
+          slotNumber: checkpoint.header.slotNumber,
+          coinbase: checkpoint.header.coinbase,
+          feeRecipient: checkpoint.header.feeRecipient,
+          gasFees: checkpoint.header.gasFees,
+        });
+        const previousHeader = previousBlockHeaders[checkpointIndex];
+        const l1ToL2Messages = this.getL1ToL2Messages(checkpoint);
+
+        this.log.verbose(`Starting processing checkpoint ${checkpoint.number}`, {
+          number: checkpoint.number,
+          checkpointHash: checkpoint.hash().toString(),
+          lastArchive: checkpoint.header.lastArchiveRoot,
+          previousHeader: previousHeader.hash(),
+          uuid: this.uuid,
         });
 
-        // TODO(#17027): Enable multiple blocks per checkpoint.
-        // Each checkpoint has only one block.
-        const totalNumBlocks = 1;
-        const checkpointIndex = block.number - fromBlock;
         await this.prover.startNewCheckpoint(
           checkpointIndex,
           checkpointConstants,
           l1ToL2Messages,
-          totalNumBlocks,
+          checkpoint.blocks.length,
           previousHeader,
         );
 
-        // Start block proving
-        await this.prover.startNewBlock(block.number, globalVariables.timestamp, txs.length);
+        for (let blockIndex = 0; blockIndex < checkpoint.blocks.length; blockIndex++) {
+          const blockTimer = new Timer();
+          const block = checkpoint.blocks[blockIndex];
+          const globalVariables = block.header.globalVariables;
+          const txs = this.getTxs(block);
+
+          this.log.verbose(`Starting processing block ${block.number}`, {
+            number: block.number,
+            blockHash: (await block.hash()).toString(),
+            lastArchive: block.header.lastArchive.root,
+            noteHashTreeRoot: block.header.state.partial.noteHashTree.root,
+            nullifierTreeRoot: block.header.state.partial.nullifierTree.root,
+            publicDataTreeRoot: block.header.state.partial.publicDataTree.root,
+            ...globalVariables,
+            numTxs: txs.length,
+          });
 
-        // Process public fns
-        const db = await this.createFork(block.number - 1, l1ToL2Messages);
-        const config = PublicSimulatorConfig.from({
-          proverId: this.prover.getProverId().toField(),
-          skipFeeEnforcement: false,
-          collectDebugLogs: false,
-          collectHints: true,
-          maxDebugLogMemoryReads: 0,
-          collectStatistics: false,
-        });
-        const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, config);
-        const processed = await this.processTxs(publicProcessor, txs);
-        await this.prover.addTxs(processed);
-        await db.close();
-        this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, {
-          blockNumber: block.number,
-          blockHash: (await block.hash()).toString(),
-          uuid: this.uuid,
-        });
+          // Start block proving
+          await this.prover.startNewBlock(block.number, globalVariables.timestamp, txs.length);
+
+          // Process public fns. L1 to L2 messages are only inserted for the first block of a checkpoint,
+          // as the fork for subsequent blocks already includes them from the previous block's synced state.
+          const db = await this.createFork(
+            BlockNumber(block.number - 1),
+            blockIndex === 0 ? l1ToL2Messages : undefined,
+          );
+          const config = PublicSimulatorConfig.from({
+            proverId: this.prover.getProverId().toField(),
+            skipFeeEnforcement: false,
+            collectDebugLogs: false,
+            collectHints: true,
+            collectPublicInputs: true,
+            collectStatistics: false,
+          });
+          const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, config);
+          const processed = await this.processTxs(publicProcessor, txs);
+          await this.prover.addTxs(processed);
+          await db.close();
+          this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, {
+            blockNumber: block.number,
+            blockHash: (await block.hash()).toString(),
+            uuid: this.uuid,
+          });
 
-        // Mark block as completed to pad it
-        const expectedBlockHeader = block.getBlockHeader();
-        await this.prover.setBlockCompleted(block.number, expectedBlockHeader);
+          // Mark block as completed to pad it
+          const expectedBlockHeader = block.header;
+          await this.prover.setBlockCompleted(block.number, expectedBlockHeader);
+          this.metrics.recordBlockProcessing(blockTimer.ms());
+        }
+        this.metrics.recordCheckpointProcessing(checkpointTimer.ms());
       });
+      this.metrics.recordAllCheckpointsProcessing(allCheckpointsTimer.ms());
 
       const executionTime = timer.ms();
 
@@ -226,16 +270,16 @@ export class EpochProvingJob implements Traceable {
 
       if (this.config.skipSubmitProof) {
         this.log.info(
-          `Proof publishing is disabled. Dropping valid proof for epoch ${epochNumber} (blocks ${fromBlock} to ${toBlock})`,
+          `Proof publishing is disabled. Dropping valid proof for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`,
         );
         this.state = 'completed';
-        this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeBlocks, epochSizeTxs);
+        this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
         return;
       }
 
       const success = await this.publisher.submitEpochProof({
-        fromBlock,
-        toBlock,
+        fromCheckpoint,
+        toCheckpoint,
         epochNumber,
         publicInputs,
         proof,
@@ -246,12 +290,12 @@ export class EpochProvingJob implements Traceable {
         throw new Error('Failed to submit epoch proof to L1');
       }
 
-      this.log.info(`Submitted proof for epoch ${epochNumber} (blocks ${fromBlock} to ${toBlock})`, {
+      this.log.info(`Submitted proof for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`, {
         epochNumber,
         uuid: this.uuid,
       });
       this.state = 'completed';
-      this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeBlocks, epochSizeTxs);
+      this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
     } catch (err: any) {
       if (err && err.name === 'HaltExecutionError') {
         this.log.warn(`Halted execution of epoch ${epochNumber} prover job`, {
@@ -274,22 +318,29 @@ export class EpochProvingJob implements Traceable {
   }
 
   /**
-   * Create a new db fork for tx processing, inserting all L1 to L2.
+   * Create a new db fork for tx processing, optionally inserting L1 to L2 messages.
+   * L1 to L2 messages should only be inserted for the first block in a checkpoint,
+   * as subsequent blocks' synced state already includes them.
    * REFACTOR: The prover already spawns a db fork of its own for each block, so we may be able to do away with just one fork.
    */
-  private async createFork(blockNumber: number, l1ToL2Messages: Fr[]) {
+  private async createFork(blockNumber: BlockNumber, l1ToL2Messages: Fr[] | undefined) {
+    this.log.verbose(`Creating fork at ${blockNumber}`, { blockNumber });
     const db = await this.dbProvider.fork(blockNumber);
-    const l1ToL2MessagesPadded = padArrayEnd<Fr, number>(
-      l1ToL2Messages,
-      Fr.ZERO,
-      NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP,
-      'Too many L1 to L2 messages',
-    );
-    this.log.verbose(`Creating fork at ${blockNumber} with ${l1ToL2Messages.length} L1 to L2 messages`, {
-      blockNumber,
-      l1ToL2Messages: l1ToL2Messages.map(m => m.toString()),
-    });
-    await db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded);
+
+    if (l1ToL2Messages !== undefined) {
+      this.log.verbose(`Inserting ${l1ToL2Messages.length} L1 to L2 messages in fork`, {
+        blockNumber,
+        l1ToL2Messages: l1ToL2Messages.map(m => m.toString()),
+      });
+      const l1ToL2MessagesPadded = padArrayEnd<Fr, number>(
+        l1ToL2Messages,
+        Fr.ZERO,
+        NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP,
+        'Too many L1 to L2 messages',
+      );
+      await db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded);
+    }
+
     return db;
   }
 
@@ -346,11 +397,12 @@ export class EpochProvingJob implements Traceable {
     const intervalMs = Math.ceil((await l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
     this.epochCheckPromise = new RunningPromise(
       async () => {
-        const blocks = await l2BlockSource.getBlockHeadersForEpoch(this.epochNumber);
-        const blockHashes = await Promise.all(blocks.map(block => block.hash()));
-        const thisBlockHashes = await Promise.all(this.blocks.map(block => block.hash()));
+        const blockHeaders = await l2BlockSource.getCheckpointedBlockHeadersForEpoch(this.epochNumber);
+        const blockHashes = await Promise.all(blockHeaders.map(header => header.hash()));
+        const thisBlocks = this.checkpoints.flatMap(checkpoint => checkpoint.blocks);
+        const thisBlockHashes = await Promise.all(thisBlocks.map(block => block.hash()));
         if (
-          blocks.length !== this.blocks.length ||
+          blockHeaders.length !== thisBlocks.length ||
           !blockHashes.every((block, i) => block.equals(thisBlockHashes[i]))
         ) {
           this.log.warn('Epoch blocks changed underfoot', {
@@ -368,30 +420,18 @@ export class EpochProvingJob implements Traceable {
     this.log.verbose(`Scheduled epoch check for epoch ${this.epochNumber} every ${intervalMs}ms`);
   }
 
-  /* Returns the header for the given block number based on the epoch proving job data. */
-  private getBlockHeader(blockNumber: number) {
-    const block = this.blocks.find(b => b.number === blockNumber);
-    if (block) {
-      return block.getBlockHeader();
-    }
-
-    if (blockNumber === Number(this.data.previousBlockHeader.getBlockNumber())) {
-      return this.data.previousBlockHeader;
-    }
-
-    throw new Error(
-      `Block header not found for block number ${blockNumber} (got ${this.blocks
-        .map(b => b.number)
-        .join(', ')} and previous header ${this.data.previousBlockHeader.getBlockNumber()})`,
-    );
+  /* Returns the last block header in the previous checkpoint for all checkpoints in the epoch */
+  private gatherPreviousBlockHeaders() {
+    const lastBlocks = this.checkpoints.map(checkpoint => checkpoint.blocks.at(-1)!);
+    return [this.data.previousBlockHeader, ...lastBlocks.map(block => block.header).slice(0, -1)];
   }
 
   private getTxs(block: L2Block): Tx[] {
     return block.body.txEffects.map(txEffect => this.txs.get(txEffect.txHash.toString())!);
   }
 
-  private getL1ToL2Messages(block: L2Block) {
-    return this.data.l1ToL2Messages[block.number];
+  private getL1ToL2Messages(checkpoint: Checkpoint) {
+    return this.data.l1ToL2Messages[checkpoint.number];
   }
 
   private async processTxs(publicProcessor: PublicProcessor, txs: Tx[]): Promise<ProcessedTx[]> {