@aztec/prover-node 4.0.0-nightly.20250907 → 4.0.0-nightly.20260108

package/src/job/epoch-proving-job.ts

@@ -1,19 +1,25 @@
-import { BatchedBlob, Blob } from '@aztec/blob-lib';
 import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
 import { asyncPool } from '@aztec/foundation/async-pool';
+import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
 import { padArrayEnd } from '@aztec/foundation/collection';
-import { Fr } from '@aztec/foundation/fields';
+import { Fr } from '@aztec/foundation/curves/bn254';
 import { createLogger } from '@aztec/foundation/log';
 import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 import { Timer } from '@aztec/foundation/timer';
+import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
+import { protocolContractsHash } from '@aztec/protocol-contracts';
+import { buildFinalBlobChallenges } from '@aztec/prover-client/helpers';
 import type { PublicProcessor, PublicProcessorFactory } from '@aztec/simulator/server';
-import type { L2Block, L2BlockSource } from '@aztec/stdlib/block';
+import { PublicSimulatorConfig } from '@aztec/stdlib/avm';
+import type { L2BlockNew, L2BlockSource } from '@aztec/stdlib/block';
+import type { Checkpoint } from '@aztec/stdlib/checkpoint';
 import {
   type EpochProver,
   type EpochProvingJobState,
   EpochProvingJobTerminalState,
   type ForkMerkleTreeOperations,
 } from '@aztec/stdlib/interfaces/server';
+import { CheckpointConstantData } from '@aztec/stdlib/rollup';
 import { MerkleTreeId } from '@aztec/stdlib/trees';
 import type { ProcessedTx, Tx } from '@aztec/stdlib/tx';
 import { Attributes, type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client';
@@ -24,6 +30,12 @@ import type { ProverNodeJobMetrics } from '../metrics.js';
 import type { ProverNodePublisher } from '../prover-node-publisher.js';
 import { type EpochProvingJobData, validateEpochProvingJobData } from './epoch-proving-job-data.js';
 
+export type EpochProvingJobOptions = {
+  parallelBlockLimit?: number;
+  skipEpochCheck?: boolean;
+  skipSubmitProof?: boolean;
+};
+
 /**
  * Job that grabs a range of blocks from the unfinalized chain from L1, gets their txs given their hashes,
  * re-executes their public calls, generates a rollup proof, and submits it to L1. This job will update the
@@ -49,7 +61,7 @@ export class EpochProvingJob implements Traceable {
     private l2BlockSource: L2BlockSource | undefined,
     private metrics: ProverNodeJobMetrics,
     private deadline: Date | undefined,
-    private config: { parallelBlockLimit?: number; skipEpochCheck?: boolean },
+    private config: EpochProvingJobOptions,
   ) {
     validateEpochProvingJobData(data);
     this.uuid = crypto.randomUUID();
@@ -64,7 +76,7 @@ export class EpochProvingJob implements Traceable {
     return this.state;
   }
 
-  public getEpochNumber(): bigint {
+  public getEpochNumber(): EpochNumber {
     return this.data.epochNumber;
   }
 
@@ -80,8 +92,8 @@ export class EpochProvingJob implements Traceable {
     return this.data.epochNumber;
   }
 
-  private get blocks() {
-    return this.data.blocks;
+  private get checkpoints() {
+    return this.data.checkpoints;
   }
 
   private get txs() {
@@ -96,7 +108,7 @@ export class EpochProvingJob implements Traceable {
    * Proves the given epoch and submits the proof to L1.
    */
   @trackSpan('EpochProvingJob.run', function () {
-    return { [Attributes.EPOCH_NUMBER]: Number(this.data.epochNumber) };
+    return { [Attributes.EPOCH_NUMBER]: this.data.epochNumber };
   })
   public async run() {
     this.scheduleDeadlineStop();
@@ -105,14 +117,22 @@ export class EpochProvingJob implements Traceable {
     }
 
     const attestations = this.attestations.map(attestation => attestation.toViem());
-    const epochNumber = Number(this.epochNumber);
-    const epochSizeBlocks = this.blocks.length;
-    const epochSizeTxs = this.blocks.reduce((total, current) => total + current.body.txEffects.length, 0);
-    const [fromBlock, toBlock] = [this.blocks[0].number, this.blocks.at(-1)!.number];
-    this.log.info(`Starting epoch ${epochNumber} proving job with blocks ${fromBlock} to ${toBlock}`, {
+    const epochNumber = this.epochNumber;
+    const epochSizeCheckpoints = this.checkpoints.length;
+    const epochSizeBlocks = this.checkpoints.reduce((accum, checkpoint) => accum + checkpoint.blocks.length, 0);
+    const epochSizeTxs = this.checkpoints.reduce(
+      (accum, checkpoint) =>
+        accum + checkpoint.blocks.reduce((accumC, block) => accumC + block.body.txEffects.length, 0),
+      0,
+    );
+    const fromCheckpoint = this.checkpoints[0].number;
+    const toCheckpoint = this.checkpoints.at(-1)!.number;
+    const fromBlock = this.checkpoints[0].blocks[0].number;
+    const toBlock = this.checkpoints.at(-1)!.blocks.at(-1)!.number;
+    this.log.info(`Starting epoch ${epochNumber} proving job with checkpoints ${fromCheckpoint} to ${toCheckpoint}`, {
       fromBlock,
       toBlock,
-      epochSizeBlocks,
+      epochSizeTxs,
       epochNumber,
       uuid: this.uuid,
     });
@@ -123,51 +143,93 @@ export class EpochProvingJob implements Traceable {
     this.runPromise = promise;
 
     try {
-      const allBlobs = (
-        await Promise.all(this.blocks.map(async block => await Blob.getBlobsPerBlock(block.body.toBlobFields())))
-      ).flat();
+      const blobFieldsPerCheckpoint = this.checkpoints.map(checkpoint => checkpoint.toBlobFields());
+      const finalBlobBatchingChallenges = await buildFinalBlobChallenges(blobFieldsPerCheckpoint);
+
+      this.prover.startNewEpoch(epochNumber, epochSizeCheckpoints, finalBlobBatchingChallenges);
+      await this.prover.startChonkVerifierCircuits(Array.from(this.txs.values()));
 
-      const finalBlobBatchingChallenges = await BatchedBlob.precomputeBatchedBlobChallenges(allBlobs);
-      this.prover.startNewEpoch(epochNumber, fromBlock, epochSizeBlocks, finalBlobBatchingChallenges);
-      await this.prover.startTubeCircuits(Array.from(this.txs.values()));
+      // Everything in the epoch should have the same chainId and version.
+      const { chainId, version } = this.checkpoints[0].blocks[0].header.globalVariables;
 
-      await asyncPool(this.config.parallelBlockLimit ?? 32, this.blocks, async block => {
+      const previousBlockHeaders = this.gatherPreviousBlockHeaders();
+
+      await asyncPool(this.config.parallelBlockLimit ?? 32, this.checkpoints, async checkpoint => {
         this.checkState();
 
-        const globalVariables = block.header.globalVariables;
-        const txs = this.getTxs(block);
-        const l1ToL2Messages = this.getL1ToL2Messages(block);
-        const previousHeader = this.getBlockHeader(block.number - 1)!;
-
-        this.log.verbose(`Starting processing block ${block.number}`, {
-          number: block.number,
-          blockHash: (await block.hash()).toString(),
-          lastArchive: block.header.lastArchive.root,
-          noteHashTreeRoot: block.header.state.partial.noteHashTree.root,
-          nullifierTreeRoot: block.header.state.partial.nullifierTree.root,
-          publicDataTreeRoot: block.header.state.partial.publicDataTree.root,
-          previousHeader: previousHeader.hash(),
-          uuid: this.uuid,
-          ...globalVariables,
+        const checkpointIndex = checkpoint.number - fromCheckpoint;
+        const checkpointConstants = CheckpointConstantData.from({
+          chainId,
+          version,
+          vkTreeRoot: getVKTreeRoot(),
+          protocolContractsHash: protocolContractsHash,
+          proverId: this.prover.getProverId().toField(),
+          slotNumber: checkpoint.header.slotNumber,
+          coinbase: checkpoint.header.coinbase,
+          feeRecipient: checkpoint.header.feeRecipient,
+          gasFees: checkpoint.header.gasFees,
         });
+        const previousHeader = previousBlockHeaders[checkpointIndex];
+        const l1ToL2Messages = this.getL1ToL2Messages(checkpoint);
 
-        // Start block proving
-        await this.prover.startNewBlock(globalVariables, l1ToL2Messages, previousHeader);
-
-        // Process public fns
-        const db = await this.createFork(block.number - 1, l1ToL2Messages);
-        const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, true);
-        const processed = await this.processTxs(publicProcessor, txs);
-        await this.prover.addTxs(processed);
-        await db.close();
-        this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, {
-          blockNumber: block.number,
-          blockHash: (await block.hash()).toString(),
+        this.log.verbose(`Starting processing checkpoint ${checkpoint.number}`, {
+          number: checkpoint.number,
+          checkpointHash: checkpoint.hash().toString(),
+          lastArchive: checkpoint.header.lastArchiveRoot,
+          previousHeader: previousHeader.hash(),
           uuid: this.uuid,
         });
 
-        // Mark block as completed to pad it
-        await this.prover.setBlockCompleted(block.number, block.header);
+        await this.prover.startNewCheckpoint(
+          checkpointIndex,
+          checkpointConstants,
+          l1ToL2Messages,
+          checkpoint.blocks.length,
+          previousHeader,
+        );
+
+        for (const block of checkpoint.blocks) {
+          const globalVariables = block.header.globalVariables;
+          const txs = this.getTxs(block);
+
+          this.log.verbose(`Starting processing block ${block.number}`, {
+            number: block.number,
+            blockHash: (await block.hash()).toString(),
+            lastArchive: block.header.lastArchive.root,
+            noteHashTreeRoot: block.header.state.partial.noteHashTree.root,
+            nullifierTreeRoot: block.header.state.partial.nullifierTree.root,
+            publicDataTreeRoot: block.header.state.partial.publicDataTree.root,
+            ...globalVariables,
+            numTxs: txs.length,
+          });
+
+          // Start block proving
+          await this.prover.startNewBlock(block.number, globalVariables.timestamp, txs.length);
+
+          // Process public fns
+          const db = await this.createFork(BlockNumber(block.number - 1), l1ToL2Messages);
+          const config = PublicSimulatorConfig.from({
+            proverId: this.prover.getProverId().toField(),
+            skipFeeEnforcement: false,
+            collectDebugLogs: false,
+            collectHints: true,
+            collectPublicInputs: true,
+            collectStatistics: false,
+          });
+          const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, config);
+          const processed = await this.processTxs(publicProcessor, txs);
+          await this.prover.addTxs(processed);
+          await db.close();
+          this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, {
+            blockNumber: block.number,
+            blockHash: (await block.hash()).toString(),
+            uuid: this.uuid,
+          });
+
+          // Mark block as completed to pad it
+          const expectedBlockHeader = block.header;
+          await this.prover.setBlockCompleted(block.number, expectedBlockHeader);
+        }
       });
 
       const executionTime = timer.ms();
@@ -178,9 +240,18 @@ export class EpochProvingJob implements Traceable {
 
       this.progressState('publishing-proof');
 
+      if (this.config.skipSubmitProof) {
+        this.log.info(
+          `Proof publishing is disabled. Dropping valid proof for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`,
+        );
+        this.state = 'completed';
+        this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
+        return;
+      }
+
       const success = await this.publisher.submitEpochProof({
-        fromBlock,
-        toBlock,
+        fromCheckpoint,
+        toCheckpoint,
         epochNumber,
         publicInputs,
         proof,
@@ -191,12 +262,12 @@ export class EpochProvingJob implements Traceable {
         throw new Error('Failed to submit epoch proof to L1');
       }
 
-      this.log.info(`Submitted proof for epoch ${epochNumber} (blocks ${fromBlock} to ${toBlock})`, {
+      this.log.info(`Submitted proof for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`, {
         epochNumber,
         uuid: this.uuid,
       });
       this.state = 'completed';
-      this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeBlocks, epochSizeTxs);
+      this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
     } catch (err: any) {
       if (err && err.name === 'HaltExecutionError') {
         this.log.warn(`Halted execution of epoch ${epochNumber} prover job`, {
@@ -222,9 +293,9 @@ export class EpochProvingJob implements Traceable {
    * Create a new db fork for tx processing, inserting all L1 to L2.
    * REFACTOR: The prover already spawns a db fork of its own for each block, so we may be able to do away with just one fork.
    */
-  private async createFork(blockNumber: number, l1ToL2Messages: Fr[]) {
+  private async createFork(blockNumber: BlockNumber, l1ToL2Messages: Fr[]) {
     const db = await this.dbProvider.fork(blockNumber);
-    const l1ToL2MessagesPadded = padArrayEnd(
+    const l1ToL2MessagesPadded = padArrayEnd<Fr, number>(
       l1ToL2Messages,
       Fr.ZERO,
       NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP,
@@ -232,7 +303,7 @@ export class EpochProvingJob implements Traceable {
     );
     this.log.verbose(`Creating fork at ${blockNumber} with ${l1ToL2Messages.length} L1 to L2 messages`, {
       blockNumber,
-      l1ToL2Messages: l1ToL2MessagesPadded.map(m => m.toString()),
+      l1ToL2Messages: l1ToL2Messages.map(m => m.toString()),
     });
     await db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded);
     return db;
@@ -252,7 +323,6 @@ export class EpochProvingJob implements Traceable {
   public async stop(state: EpochProvingJobTerminalState = 'stopped') {
     this.state = state;
     this.prover.cancel();
-    // TODO(palla/prover): Stop the publisher as well
     if (this.runPromise) {
       await this.runPromise;
     }
@@ -294,11 +364,9 @@ export class EpochProvingJob implements Traceable {
       async () => {
         const blocks = await l2BlockSource.getBlockHeadersForEpoch(this.epochNumber);
         const blockHashes = await Promise.all(blocks.map(block => block.hash()));
-        const thisBlockHashes = await Promise.all(this.blocks.map(block => block.hash()));
-        if (
-          blocks.length !== this.blocks.length ||
-          !blockHashes.every((block, i) => block.equals(thisBlockHashes[i]))
-        ) {
+        const thisBlocks = this.checkpoints.flatMap(checkpoint => checkpoint.blocks);
+        const thisBlockHashes = await Promise.all(thisBlocks.map(block => block.hash()));
+        if (blocks.length !== thisBlocks.length || !blockHashes.every((block, i) => block.equals(thisBlockHashes[i]))) {
           this.log.warn('Epoch blocks changed underfoot', {
             uuid: this.uuid,
             epochNumber: this.epochNumber,
@@ -314,30 +382,18 @@ export class EpochProvingJob implements Traceable {
     this.log.verbose(`Scheduled epoch check for epoch ${this.epochNumber} every ${intervalMs}ms`);
   }
 
-  /* Returns the header for the given block number based on the epoch proving job data. */
-  private getBlockHeader(blockNumber: number) {
-    const block = this.blocks.find(b => b.number === blockNumber);
-    if (block) {
-      return block.header;
-    }
-
-    if (blockNumber === Number(this.data.previousBlockHeader.getBlockNumber())) {
-      return this.data.previousBlockHeader;
-    }
-
-    throw new Error(
-      `Block header not found for block number ${blockNumber} (got ${this.blocks
-        .map(b => b.number)
-        .join(', ')} and previous header ${this.data.previousBlockHeader.getBlockNumber()})`,
-    );
+  /* Returns the last block header in the previous checkpoint for all checkpoints in the epoch */
+  private gatherPreviousBlockHeaders() {
+    const lastBlocks = this.checkpoints.map(checkpoint => checkpoint.blocks.at(-1)!);
+    return [this.data.previousBlockHeader, ...lastBlocks.map(block => block.header).slice(0, -1)];
   }
 
-  private getTxs(block: L2Block): Tx[] {
+  private getTxs(block: L2BlockNew): Tx[] {
     return block.body.txEffects.map(txEffect => this.txs.get(txEffect.txHash.toString())!);
   }
 
-  private getL1ToL2Messages(block: L2Block) {
-    return this.data.l1ToL2Messages[block.number];
+  private getL1ToL2Messages(checkpoint: Checkpoint) {
+    return this.data.l1ToL2Messages[checkpoint.number];
   }
 
   private async processTxs(publicProcessor: PublicProcessor, txs: Tx[]): Promise<ProcessedTx[]> {

package/src/metrics.ts

@@ -1,4 +1,4 @@
-import type { RollupContract } from '@aztec/ethereum';
+import type { RollupContract } from '@aztec/ethereum/contracts';
 import type { EthAddress } from '@aztec/foundation/eth-address';
 import { createLogger } from '@aztec/foundation/log';
 import type { L1PublishProofStats, L1PublishStats } from '@aztec/stdlib/stats';
@@ -21,6 +21,7 @@ import { formatEther, formatUnits } from 'viem';
 export class ProverNodeJobMetrics {
   proverEpochExecutionDuration: Histogram;
   provingJobDuration: Histogram;
+  provingJobCheckpoints: Gauge;
   provingJobBlocks: Gauge;
   provingJobTransactions: Gauge;
 
@@ -39,6 +40,10 @@ export class ProverNodeJobMetrics {
       unit: 's',
       valueType: ValueType.DOUBLE,
     });
+    this.provingJobCheckpoints = this.meter.createGauge(Metrics.PROVER_NODE_JOB_CHECKPOINTS, {
+      description: 'Number of checkpoints in a proven epoch',
+      valueType: ValueType.INT,
+    });
     this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
       description: 'Number of blocks in a proven epoch',
       valueType: ValueType.INT,
@@ -49,9 +54,16 @@ export class ProverNodeJobMetrics {
     });
   }
 
-  public recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number) {
+  public recordProvingJob(
+    executionTimeMs: number,
+    totalTimeMs: number,
+    numCheckpoints: number,
+    numBlocks: number,
+    numTxs: number,
+  ) {
     this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
     this.provingJobDuration.record(totalTimeMs / 1000);
+    this.provingJobCheckpoints.record(Math.floor(numCheckpoints));
     this.provingJobBlocks.record(Math.floor(numBlocks));
     this.provingJobTransactions.record(Math.floor(numTxs));
   }
@@ -91,13 +103,13 @@ export class ProverNodeRewardsMetrics {
   }
 
   private observe = async (observer: BatchObservableResult): Promise<void> => {
-    const epoch = await this.rollup.getEpochNumber();
+    const epoch = await this.rollup.getCurrentEpochNumber();
 
     if (epoch > this.proofSubmissionEpochs) {
       // look at the prev epoch so that we get an accurate value, after proof submission window has closed
       // For example, if proof submission window is 1 epoch, and we are in epoch 2, we should be looking at epoch 0.
       // Similarly, if the proof submission window is 0, and we are in epoch 1, we should be looking at epoch 0.
-      const closedEpoch = epoch - BigInt(this.proofSubmissionEpochs) - 1n;
+      const closedEpoch = BigInt(epoch) - BigInt(this.proofSubmissionEpochs) - 1n;
       const rewards = await this.rollup.getSpecificProverRewardsForEpoch(closedEpoch, this.coinbase);
 
       const fmt = parseFloat(formatUnits(rewards, 18));

package/src/monitors/epoch-monitor.ts

@@ -1,17 +1,13 @@
+import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
 import { createLogger } from '@aztec/foundation/log';
 import { RunningPromise } from '@aztec/foundation/running-promise';
+import { sleep } from '@aztec/foundation/sleep';
 import type { L2BlockSource } from '@aztec/stdlib/block';
 import { type L1RollupConstants, getEpochAtSlot } from '@aztec/stdlib/epoch-helpers';
-import {
-  type TelemetryClient,
-  type Traceable,
-  type Tracer,
-  getTelemetryClient,
-  trackSpan,
-} from '@aztec/telemetry-client';
+import { type TelemetryClient, type Traceable, type Tracer, getTelemetryClient } from '@aztec/telemetry-client';
 
 export interface EpochMonitorHandler {
-  handleEpochReadyToProve(epochNumber: bigint): Promise<boolean>;
+  handleEpochReadyToProve(epochNumber: EpochNumber): Promise<boolean>;
 }
 
 /**
@@ -31,21 +27,24 @@ export class EpochMonitor implements Traceable {
   public readonly tracer: Tracer;
 
   private handler: EpochMonitorHandler | undefined;
-  private latestEpochNumber: bigint | undefined;
+  private latestEpochNumber: EpochNumber | undefined;
 
   constructor(
     private readonly l2BlockSource: L2BlockSource,
     private readonly l1Constants: Pick<L1RollupConstants, 'epochDuration'>,
-    private options: { pollingIntervalMs: number },
+    private options: { pollingIntervalMs: number; provingDelayMs?: number },
     telemetry: TelemetryClient = getTelemetryClient(),
   ) {
     this.tracer = telemetry.getTracer('EpochMonitor');
     this.runningPromise = new RunningPromise(this.work.bind(this), this.log, this.options.pollingIntervalMs);
+    if (this.options.provingDelayMs) {
+      this.log.warn(`Prover node epoch monitor running with delay of ${this.options.provingDelayMs}ms`);
+    }
   }
 
   public static async create(
     l2BlockSource: L2BlockSource,
-    options: { pollingIntervalMs: number },
+    options: { pollingIntervalMs: number; provingDelayMs?: number },
     telemetry: TelemetryClient = getTelemetryClient(),
   ): Promise<EpochMonitor> {
     const l1Constants = await l2BlockSource.getL1Constants();
@@ -68,7 +67,6 @@ export class EpochMonitor implements Traceable {
     this.log.info('Stopped EpochMonitor');
   }
 
-  @trackSpan('EpochMonitor.work')
   public async work() {
     const { epochToProve, blockNumber, slotNumber } = await this.getEpochNumberToProve();
     this.log.debug(`Epoch to prove: ${epochToProve}`, { blockNumber, slotNumber });
@@ -87,6 +85,11 @@ export class EpochMonitor implements Traceable {
       return;
     }
 
+    if (this.options.provingDelayMs) {
+      this.log.debug(`Waiting ${this.options.provingDelayMs}ms before proving epoch ${epochToProve}`);
+      await sleep(this.options.provingDelayMs);
+    }
+
     this.log.debug(`Epoch ${epochToProve} is ready to be proven`);
     if (await this.handler?.handleEpochReadyToProve(epochToProve)) {
       this.latestEpochNumber = epochToProve;
@@ -95,7 +98,7 @@ export class EpochMonitor implements Traceable {
 
   private async getEpochNumberToProve() {
     const lastBlockProven = await this.l2BlockSource.getProvenBlockNumber();
-    const firstBlockToProve = lastBlockProven + 1;
+    const firstBlockToProve = BlockNumber(lastBlockProven + 1);
     const firstBlockHeaderToProve = await this.l2BlockSource.getBlockHeader(firstBlockToProve);
     if (!firstBlockHeaderToProve) {
       return { epochToProve: undefined, blockNumber: firstBlockToProve };