@aztec/prover-node 0.0.0-test.1 → 0.0.1-commit.b655e406

package/src/index.ts

@@ -1,5 +1,5 @@
+export * from './actions/index.js';
 export * from './config.js';
 export * from './factory.js';
-export * from './http.js';
 export * from './prover-node-publisher.js';
 export * from './prover-node.js';

package/src/job/epoch-proving-job-data.ts

@@ -0,0 +1,76 @@
+import { Fr } from '@aztec/foundation/fields';
+import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize';
+import { CommitteeAttestation, L2Block } from '@aztec/stdlib/block';
+import { BlockHeader, Tx } from '@aztec/stdlib/tx';
+
+/** All data from an epoch used in proving. */
+export type EpochProvingJobData = {
+  epochNumber: bigint;
+  blocks: L2Block[];
+  txs: Map<string, Tx>;
+  l1ToL2Messages: Record<number, Fr[]>;
+  previousBlockHeader: BlockHeader;
+  attestations: CommitteeAttestation[];
+};
+
+export function validateEpochProvingJobData(data: EpochProvingJobData) {
+  if (data.blocks.length > 0 && data.previousBlockHeader.getBlockNumber() + 1 !== data.blocks[0].number) {
+    throw new Error(
+      `Initial block number ${
+        data.blocks[0].number
+      } does not match previous block header ${data.previousBlockHeader.getBlockNumber()}`,
+    );
+  }
+
+  for (const blockNumber of data.blocks.map(block => block.number)) {
+    if (!(blockNumber in data.l1ToL2Messages)) {
+      throw new Error(`Missing L1 to L2 messages for block number ${blockNumber}`);
+    }
+  }
+}
+
+export function serializeEpochProvingJobData(data: EpochProvingJobData): Buffer {
+  const blocks = data.blocks.map(block => block.toBuffer());
+  const txs = Array.from(data.txs.values()).map(tx => tx.toBuffer());
+  const l1ToL2Messages = Object.entries(data.l1ToL2Messages).map(([blockNumber, messages]) => [
+    Number(blockNumber),
+    messages.length,
+    ...messages,
+  ]);
+  const attestations = data.attestations.map(attestation => attestation.toBuffer());
+
+  return serializeToBuffer(
+    Number(data.epochNumber),
+    data.previousBlockHeader,
+    blocks.length,
+    ...blocks,
+    txs.length,
+    ...txs,
+    l1ToL2Messages.length,
+    ...l1ToL2Messages,
+    attestations.length,
+    ...attestations,
+  );
+}
+
+export function deserializeEpochProvingJobData(buf: Buffer): EpochProvingJobData {
+  const reader = BufferReader.asReader(buf);
+  const epochNumber = BigInt(reader.readNumber());
+  const previousBlockHeader = reader.readObject(BlockHeader);
+  const blocks = reader.readVector(L2Block);
+  const txArray = reader.readVector(Tx);
+
+  const l1ToL2MessageBlockCount = reader.readNumber();
+  const l1ToL2Messages: Record<number, Fr[]> = {};
+  for (let i = 0; i < l1ToL2MessageBlockCount; i++) {
+    const blockNumber = reader.readNumber();
+    const messages = reader.readVector(Fr);
+    l1ToL2Messages[blockNumber] = messages;
+  }
+
+  const attestations = reader.readVector(CommitteeAttestation);
+
+  const txs = new Map<string, Tx>(txArray.map(tx => [tx.getTxHash().toString(), tx]));
+
+  return { epochNumber, previousBlockHeader, blocks, txs, l1ToL2Messages, attestations };
+}

package/src/job/epoch-proving-job.ts

@@ -1,7 +1,13 @@
+import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
 import { asyncPool } from '@aztec/foundation/async-pool';
+import { padArrayEnd } from '@aztec/foundation/collection';
+import { Fr } from '@aztec/foundation/fields';
 import { createLogger } from '@aztec/foundation/log';
-import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 import { Timer } from '@aztec/foundation/timer';
+import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
+import { protocolContractsHash } from '@aztec/protocol-contracts';
+import { buildFinalBlobChallenges } from '@aztec/prover-client/helpers';
 import type { PublicProcessor, PublicProcessorFactory } from '@aztec/simulator/server';
 import type { L2Block, L2BlockSource } from '@aztec/stdlib/block';
 import {
@@ -10,17 +16,25 @@ import {
   EpochProvingJobTerminalState,
   type ForkMerkleTreeOperations,
 } from '@aztec/stdlib/interfaces/server';
-import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
+import { CheckpointConstantData } from '@aztec/stdlib/rollup';
+import { MerkleTreeId } from '@aztec/stdlib/trees';
 import type { ProcessedTx, Tx } from '@aztec/stdlib/tx';
 import { Attributes, type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client';
 
 import * as crypto from 'node:crypto';
 
-import type { ProverNodeMetrics } from '../metrics.js';
+import type { ProverNodeJobMetrics } from '../metrics.js';
 import type { ProverNodePublisher } from '../prover-node-publisher.js';
+import { type EpochProvingJobData, validateEpochProvingJobData } from './epoch-proving-job-data.js';
+
+export type EpochProvingJobOptions = {
+  parallelBlockLimit?: number;
+  skipEpochCheck?: boolean;
+  skipSubmitProof?: boolean;
+};
 
 /**
- * Job that grabs a range of blocks from the unfinalised chain from L1, gets their txs given their hashes,
+ * Job that grabs a range of blocks from the unfinalized chain from L1, gets their txs given their hashes,
  * re-executes their public calls, generates a rollup proof, and submits it to L1. This job will update the
  * world state as part of public call execution via the public processor.
  */
@@ -30,27 +44,25 @@ export class EpochProvingJob implements Traceable {
   private uuid: string;
 
   private runPromise: Promise<void> | undefined;
+  private epochCheckPromise: RunningPromise | undefined;
   private deadlineTimeoutHandler: NodeJS.Timeout | undefined;
 
   public readonly tracer: Tracer;
 
   constructor(
-    private dbProvider: ForkMerkleTreeOperations,
-    private epochNumber: bigint,
-    private blocks: L2Block[],
-    private txs: Tx[],
+    private data: EpochProvingJobData,
+    private dbProvider: Pick<ForkMerkleTreeOperations, 'fork'>,
     private prover: EpochProver,
     private publicProcessorFactory: PublicProcessorFactory,
-    private publisher: ProverNodePublisher,
-    private l2BlockSource: L2BlockSource,
-    private l1ToL2MessageSource: L1ToL2MessageSource,
-    private metrics: ProverNodeMetrics,
+    private publisher: Pick<ProverNodePublisher, 'submitEpochProof'>,
+    private l2BlockSource: L2BlockSource | undefined,
+    private metrics: ProverNodeJobMetrics,
     private deadline: Date | undefined,
-    private config: { parallelBlockLimit: number } = { parallelBlockLimit: 32 },
-    private cleanUp: (job: EpochProvingJob) => Promise<void> = () => Promise.resolve(),
+    private config: EpochProvingJobOptions,
   ) {
+    validateEpochProvingJobData(data);
     this.uuid = crypto.randomUUID();
-    this.tracer = metrics.client.getTracer('EpochProvingJob');
+    this.tracer = metrics.tracer;
   }
 
   public getId(): string {
@@ -62,18 +74,46 @@ export class EpochProvingJob implements Traceable {
   }
 
   public getEpochNumber(): bigint {
-    return this.epochNumber;
+    return this.data.epochNumber;
+  }
+
+  public getDeadline(): Date | undefined {
+    return this.deadline;
+  }
+
+  public getProvingData(): EpochProvingJobData {
+    return this.data;
+  }
+
+  private get epochNumber() {
+    return this.data.epochNumber;
+  }
+
+  private get blocks() {
+    return this.data.blocks;
+  }
+
+  private get txs() {
+    return this.data.txs;
+  }
+
+  private get attestations() {
+    return this.data.attestations;
   }
 
   /**
    * Proves the given epoch and submits the proof to L1.
    */
   @trackSpan('EpochProvingJob.run', function () {
-    return { [Attributes.EPOCH_NUMBER]: Number(this.epochNumber) };
+    return { [Attributes.EPOCH_NUMBER]: Number(this.data.epochNumber) };
   })
   public async run() {
     this.scheduleDeadlineStop();
+    if (!this.config.skipEpochCheck) {
+      await this.scheduleEpochCheck();
+    }
 
+    const attestations = this.attestations.map(attestation => attestation.toViem());
     const epochNumber = Number(this.epochNumber);
     const epochSizeBlocks = this.blocks.length;
     const epochSizeTxs = this.blocks.reduce((total, current) => total + current.body.txEffects.length, 0);
@@ -92,16 +132,23 @@ export class EpochProvingJob implements Traceable {
     this.runPromise = promise;
 
     try {
-      this.prover.startNewEpoch(epochNumber, fromBlock, epochSizeBlocks);
-      await this.prover.startTubeCircuits(this.txs);
+      const blobFieldsPerCheckpoint = this.blocks.map(block => block.getCheckpointBlobFields());
+      const finalBlobBatchingChallenges = await buildFinalBlobChallenges(blobFieldsPerCheckpoint);
 
-      await asyncPool(this.config.parallelBlockLimit, this.blocks, async block => {
+      // TODO(#17027): Enable multiple blocks per checkpoint.
+      // Total number of checkpoints equals number of blocks because we currently build a checkpoint with only one block.
+      const totalNumCheckpoints = epochSizeBlocks;
+
+      this.prover.startNewEpoch(epochNumber, totalNumCheckpoints, finalBlobBatchingChallenges);
+      await this.prover.startChonkVerifierCircuits(Array.from(this.txs.values()));
+
+      await asyncPool(this.config.parallelBlockLimit ?? 32, this.blocks, async block => {
         this.checkState();
 
         const globalVariables = block.header.globalVariables;
-        const txs = await this.getTxs(block);
-        const l1ToL2Messages = await this.getL1ToL2Messages(block);
-        const previousHeader = (await this.getBlockHeader(block.number - 1))!;
+        const txs = this.getTxs(block);
+        const l1ToL2Messages = this.getL1ToL2Messages(block);
+        const previousHeader = this.getBlockHeader(block.number - 1)!;
 
         this.log.verbose(`Starting processing block ${block.number}`, {
           number: block.number,
@@ -115,12 +162,41 @@ export class EpochProvingJob implements Traceable {
           ...globalVariables,
         });
 
+        const checkpointConstants = CheckpointConstantData.from({
+          chainId: globalVariables.chainId,
+          version: globalVariables.version,
+          vkTreeRoot: getVKTreeRoot(),
+          protocolContractsHash: protocolContractsHash,
+          proverId: this.prover.getProverId().toField(),
+          slotNumber: globalVariables.slotNumber,
+          coinbase: globalVariables.coinbase,
+          feeRecipient: globalVariables.feeRecipient,
+          gasFees: globalVariables.gasFees,
+        });
+
+        // TODO(#17027): Enable multiple blocks per checkpoint.
+        // Each checkpoint has only one block.
+        const totalNumBlocks = 1;
+        const checkpointIndex = block.number - fromBlock;
+        await this.prover.startNewCheckpoint(
+          checkpointIndex,
+          checkpointConstants,
+          l1ToL2Messages,
+          totalNumBlocks,
+          blobFieldsPerCheckpoint[checkpointIndex].length,
+          previousHeader,
+        );
+
         // Start block proving
-        await this.prover.startNewBlock(globalVariables, l1ToL2Messages, previousHeader);
+        await this.prover.startNewBlock(block.number, globalVariables.timestamp, txs.length);
 
         // Process public fns
-        const db = await this.dbProvider.fork(block.number - 1);
-        const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, true);
+        const db = await this.createFork(block.number - 1, l1ToL2Messages);
+        const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, {
+          skipFeeEnforcement: true,
+          clientInitiatedSimulation: false,
+          proverId: this.prover.getProverId().toField(),
+        });
         const processed = await this.processTxs(publicProcessor, txs);
         await this.prover.addTxs(processed);
         await db.close();
@@ -131,17 +207,36 @@ export class EpochProvingJob implements Traceable {
         });
 
         // Mark block as completed to pad it
-        await this.prover.setBlockCompleted(block.number, block.header);
+        const expectedBlockHeader = block.getBlockHeader();
+        await this.prover.setBlockCompleted(block.number, expectedBlockHeader);
       });
 
       const executionTime = timer.ms();
 
       this.progressState('awaiting-prover');
-      const { publicInputs, proof } = await this.prover.finaliseEpoch();
-      this.log.info(`Finalised proof for epoch ${epochNumber}`, { epochNumber, uuid: this.uuid, duration: timer.ms() });
+      const { publicInputs, proof, batchedBlobInputs } = await this.prover.finalizeEpoch();
+      this.log.info(`Finalized proof for epoch ${epochNumber}`, { epochNumber, uuid: this.uuid, duration: timer.ms() });
 
       this.progressState('publishing-proof');
-      const success = await this.publisher.submitEpochProof({ fromBlock, toBlock, epochNumber, publicInputs, proof });
+
+      if (this.config.skipSubmitProof) {
+        this.log.info(
+          `Proof publishing is disabled. Dropping valid proof for epoch ${epochNumber} (blocks ${fromBlock} to ${toBlock})`,
+        );
+        this.state = 'completed';
+        this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeBlocks, epochSizeTxs);
+        return;
+      }
+
+      const success = await this.publisher.submitEpochProof({
+        fromBlock,
+        toBlock,
+        epochNumber,
+        publicInputs,
+        proof,
+        batchedBlobInputs,
+        attestations,
+      });
       if (!success) {
         throw new Error('Failed to submit epoch proof to L1');
       }
@@ -154,34 +249,59 @@ export class EpochProvingJob implements Traceable {
       this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeBlocks, epochSizeTxs);
     } catch (err: any) {
       if (err && err.name === 'HaltExecutionError') {
-        this.log.warn(`Halted execution of epoch ${epochNumber} prover job`, { uuid: this.uuid, epochNumber });
+        this.log.warn(`Halted execution of epoch ${epochNumber} prover job`, {
+          uuid: this.uuid,
+          epochNumber,
+          details: err.message,
+        });
         return;
       }
       this.log.error(`Error running epoch ${epochNumber} prover job`, err, { uuid: this.uuid, epochNumber });
-      this.state = 'failed';
+      if (this.state === 'processing' || this.state === 'awaiting-prover' || this.state === 'publishing-proof') {
+        this.state = 'failed';
+      }
     } finally {
       clearTimeout(this.deadlineTimeoutHandler);
-      await this.cleanUp(this);
+      await this.epochCheckPromise?.stop();
       await this.prover.stop();
       resolve();
     }
   }
 
+  /**
+   * Create a new db fork for tx processing, inserting all L1 to L2.
+   * REFACTOR: The prover already spawns a db fork of its own for each block, so we may be able to do away with just one fork.
+   */
+  private async createFork(blockNumber: number, l1ToL2Messages: Fr[]) {
+    const db = await this.dbProvider.fork(blockNumber);
+    const l1ToL2MessagesPadded = padArrayEnd<Fr, number>(
+      l1ToL2Messages,
+      Fr.ZERO,
+      NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP,
+      'Too many L1 to L2 messages',
+    );
+    this.log.verbose(`Creating fork at ${blockNumber} with ${l1ToL2Messages.length} L1 to L2 messages`, {
+      blockNumber,
+      l1ToL2Messages: l1ToL2MessagesPadded.map(m => m.toString()),
+    });
+    await db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded);
+    return db;
+  }
+
   private progressState(state: EpochProvingJobState) {
     this.checkState();
     this.state = state;
   }
 
   private checkState() {
-    if (this.state === 'timed-out' || this.state === 'stopped' || this.state === 'failed') {
+    if (this.state === 'timed-out' || this.state === 'stopped' || this.state === 'failed' || this.state === 'reorg') {
       throw new HaltExecutionError(this.state);
     }
   }
 
-  public async stop(state: EpochProvingJobState = 'stopped') {
+  public async stop(state: EpochProvingJobTerminalState = 'stopped') {
     this.state = state;
     this.prover.cancel();
-    // TODO(palla/prover): Stop the publisher as well
     if (this.runPromise) {
       await this.runPromise;
     }
@@ -207,24 +327,66 @@ export class EpochProvingJob implements Traceable {
     }
   }
 
-  /* Returns the header for the given block number, or the genesis block for block zero. */
-  private async getBlockHeader(blockNumber: number) {
-    if (blockNumber === 0) {
-      return (await this.dbProvider.fork()).getInitialHeader();
+  /**
+   * Kicks off a running promise that queries the archiver for the set of L2 blocks of the current epoch.
+   * If those change, stops the proving job with a `rerun` state, so the node re-enqueues it.
+   */
+  private async scheduleEpochCheck() {
+    const l2BlockSource = this.l2BlockSource;
+    if (!l2BlockSource) {
+      this.log.warn(`No L2 block source available, skipping epoch check`);
+      return;
     }
-    return this.l2BlockSource.getBlockHeader(blockNumber);
+
+    const intervalMs = Math.ceil((await l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
+    this.epochCheckPromise = new RunningPromise(
+      async () => {
+        const blocks = await l2BlockSource.getBlockHeadersForEpoch(this.epochNumber);
+        const blockHashes = await Promise.all(blocks.map(block => block.hash()));
+        const thisBlockHashes = await Promise.all(this.blocks.map(block => block.hash()));
+        if (
+          blocks.length !== this.blocks.length ||
+          !blockHashes.every((block, i) => block.equals(thisBlockHashes[i]))
+        ) {
+          this.log.warn('Epoch blocks changed underfoot', {
+            uuid: this.uuid,
+            epochNumber: this.epochNumber,
+            oldBlockHashes: thisBlockHashes,
+            newBlockHashes: blockHashes,
+          });
+          void this.stop('reorg');
+        }
+      },
+      this.log,
+      intervalMs,
+    ).start();
+    this.log.verbose(`Scheduled epoch check for epoch ${this.epochNumber} every ${intervalMs}ms`);
+  }
+
+  /* Returns the header for the given block number based on the epoch proving job data. */
+  private getBlockHeader(blockNumber: number) {
+    const block = this.blocks.find(b => b.number === blockNumber);
+    if (block) {
+      return block.getBlockHeader();
+    }
+
+    if (blockNumber === Number(this.data.previousBlockHeader.getBlockNumber())) {
+      return this.data.previousBlockHeader;
+    }
+
+    throw new Error(
+      `Block header not found for block number ${blockNumber} (got ${this.blocks
+        .map(b => b.number)
+        .join(', ')} and previous header ${this.data.previousBlockHeader.getBlockNumber()})`,
+    );
   }
 
-  private async getTxs(block: L2Block): Promise<Tx[]> {
-    const txHashes = block.body.txEffects.map(tx => tx.txHash.toBigInt());
-    const txsAndHashes = await Promise.all(this.txs.map(async tx => ({ tx, hash: await tx.getTxHash() })));
-    return txsAndHashes
-      .filter(txAndHash => txHashes.includes(txAndHash.hash.toBigInt()))
-      .map(txAndHash => txAndHash.tx);
+  private getTxs(block: L2Block): Tx[] {
+    return block.body.txEffects.map(txEffect => this.txs.get(txEffect.txHash.toString())!);
   }
 
   private getL1ToL2Messages(block: L2Block) {
-    return this.l1ToL2MessageSource.getL1ToL2Messages(BigInt(block.number));
+    return this.data.l1ToL2Messages[block.number];
   }
 
   private async processTxs(publicProcessor: PublicProcessor, txs: Tx[]): Promise<ProcessedTx[]> {
@@ -232,8 +394,11 @@ export class EpochProvingJob implements Traceable {
     const [processedTxs, failedTxs] = await publicProcessor.process(txs, { deadline });
 
     if (failedTxs.length) {
+      const failedTxHashes = await Promise.all(failedTxs.map(({ tx }) => tx.getTxHash()));
       throw new Error(
-        `Txs failed processing: ${failedTxs.map(({ tx, error }) => `${tx.getTxHash()} (${error})`).join(', ')}`,
+        `Txs failed processing: ${failedTxs
+          .map(({ error }, index) => `${failedTxHashes[index]} (${error})`)
+          .join(', ')}`,
       );
     }
 
@@ -246,7 +411,7 @@ export class EpochProvingJob implements Traceable {
 }
 
 class HaltExecutionError extends Error {
-  constructor(state: EpochProvingJobState) {
+  constructor(public readonly state: EpochProvingJobState) {
     super(`Halted execution due to state ${state}`);
     this.name = 'HaltExecutionError';
   }