@aztec/prover-node 0.86.0 → 0.87.0

package/src/job/epoch-proving-job.ts

@@ -10,7 +10,6 @@ import {
   EpochProvingJobTerminalState,
   type ForkMerkleTreeOperations,
 } from '@aztec/stdlib/interfaces/server';
-import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
 import type { ProcessedTx, Tx } from '@aztec/stdlib/tx';
 import { Attributes, type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client';
 
@@ -18,6 +17,7 @@ import * as crypto from 'node:crypto';
 
 import type { ProverNodeJobMetrics } from '../metrics.js';
 import type { ProverNodePublisher } from '../prover-node-publisher.js';
+import { type EpochProvingJobData, validateEpochProvingJobData } from './epoch-proving-job-data.js';
 
 /**
  * Job that grabs a range of blocks from the unfinalised chain from L1, gets their txs given their hashes,
@@ -36,19 +36,17 @@ export class EpochProvingJob implements Traceable {
   public readonly tracer: Tracer;
 
   constructor(
-    private dbProvider: ForkMerkleTreeOperations,
-    private epochNumber: bigint,
-    private blocks: L2Block[],
-    private txs: Tx[],
+    private data: EpochProvingJobData,
+    private dbProvider: Pick<ForkMerkleTreeOperations, 'fork'>,
     private prover: EpochProver,
     private publicProcessorFactory: PublicProcessorFactory,
-    private publisher: ProverNodePublisher,
-    private l2BlockSource: L2BlockSource,
-    private l1ToL2MessageSource: L1ToL2MessageSource,
+    private publisher: Pick<ProverNodePublisher, 'submitEpochProof'>,
+    private l2BlockSource: L2BlockSource | undefined,
     private metrics: ProverNodeJobMetrics,
     private deadline: Date | undefined,
-    private config: { parallelBlockLimit: number } = { parallelBlockLimit: 32 },
+    private config: { parallelBlockLimit?: number; skipEpochCheck?: boolean },
   ) {
+    validateEpochProvingJobData(data);
     this.uuid = crypto.randomUUID();
     this.tracer = metrics.tracer;
   }
@@ -62,22 +60,40 @@ export class EpochProvingJob implements Traceable {
   }
 
   public getEpochNumber(): bigint {
-    return this.epochNumber;
+    return this.data.epochNumber;
   }
 
   public getDeadline(): Date | undefined {
     return this.deadline;
   }
 
+  public getProvingData(): EpochProvingJobData {
+    return this.data;
+  }
+
+  private get epochNumber() {
+    return this.data.epochNumber;
+  }
+
+  private get blocks() {
+    return this.data.blocks;
+  }
+
+  private get txs() {
+    return this.data.txs;
+  }
+
   /**
    * Proves the given epoch and submits the proof to L1.
    */
   @trackSpan('EpochProvingJob.run', function () {
-    return { [Attributes.EPOCH_NUMBER]: Number(this.epochNumber) };
+    return { [Attributes.EPOCH_NUMBER]: Number(this.data.epochNumber) };
   })
   public async run() {
     this.scheduleDeadlineStop();
-    await this.scheduleEpochCheck();
+    if (!this.config.skipEpochCheck) {
+      await this.scheduleEpochCheck();
+    }
 
     const epochNumber = Number(this.epochNumber);
     const epochSizeBlocks = this.blocks.length;
@@ -100,13 +116,13 @@ export class EpochProvingJob implements Traceable {
       this.prover.startNewEpoch(epochNumber, fromBlock, epochSizeBlocks);
       await this.prover.startTubeCircuits(this.txs);
 
-      await asyncPool(this.config.parallelBlockLimit, this.blocks, async block => {
+      await asyncPool(this.config.parallelBlockLimit ?? 32, this.blocks, async block => {
         this.checkState();
 
         const globalVariables = block.header.globalVariables;
         const txs = await this.getTxs(block);
-        const l1ToL2Messages = await this.getL1ToL2Messages(block);
-        const previousHeader = (await this.getBlockHeader(block.number - 1))!;
+        const l1ToL2Messages = this.getL1ToL2Messages(block);
+        const previousHeader = this.getBlockHeader(block.number - 1)!;
 
         this.log.verbose(`Starting processing block ${block.number}`, {
           number: block.number,
@@ -221,10 +237,16 @@ export class EpochProvingJob implements Traceable {
    * If those change, stops the proving job with a `rerun` state, so the node re-enqueues it.
    */
   private async scheduleEpochCheck() {
-    const intervalMs = Math.ceil((await this.l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
+    const l2BlockSource = this.l2BlockSource;
+    if (!l2BlockSource) {
+      this.log.warn(`No L2 block source available, skipping epoch check`);
+      return;
+    }
+
+    const intervalMs = Math.ceil((await l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
     this.epochCheckPromise = new RunningPromise(
       async () => {
-        const blocks = await this.l2BlockSource.getBlockHeadersForEpoch(this.epochNumber);
+        const blocks = await l2BlockSource.getBlockHeadersForEpoch(this.epochNumber);
         const blockHashes = await Promise.all(blocks.map(block => block.hash()));
         const thisBlockHashes = await Promise.all(this.blocks.map(block => block.hash()));
         if (
@@ -246,12 +268,22 @@ export class EpochProvingJob implements Traceable {
     this.log.verbose(`Scheduled epoch check for epoch ${this.epochNumber} every ${intervalMs}ms`);
   }
 
-  /* Returns the header for the given block number, or the genesis block for block zero. */
-  private async getBlockHeader(blockNumber: number) {
-    if (blockNumber === 0) {
-      return (await this.dbProvider.fork()).getInitialHeader();
+  /* Returns the header for the given block number based on the epoch proving job data. */
+  private getBlockHeader(blockNumber: number) {
+    const block = this.blocks.find(b => b.number === blockNumber);
+    if (block) {
+      return block.header;
     }
-    return this.l2BlockSource.getBlockHeader(blockNumber);
+
+    if (blockNumber === Number(this.data.previousBlockHeader.getBlockNumber())) {
+      return this.data.previousBlockHeader;
+    }
+
+    throw new Error(
+      `Block header not found for block number ${blockNumber} (got ${this.blocks
+        .map(b => b.number)
+        .join(', ')} and previous header ${this.data.previousBlockHeader.getBlockNumber()})`,
+    );
   }
 
   private async getTxs(block: L2Block): Promise<Tx[]> {
@@ -263,7 +295,7 @@ export class EpochProvingJob implements Traceable {
   }
 
   private getL1ToL2Messages(block: L2Block) {
-    return this.l1ToL2MessageSource.getL1ToL2Messages(BigInt(block.number));
+    return this.data.l1ToL2Messages[block.number];
   }
 
   private async processTxs(publicProcessor: PublicProcessor, txs: Tx[]): Promise<ProcessedTx[]> {

package/src/metrics.ts

@@ -242,7 +242,7 @@ export class ProverNodePublisherMetrics {
 
     try {
       this.gasPrice.record(parseInt(formatEther(stats.gasPrice, 'gwei'), 10));
-    } catch (e) {
+    } catch {
       // ignore
     }
 
@@ -252,7 +252,7 @@ export class ProverNodePublisherMetrics {
 
     try {
       this.txTotalFee.record(parseFloat(formatEther(totalFee)));
-    } catch (e) {
+    } catch {
       // ignore
     }
   }

package/src/prover-coordination/combined-prover-coordination.ts

@@ -94,7 +94,7 @@ export class CombinedProverCoordination implements ProverCoordination {
     const notFound = txHashes.filter((_, index) => !availability[index]);
     const txsToFind = new Set(notFound.map(tx => tx.toString()));
     if (txsToFind.size === 0) {
-      this.log.info(`Check for ${txHashes.length} txs found all in the pool}`);
+      this.log.info(`Check for ${txHashes.length} txs found all in the pool`);
       return;
     }
     this.log.info(`Check for ${txHashes.length} txs found ${txsToFind.size} missing. Will gather from nodes and p2p`);
@@ -107,16 +107,19 @@ export class CombinedProverCoordination implements ProverCoordination {
     const toFindFromP2P = txsToFind.size;
     this.log.verbose(`Gathering ${toFindFromP2P} txs from p2p network`);
     const foundFromP2P = await pool.getTxsByHash([...txsToFind].map(tx => TxHash.fromString(tx)));
+
+    // TODO(!!): test for this
+    // getTxsByHash returns undefined for transactions that are not found, so it must be filtered to find the true length
+    const foundFromP2PLength = foundFromP2P.filter(tx => !!tx).length;
+
     const numFoundFromNodes = originalToFind - toFindFromP2P;
-    const numNotFound = toFindFromP2P - foundFromP2P.length;
+    const numNotFound = toFindFromP2P - foundFromP2PLength;
     if (numNotFound === 0) {
-      this.log.info(
-        `Found all ${originalToFind} txs. ${numFoundFromNodes} from nodes, ${foundFromP2P.length} from p2p`,
-      );
+      this.log.info(`Found all ${originalToFind} txs. ${numFoundFromNodes} from nodes, ${foundFromP2PLength} from p2p`);
       return;
     }
     this.log.warn(
-      `Failed to find ${numNotFound} txs from any source. Found ${foundFromP2P.length} from p2p and ${numFoundFromNodes} from nodes`,
+      `Failed to find ${numNotFound} txs from any source. Found ${foundFromP2PLength} from p2p and ${numFoundFromNodes} from nodes`,
     );
   }
 

package/src/prover-coordination/config.ts

@@ -9,6 +9,7 @@ export const proverCoordinationConfigMappings: ConfigMappingsType<ProverCoordina
     env: 'PROVER_COORDINATION_NODE_URLS',
     description: 'The URLs of the tx provider nodes',
     parseEnv: (val: string) => val.split(',').map(url => url.trim().replace(/\/$/, '')),
+    defaultValue: [],
   },
 };
 

package/src/prover-coordination/factory.ts

@@ -9,6 +9,7 @@ import { protocolContractTreeRoot } from '@aztec/protocol-contracts';
 import { type AztecNode, createAztecNodeClient } from '@aztec/stdlib/interfaces/client';
 import type { ProverCoordination, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
 import { P2PClientType } from '@aztec/stdlib/p2p';
+import { getPackageVersion } from '@aztec/stdlib/update-checker';
 import { getComponentsVersionsFromConfig } from '@aztec/stdlib/versioning';
 import { type TelemetryClient, makeTracedFetch } from '@aztec/telemetry-client';
 
@@ -59,12 +60,13 @@ export async function createProverCoordination(
   }
 
   let nodes: AztecNode[] = [];
-  if (config.proverCoordinationNodeUrls.length > 0) {
+  if (config.proverCoordinationNodeUrls && config.proverCoordinationNodeUrls.length > 0) {
     log.info('Using prover coordination via node urls');
     const versions = getComponentsVersionsFromConfig(config, protocolContractTreeRoot, getVKTreeRoot());
-    nodes = config.proverCoordinationNodeUrls.map(url =>
-      createAztecNodeClient(url, versions, makeTracedFetch([1, 2, 3], false)),
-    );
+    nodes = config.proverCoordinationNodeUrls.map(url => {
+      log.info(`Creating aztec node client for prover coordination with url: ${url}`);
+      return createAztecNodeClient(url, versions, makeTracedFetch([1, 2, 3], false));
+    });
   }
 
   const proofVerifier = config.realProofs ? await BBCircuitVerifier.new(config) : new TestCircuitVerifier();
@@ -75,6 +77,7 @@ export async function createProverCoordination(
     proofVerifier,
     deps.worldStateSynchronizer,
     deps.epochCache,
+    getPackageVersion() ?? '',
     deps.telemetry,
   );
   await p2pClient.start();

package/src/prover-node.ts

@@ -1,13 +1,16 @@
+import type { Archiver } from '@aztec/archiver';
 import type { ViemPublicClient } from '@aztec/ethereum';
-import { compact } from '@aztec/foundation/collection';
+import { assertRequired, compact, pick, sum } from '@aztec/foundation/collection';
 import { memoize } from '@aztec/foundation/decorators';
 import { EthAddress } from '@aztec/foundation/eth-address';
+import type { Fr } from '@aztec/foundation/fields';
 import { createLogger } from '@aztec/foundation/log';
 import { RunningPromise } from '@aztec/foundation/running-promise';
 import { DateProvider } from '@aztec/foundation/timer';
-import type { Maybe } from '@aztec/foundation/types';
+import type { DataStoreConfig } from '@aztec/kv-store/config';
 import { PublicProcessorFactory } from '@aztec/simulator/server';
 import type { L2Block, L2BlockSource } from '@aztec/stdlib/block';
+import type { ChainConfig } from '@aztec/stdlib/config';
 import type { ContractDataSource } from '@aztec/stdlib/contract';
 import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
 import {
@@ -21,7 +24,7 @@ import {
   tryStop,
 } from '@aztec/stdlib/interfaces/server';
 import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
-import type { Tx, TxHash } from '@aztec/stdlib/tx';
+import type { TxHash } from '@aztec/stdlib/tx';
 import {
   Attributes,
   L1Metrics,
@@ -32,22 +35,20 @@ import {
   trackSpan,
 } from '@aztec/telemetry-client';
 
+import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
+import type { SpecificProverNodeConfig } from './config.js';
+import type { EpochProvingJobData } from './job/epoch-proving-job-data.js';
 import { EpochProvingJob, type EpochProvingJobState } from './job/epoch-proving-job.js';
 import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
 import type { EpochMonitor, EpochMonitorHandler } from './monitors/epoch-monitor.js';
 import type { ProverNodePublisher } from './prover-node-publisher.js';
 
-export type ProverNodeOptions = {
-  pollingIntervalMs: number;
-  maxPendingJobs: number;
-  maxParallelBlocksPerEpoch: number;
-  txGatheringIntervalMs: number;
-};
+type ProverNodeOptions = SpecificProverNodeConfig & Partial<DataStoreOptions>;
+type DataStoreOptions = Pick<DataStoreConfig, 'dataDirectory'> & Pick<ChainConfig, 'l1ChainId' | 'rollupVersion'>;
 
 /**
- * An Aztec Prover Node is a standalone process that monitors the unfinalised chain on L1 for unproven blocks,
- * submits bids for proving them, and monitors if they are accepted. If so, the prover node fetches the txs
- * from a tx source in the p2p network or an external node, re-executes their public functions, creates a rollup
+ * An Aztec Prover Node is a standalone process that monitors the unfinalised chain on L1 for unproven epochs,
+ * fetches their txs from the p2p network or external nodes, re-executes their public functions, creates a rollup
  * proof for the epoch, and submits it to L1.
  */
 export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable {
@@ -55,7 +56,7 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
   private dateProvider = new DateProvider();
 
   private jobs: Map<string, EpochProvingJob> = new Map();
-  private options: ProverNodeOptions;
+  private config: ProverNodeOptions;
   private jobMetrics: ProverNodeJobMetrics;
   private rewardsMetrics: ProverNodeRewardsMetrics;
   private l1Metrics: L1Metrics;
@@ -68,13 +69,13 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
   constructor(
     protected readonly prover: EpochProverManager,
     protected readonly publisher: ProverNodePublisher,
-    protected readonly l2BlockSource: L2BlockSource & Maybe<Service>,
+    protected readonly l2BlockSource: L2BlockSource & Partial<Service>,
     protected readonly l1ToL2MessageSource: L1ToL2MessageSource,
     protected readonly contractDataSource: ContractDataSource,
     protected readonly worldState: WorldStateSynchronizer,
-    protected readonly coordination: ProverCoordination,
+    protected readonly coordination: ProverCoordination & Partial<Service>,
     protected readonly epochsMonitor: EpochMonitor,
-    options: Partial<ProverNodeOptions> = {},
+    config: Partial<ProverNodeOptions> = {},
     protected readonly telemetryClient: TelemetryClient = getTelemetryClient(),
   ) {
     this.l1Metrics = new L1Metrics(
@@ -83,14 +84,19 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
       [publisher.getSenderAddress()],
     );
 
-    this.options = {
-      pollingIntervalMs: 1_000,
-      maxPendingJobs: 100,
-      maxParallelBlocksPerEpoch: 32,
+    this.config = {
+      proverNodePollingIntervalMs: 1_000,
+      proverNodeMaxPendingJobs: 100,
+      proverNodeMaxParallelBlocksPerEpoch: 32,
       txGatheringIntervalMs: 1_000,
-      ...compact(options),
+      txGatheringBatchSize: 10,
+      txGatheringMaxParallelRequestsPerNode: 100,
+      proverNodeFailedEpochStore: undefined,
+      ...compact(config),
     };
 
+    this.validateConfig();
+
     const meter = telemetryClient.getMeter('ProverNode');
     this.tracer = telemetryClient.getTracer('ProverNode');
 
@@ -102,7 +108,7 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
       this.publisher.getRollupContract(),
     );
 
-    this.txFetcher = new RunningPromise(() => this.checkForTxs(), this.log, this.options.txGatheringIntervalMs);
+    this.txFetcher = new RunningPromise(() => this.checkForTxs(), this.log, this.config.txGatheringIntervalMs);
   }
 
   public getProverId() {
@@ -151,7 +157,7 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
     this.epochsMonitor.start(this);
     this.l1Metrics.start();
     await this.rewardsMetrics.start();
-    this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.options);
+    this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.config);
   }
 
   /**
@@ -188,28 +194,49 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
    * Starts a proving process and returns immediately.
    */
   public async startProof(epochNumber: number | bigint) {
-    const job = await this.createProvingJob(BigInt(epochNumber));
+    const job = await this.createProvingJob(BigInt(epochNumber), { skipEpochCheck: true });
     void this.runJob(job);
   }
 
   private async runJob(job: EpochProvingJob) {
-    const ctx = { id: job.getId(), epochNumber: job.getEpochNumber() };
+    const epochNumber = job.getEpochNumber();
+    const ctx = { id: job.getId(), epochNumber, state: undefined as EpochProvingJobState | undefined };
+
     try {
       await job.run();
       const state = job.getState();
+      ctx.state = state;
+
       if (state === 'reorg') {
-        this.log.warn(`Running new job for epoch ${job.getEpochNumber()} due to reorg`, ctx);
-        await this.createProvingJob(job.getEpochNumber());
+        this.log.warn(`Running new job for epoch ${epochNumber} due to reorg`, ctx);
+        await this.createProvingJob(epochNumber);
+      } else if (state === 'failed') {
+        this.log.error(`Job for ${epochNumber} exited with state ${state}`, ctx);
+        await this.tryUploadEpochFailure(job);
       } else {
-        this.log.verbose(`Job for ${job.getEpochNumber()} exited with state ${state}`, ctx);
+        this.log.verbose(`Job for ${epochNumber} exited with state ${state}`, ctx);
       }
     } catch (err) {
-      this.log.error(`Error proving epoch ${job.getEpochNumber()}`, err, ctx);
+      this.log.error(`Error proving epoch ${epochNumber}`, err, ctx);
     } finally {
       this.jobs.delete(job.getId());
     }
   }
 
+  protected async tryUploadEpochFailure(job: EpochProvingJob) {
+    if (this.config.proverNodeFailedEpochStore) {
+      return await uploadEpochProofFailure(
+        this.config.proverNodeFailedEpochStore,
+        job.getId(),
+        job.getProvingData(),
+        this.l2BlockSource as Archiver,
+        this.worldState,
+        assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')),
+        this.log,
+      );
+    }
+  }
+
   /**
    * Returns the prover instance.
    */
@@ -239,36 +266,38 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
   }
 
   private checkMaximumPendingJobs() {
-    const { maxPendingJobs } = this.options;
-    return maxPendingJobs === 0 || this.jobs.size < maxPendingJobs;
+    const { proverNodeMaxPendingJobs: maxPendingJobs } = this.config;
+    if (maxPendingJobs > 0 && this.jobs.size >= maxPendingJobs) {
+      throw new Error(`Maximum pending proving jobs ${maxPendingJobs} reached. Cannot create new job.`);
+    }
   }
 
   @trackSpan('ProverNode.createProvingJob', epochNumber => ({ [Attributes.EPOCH_NUMBER]: Number(epochNumber) }))
-  private async createProvingJob(epochNumber: bigint) {
-    if (!this.checkMaximumPendingJobs()) {
-      throw new Error(`Maximum pending proving jobs ${this.options.maxPendingJobs} reached. Cannot create new job.`);
-    }
+  private async createProvingJob(epochNumber: bigint, opts: { skipEpochCheck?: boolean } = {}) {
+    this.checkMaximumPendingJobs();
 
-    // Gather blocks for this epoch
-    const { blocks, txs } = await this.gatherEpochData(epochNumber);
+    // Gather all data for this epoch
+    const epochData = await this.gatherEpochData(epochNumber);
 
-    const fromBlock = blocks[0].number;
-    const toBlock = blocks.at(-1)!.number;
+    const fromBlock = epochData.blocks[0].number;
+    const toBlock = epochData.blocks.at(-1)!.number;
+    this.log.verbose(`Creating proving job for epoch ${epochNumber} for block range ${fromBlock} to ${toBlock}`);
 
     // Fast forward world state to right before the target block and get a fork
-    this.log.verbose(`Creating proving job for epoch ${epochNumber} for block range ${fromBlock} to ${toBlock}`);
     await this.worldState.syncImmediate(toBlock);
 
-    // Create a processor using the forked world state
+    // Create a processor factory
     const publicProcessorFactory = new PublicProcessorFactory(
       this.contractDataSource,
       this.dateProvider,
       this.telemetryClient,
     );
 
+    // Set deadline for this job to run. It will abort if it takes too long.
     const deadlineTs = getProofSubmissionDeadlineTimestamp(epochNumber, await this.getL1Constants());
     const deadline = new Date(Number(deadlineTs) * 1000);
-    const job = this.doCreateEpochProvingJob(epochNumber, deadline, blocks, txs, publicProcessorFactory);
+
+    const job = this.doCreateEpochProvingJob(epochData, deadline, publicProcessorFactory, opts);
     this.jobs.set(job.getId(), job);
     return job;
   }
@@ -295,12 +324,13 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
   }
 
   @trackSpan('ProverNode.gatherEpochData', epochNumber => ({ [Attributes.EPOCH_NUMBER]: Number(epochNumber) }))
-  private async gatherEpochData(epochNumber: bigint) {
-    // Gather blocks for this epoch and their txs
+  private async gatherEpochData(epochNumber: bigint): Promise<EpochProvingJobData> {
     const blocks = await this.gatherBlocks(epochNumber);
     const txs = await this.gatherTxs(epochNumber, blocks);
+    const l1ToL2Messages = await this.gatherMessages(epochNumber, blocks);
+    const previousBlockHeader = await this.gatherPreviousBlockHeader(epochNumber, blocks[0]);
 
-    return { blocks, txs };
+    return { blocks, txs, l1ToL2Messages, epochNumber, previousBlockHeader };
   }
 
   private async gatherBlocks(epochNumber: bigint) {
@@ -328,27 +358,49 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
     throw new Error(`Txs not found for epoch ${epochNumber}: ${missingTxHashes}`);
   }
 
+  private async gatherMessages(epochNumber: bigint, blocks: L2Block[]) {
+    const messages = await Promise.all(blocks.map(b => this.l1ToL2MessageSource.getL1ToL2Messages(BigInt(b.number))));
+    const messageCount = sum(messages.map(m => m.length));
+    this.log.verbose(`Gathered all ${messageCount} messages for epoch ${epochNumber}`, { epochNumber });
+    const messagesByBlock: Record<number, Fr[]> = {};
+    for (let i = 0; i < blocks.length; i++) {
+      messagesByBlock[blocks[i].number] = messages[i];
+    }
+    return messagesByBlock;
+  }
+
+  private async gatherPreviousBlockHeader(epochNumber: bigint, initialBlock: L2Block) {
+    const previousBlockNumber = initialBlock.number - 1;
+    const header = await (previousBlockNumber === 0
+      ? this.worldState.getCommitted().getInitialHeader()
+      : this.l2BlockSource.getBlockHeader(previousBlockNumber));
+
+    if (!header) {
+      throw new Error(`Previous block header ${initialBlock.number} not found for proving epoch ${epochNumber}`);
+    }
+
+    this.log.verbose(`Gathered previous block header ${header.getBlockNumber()} for epoch ${epochNumber}`);
+    return header;
+  }
+
   /** Extracted for testing purposes. */
   protected doCreateEpochProvingJob(
-    epochNumber: bigint,
+    data: EpochProvingJobData,
     deadline: Date | undefined,
-    blocks: L2Block[],
-    txs: Tx[],
     publicProcessorFactory: PublicProcessorFactory,
+    opts: { skipEpochCheck?: boolean } = {},
   ) {
+    const { proverNodeMaxParallelBlocksPerEpoch: parallelBlockLimit } = this.config;
     return new EpochProvingJob(
+      data,
       this.worldState,
-      epochNumber,
-      blocks,
-      txs,
       this.prover.createEpochProver(),
       publicProcessorFactory,
       this.publisher,
       this.l2BlockSource,
-      this.l1ToL2MessageSource,
       this.jobMetrics,
       deadline,
-      { parallelBlockLimit: this.options.maxParallelBlocksPerEpoch },
+      { parallelBlockLimit, ...opts },
     );
   }
 
@@ -356,6 +408,21 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
   protected async triggerMonitors() {
     await this.epochsMonitor.work();
   }
+
+  private validateConfig() {
+    if (
+      this.config.proverNodeFailedEpochStore &&
+      (!this.config.dataDirectory || !this.config.l1ChainId || !this.config.rollupVersion)
+    ) {
+      this.log.warn(
+        `Invalid prover-node config (missing dataDirectory, l1ChainId, or rollupVersion)`,
+        pick(this.config, 'proverNodeFailedEpochStore', 'dataDirectory', 'l1ChainId', 'rollupVersion'),
+      );
+      throw new Error(
+        'All of dataDirectory, l1ChainId, and rollupVersion are required if proverNodeFailedEpochStore is set.',
+      );
+    }
+  }
 }
 
 class EmptyEpochError extends Error {

package/src/test/index.ts

@@ -1,11 +1,14 @@
 import type { EpochProverManager } from '@aztec/stdlib/interfaces/server';
 
+import type { EpochProvingJob } from '../job/epoch-proving-job.js';
 import type { ProverNodePublisher } from '../prover-node-publisher.js';
 import { ProverNode } from '../prover-node.js';
 
-class TestProverNode_ extends ProverNode {
-  public declare prover: EpochProverManager;
-  public declare publisher: ProverNodePublisher;
+abstract class TestProverNodeClass extends ProverNode {
+  declare public prover: EpochProverManager;
+  declare public publisher: ProverNodePublisher;
+
+  public abstract override tryUploadEpochFailure(job: EpochProvingJob): Promise<string | undefined>;
 }
 
-export type TestProverNode = TestProverNode_;
+export type TestProverNode = TestProverNodeClass;