@aztec/prover-node 0.0.0-test.1 → 0.0.1-commit.0b941701

package/src/prover-node.ts

@@ -1,28 +1,34 @@
-import { compact } from '@aztec/foundation/collection';
+import type { Archiver } from '@aztec/archiver';
+import type { RollupContract } from '@aztec/ethereum/contracts';
+import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
+import { assertRequired, compact, pick, sum } from '@aztec/foundation/collection';
+import type { Fr } from '@aztec/foundation/curves/bn254';
 import { memoize } from '@aztec/foundation/decorators';
 import { createLogger } from '@aztec/foundation/log';
-import { RunningPromise } from '@aztec/foundation/running-promise';
 import { DateProvider } from '@aztec/foundation/timer';
-import type { Maybe } from '@aztec/foundation/types';
-import type { P2P } from '@aztec/p2p';
+import type { DataStoreConfig } from '@aztec/kv-store/config';
+import type { P2PClient } from '@aztec/p2p';
 import { PublicProcessorFactory } from '@aztec/simulator/server';
-import type { L2Block, L2BlockSource } from '@aztec/stdlib/block';
+import type { L2BlockSource } from '@aztec/stdlib/block';
+import type { Checkpoint } from '@aztec/stdlib/checkpoint';
+import type { ChainConfig } from '@aztec/stdlib/config';
 import type { ContractDataSource } from '@aztec/stdlib/contract';
-import { getTimestampRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
+import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
 import {
   type EpochProverManager,
   EpochProvingJobTerminalState,
-  type ProverCoordination,
   type ProverNodeApi,
   type Service,
+  type WorldStateSyncStatus,
   type WorldStateSynchronizer,
   tryStop,
 } from '@aztec/stdlib/interfaces/server';
 import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
 import type { P2PClientType } from '@aztec/stdlib/p2p';
-import type { Tx, TxHash } from '@aztec/stdlib/tx';
+import type { Tx } from '@aztec/stdlib/tx';
 import {
   Attributes,
+  L1Metrics,
   type TelemetryClient,
   type Traceable,
   type Tracer,
@@ -30,66 +36,71 @@ import {
   trackSpan,
 } from '@aztec/telemetry-client';
 
+import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
+import type { SpecificProverNodeConfig } from './config.js';
+import type { EpochProvingJobData } from './job/epoch-proving-job-data.js';
 import { EpochProvingJob, type EpochProvingJobState } from './job/epoch-proving-job.js';
-import { ProverNodeMetrics } from './metrics.js';
+import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
 import type { EpochMonitor, EpochMonitorHandler } from './monitors/epoch-monitor.js';
 import type { ProverNodePublisher } from './prover-node-publisher.js';
+import type { ProverPublisherFactory } from './prover-publisher-factory.js';
 
-export type ProverNodeOptions = {
-  pollingIntervalMs: number;
-  maxPendingJobs: number;
-  maxParallelBlocksPerEpoch: number;
-  txGatheringTimeoutMs: number;
-  txGatheringIntervalMs: number;
-  txGatheringMaxParallelRequests: number;
-};
+type ProverNodeOptions = SpecificProverNodeConfig & Partial<DataStoreOptions>;
+type DataStoreOptions = Pick<DataStoreConfig, 'dataDirectory'> & Pick<ChainConfig, 'l1ChainId' | 'rollupVersion'>;
 
 /**
- * An Aztec Prover Node is a standalone process that monitors the unfinalised chain on L1 for unproven blocks,
- * submits bids for proving them, and monitors if they are accepted. If so, the prover node fetches the txs
- * from a tx source in the p2p network or an external node, re-executes their public functions, creates a rollup
+ * An Aztec Prover Node is a standalone process that monitors the unfinalized chain on L1 for unproven epochs,
+ * fetches their txs from the p2p network or external nodes, re-executes their public functions, creates a rollup
  * proof for the epoch, and submits it to L1.
  */
 export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable {
   private log = createLogger('prover-node');
   private dateProvider = new DateProvider();
 
-  private latestEpochWeAreProving: bigint | undefined;
   private jobs: Map<string, EpochProvingJob> = new Map();
-  private cachedEpochData: { epochNumber: bigint; blocks: L2Block[]; txs: Tx[] } | undefined = undefined;
-  private options: ProverNodeOptions;
-  private metrics: ProverNodeMetrics;
-
-  private txFetcher: RunningPromise;
-  private lastBlockNumber: number | undefined;
+  private config: ProverNodeOptions;
+  private jobMetrics: ProverNodeJobMetrics;
+  private rewardsMetrics: ProverNodeRewardsMetrics;
 
   public readonly tracer: Tracer;
 
+  protected publisher: ProverNodePublisher | undefined;
+
   constructor(
     protected readonly prover: EpochProverManager,
-    protected readonly publisher: ProverNodePublisher,
-    protected readonly l2BlockSource: L2BlockSource & Maybe<Service>,
+    protected readonly publisherFactory: ProverPublisherFactory,
+    protected readonly l2BlockSource: L2BlockSource & Partial<Service>,
     protected readonly l1ToL2MessageSource: L1ToL2MessageSource,
     protected readonly contractDataSource: ContractDataSource,
     protected readonly worldState: WorldStateSynchronizer,
-    protected readonly coordination: ProverCoordination & Maybe<Service>,
+    protected readonly p2pClient: Pick<P2PClient<P2PClientType.Prover>, 'getTxProvider'> & Partial<Service>,
     protected readonly epochsMonitor: EpochMonitor,
-    options: Partial<ProverNodeOptions> = {},
+    protected readonly rollupContract: RollupContract,
+    protected readonly l1Metrics: L1Metrics,
+    config: Partial<ProverNodeOptions> = {},
     protected readonly telemetryClient: TelemetryClient = getTelemetryClient(),
   ) {
-    this.options = {
-      pollingIntervalMs: 1_000,
-      maxPendingJobs: 100,
-      maxParallelBlocksPerEpoch: 32,
-      txGatheringTimeoutMs: 60_000,
+    this.config = {
+      proverNodePollingIntervalMs: 1_000,
+      proverNodeMaxPendingJobs: 100,
+      proverNodeMaxParallelBlocksPerEpoch: 32,
       txGatheringIntervalMs: 1_000,
-      txGatheringMaxParallelRequests: 100,
-      ...compact(options),
+      txGatheringBatchSize: 10,
+      txGatheringMaxParallelRequestsPerNode: 100,
+      txGatheringTimeoutMs: 120_000,
+      proverNodeFailedEpochStore: undefined,
+      proverNodeEpochProvingDelayMs: undefined,
+      ...compact(config),
     };
 
-    this.metrics = new ProverNodeMetrics(telemetryClient, 'ProverNode');
+    this.validateConfig();
+
+    const meter = telemetryClient.getMeter('ProverNode');
     this.tracer = telemetryClient.getTracer('ProverNode');
-    this.txFetcher = new RunningPromise(() => this.checkForTxs(), this.log, this.options.txGatheringIntervalMs);
+
+    this.jobMetrics = new ProverNodeJobMetrics(meter, telemetryClient.getTracer('EpochProvingJob'));
+
+    this.rewardsMetrics = new ProverNodeRewardsMetrics(meter, this.prover.getProverId(), rollupContract);
   }
 
   public getProverId() {
@@ -97,44 +108,49 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
   }
 
   public getP2P() {
-    const asP2PClient = this.coordination as P2P<P2PClientType.Prover>;
-    if (typeof asP2PClient.isP2PClient === 'function' && asP2PClient.isP2PClient()) {
-      return asP2PClient;
-    }
-    return undefined;
+    return this.p2pClient;
   }
 
   /**
    * Handles an epoch being completed by starting a proof for it if there are no active jobs for it.
    * @param epochNumber - The epoch number that was just completed.
+   * @returns false if there is an error, true otherwise
    */
-  async handleEpochReadyToProve(epochNumber: bigint): Promise<void> {
+  async handleEpochReadyToProve(epochNumber: EpochNumber): Promise<boolean> {
     try {
-      this.log.debug('jobs', JSON.stringify(this.jobs, null, 2));
+      this.log.debug(`Running jobs as ${epochNumber} is ready to prove`, {
+        jobs: Array.from(this.jobs.values()).map(job => `${job.getEpochNumber()}:${job.getId()}`),
+      });
       const activeJobs = await this.getActiveJobsForEpoch(epochNumber);
       if (activeJobs.length > 0) {
-        this.log.info(`Not starting proof for ${epochNumber} since there are active jobs`);
-        return;
+        this.log.warn(`Not starting proof for ${epochNumber} since there are active jobs for the epoch`, {
+          activeJobs: activeJobs.map(job => job.uuid),
+        });
+        return true;
       }
-      // TODO: we probably want to skip starting a proof if we are too far into the current epoch
       await this.startProof(epochNumber);
+      return true;
     } catch (err) {
       if (err instanceof EmptyEpochError) {
         this.log.info(`Not starting proof for ${epochNumber} since no blocks were found`);
       } else {
         this.log.error(`Error handling epoch completed`, err);
       }
+      return false;
     }
   }
 
   /**
-   * Starts the prover node so it periodically checks for unproven epochs in the unfinalised chain from L1 and
+   * Starts the prover node so it periodically checks for unproven epochs in the unfinalized chain from L1 and
    * starts proving jobs for them.
    */
-  start() {
-    this.txFetcher.start();
+  async start() {
     this.epochsMonitor.start(this);
-    this.log.info('Started ProverNode', this.options);
+    await this.publisherFactory.start();
+    this.publisher = await this.publisherFactory.create();
+    await this.rewardsMetrics.start();
+    this.l1Metrics.start();
+    this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.config);
   }
 
   /**
@@ -142,32 +158,76 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
    */
   async stop() {
     this.log.info('Stopping ProverNode');
-    await this.txFetcher.stop();
     await this.epochsMonitor.stop();
     await this.prover.stop();
+    await tryStop(this.p2pClient);
     await tryStop(this.l2BlockSource);
-    this.publisher.interrupt();
+    await tryStop(this.publisherFactory);
+    this.publisher?.interrupt();
     await Promise.all(Array.from(this.jobs.values()).map(job => job.stop()));
     await this.worldState.stop();
-    await tryStop(this.coordination);
+    this.rewardsMetrics.stop();
+    this.l1Metrics.stop();
     await this.telemetryClient.stop();
     this.log.info('Stopped ProverNode');
   }
 
-  /**
-   * Creates a proof for a block range. Returns once the proof has been submitted to L1.
-   */
-  public async prove(epochNumber: number | bigint) {
-    const job = await this.createProvingJob(BigInt(epochNumber));
-    return job.run();
+  /** Returns world state status. */
+  public async getWorldStateSyncStatus(): Promise<WorldStateSyncStatus> {
+    const { syncSummary } = await this.worldState.status();
+    return syncSummary;
+  }
+
+  /** Returns archiver status. */
+  public getL2Tips() {
+    return this.l2BlockSource.getL2Tips();
   }
 
   /**
    * Starts a proving process and returns immediately.
    */
-  public async startProof(epochNumber: number | bigint) {
-    const job = await this.createProvingJob(BigInt(epochNumber));
-    void job.run().catch(err => this.log.error(`Error proving epoch ${epochNumber}`, err));
+  public async startProof(epochNumber: EpochNumber) {
+    const job = await this.createProvingJob(epochNumber, { skipEpochCheck: true });
+    void this.runJob(job);
+  }
+
+  private async runJob(job: EpochProvingJob) {
+    const epochNumber = job.getEpochNumber();
+    const ctx = { id: job.getId(), epochNumber, state: undefined as EpochProvingJobState | undefined };
+
+    try {
+      await job.run();
+      const state = job.getState();
+      ctx.state = state;
+
+      if (state === 'reorg') {
+        this.log.warn(`Running new job for epoch ${epochNumber} due to reorg`, ctx);
+        await this.createProvingJob(epochNumber);
+      } else if (state === 'failed') {
+        this.log.error(`Job for ${epochNumber} exited with state ${state}`, ctx);
+        await this.tryUploadEpochFailure(job);
+      } else {
+        this.log.verbose(`Job for ${epochNumber} exited with state ${state}`, ctx);
+      }
+    } catch (err) {
+      this.log.error(`Error proving epoch ${epochNumber}`, err, ctx);
+    } finally {
+      this.jobs.delete(job.getId());
+    }
+  }
+
+  protected async tryUploadEpochFailure(job: EpochProvingJob) {
+    if (this.config.proverNodeFailedEpochStore) {
+      return await uploadEpochProofFailure(
+        this.config.proverNodeFailedEpochStore,
+        job.getId(),
+        job.getProvingData(),
+        this.l2BlockSource as Archiver,
+        this.worldState,
+        assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')),
+        this.log,
+      );
+    }
   }
 
   /**
@@ -180,62 +240,60 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
   /**
    * Returns an array of jobs being processed.
    */
-  public getJobs(): Promise<{ uuid: string; status: EpochProvingJobState; epochNumber: number }[]> {
+  public getJobs(): Promise<{ uuid: string; status: EpochProvingJobState; epochNumber: EpochNumber }[]> {
     return Promise.resolve(
       Array.from(this.jobs.entries()).map(([uuid, job]) => ({
         uuid,
         status: job.getState(),
-        epochNumber: Number(job.getEpochNumber()),
+        epochNumber: job.getEpochNumber(),
       })),
     );
   }
 
   protected async getActiveJobsForEpoch(
-    epochBigInt: bigint,
+    epochNumber: EpochNumber,
   ): Promise<{ uuid: string; status: EpochProvingJobState }[]> {
     const jobs = await this.getJobs();
-    const epochNumber = Number(epochBigInt);
     return jobs.filter(job => job.epochNumber === epochNumber && !EpochProvingJobTerminalState.includes(job.status));
   }
 
   private checkMaximumPendingJobs() {
-    const { maxPendingJobs } = this.options;
-    return maxPendingJobs === 0 || this.jobs.size < maxPendingJobs;
+    const { proverNodeMaxPendingJobs: maxPendingJobs } = this.config;
+    if (maxPendingJobs > 0 && this.jobs.size >= maxPendingJobs) {
+      throw new Error(`Maximum pending proving jobs ${maxPendingJobs} reached. Cannot create new job.`);
+    }
   }
 
-  @trackSpan('ProverNode.createProvingJob', epochNumber => ({ [Attributes.EPOCH_NUMBER]: Number(epochNumber) }))
-  private async createProvingJob(epochNumber: bigint) {
-    if (!this.checkMaximumPendingJobs()) {
-      throw new Error(`Maximum pending proving jobs ${this.options.maxPendingJobs} reached. Cannot create new job.`);
-    }
+  @trackSpan('ProverNode.createProvingJob', epochNumber => ({ [Attributes.EPOCH_NUMBER]: epochNumber }))
+  private async createProvingJob(epochNumber: EpochNumber, opts: { skipEpochCheck?: boolean } = {}) {
+    this.checkMaximumPendingJobs();
 
-    // Gather blocks for this epoch
-    const cachedEpochData = this.cachedEpochData?.epochNumber === epochNumber ? this.cachedEpochData : undefined;
-    const { blocks, txs } = cachedEpochData ?? (await this.gatherEpochData(epochNumber));
+    this.publisher = await this.publisherFactory.create();
 
-    const fromBlock = blocks[0].number;
-    const toBlock = blocks.at(-1)!.number;
+    // Gather all data for this epoch
+    const epochData = await this.gatherEpochData(epochNumber);
+    const fromCheckpoint = epochData.checkpoints[0].number;
+    const toCheckpoint = epochData.checkpoints.at(-1)!.number;
+    const fromBlock = epochData.checkpoints[0].blocks[0].number;
+    const toBlock = epochData.checkpoints.at(-1)!.blocks.at(-1)!.number;
+    this.log.verbose(
+      `Creating proving job for epoch ${epochNumber} for checkpoint range ${fromCheckpoint} to ${toCheckpoint} and block range ${fromBlock} to ${toBlock}`,
+    );
 
     // Fast forward world state to right before the target block and get a fork
-    this.log.verbose(`Creating proving job for epoch ${epochNumber} for block range ${fromBlock} to ${toBlock}`);
     await this.worldState.syncImmediate(toBlock);
 
-    // Create a processor using the forked world state
+    // Create a processor factory
     const publicProcessorFactory = new PublicProcessorFactory(
       this.contractDataSource,
       this.dateProvider,
       this.telemetryClient,
     );
 
-    const cleanUp = () => {
-      this.jobs.delete(job.getId());
-      return Promise.resolve();
-    };
-
-    const [_, endTimestamp] = getTimestampRangeForEpoch(epochNumber + 1n, await this.getL1Constants());
-    const deadline = new Date(Number(endTimestamp) * 1000);
-
-    const job = this.doCreateEpochProvingJob(epochNumber, deadline, blocks, txs, publicProcessorFactory, cleanUp);
+    // Set deadline for this job to run. It will abort if it takes too long.
+    const deadlineTs = getProofSubmissionDeadlineTimestamp(epochNumber, await this.getL1Constants());
+    const deadline = new Date(Number(deadlineTs) * 1000);
+    const job = this.doCreateEpochProvingJob(epochData, deadline, publicProcessorFactory, this.publisher, opts);
     this.jobs.set(job.getId(), job);
     return job;
   }
@@ -245,79 +303,87 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
     return this.l2BlockSource.getL1Constants();
   }
 
-  /** Monitors for new blocks and requests their txs from the p2p layer to ensure they are available for proving. */
-  @trackSpan('ProverNode.checkForTxs')
-  private async checkForTxs() {
-    const blockNumber = await this.l2BlockSource.getBlockNumber();
-    if (this.lastBlockNumber === undefined || blockNumber > this.lastBlockNumber) {
-      const block = await this.l2BlockSource.getBlock(blockNumber);
-      if (!block) {
-        return;
-      }
-      const txHashes = block.body.txEffects.map(tx => tx.txHash);
-      this.log.verbose(`Fetching ${txHashes.length} tx hashes for block number ${blockNumber} from coordination`);
-      await this.coordination.getTxsByHash(txHashes); // This stores the txs in the tx pool, no need to persist them here
-      this.lastBlockNumber = blockNumber;
-    }
-  }
-
-  @trackSpan('ProverNode.gatherEpochData', epochNumber => ({ [Attributes.EPOCH_NUMBER]: Number(epochNumber) }))
-  private async gatherEpochData(epochNumber: bigint) {
-    // Gather blocks for this epoch and their txs
-    const blocks = await this.gatherBlocks(epochNumber);
-    const txs = await this.gatherTxs(epochNumber, blocks);
-
-    return { blocks, txs };
+  @trackSpan('ProverNode.gatherEpochData', epochNumber => ({ [Attributes.EPOCH_NUMBER]: epochNumber }))
+  private async gatherEpochData(epochNumber: EpochNumber): Promise<EpochProvingJobData> {
+    const checkpoints = await this.gatherCheckpoints(epochNumber);
+    const txArray = await this.gatherTxs(epochNumber, checkpoints);
+    const txs = new Map<string, Tx>(txArray.map(tx => [tx.getTxHash().toString(), tx]));
+    const l1ToL2Messages = await this.gatherMessages(epochNumber, checkpoints);
+    const [firstBlock] = checkpoints[0].blocks;
+    const previousBlockHeader = await this.gatherPreviousBlockHeader(epochNumber, firstBlock.number - 1);
+    const [lastPublishedCheckpoint] = await this.l2BlockSource.getCheckpoints(checkpoints.at(-1)!.number, 1);
+    const attestations = lastPublishedCheckpoint?.attestations ?? [];
+
+    return { checkpoints, txs, l1ToL2Messages, epochNumber, previousBlockHeader, attestations };
   }
 
-  private async gatherBlocks(epochNumber: bigint) {
-    const blocks = await this.l2BlockSource.getBlocksForEpoch(epochNumber);
-    if (blocks.length === 0) {
+  private async gatherCheckpoints(epochNumber: EpochNumber) {
+    const checkpoints = await this.l2BlockSource.getCheckpointsForEpoch(epochNumber);
+    if (checkpoints.length === 0) {
       throw new EmptyEpochError(epochNumber);
     }
-    return blocks;
+    return checkpoints;
   }
 
-  private async gatherTxs(epochNumber: bigint, blocks: L2Block[]) {
-    const txsToFind: TxHash[] = blocks.flatMap(block => block.body.txEffects.map(tx => tx.txHash));
-    const txs = await this.coordination.getTxsByHash(txsToFind);
+  private async gatherTxs(epochNumber: EpochNumber, checkpoints: Checkpoint[]) {
+    const deadline = new Date(this.dateProvider.now() + this.config.txGatheringTimeoutMs);
+    const txProvider = this.p2pClient.getTxProvider();
+    const blocks = checkpoints.flatMap(checkpoint => checkpoint.blocks);
+    const txsByBlock = await Promise.all(blocks.map(block => txProvider.getTxsForBlock(block, { deadline })));
+    const txs = txsByBlock.map(({ txs }) => txs).flat();
+    const missingTxs = txsByBlock.map(({ missingTxs }) => missingTxs).flat();
 
-    if (txs.length === txsToFind.length) {
+    if (missingTxs.length === 0) {
       this.log.verbose(`Gathered all ${txs.length} txs for epoch ${epochNumber}`, { epochNumber });
       return txs;
     }
 
-    const txHashesFound = await Promise.all(txs.map(tx => tx.getTxHash()));
-    const missingTxHashes = txsToFind
-      .filter(txHashToFind => !txHashesFound.some(txHashFound => txHashToFind.equals(txHashFound)))
-      .join(', ');
+    throw new Error(`Txs not found for epoch ${epochNumber}: ${missingTxs.map(hash => hash.toString()).join(', ')}`);
+  }
 
-    throw new Error(`Txs not found for epoch ${epochNumber}: ${missingTxHashes}`);
+  private async gatherMessages(epochNumber: EpochNumber, checkpoints: Checkpoint[]) {
+    const messages = await Promise.all(checkpoints.map(c => this.l1ToL2MessageSource.getL1ToL2Messages(c.number)));
+    const messageCount = sum(messages.map(m => m.length));
+    this.log.verbose(`Gathered all ${messageCount} messages for epoch ${epochNumber}`, { epochNumber });
+    const messagesByCheckpoint: Record<CheckpointNumber, Fr[]> = {};
+    for (let i = 0; i < checkpoints.length; i++) {
+      messagesByCheckpoint[checkpoints[i].number] = messages[i];
+    }
+    return messagesByCheckpoint;
+  }
+
+  private async gatherPreviousBlockHeader(epochNumber: EpochNumber, previousBlockNumber: number) {
+    const header = await (previousBlockNumber === 0
+      ? this.worldState.getCommitted().getInitialHeader()
+      : this.l2BlockSource.getBlockHeader(BlockNumber(previousBlockNumber)));
+
+    if (!header) {
+      throw new Error(`Previous block header ${previousBlockNumber} not found for proving epoch ${epochNumber}`);
+    }
+
+    this.log.verbose(`Gathered previous block header ${header.getBlockNumber()} for epoch ${epochNumber}`);
+    return header;
   }
 
   /** Extracted for testing purposes. */
   protected doCreateEpochProvingJob(
-    epochNumber: bigint,
+    data: EpochProvingJobData,
     deadline: Date | undefined,
-    blocks: L2Block[],
-    txs: Tx[],
     publicProcessorFactory: PublicProcessorFactory,
-    cleanUp: () => Promise<void>,
+    publisher: ProverNodePublisher,
+    opts: { skipEpochCheck?: boolean } = {},
   ) {
+    const { proverNodeMaxParallelBlocksPerEpoch: parallelBlockLimit, proverNodeDisableProofPublish } = this.config;
     return new EpochProvingJob(
+      data,
       this.worldState,
-      epochNumber,
-      blocks,
-      txs,
       this.prover.createEpochProver(),
       publicProcessorFactory,
-      this.publisher,
+      publisher,
       this.l2BlockSource,
-      this.l1ToL2MessageSource,
-      this.metrics,
+      this.jobMetrics,
       deadline,
-      { parallelBlockLimit: this.options.maxParallelBlocksPerEpoch },
-      cleanUp,
+      { parallelBlockLimit, skipSubmitProof: proverNodeDisableProofPublish, ...opts },
     );
   }
 
@@ -325,10 +391,25 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
   protected async triggerMonitors() {
     await this.epochsMonitor.work();
   }
+
+  private validateConfig() {
+    if (
+      this.config.proverNodeFailedEpochStore &&
+      (!this.config.dataDirectory || !this.config.l1ChainId || this.config.rollupVersion === undefined)
+    ) {
+      this.log.warn(
+        `Invalid prover-node config (missing dataDirectory, l1ChainId, or rollupVersion)`,
+        pick(this.config, 'proverNodeFailedEpochStore', 'dataDirectory', 'l1ChainId', 'rollupVersion'),
+      );
+      throw new Error(
+        'All of dataDirectory, l1ChainId, and rollupVersion are required if proverNodeFailedEpochStore is set.',
+      );
+    }
+  }
 }
 
 class EmptyEpochError extends Error {
-  constructor(epochNumber: bigint) {
+  constructor(epochNumber: EpochNumber) {
     super(`No blocks found for epoch ${epochNumber}`);
     this.name = 'EmptyEpochError';
   }

package/src/prover-publisher-factory.ts

@@ -0,0 +1,39 @@
+import type { RollupContract } from '@aztec/ethereum/contracts';
+import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils';
+import type { PublisherManager } from '@aztec/ethereum/publisher-manager';
+import type { PublisherConfig, TxSenderConfig } from '@aztec/sequencer-client';
+import type { TelemetryClient } from '@aztec/telemetry-client';
+
+import { ProverNodePublisher } from './prover-node-publisher.js';
+
+export class ProverPublisherFactory {
+  constructor(
+    private config: TxSenderConfig & PublisherConfig,
+    private deps: {
+      rollupContract: RollupContract;
+      publisherManager: PublisherManager<L1TxUtils>;
+      telemetry?: TelemetryClient;
+    },
+  ) {}
+
+  public async start() {
+    await this.deps.publisherManager.loadState();
+  }
+
+  public stop() {
+    this.deps.publisherManager.interrupt();
+  }
+
+  /**
+   * Creates a new Prover Publisher instance.
+   * @returns A new ProverNodePublisher instance.
+   */
+  public async create(): Promise<ProverNodePublisher> {
+    const l1Publisher = await this.deps.publisherManager.getAvailablePublisher();
+    return new ProverNodePublisher(this.config, {
+      rollupContract: this.deps.rollupContract,
+      l1TxUtils: l1Publisher,
+      telemetry: this.deps.telemetry,
+    });
+  }
+}

package/src/test/index.ts

@@ -1,11 +1,14 @@
 import type { EpochProverManager } from '@aztec/stdlib/interfaces/server';
 
+import type { EpochProvingJob } from '../job/epoch-proving-job.js';
 import type { ProverNodePublisher } from '../prover-node-publisher.js';
 import { ProverNode } from '../prover-node.js';
 
-class TestProverNode_ extends ProverNode {
-  public override prover!: EpochProverManager;
-  public override publisher!: ProverNodePublisher;
+abstract class TestProverNodeClass extends ProverNode {
+  declare public prover: EpochProverManager;
+  declare public publisher: ProverNodePublisher;
+
+  public abstract override tryUploadEpochFailure(job: EpochProvingJob): Promise<string | undefined>;
 }
 
-export type TestProverNode = TestProverNode_;
+export type TestProverNode = TestProverNodeClass;

package/dest/http.d.ts

@@ -1,8 +0,0 @@
-import type { ProverNode } from './prover-node.js';
-/**
- * Wrap a ProverNode instance with a JSON RPC HTTP server.
- * @param node - The ProverNode
- * @returns An JSON-RPC HTTP server
- */
-export declare function createProverNodeRpcServer(node: ProverNode): import("@aztec/foundation/json-rpc/server").SafeJsonRpcServer;
-//# sourceMappingURL=http.d.ts.map

package/dest/http.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnD;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,UAAU,iEAEzD"}

package/dest/http.js

@@ -1,9 +0,0 @@
-import { ProverNodeApiSchema } from '@aztec/stdlib/interfaces/server';
-import { createTracedJsonRpcServer } from '@aztec/telemetry-client';
-/**
- * Wrap a ProverNode instance with a JSON RPC HTTP server.
- * @param node - The ProverNode
- * @returns An JSON-RPC HTTP server
- */ export function createProverNodeRpcServer(node) {
-    return createTracedJsonRpcServer(node, ProverNodeApiSchema);
-}

package/dest/prover-coordination/config.d.ts

@@ -1,7 +0,0 @@
-import { type ConfigMappingsType } from '@aztec/foundation/config';
-export type ProverCoordinationConfig = {
-    proverCoordinationNodeUrl: string | undefined;
-};
-export declare const proverCoordinationConfigMappings: ConfigMappingsType<ProverCoordinationConfig>;
-export declare function getTxProviderConfigFromEnv(): ProverCoordinationConfig;
-//# sourceMappingURL=config.d.ts.map

package/dest/prover-coordination/config.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/prover-coordination/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,kBAAkB,EAAyB,MAAM,0BAA0B,CAAC;AAE1F,MAAM,MAAM,wBAAwB,GAAG;IACrC,yBAAyB,EAAE,MAAM,GAAG,SAAS,CAAC;CAC/C,CAAC;AAEF,eAAO,MAAM,gCAAgC,EAAE,kBAAkB,CAAC,wBAAwB,CAMzF,CAAC;AAEF,wBAAgB,0BAA0B,IAAI,wBAAwB,CAErE"}

package/dest/prover-coordination/config.js

@@ -1,11 +0,0 @@
-import { getConfigFromMappings } from '@aztec/foundation/config';
-export const proverCoordinationConfigMappings = {
-    proverCoordinationNodeUrl: {
-        env: 'PROVER_COORDINATION_NODE_URL',
-        description: 'The URL of the tx provider node',
-        parseEnv: (val)=>val
-    }
-};
-export function getTxProviderConfigFromEnv() {
-    return getConfigFromMappings(proverCoordinationConfigMappings);
-}