@aztec/prover-node 0.85.0-alpha-testnet.2 → 0.85.0-alpha-testnet.4

package/src/prover-node.ts

@@ -1,12 +1,16 @@
-import { compact } from '@aztec/foundation/collection';
+import type { Archiver } from '@aztec/archiver';
+import { assertRequired, compact, pick, sum } from '@aztec/foundation/collection';
 import { memoize } from '@aztec/foundation/decorators';
+import { EthAddress } from '@aztec/foundation/eth-address';
+import type { Fr } from '@aztec/foundation/fields';
 import { createLogger } from '@aztec/foundation/log';
 import { RunningPromise } from '@aztec/foundation/running-promise';
 import { DateProvider } from '@aztec/foundation/timer';
 import type { Maybe } from '@aztec/foundation/types';
-import type { P2P } from '@aztec/p2p';
+import type { DataStoreConfig } from '@aztec/kv-store/config';
 import { PublicProcessorFactory } from '@aztec/simulator/server';
 import type { L2Block, L2BlockSource } from '@aztec/stdlib/block';
+import type { ChainConfig } from '@aztec/stdlib/config';
 import type { ContractDataSource } from '@aztec/stdlib/contract';
 import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
 import {
@@ -20,8 +24,7 @@ import {
   tryStop,
 } from '@aztec/stdlib/interfaces/server';
 import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
-import type { P2PClientType } from '@aztec/stdlib/p2p';
-import type { Tx, TxHash } from '@aztec/stdlib/tx';
+import type { TxHash } from '@aztec/stdlib/tx';
 import {
   Attributes,
   L1Metrics,
@@ -32,24 +35,20 @@ import {
   trackSpan,
 } from '@aztec/telemetry-client';
 
+import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
+import type { SpecificProverNodeConfig } from './config.js';
+import type { EpochProvingJobData } from './job/epoch-proving-job-data.js';
 import { EpochProvingJob, type EpochProvingJobState } from './job/epoch-proving-job.js';
-import { ProverNodeMetrics } from './metrics.js';
+import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
 import type { EpochMonitor, EpochMonitorHandler } from './monitors/epoch-monitor.js';
 import type { ProverNodePublisher } from './prover-node-publisher.js';
 
-export type ProverNodeOptions = {
-  pollingIntervalMs: number;
-  maxPendingJobs: number;
-  maxParallelBlocksPerEpoch: number;
-  txGatheringTimeoutMs: number;
-  txGatheringIntervalMs: number;
-  txGatheringMaxParallelRequests: number;
-};
+type ProverNodeOptions = SpecificProverNodeConfig & Partial<DataStoreOptions>;
+type DataStoreOptions = Pick<DataStoreConfig, 'dataDirectory'> & Pick<ChainConfig, 'l1ChainId' | 'rollupVersion'>;
 
 /**
- * An Aztec Prover Node is a standalone process that monitors the unfinalised chain on L1 for unproven blocks,
- * submits bids for proving them, and monitors if they are accepted. If so, the prover node fetches the txs
- * from a tx source in the p2p network or an external node, re-executes their public functions, creates a rollup
+ * An Aztec Prover Node is a standalone process that monitors the unfinalised chain on L1 for unproven epochs,
+ * fetches their txs from the p2p network or external nodes, re-executes their public functions, creates a rollup
  * proof for the epoch, and submits it to L1.
  */
 export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable {
@@ -57,8 +56,9 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
   private dateProvider = new DateProvider();
 
   private jobs: Map<string, EpochProvingJob> = new Map();
-  private options: ProverNodeOptions;
-  private metrics: ProverNodeMetrics;
+  private config: ProverNodeOptions;
+  private jobMetrics: ProverNodeJobMetrics;
+  private rewardsMetrics: ProverNodeRewardsMetrics;
   private l1Metrics: L1Metrics;
 
   private txFetcher: RunningPromise;
@@ -73,28 +73,40 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
     protected readonly l1ToL2MessageSource: L1ToL2MessageSource,
     protected readonly contractDataSource: ContractDataSource,
     protected readonly worldState: WorldStateSynchronizer,
-    protected readonly coordination: ProverCoordination & Maybe<Service>,
+    protected readonly coordination: ProverCoordination,
     protected readonly epochsMonitor: EpochMonitor,
-    options: Partial<ProverNodeOptions> = {},
+    config: Partial<ProverNodeOptions> = {},
     protected readonly telemetryClient: TelemetryClient = getTelemetryClient(),
   ) {
     this.l1Metrics = new L1Metrics(telemetryClient.getMeter('ProverNodeL1Metrics'), publisher.l1TxUtils.publicClient, [
       publisher.getSenderAddress(),
     ]);
 
-    this.options = {
-      pollingIntervalMs: 1_000,
-      maxPendingJobs: 100,
-      maxParallelBlocksPerEpoch: 32,
-      txGatheringTimeoutMs: 60_000,
+    this.config = {
+      proverNodePollingIntervalMs: 1_000,
+      proverNodeMaxPendingJobs: 100,
+      proverNodeMaxParallelBlocksPerEpoch: 32,
       txGatheringIntervalMs: 1_000,
-      txGatheringMaxParallelRequests: 100,
-      ...compact(options),
+      txGatheringBatchSize: 10,
+      txGatheringMaxParallelRequestsPerNode: 100,
+      proverNodeFailedEpochStore: undefined,
+      ...compact(config),
     };
 
-    this.metrics = new ProverNodeMetrics(telemetryClient, 'ProverNode');
+    this.validateConfig();
+
+    const meter = telemetryClient.getMeter('ProverNode');
     this.tracer = telemetryClient.getTracer('ProverNode');
-    this.txFetcher = new RunningPromise(() => this.checkForTxs(), this.log, this.options.txGatheringIntervalMs);
+
+    this.jobMetrics = new ProverNodeJobMetrics(meter, telemetryClient.getTracer('EpochProvingJob'));
+
+    this.rewardsMetrics = new ProverNodeRewardsMetrics(
+      meter,
+      EthAddress.fromField(this.prover.getProverId()),
+      this.publisher.getRollupContract(),
+    );
+
+    this.txFetcher = new RunningPromise(() => this.checkForTxs(), this.log, this.config.txGatheringIntervalMs);
   }
 
   public getProverId() {
@@ -102,18 +114,15 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
   }
 
   public getP2P() {
-    const asP2PClient = this.coordination as P2P<P2PClientType.Prover>;
-    if (typeof asP2PClient.isP2PClient === 'function' && asP2PClient.isP2PClient()) {
-      return asP2PClient;
-    }
-    return undefined;
+    return this.coordination.getP2PClient();
   }
 
   /**
    * Handles an epoch being completed by starting a proof for it if there are no active jobs for it.
    * @param epochNumber - The epoch number that was just completed.
+   * @returns false if there is an error, true otherwise
    */
-  async handleEpochReadyToProve(epochNumber: bigint): Promise<void> {
+  async handleEpochReadyToProve(epochNumber: bigint): Promise<boolean> {
     try {
       this.log.debug(`Running jobs as ${epochNumber} is ready to prove`, {
         jobs: Array.from(this.jobs.values()).map(job => `${job.getEpochNumber()}:${job.getId()}`),
@@ -123,15 +132,17 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
         this.log.warn(`Not starting proof for ${epochNumber} since there are active jobs for the epoch`, {
           activeJobs: activeJobs.map(job => job.uuid),
         });
-        return;
+        return true;
       }
       await this.startProof(epochNumber);
+      return true;
     } catch (err) {
       if (err instanceof EmptyEpochError) {
         this.log.info(`Not starting proof for ${epochNumber} since no blocks were found`);
       } else {
         this.log.error(`Error handling epoch completed`, err);
       }
+      return false;
     }
   }
 
@@ -139,11 +150,12 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
    * Starts the prover node so it periodically checks for unproven epochs in the unfinalised chain from L1 and
    * starts proving jobs for them.
    */
-  start() {
+  async start() {
     this.txFetcher.start();
     this.epochsMonitor.start(this);
     this.l1Metrics.start();
-    this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.options);
+    await this.rewardsMetrics.start();
+    this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.config);
   }
 
   /**
@@ -160,13 +172,15 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
     await this.worldState.stop();
     await tryStop(this.coordination);
     this.l1Metrics.stop();
+    this.rewardsMetrics.stop();
     await this.telemetryClient.stop();
     this.log.info('Stopped ProverNode');
   }
 
   /** Returns world state status. */
-  public getWorldStateSyncStatus(): Promise<WorldStateSyncStatus> {
-    return this.worldState.status().then(s => s.syncSummary);
+  public async getWorldStateSyncStatus(): Promise<WorldStateSyncStatus> {
+    const { syncSummary } = await this.worldState.status();
+    return syncSummary;
   }
 
   /** Returns archiver status. */
@@ -183,23 +197,44 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
   }
 
   private async runJob(job: EpochProvingJob) {
-    const ctx = { id: job.getId(), epochNumber: job.getEpochNumber() };
+    const epochNumber = job.getEpochNumber();
+    const ctx = { id: job.getId(), epochNumber, state: undefined as EpochProvingJobState | undefined };
+
     try {
       await job.run();
       const state = job.getState();
+      ctx.state = state;
+
       if (state === 'reorg') {
-        this.log.warn(`Running new job for epoch ${job.getEpochNumber()} due to reorg`, ctx);
-        await this.createProvingJob(job.getEpochNumber());
+        this.log.warn(`Running new job for epoch ${epochNumber} due to reorg`, ctx);
+        await this.createProvingJob(epochNumber);
+      } else if (state === 'failed') {
+        this.log.error(`Job for ${epochNumber} exited with state ${state}`, ctx);
+        await this.tryUploadEpochFailure(job);
       } else {
-        this.log.verbose(`Job for ${job.getEpochNumber()} exited with state ${state}`, ctx);
+        this.log.verbose(`Job for ${epochNumber} exited with state ${state}`, ctx);
       }
     } catch (err) {
-      this.log.error(`Error proving epoch ${job.getEpochNumber()}`, err, ctx);
+      this.log.error(`Error proving epoch ${epochNumber}`, err, ctx);
     } finally {
       this.jobs.delete(job.getId());
     }
   }
 
+  protected async tryUploadEpochFailure(job: EpochProvingJob) {
+    if (this.config.proverNodeFailedEpochStore) {
+      return await uploadEpochProofFailure(
+        this.config.proverNodeFailedEpochStore,
+        job.getId(),
+        job.getProvingData(),
+        this.l2BlockSource as Archiver,
+        this.worldState,
+        assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')),
+        this.log,
+      );
+    }
+  }
+
   /**
    * Returns the prover instance.
    */
@@ -229,36 +264,38 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
   }
 
   private checkMaximumPendingJobs() {
-    const { maxPendingJobs } = this.options;
-    return maxPendingJobs === 0 || this.jobs.size < maxPendingJobs;
+    const { proverNodeMaxPendingJobs: maxPendingJobs } = this.config;
+    if (maxPendingJobs > 0 && this.jobs.size >= maxPendingJobs) {
+      throw new Error(`Maximum pending proving jobs ${maxPendingJobs} reached. Cannot create new job.`);
+    }
   }
 
   @trackSpan('ProverNode.createProvingJob', epochNumber => ({ [Attributes.EPOCH_NUMBER]: Number(epochNumber) }))
   private async createProvingJob(epochNumber: bigint) {
-    if (!this.checkMaximumPendingJobs()) {
-      throw new Error(`Maximum pending proving jobs ${this.options.maxPendingJobs} reached. Cannot create new job.`);
-    }
+    this.checkMaximumPendingJobs();
 
-    // Gather blocks for this epoch
-    const { blocks, txs } = await this.gatherEpochData(epochNumber);
+    // Gather all data for this epoch
+    const epochData = await this.gatherEpochData(epochNumber);
 
-    const fromBlock = blocks[0].number;
-    const toBlock = blocks.at(-1)!.number;
+    const fromBlock = epochData.blocks[0].number;
+    const toBlock = epochData.blocks.at(-1)!.number;
+    this.log.verbose(`Creating proving job for epoch ${epochNumber} for block range ${fromBlock} to ${toBlock}`);
 
     // Fast forward world state to right before the target block and get a fork
-    this.log.verbose(`Creating proving job for epoch ${epochNumber} for block range ${fromBlock} to ${toBlock}`);
     await this.worldState.syncImmediate(toBlock);
 
-    // Create a processor using the forked world state
+    // Create a processor factory
     const publicProcessorFactory = new PublicProcessorFactory(
       this.contractDataSource,
       this.dateProvider,
       this.telemetryClient,
     );
 
+    // Set deadline for this job to run. It will abort if it takes too long.
     const deadlineTs = getProofSubmissionDeadlineTimestamp(epochNumber, await this.getL1Constants());
     const deadline = new Date(Number(deadlineTs) * 1000);
-    const job = this.doCreateEpochProvingJob(epochNumber, deadline, blocks, txs, publicProcessorFactory);
+
+    const job = this.doCreateEpochProvingJob(epochData, deadline, publicProcessorFactory);
     this.jobs.set(job.getId(), job);
     return job;
   }
@@ -279,18 +316,19 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
       }
       const txHashes = block.body.txEffects.map(tx => tx.txHash);
       this.log.verbose(`Fetching ${txHashes.length} tx hashes for block number ${blockNumber} from coordination`);
-      await this.coordination.getTxsByHash(txHashes); // This stores the txs in the tx pool, no need to persist them here
+      await this.coordination.gatherTxs(txHashes); // This stores the txs in the tx pool, no need to persist them here
       this.lastBlockNumber = blockNumber;
     }
   }
 
   @trackSpan('ProverNode.gatherEpochData', epochNumber => ({ [Attributes.EPOCH_NUMBER]: Number(epochNumber) }))
-  private async gatherEpochData(epochNumber: bigint) {
-    // Gather blocks for this epoch and their txs
+  private async gatherEpochData(epochNumber: bigint): Promise<EpochProvingJobData> {
     const blocks = await this.gatherBlocks(epochNumber);
     const txs = await this.gatherTxs(epochNumber, blocks);
+    const l1ToL2Messages = await this.gatherMessages(epochNumber, blocks);
+    const previousBlockHeader = await this.gatherPreviousBlockHeader(epochNumber, blocks[0]);
 
-    return { blocks, txs };
+    return { blocks, txs, l1ToL2Messages, epochNumber, previousBlockHeader };
   }
 
   private async gatherBlocks(epochNumber: bigint) {
@@ -318,27 +356,48 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
     throw new Error(`Txs not found for epoch ${epochNumber}: ${missingTxHashes}`);
   }
 
+  private async gatherMessages(epochNumber: bigint, blocks: L2Block[]) {
+    const messages = await Promise.all(blocks.map(b => this.l1ToL2MessageSource.getL1ToL2Messages(BigInt(b.number))));
+    const messageCount = sum(messages.map(m => m.length));
+    this.log.verbose(`Gathered all ${messageCount} messages for epoch ${epochNumber}`, { epochNumber });
+    const messagesByBlock: Record<number, Fr[]> = {};
+    for (let i = 0; i < blocks.length; i++) {
+      messagesByBlock[blocks[i].number] = messages[i];
+    }
+    return messagesByBlock;
+  }
+
+  private async gatherPreviousBlockHeader(epochNumber: bigint, initialBlock: L2Block) {
+    const previousBlockNumber = initialBlock.number - 1;
+    const header = await (previousBlockNumber === 0
+      ? this.worldState.getCommitted().getInitialHeader()
+      : this.l2BlockSource.getBlockHeader(previousBlockNumber));
+
+    if (!header) {
+      throw new Error(`Previous block header ${initialBlock.number} not found for proving epoch ${epochNumber}`);
+    }
+
+    this.log.verbose(`Gathered previous block header ${header.getBlockNumber()} for epoch ${epochNumber}`);
+    return header;
+  }
+
   /** Extracted for testing purposes. */
   protected doCreateEpochProvingJob(
-    epochNumber: bigint,
+    data: EpochProvingJobData,
     deadline: Date | undefined,
-    blocks: L2Block[],
-    txs: Tx[],
     publicProcessorFactory: PublicProcessorFactory,
   ) {
+    const { proverNodeMaxParallelBlocksPerEpoch: parallelBlockLimit } = this.config;
     return new EpochProvingJob(
+      data,
       this.worldState,
-      epochNumber,
-      blocks,
-      txs,
       this.prover.createEpochProver(),
       publicProcessorFactory,
       this.publisher,
       this.l2BlockSource,
-      this.l1ToL2MessageSource,
-      this.metrics,
+      this.jobMetrics,
       deadline,
-      { parallelBlockLimit: this.options.maxParallelBlocksPerEpoch },
+      { parallelBlockLimit },
     );
   }
 
@@ -346,6 +405,21 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
   protected async triggerMonitors() {
     await this.epochsMonitor.work();
   }
+
+  private validateConfig() {
+    if (
+      this.config.proverNodeFailedEpochStore &&
+      (!this.config.dataDirectory || !this.config.l1ChainId || !this.config.rollupVersion)
+    ) {
+      this.log.warn(
+        `Invalid prover-node config (missing dataDirectory, l1ChainId, or rollupVersion)`,
+        pick(this.config, 'proverNodeFailedEpochStore', 'dataDirectory', 'l1ChainId', 'rollupVersion'),
+      );
+      throw new Error(
+        'All of dataDirectory, l1ChainId, and rollupVersion are required if proverNodeFailedEpochStore is set.',
+      );
+    }
+  }
 }
 
 class EmptyEpochError extends Error {

package/src/test/index.ts

@@ -1,11 +1,14 @@
 import type { EpochProverManager } from '@aztec/stdlib/interfaces/server';
 
+import type { EpochProvingJob } from '../job/epoch-proving-job.js';
 import type { ProverNodePublisher } from '../prover-node-publisher.js';
 import { ProverNode } from '../prover-node.js';
 
-class TestProverNode_ extends ProverNode {
+abstract class TestProverNodeClass extends ProverNode {
   public declare prover: EpochProverManager;
   public declare publisher: ProverNodePublisher;
+
+  public abstract override tryUploadEpochFailure(job: EpochProvingJob): Promise<string | undefined>;
 }
 
-export type TestProverNode = TestProverNode_;
+export type TestProverNode = TestProverNodeClass;