@aztec/prover-node 0.69.0 → 0.69.1-devnet

package/src/metrics.ts

@@ -1,13 +1,18 @@
-import { type Timer } from '@aztec/foundation/timer';
 import { type Histogram, Metrics, type TelemetryClient, ValueType } from '@aztec/telemetry-client';
 
 export class ProverNodeMetrics {
+  proverEpochExecutionDuration: Histogram;
   provingJobDuration: Histogram;
   provingJobBlocks: Histogram;
   provingJobTransactions: Histogram;
 
   constructor(public readonly client: TelemetryClient, name = 'ProverNode') {
     const meter = client.getMeter(name);
+    this.proverEpochExecutionDuration = meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, {
+      description: 'Duration of execution of an epoch by the prover',
+      unit: 'ms',
+      valueType: ValueType.INT,
+    });
     this.provingJobDuration = meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
       description: 'Duration of proving job',
       unit: 'ms',
@@ -23,9 +28,9 @@ export class ProverNodeMetrics {
     });
   }
 
-  public recordProvingJob(timerOrMs: Timer | number, numBlocks: number, numTxs: number) {
-    const ms = Math.ceil(typeof timerOrMs === 'number' ? timerOrMs : timerOrMs.ms());
-    this.provingJobDuration.record(ms);
+  public recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number) {
+    this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
+    this.provingJobDuration.record(Math.ceil(totalTimeMs));
     this.provingJobBlocks.record(Math.floor(numBlocks));
     this.provingJobTransactions.record(Math.floor(numTxs));
   }

package/src/prover-node.ts

@@ -10,12 +10,19 @@ import {
   type ProverCoordination,
   type ProverNodeApi,
   type Service,
+  type Tx,
+  type TxHash,
   type WorldStateSynchronizer,
+  getTimestampRangeForEpoch,
   tryStop,
 } from '@aztec/circuit-types';
 import { type ContractDataSource } from '@aztec/circuits.js';
+import { asyncPool } from '@aztec/foundation/async-pool';
 import { compact } from '@aztec/foundation/collection';
+import { memoize } from '@aztec/foundation/decorators';
+import { TimeoutError } from '@aztec/foundation/error';
 import { createLogger } from '@aztec/foundation/log';
+import { retryUntil } from '@aztec/foundation/retry';
 import { DateProvider } from '@aztec/foundation/timer';
 import { type Maybe } from '@aztec/foundation/types';
 import { type P2P } from '@aztec/p2p';
@@ -35,6 +42,9 @@ export type ProverNodeOptions = {
   pollingIntervalMs: number;
   maxPendingJobs: number;
   maxParallelBlocksPerEpoch: number;
+  txGatheringTimeoutMs: number;
+  txGatheringIntervalMs: number;
+  txGatheringMaxParallelRequests: number;
 };
 
 /**
@@ -49,31 +59,35 @@ export class ProverNode implements ClaimsMonitorHandler, EpochMonitorHandler, Pr
 
   private latestEpochWeAreProving: bigint | undefined;
   private jobs: Map<string, EpochProvingJob> = new Map();
+  private cachedEpochData: { epochNumber: bigint; blocks: L2Block[]; txs: Tx[] } | undefined = undefined;
   private options: ProverNodeOptions;
   private metrics: ProverNodeMetrics;
 
   public readonly tracer: Tracer;
 
   constructor(
-    private readonly prover: EpochProverManager,
-    private readonly publisher: L1Publisher,
-    private readonly l2BlockSource: L2BlockSource & Maybe<Service>,
-    private readonly l1ToL2MessageSource: L1ToL2MessageSource,
-    private readonly contractDataSource: ContractDataSource,
-    private readonly worldState: WorldStateSynchronizer,
-    private readonly coordination: ProverCoordination & Maybe<Service>,
-    private readonly quoteProvider: QuoteProvider,
-    private readonly quoteSigner: QuoteSigner,
-    private readonly claimsMonitor: ClaimsMonitor,
-    private readonly epochsMonitor: EpochMonitor,
-    private readonly bondManager: BondManager,
-    private readonly telemetryClient: TelemetryClient,
+    protected readonly prover: EpochProverManager,
+    protected readonly publisher: L1Publisher,
+    protected readonly l2BlockSource: L2BlockSource & Maybe<Service>,
+    protected readonly l1ToL2MessageSource: L1ToL2MessageSource,
+    protected readonly contractDataSource: ContractDataSource,
+    protected readonly worldState: WorldStateSynchronizer,
+    protected readonly coordination: ProverCoordination & Maybe<Service>,
+    protected readonly quoteProvider: QuoteProvider,
+    protected readonly quoteSigner: QuoteSigner,
+    protected readonly claimsMonitor: ClaimsMonitor,
+    protected readonly epochsMonitor: EpochMonitor,
+    protected readonly bondManager: BondManager,
+    protected readonly telemetryClient: TelemetryClient,
     options: Partial<ProverNodeOptions> = {},
   ) {
     this.options = {
       pollingIntervalMs: 1_000,
       maxPendingJobs: 100,
       maxParallelBlocksPerEpoch: 32,
+      txGatheringTimeoutMs: 60_000,
+      txGatheringIntervalMs: 1_000,
+      txGatheringMaxParallelRequests: 100,
       ...compact(options),
     };
 
@@ -139,13 +153,12 @@ export class ProverNode implements ClaimsMonitorHandler, EpochMonitorHandler, Pr
    */
   async handleEpochCompleted(epochNumber: bigint): Promise<void> {
     try {
-      // Construct a quote for the epoch
-      const blocks = await this.l2BlockSource.getBlocksForEpoch(epochNumber);
-      if (blocks.length === 0) {
-        this.log.info(`No blocks found for epoch ${epochNumber}`);
-        return;
-      }
+      // Gather data for the epoch
+      const epochData = await this.gatherEpochData(epochNumber);
+      const { blocks } = epochData;
+      this.cachedEpochData = { epochNumber, ...epochData };
 
+      // Construct a quote for the epoch
       const partialQuote = await this.quoteProvider.getQuote(Number(epochNumber), blocks);
       if (!partialQuote) {
         this.log.info(`No quote produced for epoch ${epochNumber}`);
@@ -256,10 +269,9 @@ export class ProverNode implements ClaimsMonitorHandler, EpochMonitorHandler, Pr
     }
 
     // Gather blocks for this epoch
-    const blocks = await this.l2BlockSource.getBlocksForEpoch(epochNumber);
-    if (blocks.length === 0) {
-      throw new Error(`No blocks found for epoch ${epochNumber}`);
-    }
+    const cachedEpochData = this.cachedEpochData?.epochNumber === epochNumber ? this.cachedEpochData : undefined;
+    const { blocks, txs } = cachedEpochData ?? (await this.gatherEpochData(epochNumber));
+
     const fromBlock = blocks[0].number;
     const toBlock = blocks.at(-1)!.number;
 
@@ -279,15 +291,107 @@ export class ProverNode implements ClaimsMonitorHandler, EpochMonitorHandler, Pr
       return Promise.resolve();
     };
 
-    const job = this.doCreateEpochProvingJob(epochNumber, blocks, publicProcessorFactory, cleanUp);
+    const [_, endTimestamp] = getTimestampRangeForEpoch(epochNumber + 1n, await this.getL1Constants());
+    const deadline = new Date(Number(endTimestamp) * 1000);
+
+    const job = this.doCreateEpochProvingJob(epochNumber, deadline, blocks, txs, publicProcessorFactory, cleanUp);
     this.jobs.set(job.getId(), job);
     return job;
   }
 
+  @memoize
+  private getL1Constants() {
+    return this.l2BlockSource.getL1Constants();
+  }
+
+  @trackSpan('ProverNode.gatherEpochData', epochNumber => ({ [Attributes.EPOCH_NUMBER]: Number(epochNumber) }))
+  private async gatherEpochData(epochNumber: bigint) {
+    // Gather blocks for this epoch and their txs
+    const blocks = await this.gatherBlocks(epochNumber);
+    const txs = await this.gatherTxs(epochNumber, blocks);
+
+    return { blocks, txs };
+  }
+
+  private async gatherBlocks(epochNumber: bigint) {
+    const blocks = await this.l2BlockSource.getBlocksForEpoch(epochNumber);
+    if (blocks.length === 0) {
+      throw new Error(`No blocks found for epoch ${epochNumber}`);
+    }
+    return blocks;
+  }
+
+  private async gatherTxs(epochNumber: bigint, blocks: L2Block[]) {
+    let txsToFind: TxHash[] = [];
+    const txHashToBlock = new Map<string, number>();
+    const results = new Map<string, Tx>();
+
+    for (const block of blocks) {
+      for (const tx of block.body.txEffects) {
+        txsToFind.push(tx.txHash);
+        txHashToBlock.set(tx.txHash.toString(), block.number);
+      }
+    }
+
+    const totalTxsRequired = txsToFind.length;
+    this.log.info(
+      `Gathering a total of ${totalTxsRequired} txs for epoch=${epochNumber} made up of ${blocks.length} blocks`,
+      { epochNumber },
+    );
+
+    let iteration = 0;
+    try {
+      await retryUntil(
+        async () => {
+          const batch = [...txsToFind];
+          txsToFind = [];
+          const batchResults = await asyncPool(this.options.txGatheringMaxParallelRequests, batch, async txHash => {
+            const tx = await this.coordination.getTxByHash(txHash);
+            return [txHash, tx] as const;
+          });
+          let found = 0;
+          for (const [txHash, maybeTx] of batchResults) {
+            if (maybeTx) {
+              found++;
+              results.set(txHash.toString(), maybeTx);
+            } else {
+              txsToFind.push(txHash);
+            }
+          }
+
+          this.log.verbose(
+            `Gathered ${found}/${batch.length} txs in iteration ${iteration} for epoch ${epochNumber}. In total ${results.size}/${totalTxsRequired} have been retrieved.`,
+            { epochNumber },
+          );
+          iteration++;
+
+          // stop when we found all transactions
+          return txsToFind.length === 0;
+        },
+        'Gather txs',
+        this.options.txGatheringTimeoutMs / 1_000,
+        this.options.txGatheringIntervalMs / 1_000,
+      );
+    } catch (err) {
+      if (err && err instanceof TimeoutError) {
+        const notFoundList = txsToFind
+          .map(txHash => `${txHash.toString()} (block ${txHashToBlock.get(txHash.toString())})`)
+          .join(', ');
+        throw new Error(`Txs not found for epoch ${epochNumber}: ${notFoundList}`);
+      } else {
+        throw err;
+      }
+    }
+
+    return Array.from(results.values());
+  }
+
   /** Extracted for testing purposes. */
   protected doCreateEpochProvingJob(
     epochNumber: bigint,
+    deadline: Date | undefined,
     blocks: L2Block[],
+    txs: Tx[],
     publicProcessorFactory: PublicProcessorFactory,
     cleanUp: () => Promise<void>,
   ) {
@@ -295,13 +399,14 @@ export class ProverNode implements ClaimsMonitorHandler, EpochMonitorHandler, Pr
       this.worldState,
       epochNumber,
       blocks,
+      txs,
       this.prover.createEpochProver(),
       publicProcessorFactory,
       this.publisher,
       this.l2BlockSource,
       this.l1ToL2MessageSource,
-      this.coordination,
       this.metrics,
+      deadline,
       { parallelBlockLimit: this.options.maxParallelBlocksPerEpoch },
       cleanUp,
     );

package/src/test/index.ts

@@ -0,0 +1,11 @@
+import { type EpochProverManager } from '@aztec/circuit-types';
+import { type L1Publisher } from '@aztec/sequencer-client';
+
+import { ProverNode } from '../prover-node.js';
+
+class TestProverNode_ extends ProverNode {
+  public override prover!: EpochProverManager;
+  public override publisher!: L1Publisher;
+}
+
+export type TestProverNode = TestProverNode_;