@aztec/prover-node 0.0.1-commit.fffb133c → 0.0.1-dev

package/src/metrics.ts

@@ -13,6 +13,7 @@ import {
   type TelemetryClient,
   type Tracer,
   type UpDownCounter,
+  createUpDownCounterWithDefault,
 } from '@aztec/telemetry-client';
 
 import { formatEther, formatUnits } from 'viem';
@@ -24,6 +25,12 @@ export class ProverNodeJobMetrics {
   provingJobBlocks: Gauge;
   provingJobTransactions: Gauge;
 
+  private blobProcessingDuration: Gauge;
+  private chonkVerifierDuration: Gauge;
+  private blockProcessingDuration: Histogram;
+  private checkpointProcessingDuration: Histogram;
+  private allCheckpointsProcessingDuration: Gauge;
+
   constructor(
     private meter: Meter,
     public readonly tracer: Tracer,
@@ -34,6 +41,14 @@ export class ProverNodeJobMetrics {
     this.provingJobCheckpoints = this.meter.createGauge(Metrics.PROVER_NODE_JOB_CHECKPOINTS);
     this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS);
     this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS);
+
+    this.blobProcessingDuration = this.meter.createGauge(Metrics.PROVER_NODE_BLOB_PROCESSING_LAST_DURATION);
+    this.chonkVerifierDuration = this.meter.createGauge(Metrics.PROVER_NODE_CHONK_VERIFIER_LAST_DURATION);
+    this.blockProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_BLOCK_PROCESSING_DURATION);
+    this.checkpointProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_PROCESSING_DURATION);
+    this.allCheckpointsProcessingDuration = this.meter.createGauge(
+      Metrics.PROVER_NODE_ALL_CHECKPOINTS_PROCESSING_LAST_DURATION,
+    );
   }
 
   public recordProvingJob(
@@ -49,6 +64,26 @@ export class ProverNodeJobMetrics {
     this.provingJobBlocks.record(Math.floor(numBlocks));
     this.provingJobTransactions.record(Math.floor(numTxs));
   }
+
+  public recordBlobProcessing(durationMs: number) {
+    this.blobProcessingDuration.record(Math.ceil(durationMs));
+  }
+
+  public recordChonkVerifier(durationMs: number) {
+    this.chonkVerifierDuration.record(Math.ceil(durationMs));
+  }
+
+  public recordBlockProcessing(durationMs: number) {
+    this.blockProcessingDuration.record(Math.ceil(durationMs));
+  }
+
+  public recordCheckpointProcessing(durationMs: number) {
+    this.checkpointProcessingDuration.record(Math.ceil(durationMs));
+  }
+
+  public recordAllCheckpointsProcessing(durationMs: number) {
+    this.allCheckpointsProcessingDuration.record(Math.ceil(durationMs));
+  }
 }
 
 export class ProverNodeRewardsMetrics {
@@ -65,7 +100,7 @@ export class ProverNodeRewardsMetrics {
   ) {
     this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH);
 
-    this.accumulatedRewards = this.meter.createUpDownCounter(Metrics.PROVER_NODE_REWARDS_TOTAL);
+    this.accumulatedRewards = createUpDownCounterWithDefault(this.meter, Metrics.PROVER_NODE_REWARDS_TOTAL);
   }
 
   public async start() {
@@ -128,7 +163,10 @@ export class ProverNodePublisherMetrics {
 
     this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE);
 
-    this.txCount = this.meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT);
+    this.txCount = createUpDownCounterWithDefault(this.meter, Metrics.L1_PUBLISHER_TX_COUNT, {
+      [Attributes.L1_TX_TYPE]: ['submitProof'],
+      [Attributes.OK]: [true, false],
+    });
 
     this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION);
 

package/src/monitors/epoch-monitor.ts

@@ -69,9 +69,8 @@ export class EpochMonitor implements Traceable {
 
   public async work() {
     const { epochToProve, blockNumber, slotNumber } = await this.getEpochNumberToProve();
-    this.log.debug(`Epoch to prove: ${epochToProve}`, { blockNumber, slotNumber });
     if (epochToProve === undefined) {
-      this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { blockNumber });
+      this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { epochToProve, blockNumber, slotNumber });
       return;
     }
     if (this.latestEpochNumber !== undefined && epochToProve <= this.latestEpochNumber) {
@@ -86,11 +85,11 @@ export class EpochMonitor implements Traceable {
     }
 
     if (this.options.provingDelayMs) {
-      this.log.debug(`Waiting ${this.options.provingDelayMs}ms before proving epoch ${epochToProve}`);
+      this.log.warn(`Waiting ${this.options.provingDelayMs}ms before proving epoch ${epochToProve}`);
       await sleep(this.options.provingDelayMs);
     }
 
-    this.log.debug(`Epoch ${epochToProve} is ready to be proven`);
+    this.log.verbose(`Epoch ${epochToProve} is ready to be proven`);
     if (await this.handler?.handleEpochReadyToProve(epochToProve)) {
       this.latestEpochNumber = epochToProve;
     }

package/src/prover-node-publisher.ts

@@ -1,5 +1,5 @@
 import { BatchedBlob, getEthBlobEvaluationInputs } from '@aztec/blob-lib';
-import { AZTEC_MAX_EPOCH_DURATION } from '@aztec/constants';
+import { MAX_CHECKPOINTS_PER_EPOCH } from '@aztec/constants';
 import type { RollupContract, ViemCommitteeAttestation } from '@aztec/ethereum/contracts';
 import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils';
 import { makeTuple } from '@aztec/foundation/array';
@@ -7,12 +7,14 @@ import { CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
 import { areArraysEqual } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import { EthAddress } from '@aztec/foundation/eth-address';
-import { createLogger } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
+import { retryUntil } from '@aztec/foundation/retry';
 import type { Tuple } from '@aztec/foundation/serialize';
 import { Timer } from '@aztec/foundation/timer';
 import { RollupAbi } from '@aztec/l1-artifacts';
 import type { PublisherConfig, TxSenderConfig } from '@aztec/sequencer-client';
 import { CommitteeAttestation, CommitteeAttestationsAndSigners } from '@aztec/stdlib/block';
+import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
 import type { Proof } from '@aztec/stdlib/proofs';
 import type { FeeRecipient, RootRollupPublicInputs } from '@aztec/stdlib/rollup';
 import type { L1PublishProofStats } from '@aztec/stdlib/stats';
@@ -31,7 +33,7 @@ export type L1SubmitEpochProofArgs = {
   endTimestamp: Fr;
   outHash: Fr;
   proverId: Fr;
-  fees: Tuple<FeeRecipient, typeof AZTEC_MAX_EPOCH_DURATION>;
+  fees: Tuple<FeeRecipient, typeof MAX_CHECKPOINTS_PER_EPOCH>;
   proof: Proof;
 };
 
@@ -39,7 +41,7 @@ export class ProverNodePublisher {
   private interrupted = false;
   private metrics: ProverNodePublisherMetrics;
 
-  protected log = createLogger('prover-node:l1-tx-publisher');
+  protected log: Logger;
 
   protected rollupContract: RollupContract;
 
@@ -52,10 +54,12 @@ export class ProverNodePublisher {
       l1TxUtils: L1TxUtils;
       telemetry?: TelemetryClient;
     },
+    bindings?: LoggerBindings,
   ) {
     const telemetry = deps.telemetry ?? getTelemetryClient();
 
     this.metrics = new ProverNodePublisherMetrics(telemetry, 'ProverNode');
+    this.log = createLogger('prover-node:l1-tx-publisher', bindings);
 
     this.rollupContract = deps.rollupContract;
     this.l1TxUtils = deps.l1TxUtils;
@@ -99,6 +103,11 @@ export class ProverNodePublisher {
     const ctx = { epochNumber, fromCheckpoint, toCheckpoint };
 
     if (!this.interrupted) {
+      if (!(await this.waitUntilStartBuildsOnProven(args))) {
+        this.log.verbose('Checkpoint data syncing interrupted', ctx);
+        return false;
+      }
+
       const timer = new Timer();
       // Validate epoch proof range and hashes are correct before submitting
       await this.validateEpochProofSubmission(args);
@@ -145,6 +154,53 @@ export class ProverNodePublisher {
     return false;
   }
 
+  private async waitUntilStartBuildsOnProven(args: { epochNumber: EpochNumber; fromCheckpoint: CheckpointNumber }) {
+    const { epochNumber, fromCheckpoint } = args;
+    const provenCheckpoint = await this.getProvenCheckpoint();
+    if (this.isStartBuildingOnProven(fromCheckpoint, provenCheckpoint)) {
+      return true;
+    }
+
+    const timeout = await this.getSecondsUntilProofSubmissionWindowEnd(epochNumber);
+    this.log.info(`Waiting for proven checkpoint to reach proof start`, {
+      epochNumber,
+      fromCheckpoint,
+      provenCheckpoint,
+      timeout,
+    });
+
+    await retryUntil(
+      async () => {
+        if (this.interrupted) {
+          return true;
+        }
+
+        const proven = await this.getProvenCheckpoint();
+        this.log.verbose(`Proven checkpoint is at ${proven} (waiting for ${fromCheckpoint - 1})`, { epochNumber });
+        return this.isStartBuildingOnProven(fromCheckpoint, proven) ? true : undefined;
+      },
+      `proven checkpoint to reach ${fromCheckpoint - 1}`,
+      timeout,
+      4,
+    );
+
+    return !this.interrupted;
+  }
+
+  private async getProvenCheckpoint() {
+    return (await this.rollupContract.getTips()).proven;
+  }
+
+  private isStartBuildingOnProven(fromCheckpoint: CheckpointNumber, provenCheckpoint: CheckpointNumber) {
+    return fromCheckpoint - 1 <= provenCheckpoint;
+  }
+
+  private async getSecondsUntilProofSubmissionWindowEnd(epochNumber: EpochNumber) {
+    const deadline = getProofSubmissionDeadlineTimestamp(epochNumber, await this.rollupContract.getRollupConstants());
+    const now = BigInt(Math.floor(Date.now() / 1000));
+    return Math.max(Number(deadline - now), 0.001);
+  }
+
   private async validateEpochProofSubmission(args: {
     fromCheckpoint: CheckpointNumber;
     toCheckpoint: CheckpointNumber;
@@ -269,7 +325,7 @@ export class ProverNodePublisher {
         outHash: args.publicInputs.outHash.toString(),
         proverId: EthAddress.fromField(args.publicInputs.constants.proverId).toString(),
       } /*_args*/,
-      makeTuple(AZTEC_MAX_EPOCH_DURATION * 2, i =>
+      makeTuple(MAX_CHECKPOINTS_PER_EPOCH * 2, i =>
         i % 2 === 0
           ? args.publicInputs.fees[i / 2].recipient.toField().toString()
           : args.publicInputs.fees[(i - 1) / 2].value.toString(),

package/src/prover-node.ts

@@ -1,5 +1,6 @@
 import type { Archiver } from '@aztec/archiver';
 import type { RollupContract } from '@aztec/ethereum/contracts';
+import type { Delayer } from '@aztec/ethereum/l1-tx-utils';
 import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
 import { assertRequired, compact, pick, sum } from '@aztec/foundation/collection';
 import type { Fr } from '@aztec/foundation/curves/bn254';
@@ -7,7 +8,6 @@ import { memoize } from '@aztec/foundation/decorators';
 import { createLogger } from '@aztec/foundation/log';
 import { DateProvider } from '@aztec/foundation/timer';
 import type { DataStoreConfig } from '@aztec/kv-store/config';
-import type { P2PClient } from '@aztec/p2p';
 import { PublicProcessorFactory } from '@aztec/simulator/server';
 import type { L2BlockSource } from '@aztec/stdlib/block';
 import type { Checkpoint } from '@aztec/stdlib/checkpoint';
@@ -17,6 +17,7 @@ import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers
 import {
   type EpochProverManager,
   EpochProvingJobTerminalState,
+  type ITxProvider,
   type ProverNodeApi,
   type Service,
   type WorldStateSyncStatus,
@@ -24,7 +25,6 @@ import {
   tryStop,
 } from '@aztec/stdlib/interfaces/server';
 import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
-import type { P2PClientType } from '@aztec/stdlib/p2p';
 import type { Tx } from '@aztec/stdlib/tx';
 import {
   Attributes,
@@ -55,7 +55,6 @@ type DataStoreOptions = Pick<DataStoreConfig, 'dataDirectory'> & Pick<ChainConfi
  */
 export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable {
   private log = createLogger('prover-node');
-  private dateProvider = new DateProvider();
 
   private jobs: Map<string, EpochProvingJob> = new Map();
   private config: ProverNodeOptions;
@@ -73,17 +72,19 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
     protected readonly l1ToL2MessageSource: L1ToL2MessageSource,
     protected readonly contractDataSource: ContractDataSource,
     protected readonly worldState: WorldStateSynchronizer,
-    protected readonly p2pClient: Pick<P2PClient<P2PClientType.Prover>, 'getTxProvider'> & Partial<Service>,
+    protected readonly p2pClient: { getTxProvider(): ITxProvider } & Partial<Service>,
     protected readonly epochsMonitor: EpochMonitor,
     protected readonly rollupContract: RollupContract,
     protected readonly l1Metrics: L1Metrics,
     config: Partial<ProverNodeOptions> = {},
     protected readonly telemetryClient: TelemetryClient = getTelemetryClient(),
+    private delayer?: Delayer,
+    private readonly dateProvider: DateProvider = new DateProvider(),
   ) {
     this.config = {
       proverNodePollingIntervalMs: 1_000,
       proverNodeMaxPendingJobs: 100,
-      proverNodeMaxParallelBlocksPerEpoch: 32,
+      proverNodeMaxParallelBlocksPerEpoch: 0,
       txGatheringIntervalMs: 1_000,
       txGatheringBatchSize: 10,
       txGatheringMaxParallelRequestsPerNode: 100,
@@ -111,6 +112,11 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
     return this.p2pClient;
   }
 
+  /** Returns the shared tx delayer for prover L1 txs, if enabled. Test-only. */
+  public getDelayer(): Delayer | undefined {
+    return this.delayer;
+  }
+
   /**
    * Handles an epoch being completed by starting a proof for it if there are no active jobs for it.
    * @param epochNumber - The epoch number that was just completed.
@@ -155,17 +161,15 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
 
   /**
    * Stops the prover node and all its dependencies.
+   * Resources not owned by this node (shared with the parent aztec-node) are skipped.
    */
   async stop() {
     this.log.info('Stopping ProverNode');
     await this.epochsMonitor.stop();
     await this.prover.stop();
-    await tryStop(this.p2pClient);
-    await tryStop(this.l2BlockSource);
     await tryStop(this.publisherFactory);
     this.publisher?.interrupt();
     await Promise.all(Array.from(this.jobs.values()).map(job => job.stop()));
-    await this.worldState.stop();
     this.rewardsMetrics.stop();
     this.l1Metrics.stop();
     await this.telemetryClient.stop();
@@ -275,19 +279,22 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
     const fromCheckpoint = epochData.checkpoints[0].number;
     const toCheckpoint = epochData.checkpoints.at(-1)!.number;
     const fromBlock = epochData.checkpoints[0].blocks[0].number;
-    const toBlock = epochData.checkpoints.at(-1)!.blocks.at(-1)!.number;
+    const lastBlock = epochData.checkpoints.at(-1)!.blocks.at(-1)!;
+    const toBlock = lastBlock.number;
     this.log.verbose(
       `Creating proving job for epoch ${epochNumber} for checkpoint range ${fromCheckpoint} to ${toCheckpoint} and block range ${fromBlock} to ${toBlock}`,
     );
 
     // Fast forward world state to right before the target block and get a fork
-    await this.worldState.syncImmediate(toBlock);
+    const lastBlockHash = await lastBlock.header.hash();
+    await this.worldState.syncImmediate(toBlock, lastBlockHash);
 
     // Create a processor factory
     const publicProcessorFactory = new PublicProcessorFactory(
       this.contractDataSource,
       this.dateProvider,
       this.telemetryClient,
+      this.log.getBindings(),
     );
 
     // Set deadline for this job to run. It will abort if it takes too long.
@@ -384,6 +391,7 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
       this.jobMetrics,
       deadline,
       { parallelBlockLimit, skipSubmitProof: proverNodeDisableProofPublish, ...opts },
+      this.log.getBindings(),
     );
   }
 

package/src/prover-publisher-factory.ts

@@ -1,19 +1,21 @@
 import type { RollupContract } from '@aztec/ethereum/contracts';
 import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils';
 import type { PublisherManager } from '@aztec/ethereum/publisher-manager';
-import type { PublisherConfig, TxSenderConfig } from '@aztec/sequencer-client';
+import type { LoggerBindings } from '@aztec/foundation/log';
+import type { ProverPublisherConfig, ProverTxSenderConfig } from '@aztec/sequencer-client';
 import type { TelemetryClient } from '@aztec/telemetry-client';
 
 import { ProverNodePublisher } from './prover-node-publisher.js';
 
 export class ProverPublisherFactory {
   constructor(
-    private config: TxSenderConfig & PublisherConfig,
+    private config: ProverTxSenderConfig & ProverPublisherConfig,
     private deps: {
       rollupContract: RollupContract;
       publisherManager: PublisherManager<L1TxUtils>;
       telemetry?: TelemetryClient;
     },
+    private bindings?: LoggerBindings,
   ) {}
 
   public async start() {
@@ -30,10 +32,14 @@ export class ProverPublisherFactory {
    */
   public async create(): Promise<ProverNodePublisher> {
     const l1Publisher = await this.deps.publisherManager.getAvailablePublisher();
-    return new ProverNodePublisher(this.config, {
-      rollupContract: this.deps.rollupContract,
-      l1TxUtils: l1Publisher,
-      telemetry: this.deps.telemetry,
-    });
+    return new ProverNodePublisher(
+      this.config,
+      {
+        rollupContract: this.deps.rollupContract,
+        l1TxUtils: l1Publisher,
+        telemetry: this.deps.telemetry,
+      },
+      this.bindings,
+    );
   }
 }