@aztec/prover-node 0.0.1-commit.b655e406 → 0.0.1-commit.b9865e97

package/src/metrics.ts

@@ -1,4 +1,4 @@
-import type { RollupContract } from '@aztec/ethereum';
+import type { RollupContract } from '@aztec/ethereum/contracts';
 import type { EthAddress } from '@aztec/foundation/eth-address';
 import { createLogger } from '@aztec/foundation/log';
 import type { L1PublishProofStats, L1PublishStats } from '@aztec/stdlib/stats';
@@ -13,7 +13,7 @@ import {
   type TelemetryClient,
   type Tracer,
   type UpDownCounter,
-  ValueType,
+  createUpDownCounterWithDefault,
 } from '@aztec/telemetry-client';
 
 import { formatEther, formatUnits } from 'viem';
@@ -21,40 +21,69 @@ import { formatEther, formatUnits } from 'viem';
 export class ProverNodeJobMetrics {
   proverEpochExecutionDuration: Histogram;
   provingJobDuration: Histogram;
+  provingJobCheckpoints: Gauge;
   provingJobBlocks: Gauge;
   provingJobTransactions: Gauge;
 
+  private blobProcessingDuration: Gauge;
+  private chonkVerifierDuration: Gauge;
+  private blockProcessingDuration: Histogram;
+  private checkpointProcessingDuration: Histogram;
+  private allCheckpointsProcessingDuration: Gauge;
+
   constructor(
     private meter: Meter,
     public readonly tracer: Tracer,
     private logger = createLogger('prover-node:publisher:metrics'),
   ) {
-    this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, {
-      description: 'Duration of execution of an epoch by the prover',
-      unit: 'ms',
-      valueType: ValueType.INT,
-    });
-    this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
-      description: 'Duration of proving job',
-      unit: 's',
-      valueType: ValueType.DOUBLE,
-    });
-    this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
-      description: 'Number of blocks in a proven epoch',
-      valueType: ValueType.INT,
-    });
-    this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
-      description: 'Number of transactions in a proven epoch',
-      valueType: ValueType.INT,
-    });
+    this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION);
+    this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION);
+    this.provingJobCheckpoints = this.meter.createGauge(Metrics.PROVER_NODE_JOB_CHECKPOINTS);
+    this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS);
+    this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS);
+
+    this.blobProcessingDuration = this.meter.createGauge(Metrics.PROVER_NODE_BLOB_PROCESSING_LAST_DURATION);
+    this.chonkVerifierDuration = this.meter.createGauge(Metrics.PROVER_NODE_CHONK_VERIFIER_LAST_DURATION);
+    this.blockProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_BLOCK_PROCESSING_DURATION);
+    this.checkpointProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_PROCESSING_DURATION);
+    this.allCheckpointsProcessingDuration = this.meter.createGauge(
+      Metrics.PROVER_NODE_ALL_CHECKPOINTS_PROCESSING_LAST_DURATION,
+    );
   }
 
-  public recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number) {
+  public recordProvingJob(
+    executionTimeMs: number,
+    totalTimeMs: number,
+    numCheckpoints: number,
+    numBlocks: number,
+    numTxs: number,
+  ) {
     this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
     this.provingJobDuration.record(totalTimeMs / 1000);
+    this.provingJobCheckpoints.record(Math.floor(numCheckpoints));
     this.provingJobBlocks.record(Math.floor(numBlocks));
     this.provingJobTransactions.record(Math.floor(numTxs));
   }
+
+  public recordBlobProcessing(durationMs: number) {
+    this.blobProcessingDuration.record(Math.ceil(durationMs));
+  }
+
+  public recordChonkVerifier(durationMs: number) {
+    this.chonkVerifierDuration.record(Math.ceil(durationMs));
+  }
+
+  public recordBlockProcessing(durationMs: number) {
+    this.blockProcessingDuration.record(Math.ceil(durationMs));
+  }
+
+  public recordCheckpointProcessing(durationMs: number) {
+    this.checkpointProcessingDuration.record(Math.ceil(durationMs));
+  }
+
+  public recordAllCheckpointsProcessing(durationMs: number) {
+    this.allCheckpointsProcessingDuration.record(Math.ceil(durationMs));
+  }
 }
 
 export class ProverNodeRewardsMetrics {
@@ -69,15 +98,9 @@ export class ProverNodeRewardsMetrics {
     private rollup: RollupContract,
     private logger = createLogger('prover-node:publisher:metrics'),
   ) {
-    this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH, {
-      valueType: ValueType.DOUBLE,
-      description: 'The rewards earned',
-    });
+    this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH);
 
-    this.accumulatedRewards = this.meter.createUpDownCounter(Metrics.PROVER_NODE_REWARDS_TOTAL, {
-      valueType: ValueType.DOUBLE,
-      description: 'The rewards earned (total)',
-    });
+    this.accumulatedRewards = createUpDownCounterWithDefault(this.meter, Metrics.PROVER_NODE_REWARDS_TOTAL);
   }
 
   public async start() {
@@ -97,7 +120,7 @@ export class ProverNodeRewardsMetrics {
       // look at the prev epoch so that we get an accurate value, after proof submission window has closed
       // For example, if proof submission window is 1 epoch, and we are in epoch 2, we should be looking at epoch 0.
       // Similarly, if the proof submission window is 0, and we are in epoch 1, we should be looking at epoch 0.
-      const closedEpoch = epoch - BigInt(this.proofSubmissionEpochs) - 1n;
+      const closedEpoch = BigInt(epoch) - BigInt(this.proofSubmissionEpochs) - 1n;
       const rewards = await this.rollup.getSpecificProverRewardsForEpoch(closedEpoch, this.coinbase);
 
       const fmt = parseFloat(formatUnits(rewards, 18));
@@ -117,6 +140,13 @@ export class ProverNodeRewardsMetrics {
   };
 }
 
+export type EstimatedSubmitProofStats = {
+  gasLimit: bigint;
+  baseFeePerGas: bigint;
+  maxPriorityFeePerGas: bigint;
+  estimatedTotalFee: bigint;
+};
+
 export class ProverNodePublisherMetrics {
   gasPrice: Histogram;
   txCount: UpDownCounter;
@@ -128,6 +158,10 @@ export class ProverNodePublisherMetrics {
   txBlobDataGasCost: Histogram;
   txTotalFee: Histogram;
 
+  private txGasEstimated: Histogram;
+  private gasPriceEstimated: Histogram;
+  private txTotalFeeEstimated: Histogram;
+
   private senderBalance: Gauge;
   private meter: Meter;
 
@@ -138,68 +172,34 @@ export class ProverNodePublisherMetrics {
   ) {
     this.meter = client.getMeter(name);
 
-    this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, {
-      description: 'The gas price used for transactions',
-      unit: 'gwei',
-      valueType: ValueType.DOUBLE,
-    });
+    this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE);
 
-    this.txCount = this.meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, {
-      description: 'The number of transactions processed',
+    this.txCount = createUpDownCounterWithDefault(this.meter, Metrics.L1_PUBLISHER_TX_COUNT, {
+      [Attributes.L1_TX_TYPE]: ['submitProof'],
+      [Attributes.OK]: [true, false],
     });
 
-    this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, {
-      description: 'The duration of transaction processing',
-      unit: 'ms',
-      valueType: ValueType.INT,
-    });
+    this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION);
 
-    this.txGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, {
-      description: 'The gas consumed by transactions',
-      unit: 'gas',
-      valueType: ValueType.INT,
-    });
+    this.txGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS);
 
-    this.txCalldataSize = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, {
-      description: 'The size of the calldata in transactions',
-      unit: 'By',
-      valueType: ValueType.INT,
-    });
+    this.txCalldataSize = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE);
 
-    this.txCalldataGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, {
-      description: 'The gas consumed by the calldata in transactions',
-      unit: 'gas',
-      valueType: ValueType.INT,
-    });
+    this.txCalldataGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS);
 
-    this.txBlobDataGasUsed = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, {
-      description: 'The amount of blob gas used in transactions',
-      unit: 'gas',
-      valueType: ValueType.INT,
-    });
+    this.txBlobDataGasUsed = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED);
 
-    this.txBlobDataGasCost = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, {
-      description: 'The gas cost of blobs in transactions',
-      unit: 'gwei',
-      valueType: ValueType.INT,
-    });
+    this.txBlobDataGasCost = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST);
 
-    this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE, {
-      description: 'How much L1 tx costs',
-      unit: 'gwei',
-      valueType: ValueType.DOUBLE,
-      advice: {
-        explicitBucketBoundaries: [
-          0.001, 0.002, 0.004, 0.008, 0.01, 0.02, 0.04, 0.08, 0.1, 0.2, 0.4, 0.8, 1, 1.2, 1.4, 1.8, 2,
-        ],
-      },
-    });
+    this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE);
 
-    this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, {
-      unit: 'eth',
-      description: 'The balance of the sender address',
-      valueType: ValueType.DOUBLE,
-    });
+    this.txGasEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_GAS);
+
+    this.gasPriceEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_GAS_PRICE);
+
+    this.txTotalFeeEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_TOTAL_FEE);
+
+    this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE);
   }
 
   recordFailedTx() {
@@ -213,6 +213,26 @@ export class ProverNodePublisherMetrics {
     this.recordTx(durationMs, stats);
   }
 
+  public recordEstimatedSubmitProof(stats: EstimatedSubmitProofStats) {
+    const attributes = { [Attributes.L1_TX_TYPE]: 'submitProof' } as const;
+
+    this.txGasEstimated.record(Number(stats.gasLimit), attributes);
+
+    try {
+      this.gasPriceEstimated.record(
+        parseInt(formatEther(stats.baseFeePerGas + stats.maxPriorityFeePerGas, 'gwei'), 10),
+      );
+    } catch {
+      // ignore
+    }
+
+    try {
+      this.txTotalFeeEstimated.record(parseFloat(formatEther(stats.estimatedTotalFee)));
+    } catch {
+      // ignore
+    }
+  }
+
   public recordSenderBalance(wei: bigint, senderAddress: string) {
     const eth = parseFloat(formatEther(wei, 'wei'));
     this.senderBalance.record(eth, {

package/src/monitors/epoch-monitor.ts

@@ -1,18 +1,13 @@
+import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
 import { createLogger } from '@aztec/foundation/log';
 import { RunningPromise } from '@aztec/foundation/running-promise';
 import { sleep } from '@aztec/foundation/sleep';
 import type { L2BlockSource } from '@aztec/stdlib/block';
 import { type L1RollupConstants, getEpochAtSlot } from '@aztec/stdlib/epoch-helpers';
-import {
-  type TelemetryClient,
-  type Traceable,
-  type Tracer,
-  getTelemetryClient,
-  trackSpan,
-} from '@aztec/telemetry-client';
+import { type TelemetryClient, type Traceable, type Tracer, getTelemetryClient } from '@aztec/telemetry-client';
 
 export interface EpochMonitorHandler {
-  handleEpochReadyToProve(epochNumber: bigint): Promise<boolean>;
+  handleEpochReadyToProve(epochNumber: EpochNumber): Promise<boolean>;
 }
 
 /**
@@ -32,7 +27,7 @@ export class EpochMonitor implements Traceable {
   public readonly tracer: Tracer;
 
   private handler: EpochMonitorHandler | undefined;
-  private latestEpochNumber: bigint | undefined;
+  private latestEpochNumber: EpochNumber | undefined;
 
   constructor(
     private readonly l2BlockSource: L2BlockSource,
@@ -72,12 +67,10 @@ export class EpochMonitor implements Traceable {
     this.log.info('Stopped EpochMonitor');
   }
 
-  @trackSpan('EpochMonitor.work')
   public async work() {
     const { epochToProve, blockNumber, slotNumber } = await this.getEpochNumberToProve();
-    this.log.debug(`Epoch to prove: ${epochToProve}`, { blockNumber, slotNumber });
     if (epochToProve === undefined) {
-      this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { blockNumber });
+      this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { epochToProve, blockNumber, slotNumber });
       return;
     }
     if (this.latestEpochNumber !== undefined && epochToProve <= this.latestEpochNumber) {
@@ -92,20 +85,20 @@ export class EpochMonitor implements Traceable {
     }
 
     if (this.options.provingDelayMs) {
-      this.log.debug(`Waiting ${this.options.provingDelayMs}ms before proving epoch ${epochToProve}`);
+      this.log.warn(`Waiting ${this.options.provingDelayMs}ms before proving epoch ${epochToProve}`);
       await sleep(this.options.provingDelayMs);
     }
 
-    this.log.debug(`Epoch ${epochToProve} is ready to be proven`);
+    this.log.verbose(`Epoch ${epochToProve} is ready to be proven`);
     if (await this.handler?.handleEpochReadyToProve(epochToProve)) {
       this.latestEpochNumber = epochToProve;
     }
   }
 
   private async getEpochNumberToProve() {
-    const lastBlockProven = await this.l2BlockSource.getProvenBlockNumber();
-    const firstBlockToProve = lastBlockProven + 1;
-    const firstBlockHeaderToProve = await this.l2BlockSource.getBlockHeader(firstBlockToProve);
+    const lastBlockProven = (await this.l2BlockSource.getBlockNumber({ tag: 'proven' })) ?? BlockNumber.ZERO;
+    const firstBlockToProve = BlockNumber(lastBlockProven + 1);
+    const firstBlockHeaderToProve = (await this.l2BlockSource.getBlockData({ number: firstBlockToProve }))?.header;
     if (!firstBlockHeaderToProve) {
       return { epochToProve: undefined, blockNumber: firstBlockToProve };
     }