@aztec/prover-node 0.0.1-commit.b655e406 → 0.0.1-commit.b8a057fa

package/src/metrics.ts

@@ -1,4 +1,4 @@
-import type { RollupContract } from '@aztec/ethereum';
+import type { RollupContract } from '@aztec/ethereum/contracts';
 import type { EthAddress } from '@aztec/foundation/eth-address';
 import { createLogger } from '@aztec/foundation/log';
 import type { L1PublishProofStats, L1PublishStats } from '@aztec/stdlib/stats';
@@ -13,48 +13,119 @@ import {
   type TelemetryClient,
   type Tracer,
   type UpDownCounter,
-  ValueType,
+  createUpDownCounterWithDefault,
 } from '@aztec/telemetry-client';
 
 import { formatEther, formatUnits } from 'viem';
 
+import type { CheckpointStore } from './checkpoint-store.js';
+import type { SessionManager } from './session-manager.js';
+
 export class ProverNodeJobMetrics {
-  proverEpochExecutionDuration: Histogram;
   provingJobDuration: Histogram;
+  provingJobCheckpoints: Gauge;
   provingJobBlocks: Gauge;
   provingJobTransactions: Gauge;
 
+  private blobProcessingDuration: Gauge;
+  private blockProcessingDuration: Histogram;
+  private checkpointProcessingDuration: Histogram;
+  private checkpointProvingDuration: Histogram;
+  private checkpointBlocks: Histogram;
+  private checkpointTransactions: Histogram;
+
+  /** Observable gauges for live state. Registered via `observeState(...)` once the
+   *  CheckpointStore and SessionManager are available. */
+  private activeCheckpoints: ObservableGauge | undefined;
+  private activeEpochSessions: ObservableGauge | undefined;
+  private stateObserver: ((observer: BatchObservableResult) => void) | undefined;
+  private stateObservedMetrics: ObservableGauge[] = [];
+
   constructor(
     private meter: Meter,
     public readonly tracer: Tracer,
     private logger = createLogger('prover-node:publisher:metrics'),
   ) {
-    this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, {
-      description: 'Duration of execution of an epoch by the prover',
-      unit: 'ms',
-      valueType: ValueType.INT,
-    });
-    this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
-      description: 'Duration of proving job',
-      unit: 's',
-      valueType: ValueType.DOUBLE,
-    });
-    this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
-      description: 'Number of blocks in a proven epoch',
-      valueType: ValueType.INT,
-    });
-    this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
-      description: 'Number of transactions in a proven epoch',
-      valueType: ValueType.INT,
-    });
+    this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION);
+    this.provingJobCheckpoints = this.meter.createGauge(Metrics.PROVER_NODE_JOB_CHECKPOINTS);
+    this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS);
+    this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS);
+
+    this.blobProcessingDuration = this.meter.createGauge(Metrics.PROVER_NODE_BLOB_PROCESSING_LAST_DURATION);
+    this.blockProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_BLOCK_PROCESSING_DURATION);
+    this.checkpointProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_PROCESSING_DURATION);
+    this.checkpointProvingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_PROVING_DURATION);
+    this.checkpointBlocks = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_BLOCKS);
+    this.checkpointTransactions = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_TRANSACTIONS);
   }
 
-  public recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number) {
-    this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
+  public recordProvingJob(totalTimeMs: number, numCheckpoints: number, numBlocks: number, numTxs: number) {
     this.provingJobDuration.record(totalTimeMs / 1000);
+    this.provingJobCheckpoints.record(Math.floor(numCheckpoints));
     this.provingJobBlocks.record(Math.floor(numBlocks));
     this.provingJobTransactions.record(Math.floor(numTxs));
   }
+
+  public recordBlobProcessing(durationMs: number) {
+    this.blobProcessingDuration.record(Math.ceil(durationMs));
+  }
+
+  public recordBlockProcessing(durationMs: number) {
+    this.blockProcessingDuration.record(Math.ceil(durationMs));
+  }
+
+  public recordCheckpointProcessing(durationMs: number, numBlocks: number, numTxs: number) {
+    this.checkpointProcessingDuration.record(Math.ceil(durationMs));
+    this.checkpointBlocks.record(Math.floor(numBlocks));
+    this.checkpointTransactions.record(Math.floor(numTxs));
+  }
+
+  public recordCheckpointProving(durationMs: number) {
+    this.checkpointProvingDuration.record(Math.ceil(durationMs));
+  }
+
+  /**
+   * Registers observable gauges for the prover-node's live state: how many canonical
+   * checkpoint provers are in the store, and how many epoch sessions are live (broken
+   * down by kind). Idempotent — repeated calls re-arm with the latest references.
+   *
+   * Call this once the `SessionManager` has been constructed (i.e. inside `ProverNode.start()`).
+   */
+  public observeState(checkpointStore: CheckpointStore, sessionManager: SessionManager): void {
+    this.stopObservingState();
+    this.activeCheckpoints = this.meter.createObservableGauge(Metrics.PROVER_NODE_ACTIVE_CHECKPOINTS);
+    this.activeEpochSessions = this.meter.createObservableGauge(Metrics.PROVER_NODE_ACTIVE_EPOCH_SESSIONS);
+    this.stateObserver = (observer: BatchObservableResult) => {
+      observer.observe(this.activeCheckpoints!, checkpointStore.listCanonical().length);
+      let full = 0;
+      let partial = 0;
+      for (const session of sessionManager.allSessions()) {
+        if (session.isTerminal()) {
+          continue;
+        }
+        if (session.getKind() === 'full') {
+          full++;
+        } else {
+          partial++;
+        }
+      }
+      observer.observe(this.activeEpochSessions!, full, { [Attributes.EPOCH_SESSION_KIND]: 'full' });
+      observer.observe(this.activeEpochSessions!, partial, { [Attributes.EPOCH_SESSION_KIND]: 'partial' });
+    };
+    this.stateObservedMetrics = [this.activeCheckpoints, this.activeEpochSessions];
+    this.meter.addBatchObservableCallback(this.stateObserver, this.stateObservedMetrics);
+  }
+
+  /** Tears down the observable callback registered by `observeState`. Idempotent. */
+  public stopObservingState(): void {
+    if (this.stateObserver) {
+      this.meter.removeBatchObservableCallback(this.stateObserver, this.stateObservedMetrics);
+      this.stateObserver = undefined;
+      this.stateObservedMetrics = [];
+      this.activeCheckpoints = undefined;
+      this.activeEpochSessions = undefined;
+    }
+  }
 }
 
 export class ProverNodeRewardsMetrics {
@@ -69,15 +140,9 @@ export class ProverNodeRewardsMetrics {
     private rollup: RollupContract,
     private logger = createLogger('prover-node:publisher:metrics'),
   ) {
-    this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH, {
-      valueType: ValueType.DOUBLE,
-      description: 'The rewards earned',
-    });
+    this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH);
 
-    this.accumulatedRewards = this.meter.createUpDownCounter(Metrics.PROVER_NODE_REWARDS_TOTAL, {
-      valueType: ValueType.DOUBLE,
-      description: 'The rewards earned (total)',
-    });
+    this.accumulatedRewards = createUpDownCounterWithDefault(this.meter, Metrics.PROVER_NODE_REWARDS_TOTAL);
   }
 
   public async start() {
@@ -97,7 +162,7 @@ export class ProverNodeRewardsMetrics {
       // look at the prev epoch so that we get an accurate value, after proof submission window has closed
       // For example, if proof submission window is 1 epoch, and we are in epoch 2, we should be looking at epoch 0.
       // Similarly, if the proof submission window is 0, and we are in epoch 1, we should be looking at epoch 0.
-      const closedEpoch = epoch - BigInt(this.proofSubmissionEpochs) - 1n;
+      const closedEpoch = BigInt(epoch) - BigInt(this.proofSubmissionEpochs) - 1n;
       const rewards = await this.rollup.getSpecificProverRewardsForEpoch(closedEpoch, this.coinbase);
 
       const fmt = parseFloat(formatUnits(rewards, 18));
@@ -117,6 +182,13 @@ export class ProverNodeRewardsMetrics {
   };
 }
 
+export type EstimatedSubmitProofStats = {
+  gasLimit: bigint;
+  baseFeePerGas: bigint;
+  maxPriorityFeePerGas: bigint;
+  estimatedTotalFee: bigint;
+};
+
 export class ProverNodePublisherMetrics {
   gasPrice: Histogram;
   txCount: UpDownCounter;
@@ -128,6 +200,10 @@ export class ProverNodePublisherMetrics {
   txBlobDataGasCost: Histogram;
   txTotalFee: Histogram;
 
+  private txGasEstimated: Histogram;
+  private gasPriceEstimated: Histogram;
+  private txTotalFeeEstimated: Histogram;
+
   private senderBalance: Gauge;
   private meter: Meter;
 
@@ -138,68 +214,34 @@ export class ProverNodePublisherMetrics {
   ) {
     this.meter = client.getMeter(name);
 
-    this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, {
-      description: 'The gas price used for transactions',
-      unit: 'gwei',
-      valueType: ValueType.DOUBLE,
-    });
+    this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE);
 
-    this.txCount = this.meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, {
-      description: 'The number of transactions processed',
+    this.txCount = createUpDownCounterWithDefault(this.meter, Metrics.L1_PUBLISHER_TX_COUNT, {
+      [Attributes.L1_TX_TYPE]: ['submitProof'],
+      [Attributes.OK]: [true, false],
     });
 
-    this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, {
-      description: 'The duration of transaction processing',
-      unit: 'ms',
-      valueType: ValueType.INT,
-    });
+    this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION);
 
-    this.txGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, {
-      description: 'The gas consumed by transactions',
-      unit: 'gas',
-      valueType: ValueType.INT,
-    });
+    this.txGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS);
 
-    this.txCalldataSize = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, {
-      description: 'The size of the calldata in transactions',
-      unit: 'By',
-      valueType: ValueType.INT,
-    });
+    this.txCalldataSize = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE);
 
-    this.txCalldataGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, {
-      description: 'The gas consumed by the calldata in transactions',
-      unit: 'gas',
-      valueType: ValueType.INT,
-    });
+    this.txCalldataGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS);
 
-    this.txBlobDataGasUsed = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, {
-      description: 'The amount of blob gas used in transactions',
-      unit: 'gas',
-      valueType: ValueType.INT,
-    });
+    this.txBlobDataGasUsed = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED);
 
-    this.txBlobDataGasCost = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, {
-      description: 'The gas cost of blobs in transactions',
-      unit: 'gwei',
-      valueType: ValueType.INT,
-    });
+    this.txBlobDataGasCost = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST);
 
-    this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE, {
-      description: 'How much L1 tx costs',
-      unit: 'gwei',
-      valueType: ValueType.DOUBLE,
-      advice: {
-        explicitBucketBoundaries: [
-          0.001, 0.002, 0.004, 0.008, 0.01, 0.02, 0.04, 0.08, 0.1, 0.2, 0.4, 0.8, 1, 1.2, 1.4, 1.8, 2,
-        ],
-      },
-    });
+    this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE);
 
-    this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, {
-      unit: 'eth',
-      description: 'The balance of the sender address',
-      valueType: ValueType.DOUBLE,
-    });
+    this.txGasEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_GAS);
+
+    this.gasPriceEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_GAS_PRICE);
+
+    this.txTotalFeeEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_TOTAL_FEE);
+
+    this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE);
   }
 
   recordFailedTx() {
@@ -213,6 +255,26 @@ export class ProverNodePublisherMetrics {
     this.recordTx(durationMs, stats);
   }
 
+  public recordEstimatedSubmitProof(stats: EstimatedSubmitProofStats) {
+    const attributes = { [Attributes.L1_TX_TYPE]: 'submitProof' } as const;
+
+    this.txGasEstimated.record(Number(stats.gasLimit), attributes);
+
+    try {
+      this.gasPriceEstimated.record(
+        parseInt(formatEther(stats.baseFeePerGas + stats.maxPriorityFeePerGas, 'gwei'), 10),
+      );
+    } catch {
+      // ignore
+    }
+
+    try {
+      this.txTotalFeeEstimated.record(parseFloat(formatEther(stats.estimatedTotalFee)));
+    } catch {
+      // ignore
+    }
+  }
+
   public recordSenderBalance(wei: bigint, senderAddress: string) {
     const eth = parseFloat(formatEther(wei, 'wei'));
     this.senderBalance.record(eth, {

package/src/monitors/epoch-monitor.ts

@@ -1,18 +1,13 @@
+import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
 import { createLogger } from '@aztec/foundation/log';
 import { RunningPromise } from '@aztec/foundation/running-promise';
 import { sleep } from '@aztec/foundation/sleep';
 import type { L2BlockSource } from '@aztec/stdlib/block';
 import { type L1RollupConstants, getEpochAtSlot } from '@aztec/stdlib/epoch-helpers';
-import {
-  type TelemetryClient,
-  type Traceable,
-  type Tracer,
-  getTelemetryClient,
-  trackSpan,
-} from '@aztec/telemetry-client';
+import { type TelemetryClient, type Traceable, type Tracer, getTelemetryClient } from '@aztec/telemetry-client';
 
 export interface EpochMonitorHandler {
-  handleEpochReadyToProve(epochNumber: bigint): Promise<boolean>;
+  handleEpochReadyToProve(epochNumber: EpochNumber): Promise<boolean>;
 }
 
 /**
@@ -32,7 +27,7 @@ export class EpochMonitor implements Traceable {
   public readonly tracer: Tracer;
 
   private handler: EpochMonitorHandler | undefined;
-  private latestEpochNumber: bigint | undefined;
+  private latestEpochNumber: EpochNumber | undefined;
 
   constructor(
     private readonly l2BlockSource: L2BlockSource,
@@ -72,12 +67,10 @@ export class EpochMonitor implements Traceable {
     this.log.info('Stopped EpochMonitor');
   }
 
-  @trackSpan('EpochMonitor.work')
   public async work() {
     const { epochToProve, blockNumber, slotNumber } = await this.getEpochNumberToProve();
-    this.log.debug(`Epoch to prove: ${epochToProve}`, { blockNumber, slotNumber });
     if (epochToProve === undefined) {
-      this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { blockNumber });
+      this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { epochToProve, blockNumber, slotNumber });
       return;
     }
     if (this.latestEpochNumber !== undefined && epochToProve <= this.latestEpochNumber) {
@@ -92,20 +85,20 @@ export class EpochMonitor implements Traceable {
     }
 
     if (this.options.provingDelayMs) {
-      this.log.debug(`Waiting ${this.options.provingDelayMs}ms before proving epoch ${epochToProve}`);
+      this.log.warn(`Waiting ${this.options.provingDelayMs}ms before proving epoch ${epochToProve}`);
       await sleep(this.options.provingDelayMs);
     }
 
-    this.log.debug(`Epoch ${epochToProve} is ready to be proven`);
+    this.log.verbose(`Epoch ${epochToProve} is ready to be proven`);
     if (await this.handler?.handleEpochReadyToProve(epochToProve)) {
       this.latestEpochNumber = epochToProve;
     }
   }
 
   private async getEpochNumberToProve() {
-    const lastBlockProven = await this.l2BlockSource.getProvenBlockNumber();
-    const firstBlockToProve = lastBlockProven + 1;
-    const firstBlockHeaderToProve = await this.l2BlockSource.getBlockHeader(firstBlockToProve);
+    const lastBlockProven = (await this.l2BlockSource.getBlockNumber({ tag: 'proven' })) ?? BlockNumber.ZERO;
+    const firstBlockToProve = BlockNumber(lastBlockProven + 1);
+    const firstBlockHeaderToProve = (await this.l2BlockSource.getBlockData({ number: firstBlockToProve }))?.header;
     if (!firstBlockHeaderToProve) {
       return { epochToProve: undefined, blockNumber: firstBlockToProve };
     }