@aztec/prover-node 0.0.1-commit.03f7ef2 → 0.0.1-commit.04d373f

package/src/job/epoch-proving-job.ts

@@ -1,17 +1,16 @@
-import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
 import { asyncPool } from '@aztec/foundation/async-pool';
 import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
-import { padArrayEnd } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
-import { createLogger } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 import { Timer } from '@aztec/foundation/timer';
+import { AVM_MAX_CONCURRENT_SIMULATIONS } from '@aztec/native';
 import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
 import { protocolContractsHash } from '@aztec/protocol-contracts';
 import { buildFinalBlobChallenges } from '@aztec/prover-client/helpers';
 import type { PublicProcessor, PublicProcessorFactory } from '@aztec/simulator/server';
 import { PublicSimulatorConfig } from '@aztec/stdlib/avm';
-import type { L2BlockNew, L2BlockSource } from '@aztec/stdlib/block';
+import type { L2Block, L2BlockSource } from '@aztec/stdlib/block';
 import type { Checkpoint } from '@aztec/stdlib/checkpoint';
 import {
   type EpochProver,
@@ -19,8 +18,8 @@ import {
   EpochProvingJobTerminalState,
   type ForkMerkleTreeOperations,
 } from '@aztec/stdlib/interfaces/server';
+import { appendL1ToL2MessagesToTree } from '@aztec/stdlib/messaging';
 import { CheckpointConstantData } from '@aztec/stdlib/rollup';
-import { MerkleTreeId } from '@aztec/stdlib/trees';
 import type { ProcessedTx, Tx } from '@aztec/stdlib/tx';
 import { Attributes, type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client';
 
@@ -43,10 +42,11 @@ export type EpochProvingJobOptions = {
  */
 export class EpochProvingJob implements Traceable {
   private state: EpochProvingJobState = 'initialized';
-  private log = createLogger('prover-node:epoch-proving-job');
+  private log: Logger;
   private uuid: string;
 
   private runPromise: Promise<void> | undefined;
+  private abortController = new AbortController();
   private epochCheckPromise: RunningPromise | undefined;
   private deadlineTimeoutHandler: NodeJS.Timeout | undefined;
 
@@ -57,14 +57,19 @@ export class EpochProvingJob implements Traceable {
     private dbProvider: Pick<ForkMerkleTreeOperations, 'fork'>,
     private prover: EpochProver,
     private publicProcessorFactory: PublicProcessorFactory,
-    private publisher: Pick<ProverNodePublisher, 'submitEpochProof'>,
+    private publisher: Pick<ProverNodePublisher, 'submitEpochProof' | 'analyzeEpochProofSubmission'>,
     private l2BlockSource: L2BlockSource | undefined,
     private metrics: ProverNodeJobMetrics,
     private deadline: Date | undefined,
     private config: EpochProvingJobOptions,
+    bindings?: LoggerBindings,
   ) {
     validateEpochProvingJobData(data);
     this.uuid = crypto.randomUUID();
+    this.log = createLogger('prover-node:epoch-proving-job', {
+      ...bindings,
+      instanceId: `epoch-${data.epochNumber}`,
+    });
     this.tracer = metrics.tracer;
   }
 
@@ -143,19 +148,32 @@ export class EpochProvingJob implements Traceable {
     this.runPromise = promise;
 
     try {
+      const blobTimer = new Timer();
       const blobFieldsPerCheckpoint = this.checkpoints.map(checkpoint => checkpoint.toBlobFields());
       const finalBlobBatchingChallenges = await buildFinalBlobChallenges(blobFieldsPerCheckpoint);
+      this.metrics.recordBlobProcessing(blobTimer.ms());
 
       this.prover.startNewEpoch(epochNumber, epochSizeCheckpoints, finalBlobBatchingChallenges);
+      const chonkTimer = new Timer();
       await this.prover.startChonkVerifierCircuits(Array.from(this.txs.values()));
+      this.metrics.recordChonkVerifier(chonkTimer.ms());
 
       // Everything in the epoch should have the same chainId and version.
       const { chainId, version } = this.checkpoints[0].blocks[0].header.globalVariables;
 
       const previousBlockHeaders = this.gatherPreviousBlockHeaders();
 
-      await asyncPool(this.config.parallelBlockLimit ?? 32, this.checkpoints, async checkpoint => {
+      const allCheckpointsTimer = new Timer();
+
+      const parallelism = this.config.parallelBlockLimit
+        ? this.config.parallelBlockLimit
+        : AVM_MAX_CONCURRENT_SIMULATIONS > 0
+          ? AVM_MAX_CONCURRENT_SIMULATIONS
+          : this.checkpoints.length;
+
+      await this.processCheckpoints(parallelism, async checkpoint => {
         this.checkState();
+        const checkpointTimer = new Timer();
 
         const checkpointIndex = checkpoint.number - fromCheckpoint;
         const checkpointConstants = CheckpointConstantData.from({
@@ -172,11 +190,12 @@ export class EpochProvingJob implements Traceable {
         const previousHeader = previousBlockHeaders[checkpointIndex];
         const l1ToL2Messages = this.getL1ToL2Messages(checkpoint);
 
-        this.log.verbose(`Starting processing checkpoint ${checkpoint.number}`, {
+        this.log.debug(`Starting processing checkpoint ${checkpoint.number}`, {
           number: checkpoint.number,
           checkpointHash: checkpoint.hash().toString(),
-          lastArchive: checkpoint.header.lastArchiveRoot,
-          previousHeader: previousHeader.hash(),
+          headerHash: checkpoint.header.hash().toString(),
+          numL1ToL2Messages: l1ToL2Messages.length,
+          previousBlockNumber: previousHeader.globalVariables.blockNumber,
           uuid: this.uuid,
         });
 
@@ -188,7 +207,9 @@ export class EpochProvingJob implements Traceable {
           previousHeader,
         );
 
-        for (const block of checkpoint.blocks) {
+        for (let blockIndex = 0; blockIndex < checkpoint.blocks.length; blockIndex++) {
+          const blockTimer = new Timer();
+          const block = checkpoint.blocks[blockIndex];
           const globalVariables = block.header.globalVariables;
           const txs = this.getTxs(block);
 
@@ -206,20 +227,28 @@ export class EpochProvingJob implements Traceable {
           // Start block proving
           await this.prover.startNewBlock(block.number, globalVariables.timestamp, txs.length);
 
-          // Process public fns
-          const db = await this.createFork(BlockNumber(block.number - 1), l1ToL2Messages);
-          const config = PublicSimulatorConfig.from({
-            proverId: this.prover.getProverId().toField(),
-            skipFeeEnforcement: false,
-            collectDebugLogs: false,
-            collectHints: true,
-            collectPublicInputs: true,
-            collectStatistics: false,
-          });
-          const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, config);
-          const processed = await this.processTxs(publicProcessor, txs);
-          await this.prover.addTxs(processed);
-          await db.close();
+          // Process public fns. L1 to L2 messages are only inserted for the first block of a checkpoint,
+          // as the fork for subsequent blocks already includes them from the previous block's synced state.
+          {
+            await using db = await this.createFork(
+              BlockNumber(block.number - 1),
+              blockIndex === 0 ? l1ToL2Messages : undefined,
+            );
+            this.checkState();
+            const config = PublicSimulatorConfig.from({
+              proverId: this.prover.getProverId().toField(),
+              skipFeeEnforcement: false,
+              collectDebugLogs: false,
+              collectHints: true,
+              collectPublicInputs: true,
+              collectStatistics: false,
+            });
+            const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, config);
+            const processed = await this.processTxs(publicProcessor, txs);
+            this.checkState();
+            await this.prover.addTxs(processed);
+          }
+          this.checkState();
           this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, {
             blockNumber: block.number,
             blockHash: (await block.hash()).toString(),
@@ -229,8 +258,11 @@ export class EpochProvingJob implements Traceable {
           // Mark block as completed to pad it
           const expectedBlockHeader = block.header;
           await this.prover.setBlockCompleted(block.number, expectedBlockHeader);
+          this.metrics.recordBlockProcessing(blockTimer.ms());
         }
+        this.metrics.recordCheckpointProcessing(checkpointTimer.ms());
       });
+      this.metrics.recordAllCheckpointsProcessing(allCheckpointsTimer.ms());
 
       const executionTime = timer.ms();
 
@@ -242,8 +274,21 @@ export class EpochProvingJob implements Traceable {
 
       if (this.config.skipSubmitProof) {
         this.log.info(
-          `Proof publishing is disabled. Dropping valid proof for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`,
+          `Proof publishing is disabled. Analyzing estimated L1 fees for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`,
         );
+        try {
+          await this.publisher.analyzeEpochProofSubmission({
+            fromCheckpoint,
+            toCheckpoint,
+            epochNumber,
+            publicInputs,
+            proof,
+            batchedBlobInputs,
+            attestations,
+          });
+        } catch (err) {
+          this.log.warn(`Failed to analyze estimated L1 fees for epoch ${epochNumber}`, err);
+        }
         this.state = 'completed';
         this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
         return;
@@ -290,25 +335,63 @@ export class EpochProvingJob implements Traceable {
   }
 
   /**
-   * Create a new db fork for tx processing, inserting all L1 to L2.
+   * Create a new db fork for tx processing, optionally inserting L1 to L2 messages.
+   * L1 to L2 messages should only be inserted for the first block in a checkpoint,
+   * as subsequent blocks' synced state already includes them.
    * REFACTOR: The prover already spawns a db fork of its own for each block, so we may be able to do away with just one fork.
    */
-  private async createFork(blockNumber: BlockNumber, l1ToL2Messages: Fr[]) {
-    const db = await this.dbProvider.fork(blockNumber);
-    const l1ToL2MessagesPadded = padArrayEnd<Fr, number>(
-      l1ToL2Messages,
-      Fr.ZERO,
-      NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP,
-      'Too many L1 to L2 messages',
-    );
-    this.log.verbose(`Creating fork at ${blockNumber} with ${l1ToL2Messages.length} L1 to L2 messages`, {
-      blockNumber,
-      l1ToL2Messages: l1ToL2Messages.map(m => m.toString()),
-    });
-    await db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded);
+  private async createFork(blockNumber: BlockNumber, l1ToL2Messages: Fr[] | undefined) {
+    this.log.verbose(`Creating fork at ${blockNumber}`, { blockNumber });
+    // temporary stack to control fork lifetime
+    await using cleanup = new AsyncDisposableStack();
+    const db = cleanup.use(await this.dbProvider.fork(blockNumber));
+
+    if (l1ToL2Messages !== undefined) {
+      this.log.verbose(`Inserting ${l1ToL2Messages.length} L1 to L2 messages in fork`, {
+        blockNumber,
+        l1ToL2Messages: l1ToL2Messages.map(m => m.toString()),
+      });
+      await appendL1ToL2MessagesToTree(db, l1ToL2Messages);
+    }
+
+    // everything run succesfully so we can release this stack and give control of the fork's lifetime to the caller
+    cleanup.move();
     return db;
   }
 
+  private async processCheckpoints(
+    parallelism: number,
+    processCheckpoint: (checkpoint: Checkpoint) => Promise<void>,
+  ): Promise<void> {
+    let hasError = false;
+    let firstError: unknown;
+
+    await asyncPool(Math.max(parallelism, 1), this.checkpoints, async checkpoint => {
+      if (hasError || this.abortController.signal.aborted) {
+        return;
+      }
+
+      try {
+        this.checkState();
+        await processCheckpoint(checkpoint);
+      } catch (err) {
+        if (!hasError) {
+          hasError = true;
+          firstError = err;
+          this.failProcessing();
+        }
+      }
+    });
+
+    if (hasError) {
+      throw firstError;
+    }
+
+    if (this.abortController.signal.aborted) {
+      this.checkState();
+    }
+  }
+
   private progressState(state: EpochProvingJobState) {
     this.checkState();
     this.state = state;
@@ -322,12 +405,24 @@ export class EpochProvingJob implements Traceable {
 
   public async stop(state: EpochProvingJobTerminalState = 'stopped') {
     this.state = state;
-    this.prover.cancel();
+    this.interruptProcessing();
     if (this.runPromise) {
       await this.runPromise;
     }
   }
 
+  private failProcessing() {
+    if (!EpochProvingJobTerminalState.includes(this.state)) {
+      this.state = 'failed';
+    }
+    this.interruptProcessing();
+  }
+
+  private interruptProcessing() {
+    this.abortController.abort();
+    this.prover.cancel();
+  }
+
   private scheduleDeadlineStop() {
     const deadline = this.deadline;
     if (deadline) {
@@ -362,11 +457,16 @@ export class EpochProvingJob implements Traceable {
     const intervalMs = Math.ceil((await l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
     this.epochCheckPromise = new RunningPromise(
       async () => {
-        const blocks = await l2BlockSource.getBlockHeadersForEpoch(this.epochNumber);
-        const blockHashes = await Promise.all(blocks.map(block => block.hash()));
+        const blockHeaders = (
+          await l2BlockSource.getBlocksData({ epoch: this.epochNumber, onlyCheckpointed: true })
+        ).map(d => d.header);
+        const blockHashes = await Promise.all(blockHeaders.map(header => header.hash()));
         const thisBlocks = this.checkpoints.flatMap(checkpoint => checkpoint.blocks);
         const thisBlockHashes = await Promise.all(thisBlocks.map(block => block.hash()));
-        if (blocks.length !== thisBlocks.length || !blockHashes.every((block, i) => block.equals(thisBlockHashes[i]))) {
+        if (
+          blockHeaders.length !== thisBlocks.length ||
+          !blockHashes.every((block, i) => block.equals(thisBlockHashes[i]))
+        ) {
           this.log.warn('Epoch blocks changed underfoot', {
             uuid: this.uuid,
             epochNumber: this.epochNumber,
@@ -388,7 +488,7 @@ export class EpochProvingJob implements Traceable {
     return [this.data.previousBlockHeader, ...lastBlocks.map(block => block.header).slice(0, -1)];
   }
 
-  private getTxs(block: L2BlockNew): Tx[] {
+  private getTxs(block: L2Block): Tx[] {
     return block.body.txEffects.map(txEffect => this.txs.get(txEffect.txHash.toString())!);
   }
 
@@ -398,7 +498,11 @@ export class EpochProvingJob implements Traceable {
 
   private async processTxs(publicProcessor: PublicProcessor, txs: Tx[]): Promise<ProcessedTx[]> {
     const { deadline } = this;
-    const [processedTxs, failedTxs] = await publicProcessor.process(txs, { deadline });
+    const [processedTxs, failedTxs] = await publicProcessor.process(txs, {
+      deadline,
+      signal: this.abortController.signal,
+    });
+    this.checkState();
 
     if (failedTxs.length) {
       const failedTxHashes = await Promise.all(failedTxs.map(({ tx }) => tx.getTxHash()));

package/src/metrics.ts

@@ -13,7 +13,7 @@ import {
   type TelemetryClient,
   type Tracer,
   type UpDownCounter,
-  ValueType,
+  createUpDownCounterWithDefault,
 } from '@aztec/telemetry-client';
 
 import { formatEther, formatUnits } from 'viem';
@@ -25,33 +25,30 @@ export class ProverNodeJobMetrics {
   provingJobBlocks: Gauge;
   provingJobTransactions: Gauge;
 
+  private blobProcessingDuration: Gauge;
+  private chonkVerifierDuration: Gauge;
+  private blockProcessingDuration: Histogram;
+  private checkpointProcessingDuration: Histogram;
+  private allCheckpointsProcessingDuration: Gauge;
+
   constructor(
     private meter: Meter,
     public readonly tracer: Tracer,
     private logger = createLogger('prover-node:publisher:metrics'),
   ) {
-    this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, {
-      description: 'Duration of execution of an epoch by the prover',
-      unit: 'ms',
-      valueType: ValueType.INT,
-    });
-    this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
-      description: 'Duration of proving job',
-      unit: 's',
-      valueType: ValueType.DOUBLE,
-    });
-    this.provingJobCheckpoints = this.meter.createGauge(Metrics.PROVER_NODE_JOB_CHECKPOINTS, {
-      description: 'Number of checkpoints in a proven epoch',
-      valueType: ValueType.INT,
-    });
-    this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
-      description: 'Number of blocks in a proven epoch',
-      valueType: ValueType.INT,
-    });
-    this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
-      description: 'Number of transactions in a proven epoch',
-      valueType: ValueType.INT,
-    });
+    this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION);
+    this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION);
+    this.provingJobCheckpoints = this.meter.createGauge(Metrics.PROVER_NODE_JOB_CHECKPOINTS);
+    this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS);
+    this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS);
+
+    this.blobProcessingDuration = this.meter.createGauge(Metrics.PROVER_NODE_BLOB_PROCESSING_LAST_DURATION);
+    this.chonkVerifierDuration = this.meter.createGauge(Metrics.PROVER_NODE_CHONK_VERIFIER_LAST_DURATION);
+    this.blockProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_BLOCK_PROCESSING_DURATION);
+    this.checkpointProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_PROCESSING_DURATION);
+    this.allCheckpointsProcessingDuration = this.meter.createGauge(
+      Metrics.PROVER_NODE_ALL_CHECKPOINTS_PROCESSING_LAST_DURATION,
+    );
   }
 
   public recordProvingJob(
@@ -67,6 +64,26 @@ export class ProverNodeJobMetrics {
     this.provingJobBlocks.record(Math.floor(numBlocks));
     this.provingJobTransactions.record(Math.floor(numTxs));
   }
+
+  public recordBlobProcessing(durationMs: number) {
+    this.blobProcessingDuration.record(Math.ceil(durationMs));
+  }
+
+  public recordChonkVerifier(durationMs: number) {
+    this.chonkVerifierDuration.record(Math.ceil(durationMs));
+  }
+
+  public recordBlockProcessing(durationMs: number) {
+    this.blockProcessingDuration.record(Math.ceil(durationMs));
+  }
+
+  public recordCheckpointProcessing(durationMs: number) {
+    this.checkpointProcessingDuration.record(Math.ceil(durationMs));
+  }
+
+  public recordAllCheckpointsProcessing(durationMs: number) {
+    this.allCheckpointsProcessingDuration.record(Math.ceil(durationMs));
+  }
 }
 
 export class ProverNodeRewardsMetrics {
@@ -81,15 +98,9 @@ export class ProverNodeRewardsMetrics {
     private rollup: RollupContract,
     private logger = createLogger('prover-node:publisher:metrics'),
   ) {
-    this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH, {
-      valueType: ValueType.DOUBLE,
-      description: 'The rewards earned',
-    });
+    this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH);
 
-    this.accumulatedRewards = this.meter.createUpDownCounter(Metrics.PROVER_NODE_REWARDS_TOTAL, {
-      valueType: ValueType.DOUBLE,
-      description: 'The rewards earned (total)',
-    });
+    this.accumulatedRewards = createUpDownCounterWithDefault(this.meter, Metrics.PROVER_NODE_REWARDS_TOTAL);
   }
 
   public async start() {
@@ -129,6 +140,13 @@ export class ProverNodeRewardsMetrics {
   };
 }
 
+export type EstimatedSubmitProofStats = {
+  gasLimit: bigint;
+  baseFeePerGas: bigint;
+  maxPriorityFeePerGas: bigint;
+  estimatedTotalFee: bigint;
+};
+
 export class ProverNodePublisherMetrics {
   gasPrice: Histogram;
   txCount: UpDownCounter;
@@ -140,6 +158,10 @@ export class ProverNodePublisherMetrics {
   txBlobDataGasCost: Histogram;
   txTotalFee: Histogram;
 
+  private txGasEstimated: Histogram;
+  private gasPriceEstimated: Histogram;
+  private txTotalFeeEstimated: Histogram;
+
   private senderBalance: Gauge;
   private meter: Meter;
 
@@ -150,68 +172,34 @@ export class ProverNodePublisherMetrics {
   ) {
     this.meter = client.getMeter(name);
 
-    this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, {
-      description: 'The gas price used for transactions',
-      unit: 'gwei',
-      valueType: ValueType.DOUBLE,
-    });
+    this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE);
 
-    this.txCount = this.meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, {
-      description: 'The number of transactions processed',
+    this.txCount = createUpDownCounterWithDefault(this.meter, Metrics.L1_PUBLISHER_TX_COUNT, {
+      [Attributes.L1_TX_TYPE]: ['submitProof'],
+      [Attributes.OK]: [true, false],
     });
 
-    this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, {
-      description: 'The duration of transaction processing',
-      unit: 'ms',
-      valueType: ValueType.INT,
-    });
+    this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION);
 
-    this.txGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, {
-      description: 'The gas consumed by transactions',
-      unit: 'gas',
-      valueType: ValueType.INT,
-    });
+    this.txGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS);
 
-    this.txCalldataSize = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, {
-      description: 'The size of the calldata in transactions',
-      unit: 'By',
-      valueType: ValueType.INT,
-    });
+    this.txCalldataSize = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE);
 
-    this.txCalldataGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, {
-      description: 'The gas consumed by the calldata in transactions',
-      unit: 'gas',
-      valueType: ValueType.INT,
-    });
+    this.txCalldataGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS);
 
-    this.txBlobDataGasUsed = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, {
-      description: 'The amount of blob gas used in transactions',
-      unit: 'gas',
-      valueType: ValueType.INT,
-    });
+    this.txBlobDataGasUsed = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED);
 
-    this.txBlobDataGasCost = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, {
-      description: 'The gas cost of blobs in transactions',
-      unit: 'gwei',
-      valueType: ValueType.INT,
-    });
+    this.txBlobDataGasCost = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST);
 
-    this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE, {
-      description: 'How much L1 tx costs',
-      unit: 'gwei',
-      valueType: ValueType.DOUBLE,
-      advice: {
-        explicitBucketBoundaries: [
-          0.001, 0.002, 0.004, 0.008, 0.01, 0.02, 0.04, 0.08, 0.1, 0.2, 0.4, 0.8, 1, 1.2, 1.4, 1.8, 2,
-        ],
-      },
-    });
+    this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE);
 
-    this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, {
-      unit: 'eth',
-      description: 'The balance of the sender address',
-      valueType: ValueType.DOUBLE,
-    });
+    this.txGasEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_GAS);
+
+    this.gasPriceEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_GAS_PRICE);
+
+    this.txTotalFeeEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_TOTAL_FEE);
+
+    this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE);
   }
 
   recordFailedTx() {
@@ -225,6 +213,26 @@ export class ProverNodePublisherMetrics {
     this.recordTx(durationMs, stats);
   }
 
+  public recordEstimatedSubmitProof(stats: EstimatedSubmitProofStats) {
+    const attributes = { [Attributes.L1_TX_TYPE]: 'submitProof' } as const;
+
+    this.txGasEstimated.record(Number(stats.gasLimit), attributes);
+
+    try {
+      this.gasPriceEstimated.record(
+        parseInt(formatEther(stats.baseFeePerGas + stats.maxPriorityFeePerGas, 'gwei'), 10),
+      );
+    } catch {
+      // ignore
+    }
+
+    try {
+      this.txTotalFeeEstimated.record(parseFloat(formatEther(stats.estimatedTotalFee)));
+    } catch {
+      // ignore
+    }
+  }
+
   public recordSenderBalance(wei: bigint, senderAddress: string) {
     const eth = parseFloat(formatEther(wei, 'wei'));
     this.senderBalance.record(eth, {

package/src/monitors/epoch-monitor.ts

@@ -4,13 +4,7 @@ import { RunningPromise } from '@aztec/foundation/running-promise';
 import { sleep } from '@aztec/foundation/sleep';
 import type { L2BlockSource } from '@aztec/stdlib/block';
 import { type L1RollupConstants, getEpochAtSlot } from '@aztec/stdlib/epoch-helpers';
-import {
-  type TelemetryClient,
-  type Traceable,
-  type Tracer,
-  getTelemetryClient,
-  trackSpan,
-} from '@aztec/telemetry-client';
+import { type TelemetryClient, type Traceable, type Tracer, getTelemetryClient } from '@aztec/telemetry-client';
 
 export interface EpochMonitorHandler {
   handleEpochReadyToProve(epochNumber: EpochNumber): Promise<boolean>;
@@ -73,12 +67,10 @@ export class EpochMonitor implements Traceable {
     this.log.info('Stopped EpochMonitor');
   }
 
-  @trackSpan('EpochMonitor.work')
   public async work() {
     const { epochToProve, blockNumber, slotNumber } = await this.getEpochNumberToProve();
-    this.log.debug(`Epoch to prove: ${epochToProve}`, { blockNumber, slotNumber });
     if (epochToProve === undefined) {
-      this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { blockNumber });
+      this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { epochToProve, blockNumber, slotNumber });
       return;
     }
     if (this.latestEpochNumber !== undefined && epochToProve <= this.latestEpochNumber) {
@@ -93,20 +85,20 @@ export class EpochMonitor implements Traceable {
     }
 
     if (this.options.provingDelayMs) {
-      this.log.debug(`Waiting ${this.options.provingDelayMs}ms before proving epoch ${epochToProve}`);
+      this.log.warn(`Waiting ${this.options.provingDelayMs}ms before proving epoch ${epochToProve}`);
       await sleep(this.options.provingDelayMs);
     }
 
-    this.log.debug(`Epoch ${epochToProve} is ready to be proven`);
+    this.log.verbose(`Epoch ${epochToProve} is ready to be proven`);
     if (await this.handler?.handleEpochReadyToProve(epochToProve)) {
       this.latestEpochNumber = epochToProve;
     }
   }
 
   private async getEpochNumberToProve() {
-    const lastBlockProven = await this.l2BlockSource.getProvenBlockNumber();
+    const lastBlockProven = (await this.l2BlockSource.getBlockNumber({ tag: 'proven' })) ?? BlockNumber.ZERO;
     const firstBlockToProve = BlockNumber(lastBlockProven + 1);
-    const firstBlockHeaderToProve = await this.l2BlockSource.getBlockHeader(firstBlockToProve);
+    const firstBlockHeaderToProve = (await this.l2BlockSource.getBlockData({ number: firstBlockToProve }))?.header;
     if (!firstBlockHeaderToProve) {
       return { epochToProve: undefined, blockNumber: firstBlockToProve };
     }