@aztec/sequencer-client 4.0.0-nightly.20250907 → 4.0.0-nightly.20260107

package/dest/sequencer/metrics.js

@@ -1,8 +1,7 @@
 import { Attributes, Metrics, ValueType } from '@aztec/telemetry-client';
 import { formatUnits } from 'viem';
-import { sequencerStateToNumber } from './utils.js';
+// TODO(palla/mbps): Review all metrics and add any missing ones per checkpoint
 export class SequencerMetrics {
-    coinbase;
     rollup;
     tracer;
     meter;
@@ -10,9 +9,6 @@ export class SequencerMetrics {
     blockBuildDuration;
     blockBuildManaPerSecond;
     stateTransitionBufferDuration;
-    currentBlockNumber;
-    currentBlockSize;
-    blockBuilderInsertions;
     // these are gauges because for individual sequencers building a block is not something that happens often enough to warrant a histogram
     timeToCollectAttestations;
     allowanceToCollectAttestations;
@@ -21,19 +17,26 @@ export class SequencerMetrics {
     rewards;
     slots;
     filledSlots;
-    missedSlots;
+    blockProposalFailed;
+    blockProposalSuccess;
+    blockProposalPrecheckFailed;
+    checkpointSuccess;
+    slashingAttempts;
+    blockAttestationDelay;
+    // Fisherman fee analysis metrics
+    fishermanWouldBeIncluded;
+    fishermanTimeBeforeBlock;
+    fishermanPendingBlobTxCount;
+    fishermanIncludedBlobTxCount;
+    fishermanCalculatedPriorityFee;
+    fishermanPriorityFeeDelta;
+    fishermanEstimatedCost;
+    fishermanEstimatedOverpayment;
+    fishermanMinedBlobTxPriorityFee;
+    fishermanMinedBlobTxTotalCost;
     lastSeenSlot;
-    constructor(client, getState, coinbase, rollup, name = 'Sequencer'){
-        this.coinbase = coinbase;
+    constructor(client, rollup, name = 'Sequencer'){
         this.rollup = rollup;
-        this.observe = async (observer)=>{
-            let rewards = 0n;
-            rewards = await this.rollup.getSequencerRewards(this.coinbase);
-            const fmt = parseFloat(formatUnits(rewards, 18));
-            observer.observe(this.rewards, fmt, {
-                [Attributes.COINBASE]: this.coinbase.toString()
-            });
-        };
         this.meter = client.getMeter(name);
         this.tracer = client.getTracer(name);
         this.blockCounter = this.meter.createUpDownCounter(Metrics.SEQUENCER_BLOCK_COUNT);
@@ -52,37 +55,19 @@ export class SequencerMetrics {
             description: 'The time difference between when the sequencer needed to transition to a new state and when it actually did.',
             valueType: ValueType.INT
         });
-        const currentState = this.meter.createObservableGauge(Metrics.SEQUENCER_CURRENT_STATE, {
-            description: 'Current state of the sequencer'
-        });
-        currentState.addCallback((observer)=>{
-            observer.observe(sequencerStateToNumber(getState()));
-        });
-        this.currentBlockNumber = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_NUMBER, {
-            description: 'Current block number',
-            valueType: ValueType.INT
-        });
-        this.currentBlockSize = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_SIZE, {
-            description: 'Current block size',
-            valueType: ValueType.INT
-        });
-        this.blockBuilderInsertions = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_BUILD_INSERTION_TIME, {
-            description: 'Timer for tree insertions performed by the block builder',
-            unit: 'us',
+        this.blockAttestationDelay = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_ATTESTATION_DELAY, {
+            unit: 'ms',
+            description: 'The time difference between block proposal and minimal attestation count reached,',
             valueType: ValueType.INT
         });
         // Init gauges and counters
-        this.setCurrentBlock(0, 0);
-        this.blockCounter.add(0, {
-            [Attributes.STATUS]: 'cancelled'
-        });
         this.blockCounter.add(0, {
             [Attributes.STATUS]: 'failed'
         });
         this.blockCounter.add(0, {
             [Attributes.STATUS]: 'built'
         });
-        this.rewards = this.meter.createObservableGauge(Metrics.SEQUENCER_CURRENT_BLOCK_REWARDS, {
+        this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_REWARDS, {
             valueType: ValueType.DOUBLE,
             description: 'The rewards earned'
         });
@@ -90,14 +75,13 @@ export class SequencerMetrics {
             valueType: ValueType.INT,
             description: 'The number of slots this sequencer was selected for'
         });
-        this.filledSlots = this.meter.createUpDownCounter(Metrics.SEQUENCER_FILLED_SLOT_COUNT, {
+        /**
+     * NOTE: we do not track missed slots as a separate metric. That would be difficult to determine
+     * Instead, use a computed metric, `slots - filledSlots` to get the number of slots a sequencer has missed.
+     */ this.filledSlots = this.meter.createUpDownCounter(Metrics.SEQUENCER_FILLED_SLOT_COUNT, {
             valueType: ValueType.INT,
             description: 'The number of slots this sequencer has filled'
         });
-        this.missedSlots = this.meter.createUpDownCounter(Metrics.SEQUENCER_MISSED_SLOT_COUNT, {
-            valueType: ValueType.INT,
-            description: 'The number of slots this sequencer has missed to fill'
-        });
         this.timeToCollectAttestations = this.meter.createGauge(Metrics.SEQUENCER_COLLECT_ATTESTATIONS_DURATION, {
             description: 'The time spent collecting attestations from committee members',
             unit: 'ms',
@@ -116,21 +100,75 @@ export class SequencerMetrics {
             valueType: ValueType.INT,
             description: 'The minimum number of attestations required to publish a block'
         });
+        this.blockProposalFailed = this.meter.createUpDownCounter(Metrics.SEQUENCER_BLOCK_PROPOSAL_FAILED_COUNT, {
+            valueType: ValueType.INT,
+            description: 'The number of times block proposal failed (including validation builds)'
+        });
+        this.blockProposalSuccess = this.meter.createUpDownCounter(Metrics.SEQUENCER_BLOCK_PROPOSAL_SUCCESS_COUNT, {
+            valueType: ValueType.INT,
+            description: 'The number of times block proposal succeeded (including validation builds)'
+        });
+        this.checkpointSuccess = this.meter.createUpDownCounter(Metrics.SEQUENCER_CHECKPOINT_SUCCESS_COUNT, {
+            valueType: ValueType.INT,
+            description: 'The number of times checkpoint publishing succeeded'
+        });
+        this.blockProposalPrecheckFailed = this.meter.createUpDownCounter(Metrics.SEQUENCER_BLOCK_PROPOSAL_PRECHECK_FAILED_COUNT, {
+            valueType: ValueType.INT,
+            description: 'The number of times block proposal pre-build checks failed'
+        });
+        this.slashingAttempts = this.meter.createUpDownCounter(Metrics.SEQUENCER_SLASHING_ATTEMPTS_COUNT, {
+            valueType: ValueType.INT,
+            description: 'The number of slashing action attempts'
+        });
+        // Fisherman fee analysis metrics
+        this.fishermanWouldBeIncluded = this.meter.createUpDownCounter(Metrics.FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED, {
+            valueType: ValueType.INT,
+            description: 'Whether the transaction would have been included in the block'
+        });
+        this.fishermanTimeBeforeBlock = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_TIME_BEFORE_BLOCK, {
+            unit: 'ms',
+            description: 'Time in ms between fee analysis and block being mined',
+            valueType: ValueType.INT
+        });
+        this.fishermanPendingBlobTxCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_PENDING_BLOB_TX_COUNT, {
+            description: 'Number of blob transactions seen in the pending block',
+            valueType: ValueType.INT
+        });
+        this.fishermanIncludedBlobTxCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_INCLUDED_BLOB_TX_COUNT, {
+            description: 'Number of blob transactions that got included in the mined block',
+            valueType: ValueType.INT
+        });
+        this.fishermanCalculatedPriorityFee = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_CALCULATED_PRIORITY_FEE, {
+            unit: 'gwei',
+            description: 'Priority fee calculated by each strategy',
+            valueType: ValueType.DOUBLE
+        });
+        this.fishermanPriorityFeeDelta = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_PRIORITY_FEE_DELTA, {
+            unit: 'gwei',
+            description: 'Difference between our priority fee and minimum included priority fee',
+            valueType: ValueType.DOUBLE
+        });
+        this.fishermanEstimatedCost = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_ESTIMATED_COST, {
+            unit: 'eth',
+            description: 'Estimated total cost in ETH for the transaction with this strategy',
+            valueType: ValueType.DOUBLE
+        });
+        this.fishermanEstimatedOverpayment = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_ESTIMATED_OVERPAYMENT, {
+            unit: 'eth',
+            description: 'Estimated overpayment in ETH vs minimum required for inclusion',
+            valueType: ValueType.DOUBLE
+        });
+        this.fishermanMinedBlobTxPriorityFee = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_PRIORITY_FEE, {
+            unit: 'gwei',
+            description: 'Priority fee per gas for blob transactions in mined blocks',
+            valueType: ValueType.DOUBLE
+        });
+        this.fishermanMinedBlobTxTotalCost = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_TOTAL_COST, {
+            unit: 'eth',
+            description: 'Total cost in ETH for blob transactions in mined blocks',
+            valueType: ValueType.DOUBLE
+        });
     }
-    setCoinbase(coinbase) {
-        this.coinbase = coinbase;
-    }
-    start() {
-        this.meter.addBatchObservableCallback(this.observe, [
-            this.rewards
-        ]);
-    }
-    stop() {
-        this.meter.removeBatchObservableCallback(this.observe, [
-            this.rewards
-        ]);
-    }
-    observe;
     recordRequiredAttestations(requiredAttestationsCount, allowanceMs) {
         this.requiredAttestions.record(requiredAttestationsCount);
         this.allowanceToCollectAttestations.record(Math.ceil(allowanceMs));
@@ -138,19 +176,13 @@ export class SequencerMetrics {
         this.collectedAttestions.record(0);
         this.timeToCollectAttestations.record(0);
     }
+    recordBlockAttestationDelay(duration) {
+        this.blockAttestationDelay.record(duration);
+    }
     recordCollectedAttestations(count, durationMs) {
         this.collectedAttestions.record(count);
         this.timeToCollectAttestations.record(Math.ceil(durationMs));
     }
-    recordBlockBuilderTreeInsertions(timeUs) {
-        this.blockBuilderInsertions.record(Math.ceil(timeUs));
-    }
-    recordCancelledBlock() {
-        this.blockCounter.add(1, {
-            [Attributes.STATUS]: 'cancelled'
-        });
-        this.setCurrentBlock(0, 0);
-    }
     recordBuiltBlock(buildDurationMs, totalMana) {
         this.blockCounter.add(1, {
             [Attributes.STATUS]: 'built'
@@ -162,41 +194,135 @@ export class SequencerMetrics {
         this.blockCounter.add(1, {
             [Attributes.STATUS]: 'failed'
         });
-        this.setCurrentBlock(0, 0);
-    }
-    recordNewBlock(blockNumber, txCount) {
-        this.setCurrentBlock(blockNumber, txCount);
     }
     recordStateTransitionBufferMs(durationMs, state) {
         this.stateTransitionBufferDuration.record(durationMs, {
             [Attributes.SEQUENCER_STATE]: state
         });
     }
-    observeSlotChange(slot, proposer) {
+    incOpenSlot(slot, proposer) {
         // sequencer went through the loop a second time. Noop
         if (slot === this.lastSeenSlot) {
             return;
         }
-        if (typeof this.lastSeenSlot === 'bigint') {
-            this.missedSlots.add(1, {
-                [Attributes.BLOCK_PROPOSER]: proposer
-            });
-        }
-        if (typeof slot === 'bigint') {
-            this.slots.add(1, {
-                [Attributes.BLOCK_PROPOSER]: proposer
-            });
-        }
+        this.slots.add(1, {
+            [Attributes.BLOCK_PROPOSER]: proposer
+        });
         this.lastSeenSlot = slot;
     }
-    incFilledSlot(proposer) {
+    async incFilledSlot(proposer, coinbase) {
         this.filledSlots.add(1, {
             [Attributes.BLOCK_PROPOSER]: proposer
         });
         this.lastSeenSlot = undefined;
+        if (coinbase) {
+            try {
+                const rewards = await this.rollup.getSequencerRewards(coinbase);
+                const fmt = parseFloat(formatUnits(rewards, 18));
+                this.rewards.record(fmt, {
+                    [Attributes.COINBASE]: coinbase.toString()
+                });
+            } catch  {
+            // no-op
+            }
+        }
+    }
+    recordCheckpointSuccess() {
+        this.checkpointSuccess.add(1);
+    }
+    recordBlockProposalFailed(reason) {
+        this.blockProposalFailed.add(1, {
+            ...reason && {
+                [Attributes.ERROR_TYPE]: reason
+            }
+        });
+    }
+    recordBlockProposalSuccess() {
+        this.blockProposalSuccess.add(1);
+    }
+    recordBlockProposalPrecheckFailed(checkType) {
+        this.blockProposalPrecheckFailed.add(1, {
+            [Attributes.ERROR_TYPE]: checkType
+        });
     }
-    setCurrentBlock(blockNumber, txCount) {
-        this.currentBlockNumber.record(blockNumber);
-        this.currentBlockSize.record(txCount);
+    recordSlashingAttempt(actionCount) {
+        this.slashingAttempts.add(actionCount);
+    }
+    /**
+   * Records metrics for a completed fisherman fee analysis
+   * @param analysis - The completed fee analysis result
+   */ recordFishermanFeeAnalysis(analysis) {
+        // In fisherman mode, we should always have strategy results
+        if (!analysis.computedPrices.strategyResults || analysis.computedPrices.strategyResults.length === 0) {
+            // This should never happen in fisherman mode - log an error
+            // We don't record metrics without strategy IDs as that defeats the purpose
+            throw new Error(`No strategy results found in fisherman fee analysis ${analysis.id}. This indicates a bug in the fee analysis.`);
+        }
+        // Record metrics for each strategy separately
+        for (const strategyResult of analysis.computedPrices.strategyResults){
+            const strategyAttributes = {
+                [Attributes.FISHERMAN_FEE_STRATEGY_ID]: strategyResult.strategyId
+            };
+            // Record pending block snapshot data (once per strategy for comparison)
+            this.fishermanPendingBlobTxCount.record(analysis.pendingSnapshot.pendingBlobTxCount, strategyAttributes);
+            // Record mined block data if available
+            if (analysis.minedBlock) {
+                this.fishermanIncludedBlobTxCount.record(analysis.minedBlock.includedBlobTxCount, strategyAttributes);
+                // Record actual fees from blob transactions in the mined block
+                for (const blobTx of analysis.minedBlock.includedBlobTxs){
+                    // Record priority fee per gas in Gwei
+                    const priorityFeeGwei = Number(blobTx.maxPriorityFeePerGas) / 1e9;
+                    this.fishermanMinedBlobTxPriorityFee.record(priorityFeeGwei, strategyAttributes);
+                    // Calculate total cost in ETH
+                    // Cost = (gas * (baseFee + priorityFee)) + (blobCount * blobGasPerBlob * blobBaseFee)
+                    const baseFee = analysis.minedBlock.baseFeePerGas;
+                    const effectiveGasPrice = baseFee + blobTx.maxPriorityFeePerGas;
+                    // Calculate execution cost using actual gas limit from the transaction
+                    const executionCost = blobTx.gas * effectiveGasPrice;
+                    // Calculate blob cost using maxFeePerBlobGas * blobCount * GAS_PER_BLOB
+                    const blobCost = blobTx.maxFeePerBlobGas * BigInt(blobTx.blobCount) * 131072n; // 128KB per blob
+                    const totalCostWei = executionCost + blobCost;
+                    const totalCostEth = Number(totalCostWei) / 1e18;
+                    this.fishermanMinedBlobTxTotalCost.record(totalCostEth, strategyAttributes);
+                }
+            }
+            // Record the calculated priority fee for this strategy
+            const calculatedPriorityFeeGwei = Number(strategyResult.calculatedPriorityFee) / 1e9;
+            this.fishermanCalculatedPriorityFee.record(calculatedPriorityFeeGwei, strategyAttributes);
+            // Record analysis results if available
+            if (analysis.analysis) {
+                this.fishermanTimeBeforeBlock.record(Math.ceil(analysis.analysis.timeBeforeBlockMs), strategyAttributes);
+                // Record strategy-specific inclusion result
+                if (strategyResult.wouldBeIncluded !== undefined) {
+                    if (strategyResult.wouldBeIncluded) {
+                        this.fishermanWouldBeIncluded.add(1, {
+                            ...strategyAttributes,
+                            [Attributes.OK]: true
+                        });
+                    } else {
+                        this.fishermanWouldBeIncluded.add(1, {
+                            ...strategyAttributes,
+                            [Attributes.OK]: false,
+                            ...strategyResult.exclusionReason && {
+                                [Attributes.ERROR_TYPE]: strategyResult.exclusionReason
+                            }
+                        });
+                    }
+                }
+                // Record strategy-specific priority fee delta
+                if (strategyResult.priorityFeeDelta !== undefined) {
+                    const priorityFeeDeltaGwei = Number(strategyResult.priorityFeeDelta) / 1e9;
+                    this.fishermanPriorityFeeDelta.record(priorityFeeDeltaGwei, strategyAttributes);
+                }
+                // Record estimated cost if available
+                if (strategyResult.estimatedCostEth !== undefined) {
+                    this.fishermanEstimatedCost.record(strategyResult.estimatedCostEth, strategyAttributes);
+                }
+                // Record estimated overpayment if available
+                if (strategyResult.estimatedOverpaymentEth !== undefined) {
+                    this.fishermanEstimatedOverpayment.record(strategyResult.estimatedOverpaymentEth, strategyAttributes);
+                }
+            }
+        }
     }
 }