@aztec/sequencer-client 0.0.1-commit.03f7ef2 → 0.0.1-commit.04d373f

package/dest/sequencer/metrics.js

@@ -1,6 +1,5 @@
-import { Attributes, Metrics, ValueType } from '@aztec/telemetry-client';
+import { Attributes, Metrics, createUpDownCounterWithDefault } from '@aztec/telemetry-client';
 import { formatUnits } from 'viem';
-// TODO(palla/mbps): Review all metrics and add any missing ones per checkpoint
 export class SequencerMetrics {
     rollup;
     tracer;
@@ -9,6 +8,7 @@ export class SequencerMetrics {
     blockBuildDuration;
     blockBuildManaPerSecond;
     stateTransitionBufferDuration;
+    stateDuration;
     // these are gauges because for individual sequencers building a block is not something that happens often enough to warrant a histogram
     timeToCollectAttestations;
     allowanceToCollectAttestations;
@@ -18,156 +18,112 @@ export class SequencerMetrics {
     slots;
     filledSlots;
     blockProposalFailed;
-    blockProposalSuccess;
-    blockProposalPrecheckFailed;
+    checkpointProposalSuccess;
+    checkpointPrecheckFailed;
+    checkpointProposalFailed;
     checkpointSuccess;
     slashingAttempts;
-    blockAttestationDelay;
+    pipelineDepth;
+    pipelineDiscards;
+    pipelineParentCheckpointMismatches;
     // Fisherman fee analysis metrics
     fishermanWouldBeIncluded;
     fishermanTimeBeforeBlock;
     fishermanPendingBlobTxCount;
     fishermanIncludedBlobTxCount;
+    fishermanPendingBlobCount;
+    fishermanIncludedBlobCount;
+    fishermanBlockBlobsFull;
+    fishermanMaxBlobCapacity;
     fishermanCalculatedPriorityFee;
     fishermanPriorityFeeDelta;
     fishermanEstimatedCost;
     fishermanEstimatedOverpayment;
     fishermanMinedBlobTxPriorityFee;
     fishermanMinedBlobTxTotalCost;
+    blockInterBlockTime;
+    lastBlockBuiltTimestamp;
+    lastBlockBuiltSlot;
     lastSeenSlot;
     constructor(client, rollup, name = 'Sequencer'){
         this.rollup = rollup;
         this.meter = client.getMeter(name);
         this.tracer = client.getTracer(name);
-        this.blockCounter = this.meter.createUpDownCounter(Metrics.SEQUENCER_BLOCK_COUNT);
-        this.blockBuildDuration = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_BUILD_DURATION, {
-            unit: 'ms',
-            description: 'Duration to build a block',
-            valueType: ValueType.INT
-        });
-        this.blockBuildManaPerSecond = this.meter.createGauge(Metrics.SEQUENCER_BLOCK_BUILD_MANA_PER_SECOND, {
-            unit: 'mana/s',
-            description: 'Mana per second when building a block',
-            valueType: ValueType.INT
-        });
-        this.stateTransitionBufferDuration = this.meter.createHistogram(Metrics.SEQUENCER_STATE_TRANSITION_BUFFER_DURATION, {
-            unit: 'ms',
-            description: 'The time difference between when the sequencer needed to transition to a new state and when it actually did.',
-            valueType: ValueType.INT
-        });
-        this.blockAttestationDelay = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_ATTESTATION_DELAY, {
-            unit: 'ms',
-            description: 'The time difference between block proposal and minimal attestation count reached,',
-            valueType: ValueType.INT
-        });
-        // Init gauges and counters
-        this.blockCounter.add(0, {
-            [Attributes.STATUS]: 'failed'
-        });
-        this.blockCounter.add(0, {
-            [Attributes.STATUS]: 'built'
-        });
-        this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_REWARDS, {
-            valueType: ValueType.DOUBLE,
-            description: 'The rewards earned'
-        });
-        this.slots = this.meter.createUpDownCounter(Metrics.SEQUENCER_SLOT_COUNT, {
-            valueType: ValueType.INT,
-            description: 'The number of slots this sequencer was selected for'
-        });
+        this.blockCounter = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_BLOCK_COUNT, {
+            [Attributes.STATUS]: [
+                'failed',
+                'built'
+            ]
+        });
+        this.blockBuildDuration = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_BUILD_DURATION);
+        this.blockBuildManaPerSecond = this.meter.createGauge(Metrics.SEQUENCER_BLOCK_BUILD_MANA_PER_SECOND);
+        this.blockInterBlockTime = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_INTER_BLOCK_TIME);
+        this.stateTransitionBufferDuration = this.meter.createHistogram(Metrics.SEQUENCER_STATE_TRANSITION_BUFFER_DURATION);
+        this.stateDuration = this.meter.createHistogram(Metrics.SEQUENCER_STATE_DURATION);
+        this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_SLOT_REWARDS);
+        this.slots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLOT_COUNT);
         /**
      * NOTE: we do not track missed slots as a separate metric. That would be difficult to determine
      * Instead, use a computed metric, `slots - filledSlots` to get the number of slots a sequencer has missed.
-     */ this.filledSlots = this.meter.createUpDownCounter(Metrics.SEQUENCER_FILLED_SLOT_COUNT, {
-            valueType: ValueType.INT,
-            description: 'The number of slots this sequencer has filled'
-        });
-        this.timeToCollectAttestations = this.meter.createGauge(Metrics.SEQUENCER_COLLECT_ATTESTATIONS_DURATION, {
-            description: 'The time spent collecting attestations from committee members',
-            unit: 'ms',
-            valueType: ValueType.INT
-        });
-        this.allowanceToCollectAttestations = this.meter.createGauge(Metrics.SEQUENCER_COLLECT_ATTESTATIONS_TIME_ALLOWANCE, {
-            description: 'Maximum amount of time to collect attestations',
-            unit: 'ms',
-            valueType: ValueType.INT
-        });
-        this.requiredAttestions = this.meter.createGauge(Metrics.SEQUENCER_REQUIRED_ATTESTATIONS_COUNT, {
-            valueType: ValueType.INT,
-            description: 'The minimum number of attestations required to publish a block'
-        });
-        this.collectedAttestions = this.meter.createGauge(Metrics.SEQUENCER_COLLECTED_ATTESTATIONS_COUNT, {
-            valueType: ValueType.INT,
-            description: 'The minimum number of attestations required to publish a block'
-        });
-        this.blockProposalFailed = this.meter.createUpDownCounter(Metrics.SEQUENCER_BLOCK_PROPOSAL_FAILED_COUNT, {
-            valueType: ValueType.INT,
-            description: 'The number of times block proposal failed (including validation builds)'
-        });
-        this.blockProposalSuccess = this.meter.createUpDownCounter(Metrics.SEQUENCER_BLOCK_PROPOSAL_SUCCESS_COUNT, {
-            valueType: ValueType.INT,
-            description: 'The number of times block proposal succeeded (including validation builds)'
-        });
-        this.checkpointSuccess = this.meter.createUpDownCounter(Metrics.SEQUENCER_CHECKPOINT_SUCCESS_COUNT, {
-            valueType: ValueType.INT,
-            description: 'The number of times checkpoint publishing succeeded'
-        });
-        this.blockProposalPrecheckFailed = this.meter.createUpDownCounter(Metrics.SEQUENCER_BLOCK_PROPOSAL_PRECHECK_FAILED_COUNT, {
-            valueType: ValueType.INT,
-            description: 'The number of times block proposal pre-build checks failed'
-        });
-        this.slashingAttempts = this.meter.createUpDownCounter(Metrics.SEQUENCER_SLASHING_ATTEMPTS_COUNT, {
-            valueType: ValueType.INT,
-            description: 'The number of slashing action attempts'
-        });
+     */ this.filledSlots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_FILLED_SLOT_COUNT);
+        this.timeToCollectAttestations = this.meter.createGauge(Metrics.SEQUENCER_COLLECT_ATTESTATIONS_DURATION);
+        this.allowanceToCollectAttestations = this.meter.createGauge(Metrics.SEQUENCER_COLLECT_ATTESTATIONS_TIME_ALLOWANCE);
+        this.requiredAttestions = this.meter.createGauge(Metrics.SEQUENCER_REQUIRED_ATTESTATIONS_COUNT);
+        this.collectedAttestions = this.meter.createGauge(Metrics.SEQUENCER_COLLECTED_ATTESTATIONS_COUNT);
+        this.blockProposalFailed = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_BLOCK_PROPOSAL_FAILED_COUNT);
+        this.checkpointProposalSuccess = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_CHECKPOINT_PROPOSAL_SUCCESS_COUNT);
+        this.checkpointSuccess = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_CHECKPOINT_SUCCESS_COUNT);
+        this.checkpointPrecheckFailed = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_CHECKPOINT_PRECHECK_FAILED_COUNT, {
+            [Attributes.ERROR_TYPE]: [
+                'slot_already_taken',
+                'rollup_contract_check_failed',
+                'slot_mismatch',
+                'block_number_mismatch'
+            ]
+        });
+        this.checkpointProposalFailed = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_CHECKPOINT_PROPOSAL_FAILED_COUNT);
+        this.slashingAttempts = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLASHING_ATTEMPTS_COUNT);
+        this.pipelineDepth = this.meter.createGauge(Metrics.SEQUENCER_PIPELINE_DEPTH);
+        this.pipelineDiscards = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_PIPELINE_DISCARDS_COUNT);
+        this.pipelineParentCheckpointMismatches = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_PIPELINE_PARENT_CHECKPOINT_MISMATCH_COUNT, {
+            [Attributes.ERROR_TYPE]: [
+                'archiver-sync-timeout',
+                'parent-not-on-l1',
+                'parent-hash-mismatch',
+                'parent-invalid-attestations',
+                'unexpected-parent-appeared'
+            ]
+        });
+        this.pipelineDepth.record(0);
         // Fisherman fee analysis metrics
-        this.fishermanWouldBeIncluded = this.meter.createUpDownCounter(Metrics.FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED, {
-            valueType: ValueType.INT,
-            description: 'Whether the transaction would have been included in the block'
-        });
-        this.fishermanTimeBeforeBlock = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_TIME_BEFORE_BLOCK, {
-            unit: 'ms',
-            description: 'Time in ms between fee analysis and block being mined',
-            valueType: ValueType.INT
-        });
-        this.fishermanPendingBlobTxCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_PENDING_BLOB_TX_COUNT, {
-            description: 'Number of blob transactions seen in the pending block',
-            valueType: ValueType.INT
-        });
-        this.fishermanIncludedBlobTxCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_INCLUDED_BLOB_TX_COUNT, {
-            description: 'Number of blob transactions that got included in the mined block',
-            valueType: ValueType.INT
-        });
-        this.fishermanCalculatedPriorityFee = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_CALCULATED_PRIORITY_FEE, {
-            unit: 'gwei',
-            description: 'Priority fee calculated by each strategy',
-            valueType: ValueType.DOUBLE
-        });
-        this.fishermanPriorityFeeDelta = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_PRIORITY_FEE_DELTA, {
-            unit: 'gwei',
-            description: 'Difference between our priority fee and minimum included priority fee',
-            valueType: ValueType.DOUBLE
-        });
-        this.fishermanEstimatedCost = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_ESTIMATED_COST, {
-            unit: 'eth',
-            description: 'Estimated total cost in ETH for the transaction with this strategy',
-            valueType: ValueType.DOUBLE
-        });
-        this.fishermanEstimatedOverpayment = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_ESTIMATED_OVERPAYMENT, {
-            unit: 'eth',
-            description: 'Estimated overpayment in ETH vs minimum required for inclusion',
-            valueType: ValueType.DOUBLE
-        });
-        this.fishermanMinedBlobTxPriorityFee = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_PRIORITY_FEE, {
-            unit: 'gwei',
-            description: 'Priority fee per gas for blob transactions in mined blocks',
-            valueType: ValueType.DOUBLE
-        });
-        this.fishermanMinedBlobTxTotalCost = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_TOTAL_COST, {
-            unit: 'eth',
-            description: 'Total cost in ETH for blob transactions in mined blocks',
-            valueType: ValueType.DOUBLE
-        });
+        this.fishermanWouldBeIncluded = createUpDownCounterWithDefault(this.meter, Metrics.FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED, {
+            [Attributes.OK]: [
+                true,
+                false
+            ],
+            [Attributes.BLOCK_FULL]: [
+                'true',
+                'false'
+            ]
+        });
+        this.fishermanTimeBeforeBlock = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_TIME_BEFORE_BLOCK);
+        this.fishermanPendingBlobTxCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_PENDING_BLOB_TX_COUNT);
+        this.fishermanIncludedBlobTxCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_INCLUDED_BLOB_TX_COUNT);
+        this.fishermanCalculatedPriorityFee = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_CALCULATED_PRIORITY_FEE);
+        this.fishermanPriorityFeeDelta = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_PRIORITY_FEE_DELTA);
+        this.fishermanEstimatedCost = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_ESTIMATED_COST);
+        this.fishermanEstimatedOverpayment = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_ESTIMATED_OVERPAYMENT);
+        this.fishermanMinedBlobTxPriorityFee = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_PRIORITY_FEE);
+        this.fishermanMinedBlobTxTotalCost = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_TOTAL_COST);
+        this.fishermanPendingBlobCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_PENDING_BLOB_COUNT);
+        this.fishermanIncludedBlobCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_INCLUDED_BLOB_COUNT);
+        this.fishermanBlockBlobsFull = createUpDownCounterWithDefault(this.meter, Metrics.FISHERMAN_FEE_ANALYSIS_BLOCK_BLOBS_FULL, {
+            [Attributes.OK]: [
+                true,
+                false
+            ]
+        });
+        this.fishermanMaxBlobCapacity = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_MAX_BLOB_CAPACITY);
     }
     recordRequiredAttestations(requiredAttestationsCount, allowanceMs) {
         this.requiredAttestions.record(requiredAttestationsCount);
@@ -176,19 +132,23 @@ export class SequencerMetrics {
         this.collectedAttestions.record(0);
         this.timeToCollectAttestations.record(0);
     }
-    recordBlockAttestationDelay(duration) {
-        this.blockAttestationDelay.record(duration);
-    }
     recordCollectedAttestations(count, durationMs) {
         this.collectedAttestions.record(count);
         this.timeToCollectAttestations.record(Math.ceil(durationMs));
     }
-    recordBuiltBlock(buildDurationMs, totalMana) {
+    recordBuiltBlock(buildDurationMs, totalMana, slot) {
         this.blockCounter.add(1, {
             [Attributes.STATUS]: 'built'
         });
         this.blockBuildDuration.record(Math.ceil(buildDurationMs));
         this.blockBuildManaPerSecond.record(Math.ceil(totalMana * 1000 / buildDurationMs));
+        // Only record inter-block time between blocks built within the same slot.
+        const now = Date.now();
+        if (this.lastBlockBuiltTimestamp !== undefined && this.lastBlockBuiltSlot === slot) {
+            this.blockInterBlockTime.record(now - this.lastBlockBuiltTimestamp);
+        }
+        this.lastBlockBuiltTimestamp = now;
+        this.lastBlockBuiltSlot = slot;
     }
     recordFailedBlock() {
         this.blockCounter.add(1, {
@@ -200,6 +160,22 @@ export class SequencerMetrics {
             [Attributes.SEQUENCER_STATE]: state
         });
     }
+    recordStateDuration(durationMs, state) {
+        this.stateDuration.record(Math.ceil(durationMs), {
+            [Attributes.SEQUENCER_STATE]: state
+        });
+    }
+    recordPipelineDepth(depth) {
+        this.pipelineDepth.record(depth);
+    }
+    recordPipelineDiscard(count = 1) {
+        this.pipelineDiscards.add(count);
+    }
+    recordPipelineParentCheckpointMismatch(reason) {
+        this.pipelineParentCheckpointMismatches.add(1, {
+            [Attributes.ERROR_TYPE]: reason
+        });
+    }
     incOpenSlot(slot, proposer) {
         // sequencer went through the loop a second time. Noop
         if (slot === this.lastSeenSlot) {
@@ -237,14 +213,21 @@ export class SequencerMetrics {
             }
         });
     }
-    recordBlockProposalSuccess() {
-        this.blockProposalSuccess.add(1);
+    recordCheckpointProposalSuccess() {
+        this.checkpointProposalSuccess.add(1);
     }
-    recordBlockProposalPrecheckFailed(checkType) {
-        this.blockProposalPrecheckFailed.add(1, {
+    recordCheckpointPrecheckFailed(checkType) {
+        this.checkpointPrecheckFailed.add(1, {
             [Attributes.ERROR_TYPE]: checkType
         });
     }
+    recordCheckpointProposalFailed(reason) {
+        this.checkpointProposalFailed.add(1, {
+            ...reason && {
+                [Attributes.ERROR_TYPE]: reason
+            }
+        });
+    }
     recordSlashingAttempt(actionCount) {
         this.slashingAttempts.add(actionCount);
     }
@@ -265,9 +248,11 @@ export class SequencerMetrics {
             };
             // Record pending block snapshot data (once per strategy for comparison)
             this.fishermanPendingBlobTxCount.record(analysis.pendingSnapshot.pendingBlobTxCount, strategyAttributes);
+            this.fishermanPendingBlobCount.record(analysis.pendingSnapshot.pendingBlobCount, strategyAttributes);
             // Record mined block data if available
             if (analysis.minedBlock) {
                 this.fishermanIncludedBlobTxCount.record(analysis.minedBlock.includedBlobTxCount, strategyAttributes);
+                this.fishermanIncludedBlobCount.record(analysis.minedBlock.includedBlobCount, strategyAttributes);
                 // Record actual fees from blob transactions in the mined block
                 for (const blobTx of analysis.minedBlock.includedBlobTxs){
                     // Record priority fee per gas in Gwei
@@ -292,16 +277,34 @@ export class SequencerMetrics {
             // Record analysis results if available
             if (analysis.analysis) {
                 this.fishermanTimeBeforeBlock.record(Math.ceil(analysis.analysis.timeBeforeBlockMs), strategyAttributes);
+                // Record whether the block reached 100% blob capacity
+                if (analysis.analysis.blockBlobsFull) {
+                    this.fishermanBlockBlobsFull.add(1, {
+                        ...strategyAttributes,
+                        [Attributes.OK]: true
+                    });
+                } else {
+                    this.fishermanBlockBlobsFull.add(1, {
+                        ...strategyAttributes,
+                        [Attributes.OK]: false
+                    });
+                }
+                // Record the max blob capacity for this block
+                this.fishermanMaxBlobCapacity.record(analysis.analysis.maxBlobCapacity, strategyAttributes);
                 // Record strategy-specific inclusion result
                 if (strategyResult.wouldBeIncluded !== undefined) {
+                    const inclusionAttributes = {
+                        ...strategyAttributes,
+                        [Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false'
+                    };
                     if (strategyResult.wouldBeIncluded) {
                         this.fishermanWouldBeIncluded.add(1, {
-                            ...strategyAttributes,
+                            ...inclusionAttributes,
                             [Attributes.OK]: true
                         });
                     } else {
                         this.fishermanWouldBeIncluded.add(1, {
-                            ...strategyAttributes,
+                            ...inclusionAttributes,
                             [Attributes.OK]: false,
                             ...strategyResult.exclusionReason && {
                                 [Attributes.ERROR_TYPE]: strategyResult.exclusionReason
@@ -312,15 +315,27 @@ export class SequencerMetrics {
                 // Record strategy-specific priority fee delta
                 if (strategyResult.priorityFeeDelta !== undefined) {
                     const priorityFeeDeltaGwei = Number(strategyResult.priorityFeeDelta) / 1e9;
-                    this.fishermanPriorityFeeDelta.record(priorityFeeDeltaGwei, strategyAttributes);
+                    const deltaAttributes = {
+                        ...strategyAttributes,
+                        [Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false'
+                    };
+                    this.fishermanPriorityFeeDelta.record(priorityFeeDeltaGwei, deltaAttributes);
                 }
                 // Record estimated cost if available
                 if (strategyResult.estimatedCostEth !== undefined) {
-                    this.fishermanEstimatedCost.record(strategyResult.estimatedCostEth, strategyAttributes);
+                    const costAttributes = {
+                        ...strategyAttributes,
+                        [Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false'
+                    };
+                    this.fishermanEstimatedCost.record(strategyResult.estimatedCostEth, costAttributes);
                 }
                 // Record estimated overpayment if available
                 if (strategyResult.estimatedOverpaymentEth !== undefined) {
-                    this.fishermanEstimatedOverpayment.record(strategyResult.estimatedOverpaymentEth, strategyAttributes);
+                    const overpaymentAttributes = {
+                        ...strategyAttributes,
+                        [Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false'
+                    };
+                    this.fishermanEstimatedOverpayment.record(strategyResult.estimatedOverpaymentEth, overpaymentAttributes);
                 }
             }
         }