@aztec/sequencer-client 0.0.1-commit.03f7ef2 → 0.0.1-commit.08c5969dc

package/src/sequencer/checkpoint_voter.ts

@@ -5,6 +5,8 @@ import type { SlasherClientInterface } from '@aztec/slasher';
 import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
 import type { ResolvedSequencerConfig } from '@aztec/stdlib/interfaces/server';
 import type { ValidatorClient } from '@aztec/validator-client';
+import { DutyAlreadySignedError } from '@aztec/validator-ha-signer/errors';
+import { DutyType, type SigningContext } from '@aztec/validator-ha-signer/types';
 
 import type { TypedDataDefinition } from 'viem';
 
@@ -17,7 +19,8 @@ import type { SequencerRollupConstants } from './types.js';
  */
 export class CheckpointVoter {
   private slotTimestamp: bigint;
-  private signer: (msg: TypedDataDefinition) => Promise<`0x${string}`>;
+  private governanceSigner: (msg: TypedDataDefinition) => Promise<`0x${string}`>;
+  private slashingSigner: (msg: TypedDataDefinition) => Promise<`0x${string}`>;
 
   constructor(
     private readonly slot: SlotNumber,
@@ -31,8 +34,16 @@ export class CheckpointVoter {
     private readonly log: Logger,
   ) {
     this.slotTimestamp = getTimestampForSlot(this.slot, this.l1Constants);
-    this.signer = (msg: TypedDataDefinition) =>
-      this.validatorClient.signWithAddress(this.attestorAddress, msg).then(s => s.toString());
+
+    // Create separate signers with appropriate duty contexts for governance and slashing votes
+    // These use HA protection to ensure only one node signs per slot/duty
+    const governanceContext: SigningContext = { slot: this.slot, dutyType: DutyType.GOVERNANCE_VOTE };
+    this.governanceSigner = (msg: TypedDataDefinition) =>
+      this.validatorClient.signWithAddress(this.attestorAddress, msg, governanceContext).then(s => s.toString());
+
+    const slashingContext: SigningContext = { slot: this.slot, dutyType: DutyType.SLASHING_VOTE };
+    this.slashingSigner = (msg: TypedDataDefinition) =>
+      this.validatorClient.signWithAddress(this.attestorAddress, msg, slashingContext).then(s => s.toString());
   }
 
   /**
@@ -68,10 +79,17 @@ export class CheckpointVoter {
         this.slot,
         this.slotTimestamp,
         this.attestorAddress,
-        this.signer,
+        this.governanceSigner,
       );
     } catch (err) {
-      this.log.error(`Error enqueuing governance vote`, err, { slot: this.slot });
+      if (err instanceof DutyAlreadySignedError) {
+        this.log.info(`Governance vote already signed by another node`, {
+          slot: this.slot,
+          signedByNode: err.signedByNode,
+        });
+      } else {
+        this.log.error(`Error enqueueing governance vote`, err);
+      }
       return false;
     }
   }
@@ -95,10 +113,17 @@ export class CheckpointVoter {
         this.slot,
         this.slotTimestamp,
         this.attestorAddress,
-        this.signer,
+        this.slashingSigner,
       );
     } catch (err) {
-      this.log.error(`Error enqueuing slashing vote`, err, { slot: this.slot });
+      if (err instanceof DutyAlreadySignedError) {
+        this.log.info(`Slashing vote already signed by another node`, {
+          slot: this.slot,
+          signedByNode: err.signedByNode,
+        });
+      } else {
+        this.log.error(`Error enqueueing slashing vote`, err);
+      }
       return false;
     }
   }

package/src/sequencer/index.ts

@@ -1,5 +1,3 @@
-export * from './block_builder.js';
-export * from './checkpoint_builder.js';
 export * from './checkpoint_proposal_job.js';
 export * from './checkpoint_voter.js';
 export * from './config.js';

package/src/sequencer/metrics.ts

@@ -11,7 +11,7 @@ import {
   type TelemetryClient,
   type Tracer,
   type UpDownCounter,
-  ValueType,
+  createUpDownCounterWithDefault,
 } from '@aztec/telemetry-client';
 
 import { type Hex, formatUnits } from 'viem';
@@ -44,13 +44,16 @@ export class SequencerMetrics {
   private blockProposalPrecheckFailed: UpDownCounter;
   private checkpointSuccess: UpDownCounter;
   private slashingAttempts: UpDownCounter;
-  private blockAttestationDelay: Histogram;
+  private checkpointAttestationDelay: Histogram;
 
   // Fisherman fee analysis metrics
   private fishermanWouldBeIncluded: UpDownCounter;
   private fishermanTimeBeforeBlock: Histogram;
   private fishermanPendingBlobTxCount: Histogram;
   private fishermanIncludedBlobTxCount: Histogram;
+  private fishermanPendingBlobCount: Histogram;
+  private fishermanIncludedBlobCount: Histogram;
+  private fishermanBlockBlobsFull: UpDownCounter;
   private fishermanCalculatedPriorityFee: Histogram;
   private fishermanPriorityFeeDelta: Histogram;
   private fishermanEstimatedCost: Histogram;
@@ -68,189 +71,109 @@ export class SequencerMetrics {
     this.meter = client.getMeter(name);
     this.tracer = client.getTracer(name);
 
-    this.blockCounter = this.meter.createUpDownCounter(Metrics.SEQUENCER_BLOCK_COUNT);
-
-    this.blockBuildDuration = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_BUILD_DURATION, {
-      unit: 'ms',
-      description: 'Duration to build a block',
-      valueType: ValueType.INT,
+    this.blockCounter = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_BLOCK_COUNT, {
+      [Attributes.STATUS]: ['failed', 'built'],
     });
 
-    this.blockBuildManaPerSecond = this.meter.createGauge(Metrics.SEQUENCER_BLOCK_BUILD_MANA_PER_SECOND, {
-      unit: 'mana/s',
-      description: 'Mana per second when building a block',
-      valueType: ValueType.INT,
-    });
+    this.blockBuildDuration = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_BUILD_DURATION);
 
-    this.stateTransitionBufferDuration = this.meter.createHistogram(
-      Metrics.SEQUENCER_STATE_TRANSITION_BUFFER_DURATION,
-      {
-        unit: 'ms',
-        description:
-          'The time difference between when the sequencer needed to transition to a new state and when it actually did.',
-        valueType: ValueType.INT,
-      },
-    );
+    this.blockBuildManaPerSecond = this.meter.createGauge(Metrics.SEQUENCER_BLOCK_BUILD_MANA_PER_SECOND);
 
-    this.blockAttestationDelay = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_ATTESTATION_DELAY, {
-      unit: 'ms',
-      description: 'The time difference between block proposal and minimal attestation count reached,',
-      valueType: ValueType.INT,
-    });
+    this.stateTransitionBufferDuration = this.meter.createHistogram(Metrics.SEQUENCER_STATE_TRANSITION_BUFFER_DURATION);
 
-    // Init gauges and counters
-    this.blockCounter.add(0, {
-      [Attributes.STATUS]: 'failed',
-    });
-    this.blockCounter.add(0, {
-      [Attributes.STATUS]: 'built',
-    });
+    this.checkpointAttestationDelay = this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_ATTESTATION_DELAY);
 
-    this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_REWARDS, {
-      valueType: ValueType.DOUBLE,
-      description: 'The rewards earned',
-    });
+    this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_REWARDS);
 
-    this.slots = this.meter.createUpDownCounter(Metrics.SEQUENCER_SLOT_COUNT, {
-      valueType: ValueType.INT,
-      description: 'The number of slots this sequencer was selected for',
-    });
+    this.slots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLOT_COUNT);
 
     /**
      * NOTE: we do not track missed slots as a separate metric. That would be difficult to determine
      * Instead, use a computed metric, `slots - filledSlots` to get the number of slots a sequencer has missed.
      */
-    this.filledSlots = this.meter.createUpDownCounter(Metrics.SEQUENCER_FILLED_SLOT_COUNT, {
-      valueType: ValueType.INT,
-      description: 'The number of slots this sequencer has filled',
-    });
+    this.filledSlots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_FILLED_SLOT_COUNT);
 
-    this.timeToCollectAttestations = this.meter.createGauge(Metrics.SEQUENCER_COLLECT_ATTESTATIONS_DURATION, {
-      description: 'The time spent collecting attestations from committee members',
-      unit: 'ms',
-      valueType: ValueType.INT,
-    });
+    this.timeToCollectAttestations = this.meter.createGauge(Metrics.SEQUENCER_COLLECT_ATTESTATIONS_DURATION);
 
-    this.allowanceToCollectAttestations = this.meter.createGauge(
-      Metrics.SEQUENCER_COLLECT_ATTESTATIONS_TIME_ALLOWANCE,
-      {
-        description: 'Maximum amount of time to collect attestations',
-        unit: 'ms',
-        valueType: ValueType.INT,
-      },
-    );
+    this.allowanceToCollectAttestations = this.meter.createGauge(Metrics.SEQUENCER_COLLECT_ATTESTATIONS_TIME_ALLOWANCE);
 
-    this.requiredAttestions = this.meter.createGauge(Metrics.SEQUENCER_REQUIRED_ATTESTATIONS_COUNT, {
-      valueType: ValueType.INT,
-      description: 'The minimum number of attestations required to publish a block',
-    });
+    this.requiredAttestions = this.meter.createGauge(Metrics.SEQUENCER_REQUIRED_ATTESTATIONS_COUNT);
 
-    this.collectedAttestions = this.meter.createGauge(Metrics.SEQUENCER_COLLECTED_ATTESTATIONS_COUNT, {
-      valueType: ValueType.INT,
-      description: 'The minimum number of attestations required to publish a block',
-    });
+    this.collectedAttestions = this.meter.createGauge(Metrics.SEQUENCER_COLLECTED_ATTESTATIONS_COUNT);
 
-    this.blockProposalFailed = this.meter.createUpDownCounter(Metrics.SEQUENCER_BLOCK_PROPOSAL_FAILED_COUNT, {
-      valueType: ValueType.INT,
-      description: 'The number of times block proposal failed (including validation builds)',
-    });
+    this.blockProposalFailed = createUpDownCounterWithDefault(
+      this.meter,
+      Metrics.SEQUENCER_BLOCK_PROPOSAL_FAILED_COUNT,
+    );
 
-    this.blockProposalSuccess = this.meter.createUpDownCounter(Metrics.SEQUENCER_BLOCK_PROPOSAL_SUCCESS_COUNT, {
-      valueType: ValueType.INT,
-      description: 'The number of times block proposal succeeded (including validation builds)',
-    });
+    this.blockProposalSuccess = createUpDownCounterWithDefault(
+      this.meter,
+      Metrics.SEQUENCER_BLOCK_PROPOSAL_SUCCESS_COUNT,
+    );
 
-    this.checkpointSuccess = this.meter.createUpDownCounter(Metrics.SEQUENCER_CHECKPOINT_SUCCESS_COUNT, {
-      valueType: ValueType.INT,
-      description: 'The number of times checkpoint publishing succeeded',
-    });
+    this.checkpointSuccess = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_CHECKPOINT_SUCCESS_COUNT);
 
-    this.blockProposalPrecheckFailed = this.meter.createUpDownCounter(
+    this.blockProposalPrecheckFailed = createUpDownCounterWithDefault(
+      this.meter,
       Metrics.SEQUENCER_BLOCK_PROPOSAL_PRECHECK_FAILED_COUNT,
       {
-        valueType: ValueType.INT,
-        description: 'The number of times block proposal pre-build checks failed',
+        [Attributes.ERROR_TYPE]: [
+          'slot_already_taken',
+          'rollup_contract_check_failed',
+          'slot_mismatch',
+          'block_number_mismatch',
+        ],
       },
     );
 
-    this.slashingAttempts = this.meter.createUpDownCounter(Metrics.SEQUENCER_SLASHING_ATTEMPTS_COUNT, {
-      valueType: ValueType.INT,
-      description: 'The number of slashing action attempts',
-    });
+    this.slashingAttempts = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLASHING_ATTEMPTS_COUNT);
 
     // Fisherman fee analysis metrics
-    this.fishermanWouldBeIncluded = this.meter.createUpDownCounter(Metrics.FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED, {
-      valueType: ValueType.INT,
-      description: 'Whether the transaction would have been included in the block',
-    });
-
-    this.fishermanTimeBeforeBlock = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_TIME_BEFORE_BLOCK, {
-      unit: 'ms',
-      description: 'Time in ms between fee analysis and block being mined',
-      valueType: ValueType.INT,
-    });
-
-    this.fishermanPendingBlobTxCount = this.meter.createHistogram(
-      Metrics.FISHERMAN_FEE_ANALYSIS_PENDING_BLOB_TX_COUNT,
+    this.fishermanWouldBeIncluded = createUpDownCounterWithDefault(
+      this.meter,
+      Metrics.FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED,
       {
-        description: 'Number of blob transactions seen in the pending block',
-        valueType: ValueType.INT,
+        [Attributes.OK]: [true, false],
       },
     );
 
+    this.fishermanTimeBeforeBlock = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_TIME_BEFORE_BLOCK);
+
+    this.fishermanPendingBlobTxCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_PENDING_BLOB_TX_COUNT);
+
     this.fishermanIncludedBlobTxCount = this.meter.createHistogram(
       Metrics.FISHERMAN_FEE_ANALYSIS_INCLUDED_BLOB_TX_COUNT,
-      {
-        description: 'Number of blob transactions that got included in the mined block',
-        valueType: ValueType.INT,
-      },
     );
 
     this.fishermanCalculatedPriorityFee = this.meter.createHistogram(
       Metrics.FISHERMAN_FEE_ANALYSIS_CALCULATED_PRIORITY_FEE,
-      {
-        unit: 'gwei',
-        description: 'Priority fee calculated by each strategy',
-        valueType: ValueType.DOUBLE,
-      },
     );
 
-    this.fishermanPriorityFeeDelta = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_PRIORITY_FEE_DELTA, {
-      unit: 'gwei',
-      description: 'Difference between our priority fee and minimum included priority fee',
-      valueType: ValueType.DOUBLE,
-    });
+    this.fishermanPriorityFeeDelta = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_PRIORITY_FEE_DELTA);
 
-    this.fishermanEstimatedCost = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_ESTIMATED_COST, {
-      unit: 'eth',
-      description: 'Estimated total cost in ETH for the transaction with this strategy',
-      valueType: ValueType.DOUBLE,
-    });
+    this.fishermanEstimatedCost = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_ESTIMATED_COST);
 
     this.fishermanEstimatedOverpayment = this.meter.createHistogram(
       Metrics.FISHERMAN_FEE_ANALYSIS_ESTIMATED_OVERPAYMENT,
-      {
-        unit: 'eth',
-        description: 'Estimated overpayment in ETH vs minimum required for inclusion',
-        valueType: ValueType.DOUBLE,
-      },
     );
 
     this.fishermanMinedBlobTxPriorityFee = this.meter.createHistogram(
       Metrics.FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_PRIORITY_FEE,
-      {
-        unit: 'gwei',
-        description: 'Priority fee per gas for blob transactions in mined blocks',
-        valueType: ValueType.DOUBLE,
-      },
     );
 
     this.fishermanMinedBlobTxTotalCost = this.meter.createHistogram(
       Metrics.FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_TOTAL_COST,
+    );
+
+    this.fishermanPendingBlobCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_PENDING_BLOB_COUNT);
+
+    this.fishermanIncludedBlobCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_INCLUDED_BLOB_COUNT);
+
+    this.fishermanBlockBlobsFull = createUpDownCounterWithDefault(
+      this.meter,
+      Metrics.FISHERMAN_FEE_ANALYSIS_BLOCK_BLOBS_FULL,
       {
-        unit: 'eth',
-        description: 'Total cost in ETH for blob transactions in mined blocks',
-        valueType: ValueType.DOUBLE,
+        [Attributes.OK]: [true, false],
       },
     );
   }
@@ -264,8 +187,8 @@ export class SequencerMetrics {
     this.timeToCollectAttestations.record(0);
   }
 
-  public recordBlockAttestationDelay(duration: number) {
-    this.blockAttestationDelay.record(duration);
+  public recordCheckpointAttestationDelay(duration: number) {
+    this.checkpointAttestationDelay.record(duration);
   }
 
   public recordCollectedAttestations(count: number, durationMs: number) {
@@ -339,7 +262,9 @@ export class SequencerMetrics {
     this.blockProposalSuccess.add(1);
   }
 
-  recordBlockProposalPrecheckFailed(checkType: string) {
+  recordBlockProposalPrecheckFailed(
+    checkType: 'slot_already_taken' | 'rollup_contract_check_failed' | 'slot_mismatch' | 'block_number_mismatch',
+  ) {
     this.blockProposalPrecheckFailed.add(1, {
       [Attributes.ERROR_TYPE]: checkType,
     });
@@ -371,10 +296,12 @@ export class SequencerMetrics {
 
       // Record pending block snapshot data (once per strategy for comparison)
       this.fishermanPendingBlobTxCount.record(analysis.pendingSnapshot.pendingBlobTxCount, strategyAttributes);
+      this.fishermanPendingBlobCount.record(analysis.pendingSnapshot.pendingBlobCount, strategyAttributes);
 
       // Record mined block data if available
       if (analysis.minedBlock) {
         this.fishermanIncludedBlobTxCount.record(analysis.minedBlock.includedBlobTxCount, strategyAttributes);
+        this.fishermanIncludedBlobCount.record(analysis.minedBlock.includedBlobCount, strategyAttributes);
 
         // Record actual fees from blob transactions in the mined block
         for (const blobTx of analysis.minedBlock.includedBlobTxs) {
@@ -408,6 +335,13 @@ export class SequencerMetrics {
       if (analysis.analysis) {
         this.fishermanTimeBeforeBlock.record(Math.ceil(analysis.analysis.timeBeforeBlockMs), strategyAttributes);
 
+        // Record whether the block reached 100% blob capacity
+        if (analysis.analysis.blockBlobsFull) {
+          this.fishermanBlockBlobsFull.add(1, { ...strategyAttributes, [Attributes.OK]: true });
+        } else {
+          this.fishermanBlockBlobsFull.add(1, { ...strategyAttributes, [Attributes.OK]: false });
+        }
+
         // Record strategy-specific inclusion result
         if (strategyResult.wouldBeIncluded !== undefined) {
           if (strategyResult.wouldBeIncluded) {