@aztec/sequencer-client 0.0.1-commit.9d2bcf6d → 0.0.1-commit.9ebd450e8

package/src/sequencer/checkpoint_proposal_job_metrics.ts

@@ -0,0 +1,128 @@
+import {
+  Attributes,
+  type Gauge,
+  type Histogram,
+  type Meter,
+  Metrics,
+  type TelemetryClient,
+} from '@aztec/telemetry-client';
+
+type CheckpointProposalJobInstruments = {
+  checkpointAttestationDelay: Histogram;
+  checkpointBuildDuration: Histogram;
+  checkpointStartToFirstBlockDuration: Histogram;
+  checkpointLastBlockToBroadcastDuration: Histogram;
+  pipelinedCheckpointBuildStartOffsetFromSlotBoundary: Histogram;
+  checkpointBlockCount: Gauge;
+  checkpointTxCount: Gauge;
+  checkpointTotalMana: Gauge;
+};
+
+/**
+ * Per-job recording surface used by {@link CheckpointProposalJob}.
+ *
+ * Ownership split:
+ * - {@link CheckpointProposalJobMetrics} owns the OpenTelemetry meter and instruments and should live for the
+ *   lifetime of the sequencer process.
+ * - A recorder owns only mutable timing state for one checkpoint proposal job so overlapping jobs cannot
+ *   overwrite each other's timing markers.
+ */
+export interface CheckpointProposalJobMetricsRecorder {
+  recordCheckpointAttestationDelay(durationMs: number): void;
+  recordCheckpointBuild(durationMs: number, blockCount: number, txCount: number, totalMana: number): void;
+  recordPipelinedCheckpointBuildStartOffsetFromSlotBoundary(offsetMs: number): void;
+  startCheckpointTiming(nowMs: number): void;
+  noteCheckpointBlockBuilt(nowMs: number, opts: { isFirstBlock: boolean; isLastBlock: boolean }): void;
+  noteCheckpointBroadcast(nowMs: number): void;
+}
+
+/**
+ * Concrete per-job recorder.
+ *
+ * This class should be short-lived: create one recorder per checkpoint proposal job and discard it when the job
+ * completes. It intentionally does not create instruments; it only holds the job-local timestamps needed to derive
+ * timing metrics safely.
+ */
+class CheckpointProposalJobMetricsRecorderImpl implements CheckpointProposalJobMetricsRecorder {
+  private checkpointStartedAt?: number;
+  private checkpointLastBlockBuiltAt?: number;
+
+  constructor(private readonly instruments: CheckpointProposalJobInstruments) {}
+
+  public recordCheckpointAttestationDelay(durationMs: number) {
+    this.instruments.checkpointAttestationDelay.record(Math.ceil(durationMs));
+  }
+
+  public recordCheckpointBuild(durationMs: number, blockCount: number, txCount: number, totalMana: number) {
+    this.instruments.checkpointBuildDuration.record(Math.ceil(durationMs));
+    this.instruments.checkpointBlockCount.record(blockCount);
+    this.instruments.checkpointTxCount.record(txCount);
+    this.instruments.checkpointTotalMana.record(totalMana);
+  }
+
+  public recordPipelinedCheckpointBuildStartOffsetFromSlotBoundary(offsetMs: number) {
+    this.instruments.pipelinedCheckpointBuildStartOffsetFromSlotBoundary.record(Math.ceil(Math.abs(offsetMs)), {
+      [Attributes.SLOT_BOUNDARY_SIDE]: offsetMs < 0 ? 'before' : 'after',
+    });
+  }
+
+  public startCheckpointTiming(nowMs: number) {
+    this.checkpointStartedAt = nowMs;
+    this.checkpointLastBlockBuiltAt = undefined;
+  }
+
+  public noteCheckpointBlockBuilt(nowMs: number, opts: { isFirstBlock: boolean; isLastBlock: boolean }) {
+    if (opts.isFirstBlock && this.checkpointStartedAt !== undefined) {
+      this.instruments.checkpointStartToFirstBlockDuration.record(Math.ceil(nowMs - this.checkpointStartedAt));
+    }
+
+    if (opts.isLastBlock) {
+      this.checkpointLastBlockBuiltAt = nowMs;
+    }
+  }
+
+  public noteCheckpointBroadcast(nowMs: number) {
+    if (this.checkpointLastBlockBuiltAt !== undefined) {
+      this.instruments.checkpointLastBlockToBroadcastDuration.record(
+        Math.ceil(nowMs - this.checkpointLastBlockBuiltAt),
+      );
+    }
+
+    this.checkpointLastBlockBuiltAt = undefined;
+  }
+}
+
+/**
+ * Long-lived owner of checkpoint proposal job telemetry instruments.
+ *
+ * The sequencer should construct this once and reuse it across jobs. Each call to {@link createRecorder} returns a
+ * lightweight recorder with isolated mutable state for a single checkpoint proposal job.
+ */
+export class CheckpointProposalJobMetrics {
+  private readonly meter: Meter;
+  private readonly instruments: CheckpointProposalJobInstruments;
+
+  constructor(client: TelemetryClient, name = 'CheckpointProposalJob') {
+    this.meter = client.getMeter(name);
+    this.instruments = {
+      checkpointAttestationDelay: this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_ATTESTATION_DELAY),
+      checkpointBuildDuration: this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_BUILD_DURATION),
+      checkpointStartToFirstBlockDuration: this.meter.createHistogram(
+        Metrics.SEQUENCER_CHECKPOINT_START_TO_FIRST_BLOCK_DURATION,
+      ),
+      checkpointLastBlockToBroadcastDuration: this.meter.createHistogram(
+        Metrics.SEQUENCER_CHECKPOINT_LAST_BLOCK_TO_BROADCAST_DURATION,
+      ),
+      pipelinedCheckpointBuildStartOffsetFromSlotBoundary: this.meter.createHistogram(
+        Metrics.SEQUENCER_PIPELINED_CHECKPOINT_BUILD_START_OFFSET_FROM_SLOT_BOUNDARY,
+      ),
+      checkpointBlockCount: this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_BLOCK_COUNT),
+      checkpointTxCount: this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_TX_COUNT),
+      checkpointTotalMana: this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_TOTAL_MANA),
+    };
+  }
+
+  public createRecorder(): CheckpointProposalJobMetricsRecorder {
+    return new CheckpointProposalJobMetricsRecorderImpl(this.instruments);
+  }
+}

package/src/sequencer/checkpoint_voter.ts

@@ -2,7 +2,6 @@ import type { SlotNumber } from '@aztec/foundation/branded-types';
 import type { EthAddress } from '@aztec/foundation/eth-address';
 import type { Logger } from '@aztec/foundation/log';
 import type { SlasherClientInterface } from '@aztec/slasher';
-import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
 import type { ResolvedSequencerConfig } from '@aztec/stdlib/interfaces/server';
 import type { ValidatorClient } from '@aztec/validator-client';
 import { DutyAlreadySignedError } from '@aztec/validator-ha-signer/errors';
@@ -18,7 +17,6 @@ import type { SequencerRollupConstants } from './types.js';
  * Handles governance and slashing voting for a given slot.
  */
 export class CheckpointVoter {
-  private slotTimestamp: bigint;
   private governanceSigner: (msg: TypedDataDefinition) => Promise<`0x${string}`>;
   private slashingSigner: (msg: TypedDataDefinition) => Promise<`0x${string}`>;
 
@@ -33,8 +31,6 @@ export class CheckpointVoter {
     private readonly metrics: SequencerMetrics,
     private readonly log: Logger,
   ) {
-    this.slotTimestamp = getTimestampForSlot(this.slot, this.l1Constants);
-
     // Create separate signers with appropriate duty contexts for governance and slashing votes
     // These use HA protection to ensure only one node signs per slot/duty
     const governanceContext: SigningContext = { slot: this.slot, dutyType: DutyType.GOVERNANCE_VOTE };
@@ -77,7 +73,6 @@ export class CheckpointVoter {
       return await this.publisher.enqueueGovernanceCastSignal(
         governanceProposerPayload,
         this.slot,
-        this.slotTimestamp,
         this.attestorAddress,
         this.governanceSigner,
       );
@@ -108,13 +103,7 @@ export class CheckpointVoter {
 
       this.metrics.recordSlashingAttempt(actions.length);
 
-      return await this.publisher.enqueueSlashingActions(
-        actions,
-        this.slot,
-        this.slotTimestamp,
-        this.attestorAddress,
-        this.slashingSigner,
-      );
+      return await this.publisher.enqueueSlashingActions(actions, this.slot, this.attestorAddress, this.slashingSigner);
     } catch (err) {
       if (err instanceof DutyAlreadySignedError) {
         this.log.info(`Slashing vote already signed by another node`, {

package/src/sequencer/events.ts

@@ -13,7 +13,7 @@ export type SequencerEvents = {
   ['proposer-rollup-check-failed']: (args: { reason: string; slot: SlotNumber }) => void;
   ['block-tx-count-check-failed']: (args: { minTxs: number; availableTxs: number; slot: SlotNumber }) => void;
   ['block-build-failed']: (args: { reason: string; slot: SlotNumber }) => void;
-  ['block-proposed']: (args: { blockNumber: BlockNumber; slot: SlotNumber }) => void;
+  ['block-proposed']: (args: { blockNumber: BlockNumber; slot: SlotNumber; buildSlot: SlotNumber }) => void;
   ['checkpoint-empty']: (args: { slot: SlotNumber }) => void;
   ['checkpoint-publish-failed']: (args: {
     slot: SlotNumber;
@@ -24,4 +24,9 @@ export type SequencerEvents = {
   }) => void;
   ['checkpoint-published']: (args: { checkpoint: CheckpointNumber; slot: SlotNumber }) => void;
   ['checkpoint-error']: (args: { error: Error }) => void;
+  ['pipelined-checkpoint-discarded']: (args: {
+    slot: SlotNumber;
+    checkpointNumber: CheckpointNumber;
+    reason: string;
+  }) => void;
 };

package/src/sequencer/metrics.ts

@@ -18,7 +18,6 @@ import { type Hex, formatUnits } from 'viem';
 
 import type { SequencerState } from './utils.js';
 
-// TODO(palla/mbps): Review all metrics and add any missing ones per checkpoint
 export class SequencerMetrics {
   public readonly tracer: Tracer;
   private meter: Meter;
@@ -40,17 +39,24 @@ export class SequencerMetrics {
   private filledSlots: UpDownCounter;
 
   private blockProposalFailed: UpDownCounter;
-  private blockProposalSuccess: UpDownCounter;
-  private blockProposalPrecheckFailed: UpDownCounter;
+  private checkpointProposalSuccess: UpDownCounter;
+  private checkpointPrecheckFailed: UpDownCounter;
+  private checkpointProposalFailed: UpDownCounter;
   private checkpointSuccess: UpDownCounter;
   private slashingAttempts: UpDownCounter;
-  private checkpointAttestationDelay: Histogram;
+  private pipelineDepth: Gauge;
+  private pipelineDiscards: UpDownCounter;
+  private pipelineParentCheckpointMismatches: UpDownCounter;
 
   // Fisherman fee analysis metrics
   private fishermanWouldBeIncluded: UpDownCounter;
   private fishermanTimeBeforeBlock: Histogram;
   private fishermanPendingBlobTxCount: Histogram;
   private fishermanIncludedBlobTxCount: Histogram;
+  private fishermanPendingBlobCount: Histogram;
+  private fishermanIncludedBlobCount: Histogram;
+  private fishermanBlockBlobsFull: UpDownCounter;
+  private fishermanMaxBlobCapacity: Histogram;
   private fishermanCalculatedPriorityFee: Histogram;
   private fishermanPriorityFeeDelta: Histogram;
   private fishermanEstimatedCost: Histogram;
@@ -58,6 +64,10 @@ export class SequencerMetrics {
   private fishermanMinedBlobTxPriorityFee: Histogram;
   private fishermanMinedBlobTxTotalCost: Histogram;
 
+  private blockInterBlockTime: Histogram;
+  private lastBlockBuiltTimestamp?: number;
+  private lastBlockBuiltSlot?: SlotNumber;
+
   private lastSeenSlot?: SlotNumber;
 
   constructor(
@@ -76,11 +86,11 @@ export class SequencerMetrics {
 
     this.blockBuildManaPerSecond = this.meter.createGauge(Metrics.SEQUENCER_BLOCK_BUILD_MANA_PER_SECOND);
 
-    this.stateTransitionBufferDuration = this.meter.createHistogram(Metrics.SEQUENCER_STATE_TRANSITION_BUFFER_DURATION);
+    this.blockInterBlockTime = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_INTER_BLOCK_TIME);
 
-    this.checkpointAttestationDelay = this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_ATTESTATION_DELAY);
+    this.stateTransitionBufferDuration = this.meter.createHistogram(Metrics.SEQUENCER_STATE_TRANSITION_BUFFER_DURATION);
 
-    this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_REWARDS);
+    this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_SLOT_REWARDS);
 
     this.slots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLOT_COUNT);
 
@@ -103,16 +113,16 @@ export class SequencerMetrics {
       Metrics.SEQUENCER_BLOCK_PROPOSAL_FAILED_COUNT,
     );
 
-    this.blockProposalSuccess = createUpDownCounterWithDefault(
+    this.checkpointProposalSuccess = createUpDownCounterWithDefault(
       this.meter,
-      Metrics.SEQUENCER_BLOCK_PROPOSAL_SUCCESS_COUNT,
+      Metrics.SEQUENCER_CHECKPOINT_PROPOSAL_SUCCESS_COUNT,
     );
 
     this.checkpointSuccess = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_CHECKPOINT_SUCCESS_COUNT);
 
-    this.blockProposalPrecheckFailed = createUpDownCounterWithDefault(
+    this.checkpointPrecheckFailed = createUpDownCounterWithDefault(
       this.meter,
-      Metrics.SEQUENCER_BLOCK_PROPOSAL_PRECHECK_FAILED_COUNT,
+      Metrics.SEQUENCER_CHECKPOINT_PRECHECK_FAILED_COUNT,
       {
         [Attributes.ERROR_TYPE]: [
           'slot_already_taken',
@@ -123,14 +133,37 @@ export class SequencerMetrics {
       },
     );
 
+    this.checkpointProposalFailed = createUpDownCounterWithDefault(
+      this.meter,
+      Metrics.SEQUENCER_CHECKPOINT_PROPOSAL_FAILED_COUNT,
+    );
+
     this.slashingAttempts = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLASHING_ATTEMPTS_COUNT);
 
+    this.pipelineDepth = this.meter.createGauge(Metrics.SEQUENCER_PIPELINE_DEPTH);
+    this.pipelineDiscards = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_PIPELINE_DISCARDS_COUNT);
+    this.pipelineParentCheckpointMismatches = createUpDownCounterWithDefault(
+      this.meter,
+      Metrics.SEQUENCER_PIPELINE_PARENT_CHECKPOINT_MISMATCH_COUNT,
+      {
+        [Attributes.ERROR_TYPE]: [
+          'archiver-sync-timeout',
+          'parent-not-on-l1',
+          'parent-hash-mismatch',
+          'parent-invalid-attestations',
+          'unexpected-parent-appeared',
+        ],
+      },
+    );
+    this.pipelineDepth.record(0);
+
     // Fisherman fee analysis metrics
     this.fishermanWouldBeIncluded = createUpDownCounterWithDefault(
       this.meter,
       Metrics.FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED,
       {
         [Attributes.OK]: [true, false],
+        [Attributes.BLOCK_FULL]: ['true', 'false'],
       },
     );
 
@@ -161,6 +194,20 @@ export class SequencerMetrics {
     this.fishermanMinedBlobTxTotalCost = this.meter.createHistogram(
       Metrics.FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_TOTAL_COST,
     );
+
+    this.fishermanPendingBlobCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_PENDING_BLOB_COUNT);
+
+    this.fishermanIncludedBlobCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_INCLUDED_BLOB_COUNT);
+
+    this.fishermanBlockBlobsFull = createUpDownCounterWithDefault(
+      this.meter,
+      Metrics.FISHERMAN_FEE_ANALYSIS_BLOCK_BLOBS_FULL,
+      {
+        [Attributes.OK]: [true, false],
+      },
+    );
+
+    this.fishermanMaxBlobCapacity = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_MAX_BLOB_CAPACITY);
   }
 
   public recordRequiredAttestations(requiredAttestationsCount: number, allowanceMs: number) {
@@ -172,21 +219,25 @@ export class SequencerMetrics {
     this.timeToCollectAttestations.record(0);
   }
 
-  public recordCheckpointAttestationDelay(duration: number) {
-    this.checkpointAttestationDelay.record(duration);
-  }
-
   public recordCollectedAttestations(count: number, durationMs: number) {
     this.collectedAttestions.record(count);
     this.timeToCollectAttestations.record(Math.ceil(durationMs));
   }
 
-  recordBuiltBlock(buildDurationMs: number, totalMana: number) {
+  recordBuiltBlock(buildDurationMs: number, totalMana: number, slot: SlotNumber) {
     this.blockCounter.add(1, {
       [Attributes.STATUS]: 'built',
     });
     this.blockBuildDuration.record(Math.ceil(buildDurationMs));
     this.blockBuildManaPerSecond.record(Math.ceil((totalMana * 1000) / buildDurationMs));
+
+    // Only record inter-block time between blocks built within the same slot.
+    const now = Date.now();
+    if (this.lastBlockBuiltTimestamp !== undefined && this.lastBlockBuiltSlot === slot) {
+      this.blockInterBlockTime.record(now - this.lastBlockBuiltTimestamp);
+    }
+    this.lastBlockBuiltTimestamp = now;
+    this.lastBlockBuiltSlot = slot;
   }
 
   recordFailedBlock() {
@@ -201,6 +252,20 @@ export class SequencerMetrics {
     });
   }
 
+  recordPipelineDepth(depth: number) {
+    this.pipelineDepth.record(depth);
+  }
+
+  recordPipelineDiscard(count = 1) {
+    this.pipelineDiscards.add(count);
+  }
+
+  recordPipelineParentCheckpointMismatch(reason: string) {
+    this.pipelineParentCheckpointMismatches.add(1, {
+      [Attributes.ERROR_TYPE]: reason,
+    });
+  }
+
   incOpenSlot(slot: SlotNumber, proposer: string) {
     // sequencer went through the loop a second time. Noop
     if (slot === this.lastSeenSlot) {
@@ -243,15 +308,19 @@ export class SequencerMetrics {
     });
   }
 
-  recordBlockProposalSuccess() {
-    this.blockProposalSuccess.add(1);
+  recordCheckpointProposalSuccess() {
+    this.checkpointProposalSuccess.add(1);
   }
 
-  recordBlockProposalPrecheckFailed(
+  recordCheckpointPrecheckFailed(
     checkType: 'slot_already_taken' | 'rollup_contract_check_failed' | 'slot_mismatch' | 'block_number_mismatch',
   ) {
-    this.blockProposalPrecheckFailed.add(1, {
-      [Attributes.ERROR_TYPE]: checkType,
+    this.checkpointPrecheckFailed.add(1, { [Attributes.ERROR_TYPE]: checkType });
+  }
+
+  recordCheckpointProposalFailed(reason?: string) {
+    this.checkpointProposalFailed.add(1, {
+      ...(reason && { [Attributes.ERROR_TYPE]: reason }),
     });
   }
 
@@ -281,10 +350,12 @@ export class SequencerMetrics {
 
       // Record pending block snapshot data (once per strategy for comparison)
       this.fishermanPendingBlobTxCount.record(analysis.pendingSnapshot.pendingBlobTxCount, strategyAttributes);
+      this.fishermanPendingBlobCount.record(analysis.pendingSnapshot.pendingBlobCount, strategyAttributes);
 
       // Record mined block data if available
       if (analysis.minedBlock) {
         this.fishermanIncludedBlobTxCount.record(analysis.minedBlock.includedBlobTxCount, strategyAttributes);
+        this.fishermanIncludedBlobCount.record(analysis.minedBlock.includedBlobCount, strategyAttributes);
 
         // Record actual fees from blob transactions in the mined block
         for (const blobTx of analysis.minedBlock.includedBlobTxs) {
@@ -318,13 +389,28 @@ export class SequencerMetrics {
       if (analysis.analysis) {
         this.fishermanTimeBeforeBlock.record(Math.ceil(analysis.analysis.timeBeforeBlockMs), strategyAttributes);
 
+        // Record whether the block reached 100% blob capacity
+        if (analysis.analysis.blockBlobsFull) {
+          this.fishermanBlockBlobsFull.add(1, { ...strategyAttributes, [Attributes.OK]: true });
+        } else {
+          this.fishermanBlockBlobsFull.add(1, { ...strategyAttributes, [Attributes.OK]: false });
+        }
+
+        // Record the max blob capacity for this block
+        this.fishermanMaxBlobCapacity.record(analysis.analysis.maxBlobCapacity, strategyAttributes);
+
         // Record strategy-specific inclusion result
         if (strategyResult.wouldBeIncluded !== undefined) {
+          const inclusionAttributes = {
+            ...strategyAttributes,
+            [Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false',
+          };
+
           if (strategyResult.wouldBeIncluded) {
-            this.fishermanWouldBeIncluded.add(1, { ...strategyAttributes, [Attributes.OK]: true });
+            this.fishermanWouldBeIncluded.add(1, { ...inclusionAttributes, [Attributes.OK]: true });
           } else {
             this.fishermanWouldBeIncluded.add(1, {
-              ...strategyAttributes,
+              ...inclusionAttributes,
               [Attributes.OK]: false,
               ...(strategyResult.exclusionReason && { [Attributes.ERROR_TYPE]: strategyResult.exclusionReason }),
             });
@@ -334,17 +420,29 @@ export class SequencerMetrics {
         // Record strategy-specific priority fee delta
         if (strategyResult.priorityFeeDelta !== undefined) {
           const priorityFeeDeltaGwei = Number(strategyResult.priorityFeeDelta) / 1e9;
-          this.fishermanPriorityFeeDelta.record(priorityFeeDeltaGwei, strategyAttributes);
+          const deltaAttributes = {
+            ...strategyAttributes,
+            [Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false',
+          };
+          this.fishermanPriorityFeeDelta.record(priorityFeeDeltaGwei, deltaAttributes);
         }
 
         // Record estimated cost if available
         if (strategyResult.estimatedCostEth !== undefined) {
-          this.fishermanEstimatedCost.record(strategyResult.estimatedCostEth, strategyAttributes);
+          const costAttributes = {
+            ...strategyAttributes,
+            [Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false',
+          };
+          this.fishermanEstimatedCost.record(strategyResult.estimatedCostEth, costAttributes);
         }
 
         // Record estimated overpayment if available
         if (strategyResult.estimatedOverpaymentEth !== undefined) {
-          this.fishermanEstimatedOverpayment.record(strategyResult.estimatedOverpaymentEth, strategyAttributes);
+          const overpaymentAttributes = {
+            ...strategyAttributes,
+            [Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false',
+          };
+          this.fishermanEstimatedOverpayment.record(strategyResult.estimatedOverpaymentEth, overpaymentAttributes);
         }
       }
     }