@aztec/sequencer-client 0.0.1-commit.1bb068fb5 → 0.0.1-commit.217f559981

package/src/sequencer/metrics.ts

@@ -18,7 +18,6 @@ import { type Hex, formatUnits } from 'viem';
 
 import type { SequencerState } from './utils.js';
 
-// TODO(palla/mbps): Review all metrics and add any missing ones per checkpoint
 export class SequencerMetrics {
   public readonly tracer: Tracer;
   private meter: Meter;
@@ -40,11 +39,16 @@ export class SequencerMetrics {
   private filledSlots: UpDownCounter;
 
   private blockProposalFailed: UpDownCounter;
-  private blockProposalSuccess: UpDownCounter;
-  private blockProposalPrecheckFailed: UpDownCounter;
+  private checkpointProposalSuccess: UpDownCounter;
+  private checkpointPrecheckFailed: UpDownCounter;
+  private checkpointProposalFailed: UpDownCounter;
   private checkpointSuccess: UpDownCounter;
   private slashingAttempts: UpDownCounter;
   private checkpointAttestationDelay: Histogram;
+  private checkpointBuildDuration: Histogram;
+  private checkpointBlockCount: Gauge;
+  private checkpointTxCount: Gauge;
+  private checkpointTotalMana: Gauge;
 
   // Fisherman fee analysis metrics
   private fishermanWouldBeIncluded: UpDownCounter;
@@ -54,6 +58,7 @@ export class SequencerMetrics {
   private fishermanPendingBlobCount: Histogram;
   private fishermanIncludedBlobCount: Histogram;
   private fishermanBlockBlobsFull: UpDownCounter;
+  private fishermanMaxBlobCapacity: Histogram;
   private fishermanCalculatedPriorityFee: Histogram;
   private fishermanPriorityFeeDelta: Histogram;
   private fishermanEstimatedCost: Histogram;
@@ -83,7 +88,7 @@ export class SequencerMetrics {
 
     this.checkpointAttestationDelay = this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_ATTESTATION_DELAY);
 
-    this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_REWARDS);
+    this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_SLOT_REWARDS);
 
     this.slots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLOT_COUNT);
 
@@ -106,16 +111,16 @@ export class SequencerMetrics {
       Metrics.SEQUENCER_BLOCK_PROPOSAL_FAILED_COUNT,
     );
 
-    this.blockProposalSuccess = createUpDownCounterWithDefault(
+    this.checkpointProposalSuccess = createUpDownCounterWithDefault(
       this.meter,
-      Metrics.SEQUENCER_BLOCK_PROPOSAL_SUCCESS_COUNT,
+      Metrics.SEQUENCER_CHECKPOINT_PROPOSAL_SUCCESS_COUNT,
     );
 
     this.checkpointSuccess = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_CHECKPOINT_SUCCESS_COUNT);
 
-    this.blockProposalPrecheckFailed = createUpDownCounterWithDefault(
+    this.checkpointPrecheckFailed = createUpDownCounterWithDefault(
       this.meter,
-      Metrics.SEQUENCER_BLOCK_PROPOSAL_PRECHECK_FAILED_COUNT,
+      Metrics.SEQUENCER_CHECKPOINT_PRECHECK_FAILED_COUNT,
       {
         [Attributes.ERROR_TYPE]: [
           'slot_already_taken',
@@ -126,6 +131,16 @@ export class SequencerMetrics {
       },
     );
 
+    this.checkpointProposalFailed = createUpDownCounterWithDefault(
+      this.meter,
+      Metrics.SEQUENCER_CHECKPOINT_PROPOSAL_FAILED_COUNT,
+    );
+
+    this.checkpointBuildDuration = this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_BUILD_DURATION);
+    this.checkpointBlockCount = this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_BLOCK_COUNT);
+    this.checkpointTxCount = this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_TX_COUNT);
+    this.checkpointTotalMana = this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_TOTAL_MANA);
+
     this.slashingAttempts = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLASHING_ATTEMPTS_COUNT);
 
     // Fisherman fee analysis metrics
@@ -134,6 +149,7 @@ export class SequencerMetrics {
       Metrics.FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED,
       {
         [Attributes.OK]: [true, false],
+        [Attributes.BLOCK_FULL]: ['true', 'false'],
       },
     );
 
@@ -176,6 +192,8 @@ export class SequencerMetrics {
         [Attributes.OK]: [true, false],
       },
     );
+
+    this.fishermanMaxBlobCapacity = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_MAX_BLOB_CAPACITY);
   }
 
   public recordRequiredAttestations(requiredAttestationsCount: number, allowanceMs: number) {
@@ -258,18 +276,30 @@ export class SequencerMetrics {
     });
   }
 
-  recordBlockProposalSuccess() {
-    this.blockProposalSuccess.add(1);
+  recordCheckpointProposalSuccess() {
+    this.checkpointProposalSuccess.add(1);
   }
 
-  recordBlockProposalPrecheckFailed(
+  recordCheckpointPrecheckFailed(
     checkType: 'slot_already_taken' | 'rollup_contract_check_failed' | 'slot_mismatch' | 'block_number_mismatch',
   ) {
-    this.blockProposalPrecheckFailed.add(1, {
-      [Attributes.ERROR_TYPE]: checkType,
+    this.checkpointPrecheckFailed.add(1, { [Attributes.ERROR_TYPE]: checkType });
+  }
+
+  recordCheckpointProposalFailed(reason?: string) {
+    this.checkpointProposalFailed.add(1, {
+      ...(reason && { [Attributes.ERROR_TYPE]: reason }),
     });
   }
 
+  /** Records aggregate metrics for a completed checkpoint build. */
+  recordCheckpointBuild(durationMs: number, blockCount: number, txCount: number, totalMana: number) {
+    this.checkpointBuildDuration.record(Math.ceil(durationMs));
+    this.checkpointBlockCount.record(blockCount);
+    this.checkpointTxCount.record(txCount);
+    this.checkpointTotalMana.record(totalMana);
+  }
+
   recordSlashingAttempt(actionCount: number) {
     this.slashingAttempts.add(actionCount);
   }
@@ -342,13 +372,21 @@ export class SequencerMetrics {
           this.fishermanBlockBlobsFull.add(1, { ...strategyAttributes, [Attributes.OK]: false });
         }
 
+        // Record the max blob capacity for this block
+        this.fishermanMaxBlobCapacity.record(analysis.analysis.maxBlobCapacity, strategyAttributes);
+
         // Record strategy-specific inclusion result
         if (strategyResult.wouldBeIncluded !== undefined) {
+          const inclusionAttributes = {
+            ...strategyAttributes,
+            [Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false',
+          };
+
           if (strategyResult.wouldBeIncluded) {
-            this.fishermanWouldBeIncluded.add(1, { ...strategyAttributes, [Attributes.OK]: true });
+            this.fishermanWouldBeIncluded.add(1, { ...inclusionAttributes, [Attributes.OK]: true });
           } else {
             this.fishermanWouldBeIncluded.add(1, {
-              ...strategyAttributes,
+              ...inclusionAttributes,
               [Attributes.OK]: false,
               ...(strategyResult.exclusionReason && { [Attributes.ERROR_TYPE]: strategyResult.exclusionReason }),
             });
@@ -358,17 +396,29 @@ export class SequencerMetrics {
         // Record strategy-specific priority fee delta
         if (strategyResult.priorityFeeDelta !== undefined) {
           const priorityFeeDeltaGwei = Number(strategyResult.priorityFeeDelta) / 1e9;
-          this.fishermanPriorityFeeDelta.record(priorityFeeDeltaGwei, strategyAttributes);
+          const deltaAttributes = {
+            ...strategyAttributes,
+            [Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false',
+          };
+          this.fishermanPriorityFeeDelta.record(priorityFeeDeltaGwei, deltaAttributes);
         }
 
         // Record estimated cost if available
         if (strategyResult.estimatedCostEth !== undefined) {
-          this.fishermanEstimatedCost.record(strategyResult.estimatedCostEth, strategyAttributes);
+          const costAttributes = {
+            ...strategyAttributes,
+            [Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false',
+          };
+          this.fishermanEstimatedCost.record(strategyResult.estimatedCostEth, costAttributes);
         }
 
         // Record estimated overpayment if available
         if (strategyResult.estimatedOverpaymentEth !== undefined) {
-          this.fishermanEstimatedOverpayment.record(strategyResult.estimatedOverpaymentEth, strategyAttributes);
+          const overpaymentAttributes = {
+            ...strategyAttributes,
+            [Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false',
+          };
+          this.fishermanEstimatedOverpayment.record(strategyResult.estimatedOverpaymentEth, overpaymentAttributes);
         }
       }
     }

package/src/sequencer/sequencer.ts

@@ -12,7 +12,7 @@ import type { DateProvider } from '@aztec/foundation/timer';
 import type { TypedEventEmitter } from '@aztec/foundation/types';
 import type { P2P } from '@aztec/p2p';
 import type { SlasherClientInterface } from '@aztec/slasher';
-import type { L2Block, L2BlockSink, L2BlockSource, ValidateCheckpointResult } from '@aztec/stdlib/block';
+import type { BlockData, L2BlockSink, L2BlockSource, ValidateCheckpointResult } from '@aztec/stdlib/block';
 import type { Checkpoint } from '@aztec/stdlib/checkpoint';
 import { getSlotAtTimestamp, getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
 import {
@@ -25,7 +25,7 @@ import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
 import { pickFromSchema } from '@aztec/stdlib/schemas';
 import { MerkleTreeId } from '@aztec/stdlib/trees';
 import { Attributes, type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
-import { FullNodeCheckpointsBuilder, type ValidatorClient } from '@aztec/validator-client';
+import { FullNodeCheckpointsBuilder, NodeKeystoreAdapter, type ValidatorClient } from '@aztec/validator-client';
 
 import EventEmitter from 'node:events';
 
@@ -75,14 +75,6 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
   /** The maximum number of seconds that the sequencer can be into a slot to transition to a particular state. */
   protected timetable!: SequencerTimetable;
 
-  // This shouldn't be here as this gets re-created each time we build/propose a block.
-  // But we have a number of tests that abuse/rely on this class having a permanent publisher.
-  // As long as those tests only configure a single publisher they will continue to work.
-  // This will get re-assigned every time the sequencer goes to build a new block to a publisher that is valid
-  // for the block proposer.
-  // TODO(palla/mbps): Remove this field and fix tests
-  protected publisher: SequencerPublisher | undefined;
-
   /** Config for the sequencer */
   protected config: ResolvedSequencerConfig = DefaultSequencerConfig;
 
@@ -134,10 +126,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     );
   }
 
-  /** Initializes the sequencer (precomputes tables and creates a publisher). Takes about 3s. */
-  public async init() {
+  /** Initializes the sequencer (precomputes tables). Takes about 3s. */
+  public init() {
     getKzg();
-    this.publisher = (await this.publisherFactory.create(undefined)).publisher;
   }
 
   /** Starts the sequencer and moves to IDLE state. */
@@ -156,7 +147,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
   public async stop(): Promise<void> {
     this.log.info(`Stopping sequencer`);
     this.setState(SequencerState.STOPPING, undefined, { force: true });
-    this.publisher?.interrupt();
+    this.publisherFactory.interruptAll();
     await this.runningPromise?.stop();
     this.setState(SequencerState.STOPPED, undefined, { force: true });
     this.log.info('Stopped sequencer');
@@ -169,7 +160,6 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     } catch (err) {
       this.emit('checkpoint-error', { error: err as Error });
       if (err instanceof SequencerTooSlowError) {
-        // TODO(palla/mbps): Add missing states
         // Log as warn only if we had to abort halfway through the block proposal
         const logLvl = [SequencerState.INITIALIZING_CHECKPOINT, SequencerState.PROPOSER_CHECK].includes(
           err.proposedState,
@@ -310,12 +300,12 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     }
 
     // Check that the slot is not taken by a block already (should never happen, since only us can propose for this slot)
-    if (syncedTo.block && syncedTo.block.header.getSlot() >= slot) {
+    if (syncedTo.blockData && syncedTo.blockData.header.getSlot() >= slot) {
       this.log.warn(
         `Cannot propose block at next L2 slot ${slot} since that slot was taken by block ${syncedTo.blockNumber}`,
-        { ...logCtx, block: syncedTo.block.header.toInspect() },
+        { ...logCtx, block: syncedTo.blockData.header.toInspect() },
       );
-      this.metrics.recordBlockProposalPrecheckFailed('slot_already_taken');
+      this.metrics.recordCheckpointPrecheckFailed('slot_already_taken');
       return undefined;
     }
 
@@ -326,7 +316,6 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     const proposerForPublisher = this.config.fishermanMode ? undefined : proposer;
     const { attestorAddress, publisher } = await this.publisherFactory.create(proposerForPublisher);
     this.log.verbose(`Created publisher at address ${publisher.getSenderAddress()} for attestor ${attestorAddress}`);
-    this.publisher = publisher;
 
     // In fisherman mode, set the actual proposer's address for simulations
     if (this.config.fishermanMode && proposer) {
@@ -351,7 +340,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
         logCtx,
       );
       this.emit('proposer-rollup-check-failed', { reason: 'Rollup contract check failed', slot });
-      this.metrics.recordBlockProposalPrecheckFailed('rollup_contract_check_failed');
+      this.metrics.recordCheckpointPrecheckFailed('rollup_contract_check_failed');
       return undefined;
     }
 
@@ -361,7 +350,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
         { ...logCtx, rollup: canProposeCheck, expectedSlot: slot },
       );
       this.emit('proposer-rollup-check-failed', { reason: 'Slot mismatch', slot });
-      this.metrics.recordBlockProposalPrecheckFailed('slot_mismatch');
+      this.metrics.recordCheckpointPrecheckFailed('slot_mismatch');
       return undefined;
     }
 
@@ -371,7 +360,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
         { ...logCtx, rollup: canProposeCheck, expectedSlot: slot },
       );
       this.emit('proposer-rollup-check-failed', { reason: 'Block mismatch', slot });
-      this.metrics.recordBlockProposalPrecheckFailed('block_number_mismatch');
+      this.metrics.recordCheckpointPrecheckFailed('block_number_mismatch');
       return undefined;
     }
 
@@ -433,6 +422,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     );
   }
 
+  /**
+   * Returns the current sequencer state.
+   */
+  public getState(): SequencerState {
+    return this.state;
+  }
+
   /**
    * Internal helper for setting the sequencer state and checks if we have enough time left in the slot to transition to the new state.
    * @param proposedState - The new state to transition to.
@@ -533,18 +529,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
       };
     }
 
-    const block = await this.l2BlockSource.getL2Block(blockNumber);
-    if (!block) {
+    const blockData = await this.l2BlockSource.getBlockData(blockNumber);
+    if (!blockData) {
       // this shouldn't really happen because a moment ago we checked that all components were in sync
-      this.log.error(`Failed to get L2 block ${blockNumber} from the archiver with all components in sync`);
+      this.log.error(`Failed to get L2 block data ${blockNumber} from the archiver with all components in sync`);
       return undefined;
     }
 
     return {
-      block,
-      blockNumber: block.number,
-      checkpointNumber: block.checkpointNumber,
-      archive: block.archive.root,
+      blockData,
+      blockNumber: blockData.header.getBlockNumber(),
+      checkpointNumber: blockData.checkpointNumber,
+      archive: blockData.archive.root,
       l1Timestamp,
       pendingChainValidationStatus,
     };
@@ -867,6 +863,11 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     return this.validatorClient?.getValidatorAddresses();
   }
 
+  /** Updates the publisher factory's node keystore adapter after a keystore reload. */
+  public updatePublisherNodeKeyStore(adapter: NodeKeystoreAdapter): void {
+    this.publisherFactory.updateNodeKeyStore(adapter);
+  }
+
   public getConfig() {
     return this.config;
   }
@@ -877,7 +878,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
 }
 
 type SequencerSyncCheckResult = {
-  block?: L2Block;
+  blockData?: BlockData;
   checkpointNumber: CheckpointNumber;
   blockNumber: BlockNumber;
   archive: Fr;

package/src/test/index.ts

@@ -1,18 +1,16 @@
-import type { L1TxUtilsWithBlobs } from '@aztec/ethereum/l1-tx-utils-with-blobs';
+import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils';
 import type { PublisherManager } from '@aztec/ethereum/publisher-manager';
 import type { PublicProcessorFactory } from '@aztec/simulator/server';
 import type { FullNodeCheckpointsBuilder, ValidatorClient } from '@aztec/validator-client';
 
 import { SequencerClient } from '../client/sequencer-client.js';
 import type { SequencerPublisherFactory } from '../publisher/sequencer-publisher-factory.js';
-import type { SequencerPublisher } from '../publisher/sequencer-publisher.js';
 import { Sequencer } from '../sequencer/sequencer.js';
 import type { SequencerTimetable } from '../sequencer/timetable.js';
 
 class TestSequencer_ extends Sequencer {
   declare public publicProcessorFactory: PublicProcessorFactory;
   declare public timetable: SequencerTimetable;
-  declare public publisher: SequencerPublisher;
   declare public publisherFactory: SequencerPublisherFactory;
   declare public validatorClient: ValidatorClient;
   declare public checkpointsBuilder: FullNodeCheckpointsBuilder;
@@ -22,7 +20,7 @@ export type TestSequencer = TestSequencer_;
 
 class TestSequencerClient_ extends SequencerClient {
   declare public sequencer: TestSequencer;
-  declare public publisherManager: PublisherManager<L1TxUtilsWithBlobs>;
+  declare public publisherManager: PublisherManager<L1TxUtils>;
 }
 
 export type TestSequencerClient = TestSequencerClient_;