@aztec/sequencer-client 0.0.1-commit.e588bc7e5 → 0.0.1-commit.e5a3663dd

package/src/sequencer/checkpoint_proposal_job_metrics.ts

@@ -0,0 +1,128 @@
+import {
+  Attributes,
+  type Gauge,
+  type Histogram,
+  type Meter,
+  Metrics,
+  type TelemetryClient,
+} from '@aztec/telemetry-client';
+
+type CheckpointProposalJobInstruments = {
+  checkpointAttestationDelay: Histogram;
+  checkpointBuildDuration: Histogram;
+  checkpointStartToFirstBlockDuration: Histogram;
+  checkpointLastBlockToBroadcastDuration: Histogram;
+  pipelinedCheckpointBuildStartOffsetFromSlotBoundary: Histogram;
+  checkpointBlockCount: Gauge;
+  checkpointTxCount: Gauge;
+  checkpointTotalMana: Gauge;
+};
+
+/**
+ * Per-job recording surface used by {@link CheckpointProposalJob}.
+ *
+ * Ownership split:
+ * - {@link CheckpointProposalJobMetrics} owns the OpenTelemetry meter and instruments and should live for the
+ *   lifetime of the sequencer process.
+ * - A recorder owns only mutable timing state for one checkpoint proposal job so overlapping jobs cannot
+ *   overwrite each other's timing markers.
+ */
+export interface CheckpointProposalJobMetricsRecorder {
+  recordCheckpointAttestationDelay(durationMs: number): void;
+  recordCheckpointBuild(durationMs: number, blockCount: number, txCount: number, totalMana: number): void;
+  recordPipelinedCheckpointBuildStartOffsetFromSlotBoundary(offsetMs: number): void;
+  startCheckpointTiming(nowMs: number): void;
+  noteCheckpointBlockBuilt(nowMs: number, opts: { isFirstBlock: boolean; isLastBlock: boolean }): void;
+  noteCheckpointBroadcast(nowMs: number): void;
+}
+
+/**
+ * Concrete per-job recorder.
+ *
+ * This class should be short-lived: create one recorder per checkpoint proposal job and discard it when the job
+ * completes. It intentionally does not create instruments; it only holds the job-local timestamps needed to derive
+ * timing metrics safely.
+ */
+class CheckpointProposalJobMetricsRecorderImpl implements CheckpointProposalJobMetricsRecorder {
+  private checkpointStartedAt?: number;
+  private checkpointLastBlockBuiltAt?: number;
+
+  constructor(private readonly instruments: CheckpointProposalJobInstruments) {}
+
+  public recordCheckpointAttestationDelay(durationMs: number) {
+    this.instruments.checkpointAttestationDelay.record(Math.ceil(durationMs));
+  }
+
+  public recordCheckpointBuild(durationMs: number, blockCount: number, txCount: number, totalMana: number) {
+    this.instruments.checkpointBuildDuration.record(Math.ceil(durationMs));
+    this.instruments.checkpointBlockCount.record(blockCount);
+    this.instruments.checkpointTxCount.record(txCount);
+    this.instruments.checkpointTotalMana.record(totalMana);
+  }
+
+  public recordPipelinedCheckpointBuildStartOffsetFromSlotBoundary(offsetMs: number) {
+    this.instruments.pipelinedCheckpointBuildStartOffsetFromSlotBoundary.record(Math.ceil(Math.abs(offsetMs)), {
+      [Attributes.SLOT_BOUNDARY_SIDE]: offsetMs < 0 ? 'before' : 'after',
+    });
+  }
+
+  public startCheckpointTiming(nowMs: number) {
+    this.checkpointStartedAt = nowMs;
+    this.checkpointLastBlockBuiltAt = undefined;
+  }
+
+  public noteCheckpointBlockBuilt(nowMs: number, opts: { isFirstBlock: boolean; isLastBlock: boolean }) {
+    if (opts.isFirstBlock && this.checkpointStartedAt !== undefined) {
+      this.instruments.checkpointStartToFirstBlockDuration.record(Math.ceil(nowMs - this.checkpointStartedAt));
+    }
+
+    if (opts.isLastBlock) {
+      this.checkpointLastBlockBuiltAt = nowMs;
+    }
+  }
+
+  public noteCheckpointBroadcast(nowMs: number) {
+    if (this.checkpointLastBlockBuiltAt !== undefined) {
+      this.instruments.checkpointLastBlockToBroadcastDuration.record(
+        Math.ceil(nowMs - this.checkpointLastBlockBuiltAt),
+      );
+    }
+
+    this.checkpointLastBlockBuiltAt = undefined;
+  }
+}
+
+/**
+ * Long-lived owner of checkpoint proposal job telemetry instruments.
+ *
+ * The sequencer should construct this once and reuse it across jobs. Each call to {@link createRecorder} returns a
+ * lightweight recorder with isolated mutable state for a single checkpoint proposal job.
+ */
+export class CheckpointProposalJobMetrics {
+  private readonly meter: Meter;
+  private readonly instruments: CheckpointProposalJobInstruments;
+
+  constructor(client: TelemetryClient, name = 'CheckpointProposalJob') {
+    this.meter = client.getMeter(name);
+    this.instruments = {
+      checkpointAttestationDelay: this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_ATTESTATION_DELAY),
+      checkpointBuildDuration: this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_BUILD_DURATION),
+      checkpointStartToFirstBlockDuration: this.meter.createHistogram(
+        Metrics.SEQUENCER_CHECKPOINT_START_TO_FIRST_BLOCK_DURATION,
+      ),
+      checkpointLastBlockToBroadcastDuration: this.meter.createHistogram(
+        Metrics.SEQUENCER_CHECKPOINT_LAST_BLOCK_TO_BROADCAST_DURATION,
+      ),
+      pipelinedCheckpointBuildStartOffsetFromSlotBoundary: this.meter.createHistogram(
+        Metrics.SEQUENCER_PIPELINED_CHECKPOINT_BUILD_START_OFFSET_FROM_SLOT_BOUNDARY,
+      ),
+      checkpointBlockCount: this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_BLOCK_COUNT),
+      checkpointTxCount: this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_TX_COUNT),
+      checkpointTotalMana: this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_TOTAL_MANA),
+    };
+  }
+
+  public createRecorder(): CheckpointProposalJobMetricsRecorder {
+    return new CheckpointProposalJobMetricsRecorderImpl(this.instruments);
+  }
+}

package/src/sequencer/events.ts

@@ -24,4 +24,9 @@ export type SequencerEvents = {
   }) => void;
   ['checkpoint-published']: (args: { checkpoint: CheckpointNumber; slot: SlotNumber }) => void;
   ['checkpoint-error']: (args: { error: Error }) => void;
+  ['pipelined-checkpoint-discarded']: (args: {
+    slot: SlotNumber;
+    checkpointNumber: CheckpointNumber;
+    reason: string;
+  }) => void;
 };

package/src/sequencer/metrics.ts

@@ -26,6 +26,7 @@ export class SequencerMetrics {
   private blockBuildDuration: Histogram;
   private blockBuildManaPerSecond: Gauge;
   private stateTransitionBufferDuration: Histogram;
+  private stateDuration: Histogram;
 
   // these are gauges because for individual sequencers building a block is not something that happens often enough to warrant a histogram
   private timeToCollectAttestations: Gauge;
@@ -44,13 +45,9 @@ export class SequencerMetrics {
   private checkpointProposalFailed: UpDownCounter;
   private checkpointSuccess: UpDownCounter;
   private slashingAttempts: UpDownCounter;
-  private checkpointAttestationDelay: Histogram;
-  private checkpointBuildDuration: Histogram;
-  private checkpointBlockCount: Gauge;
-  private checkpointTxCount: Gauge;
-  private checkpointTotalMana: Gauge;
   private pipelineDepth: Gauge;
   private pipelineDiscards: UpDownCounter;
+  private pipelineParentCheckpointMismatches: UpDownCounter;
 
   // Fisherman fee analysis metrics
   private fishermanWouldBeIncluded: UpDownCounter;
@@ -68,6 +65,10 @@ export class SequencerMetrics {
   private fishermanMinedBlobTxPriorityFee: Histogram;
   private fishermanMinedBlobTxTotalCost: Histogram;
 
+  private blockInterBlockTime: Histogram;
+  private lastBlockBuiltTimestamp?: number;
+  private lastBlockBuiltSlot?: SlotNumber;
+
   private lastSeenSlot?: SlotNumber;
 
   constructor(
@@ -86,9 +87,11 @@ export class SequencerMetrics {
 
     this.blockBuildManaPerSecond = this.meter.createGauge(Metrics.SEQUENCER_BLOCK_BUILD_MANA_PER_SECOND);
 
+    this.blockInterBlockTime = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_INTER_BLOCK_TIME);
+
     this.stateTransitionBufferDuration = this.meter.createHistogram(Metrics.SEQUENCER_STATE_TRANSITION_BUFFER_DURATION);
 
-    this.checkpointAttestationDelay = this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_ATTESTATION_DELAY);
+    this.stateDuration = this.meter.createHistogram(Metrics.SEQUENCER_STATE_DURATION);
 
     this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_SLOT_REWARDS);
 
@@ -138,15 +141,23 @@ export class SequencerMetrics {
       Metrics.SEQUENCER_CHECKPOINT_PROPOSAL_FAILED_COUNT,
     );
 
-    this.checkpointBuildDuration = this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_BUILD_DURATION);
-    this.checkpointBlockCount = this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_BLOCK_COUNT);
-    this.checkpointTxCount = this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_TX_COUNT);
-    this.checkpointTotalMana = this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_TOTAL_MANA);
-
     this.slashingAttempts = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLASHING_ATTEMPTS_COUNT);
 
     this.pipelineDepth = this.meter.createGauge(Metrics.SEQUENCER_PIPELINE_DEPTH);
     this.pipelineDiscards = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_PIPELINE_DISCARDS_COUNT);
+    this.pipelineParentCheckpointMismatches = createUpDownCounterWithDefault(
+      this.meter,
+      Metrics.SEQUENCER_PIPELINE_PARENT_CHECKPOINT_MISMATCH_COUNT,
+      {
+        [Attributes.ERROR_TYPE]: [
+          'archiver-sync-timeout',
+          'parent-not-on-l1',
+          'parent-hash-mismatch',
+          'parent-invalid-attestations',
+          'unexpected-parent-appeared',
+        ],
+      },
+    );
     this.pipelineDepth.record(0);
 
     // Fisherman fee analysis metrics
@@ -211,21 +222,25 @@ export class SequencerMetrics {
     this.timeToCollectAttestations.record(0);
   }
 
-  public recordCheckpointAttestationDelay(duration: number) {
-    this.checkpointAttestationDelay.record(duration);
-  }
-
   public recordCollectedAttestations(count: number, durationMs: number) {
     this.collectedAttestions.record(count);
     this.timeToCollectAttestations.record(Math.ceil(durationMs));
   }
 
-  recordBuiltBlock(buildDurationMs: number, totalMana: number) {
+  recordBuiltBlock(buildDurationMs: number, totalMana: number, slot: SlotNumber) {
     this.blockCounter.add(1, {
       [Attributes.STATUS]: 'built',
     });
     this.blockBuildDuration.record(Math.ceil(buildDurationMs));
     this.blockBuildManaPerSecond.record(Math.ceil((totalMana * 1000) / buildDurationMs));
+
+    // Only record inter-block time between blocks built within the same slot.
+    const now = Date.now();
+    if (this.lastBlockBuiltTimestamp !== undefined && this.lastBlockBuiltSlot === slot) {
+      this.blockInterBlockTime.record(now - this.lastBlockBuiltTimestamp);
+    }
+    this.lastBlockBuiltTimestamp = now;
+    this.lastBlockBuiltSlot = slot;
   }
 
   recordFailedBlock() {
@@ -240,6 +255,12 @@ export class SequencerMetrics {
     });
   }
 
+  recordStateDuration(durationMs: number, state: SequencerState) {
+    this.stateDuration.record(Math.ceil(durationMs), {
+      [Attributes.SEQUENCER_STATE]: state,
+    });
+  }
+
   recordPipelineDepth(depth: number) {
     this.pipelineDepth.record(depth);
   }
@@ -248,6 +269,12 @@ export class SequencerMetrics {
     this.pipelineDiscards.add(count);
   }
 
+  recordPipelineParentCheckpointMismatch(reason: string) {
+    this.pipelineParentCheckpointMismatches.add(1, {
+      [Attributes.ERROR_TYPE]: reason,
+    });
+  }
+
   incOpenSlot(slot: SlotNumber, proposer: string) {
     // sequencer went through the loop a second time. Noop
     if (slot === this.lastSeenSlot) {
@@ -306,14 +333,6 @@ export class SequencerMetrics {
     });
   }
 
-  /** Records aggregate metrics for a completed checkpoint build. */
-  recordCheckpointBuild(durationMs: number, blockCount: number, txCount: number, totalMana: number) {
-    this.checkpointBuildDuration.record(Math.ceil(durationMs));
-    this.checkpointBlockCount.record(blockCount);
-    this.checkpointTxCount.record(txCount);
-    this.checkpointTotalMana.record(totalMana);
-  }
-
   recordSlashingAttempt(actionCount: number) {
     this.slashingAttempts.add(actionCount);
   }

package/src/sequencer/sequencer.ts

@@ -1,12 +1,12 @@
 import { getKzg } from '@aztec/blob-lib';
 import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
 import type { EpochCache } from '@aztec/epoch-cache';
-import { NoCommitteeError, type RollupContract } from '@aztec/ethereum/contracts';
+import { NoCommitteeError, type RollupContract, SimulationOverridesBuilder } from '@aztec/ethereum/contracts';
 import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
 import { merge, omit, pick } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import { EthAddress } from '@aztec/foundation/eth-address';
-import { createLogger } from '@aztec/foundation/log';
+import { type Logger, createLogger } from '@aztec/foundation/log';
 import { RunningPromise } from '@aztec/foundation/running-promise';
 import type { DateProvider } from '@aztec/foundation/timer';
 import type { TypedEventEmitter } from '@aztec/foundation/types';
@@ -14,6 +14,7 @@ import type { P2P } from '@aztec/p2p';
 import type { SlasherClientInterface } from '@aztec/slasher';
 import type { BlockData, L2BlockSink, L2BlockSource, ValidateCheckpointResult } from '@aztec/stdlib/block';
 import type { Checkpoint, ProposedCheckpointData } from '@aztec/stdlib/checkpoint';
+import type { ChainConfig } from '@aztec/stdlib/config';
 import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
 import {
   type ResolvedSequencerConfig,
@@ -22,6 +23,7 @@ import {
   type WorldStateSynchronizer,
 } from '@aztec/stdlib/interfaces/server';
 import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
+import type { CoordinationSignatureContext } from '@aztec/stdlib/p2p';
 import { pickFromSchema } from '@aztec/stdlib/schemas';
 import { MerkleTreeId } from '@aztec/stdlib/trees';
 import { Attributes, type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
@@ -34,6 +36,7 @@ import type { GlobalVariableBuilder } from '../global_variable_builder/global_bu
 import type { SequencerPublisherFactory } from '../publisher/sequencer-publisher-factory.js';
 import type { InvalidateCheckpointRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js';
 import { CheckpointProposalJob } from './checkpoint_proposal_job.js';
+import { CheckpointProposalJobMetrics } from './checkpoint_proposal_job_metrics.js';
 import { CheckpointVoter } from './checkpoint_voter.js';
 import { SequencerInterruptedError, SequencerTooSlowError } from './errors.js';
 import type { SequencerEvents } from './events.js';
@@ -55,7 +58,11 @@ export { SequencerState };
 export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<SequencerEvents>) {
   private runningPromise?: RunningPromise;
   private state = SequencerState.STOPPED;
+  private stateSlotNumber: SlotNumber | undefined;
+  private stateEnteredAtMs = performance.now();
   private metrics: SequencerMetrics;
+  private checkpointProposalJobMetrics: CheckpointProposalJobMetrics;
+  private readonly stateLog: Logger;
 
   /** The last slot for which we attempted to perform our voting duties with degraded block production */
   private lastSlotForFallbackVote: SlotNumber | undefined;
@@ -73,13 +80,14 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
   private lastEpochForStrategyComparison: EpochNumber | undefined;
 
   /** The last checkpoint proposal job, tracked so we can await its pending L1 submission during shutdown. */
-  private lastCheckpointProposalJob: CheckpointProposalJob | undefined;
+  protected lastCheckpointProposalJob: CheckpointProposalJob | undefined;
 
   /** The maximum number of seconds that the sequencer can be into a slot to transition to a particular state. */
   protected timetable!: SequencerTimetable;
 
   /** Config for the sequencer */
   protected config: ResolvedSequencerConfig = DefaultSequencerConfig;
+  private readonly signatureContext: CoordinationSignatureContext;
 
   constructor(
     protected publisherFactory: SequencerPublisherFactory,
@@ -95,18 +103,24 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     protected dateProvider: DateProvider,
     protected epochCache: EpochCache,
     protected rollupContract: RollupContract,
-    config: SequencerConfig,
+    config: SequencerConfig & Pick<ChainConfig, 'l1ChainId' | 'l1Contracts'>,
     protected telemetry: TelemetryClient = getTelemetryClient(),
     protected log = createLogger('sequencer'),
   ) {
     super();
+    this.stateLog = log.createChild('state');
 
     // Add [FISHERMAN] prefix to logger if in fisherman mode
     if (config.fishermanMode) {
       this.log = log.createChild('[FISHERMAN]');
     }
 
+    this.signatureContext = {
+      chainId: config.l1ChainId,
+      rollupAddress: config.l1Contracts.rollupAddress,
+    };
     this.metrics = new SequencerMetrics(telemetry, this.rollupContract, 'Sequencer');
+    this.checkpointProposalJobMetrics = new CheckpointProposalJobMetrics(telemetry);
     this.updateConfig(config);
   }
 
@@ -115,14 +129,16 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     const filteredConfig = pickFromSchema(config, SequencerConfigSchema);
     this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowListExtend'));
     this.config = merge(this.config, filteredConfig);
+    const p2pPropagationTime = this.config.attestationPropagationTime;
     this.timetable = new SequencerTimetable(
       {
         ethereumSlotDuration: this.l1Constants.ethereumSlotDuration,
         aztecSlotDuration: this.aztecSlotDuration,
         l1PublishingTime: this.l1PublishingTime,
-        p2pPropagationTime: this.config.attestationPropagationTime,
+        p2pPropagationTime,
         blockDurationMs: this.config.blockDurationMs,
         enforce: this.config.enforceTimeTable,
+        pipelining: this.epochCache.isProposerPipeliningEnabled(),
       },
       this.metrics,
       this.log,
@@ -146,6 +162,11 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     this.log.info('Started sequencer');
   }
 
+  /** Triggers an immediate run of the sequencer, bypassing the polling interval. */
+  public trigger() {
+    return this.runningPromise?.trigger();
+  }
+
   /** Stops the sequencer from building blocks and moves to STOPPED state. */
   public async stop(): Promise<void> {
     this.log.info(`Stopping sequencer`);
@@ -299,16 +320,6 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     // Next checkpoint follows from the last synced one
     const checkpointNumber = CheckpointNumber(syncedTo.checkpointNumber + 1);
 
-    // Guard: don't exceed 1-deep pipeline. Without a proposed checkpoint, we can only build
-    // confirmed + 1. With a proposed checkpoint, we can build confirmed + 2.
-    const confirmedCkpt = syncedTo.checkpointedCheckpointNumber;
-    if (checkpointNumber > confirmedCkpt + 2) {
-      this.log.warn(
-        `Skipping slot ${targetSlot}: checkpoint ${checkpointNumber} exceeds max pipeline depth (confirmed=${confirmedCkpt})`,
-      );
-      return undefined;
-    }
-
     const logCtx = {
       nowSeconds,
       syncedToL2Slot: syncedTo.syncedL2Slot,
@@ -329,6 +340,16 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
       return undefined;
     }
 
+    // Guard: don't exceed 1-deep pipeline. Without a proposed checkpoint, we can only build
+    // confirmed + 1. With a proposed checkpoint, we can build confirmed + 2.
+    const confirmedCkpt = syncedTo.checkpointedCheckpointNumber;
+    if (checkpointNumber > confirmedCkpt + 2) {
+      this.log.verbose(
+        `Skipping slot ${targetSlot}: checkpoint ${checkpointNumber} exceeds max pipeline depth (confirmed=${confirmedCkpt})`,
+      );
+      return undefined;
+    }
+
     // Check that the target slot is not taken by a block already (should never happen, since only us can propose for this slot)
     if (syncedTo.blockData && syncedTo.blockData.header.getSlot() >= targetSlot) {
       this.log.warn(
@@ -360,18 +381,14 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     // The L1 contract reads archives[proposedCheckpointNumber] and compares it with the provided archive.
     // When invalidating or pipelining, the local archive may differ from L1's, so we adjust accordingly.
     let archiveForCheck = syncedTo.archive;
-    const l1Overrides: {
-      forcePendingCheckpointNumber?: CheckpointNumber;
-      forceArchive?: { checkpointNumber: CheckpointNumber; archive: Fr };
-    } = {};
+    const l1SimulationOverridesBuilder = new SimulationOverridesBuilder();
 
     if (this.epochCache.isProposerPipeliningEnabled() && syncedTo.hasProposedCheckpoint) {
       // Parent checkpoint hasn't landed on L1 yet. Override both the proposed checkpoint number
       // and the archive at that checkpoint so L1 simulation sees the correct chain tip.
       const parentCheckpointNumber = CheckpointNumber(checkpointNumber - 1);
-      l1Overrides.forcePendingCheckpointNumber = parentCheckpointNumber;
-      l1Overrides.forceArchive = { checkpointNumber: parentCheckpointNumber, archive: syncedTo.archive };
-      this.metrics.recordPipelineDepth(1);
+      l1SimulationOverridesBuilder.forPendingCheckpoint(parentCheckpointNumber).withPendingArchive(syncedTo.archive);
+      this.metrics.recordPipelineDepth(syncedTo.checkpointNumber - syncedTo.checkpointedCheckpointNumber);
 
       this.log.verbose(
         `Building on top of proposed checkpoint (pending=${syncedTo.proposedCheckpointData?.checkpointNumber})`,
@@ -382,13 +399,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
       // After invalidation, L1 will roll back to checkpoint N-1. The archive at N-1 already
       // exists on L1, so we just pass the matching archive (the lastArchive of the invalid checkpoint).
       archiveForCheck = invalidateCheckpoint.lastArchive;
-      l1Overrides.forcePendingCheckpointNumber = invalidateCheckpoint.forcePendingCheckpointNumber;
+      l1SimulationOverridesBuilder.forPendingCheckpoint(invalidateCheckpoint.forcePendingCheckpointNumber);
       this.metrics.recordPipelineDepth(0);
     } else {
       this.metrics.recordPipelineDepth(0);
     }
 
-    const canProposeCheck = await publisher.canProposeAt(archiveForCheck, proposer ?? EthAddress.ZERO, l1Overrides);
+    const simulationOverridesPlan = l1SimulationOverridesBuilder.build();
+    const canProposeCheck = await publisher.canProposeAt(
+      archiveForCheck,
+      proposer ?? EthAddress.ZERO,
+      simulationOverridesPlan,
+    );
 
     if (canProposeCheck === undefined) {
       this.log.warn(
@@ -478,12 +500,14 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
       this.checkpointsBuilder,
       this.l2BlockSource,
       this.l1Constants,
+      this.signatureContext,
       this.config,
       this.timetable,
       this.slasherClient,
       this.epochCache,
       this.dateProvider,
       this.metrics,
+      this.checkpointProposalJobMetrics.createRecorder(),
       this,
       this.setState.bind(this),
       this.tracer,
@@ -524,19 +548,35 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
       this.timetable.assertTimeLeft(proposedState, secondsIntoSlot);
     }
 
+    const oldState = this.state;
+    const oldStateSlotNumber = this.stateSlotNumber;
+    const stateChanged = proposedState !== oldState;
+    const transitionAtMs = performance.now();
+    const stateDurationMs = transitionAtMs - this.stateEnteredAtMs;
+
     const boringStates = [SequencerState.IDLE, SequencerState.SYNCHRONIZING];
     const logLevel =
-      boringStates.includes(proposedState) && boringStates.includes(this.state)
-        ? ('trace' as const)
-        : ('debug' as const);
-    this.log[logLevel](`Transitioning from ${this.state} to ${proposedState}`, { slotNumber, secondsIntoSlot });
+      boringStates.includes(proposedState) && boringStates.includes(oldState) ? ('trace' as const) : ('debug' as const);
+    this.stateLog[logLevel](`Transitioning from ${oldState} to ${proposedState}`, {
+      oldState,
+      newState: proposedState,
+      slotNumber,
+      stateSlotNumber: oldStateSlotNumber,
+      secondsIntoSlot,
+      ...(stateChanged && { stateDurationMs: Math.ceil(stateDurationMs) }),
+    });
 
     this.emit('state-changed', {
-      oldState: this.state,
+      oldState,
       newState: proposedState,
       secondsIntoSlot,
       slot: slotNumber,
     });
+    if (stateChanged) {
+      this.metrics.recordStateDuration(stateDurationMs, oldState);
+      this.stateEnteredAtMs = transitionAtMs;
+      this.stateSlotNumber = slotNumber;
+    }
     this.state = proposedState;
   }
 
@@ -567,12 +607,12 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
         .getL2Tips()
         .then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed, proposedCheckpoint: t.proposedCheckpoint })),
       this.p2pClient.getStatus().then(p2p => p2p.syncedToL2Block),
-      this.l1ToL2MessageSource.getL2Tips().then(t => t.proposed),
+      this.l1ToL2MessageSource.getL2Tips().then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed })),
       this.l2BlockSource.getPendingChainValidationStatus(),
-      this.l2BlockSource.getProposedCheckpointOnly(),
+      this.l2BlockSource.getLastProposedCheckpoint(),
     ] as const);
 
-    const [worldState, l2Tips, p2p, l1ToL2MessageSource, pendingChainValidationStatus, proposedCheckpointData] =
+    const [worldState, l2Tips, p2p, l1ToL2MessageSourceTips, pendingChainValidationStatus, proposedCheckpointData] =
       syncedBlocks;
 
     // Handle zero as a special case, since the block hash won't match across services if we're changing the prefilled data for the genesis block,
@@ -580,19 +620,25 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     // TODO(palla/mbps): Fix the above. All components should be able to handle dynamic genesis block hashes.
     const result =
       (l2Tips.proposed.number === 0 &&
+        l2Tips.checkpointed.block.number === 0 &&
+        l2Tips.checkpointed.checkpoint.number === 0 &&
         worldState.number === 0 &&
         p2p.number === 0 &&
-        l1ToL2MessageSource.number === 0) ||
+        l1ToL2MessageSourceTips.proposed.number === 0 &&
+        l1ToL2MessageSourceTips.checkpointed.block.number === 0 &&
+        l1ToL2MessageSourceTips.checkpointed.checkpoint.number === 0) ||
       (worldState.hash === l2Tips.proposed.hash &&
         p2p.hash === l2Tips.proposed.hash &&
-        l1ToL2MessageSource.hash === l2Tips.proposed.hash);
+        l1ToL2MessageSourceTips.proposed.hash === l2Tips.proposed.hash &&
+        l1ToL2MessageSourceTips.checkpointed.block.hash === l2Tips.checkpointed.block.hash &&
+        l1ToL2MessageSourceTips.checkpointed.checkpoint.hash === l2Tips.checkpointed.checkpoint.hash);
 
     if (!result) {
       this.log.debug(`Sequencer sync check failed`, {
         worldState,
         l2BlockSource: l2Tips.proposed,
         p2p,
-        l1ToL2MessageSource,
+        l1ToL2MessageSourceTips,
       });
       return undefined;
     }