@aztec/sequencer-client 0.0.1-commit.b6e433891 → 0.0.1-commit.b9865e97

package/src/sequencer/sequencer.ts

@@ -1,20 +1,26 @@
 import { getKzg } from '@aztec/blob-lib';
-import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
-import type { EpochCache } from '@aztec/epoch-cache';
+import { type EpochCache, PROPOSER_PIPELINING_SLOT_OFFSET } from '@aztec/epoch-cache';
 import { NoCommitteeError, type RollupContract } from '@aztec/ethereum/contracts';
 import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
 import { merge, omit, pick } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import { EthAddress } from '@aztec/foundation/eth-address';
-import { createLogger } from '@aztec/foundation/log';
+import { type Logger, createLogger } from '@aztec/foundation/log';
 import { RunningPromise } from '@aztec/foundation/running-promise';
 import type { DateProvider } from '@aztec/foundation/timer';
 import type { TypedEventEmitter } from '@aztec/foundation/types';
 import type { P2P } from '@aztec/p2p';
 import type { SlasherClientInterface } from '@aztec/slasher';
-import type { BlockData, L2BlockSink, L2BlockSource, ValidateCheckpointResult } from '@aztec/stdlib/block';
-import type { Checkpoint } from '@aztec/stdlib/checkpoint';
-import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
+import type {
+  BlockData,
+  L2BlockSink,
+  L2BlockSource,
+  ProposedCheckpointSink,
+  ValidateCheckpointResult,
+} from '@aztec/stdlib/block';
+import type { Checkpoint, ProposedCheckpointData } from '@aztec/stdlib/checkpoint';
+import type { ChainConfig } from '@aztec/stdlib/config';
+import { getEpochAtSlot, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
 import {
   type ResolvedSequencerConfig,
   type SequencerConfig,
@@ -22,8 +28,9 @@ import {
   type WorldStateSynchronizer,
 } from '@aztec/stdlib/interfaces/server';
 import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
+import type { CoordinationSignatureContext } from '@aztec/stdlib/p2p';
 import { pickFromSchema } from '@aztec/stdlib/schemas';
-import { MerkleTreeId } from '@aztec/stdlib/trees';
+import { MIN_EXECUTION_TIME } from '@aztec/stdlib/timetable';
 import { Attributes, type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
 import { FullNodeCheckpointsBuilder, NodeKeystoreAdapter, type ValidatorClient } from '@aztec/validator-client';
 
@@ -33,7 +40,9 @@ import { DefaultSequencerConfig } from '../config.js';
 import type { GlobalVariableBuilder } from '../global_variable_builder/global_builder.js';
 import type { SequencerPublisherFactory } from '../publisher/sequencer-publisher-factory.js';
 import type { InvalidateCheckpointRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js';
+import { buildCheckpointSimulationOverridesPlan } from './chain_state_overrides.js';
 import { CheckpointProposalJob } from './checkpoint_proposal_job.js';
+import { CheckpointProposalJobMetrics } from './checkpoint_proposal_job_metrics.js';
 import { CheckpointVoter } from './checkpoint_voter.js';
 import { SequencerInterruptedError, SequencerTooSlowError } from './errors.js';
 import type { SequencerEvents } from './events.js';
@@ -44,6 +53,16 @@ import { SequencerState } from './utils.js';
 
 export { SequencerState };
 
+/** Slot snapshot used to prepare a checkpoint proposal. */
+type SequencerSlotContext = {
+  slot: SlotNumber;
+  targetSlot: SlotNumber;
+  epoch: EpochNumber;
+  targetEpoch: EpochNumber;
+  ts: bigint;
+  nowSeconds: bigint;
+};
+
 /**
  * Sequencer client
  * - Checks whether it is elected as proposer for the next slot
@@ -55,7 +74,11 @@ export { SequencerState };
 export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<SequencerEvents>) {
   private runningPromise?: RunningPromise;
   private state = SequencerState.STOPPED;
+  private stateSlotNumber: SlotNumber | undefined;
+  private stateEnteredAtMs = performance.now();
   private metrics: SequencerMetrics;
+  private checkpointProposalJobMetrics: CheckpointProposalJobMetrics;
+  private readonly stateLog: Logger;
 
   /** The last slot for which we attempted to perform our voting duties with degraded block production */
   private lastSlotForFallbackVote: SlotNumber | undefined;
@@ -72,11 +95,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
   /** The last epoch for which we logged strategy comparison in fisherman mode. */
   private lastEpochForStrategyComparison: EpochNumber | undefined;
 
+  /** The last checkpoint proposal job, tracked so we can await its pending L1 submission during shutdown. */
+  protected lastCheckpointProposalJob: CheckpointProposalJob | undefined;
+
   /** The maximum number of seconds that the sequencer can be into a slot to transition to a particular state. */
   protected timetable!: SequencerTimetable;
 
   /** Config for the sequencer */
   protected config: ResolvedSequencerConfig = DefaultSequencerConfig;
+  private readonly signatureContext: CoordinationSignatureContext;
 
   constructor(
     protected publisherFactory: SequencerPublisherFactory,
@@ -85,25 +112,31 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     protected p2pClient: P2P,
     protected worldState: WorldStateSynchronizer,
     protected slasherClient: SlasherClientInterface | undefined,
-    protected l2BlockSource: L2BlockSource & L2BlockSink,
+    protected l2BlockSource: L2BlockSource & L2BlockSink & ProposedCheckpointSink,
     protected l1ToL2MessageSource: L1ToL2MessageSource,
     protected checkpointsBuilder: FullNodeCheckpointsBuilder,
     protected l1Constants: SequencerRollupConstants,
     protected dateProvider: DateProvider,
     protected epochCache: EpochCache,
     protected rollupContract: RollupContract,
-    config: SequencerConfig,
+    config: SequencerConfig & Pick<ChainConfig, 'l1ChainId' | 'rollupAddress'>,
     protected telemetry: TelemetryClient = getTelemetryClient(),
     protected log = createLogger('sequencer'),
   ) {
     super();
+    this.stateLog = log.createChild('state');
 
     // Add [FISHERMAN] prefix to logger if in fisherman mode
     if (config.fishermanMode) {
       this.log = log.createChild('[FISHERMAN]');
     }
 
+    this.signatureContext = {
+      chainId: config.l1ChainId,
+      rollupAddress: config.rollupAddress,
+    };
     this.metrics = new SequencerMetrics(telemetry, this.rollupContract, 'Sequencer');
+    this.checkpointProposalJobMetrics = new CheckpointProposalJobMetrics(telemetry);
     this.updateConfig(config);
   }
 
@@ -112,12 +145,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     const filteredConfig = pickFromSchema(config, SequencerConfigSchema);
     this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowListExtend'));
     this.config = merge(this.config, filteredConfig);
+    const p2pPropagationTime = this.config.attestationPropagationTime;
     this.timetable = new SequencerTimetable(
       {
         ethereumSlotDuration: this.l1Constants.ethereumSlotDuration,
         aztecSlotDuration: this.aztecSlotDuration,
         l1PublishingTime: this.l1PublishingTime,
-        p2pPropagationTime: this.config.attestationPropagationTime,
+        p2pPropagationTime,
         blockDurationMs: this.config.blockDurationMs,
         enforce: this.config.enforceTimeTable,
       },
@@ -143,12 +177,19 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     this.log.info('Started sequencer');
   }
 
+  /** Triggers an immediate run of the sequencer, bypassing the polling interval. */
+  public trigger() {
+    return this.runningPromise?.trigger();
+  }
+
   /** Stops the sequencer from building blocks and moves to STOPPED state. */
   public async stop(): Promise<void> {
     this.log.info(`Stopping sequencer`);
     this.setState(SequencerState.STOPPING, undefined, { force: true });
-    this.publisherFactory.interruptAll();
+    this.lastCheckpointProposalJob?.interrupt();
+    await this.publisherFactory.stopAll();
     await this.runningPromise?.stop();
+    await this.lastCheckpointProposalJob?.awaitPendingSubmission();
     this.setState(SequencerState.STOPPED, undefined, { force: true });
     this.log.info('Stopped sequencer');
   }
@@ -158,17 +199,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     try {
       await this.work();
     } catch (err) {
-      this.emit('checkpoint-error', { error: err as Error });
-      if (err instanceof SequencerTooSlowError) {
-        // Log as warn only if we had to abort halfway through the block proposal
-        const logLvl = [SequencerState.INITIALIZING_CHECKPOINT, SequencerState.PROPOSER_CHECK].includes(
-          err.proposedState,
-        )
-          ? ('debug' as const)
-          : ('warn' as const);
-        this.log[logLvl](err.message, { now: this.dateProvider.nowInSeconds() });
+      if (
+        err instanceof SequencerTooSlowError &&
+        [SequencerState.INITIALIZING_CHECKPOINT, SequencerState.PROPOSER_CHECK].includes(err.proposedState)
+      ) {
+        // No need to alert if we just didn't get to start in time
+        this.log.debug(err.message, { now: this.dateProvider.nowInSeconds() });
       } else {
         // Re-throw other errors
+        this.emit('checkpoint-error', { error: err as Error });
         throw err;
       }
     } finally {
@@ -192,8 +231,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
   @trackSpan('Sequencer.work')
   protected async work() {
     this.setState(SequencerState.SYNCHRONIZING, undefined);
-    const { slot, ts, nowSeconds, epoch } = this.epochCache.getEpochAndSlotInNextL1Slot();
-    const { slot: targetSlot, epoch: targetEpoch } = this.epochCache.getTargetEpochAndSlotInNextL1Slot();
+    const { slot, targetSlot, epoch, targetEpoch, ts, nowSeconds } = this.getSlotContextInNextL1Slot();
 
     // Check if we are synced and it's our slot, grab a publisher, check previous block invalidation, etc
     const checkpointProposalJob = await this.prepareCheckpointProposal(
@@ -208,6 +246,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
       return;
     }
 
+    // Track the job so we can await its pending L1 submission during shutdown
+    this.lastCheckpointProposalJob = checkpointProposalJob;
+
     // Execute the checkpoint proposal job
     const checkpoint = await checkpointProposalJob.execute();
 
@@ -228,13 +269,20 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     return checkpoint;
   }
 
+  /** Returns slot and target slot from a single clock snapshot. */
+  protected getSlotContextInNextL1Slot(): SequencerSlotContext {
+    const { slot, ts, nowSeconds, epoch } = this.epochCache.getEpochAndSlotInNextL1Slot();
+    const targetSlot = SlotNumber(slot + PROPOSER_PIPELINING_SLOT_OFFSET);
+    return { slot, targetSlot, epoch, targetEpoch: getEpochAtSlot(targetSlot, this.l1Constants), ts, nowSeconds };
+  }
+
   /**
    * Prepares the checkpoint proposal by performing all necessary checks and setup.
    * This is the initial step in the main loop.
    * @returns CheckpointProposalJob if successful, undefined if we are not yet synced or are not the proposer.
    */
   @trackSpan('Sequencer.prepareCheckpointProposal')
-  private async prepareCheckpointProposal(
+  protected async prepareCheckpointProposal(
     slot: SlotNumber,
     targetSlot: SlotNumber,
     epoch: EpochNumber,
@@ -262,6 +310,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
       return undefined;
     }
 
+    // Test-only: skip proposing for explicitly paused slots. Attestation paths run in the validator
+    // client and are not gated by this hook, so paused proposers still attest to others' proposals.
+    if (this.config.pauseProposingForSlots?.some(s => s === targetSlot)) {
+      this.log.warn(`Skipping proposal for paused slot ${targetSlot} (test-only pauseProposingForSlots hook)`, {
+        targetSlot,
+      });
+      return undefined;
+    }
+
     // Check all components are synced to latest as seen by the archiver (queries all subsystems)
     const syncedTo = await this.checkSync({ ts, slot });
     if (!syncedTo) {
@@ -278,7 +335,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
       this.setState(SequencerState.PROPOSER_CHECK, slot);
       const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
       if (canPropose) {
-        await this.tryVoteWhenEscapeHatchOpen({ slot, proposer });
+        await this.tryVoteWhenEscapeHatchOpen({ slot, targetSlot, proposer });
       } else {
         this.log.trace(`Escape hatch open but we are not proposer, skipping vote-only actions`, {
           slot,
@@ -312,6 +369,16 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
       return undefined;
     }
 
+    // Guard: don't exceed 1-deep pipeline. Without a proposed checkpoint, we can only build
+    // confirmed + 1. With a proposed checkpoint, we can build confirmed + 2.
+    const confirmedCkpt = syncedTo.checkpointedCheckpointNumber;
+    if (checkpointNumber > confirmedCkpt + 2) {
+      this.log.verbose(
+        `Skipping slot ${targetSlot}: checkpoint ${checkpointNumber} exceeds max pipeline depth (confirmed=${confirmedCkpt})`,
+      );
+      return undefined;
+    }
+
     // Check that the target slot is not taken by a block already (should never happen, since only us can propose for this slot)
     if (syncedTo.blockData && syncedTo.blockData.header.getSlot() >= targetSlot) {
       this.log.warn(
@@ -330,25 +397,96 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     const { attestorAddress, publisher } = await this.publisherFactory.create(proposerForPublisher);
     this.log.verbose(`Created publisher at address ${publisher.getSenderAddress()} for attestor ${attestorAddress}`);
 
-    // In fisherman mode, set the actual proposer's address for simulations
-    if (this.config.fishermanMode && proposer) {
-      publisher.setProposerAddressForSimulation(proposer);
-      this.log.debug(`Set proposer address ${proposer} for simulation in fisherman mode`);
+    // Prepare invalidation request if the pending chain is invalid (returns undefined if no need).
+    // Only simulate invalidation when there's no proposed parent, since we assume the proposed parent
+    // will invalidate the currently invalid checkpoint on L1.
+    const invalidateCheckpoint =
+      syncedTo.hasProposedCheckpoint || syncedTo.pendingChainValidationStatus.valid
+        ? undefined
+        : await publisher.simulateInvalidateCheckpoint(syncedTo.pendingChainValidationStatus);
+
+    // Determine the correct archive and L1 state overrides for the canProposeAt check.
+    // The L1 contract reads archives[proposedCheckpointNumber] and compares it with the provided archive.
+    // When invalidating or pipelining, the local archive may differ from L1's, so we adjust accordingly.
+    let archiveForCheck = syncedTo.archive;
+
+    if (syncedTo.hasProposedCheckpoint) {
+      this.metrics.recordPipelineDepth(syncedTo.checkpointNumber - syncedTo.checkpointedCheckpointNumber);
+      this.log.verbose(
+        `Building on top of proposed checkpoint (pending=${syncedTo.proposedCheckpointData?.checkpointNumber}) for target slot ${targetSlot}`,
+        { targetSlot, parentCheckpointNumber: CheckpointNumber(checkpointNumber - 1) },
+      );
+      // Match what L1 will see at archives[pending] once the proposed parent lands: the parent's
+      // own archive root from the gossiped proposal. `syncedTo.archive` is the world-state-local
+      // view and can transiently diverge from the proposed parent (e.g. before the proposed
+      // parent's blocks have been applied locally); diverging here would cause the canProposeAt
+      // override to set archives[pending] to one value while we present another for comparison.
+      archiveForCheck = syncedTo.proposedCheckpointData!.archive.root;
+    } else if (invalidateCheckpoint) {
+      // After invalidation, L1 will roll back to checkpoint N-1. The archive at N-1 already
+      // exists on L1, so we just pass the matching archive (the lastArchive of the invalid checkpoint).
+      archiveForCheck = invalidateCheckpoint.lastArchive;
+      this.metrics.recordPipelineDepth(0);
+    } else {
+      this.metrics.recordPipelineDepth(0);
     }
 
-    // Prepare invalidation request if the pending chain is invalid (returns undefined if no need)
-    const invalidateCheckpoint = await publisher.simulateInvalidateCheckpoint(syncedTo.pendingChainValidationStatus);
+    // Build the simulation plan: pending/proven override from pipelining or invalidation (or the
+    // current snapshot when neither applies, to short-circuit any pending prune in simulation),
+    // plus the parent checkpoint cell and fee header when pipelining.
+    const simulationOverridesPlan = await buildCheckpointSimulationOverridesPlan({
+      checkpointNumber,
+      proposedCheckpointData: syncedTo.hasProposedCheckpoint ? syncedTo.proposedCheckpointData : undefined,
+      invalidateToPendingCheckpointNumber: invalidateCheckpoint?.forcePendingCheckpointNumber,
+      checkpointedCheckpointNumber: syncedTo.checkpointedCheckpointNumber,
+      rollup: this.rollupContract,
+      signatureContext: this.signatureContext,
+      log: this.log,
+    });
+
+    // The plan always pins both pending/proven (to short-circuit `canPruneAtTime` in simulation),
+    // so `provenOverride` always reflects the assumed proven checkpoint we are pinning the
+    // simulation to. We additionally warn when the pin is load-bearing — i.e. when a prune would
+    // actually fire at the target slot without it — so observers can spot "we are building
+    // optimistically across a pruning boundary" in the logs.
+    const provenOverride = simulationOverridesPlan?.chainTipsOverride?.proven;
+    if (provenOverride !== undefined && (await this.l2BlockSource.isPruneDueAtSlot(targetSlot))) {
+      this.log.warn(
+        `Assuming proof for epoch ending at checkpoint ${provenOverride} lands by target slot ${targetSlot}`,
+        { checkpointNumber, slot, targetSlot, provenOverride },
+      );
+    }
 
-    // Check with the rollup contract if we can indeed propose at the target slot. This check should not fail
-    // if all the previous checks are good, but we do it just in case.
-    const canProposeCheck = await publisher.canProposeAt(syncedTo.archive, proposer ?? EthAddress.ZERO, {
-      ...invalidateCheckpoint,
+    this.emit('preparing-checkpoint', {
+      targetSlot,
+      checkpointNumber,
+      hadProposedParent: syncedTo.hasProposedCheckpoint,
+      provenOverride,
+      simulatedPending: simulationOverridesPlan?.chainTipsOverride?.pending,
     });
 
+    const canProposeCheck = await publisher.canProposeAt(
+      archiveForCheck,
+      proposer ?? EthAddress.ZERO,
+      simulationOverridesPlan,
+    );
+
+    const proposeContext = {
+      hasProposedCheckpoint: syncedTo.hasProposedCheckpoint,
+      proposedCheckpointNumber: syncedTo.proposedCheckpointData?.checkpointNumber,
+      checkpointedCheckpointNumber: syncedTo.checkpointedCheckpointNumber,
+      isInvalidating: !!invalidateCheckpoint,
+      invalidatingCheckpointNumber: invalidateCheckpoint?.checkpointNumber,
+      archiveForCheck: archiveForCheck.toString(),
+      overridePendingCheckpointNumber: simulationOverridesPlan?.chainTipsOverride?.pending,
+      overrideArchive: simulationOverridesPlan?.pendingCheckpointState?.archive,
+      overrideFeeHeader: simulationOverridesPlan?.pendingCheckpointState?.feeHeader,
+    };
+
     if (canProposeCheck === undefined) {
       this.log.warn(
         `Cannot propose checkpoint ${checkpointNumber} at slot ${slot} due to failed rollup contract check`,
-        logCtx,
+        { ...logCtx, ...proposeContext },
       );
       this.emit('proposer-rollup-check-failed', { reason: 'Rollup contract check failed', slot });
       this.metrics.recordCheckpointPrecheckFailed('rollup_contract_check_failed');
@@ -358,7 +496,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     if (canProposeCheck.slot !== targetSlot) {
       this.log.warn(
         `Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${targetSlot} but got ${canProposeCheck.slot}.`,
-        { ...logCtx, rollup: canProposeCheck, expectedSlot: targetSlot },
+        { ...logCtx, ...proposeContext, rollup: canProposeCheck, expectedSlot: targetSlot },
       );
       this.emit('proposer-rollup-check-failed', { reason: 'Slot mismatch', slot });
       this.metrics.recordCheckpointPrecheckFailed('slot_mismatch');
@@ -368,7 +506,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     if (canProposeCheck.checkpointNumber !== checkpointNumber) {
       this.log.warn(
         `Cannot propose due to block mismatch with rollup contract (this can be caused by a pending archiver sync). Expected checkpoint ${checkpointNumber} but got ${canProposeCheck.checkpointNumber}.`,
-        { ...logCtx, rollup: canProposeCheck, expectedSlot: slot },
+        { ...logCtx, ...proposeContext, rollup: canProposeCheck, expectedSlot: slot },
       );
       this.emit('proposer-rollup-check-failed', { reason: 'Block mismatch', slot });
       this.metrics.recordCheckpointPrecheckFailed('block_number_mismatch');
@@ -382,8 +520,8 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
       `Preparing checkpoint proposal ${checkpointNumber} for target slot ${targetSlot} during wall-clock slot ${slot}`,
       {
         ...logCtx,
+        ...proposeContext,
         proposer,
-        pipeliningEnabled: this.epochCache.isProposerPipeliningEnabled(),
       },
     );
 
@@ -391,36 +529,38 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     return this.createCheckpointProposalJob(
       slot,
       targetSlot,
-      epoch,
       targetEpoch,
       checkpointNumber,
       syncedTo.blockNumber,
+      syncedTo.checkpointedCheckpointNumber,
       proposer,
       publisher,
       attestorAddress,
       invalidateCheckpoint,
+      syncedTo.proposedCheckpointData,
     );
   }
 
   protected createCheckpointProposalJob(
     slot: SlotNumber,
     targetSlot: SlotNumber,
-    epoch: EpochNumber,
     targetEpoch: EpochNumber,
     checkpointNumber: CheckpointNumber,
     syncedToBlockNumber: BlockNumber,
+    checkpointedCheckpointNumber: CheckpointNumber,
     proposer: EthAddress | undefined,
     publisher: SequencerPublisher,
     attestorAddress: EthAddress,
     invalidateCheckpoint: InvalidateCheckpointRequest | undefined,
+    proposedCheckpointData?: ProposedCheckpointData,
   ): CheckpointProposalJob {
     return new CheckpointProposalJob(
       slot,
       targetSlot,
-      epoch,
       targetEpoch,
       checkpointNumber,
       syncedToBlockNumber,
+      checkpointedCheckpointNumber,
       proposer,
       publisher,
       attestorAddress,
@@ -434,16 +574,19 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
       this.checkpointsBuilder,
       this.l2BlockSource,
       this.l1Constants,
+      this.signatureContext,
       this.config,
       this.timetable,
       this.slasherClient,
       this.epochCache,
       this.dateProvider,
       this.metrics,
+      this.checkpointProposalJobMetrics.createRecorder(),
       this,
-      this.setState.bind(this),
+      (state, slot, timeReferenceSlot) => this.setState(state, slot, { timeReferenceSlot }),
       this.tracer,
       this.log.getBindings(),
+      proposedCheckpointData,
     );
   }
 
@@ -457,13 +600,14 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
   /**
    * Internal helper for setting the sequencer state and checks if we have enough time left in the slot to transition to the new state.
    * @param proposedState - The new state to transition to.
-   * @param slotNumber - The current slot number.
+   * @param slotNumber - The logical slot associated with the new state.
    * @param force - Whether to force the transition even if the sequencer is stopped.
+   * @param timeReferenceSlot - The slot to use for deadline checks when it differs from the logical slot.
    */
   protected setState(
     proposedState: SequencerState,
     slotNumber: SlotNumber | undefined,
-    opts: { force?: boolean } = {},
+    opts: { force?: boolean; timeReferenceSlot?: SlotNumber } = {},
   ): void {
     if (this.state === SequencerState.STOPPING && proposedState !== SequencerState.STOPPED && !opts.force) {
       this.log.warn(`Cannot set sequencer to ${proposedState} as it is stopping.`);
@@ -474,24 +618,43 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
       return;
     }
     let secondsIntoSlot = undefined;
-    if (slotNumber !== undefined) {
-      secondsIntoSlot = this.getSecondsIntoSlot(slotNumber);
+    const timeReferenceSlot = opts.timeReferenceSlot ?? slotNumber;
+    if (timeReferenceSlot !== undefined) {
+      secondsIntoSlot = this.getSecondsIntoSlot(timeReferenceSlot);
       this.timetable.assertTimeLeft(proposedState, secondsIntoSlot);
     }
 
+    const oldState = this.state;
+    const oldStateSlotNumber = this.stateSlotNumber;
+    const stateChanged = proposedState !== oldState;
+    const transitionAtMs = performance.now();
+    const stateDurationMs = transitionAtMs - this.stateEnteredAtMs;
+
     const boringStates = [SequencerState.IDLE, SequencerState.SYNCHRONIZING];
     const logLevel =
-      boringStates.includes(proposedState) && boringStates.includes(this.state)
-        ? ('trace' as const)
-        : ('debug' as const);
-    this.log[logLevel](`Transitioning from ${this.state} to ${proposedState}`, { slotNumber, secondsIntoSlot });
+      boringStates.includes(proposedState) && boringStates.includes(oldState) ? ('trace' as const) : ('debug' as const);
+    this.stateLog[logLevel](`Transitioning from ${oldState} to ${proposedState}`, {
+      oldState,
+      newState: proposedState,
+      slotNumber,
+      timeReferenceSlot,
+      stateSlotNumber: oldStateSlotNumber,
+      secondsIntoSlot,
+      ...(stateChanged && { stateDurationMs: Math.ceil(stateDurationMs) }),
+    });
 
     this.emit('state-changed', {
-      oldState: this.state,
+      oldState,
       newState: proposedState,
       secondsIntoSlot,
       slot: slotNumber,
+      timeReferenceSlot,
     });
+    if (stateChanged) {
+      this.metrics.recordStateDuration(stateDurationMs, oldState);
+      this.stateEnteredAtMs = transitionAtMs;
+      this.stateSlotNumber = slotNumber;
+    }
     this.state = proposedState;
   }
 
@@ -501,8 +664,8 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
    */
   protected async checkSync(args: { ts: bigint; slot: SlotNumber }): Promise<SequencerSyncCheckResult | undefined> {
     // Check that the archiver has fully synced the L2 slot before the one we want to propose in.
-    // TODO(#14766): Archiver reports L1 timestamp based on L1 blocks seen, which means that a missed L1 block will
-    // cause the archiver L1 timestamp to fall behind, and cause this sequencer to start processing one L1 slot later.
+    // The archiver reports sync progress via L1 block timestamps and synced checkpoint slots.
+    // See getSyncedL2SlotNumber for how missed L1 blocks are handled.
     const syncedL2Slot = await this.l2BlockSource.getSyncedL2SlotNumber();
     const { slot } = args;
     if (syncedL2Slot === undefined || syncedL2Slot + 1 < slot) {
@@ -518,45 +681,113 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
         number: syncSummary.latestBlockNumber,
         hash: syncSummary.latestBlockHash,
       })),
-      this.l2BlockSource.getL2Tips().then(t => t.proposed),
+      this.l2BlockSource
+        .getL2Tips()
+        .then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed, proposedCheckpoint: t.proposedCheckpoint })),
       this.p2pClient.getStatus().then(p2p => p2p.syncedToL2Block),
-      this.l1ToL2MessageSource.getL2Tips().then(t => t.proposed),
+      this.l1ToL2MessageSource.getL2Tips().then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed })),
       this.l2BlockSource.getPendingChainValidationStatus(),
+      this.l2BlockSource.getProposedCheckpointData(),
     ] as const);
 
-    const [worldState, l2BlockSource, p2p, l1ToL2MessageSource, pendingChainValidationStatus] = syncedBlocks;
+    const [worldState, l2Tips, p2p, l1ToL2MessageSourceTips, pendingChainValidationStatus, proposedCheckpointData] =
+      syncedBlocks;
 
-    // Handle zero as a special case, since the block hash won't match across services if we're changing the prefilled data for the genesis block,
-    // as the world state can compute the new genesis block hash, but other components use the hardcoded constant.
-    // TODO(palla/mbps): Fix the above. All components should be able to handle dynamic genesis block hashes.
     const result =
-      (l2BlockSource.number === 0 && worldState.number === 0 && p2p.number === 0 && l1ToL2MessageSource.number === 0) ||
-      (worldState.hash === l2BlockSource.hash &&
-        p2p.hash === l2BlockSource.hash &&
-        l1ToL2MessageSource.hash === l2BlockSource.hash);
+      worldState.hash === l2Tips.proposed.hash &&
+      p2p.hash === l2Tips.proposed.hash &&
+      l1ToL2MessageSourceTips.proposed.hash === l2Tips.proposed.hash &&
+      l1ToL2MessageSourceTips.checkpointed.block.hash === l2Tips.checkpointed.block.hash &&
+      l1ToL2MessageSourceTips.checkpointed.checkpoint.hash === l2Tips.checkpointed.checkpoint.hash;
 
     if (!result) {
-      this.log.debug(`Sequencer sync check failed`, { worldState, l2BlockSource, p2p, l1ToL2MessageSource });
+      this.log.debug(`Sequencer sync check failed`, {
+        worldState,
+        l2BlockSource: l2Tips.proposed,
+        p2p,
+        l1ToL2MessageSourceTips,
+      });
       return undefined;
     }
 
-    // Special case for genesis state
     const blockNumber = worldState.number;
-    if (blockNumber < INITIAL_L2_BLOCK_NUM) {
-      const archive = new Fr((await this.worldState.getCommitted().getTreeInfo(MerkleTreeId.ARCHIVE)).root);
-      return {
-        checkpointNumber: CheckpointNumber.ZERO,
-        blockNumber: BlockNumber.ZERO,
-        archive,
+    const blockData = await this.l2BlockSource.getBlockData({ number: blockNumber });
+    if (!blockData) {
+      this.log.warn(`Sequencer sync check failed: failed to get L2 block data ${blockNumber} from the archiver`, {
+        blockNumber,
+        l2Tips,
+        syncedL2Slot,
+        ...args,
+      });
+      return undefined;
+    }
+
+    // Refuse to build a checkpoint on top of a proposed block whose enclosing checkpoint was never
+    // proposed. Under pipelining we may have received and reexecuted such a block locally — advancing
+    // our world-state tip past the checkpointed tip — while the proposing node never published the
+    // matching proposed checkpoint (e.g. it crashed before assembling it). Building on this orphan block
+    // would fork the chain off a tip no other node can follow. The archiver prunes these orphan blocks
+    // once their build slot ends; this guard is the correctness barrier during the grace window before.
+    if (
+      blockData.checkpointNumber > l2Tips.checkpointed.checkpoint.number &&
+      (l2Tips.proposedCheckpoint.checkpoint.number !== blockData.checkpointNumber ||
+        proposedCheckpointData?.checkpointNumber !== blockData.checkpointNumber)
+    ) {
+      const logCtx = {
+        blockCheckpointNumber: blockData.checkpointNumber,
+        checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
+        proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number,
+        proposedCheckpointDataNumber: proposedCheckpointData?.checkpointNumber,
+        blockNumber: blockData.header.getBlockNumber(),
+        blockSlot: blockData.header.getSlot(),
         syncedL2Slot,
-        pendingChainValidationStatus,
+        ...args,
       };
+
+      // Under pipelining the block proposal for a checkpoint leads its checkpoint proposal by up to one
+      // slot, so a world-state tip sitting in an as-yet-unproposed checkpoint is the expected steady state
+      // until that checkpoint is due. Only treat it as abnormal — and warn — once the checkpoint is overdue
+      // by the same deadline the archiver uses to prune the orphan block (see pruneOrphanProposedBlocks).
+      // Before then this is normal pipelining and we wait it out quietly.
+      if (this.isProposedCheckpointOverdue(blockData.header.getSlot())) {
+        this.log.warn(`Sequencer sync check failed: proposed block has no matching proposed checkpoint`, logCtx);
+      } else {
+        this.log.debug(`Waiting for proposed checkpoint to catch up with reexecuted block`, logCtx);
+      }
+      return undefined;
     }
 
-    const blockData = await this.l2BlockSource.getBlockData(blockNumber);
-    if (!blockData) {
-      // this shouldn't really happen because a moment ago we checked that all components were in sync
-      this.log.error(`Failed to get L2 block data ${blockNumber} from the archiver with all components in sync`);
+    const hasProposedCheckpoint = l2Tips.proposedCheckpoint.checkpoint.number > l2Tips.checkpointed.checkpoint.number;
+
+    // The l2Tips and proposedCheckpointData reads above come from independent archiver snapshots
+    // (a JS-side tips cache vs. a direct store read on `#proposedCheckpoints`). A concurrent archiver
+    // write that mutates both can be observed split, leaving us with `hasProposedCheckpoint=true` but
+    // no proposedCheckpointData (or one whose number doesn't match the tip). Refuse to proceed in that
+    // window — the next checkSync tick will see a coherent snapshot.
+    if (
+      hasProposedCheckpoint &&
+      (!proposedCheckpointData ||
+        proposedCheckpointData.checkpointNumber !== l2Tips.proposedCheckpoint.checkpoint.number)
+    ) {
+      this.log.warn(`Sequencer sync check failed: inconsistent proposed-checkpoint state`, {
+        proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number,
+        checkpointedTipNumber: l2Tips.checkpointed.checkpoint.number,
+        proposedCheckpointDataNumber: proposedCheckpointData?.checkpointNumber,
+        syncedL2Slot,
+        ...args,
+      });
+      return undefined;
+    }
+
+    // Check that the proposed checkpoint is indeed the parent of the checkpoint we'll be building
+    // The checkpoint number to build is derived as blockData.checkpointNumber + 1
+    if (proposedCheckpointData && proposedCheckpointData.checkpointNumber !== blockData.checkpointNumber) {
+      this.log.warn(`Sequencer sync check failed: proposed checkpoint number mismatch`, {
+        proposedCheckpointNumber: proposedCheckpointData.checkpointNumber,
+        blockCheckpointNumber: blockData.checkpointNumber,
+        syncedL2Slot,
+        ...args,
+      });
       return undefined;
     }
 
@@ -564,12 +795,30 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
       blockData,
       blockNumber: blockData.header.getBlockNumber(),
       checkpointNumber: blockData.checkpointNumber,
+      checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
       archive: blockData.archive.root,
+      hasProposedCheckpoint,
+      proposedCheckpointData,
       syncedL2Slot,
       pendingChainValidationStatus,
     };
   }
 
+  /**
+   * Whether the enclosing checkpoint of a reexecuted block is overdue: past the deadline by which a
+   * well-behaved proposer should have published it. Mirrors the archiver's orphan-prune deadline (the
+   * start of the slot after the block's build slot, plus a grace period) so the sequencer only warns
+   * about a missing proposed checkpoint once the archiver itself would prune the orphan block. The grace
+   * is derived from the block build duration the same way the archiver defaults it at node wiring.
+   */
+  private isProposedCheckpointOverdue(blockSlot: SlotNumber): boolean {
+    const expectedBySlot = SlotNumber(Number(blockSlot) - PROPOSER_PIPELINING_SLOT_OFFSET + 1);
+    const graceSeconds =
+      this.config.blockDurationMs !== undefined ? Math.ceil(this.config.blockDurationMs / 1000) : MIN_EXECUTION_TIME;
+    const expectedByTime = getTimestampForSlot(expectedBySlot, this.l1Constants) + BigInt(graceSeconds);
+    return BigInt(this.dateProvider.nowInSeconds()) >= expectedByTime;
+  }
+
   /**
    * Checks if we are the proposer for the next slot.
    * @returns True if we can propose, and the proposer address (undefined if anyone can propose)
@@ -612,7 +861,10 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
       return [false, proposer];
     }
 
-    this.log.debug(`We are the proposer for target slot ${targetSlot}`, { targetSlot, proposer });
+    this.log.info(`We are the proposer for pipeline slot ${targetSlot}`, {
+      targetSlot,
+      proposer,
+    });
     return [true, proposer];
   }
 
@@ -688,7 +940,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     }
 
     this.log.info(`Voting in slot ${slot} despite sync failure`, { slot });
-    await publisher.sendRequests();
+    // Votes are EIP-712-signed for `targetSlot` (the pipelined slot in which the multicall is
+    // expected to mine). Delay submission to the start of `targetSlot` so the tx mines in the
+    // slot the votes were signed for. We fire-and-forget so we don't block the sequencer's
+    // work loop while waiting for the target slot to start.
+    void publisher.sendRequestsAt(targetSlot).catch(err => {
+      this.log.error(`Failed to publish votes despite sync failure for slot ${slot}`, err, { slot });
+    });
   }
 
   /**
@@ -698,9 +956,10 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
   @trackSpan('Sequencer.tryVoteWhenEscapeHatchOpen', ({ slot }) => ({ [Attributes.SLOT_NUMBER]: slot }))
   protected async tryVoteWhenEscapeHatchOpen(args: {
     slot: SlotNumber;
+    targetSlot: SlotNumber;
     proposer: EthAddress | undefined;
   }): Promise<void> {
-    const { slot, proposer } = args;
+    const { slot, targetSlot, proposer } = args;
 
     // Prevent duplicate attempts in the same slot
     if (this.lastSlotForFallbackVote === slot) {
@@ -713,10 +972,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
 
     const { attestorAddress, publisher } = await this.publisherFactory.create(proposer);
 
-    this.log.debug(`Escape hatch open for slot ${slot}, attempting vote-only actions`, { slot, attestorAddress });
+    this.log.debug(`Escape hatch open for slot ${slot}, attempting vote-only actions`, {
+      slot,
+      targetSlot,
+      attestorAddress,
+    });
 
+    // Under proposer pipelining, the multicall is expected to mine in `targetSlot` (slot + 1).
+    // Governance and slashing votes are EIP-712-signed against the slot they will mine in, and the
+    // L1 contract checks `msg.sender == getCurrentProposer()` using the mining slot. So we must
+    // sign for `targetSlot` and delay submission to the start of `targetSlot`.
     const voter = new CheckpointVoter(
-      slot,
+      targetSlot,
       publisher,
       attestorAddress,
       this.validatorClient,
@@ -735,8 +1002,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
       return;
     }
 
-    this.log.info(`Voting in slot ${slot} (escape hatch open)`, { slot });
-    await publisher.sendRequests();
+    this.log.info(`Voting in slot ${slot} (escape hatch open)`, { slot, targetSlot });
+    // Votes are EIP-712-signed for `targetSlot`. Delay submission to the start of `targetSlot` so
+    // the multicall mines in the slot the votes were signed for; otherwise the L1 contract reads
+    // `signaler = getCurrentProposer()` against the wrong slot and signature verification fails
+    // silently inside Multicall3. Fire-and-forget so we don't block the sequencer's work loop while
+    // waiting for the target slot to start, mirroring tryVoteWhenSyncFails.
+    void publisher.sendRequestsAt(targetSlot).catch(err => {
+      this.log.error(`Failed to publish escape-hatch votes for slot ${slot}`, err, { slot, targetSlot });
+    });
   }
 
   /**
@@ -909,8 +1183,11 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
 type SequencerSyncCheckResult = {
   blockData?: BlockData;
   checkpointNumber: CheckpointNumber;
+  checkpointedCheckpointNumber: CheckpointNumber;
   blockNumber: BlockNumber;
   archive: Fr;
+  hasProposedCheckpoint: boolean;
+  proposedCheckpointData?: ProposedCheckpointData;
   syncedL2Slot: SlotNumber;
   pendingChainValidationStatus: ValidateCheckpointResult;
 };