@aztec/sequencer-client 0.0.1-commit.b2a5d0dd1 → 0.0.1-commit.b3d3157a

package/src/sequencer/checkpoint_proposal_job.ts

@@ -14,12 +14,12 @@ import {
   generateUnrecoverableSignature,
 } from '@aztec/foundation/crypto/secp256k1-signer';
 import { Fr } from '@aztec/foundation/curves/bn254';
+import { InterruptError, TimeoutError } from '@aztec/foundation/error';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { Signature } from '@aztec/foundation/eth-signature';
 import { filter } from '@aztec/foundation/iterator';
 import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
-import { retryUntil } from '@aztec/foundation/retry';
-import { sleep, sleepUntil } from '@aztec/foundation/sleep';
+import { InterruptibleSleep } from '@aztec/foundation/sleep';
 import { type DateProvider, Timer } from '@aztec/foundation/timer';
 import { type TypedEventEmitter, isErrorClass, unfreeze } from '@aztec/foundation/types';
 import type { P2P } from '@aztec/p2p';
@@ -31,9 +31,15 @@ import {
   type L2BlockSink,
   type L2BlockSource,
   MaliciousCommitteeAttestationsAndSigners,
+  type ProposedCheckpointSink,
   type ValidateCheckpointResult,
 } from '@aztec/stdlib/block';
-import { type Checkpoint, type ProposedCheckpointData, validateCheckpoint } from '@aztec/stdlib/checkpoint';
+import {
+  type Checkpoint,
+  type ProposedCheckpointData,
+  getPreviousCheckpointOutHashes,
+  validateCheckpoint,
+} from '@aztec/stdlib/checkpoint';
 import { computeQuorum, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
 import { Gas } from '@aztec/stdlib/gas';
 import {
@@ -46,8 +52,10 @@ import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@azte
 import type {
   BlockProposal,
   BlockProposalOptions,
+  CheckpointAttestation,
   CheckpointProposal,
   CheckpointProposalOptions,
+  CoordinationSignatureContext,
 } from '@aztec/stdlib/p2p';
 import { orderAttestations, trimAttestations } from '@aztec/stdlib/p2p';
 import type { L2BlockBuiltStats } from '@aztec/stdlib/stats';
@@ -59,10 +67,7 @@ import { DutyAlreadySignedError, SlashingProtectionError } from '@aztec/validato
 
 import type { GlobalVariableBuilder } from '../global_variable_builder/global_builder.js';
 import type { InvalidateCheckpointRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js';
-import {
-  buildPipelinedParentSimulationOverridesPlan,
-  buildSubmissionSimulationOverridesPlan,
-} from './chain_state_overrides.js';
+import { buildCheckpointSimulationOverridesPlan } from './chain_state_overrides.js';
 import type { CheckpointProposalJobMetricsRecorder } from './checkpoint_proposal_job_metrics.js';
 import { CheckpointVoter } from './checkpoint_voter.js';
 import { SequencerInterruptedError } from './errors.js';
@@ -74,6 +79,7 @@ import { SequencerState } from './utils.js';
 
 /** How much time to sleep while waiting for min transactions to accumulate for a block */
 const TXS_POLLING_MS = 500;
+const ARCHIVER_SYNC_POLLING_MS = 200;
 
 /** Result from proposeCheckpoint when a checkpoint was successfully built and broadcast. */
 type CheckpointProposalBroadcast = {
@@ -97,12 +103,24 @@ type CheckpointProposalResult = {
  */
 export class CheckpointProposalJob implements Traceable {
   protected readonly log: Logger;
+  private readonly checkpointEventLog: Logger;
 
   /** Tracks the fire-and-forget L1 submission promise so it can be awaited during shutdown. */
   private pendingL1Submission: Promise<void> | undefined;
+  private readonly interruptibleSleep = new InterruptibleSleep();
+  private interrupted = false;
+
+  /**
+   * Chain state overrides built once per slot in proposeCheckpoint after the checkpoint is
+   * complete. Carries the pending parent override (archive + slot + fee header) for pipelining,
+   * or the invalidation pending override when rolling back. Consumed by
+   * publisher.validateBlockHeader before broadcast.
+   */
+  private checkpointSimulationOverridesPlan?: SimulationOverridesPlan;
 
-  /** Pipelined parent chain state used while building and later submitting this checkpoint. */
-  private pipelinedParentSimulationOverridesPlan?: SimulationOverridesPlan;
+  private getSignatureContext(): CoordinationSignatureContext {
+    return this.signatureContext;
+  }
 
   constructor(
     private readonly slotNow: SlotNumber,
@@ -110,6 +128,7 @@ export class CheckpointProposalJob implements Traceable {
     private readonly targetEpoch: EpochNumber,
     private readonly checkpointNumber: CheckpointNumber,
     private readonly syncedToBlockNumber: BlockNumber,
+    private readonly checkpointedCheckpointNumber: CheckpointNumber,
     // TODO(palla/mbps): Can we remove the proposer in favor of attestorAddress? Need to check fisherman-node flows.
     private readonly proposer: EthAddress | undefined,
     private readonly publisher: SequencerPublisher,
@@ -122,8 +141,9 @@ export class CheckpointProposalJob implements Traceable {
     private readonly l1ToL2MessageSource: L1ToL2MessageSource,
     private readonly l2BlockSource: L2BlockSource,
     private readonly checkpointsBuilder: FullNodeCheckpointsBuilder,
-    private readonly blockSink: L2BlockSink,
+    private readonly blockSink: L2BlockSink & ProposedCheckpointSink,
     private readonly l1Constants: SequencerRollupConstants,
+    private readonly signatureContext: CoordinationSignatureContext,
     protected config: ResolvedSequencerConfig,
     protected timetable: SequencerTimetable,
     private readonly slasherClient: SlasherClientInterface | undefined,
@@ -132,7 +152,7 @@ export class CheckpointProposalJob implements Traceable {
     private readonly metrics: SequencerMetrics,
     private readonly checkpointMetrics: CheckpointProposalJobMetricsRecorder,
     protected readonly eventEmitter: TypedEventEmitter<SequencerEvents>,
-    private readonly setStateFn: (state: SequencerState, slot?: SlotNumber) => void,
+    private readonly setStateFn: (state: SequencerState, slot?: SlotNumber, timeReferenceSlot?: SlotNumber) => void,
     public readonly tracer: Tracer,
     bindings?: LoggerBindings,
     private readonly proposedCheckpointData?: ProposedCheckpointData,
@@ -141,6 +161,10 @@ export class CheckpointProposalJob implements Traceable {
       ...bindings,
       instanceId: `slot-${this.slotNow}`,
     });
+    this.checkpointEventLog = createLogger('sequencer:checkpoint-events', {
+      ...bindings,
+      instanceId: `slot-${this.slotNow}`,
+    });
   }
 
   /** Awaits the pending L1 submission if one is in progress. Call during shutdown. */
@@ -149,6 +173,37 @@ export class CheckpointProposalJob implements Traceable {
     await this.pendingL1Submission;
   }
 
+  /** Interrupts job-owned waits, including the publisher's send-at-slot sleep, so shutdown can finish. */
+  public interrupt(): void {
+    this.interrupted = true;
+    this.interruptibleSleep.interrupt(true);
+    this.publisher.interrupt();
+  }
+
+  private async awaitInterruptibleSleep(ms: number): Promise<void> {
+    if (this.interrupted) {
+      throw new SequencerInterruptedError();
+    }
+    if (ms <= 0) {
+      return;
+    }
+    try {
+      await this.interruptibleSleep.sleep(ms);
+    } catch (err) {
+      if (err instanceof InterruptError) {
+        throw new SequencerInterruptedError();
+      }
+      throw err;
+    }
+  }
+
+  private logCheckpointEvent(eventName: string, message: string, fields: Record<string, unknown>): void {
+    this.checkpointEventLog.debug(message, {
+      eventName: `sequencer-checkpoint-${eventName}`,
+      ...fields,
+    });
+  }
+
   /**
    * Executes the checkpoint proposal job.
    * Builds blocks, assembles checkpoint, and broadcasts the proposal (blocking).
@@ -179,9 +234,14 @@ export class CheckpointProposalJob implements Traceable {
 
     if (!broadcast) {
       await Promise.all(votesPromises);
-      // Still submit votes even without a checkpoint
+      // Still submit votes even without a checkpoint.
+      // Under proposer pipelining, vote-offenses signatures are EIP-712-bound to `targetSlot`
+      // (the pipelined slot in which the multicall is expected to mine). Submitting at the
+      // wall-clock time would let the multicall mine in a different L2 slot, causing
+      // signature verification to fail silently inside Multicall3. Delay submission to the
+      // start of `targetSlot` so the tx mines in the slot the vote was signed for.
       if (!this.config.fishermanMode) {
-        this.pendingL1Submission = this.publisher.sendRequestsAt(this.dateProvider.nowAsDate()).then(() => {});
+        this.pendingL1Submission = this.publisher.sendRequestsAt(this.targetSlot).then(() => {});
       }
       return undefined;
     }
@@ -211,7 +271,6 @@ export class CheckpointProposalJob implements Traceable {
     votesPromises: Promise<unknown>[],
   ): Promise<void> {
     const { checkpoint } = broadcast;
-    const isPipelining = this.epochCache.isProposerPipeliningEnabled();
 
     try {
       // Wait for all votes actions, enqueued at the beginning, to resolve
@@ -220,15 +279,14 @@ export class CheckpointProposalJob implements Traceable {
       // Try to collect attestations from the committee
       const signedAttestations = await this.getSignedCommitteeAttestations(broadcast);
 
-      // If pipelining, wait for the previous checkpoint to land on L1 before submitting,
-      // so we can check it matches the proposed checkpoint we used as parent, and has valid attestations.
-      if (signedAttestations && (!isPipelining || (await this.waitForValidParentCheckpointOnL1()))) {
+      // Wait for the previous checkpoint to land on L1 before submitting, so we can check it
+      // matches the proposed checkpoint we used as parent, and has valid attestations.
+      if (signedAttestations && (await this.waitForValidParentCheckpointOnL1())) {
         await this.enqueueCheckpointForSubmission({ checkpoint, ...signedAttestations });
       }
 
       // If we failed to collect attestations, at least check if we need to issue an invalidation
-      // Note that if we are not pipelining, we enqueued the invalidation at the beginning
-      if (!signedAttestations && isPipelining && (await this.waitForSyncedL2SlotNumber(this.slotNow))) {
+      if (!signedAttestations && (await this.waitForSyncedL2SlotNumber(this.slotNow))) {
         const validationStatus = await this.l2BlockSource.getPendingChainValidationStatus();
         if (!validationStatus.valid) {
           this.log.warn(
@@ -240,32 +298,56 @@ export class CheckpointProposalJob implements Traceable {
       }
 
       // Send whatever was enqueued: votes + (propose | invalidation | nothing).
-      // Compute the earliest time to submit: pipeline slot start when pipelining, now otherwise.
-      const submitAfter = isPipelining
-        ? new Date(Number(getTimestampForSlot(this.targetSlot, this.l1Constants)) * 1000)
-        : new Date(this.dateProvider.now());
-
-      const l1Response = await this.publisher.sendRequestsAt(submitAfter);
+      const l1Response = await this.publisher.sendRequestsAt(this.targetSlot);
       const proposedAction = l1Response?.successfulActions.find(a => a === 'propose');
       if (proposedAction) {
+        this.logCheckpointEvent('published', `Checkpoint published for slot ${this.targetSlot}`, {
+          slot: this.targetSlot,
+          checkpointNumber: this.checkpointNumber,
+          successfulActions: l1Response?.successfulActions,
+          sentActions: l1Response?.sentActions,
+        });
         this.eventEmitter.emit('checkpoint-published', { checkpoint: this.checkpointNumber, slot: this.targetSlot });
         const coinbase = checkpoint.header.coinbase;
         await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
       } else {
+        this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
+          slot: this.targetSlot,
+          checkpointNumber: this.checkpointNumber,
+          successfulActions: l1Response?.successfulActions,
+          failedActions: l1Response?.failedActions,
+          sentActions: l1Response?.sentActions,
+          expiredActions: l1Response?.expiredActions,
+          reason: 'propose_action_not_successful',
+        });
+        this.log.warn(`Checkpoint publish failed for slot ${this.targetSlot}`, {
+          slot: this.targetSlot,
+          checkpointNumber: this.checkpointNumber,
+          successfulActions: l1Response?.successfulActions,
+          failedActions: l1Response?.failedActions,
+          sentActions: l1Response?.sentActions,
+          expiredActions: l1Response?.expiredActions,
+          reason: 'propose_action_not_successful',
+        });
         this.eventEmitter.emit('checkpoint-publish-failed', { ...l1Response, slot: this.targetSlot });
-        if (isPipelining) {
-          this.metrics.recordPipelineDiscard();
-        }
+        this.metrics.recordPipelineDiscard();
       }
     } catch (err) {
       if (err instanceof SequencerInterruptedError) {
         return;
       }
-      this.log.error(`Background attestation/L1 pipeline failed for slot ${this.targetSlot}`, err);
+      this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
+        slot: this.targetSlot,
+        checkpointNumber: this.checkpointNumber,
+        reason: err instanceof Error ? err.message : String(err),
+      });
+      this.log.error(`Background attestation/L1 pipeline failed for slot ${this.targetSlot}`, err, {
+        slot: this.targetSlot,
+        checkpointNumber: this.checkpointNumber,
+        reason: err instanceof Error ? err.message : String(err),
+      });
       this.eventEmitter.emit('checkpoint-publish-failed', { slot: this.targetSlot });
-      if (isPipelining) {
-        this.metrics.recordPipelineDiscard();
-      }
+      this.metrics.recordPipelineDiscard();
     }
   }
 
@@ -273,7 +355,8 @@ export class CheckpointProposalJob implements Traceable {
   private async enqueueCheckpointForSubmission(result: CheckpointProposalResult): Promise<void> {
     const { checkpoint, attestations, attestationsSignature } = result;
 
-    this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.targetSlot);
+    // Build-frame deadlines are measured against `slotNow`; observers still see `targetSlot`.
+    this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.targetSlot, this.slotNow);
     const aztecSlotDuration = this.l1Constants.slotDuration;
     const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
     const txTimeoutAt = new Date((submissionSlotStart + aztecSlotDuration) * 1000);
@@ -292,17 +375,8 @@ export class CheckpointProposalJob implements Traceable {
       }
     }
 
-    const isPipelining = this.epochCache.isProposerPipeliningEnabled();
-    const submissionSimulationOverridesPlan = buildSubmissionSimulationOverridesPlan({
-      pipelinedParentPlan: this.pipelinedParentSimulationOverridesPlan,
-      invalidateToPendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
-      lastArchiveRoot: checkpoint.header.lastArchiveRoot,
-      pipeliningEnabled: isPipelining,
-    });
-
     await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
       txTimeoutAt,
-      ...(submissionSimulationOverridesPlan ? { simulationOverridesPlan: submissionSimulationOverridesPlan } : {}),
     });
   }
 
@@ -318,16 +392,21 @@ export class CheckpointProposalJob implements Traceable {
     const timeoutSeconds = Math.max(0.1, (targetSlotEndMs + syncDelayTolerance - this.dateProvider.now()) / 1000);
 
     try {
-      return await retryUntil(
-        async () => {
-          const syncedSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
-          return syncedSlot !== undefined && syncedSlot >= waitForSlot;
-        },
-        `archiver sync past slot ${waitForSlot}`,
-        timeoutSeconds,
-        0.2,
-      );
-    } catch {
+      const timer = new Timer();
+      while (true) {
+        const syncedSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
+        if (syncedSlot !== undefined && syncedSlot >= waitForSlot) {
+          return true;
+        }
+        if (timeoutSeconds && timer.s() > timeoutSeconds) {
+          throw new TimeoutError(`Timeout awaiting archiver sync past slot ${waitForSlot}`);
+        }
+        await this.awaitInterruptibleSleep(ARCHIVER_SYNC_POLLING_MS);
+      }
+    } catch (err) {
+      if (err instanceof SequencerInterruptedError) {
+        throw err;
+      }
       this.log.warn(
         `Archiver did not sync L1 past slot ${waitForSlot} before slot ${this.targetSlot} expired, discarding pipelined work`,
         { checkpointNumber: this.checkpointNumber },
@@ -345,6 +424,13 @@ export class CheckpointProposalJob implements Traceable {
    * If the parent has invalid attestations, enqueues an invalidation. Returns whether to proceed with the proposal.
    */
   protected async waitForValidParentCheckpointOnL1(): Promise<boolean> {
+    if (this.config.skipWaitForValidParentCheckpointOnL1) {
+      this.log.warn(`Skipping waitForValidParentCheckpointOnL1 due to test configuration`, {
+        checkpointNumber: this.checkpointNumber,
+      });
+      return true;
+    }
+
     const parentCheckpointNumber = CheckpointNumber(this.checkpointNumber - 1);
 
     // Wait until archiver has synced L1 past the parent's slot (slotNow)
@@ -451,7 +537,7 @@ export class CheckpointProposalJob implements Traceable {
   private async proposeCheckpoint(): Promise<CheckpointProposalBroadcast | undefined> {
     try {
       const now = this.dateProvider.now();
-      if (this.epochCache.isProposerPipeliningEnabled() && this.proposedCheckpointData) {
+      if (this.proposedCheckpointData) {
         // Measure against the wall-clock slot whose build window we are currently using.
         // In pipelining mode `targetSlot` is intentionally one slot ahead, which makes the
         // target-slot boundary a full slot away from the actual build start time.
@@ -465,12 +551,15 @@ export class CheckpointProposalJob implements Traceable {
       const feeRecipient = this.validatorClient.getFeeRecipientForAttestor(this.attestorAddress);
 
       // Start the checkpoint
-      this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.targetSlot);
-      this.log.info(`Starting checkpoint proposal`, {
+      this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.targetSlot, this.slotNow);
+      this.logCheckpointEvent('slot-started', `Starting checkpoint proposal for slot ${this.targetSlot}`, {
         buildSlot: this.slotNow,
         submissionSlot: this.targetSlot,
-        pipelining: this.epochCache.isProposerPipeliningEnabled(),
+        slot: this.targetSlot,
+        checkpointNumber: this.checkpointNumber,
         proposer: this.proposer?.toString(),
+        attestorAddress: this.attestorAddress.toString(),
+        publisherAddress: this.publisher.getSenderAddress().toString(),
         coinbase: coinbase.toString(),
       });
       this.metrics.incOpenSlot(this.targetSlot, this.proposer?.toString() ?? 'unknown');
@@ -480,37 +569,51 @@ export class CheckpointProposalJob implements Traceable {
         this.publisher.enqueueInvalidateCheckpoint(this.invalidateCheckpoint);
       }
 
-      // Create checkpoint builder for the slot.
-      // When pipelining, force the proposed checkpoint number and fee header to our parent so the
-      // fee computation sees the same chain tip that L1 will see once the previous pipelined checkpoint lands.
-      const isPipelining = this.epochCache.isProposerPipeliningEnabled();
-      this.pipelinedParentSimulationOverridesPlan = isPipelining
-        ? await buildPipelinedParentSimulationOverridesPlan({
-            checkpointNumber: this.checkpointNumber,
-            proposedCheckpointData: this.proposedCheckpointData,
-            rollup: this.publisher.rollupContract,
-            log: this.log,
-          })
-        : undefined;
+      // Build the simulation plan for this slot. When pipelining, this overrides L1's view of
+      // pending/archive/fee-header to "as if the proposed parent had landed", so both the
+      // mana-min-fee simulation (in the globals builder) and the pre-broadcast
+      // validateBlockHeader see the chain tip the eventual L1 send will see.
+      this.checkpointSimulationOverridesPlan = await buildCheckpointSimulationOverridesPlan({
+        checkpointNumber: this.checkpointNumber,
+        proposedCheckpointData: this.proposedCheckpointData,
+        invalidateToPendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
+        checkpointedCheckpointNumber: this.checkpointedCheckpointNumber,
+        rollup: this.publisher.rollupContract,
+        signatureContext: this.signatureContext,
+        log: this.log,
+      });
 
       const checkpointGlobalVariables = await this.globalsBuilder.buildCheckpointGlobalVariables(
         coinbase,
         feeRecipient,
         this.targetSlot,
-        this.pipelinedParentSimulationOverridesPlan,
+        this.checkpointSimulationOverridesPlan,
       );
 
       // Collect L1 to L2 messages for the checkpoint and compute their hash
       const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(this.checkpointNumber);
       const inHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
 
-      // Collect the out hashes of all the checkpoints before this one in the same epoch
-      const previousCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsDataForEpoch(this.targetEpoch))
-        .filter(c => c.checkpointNumber < this.checkpointNumber)
-        .map(c => c.checkpointOutHash);
+      // Collect the out hashes of all the checkpoints before this one in the same epoch.
+      // Under pipelining the parent checkpoint may not be on L1 yet at build time, so the helper
+      // splices in the parent's checkpointOutHash from the locally-known proposed checkpoint so
+      // the resulting `epochOutHash` matches what validators (and L1) compute once the parent
+      // lands on L1.
+      const previousCheckpointOutHashes = await getPreviousCheckpointOutHashes({
+        blockSource: this.l2BlockSource,
+        epoch: this.targetEpoch,
+        checkpointNumber: this.checkpointNumber,
+        l1Constants: this.epochCache.getL1Constants(),
+        pipeliningEnabled: true,
+        proposedCheckpointData: this.proposedCheckpointData,
+        log: this.log,
+      });
 
-      // Get the fee asset price modifier from the oracle
-      const feeAssetPriceModifier = await this.publisher.getFeeAssetPriceModifier();
+      // Anchor the modifier to the predicted parent fee header: L1 will apply it against
+      // that, not against the latest published checkpoint (which lags by one under pipelining).
+      const predictedParentEthPerFeeAssetE12 =
+        this.checkpointSimulationOverridesPlan?.pendingCheckpointState?.feeHeader?.ethPerFeeAsset;
+      const feeAssetPriceModifier = await this.publisher.getFeeAssetPriceModifier(predictedParentEthPerFeeAssetE12);
 
       // Create a long-lived forked world state for the checkpoint builder
       await using fork = await this.worldState.fork(this.syncedToBlockNumber, { closeDelayMs: 12_000 });
@@ -534,7 +637,8 @@ export class CheckpointProposalJob implements Traceable {
 
       const checkpointProposalOptions: CheckpointProposalOptions = {
         publishFullTxs: !!this.config.publishTxsWithProposals,
-        broadcastInvalidCheckpointProposal: this.config.broadcastInvalidBlockProposal,
+        broadcastInvalidCheckpointProposal:
+          this.config.broadcastInvalidCheckpointProposalOnly || this.config.broadcastInvalidBlockProposal,
       };
 
       let blocksInCheckpoint: L2Block[] = [];
@@ -562,23 +666,45 @@ export class CheckpointProposalJob implements Traceable {
       }
 
       if (blocksInCheckpoint.length === 0) {
-        this.log.warn(`No blocks were built for slot ${this.targetSlot}`, { slot: this.targetSlot });
+        this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
+          slot: this.targetSlot,
+          checkpointNumber: this.checkpointNumber,
+          reason: 'no_blocks_built',
+        });
+        this.log.warn(`No blocks were built for slot ${this.targetSlot}`, {
+          slot: this.targetSlot,
+          checkpointNumber: this.checkpointNumber,
+          reason: 'no_blocks_built',
+        });
         this.eventEmitter.emit('checkpoint-empty', { slot: this.targetSlot });
         return undefined;
       }
 
       const minBlocksForCheckpoint = this.config.minBlocksForCheckpoint;
       if (minBlocksForCheckpoint !== undefined && blocksInCheckpoint.length < minBlocksForCheckpoint) {
+        this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
+          slot: this.targetSlot,
+          checkpointNumber: this.checkpointNumber,
+          blocksBuilt: blocksInCheckpoint.length,
+          minBlocksForCheckpoint,
+          reason: 'min_blocks_not_met',
+        });
         this.log.warn(
           `Checkpoint has fewer blocks than minimum (${blocksInCheckpoint.length} < ${minBlocksForCheckpoint}), skipping proposal`,
-          { slot: this.targetSlot, blocksBuilt: blocksInCheckpoint.length, minBlocksForCheckpoint },
+          {
+            slot: this.targetSlot,
+            checkpointNumber: this.checkpointNumber,
+            blocksBuilt: blocksInCheckpoint.length,
+            minBlocksForCheckpoint,
+            reason: 'min_blocks_not_met',
+          },
         );
         return undefined;
       }
 
       // Assemble and broadcast the checkpoint proposal, including the last block that was not
       // broadcasted yet, and wait to collect the committee attestations.
-      this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.targetSlot);
+      this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.targetSlot, this.slotNow);
       const checkpoint = await checkpointBuilder.completeCheckpoint();
 
       // Final validation: per-block limits are only checked if the operator set them explicitly.
@@ -592,7 +718,18 @@ export class CheckpointProposalJob implements Traceable {
           maxTxsPerCheckpoint: this.config.maxTxsPerCheckpoint,
         });
       } catch (err) {
+        this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
+          slot: this.targetSlot,
+          checkpointNumber: this.checkpointNumber,
+          blocksBuilt: blocksInCheckpoint.length,
+          reason: 'invalid_checkpoint',
+          checkpoint: checkpoint.header.toInspect(),
+        });
         this.log.error(`Built an invalid checkpoint at slot ${this.slotNow} (skipping proposal)`, err, {
+          slot: this.targetSlot,
+          checkpointNumber: this.checkpointNumber,
+          blocksBuilt: blocksInCheckpoint.length,
+          reason: 'invalid_checkpoint',
           checkpoint: checkpoint.header.toInspect(),
         });
         return undefined;
@@ -605,6 +742,17 @@ export class CheckpointProposalJob implements Traceable {
         checkpoint.getStats().txCount,
         Number(checkpoint.header.totalManaUsed.toBigInt()),
       );
+      this.logCheckpointEvent('built', `Checkpoint built for slot ${this.targetSlot}`, {
+        slot: this.targetSlot,
+        buildSlot: this.slotNow,
+        checkpointNumber: this.checkpointNumber,
+        proposer: this.proposer?.toString(),
+        attestorAddress: this.attestorAddress.toString(),
+        publisherAddress: this.publisher.getSenderAddress().toString(),
+        blocksBuilt: blocksInCheckpoint.length,
+        txCount: checkpoint.getStats().txCount,
+        totalMana: Number(checkpoint.header.totalManaUsed.toBigInt()),
+      });
 
       // In fisherman mode, return the checkpoint without broadcasting or collecting attestations
       if (this.config.fishermanMode) {
@@ -622,6 +770,25 @@ export class CheckpointProposalJob implements Traceable {
         return { checkpoint, proposal: undefined!, blockProposedAt: this.dateProvider.now() };
       }
 
+      // Validate the header against L1 state before broadcasting.
+      // If this fails the slot is aborted before any gossip work; state drift between here
+      // and the eventual L1 send is caught by the bundle simulate at send time.
+      try {
+        await this.publisher.validateBlockHeader(checkpoint.header, this.checkpointSimulationOverridesPlan);
+      } catch (err) {
+        this.log.error(`Pre-broadcast header validation failed for slot ${this.targetSlot}; aborting`, err, {
+          slot: this.targetSlot,
+          checkpointNumber: this.checkpointNumber,
+        });
+        this.metrics.recordCheckpointProposalFailed('header_validation_failed');
+        this.eventEmitter.emit('header-validation-failed', {
+          slot: this.targetSlot,
+          checkpointNumber: this.checkpointNumber,
+          reason: err instanceof Error ? err.message : String(err),
+        });
+        return undefined;
+      }
+
       // Create the checkpoint proposal and broadcast it
       const proposal = await this.validatorClient.createCheckpointProposal(
         checkpoint.header,
@@ -633,9 +800,26 @@ export class CheckpointProposalJob implements Traceable {
         checkpointProposalOptions,
       );
 
+      // Advance our own optimistic proposed-checkpoint tip locally before gossiping. Gossipsub
+      // doesn't echo our own messages back, so this is how the proposer makes its own proposed
+      // checkpoint visible for pipelining the next slot. Built from local checkpoint data — never
+      // from the broadcast proposal archive, which may be deliberately corrupted under test flags.
+      // Fail closed: if this throws, the outer catch aborts the slot before gossiping.
+      await this.syncProposedCheckpointToArchiver(checkpoint, blocksInCheckpoint.length, feeAssetPriceModifier);
+
       const blockProposedAt = this.dateProvider.now();
-      await this.p2pClient.broadcastCheckpointProposal(proposal);
-      this.checkpointMetrics.noteCheckpointBroadcast(this.dateProvider.now());
+      if (this.config.skipBroadcastCheckpointProposal) {
+        // Test-only: suppress the CheckpointProposal so peers never see a proposed checkpoint for
+        // this slot, but still broadcast the held last block standalone so peers' archivers ingest
+        // it as a proposed-but-uncheckpointed tip — the exact orphan-block state that
+        // pruneOrphanProposedBlocks / checkSync must handle.
+        if (blockPendingBroadcast && !this.config.skipBroadcastProposals) {
+          await this.p2pClient.broadcastProposal(blockPendingBroadcast);
+        }
+      } else if (!this.config.skipBroadcastProposals) {
+        await this.p2pClient.broadcastCheckpointProposal(proposal);
+        this.checkpointMetrics.noteCheckpointBroadcast(this.dateProvider.now());
+      }
 
       // Return immediately after broadcast — attestation collection happens in the background
       return { checkpoint, proposal, blockProposedAt };
@@ -675,6 +859,15 @@ export class CheckpointProposalJob implements Traceable {
       const indexWithinCheckpoint = IndexWithinCheckpoint(blocksBuilt);
       const blockNumber = BlockNumber(initialBlockNumber + blocksBuilt);
 
+      if (blocksBuilt >= this.config.maxBlocksPerCheckpoint) {
+        this.log.debug(`Reached max blocks per checkpoint`, {
+          slot: this.targetSlot,
+          blocksBuilt,
+          maxBlocksPerCheckpoint: this.config.maxBlocksPerCheckpoint,
+        });
+        break;
+      }
+
       const secondsIntoSlot = this.getSecondsIntoSlot();
       const timingInfo = this.timetable.canStartNextBlock(secondsIntoSlot);
 
@@ -735,7 +928,12 @@ export class CheckpointProposalJob implements Traceable {
       usedTxs.forEach(tx => txHashesAlreadyIncluded.add(tx.txHash.toString()));
 
       // Sign the block proposal. This will throw if HA signing fails.
-      const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions);
+      const proposal = await this.createBlockProposal(block, inHash, usedTxs, {
+        ...blockProposalOptions,
+        broadcastInvalidBlockProposal:
+          blockProposalOptions.broadcastInvalidBlockProposal ||
+          block.indexWithinCheckpoint === this.config.invalidBlockProposalIndexWithinCheckpoint,
+      });
 
       // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal,
       // so we avoid polluting our archive with a block that would fail.
@@ -756,7 +954,9 @@ export class CheckpointProposalJob implements Traceable {
       }
 
       // Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it.
-      proposal && (await this.p2pClient.broadcastProposal(proposal));
+      if (proposal && !this.config.skipBroadcastProposals) {
+        await this.p2pClient.broadcastProposal(proposal);
+      }
 
       // Wait until the next block's start time
       await this.waitUntilNextSubslot(timingInfo.deadline);
@@ -796,7 +996,7 @@ export class CheckpointProposalJob implements Traceable {
   /** Sleeps until it is time to produce the next block in the slot */
   @trackSpan('CheckpointProposalJob.waitUntilNextSubslot')
   private async waitUntilNextSubslot(nextSubslotStart: number) {
-    this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.targetSlot);
+    this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.targetSlot, this.slotNow);
     this.log.verbose(`Waiting until time for the next block at ${nextSubslotStart}s into slot`, {
       slot: this.targetSlot,
     });
@@ -830,9 +1030,26 @@ export class CheckpointProposalJob implements Traceable {
       // Wait until we have enough txs to build the block
       const { availableTxs, canStartBuilding, minTxs } = await this.waitForMinTxs(opts);
       if (!canStartBuilding) {
+        this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
+          reason: 'insufficient_txs',
+          blockNumber,
+          slot: this.targetSlot,
+          checkpointNumber: this.checkpointNumber,
+          indexWithinCheckpoint,
+          availableTxs,
+          minTxs,
+        });
         this.log.warn(
           `Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (got ${availableTxs} txs but needs ${minTxs})`,
-          { blockNumber, slot: this.targetSlot, indexWithinCheckpoint },
+          {
+            reason: 'insufficient_txs',
+            blockNumber,
+            slot: this.targetSlot,
+            checkpointNumber: this.checkpointNumber,
+            indexWithinCheckpoint,
+            availableTxs,
+            minTxs,
+          },
         );
         this.eventEmitter.emit('block-tx-count-check-failed', { minTxs, availableTxs, slot: this.targetSlot });
         this.metrics.recordBlockProposalFailed('insufficient_txs');
@@ -850,7 +1067,7 @@ export class CheckpointProposalJob implements Traceable {
         `Building block ${blockNumber} at index ${indexWithinCheckpoint} for slot ${this.targetSlot} with ${availableTxs} available txs`,
         { slot: this.targetSlot, blockNumber, indexWithinCheckpoint },
       );
-      this.setStateFn(SequencerState.CREATING_BLOCK, this.targetSlot);
+      this.setStateFn(SequencerState.CREATING_BLOCK, this.targetSlot, this.slotNow);
 
       // Per-block limits are operator overrides (from SEQ_MAX_L2_BLOCK_GAS etc.) further capped
       // by remaining checkpoint-level budgets inside CheckpointBuilder before each block is built.
@@ -884,10 +1101,21 @@ export class CheckpointProposalJob implements Traceable {
       await this.dropFailedTxsFromP2P(buildResult.failedTxs);
 
       if (buildResult.status === 'insufficient-valid-txs') {
+        this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
+          reason: 'insufficient_valid_txs',
+          slot: this.targetSlot,
+          checkpointNumber: this.checkpointNumber,
+          blockNumber,
+          numTxs: buildResult.processedCount,
+          indexWithinCheckpoint,
+          minValidTxs,
+        });
         this.log.warn(
           `Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.targetSlot} has too few valid txs to be proposed`,
           {
+            reason: 'insufficient_valid_txs',
             slot: this.targetSlot,
+            checkpointNumber: this.checkpointNumber,
             blockNumber,
             numTxs: buildResult.processedCount,
             indexWithinCheckpoint,
@@ -925,6 +1153,9 @@ export class CheckpointProposalJob implements Traceable {
       // `buildSlot` is the wall-clock slot during which the block was actually built.
       this.eventEmitter.emit('block-proposed', {
         blockNumber: block.number,
+        blockHash,
+        checkpointNumber: this.checkpointNumber,
+        indexWithinCheckpoint: block.indexWithinCheckpoint,
         slot: this.targetSlot,
         buildSlot: this.slotNow,
       });
@@ -936,7 +1167,18 @@ export class CheckpointProposalJob implements Traceable {
         reason: err.message,
         slot: this.targetSlot,
       });
-      this.log.error(`Error building block`, err, { blockNumber, slot: this.targetSlot });
+      this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
+        reason: err instanceof Error ? err.message : String(err),
+        slot: this.targetSlot,
+        checkpointNumber: this.checkpointNumber,
+        blockNumber,
+      });
+      this.log.error(`Error building block`, err, {
+        reason: err instanceof Error ? err.message : String(err),
+        slot: this.targetSlot,
+        checkpointNumber: this.checkpointNumber,
+        blockNumber,
+      });
       this.metrics.recordBlockProposalFailed(err.name || 'unknown_error');
       this.metrics.recordFailedBlock();
       return { error: err };
@@ -996,7 +1238,7 @@ export class CheckpointProposalJob implements Traceable {
       }
 
       // Wait a bit before checking again
-      this.setStateFn(SequencerState.WAITING_FOR_TXS, this.targetSlot);
+      this.setStateFn(SequencerState.WAITING_FOR_TXS, this.targetSlot, this.slotNow);
       this.log.verbose(
         `Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (have ${availableTxs} but need ${minTxs})`,
         { blockNumber, slot: this.targetSlot, indexWithinCheckpoint },
@@ -1012,7 +1254,7 @@ export class CheckpointProposalJob implements Traceable {
     broadcast: CheckpointProposalBroadcast,
   ): Promise<{ attestations: CommitteeAttestationsAndSigners; attestationsSignature: Signature } | undefined> {
     const { proposal, blockProposedAt } = broadcast;
-    this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot);
+    this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot, this.slotNow);
     const attestations = await this.waitForAttestations(proposal);
     if (!attestations) {
       return undefined;
@@ -1048,7 +1290,7 @@ export class CheckpointProposalJob implements Traceable {
   ): Promise<CommitteeAttestationsAndSigners | undefined> {
     if (this.config.fishermanMode) {
       this.log.debug('Skipping attestation collection in fisherman mode');
-      return CommitteeAttestationsAndSigners.empty();
+      return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
     }
 
     const slotNumber = proposal.slotNumber;
@@ -1058,7 +1300,7 @@ export class CheckpointProposalJob implements Traceable {
       throw new Error('No committee when collecting attestations');
     } else if (committee.length === 0) {
       this.log.verbose(`Attesting committee is empty`);
-      return CommitteeAttestationsAndSigners.empty();
+      return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
     } else {
       this.log.debug(`Attesting committee length is ${committee.length}`, { committee });
     }
@@ -1068,7 +1310,13 @@ export class CheckpointProposalJob implements Traceable {
     if (this.config.skipCollectingAttestations) {
       this.log.warn('Skipping attestation collection as per config (attesting with own keys only)');
       const attestations = await this.validatorClient?.collectOwnAttestations(proposal, this.checkpointNumber);
-      return new CommitteeAttestationsAndSigners(orderAttestations(attestations ?? [], committee));
+      this.logCheckpointAttestations('collected', committee, attestations ?? [], numberOfRequiredAttestations, {
+        reason: 'collect_own_only',
+      });
+      return new CommitteeAttestationsAndSigners(
+        orderAttestations(attestations ?? [], committee),
+        this.getSignatureContext(),
+      );
     }
 
     const attestationTimeAllowed = this.config.enforceTimeTable
@@ -1104,6 +1352,9 @@ export class CheckpointProposalJob implements Traceable {
 
       // Rollup contract requires that the signatures are provided in the order of the committee
       const sorted = orderAttestations(trimmed, committee);
+      this.logCheckpointAttestations('collected', committee, attestations, numberOfRequiredAttestations, {
+        submittedCount: trimmed.length,
+      });
 
       // Manipulate the attestations if we've been configured to do so
       if (
@@ -1115,15 +1366,23 @@ export class CheckpointProposalJob implements Traceable {
         return this.manipulateAttestations(proposal.slotNumber, epoch, seed, committee, sorted);
       }
 
-      return new CommitteeAttestationsAndSigners(sorted);
+      return new CommitteeAttestationsAndSigners(sorted, this.getSignatureContext());
     } catch (err) {
       if (err && err instanceof AttestationTimeoutError) {
         collectedAttestationsCount = err.collectedCount;
+        this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
+          collectedCount: collectedAttestationsCount,
+          reason: 'timeout',
+        });
         this.log.error(
           `Timeout while waiting for attestations for checkpoint proposal at slot ${proposal.slotNumber} (collected ${collectedAttestationsCount}/${numberOfRequiredAttestations})`,
           err,
         );
       } else {
+        this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
+          collectedCount: collectedAttestationsCount,
+          reason: err instanceof Error ? err.message : String(err),
+        });
         this.log.error(`Error collecting attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
       }
       return undefined;
@@ -1132,6 +1391,31 @@ export class CheckpointProposalJob implements Traceable {
     }
   }
 
+  private logCheckpointAttestations(
+    status: 'collected' | 'failed',
+    committee: EthAddress[],
+    attestations: CheckpointAttestation[] | undefined,
+    requiredAttestations: number,
+    opts: { collectedCount?: number; submittedCount?: number; reason?: string } = {},
+  ) {
+    const signedValidators =
+      attestations
+        ?.map(attestation => attestation.getSender()?.toString())
+        .filter((address): address is `0x${string}` => address !== undefined) ?? [];
+    const collectedCount = opts.collectedCount ?? new Set(signedValidators).size;
+    const missingValidatorCount = status === 'failed' ? Math.max(0, requiredAttestations - collectedCount) : undefined;
+    this.logCheckpointEvent(`attestations-${status}`, `Checkpoint attestations ${status} for slot ${this.targetSlot}`, {
+      slot: this.targetSlot,
+      checkpointNumber: this.checkpointNumber,
+      committeeSize: committee.length,
+      requiredAttestations,
+      collectedAttestations: collectedCount,
+      ...(opts.submittedCount !== undefined && { submittedAttestations: opts.submittedCount }),
+      ...(missingValidatorCount !== undefined && { missingValidatorCount }),
+      ...(opts.reason !== undefined && { reason: opts.reason }),
+    });
+  }
+
   /** Breaks the attestations before publishing based on attack configs */
   private manipulateAttestations(
     slotNumber: SlotNumber,
@@ -1175,7 +1459,7 @@ export class CheckpointProposalJob implements Traceable {
           unfreeze(attestations[targetIndex]).signature = generateRecoverableSignature();
         }
       }
-      return new CommitteeAttestationsAndSigners(attestations);
+      return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
     }
 
     if (this.config.shuffleAttestationOrdering) {
@@ -1197,11 +1481,11 @@ export class CheckpointProposalJob implements Traceable {
         [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
       }
 
-      const signers = new CommitteeAttestationsAndSigners(attestations).getSigners();
-      return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers);
+      const signers = new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext()).getSigners();
+      return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers, this.getSignatureContext());
     }
 
-    return new CommitteeAttestationsAndSigners(attestations);
+    return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
   }
 
   private async dropFailedTxsFromP2P(failedTxs: FailedTx[]) {
@@ -1238,6 +1522,41 @@ export class CheckpointProposalJob implements Traceable {
     await this.blockSink.addBlock(block);
   }
 
+  /**
+   * Adds the proposed checkpoint to the archiver so the proposer's optimistic proposed-checkpoint
+   * tip advances locally. Gossip doesn't echo our own messages back, so without this the proposer
+   * would never see its own proposed checkpoint and couldn't pipeline the next slot.
+   *
+   * Skipped whenever proposed blocks aren't pushed (`skipPushProposedBlocksToArchiver`, fisherman
+   * mode): the archiver derives the checkpoint archive from its stored blocks, so without them the
+   * push would fail. All blocks were already added (and awaited) during block building, so this
+   * needs no retry — they are guaranteed present by the time we get here.
+   */
+  private async syncProposedCheckpointToArchiver(
+    checkpoint: Checkpoint,
+    blockCount: number,
+    feeAssetPriceModifier: bigint,
+  ): Promise<void> {
+    if (this.config.skipPushProposedBlocksToArchiver || this.config.fishermanMode) {
+      return;
+    }
+    const startBlock = BlockNumber(this.syncedToBlockNumber + 1);
+    this.log.debug(`Syncing proposed checkpoint ${this.checkpointNumber} to archiver`, {
+      checkpointNumber: this.checkpointNumber,
+      slot: this.targetSlot,
+      startBlock,
+      blockCount,
+    });
+    await this.blockSink.addProposedCheckpoint({
+      header: checkpoint.header,
+      checkpointNumber: this.checkpointNumber,
+      startBlock,
+      blockCount,
+      totalManaUsed: checkpoint.header.totalManaUsed.toBigInt(),
+      feeAssetPriceModifier,
+    });
+  }
+
   /** Runs fee analysis and logs checkpoint outcome as fisherman */
   private async handleCheckpointEndAsFisherman(checkpoint: Checkpoint | undefined) {
     // Perform L1 fee analysis before clearing requests
@@ -1290,12 +1609,13 @@ export class CheckpointProposalJob implements Traceable {
   protected async waitUntilTimeInSlot(targetSecondsIntoSlot: number): Promise<void> {
     const slotStartTimestamp = this.getSlotStartBuildTimestamp();
     const targetTimestamp = slotStartTimestamp + targetSecondsIntoSlot;
-    await sleepUntil(new Date(targetTimestamp * 1000), this.dateProvider.nowAsDate());
+    const delayMs = targetTimestamp * 1000 - this.dateProvider.nowAsDate().getTime();
+    await this.awaitInterruptibleSleep(delayMs);
   }
 
-  /** Waits the polling interval for transactions. Extracted for test overriding. */
+  /** Pause between mempool polls in waitForMinTxs. Extracted for test overriding. */
   protected async waitForTxsPollingInterval(): Promise<void> {
-    await sleep(TXS_POLLING_MS);
+    await this.awaitInterruptibleSleep(TXS_POLLING_MS);
   }
 
   private getSlotStartBuildTimestamp(): number {