@aztec/sequencer-client 0.0.1-commit.ef17749e1 → 0.0.1-commit.f1b29a41e

package/src/sequencer/checkpoint_proposal_job.ts

@@ -1,4 +1,5 @@
 import type { EpochCache } from '@aztec/epoch-cache';
+import { type FeeHeader, RollupContract } from '@aztec/ethereum/contracts';
 import {
   BlockNumber,
   CheckpointNumber,
@@ -30,17 +31,22 @@ import {
   type L2BlockSource,
   MaliciousCommitteeAttestationsAndSigners,
 } from '@aztec/stdlib/block';
-import { type Checkpoint, validateCheckpoint } from '@aztec/stdlib/checkpoint';
-import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
+import { type Checkpoint, type ProposedCheckpointData, validateCheckpoint } from '@aztec/stdlib/checkpoint';
+import { computeQuorum, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
 import { Gas } from '@aztec/stdlib/gas';
 import {
-  NoValidTxsError,
-  type PublicProcessorLimits,
+  type BlockBuilderOptions,
+  InsufficientValidTxsError,
   type ResolvedSequencerConfig,
   type WorldStateSynchronizer,
 } from '@aztec/stdlib/interfaces/server';
 import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
-import type { BlockProposalOptions, CheckpointProposal, CheckpointProposalOptions } from '@aztec/stdlib/p2p';
+import type {
+  BlockProposal,
+  BlockProposalOptions,
+  CheckpointProposal,
+  CheckpointProposalOptions,
+} from '@aztec/stdlib/p2p';
 import { orderAttestations, trimAttestations } from '@aztec/stdlib/p2p';
 import type { L2BlockBuiltStats } from '@aztec/stdlib/stats';
 import { type FailedTx, Tx } from '@aztec/stdlib/tx';
@@ -62,6 +68,13 @@ import { SequencerState } from './utils.js';
 /** How much time to sleep while waiting for min transactions to accumulate for a block */
 const TXS_POLLING_MS = 500;
 
+/** Result from proposeCheckpoint when a checkpoint was successfully built and attested. */
+type CheckpointProposalResult = {
+  checkpoint: Checkpoint;
+  attestations: CommitteeAttestationsAndSigners;
+  attestationsSignature: Signature;
+};
+
 /**
  * Handles the execution of a checkpoint proposal after the initial preparation phase.
  * This includes building blocks, collecting attestations, and publishing the checkpoint to L1,
@@ -71,9 +84,16 @@ const TXS_POLLING_MS = 500;
 export class CheckpointProposalJob implements Traceable {
   protected readonly log: Logger;
 
+  /** Tracks the fire-and-forget L1 submission promise so it can be awaited during shutdown. */
+  private pendingL1Submission: Promise<void> | undefined;
+
+  /** Fee header override computed during proposeCheckpoint, reused in enqueueCheckpointForSubmission. */
+  private computedForceProposedFeeHeader?: { checkpointNumber: CheckpointNumber; feeHeader: FeeHeader };
+
   constructor(
-    private readonly epoch: EpochNumber,
-    private readonly slot: SlotNumber,
+    private readonly slotNow: SlotNumber,
+    private readonly targetSlot: SlotNumber,
+    private readonly targetEpoch: EpochNumber,
     private readonly checkpointNumber: CheckpointNumber,
     private readonly syncedToBlockNumber: BlockNumber,
     // TODO(palla/mbps): Can we remove the proposer in favor of attestorAddress? Need to check fisherman-node flows.
@@ -100,13 +120,25 @@ export class CheckpointProposalJob implements Traceable {
     private readonly setStateFn: (state: SequencerState, slot?: SlotNumber) => void,
     public readonly tracer: Tracer,
     bindings?: LoggerBindings,
+    private readonly proposedCheckpointData?: ProposedCheckpointData,
   ) {
-    this.log = createLogger('sequencer:checkpoint-proposal', { ...bindings, instanceId: `slot-${slot}` });
+    this.log = createLogger('sequencer:checkpoint-proposal', {
+      ...bindings,
+      instanceId: `slot-${this.slotNow}`,
+    });
+  }
+
+  /** Awaits the pending L1 submission if one is in progress. Call during shutdown. */
+  public async awaitPendingSubmission(): Promise<void> {
+    this.log.info('Awaiting pending L1 payload submission');
+    await this.pendingL1Submission;
   }
 
   /**
    * Executes the checkpoint proposal job.
-   * Returns the published checkpoint if successful, undefined otherwise.
+   * Builds blocks, collects attestations, enqueues requests, and schedules L1 submission as a
+   * background task so the work loop can return to IDLE immediately.
+   * Returns the built checkpoint if successful, undefined otherwise.
    */
   @trackSpan('CheckpointProposalJob.execute')
   public async execute(): Promise<Checkpoint | undefined> {
@@ -114,7 +146,7 @@ export class CheckpointProposalJob implements Traceable {
     // In fisherman mode, we simulate slashing but don't actually publish to L1
     // These are constant for the whole slot, so we only enqueue them once
     const votesPromises = new CheckpointVoter(
-      this.slot,
+      this.targetSlot,
       this.publisher,
       this.attestorAddress,
       this.validatorClient,
@@ -125,8 +157,10 @@ export class CheckpointProposalJob implements Traceable {
       this.log,
     ).enqueueVotes();
 
-    // Build and propose the checkpoint. This will enqueue the request on the publisher if a checkpoint is built.
-    const checkpoint = await this.proposeCheckpoint();
+    // Build and propose the checkpoint. Builds blocks, broadcasts, collects attestations, and signs.
+    // Does NOT enqueue to L1 yet — that happens after the pipeline sleep.
+    const proposalResult = await this.proposeCheckpoint();
+    const checkpoint = proposalResult?.checkpoint;
 
     // Wait until the voting promises have resolved, so all requests are enqueued (not sent)
     await Promise.all(votesPromises);
@@ -141,47 +175,135 @@ export class CheckpointProposalJob implements Traceable {
       return;
     }
 
-    // Then send everything to L1
-    const l1Response = await this.publisher.sendRequests();
-    const proposedAction = l1Response?.successfulActions.find(a => a === 'propose');
-    if (proposedAction) {
-      this.eventEmitter.emit('checkpoint-published', { checkpoint: this.checkpointNumber, slot: this.slot });
-      const coinbase = checkpoint?.header.coinbase;
-      await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
-      return checkpoint;
-    } else if (checkpoint) {
-      this.eventEmitter.emit('checkpoint-publish-failed', { ...l1Response, slot: this.slot });
-      return undefined;
+    // Enqueue the checkpoint for L1 submission
+    if (proposalResult) {
+      try {
+        await this.enqueueCheckpointForSubmission(proposalResult);
+      } catch (err) {
+        this.log.error(`Failed to enqueue checkpoint for L1 submission at slot ${this.targetSlot}`, err);
+        // Continue to sendRequestsAt so votes are still sent
+      }
     }
+
+    // Compute the earliest time to submit: pipeline slot start when pipelining, now otherwise.
+    const submitAfter = this.epochCache.isProposerPipeliningEnabled()
+      ? new Date(Number(getTimestampForSlot(this.targetSlot, this.l1Constants)) * 1000)
+      : new Date(this.dateProvider.now());
+
+    // TODO(https://github.com/AztecProtocol/aztec-packages/pull/21250): should discard the pending submission if a reorg occurs underneath
+
+    // Schedule L1 submission in the background so the work loop returns immediately.
+    // The publisher will sleep until submitAfter, then send the bundled requests.
+    // The promise is stored so it can be awaited during shutdown.
+    this.pendingL1Submission = this.publisher
+      .sendRequestsAt(submitAfter)
+      .then(async l1Response => {
+        const proposedAction = l1Response?.successfulActions.find(a => a === 'propose');
+        if (proposedAction) {
+          this.eventEmitter.emit('checkpoint-published', { checkpoint: this.checkpointNumber, slot: this.targetSlot });
+          const coinbase = checkpoint?.header.coinbase;
+          await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
+        } else if (checkpoint) {
+          this.eventEmitter.emit('checkpoint-publish-failed', { ...l1Response, slot: this.targetSlot });
+
+          if (this.epochCache.isProposerPipeliningEnabled()) {
+            this.metrics.recordPipelineDiscard();
+          }
+        }
+      })
+      .catch(err => {
+        this.log.error(`Background L1 submission failed for slot ${this.targetSlot}`, err);
+        if (checkpoint) {
+          this.eventEmitter.emit('checkpoint-publish-failed', { slot: this.targetSlot });
+
+          if (this.epochCache.isProposerPipeliningEnabled()) {
+            this.metrics.recordPipelineDiscard();
+          }
+        }
+      });
+
+    // Return the built checkpoint immediately — the work loop is now unblocked
+    return checkpoint;
+  }
+
+  /** Enqueues the checkpoint for L1 submission. Called after pipeline sleep in execute(). */
+  private async enqueueCheckpointForSubmission(result: CheckpointProposalResult): Promise<void> {
+    const { checkpoint, attestations, attestationsSignature } = result;
+
+    this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.targetSlot);
+    const aztecSlotDuration = this.l1Constants.slotDuration;
+    const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
+    const txTimeoutAt = new Date((submissionSlotStart + aztecSlotDuration) * 1000);
+
+    // If we have been configured to potentially skip publishing checkpoint then roll the dice here
+    if (
+      this.config.skipPublishingCheckpointsPercent !== undefined &&
+      this.config.skipPublishingCheckpointsPercent > 0
+    ) {
+      const roll = Math.max(0, randomInt(100));
+      if (roll < this.config.skipPublishingCheckpointsPercent) {
+        this.log.warn(
+          `Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${roll}`,
+        );
+        return;
+      }
+    }
+
+    await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
+      txTimeoutAt,
+      forcePendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
+      forceProposedFeeHeader: this.computedForceProposedFeeHeader,
+    });
   }
 
   @trackSpan('CheckpointProposalJob.proposeCheckpoint', function () {
     return {
       // nullish operator needed for tests
       [Attributes.COINBASE]: this.validatorClient.getCoinbaseForAttestor(this.attestorAddress)?.toString(),
-      [Attributes.SLOT_NUMBER]: this.slot,
+      [Attributes.SLOT_NUMBER]: this.targetSlot,
     };
   })
-  private async proposeCheckpoint(): Promise<Checkpoint | undefined> {
+  private async proposeCheckpoint(): Promise<CheckpointProposalResult | undefined> {
     try {
       // Get operator configured coinbase and fee recipient for this attestor
       const coinbase = this.validatorClient.getCoinbaseForAttestor(this.attestorAddress);
       const feeRecipient = this.validatorClient.getFeeRecipientForAttestor(this.attestorAddress);
 
       // Start the checkpoint
-      this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.slot);
-      this.metrics.incOpenSlot(this.slot, this.proposer?.toString() ?? 'unknown');
+      this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.targetSlot);
+      this.log.info(`Starting checkpoint proposal`, {
+        buildSlot: this.slotNow,
+        submissionSlot: this.targetSlot,
+        pipelining: this.epochCache.isProposerPipeliningEnabled(),
+        proposer: this.proposer?.toString(),
+        coinbase: coinbase.toString(),
+      });
+      this.metrics.incOpenSlot(this.targetSlot, this.proposer?.toString() ?? 'unknown');
 
       // Enqueues checkpoint invalidation (constant for the whole slot)
       if (this.invalidateCheckpoint && !this.config.skipInvalidateBlockAsProposer) {
         this.publisher.enqueueInvalidateCheckpoint(this.invalidateCheckpoint);
       }
 
-      // Create checkpoint builder for the slot
+      // Create checkpoint builder for the slot.
+      // When pipelining, force the proposed checkpoint number and fee header to our parent so the
+      // fee computation sees the same chain tip that L1 will see once the previous pipelined checkpoint lands.
+      const isPipelining = this.epochCache.isProposerPipeliningEnabled();
+      const parentCheckpointNumber = isPipelining ? CheckpointNumber(this.checkpointNumber - 1) : undefined;
+
+      // Compute the parent's fee header override when pipelining
+      if (isPipelining && this.proposedCheckpointData) {
+        this.computedForceProposedFeeHeader = await this.computeForceProposedFeeHeader(parentCheckpointNumber!);
+      }
+
       const checkpointGlobalVariables = await this.globalsBuilder.buildCheckpointGlobalVariables(
         coinbase,
         feeRecipient,
-        this.slot,
+        this.targetSlot,
+        {
+          forcePendingCheckpointNumber: parentCheckpointNumber,
+          forceProposedFeeHeader: this.computedForceProposedFeeHeader,
+        },
       );
 
       // Collect L1 to L2 messages for the checkpoint and compute their hash
@@ -189,7 +311,7 @@ export class CheckpointProposalJob implements Traceable {
       const inHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
 
       // Collect the out hashes of all the checkpoints before this one in the same epoch
-      const previousCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsDataForEpoch(this.epoch))
+      const previousCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsDataForEpoch(this.targetEpoch))
         .filter(c => c.checkpointNumber < this.checkpointNumber)
         .map(c => c.checkpointOutHash);
 
@@ -246,8 +368,8 @@ export class CheckpointProposalJob implements Traceable {
       }
 
       if (blocksInCheckpoint.length === 0) {
-        this.log.warn(`No blocks were built for slot ${this.slot}`, { slot: this.slot });
-        this.eventEmitter.emit('checkpoint-empty', { slot: this.slot });
+        this.log.warn(`No blocks were built for slot ${this.targetSlot}`, { slot: this.targetSlot });
+        this.eventEmitter.emit('checkpoint-empty', { slot: this.targetSlot });
         return undefined;
       }
 
@@ -255,17 +377,18 @@ export class CheckpointProposalJob implements Traceable {
       if (minBlocksForCheckpoint !== undefined && blocksInCheckpoint.length < minBlocksForCheckpoint) {
         this.log.warn(
           `Checkpoint has fewer blocks than minimum (${blocksInCheckpoint.length} < ${minBlocksForCheckpoint}), skipping proposal`,
-          { slot: this.slot, blocksBuilt: blocksInCheckpoint.length, minBlocksForCheckpoint },
+          { slot: this.targetSlot, blocksBuilt: blocksInCheckpoint.length, minBlocksForCheckpoint },
         );
         return undefined;
       }
 
       // Assemble and broadcast the checkpoint proposal, including the last block that was not
       // broadcasted yet, and wait to collect the committee attestations.
-      this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.slot);
+      this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.targetSlot);
       const checkpoint = await checkpointBuilder.completeCheckpoint();
 
-      // Final validation round for the checkpoint before we propose it, just for safety
+      // Final validation: per-block limits are only checked if the operator set them explicitly.
+      // Otherwise, checkpoint-level budgets were already enforced by the redistribution logic.
       try {
         validateCheckpoint(checkpoint, {
           rollupManaLimit: this.l1Constants.rollupManaLimit,
@@ -275,7 +398,7 @@ export class CheckpointProposalJob implements Traceable {
           maxTxsPerCheckpoint: this.config.maxTxsPerCheckpoint,
         });
       } catch (err) {
-        this.log.error(`Built an invalid checkpoint at slot ${this.slot} (skipping proposal)`, err, {
+        this.log.error(`Built an invalid checkpoint at slot ${this.slotNow} (skipping proposal)`, err, {
           checkpoint: checkpoint.header.toInspect(),
         });
         return undefined;
@@ -292,16 +415,20 @@ export class CheckpointProposalJob implements Traceable {
       // Do not collect attestations nor publish to L1 in fisherman mode
       if (this.config.fishermanMode) {
         this.log.info(
-          `Built checkpoint for slot ${this.slot} with ${blocksInCheckpoint.length} blocks. ` +
+          `Built checkpoint for slot ${this.targetSlot} with ${blocksInCheckpoint.length} blocks. ` +
             `Skipping proposal in fisherman mode.`,
           {
-            slot: this.slot,
+            slot: this.targetSlot,
             checkpoint: checkpoint.header.toInspect(),
             blocksBuilt: blocksInCheckpoint.length,
           },
         );
         this.metrics.recordCheckpointSuccess();
-        return checkpoint;
+        return {
+          checkpoint,
+          attestations: CommitteeAttestationsAndSigners.empty(),
+          attestationsSignature: Signature.empty(),
+        };
       }
 
       // Include the block pending broadcast in the checkpoint proposal if any
@@ -324,7 +451,7 @@ export class CheckpointProposalJob implements Traceable {
       const blockProposedAt = this.dateProvider.now();
       await this.p2pClient.broadcastCheckpointProposal(proposal);
 
-      this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.slot);
+      this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot);
       const attestations = await this.waitForAttestations(proposal);
       const blockAttestedAt = this.dateProvider.now();
 
@@ -337,7 +464,7 @@ export class CheckpointProposalJob implements Traceable {
         attestationsSignature = await this.validatorClient.signAttestationsAndSigners(
           attestations,
           signer,
-          this.slot,
+          this.targetSlot,
           this.checkpointNumber,
         );
       } catch (err) {
@@ -349,39 +476,15 @@ export class CheckpointProposalJob implements Traceable {
         throw err;
       }
 
-      // Enqueue publishing the checkpoint to L1
-      this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.slot);
-      const aztecSlotDuration = this.l1Constants.slotDuration;
-      const slotStartBuildTimestamp = this.getSlotStartBuildTimestamp();
-      const txTimeoutAt = new Date((slotStartBuildTimestamp + aztecSlotDuration) * 1000);
-
-      // If we have been configured to potentially skip publishing checkpoint then roll the dice here
-      if (
-        this.config.skipPublishingCheckpointsPercent !== undefined &&
-        this.config.skipPublishingCheckpointsPercent > 0
-      ) {
-        const result = Math.max(0, randomInt(100));
-        if (result < this.config.skipPublishingCheckpointsPercent) {
-          this.log.warn(
-            `Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${result}`,
-          );
-          return checkpoint;
-        }
-      }
-
-      await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
-        txTimeoutAt,
-        forcePendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
-      });
-
-      return checkpoint;
+      // Return the result for the caller to enqueue after the pipeline sleep
+      return { checkpoint, attestations, attestationsSignature };
     } catch (err) {
       if (err && (err instanceof DutyAlreadySignedError || err instanceof SlashingProtectionError)) {
         // swallow this error. It's already been logged by a function deeper in the stack
         return undefined;
       }
 
-      this.log.error(`Error building checkpoint at slot ${this.slot}`, err);
+      this.log.error(`Error building checkpoint at slot ${this.targetSlot}`, err);
       return undefined;
     }
   }
@@ -416,7 +519,7 @@ export class CheckpointProposalJob implements Traceable {
 
       if (!timingInfo.canStart) {
         this.log.debug(`Not enough time left in slot to start another block`, {
-          slot: this.slot,
+          slot: this.targetSlot,
           blocksBuilt,
           secondsIntoSlot,
         });
@@ -451,8 +554,8 @@ export class CheckpointProposalJob implements Traceable {
       } else if ('error' in buildResult) {
         // If there was an error building the block, just exit the loop and give up the rest of the slot
         if (!(buildResult.error instanceof SequencerInterruptedError)) {
-          this.log.warn(`Halting block building for slot ${this.slot}`, {
-            slot: this.slot,
+          this.log.warn(`Halting block building for slot ${this.targetSlot}`, {
+            slot: this.targetSlot,
             blocksBuilt,
             error: buildResult.error,
           });
@@ -462,21 +565,14 @@ export class CheckpointProposalJob implements Traceable {
 
       const { block, usedTxs } = buildResult;
       blocksInCheckpoint.push(block);
-
-      // Sync the proposed block to the archiver to make it available
-      // Note that the checkpoint builder uses its own fork so it should not need to wait for this syncing
-      // Eventually we should refactor the checkpoint builder to not need a separate long-lived fork
-      // Fire and forget - don't block the critical path, but log errors
-      this.syncProposedBlockToArchiver(block).catch(err => {
-        this.log.error(`Failed to sync proposed block ${block.number} to archiver`, { blockNumber: block.number, err });
-      });
-
       usedTxs.forEach(tx => txHashesAlreadyIncluded.add(tx.txHash.toString()));
 
-      // If this is the last block, exit the loop now so we start collecting attestations
+      // If this is the last block, sync it to the archiver and exit the loop
+      // so we can build the checkpoint and start collecting attestations.
       if (timingInfo.isLastBlock) {
-        this.log.verbose(`Completed final block ${blockNumber} for slot ${this.slot}`, {
-          slot: this.slot,
+        await this.syncProposedBlockToArchiver(block);
+        this.log.verbose(`Completed final block ${blockNumber} for slot ${this.targetSlot}`, {
+          slot: this.targetSlot,
           blockNumber,
           blocksBuilt,
         });
@@ -484,38 +580,61 @@ export class CheckpointProposalJob implements Traceable {
         break;
       }
 
-      // For non-last blocks, broadcast the block proposal (unless we're in fisherman mode)
-      // If the block is the last one, we'll broadcast it along with the checkpoint at the end of the loop
-      if (!this.config.fishermanMode) {
-        const proposal = await this.validatorClient.createBlockProposal(
-          block.header,
-          block.indexWithinCheckpoint,
-          inHash,
-          block.archive.root,
-          usedTxs,
-          this.proposer,
-          blockProposalOptions,
-        );
-        await this.p2pClient.broadcastProposal(proposal);
-      }
+      // Broadcast the block proposal (unless we're in fisherman mode) unless the block is the last one,
+      // in which case we'll broadcast it along with the checkpoint at the end of the loop.
+      // Note that we only send the block to the archiver if we manage to create the proposal, so if there's
+      // a HA error we don't pollute our archiver with a block that won't make it to the chain.
+      const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions);
+
+      // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal.
+      // We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
+      // If this throws, we abort the entire checkpoint.
+      await this.syncProposedBlockToArchiver(block);
+
+      // Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it.
+      proposal && (await this.p2pClient.broadcastProposal(proposal));
 
       // Wait until the next block's start time
       await this.waitUntilNextSubslot(timingInfo.deadline);
     }
 
-    this.log.verbose(`Block building loop completed for slot ${this.slot}`, {
-      slot: this.slot,
+    this.log.verbose(`Block building loop completed for slot ${this.targetSlot}`, {
+      slot: this.targetSlot,
       blocksBuilt: blocksInCheckpoint.length,
     });
 
     return { blocksInCheckpoint, blockPendingBroadcast };
   }
 
+  /** Creates a block proposal for a given block via the validator client (unless in fisherman mode) */
+  private createBlockProposal(
+    block: L2Block,
+    inHash: Fr,
+    usedTxs: Tx[],
+    blockProposalOptions: BlockProposalOptions,
+  ): Promise<BlockProposal | undefined> {
+    if (this.config.fishermanMode) {
+      this.log.info(`Skipping block proposal for block ${block.number} in fisherman mode`);
+      return Promise.resolve(undefined);
+    }
+    return this.validatorClient.createBlockProposal(
+      block.header,
+      block.indexWithinCheckpoint,
+      inHash,
+      block.archive.root,
+      usedTxs,
+      this.proposer,
+      blockProposalOptions,
+    );
+  }
+
   /** Sleeps until it is time to produce the next block in the slot */
   @trackSpan('CheckpointProposalJob.waitUntilNextSubslot')
   private async waitUntilNextSubslot(nextSubslotStart: number) {
-    this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.slot);
-    this.log.verbose(`Waiting until time for the next block at ${nextSubslotStart}s into slot`, { slot: this.slot });
+    this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.targetSlot);
+    this.log.verbose(`Waiting until time for the next block at ${nextSubslotStart}s into slot`, {
+      slot: this.targetSlot,
+    });
     await this.waitUntilTimeInSlot(nextSubslotStart);
   }
 
@@ -536,20 +655,19 @@ export class CheckpointProposalJob implements Traceable {
       opts;
 
     this.log.verbose(
-      `Preparing block ${blockNumber} index ${indexWithinCheckpoint} at checkpoint ${this.checkpointNumber} for slot ${this.slot}`,
+      `Preparing block ${blockNumber} index ${indexWithinCheckpoint} at checkpoint ${this.checkpointNumber} for slot ${this.targetSlot}`,
       { ...checkpointBuilder.getConstantData(), ...opts },
     );
 
     try {
       // Wait until we have enough txs to build the block
-      const minTxs = this.config.minTxsPerBlock;
-      const { availableTxs, canStartBuilding } = await this.waitForMinTxs(opts);
+      const { availableTxs, canStartBuilding, minTxs } = await this.waitForMinTxs(opts);
       if (!canStartBuilding) {
         this.log.warn(
-          `Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.slot} (got ${availableTxs} txs but needs ${minTxs})`,
-          { blockNumber, slot: this.slot, indexWithinCheckpoint },
+          `Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (got ${availableTxs} txs but needs ${minTxs})`,
+          { blockNumber, slot: this.targetSlot, indexWithinCheckpoint },
         );
-        this.eventEmitter.emit('block-tx-count-check-failed', { minTxs, availableTxs, slot: this.slot });
+        this.eventEmitter.emit('block-tx-count-check-failed', { minTxs, availableTxs, slot: this.targetSlot });
         this.metrics.recordBlockProposalFailed('insufficient_txs');
         return undefined;
       }
@@ -562,14 +680,16 @@ export class CheckpointProposalJob implements Traceable {
       );
 
       this.log.debug(
-        `Building block ${blockNumber} at index ${indexWithinCheckpoint} for slot ${this.slot} with ${availableTxs} available txs`,
-        { slot: this.slot, blockNumber, indexWithinCheckpoint },
+        `Building block ${blockNumber} at index ${indexWithinCheckpoint} for slot ${this.targetSlot} with ${availableTxs} available txs`,
+        { slot: this.targetSlot, blockNumber, indexWithinCheckpoint },
       );
-      this.setStateFn(SequencerState.CREATING_BLOCK, this.slot);
+      this.setStateFn(SequencerState.CREATING_BLOCK, this.targetSlot);
 
-      // Per-block limits derived at startup by computeBlockLimits(), further capped
+      // Per-block limits are operator overrides (from SEQ_MAX_L2_BLOCK_GAS etc.) further capped
       // by remaining checkpoint-level budgets inside CheckpointBuilder before each block is built.
-      const blockBuilderOptions: PublicProcessorLimits = {
+      // minValidTxs is passed into the builder so it can reject the block *before* updating state.
+      const minValidTxs = forceCreate ? 0 : (this.config.minValidTxsPerBlock ?? minTxs);
+      const blockBuilderOptions: BlockBuilderOptions = {
         maxTransactions: this.config.maxTxsPerBlock,
         maxBlockGas:
           this.config.maxL2BlockGas !== undefined || this.config.maxDABlockGas !== undefined
@@ -577,9 +697,14 @@ export class CheckpointProposalJob implements Traceable {
             : undefined,
         deadline: buildDeadline,
         isBuildingProposal: true,
+        minValidTxs,
+        maxBlocksPerCheckpoint: this.timetable.maxNumberOfBlocks,
+        perBlockAllocationMultiplier: this.config.perBlockAllocationMultiplier,
       };
 
-      // Actually build the block by executing txs
+      // Actually build the block by executing txs. The builder throws InsufficientValidTxsError
+      // if the number of successfully processed txs is below minValidTxs, ensuring state is not
+      // updated for blocks that will be discarded.
       const buildResult = await this.buildSingleBlockWithCheckpointBuilder(
         checkpointBuilder,
         pendingTxs,
@@ -591,22 +716,27 @@ export class CheckpointProposalJob implements Traceable {
       // If any txs failed during execution, drop them from the mempool so we don't pick them up again
       await this.dropFailedTxsFromP2P(buildResult.failedTxs);
 
-      // Check if we have created a block with enough txs. If there were invalid txs in the pool, or if execution took
-      // too long, then we may not get to minTxsPerBlock after executing public functions.
-      const minValidTxs = this.config.minValidTxsPerBlock ?? minTxs;
-      const numTxs = buildResult.status === 'no-valid-txs' ? 0 : buildResult.numTxs;
-      if (buildResult.status === 'no-valid-txs' || (!forceCreate && numTxs < minValidTxs)) {
+      if (buildResult.status === 'insufficient-valid-txs') {
         this.log.warn(
-          `Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.slot} has too few valid txs to be proposed`,
-          { slot: this.slot, blockNumber, numTxs, indexWithinCheckpoint, minValidTxs, buildResult: buildResult.status },
+          `Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.targetSlot} has too few valid txs to be proposed`,
+          {
+            slot: this.targetSlot,
+            blockNumber,
+            numTxs: buildResult.processedCount,
+            indexWithinCheckpoint,
+            minValidTxs,
+          },
         );
-        this.eventEmitter.emit('block-build-failed', { reason: `Insufficient valid txs`, slot: this.slot });
+        this.eventEmitter.emit('block-build-failed', {
+          reason: `Insufficient valid txs`,
+          slot: this.targetSlot,
+        });
         this.metrics.recordBlockProposalFailed('insufficient_valid_txs');
         return undefined;
       }
 
       // Block creation succeeded, emit stats and metrics
-      const { block, publicProcessorDuration, usedTxs, blockBuildDuration } = buildResult;
+      const { block, publicProcessorDuration, usedTxs, blockBuildDuration, numTxs } = buildResult;
 
       const blockStats = {
         eventName: 'l2-block-built',
@@ -620,30 +750,39 @@ export class CheckpointProposalJob implements Traceable {
       const manaPerSec = block.header.totalManaUsed.toNumberUnsafe() / (blockBuildDuration / 1000);
 
       this.log.info(
-        `Built block ${block.number} at checkpoint ${this.checkpointNumber} for slot ${this.slot} with ${numTxs} txs`,
+        `Built block ${block.number} at checkpoint ${this.checkpointNumber} for slot ${this.targetSlot} with ${numTxs} txs`,
         { blockHash, txHashes, manaPerSec, ...blockStats },
       );
 
-      this.eventEmitter.emit('block-proposed', { blockNumber: block.number, slot: this.slot });
+      // `slot` is the target/submission slot (may be one ahead when pipelining),
+      // `buildSlot` is the wall-clock slot during which the block was actually built.
+      this.eventEmitter.emit('block-proposed', {
+        blockNumber: block.number,
+        slot: this.targetSlot,
+        buildSlot: this.slotNow,
+      });
       this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe());
 
       return { block, usedTxs };
     } catch (err: any) {
-      this.eventEmitter.emit('block-build-failed', { reason: err.message, slot: this.slot });
-      this.log.error(`Error building block`, err, { blockNumber, slot: this.slot });
+      this.eventEmitter.emit('block-build-failed', {
+        reason: err.message,
+        slot: this.targetSlot,
+      });
+      this.log.error(`Error building block`, err, { blockNumber, slot: this.targetSlot });
       this.metrics.recordBlockProposalFailed(err.name || 'unknown_error');
       this.metrics.recordFailedBlock();
       return { error: err };
     }
   }
 
-  /** Uses the checkpoint builder to build a block, catching specific txs */
+  /** Uses the checkpoint builder to build a block, catching InsufficientValidTxsError. */
   private async buildSingleBlockWithCheckpointBuilder(
     checkpointBuilder: CheckpointBuilder,
     pendingTxs: AsyncIterable<Tx>,
     blockNumber: BlockNumber,
     blockTimestamp: bigint,
-    blockBuilderOptions: PublicProcessorLimits,
+    blockBuilderOptions: BlockBuilderOptions,
   ) {
     try {
       const workTimer = new Timer();
@@ -651,8 +790,12 @@ export class CheckpointProposalJob implements Traceable {
       const blockBuildDuration = workTimer.ms();
       return { ...result, blockBuildDuration, status: 'success' as const };
     } catch (err: unknown) {
-      if (isErrorClass(err, NoValidTxsError)) {
-        return { failedTxs: err.failedTxs, status: 'no-valid-txs' as const };
+      if (isErrorClass(err, InsufficientValidTxsError)) {
+        return {
+          failedTxs: err.failedTxs,
+          processedCount: err.processedCount,
+          status: 'insufficient-valid-txs' as const,
+        };
       }
       throw err;
     }
@@ -665,7 +808,7 @@ export class CheckpointProposalJob implements Traceable {
     blockNumber: BlockNumber;
     indexWithinCheckpoint: IndexWithinCheckpoint;
     buildDeadline: Date | undefined;
-  }): Promise<{ canStartBuilding: boolean; availableTxs: number }> {
+  }): Promise<{ canStartBuilding: boolean; availableTxs: number; minTxs: number }> {
     const { indexWithinCheckpoint, blockNumber, buildDeadline, forceCreate } = opts;
 
     // We only allow a block with 0 txs in the first block of the checkpoint
@@ -682,20 +825,20 @@ export class CheckpointProposalJob implements Traceable {
       // If we're past deadline, or we have no deadline, give up
       const now = this.dateProvider.nowAsDate();
       if (startBuildingDeadline === undefined || now >= startBuildingDeadline) {
-        return { canStartBuilding: false, availableTxs: availableTxs };
+        return { canStartBuilding: false, availableTxs, minTxs };
       }
 
       // Wait a bit before checking again
-      this.setStateFn(SequencerState.WAITING_FOR_TXS, this.slot);
+      this.setStateFn(SequencerState.WAITING_FOR_TXS, this.targetSlot);
       this.log.verbose(
-        `Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.slot} (have ${availableTxs} but need ${minTxs})`,
-        { blockNumber, slot: this.slot, indexWithinCheckpoint },
+        `Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (have ${availableTxs} but need ${minTxs})`,
+        { blockNumber, slot: this.targetSlot, indexWithinCheckpoint },
       );
       await this.waitForTxsPollingInterval();
       availableTxs = await this.p2pClient.getPendingTxCount();
     }
 
-    return { canStartBuilding: true, availableTxs };
+    return { canStartBuilding: true, availableTxs, minTxs };
   }
 
   /**
@@ -721,7 +864,7 @@ export class CheckpointProposalJob implements Traceable {
       this.log.debug(`Attesting committee length is ${committee.length}`, { committee });
     }
 
-    const numberOfRequiredAttestations = Math.floor((committee.length * 2) / 3) + 1;
+    const numberOfRequiredAttestations = computeQuorum(committee.length);
 
     if (this.config.skipCollectingAttestations) {
       this.log.warn('Skipping attestation collection as per config (attesting with own keys only)');
@@ -871,7 +1014,7 @@ export class CheckpointProposalJob implements Traceable {
    * would never receive its own block without this explicit sync.
    */
   private async syncProposedBlockToArchiver(block: L2Block): Promise<void> {
-    if (this.config.skipPushProposedBlocksToArchiver !== false) {
+    if (this.config.skipPushProposedBlocksToArchiver) {
       this.log.warn(`Skipping push of proposed block ${block.number} to archiver`, {
         blockNumber: block.number,
         slot: block.header.globalVariables.slotNumber,
@@ -889,19 +1032,19 @@ export class CheckpointProposalJob implements Traceable {
   private async handleCheckpointEndAsFisherman(checkpoint: Checkpoint | undefined) {
     // Perform L1 fee analysis before clearing requests
     // The callback is invoked asynchronously after the next block is mined
-    const feeAnalysis = await this.publisher.analyzeL1Fees(this.slot, analysis =>
+    const feeAnalysis = await this.publisher.analyzeL1Fees(this.targetSlot, analysis =>
       this.metrics.recordFishermanFeeAnalysis(analysis),
     );
 
     if (checkpoint) {
-      this.log.info(`Validation checkpoint building SUCCEEDED for slot ${this.slot}`, {
+      this.log.info(`Validation checkpoint building SUCCEEDED for slot ${this.targetSlot}`, {
         ...checkpoint.toCheckpointInfo(),
         ...checkpoint.getStats(),
         feeAnalysisId: feeAnalysis?.id,
       });
     } else {
-      this.log.warn(`Validation block building FAILED for slot ${this.slot}`, {
-        slot: this.slot,
+      this.log.warn(`Validation block building FAILED for slot ${this.targetSlot}`, {
+        slot: this.targetSlot,
         feeAnalysisId: feeAnalysis?.id,
       });
       this.metrics.recordCheckpointProposalFailed('block_build_failed');
@@ -915,15 +1058,15 @@ export class CheckpointProposalJob implements Traceable {
    */
   private handleHASigningError(err: any, errorContext: string): boolean {
     if (err instanceof DutyAlreadySignedError) {
-      this.log.info(`${errorContext} for slot ${this.slot} already signed by another HA node, yielding`, {
-        slot: this.slot,
+      this.log.info(`${errorContext} for slot ${this.targetSlot} already signed by another HA node, yielding`, {
+        slot: this.targetSlot,
         signedByNode: err.signedByNode,
       });
       return true;
     }
     if (err instanceof SlashingProtectionError) {
-      this.log.info(`${errorContext} for slot ${this.slot} blocked by slashing protection, yielding`, {
-        slot: this.slot,
+      this.log.info(`${errorContext} for slot ${this.targetSlot} blocked by slashing protection, yielding`, {
+        slot: this.targetSlot,
         existingMessageHash: err.existingMessageHash,
         attemptedMessageHash: err.attemptedMessageHash,
       });
@@ -932,6 +1075,56 @@ export class CheckpointProposalJob implements Traceable {
     return false;
   }
 
+  /**
+   * In times of congestion we need to simulate using the correct fee header override for the previous block
+   * We calculate the correct fee header values.
+   *
+   * If we are in block 1, or the checkpoint we are querying does not exist, we return undefined. However
+   * If we are pipelining - where this function is called, the grandparentCheckpointNumber should always exist
+   * @param parentCheckpointNumber
+   * @returns
+   */
+  protected async computeForceProposedFeeHeader(parentCheckpointNumber: CheckpointNumber): Promise<
+    | {
+        checkpointNumber: CheckpointNumber;
+        feeHeader: FeeHeader;
+      }
+    | undefined
+  > {
+    if (!this.proposedCheckpointData) {
+      return undefined;
+    }
+
+    const rollup = this.publisher.rollupContract;
+    const grandparentCheckpointNumber = CheckpointNumber(this.checkpointNumber - 2);
+    try {
+      const [grandparentCheckpoint, manaTarget] = await Promise.all([
+        rollup.getCheckpoint(grandparentCheckpointNumber),
+        rollup.getManaTarget(),
+      ]);
+
+      if (!grandparentCheckpoint || !grandparentCheckpoint.feeHeader) {
+        this.log.error(
+          `Grandparent checkpoint or its feeHeader is undefined for checkpointNumber=${grandparentCheckpointNumber.toString()}`,
+        );
+        return undefined;
+      } else {
+        const parentFeeHeader = RollupContract.computeChildFeeHeader(
+          grandparentCheckpoint.feeHeader,
+          this.proposedCheckpointData.totalManaUsed,
+          this.proposedCheckpointData.feeAssetPriceModifier,
+          manaTarget,
+        );
+        return { checkpointNumber: parentCheckpointNumber, feeHeader: parentFeeHeader };
+      }
+    } catch (err) {
+      this.log.error(
+        `Failed to fetch grandparent checkpoint or mana target for checkpointNumber=${grandparentCheckpointNumber.toString()}: ${err}`,
+      );
+      return undefined;
+    }
+  }
+
   /** Waits until a specific time within the current slot */
   @trackSpan('CheckpointProposalJob.waitUntilTimeInSlot')
   protected async waitUntilTimeInSlot(targetSecondsIntoSlot: number): Promise<void> {
@@ -946,7 +1139,7 @@ export class CheckpointProposalJob implements Traceable {
   }
 
   private getSlotStartBuildTimestamp(): number {
-    return getSlotStartBuildTimestamp(this.slot, this.l1Constants);
+    return getSlotStartBuildTimestamp(this.slotNow, this.l1Constants);
   }
 
   private getSecondsIntoSlot(): number {