@aztec/sequencer-client 0.0.1-commit.3f296a7d2 → 0.0.1-commit.3f5453c7b

package/dest/sequencer/checkpoint_proposal_job.js

@@ -446,7 +446,7 @@ import { Timer } from '@aztec/foundation/timer';
 import { isErrorClass, unfreeze } from '@aztec/foundation/types';
 import { CommitteeAttestationsAndSigners, MaliciousCommitteeAttestationsAndSigners } from '@aztec/stdlib/block';
 import { validateCheckpoint } from '@aztec/stdlib/checkpoint';
-import { getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
+import { computeQuorum, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
 import { Gas } from '@aztec/stdlib/gas';
 import { InsufficientValidTxsError } from '@aztec/stdlib/interfaces/server';
 import { computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
@@ -454,6 +454,7 @@ import { orderAttestations, trimAttestations } from '@aztec/stdlib/p2p';
 import { AttestationTimeoutError } from '@aztec/stdlib/validators';
 import { Attributes, trackSpan } from '@aztec/telemetry-client';
 import { DutyAlreadySignedError, SlashingProtectionError } from '@aztec/validator-ha-signer/errors';
+import { buildPipelinedParentSimulationOverridesPlan, buildSubmissionSimulationOverridesPlan } from './chain_state_overrides.js';
 import { CheckpointVoter } from './checkpoint_voter.js';
 import { SequencerInterruptedError } from './errors.js';
 import { SequencerState } from './utils.js';
@@ -473,7 +474,6 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
  */ export class CheckpointProposalJob {
     slotNow;
     targetSlot;
-    epochNow;
     targetEpoch;
     checkpointNumber;
     syncedToBlockNumber;
@@ -496,9 +496,11 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
     epochCache;
     dateProvider;
     metrics;
+    checkpointMetrics;
     eventEmitter;
     setStateFn;
     tracer;
+    proposedCheckpointData;
     static{
         ({ e: [_initProto] } = _apply_decs_2203_r(this, [
             [
@@ -544,11 +546,12 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
         ], []));
     }
     log;
-    constructor(slotNow, targetSlot, epochNow, targetEpoch, checkpointNumber, syncedToBlockNumber, // TODO(palla/mbps): Can we remove the proposer in favor of attestorAddress? Need to check fisherman-node flows.
-    proposer, publisher, attestorAddress, invalidateCheckpoint, validatorClient, globalsBuilder, p2pClient, worldState, l1ToL2MessageSource, l2BlockSource, checkpointsBuilder, blockSink, l1Constants, config, timetable, slasherClient, epochCache, dateProvider, metrics, eventEmitter, setStateFn, tracer, bindings){
+    /** Tracks the fire-and-forget L1 submission promise so it can be awaited during shutdown. */ pendingL1Submission;
+    /** Pipelined parent chain state used while building and later submitting this checkpoint. */ pipelinedParentSimulationOverridesPlan;
+    constructor(slotNow, targetSlot, targetEpoch, checkpointNumber, syncedToBlockNumber, // TODO(palla/mbps): Can we remove the proposer in favor of attestorAddress? Need to check fisherman-node flows.
+    proposer, publisher, attestorAddress, invalidateCheckpoint, validatorClient, globalsBuilder, p2pClient, worldState, l1ToL2MessageSource, l2BlockSource, checkpointsBuilder, blockSink, l1Constants, config, timetable, slasherClient, epochCache, dateProvider, metrics, checkpointMetrics, eventEmitter, setStateFn, tracer, bindings, proposedCheckpointData){
         this.slotNow = slotNow;
         this.targetSlot = targetSlot;
-        this.epochNow = epochNow;
         this.targetEpoch = targetEpoch;
         this.checkpointNumber = checkpointNumber;
         this.syncedToBlockNumber = syncedToBlockNumber;
@@ -571,77 +574,142 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
         this.epochCache = epochCache;
         this.dateProvider = dateProvider;
         this.metrics = metrics;
+        this.checkpointMetrics = checkpointMetrics;
         this.eventEmitter = eventEmitter;
         this.setStateFn = setStateFn;
         this.tracer = tracer;
+        this.proposedCheckpointData = proposedCheckpointData;
         _initProto(this);
         this.log = createLogger('sequencer:checkpoint-proposal', {
             ...bindings,
             instanceId: `slot-${this.slotNow}`
         });
     }
-    /** The wall-clock slot during which the proposer builds. */ get slot() {
-        return this.slotNow;
-    }
-    /** The wall-clock epoch. */ get epoch() {
-        return this.epochNow;
+    /** Awaits the pending L1 submission if one is in progress. Call during shutdown. */ async awaitPendingSubmission() {
+        this.log.info('Awaiting pending L1 payload submission');
+        await this.pendingL1Submission;
     }
     /**
    * Executes the checkpoint proposal job.
-   * Returns the published checkpoint if successful, undefined otherwise.
+   * Builds blocks, assembles checkpoint, and broadcasts the proposal (blocking).
+   * Attestation collection, signing, and L1 submission are backgrounded so the
+   * work loop can return to IDLE immediately for consecutive slot proposals.
+   * Returns the built checkpoint if successful, undefined otherwise.
    */ async execute() {
         // Enqueue governance and slashing votes (returns promises that will be awaited later)
         // In fisherman mode, we simulate slashing but don't actually publish to L1
         // These are constant for the whole slot, so we only enqueue them once
         const votesPromises = new CheckpointVoter(this.targetSlot, this.publisher, this.attestorAddress, this.validatorClient, this.slasherClient, this.l1Constants, this.config, this.metrics, this.log).enqueueVotes();
-        // Build and propose the checkpoint. This will enqueue the request on the publisher if a checkpoint is built.
-        const checkpoint = await this.proposeCheckpoint();
-        // Wait until the voting promises have resolved, so all requests are enqueued (not sent)
-        await Promise.all(votesPromises);
-        if (checkpoint) {
-            this.metrics.recordCheckpointProposalSuccess();
+        // Build blocks, assemble checkpoint, and broadcast proposal (BLOCKING).
+        // Returns after broadcast — attestation collection is deferred.
+        const broadcast = await this.proposeCheckpoint();
+        if (!broadcast) {
+            await Promise.all(votesPromises);
+            // Still submit votes even without a checkpoint
+            if (!this.config.fishermanMode) {
+                this.pendingL1Submission = this.publisher.sendRequestsAt(this.dateProvider.nowAsDate()).then(()=>{});
+            }
+            return undefined;
         }
+        const { checkpoint } = broadcast;
+        this.metrics.recordCheckpointProposalSuccess();
         // Do not post anything to L1 if we are fishermen, but do perform L1 fee analysis
         if (this.config.fishermanMode) {
             await this.handleCheckpointEndAsFisherman(checkpoint);
-            return;
+            return checkpoint;
         }
-        // If pipelining, wait until the submission slot so L1 recognizes the pipelined proposer
-        if (this.epochCache.isProposerPipeliningEnabled()) {
-            const submissionSlotTimestamp = getTimestampForSlot(this.targetSlot, this.l1Constants) - BigInt(this.l1Constants.ethereumSlotDuration);
-            this.log.info(`Waiting until submission slot ${this.targetSlot} for L1 submission`, {
-                slot: this.slot,
-                submissionSlot: this.targetSlot,
-                submissionSlotTimestamp
-            });
-            await sleepUntil(new Date(Number(submissionSlotTimestamp) * 1000), this.dateProvider.nowAsDate());
-            // After waking, verify the parent checkpoint wasn't pruned during the sleep.
-            // We check L1's pending tip directly instead of canProposeAt, which also validates the proposer
-            // identity and would fail because the timestamp resolves to a different slot's proposer.
-            const l1Tips = await this.publisher.rollupContract.getTips();
-            if (l1Tips.pending < this.checkpointNumber - 1) {
-                this.log.warn(`Parent checkpoint was pruned during pipelining sleep (L1 pending=${l1Tips.pending}, expected>=${this.checkpointNumber - 1}), skipping L1 submission for checkpoint ${this.checkpointNumber}`);
-                return undefined;
+        // Background the attestation → signing → L1 pipeline so the work loop is unblocked
+        this.pendingL1Submission = this.waitForAttestationsAndEnqueueSubmissionAsync(broadcast, votesPromises);
+        // Return the built checkpoint immediately — the work loop is now unblocked
+        return checkpoint;
+    }
+    /**
+   * Background pipeline: collects attestations, signs them, enqueues the checkpoint, and submits to L1.
+   * Runs as a fire-and-forget task stored in `pendingL1Submission` so the work loop is unblocked.
+   */ async waitForAttestationsAndEnqueueSubmissionAsync(broadcast, votesPromises) {
+        const { checkpoint, proposal, blockProposedAt } = broadcast;
+        try {
+            await Promise.all(votesPromises);
+            this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot);
+            const attestations = await this.waitForAttestations(proposal);
+            this.checkpointMetrics.recordCheckpointAttestationDelay(this.dateProvider.now() - blockProposedAt);
+            // Proposer must sign over the attestations before pushing them to L1
+            const signer = this.proposer ?? this.publisher.getSenderAddress();
+            let attestationsSignature;
+            try {
+                attestationsSignature = await this.validatorClient.signAttestationsAndSigners(attestations, signer, this.targetSlot, this.checkpointNumber);
+            } catch (err) {
+                if (this.handleHASigningError(err, 'Attestations signature')) {
+                    return;
+                }
+                throw err;
             }
-        }
-        // Then send everything to L1
-        const l1Response = await this.publisher.sendRequests();
-        const proposedAction = l1Response?.successfulActions.find((a)=>a === 'propose');
-        if (proposedAction) {
-            this.eventEmitter.emit('checkpoint-published', {
-                checkpoint: this.checkpointNumber,
-                slot: this.slot
+            // Enqueue the checkpoint for L1 submission
+            await this.enqueueCheckpointForSubmission({
+                checkpoint,
+                attestations,
+                attestationsSignature
             });
-            const coinbase = checkpoint?.header.coinbase;
-            await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
-            return checkpoint;
-        } else if (checkpoint) {
+            // Compute the earliest time to submit: pipeline slot start when pipelining, now otherwise.
+            const submitAfter = this.epochCache.isProposerPipeliningEnabled() ? new Date(Number(getTimestampForSlot(this.targetSlot, this.l1Constants)) * 1000) : new Date(this.dateProvider.now());
+            const l1Response = await this.publisher.sendRequestsAt(submitAfter);
+            const proposedAction = l1Response?.successfulActions.find((a)=>a === 'propose');
+            if (proposedAction) {
+                this.eventEmitter.emit('checkpoint-published', {
+                    checkpoint: this.checkpointNumber,
+                    slot: this.targetSlot
+                });
+                const coinbase = checkpoint.header.coinbase;
+                await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
+            } else {
+                this.eventEmitter.emit('checkpoint-publish-failed', {
+                    ...l1Response,
+                    slot: this.targetSlot
+                });
+                if (this.epochCache.isProposerPipeliningEnabled()) {
+                    this.metrics.recordPipelineDiscard();
+                }
+            }
+        } catch (err) {
+            if (err instanceof SequencerInterruptedError) {
+                return;
+            }
+            this.log.error(`Background attestation/L1 pipeline failed for slot ${this.targetSlot}`, err);
             this.eventEmitter.emit('checkpoint-publish-failed', {
-                ...l1Response,
-                slot: this.slot
+                slot: this.targetSlot
             });
-            return undefined;
+            if (this.epochCache.isProposerPipeliningEnabled()) {
+                this.metrics.recordPipelineDiscard();
+            }
+        }
+    }
+    /** Enqueues the checkpoint for L1 submission. Called after pipeline sleep in execute(). */ async enqueueCheckpointForSubmission(result) {
+        const { checkpoint, attestations, attestationsSignature } = result;
+        this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.targetSlot);
+        const aztecSlotDuration = this.l1Constants.slotDuration;
+        const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
+        const txTimeoutAt = new Date((submissionSlotStart + aztecSlotDuration) * 1000);
+        // If we have been configured to potentially skip publishing checkpoint then roll the dice here
+        if (this.config.skipPublishingCheckpointsPercent !== undefined && this.config.skipPublishingCheckpointsPercent > 0) {
+            const roll = Math.max(0, randomInt(100));
+            if (roll < this.config.skipPublishingCheckpointsPercent) {
+                this.log.warn(`Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${roll}`);
+                return;
+            }
         }
+        const isPipelining = this.epochCache.isProposerPipeliningEnabled();
+        const submissionSimulationOverridesPlan = buildSubmissionSimulationOverridesPlan({
+            pipelinedParentPlan: this.pipelinedParentSimulationOverridesPlan,
+            invalidateToPendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
+            lastArchiveRoot: checkpoint.header.lastArchiveRoot,
+            pipeliningEnabled: isPipelining
+        });
+        await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
+            txTimeoutAt,
+            ...submissionSimulationOverridesPlan ? {
+                simulationOverridesPlan: submissionSimulationOverridesPlan
+            } : {}
+        });
     }
     async proposeCheckpoint() {
         try {
@@ -651,13 +719,22 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
                 hasError: false
             };
             try {
+                const now = this.dateProvider.now();
+                if (this.epochCache.isProposerPipeliningEnabled() && this.proposedCheckpointData) {
+                    // Measure against the wall-clock slot whose build window we are currently using.
+                    // In pipelining mode `targetSlot` is intentionally one slot ahead, which makes the
+                    // target-slot boundary a full slot away from the actual build start time.
+                    const slotBoundaryMs = Number(getTimestampForSlot(this.slotNow, this.l1Constants)) * 1000;
+                    this.checkpointMetrics.recordPipelinedCheckpointBuildStartOffsetFromSlotBoundary(now - slotBoundaryMs);
+                }
+                this.checkpointMetrics.startCheckpointTiming(now);
                 // Get operator configured coinbase and fee recipient for this attestor
                 const coinbase = this.validatorClient.getCoinbaseForAttestor(this.attestorAddress);
                 const feeRecipient = this.validatorClient.getFeeRecipientForAttestor(this.attestorAddress);
                 // Start the checkpoint
                 this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.targetSlot);
                 this.log.info(`Starting checkpoint proposal`, {
-                    buildSlot: this.slot,
+                    buildSlot: this.slotNow,
                     submissionSlot: this.targetSlot,
                     pipelining: this.epochCache.isProposerPipeliningEnabled(),
                     proposer: this.proposer?.toString(),
@@ -668,8 +745,17 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
                 if (this.invalidateCheckpoint && !this.config.skipInvalidateBlockAsProposer) {
                     this.publisher.enqueueInvalidateCheckpoint(this.invalidateCheckpoint);
                 }
-                // Create checkpoint builder for the slot
-                const checkpointGlobalVariables = await this.globalsBuilder.buildCheckpointGlobalVariables(coinbase, feeRecipient, this.targetSlot);
+                // Create checkpoint builder for the slot.
+                // When pipelining, force the proposed checkpoint number and fee header to our parent so the
+                // fee computation sees the same chain tip that L1 will see once the previous pipelined checkpoint lands.
+                const isPipelining = this.epochCache.isProposerPipeliningEnabled();
+                this.pipelinedParentSimulationOverridesPlan = isPipelining ? await buildPipelinedParentSimulationOverridesPlan({
+                    checkpointNumber: this.checkpointNumber,
+                    proposedCheckpointData: this.proposedCheckpointData,
+                    rollup: this.publisher.rollupContract,
+                    log: this.log
+                }) : undefined;
+                const checkpointGlobalVariables = await this.globalsBuilder.buildCheckpointGlobalVariables(coinbase, feeRecipient, this.targetSlot, this.pipelinedParentSimulationOverridesPlan);
                 // Collect L1 to L2 messages for the checkpoint and compute their hash
                 const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(this.checkpointNumber);
                 const inHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
@@ -741,14 +827,14 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
                         maxTxsPerCheckpoint: this.config.maxTxsPerCheckpoint
                     });
                 } catch (err) {
-                    this.log.error(`Built an invalid checkpoint at slot ${this.slot} (skipping proposal)`, err, {
+                    this.log.error(`Built an invalid checkpoint at slot ${this.slotNow} (skipping proposal)`, err, {
                         checkpoint: checkpoint.header.toInspect()
                     });
                     return undefined;
                 }
                 // Record checkpoint-level build metrics
-                this.metrics.recordCheckpointBuild(checkpointBuildTimer.ms(), blocksInCheckpoint.length, checkpoint.getStats().txCount, Number(checkpoint.header.totalManaUsed.toBigInt()));
-                // Do not collect attestations nor publish to L1 in fisherman mode
+                this.checkpointMetrics.recordCheckpointBuild(checkpointBuildTimer.ms(), blocksInCheckpoint.length, checkpoint.getStats().txCount, Number(checkpoint.header.totalManaUsed.toBigInt()));
+                // In fisherman mode, return the checkpoint without broadcasting or collecting attestations
                 if (this.config.fishermanMode) {
                     this.log.info(`Built checkpoint for slot ${this.targetSlot} with ${blocksInCheckpoint.length} blocks. ` + `Skipping proposal in fisherman mode.`, {
                         slot: this.targetSlot,
@@ -756,53 +842,24 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
                         blocksBuilt: blocksInCheckpoint.length
                     });
                     this.metrics.recordCheckpointSuccess();
-                    return checkpoint;
+                    // Return a broadcast result with a dummy proposal — fisherman mode skips attestation collection
+                    return {
+                        checkpoint,
+                        proposal: undefined,
+                        blockProposedAt: this.dateProvider.now()
+                    };
                 }
-                // Include the block pending broadcast in the checkpoint proposal if any
-                const lastBlock = blockPendingBroadcast && {
-                    blockHeader: blockPendingBroadcast.block.header,
-                    indexWithinCheckpoint: blockPendingBroadcast.block.indexWithinCheckpoint,
-                    txs: blockPendingBroadcast.txs
-                };
                 // Create the checkpoint proposal and broadcast it
-                const proposal = await this.validatorClient.createCheckpointProposal(checkpoint.header, checkpoint.archive.root, feeAssetPriceModifier, lastBlock, this.proposer, checkpointProposalOptions);
+                const proposal = await this.validatorClient.createCheckpointProposal(checkpoint.header, checkpoint.archive.root, feeAssetPriceModifier, blockPendingBroadcast, this.proposer, checkpointProposalOptions);
                 const blockProposedAt = this.dateProvider.now();
                 await this.p2pClient.broadcastCheckpointProposal(proposal);
-                this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot);
-                const attestations = await this.waitForAttestations(proposal);
-                const blockAttestedAt = this.dateProvider.now();
-                this.metrics.recordCheckpointAttestationDelay(blockAttestedAt - blockProposedAt);
-                // Proposer must sign over the attestations before pushing them to L1
-                const signer = this.proposer ?? this.publisher.getSenderAddress();
-                let attestationsSignature;
-                try {
-                    attestationsSignature = await this.validatorClient.signAttestationsAndSigners(attestations, signer, this.targetSlot, this.checkpointNumber);
-                } catch (err) {
-                    // We shouldn't really get here since we yield to another HA node
-                    // as soon as we see these errors when creating block or checkpoint proposals.
-                    if (this.handleHASigningError(err, 'Attestations signature')) {
-                        return undefined;
-                    }
-                    throw err;
-                }
-                // Enqueue publishing the checkpoint to L1
-                this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.targetSlot);
-                const aztecSlotDuration = this.l1Constants.slotDuration;
-                const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
-                const txTimeoutAt = new Date((submissionSlotStart + aztecSlotDuration) * 1000);
-                // If we have been configured to potentially skip publishing checkpoint then roll the dice here
-                if (this.config.skipPublishingCheckpointsPercent !== undefined && this.config.skipPublishingCheckpointsPercent > 0) {
-                    const result = Math.max(0, randomInt(100));
-                    if (result < this.config.skipPublishingCheckpointsPercent) {
-                        this.log.warn(`Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${result}`);
-                        return checkpoint;
-                    }
-                }
-                await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
-                    txTimeoutAt,
-                    forcePendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber
-                });
-                return checkpoint;
+                this.checkpointMetrics.noteCheckpointBroadcast(this.dateProvider.now());
+                // Return immediately after broadcast — attestation collection happens in the background
+                return {
+                    checkpoint,
+                    proposal,
+                    blockProposedAt
+                };
             } catch (e) {
                 env.error = e;
                 env.hasError = true;
@@ -815,7 +872,7 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
                 // swallow this error. It's already been logged by a function deeper in the stack
                 return undefined;
             }
-            this.log.error(`Error building checkpoint at slot ${this.slot}`, err);
+            this.log.error(`Error building checkpoint at slot ${this.targetSlot}`, err);
             return undefined;
         }
     }
@@ -852,17 +909,19 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
                 indexWithinCheckpoint,
                 txHashesAlreadyIncluded
             });
-            // TODO(palla/mbps): Review these conditions. We may want to keep trying in some scenarios.
-            if (!buildResult && timingInfo.isLastBlock) {
-                break;
-            } else if (!buildResult && timingInfo.deadline !== undefined) {
-                // But if there is still time for more blocks, wait until the next subslot and try again
+            // If we failed to build the block due to insufficient txs, we try again if there is still time left in the slot
+            if ('failure' in buildResult) {
+                // If this was the last subslot, or we're running with a single block per slot, we're done
+                if (timingInfo.isLastBlock || timingInfo.deadline === undefined) {
+                    break;
+                }
+                // Otherwise, if there is still time for more blocks, we wait until the next subslot and try again
                 await this.waitUntilNextSubslot(timingInfo.deadline);
                 continue;
-            } else if (!buildResult) {
-                break;
-            } else if ('error' in buildResult) {
-                // If there was an error building the block, just exit the loop and give up the rest of the slot
+            }
+            // If there was an error building the block, we just exit the loop and give up the rest of the slot.
+            // We don't want to risk building more blocks if something went wrong.
+            if ('error' in buildResult) {
                 if (!(buildResult.error instanceof SequencerInterruptedError)) {
                     this.log.warn(`Halting block building for slot ${this.targetSlot}`, {
                         slot: this.targetSlot,
@@ -873,32 +932,29 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
                 break;
             }
             const { block, usedTxs } = buildResult;
+            this.checkpointMetrics.noteCheckpointBlockBuilt(this.dateProvider.now(), {
+                isFirstBlock: blocksBuilt === 0,
+                isLastBlock: timingInfo.isLastBlock
+            });
             blocksInCheckpoint.push(block);
             usedTxs.forEach((tx)=>txHashesAlreadyIncluded.add(tx.txHash.toString()));
-            // If this is the last block, sync it to the archiver and exit the loop
-            // so we can build the checkpoint and start collecting attestations.
+            // Sign the block proposal. This will throw if HA signing fails.
+            const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions);
+            // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal,
+            // so we avoid polluting our archive with a block that would fail.
+            // We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
+            // If this throws, we abort the entire checkpoint.
+            await this.syncProposedBlockToArchiver(block);
+            // If this is the last block, do not broadcast it, since it will be included in the checkpoint proposal.
             if (timingInfo.isLastBlock) {
-                await this.syncProposedBlockToArchiver(block);
                 this.log.verbose(`Completed final block ${blockNumber} for slot ${this.targetSlot}`, {
                     slot: this.targetSlot,
                     blockNumber,
                     blocksBuilt
                 });
-                blockPendingBroadcast = {
-                    block,
-                    txs: usedTxs
-                };
+                blockPendingBroadcast = proposal;
                 break;
             }
-            // Broadcast the block proposal (unless we're in fisherman mode) unless the block is the last one,
-            // in which case we'll broadcast it along with the checkpoint at the end of the loop.
-            // Note that we only send the block to the archiver if we manage to create the proposal, so if there's
-            // a HA error we don't pollute our archiver with a block that won't make it to the chain.
-            const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions);
-            // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal.
-            // We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
-            // If this throws, we abort the entire checkpoint.
-            await this.syncProposedBlockToArchiver(block);
             // Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it.
             proposal && await this.p2pClient.broadcastProposal(proposal);
             // Wait until the next block's start time
@@ -948,7 +1004,9 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
                     slot: this.targetSlot
                 });
                 this.metrics.recordBlockProposalFailed('insufficient_txs');
-                return undefined;
+                return {
+                    failure: 'insufficient-txs'
+                };
             }
             // Create iterator to pending txs. We filter out txs already included in previous blocks in the checkpoint
             // just in case p2p failed to sync the provisional block and didn't get to remove those txs from the mempool yet.
@@ -991,7 +1049,9 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
                     slot: this.targetSlot
                 });
                 this.metrics.recordBlockProposalFailed('insufficient_valid_txs');
-                return undefined;
+                return {
+                    failure: 'insufficient-valid-txs'
+                };
             }
             // Block creation succeeded, emit stats and metrics
             const { block, publicProcessorDuration, usedTxs, blockBuildDuration, numTxs } = buildResult;
@@ -1010,12 +1070,14 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
                 manaPerSec,
                 ...blockStats
             });
+            // `slot` is the target/submission slot (may be one ahead when pipelining),
+            // `buildSlot` is the wall-clock slot during which the block was actually built.
             this.eventEmitter.emit('block-proposed', {
                 blockNumber: block.number,
                 slot: this.targetSlot,
                 buildSlot: this.slotNow
             });
-            this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe());
+            this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe(), this.targetSlot);
             return {
                 block,
                 usedTxs
@@ -1110,13 +1172,13 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
                 committee
             });
         }
-        const numberOfRequiredAttestations = Math.floor(committee.length * 2 / 3) + 1;
+        const numberOfRequiredAttestations = computeQuorum(committee.length);
         if (this.config.skipCollectingAttestations) {
             this.log.warn('Skipping attestation collection as per config (attesting with own keys only)');
             const attestations = await this.validatorClient?.collectOwnAttestations(proposal);
             return new CommitteeAttestationsAndSigners(orderAttestations(attestations ?? [], committee));
         }
-        const attestationTimeAllowed = this.config.enforceTimeTable ? this.timetable.getMaxAllowedTime(SequencerState.PUBLISHING_CHECKPOINT) : this.l1Constants.slotDuration;
+        const attestationTimeAllowed = this.config.enforceTimeTable ? this.timetable.getCheckpointAttestationDeadline() : this.l1Constants.slotDuration;
         const attestationDeadline = new Date((this.getSlotStartBuildTimestamp() + attestationTimeAllowed) * 1000);
         this.metrics.recordRequiredAttestations(numberOfRequiredAttestations, attestationTimeAllowed);
         const collectAttestationsTimer = new Timer();
@@ -1215,8 +1277,12 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
    * Adds the proposed block to the archiver so it's available via P2P.
    * Gossip doesn't echo messages back to the sender, so the proposer's archiver/world-state
    * would never receive its own block without this explicit sync.
+   *
+   * In fisherman mode we skip this push: the fisherman builds blocks locally for validation
+   * and fee analysis only, and pushing them to the archiver causes spurious reorg cascades
+   * whenever the real proposer's block arrives from L1.
    */ async syncProposedBlockToArchiver(block) {
-        if (this.config.skipPushProposedBlocksToArchiver !== false) {
+        if (this.config.skipPushProposedBlocksToArchiver || this.config.fishermanMode) {
             this.log.warn(`Skipping push of proposed block ${block.number} to archiver`, {
                 blockNumber: block.number,
                 slot: block.header.globalVariables.slotNumber
@@ -1277,7 +1343,7 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
         await sleep(TXS_POLLING_MS);
     }
     getSlotStartBuildTimestamp() {
-        return getSlotStartBuildTimestamp(this.slot, this.l1Constants);
+        return getSlotStartBuildTimestamp(this.slotNow, this.l1Constants);
     }
     getSecondsIntoSlot() {
         const slotStartTimestamp = this.getSlotStartBuildTimestamp();

package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts

@@ -0,0 +1,34 @@
+import { type TelemetryClient } from '@aztec/telemetry-client';
+/**
+ * Per-job recording surface used by {@link CheckpointProposalJob}.
+ *
+ * Ownership split:
+ * - {@link CheckpointProposalJobMetrics} owns the OpenTelemetry meter and instruments and should live for the
+ *   lifetime of the sequencer process.
+ * - A recorder owns only mutable timing state for one checkpoint proposal job so overlapping jobs cannot
+ *   overwrite each other's timing markers.
+ */
+export interface CheckpointProposalJobMetricsRecorder {
+    recordCheckpointAttestationDelay(durationMs: number): void;
+    recordCheckpointBuild(durationMs: number, blockCount: number, txCount: number, totalMana: number): void;
+    recordPipelinedCheckpointBuildStartOffsetFromSlotBoundary(offsetMs: number): void;
+    startCheckpointTiming(nowMs: number): void;
+    noteCheckpointBlockBuilt(nowMs: number, opts: {
+        isFirstBlock: boolean;
+        isLastBlock: boolean;
+    }): void;
+    noteCheckpointBroadcast(nowMs: number): void;
+}
+/**
+ * Long-lived owner of checkpoint proposal job telemetry instruments.
+ *
+ * The sequencer should construct this once and reuse it across jobs. Each call to {@link createRecorder} returns a
+ * lightweight recorder with isolated mutable state for a single checkpoint proposal job.
+ */
+export declare class CheckpointProposalJobMetrics {
+    private readonly meter;
+    private readonly instruments;
+    constructor(client: TelemetryClient, name?: string);
+    createRecorder(): CheckpointProposalJobMetricsRecorder;
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY2hlY2twb2ludF9wcm9wb3NhbF9qb2JfbWV0cmljcy5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL3NlcXVlbmNlci9jaGVja3BvaW50X3Byb3Bvc2FsX2pvYl9tZXRyaWNzLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFNTCxLQUFLLGVBQWUsRUFDckIsTUFBTSx5QkFBeUIsQ0FBQztBQWFqQzs7Ozs7Ozs7R0FRRztBQUNILE1BQU0sV0FBVyxvQ0FBb0M7SUFDbkQsZ0NBQWdDLENBQUMsVUFBVSxFQUFFLE1BQU0sR0FBRyxJQUFJLENBQUM7SUFDM0QscUJBQXFCLENBQUMsVUFBVSxFQUFFLE1BQU0sRUFBRSxVQUFVLEVBQUUsTUFBTSxFQUFFLE9BQU8sRUFBRSxNQUFNLEVBQUUsU0FBUyxFQUFFLE1BQU0sR0FBRyxJQUFJLENBQUM7SUFDeEcseURBQXlELENBQUMsUUFBUSxFQUFFLE1BQU0sR0FBRyxJQUFJLENBQUM7SUFDbEYscUJBQXFCLENBQUMsS0FBSyxFQUFFLE1BQU0sR0FBRyxJQUFJLENBQUM7SUFDM0Msd0JBQXdCLENBQUMsS0FBSyxFQUFFLE1BQU0sRUFBRSxJQUFJLEVBQUU7UUFBRSxZQUFZLEVBQUUsT0FBTyxDQUFDO1FBQUMsV0FBVyxFQUFFLE9BQU8sQ0FBQTtLQUFFLEdBQUcsSUFBSSxDQUFDO0lBQ3JHLHVCQUF1QixDQUFDLEtBQUssRUFBRSxNQUFNLEdBQUcsSUFBSSxDQUFDO0NBQzlDO0FBMEREOzs7OztHQUtHO0FBQ0gscUJBQWEsNEJBQTRCO0lBQ3ZDLE9BQU8sQ0FBQyxRQUFRLENBQUMsS0FBSyxDQUFRO0lBQzlCLE9BQU8sQ0FBQyxRQUFRLENBQUMsV0FBVyxDQUFtQztJQUUvRCxZQUFZLE1BQU0sRUFBRSxlQUFlLEVBQUUsSUFBSSxTQUEwQixFQWtCbEU7SUFFTSxjQUFjLElBQUksb0NBQW9DLENBRTVEO0NBQ0YifQ==

package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoint_proposal_job_metrics.d.ts","sourceRoot":"","sources":["../../src/sequencer/checkpoint_proposal_job_metrics.ts"],"names":[],"mappings":"AAAA,OAAO,EAML,KAAK,eAAe,EACrB,MAAM,yBAAyB,CAAC;AAajC;;;;;;;;GAQG;AACH,MAAM,WAAW,oCAAoC;IACnD,gCAAgC,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3D,qBAAqB,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACxG,yDAAyD,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IAClF,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,YAAY,EAAE,OAAO,CAAC;QAAC,WAAW,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;IACrG,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9C;AA0DD;;;;;GAKG;AACH,qBAAa,4BAA4B;IACvC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAQ;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAmC;IAE/D,YAAY,MAAM,EAAE,eAAe,EAAE,IAAI,SAA0B,EAkBlE;IAEM,cAAc,IAAI,oCAAoC,CAE5D;CACF"}