@aztec/sequencer-client 0.0.1-commit.5914bae → 0.0.1-commit.59a0419c6

package/dest/publisher/sequencer-publisher.js

@@ -372,19 +372,20 @@ function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
 }
 var _dec, _dec1, _dec2, _initProto;
 import { Blob, getBlobsPerL1Block, getPrefixedEthBlobCommitments } from '@aztec/blob-lib';
-import { FeeAssetPriceOracle, MULTI_CALL_3_ADDRESS, Multicall3, RollupContract } from '@aztec/ethereum/contracts';
+import { FeeAssetPriceOracle, MULTI_CALL_3_ADDRESS, Multicall3, SimulationOverridesBuilder, buildSimulationOverridesStateOverride } from '@aztec/ethereum/contracts';
 import { L1FeeAnalyzer } from '@aztec/ethereum/l1-fee-analysis';
 import { MAX_L1_TX_LIMIT, WEI_CONST } from '@aztec/ethereum/l1-tx-utils';
 import { FormattedViemError, formatViemError, mergeAbis, tryExtractEvent } from '@aztec/ethereum/utils';
 import { sumBigint } from '@aztec/foundation/bigint';
-import { toHex as toPaddedHex } from '@aztec/foundation/bigint-buffer';
 import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types';
 import { trimmedBytesLength } from '@aztec/foundation/buffer';
 import { pick } from '@aztec/foundation/collection';
+import { TimeoutError } from '@aztec/foundation/error';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { Signature } from '@aztec/foundation/eth-signature';
 import { createLogger } from '@aztec/foundation/log';
 import { makeBackoff, retry } from '@aztec/foundation/retry';
+import { InterruptibleSleep } from '@aztec/foundation/sleep';
 import { bufferToHex } from '@aztec/foundation/string';
 import { Timer } from '@aztec/foundation/timer';
 import { EmpireBaseAbi, ErrorsAbi, RollupAbi } from '@aztec/l1-artifacts';
@@ -392,16 +393,14 @@ import { encodeSlashConsensusVotes } from '@aztec/slasher';
 import { CommitteeAttestationsAndSigners } from '@aztec/stdlib/block';
 import { getLastL1SlotTimestampForL2Slot, getNextL1SlotTimestamp } from '@aztec/stdlib/epoch-helpers';
 import { getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
-import { encodeFunctionData, toHex } from 'viem';
+import { encodeFunctionData, keccak256, multicall3Abi, toHex } from 'viem';
+import { createL1TxFailedStore } from './l1_tx_failed_store/index.js';
 import { SequencerPublisherMetrics } from './sequencer-publisher-metrics.js';
 export const Actions = [
     'invalidate-by-invalid-attestation',
     'invalidate-by-insufficient-attestations',
     'propose',
     'governance-signal',
-    'empire-slashing-signal',
-    'create-empire-payload',
-    'execute-empire-payload',
     'vote-offenses',
     'execute-slash'
 ];
@@ -432,6 +431,7 @@ export class SequencerPublisher {
     interrupted;
     metrics;
     epochCache;
+    failedTxStore;
     governanceLog;
     slashingLog;
     lastActions;
@@ -440,11 +440,13 @@ export class SequencerPublisher {
     log;
     ethereumSlotDuration;
     aztecSlotDuration;
-    dateProvider;
+    /** Date provider for wall-clock time. */ dateProvider;
     blobClient;
     /** Address to use for simulations in fisherman mode (actual proposer's address) */ proposerAddressForSimulation;
+    /** Optional callback to obtain a replacement publisher when the current one fails to send. */ getNextPublisher;
     /** L1 fee analyzer for fisherman mode */ l1FeeAnalyzer;
     /** Fee asset price oracle for computing price modifiers from Uniswap V4 */ feeAssetPriceOracle;
+    /** Interruptible sleep used by sendRequestsAt to wait until a target timestamp. */ interruptibleSleep;
     // A CALL to a cold address is 2700 gas
     static MULTICALL_OVERHEAD_GAS_GUESS = 5000n;
     // Gas report for VotingWithSigTest shows a max gas of 100k, but we've seen it cost 700k+ in testnet
@@ -453,7 +455,6 @@ export class SequencerPublisher {
     rollupContract;
     govProposerContract;
     slashingProposerContract;
-    slashFactoryContract;
     tracer;
     requests;
     constructor(config, deps){
@@ -464,6 +465,7 @@ export class SequencerPublisher {
         this.lastActions = {};
         this.isPayloadEmptyCache = new Map();
         this.payloadProposedCache = new Set();
+        this.interruptibleSleep = new InterruptibleSleep();
         this.requests = [];
         this.log = deps.log ?? createLogger('sequencer:publisher');
         this.ethereumSlotDuration = BigInt(config.ethereumSlotDuration);
@@ -472,10 +474,12 @@ export class SequencerPublisher {
         this.epochCache = deps.epochCache;
         this.lastActions = deps.lastActions;
         this.blobClient = deps.blobClient;
+        this.dateProvider = deps.dateProvider;
         const telemetry = deps.telemetry ?? getTelemetryClient();
         this.metrics = deps.metrics ?? new SequencerPublisherMetrics(telemetry, 'SequencerPublisher');
         this.tracer = telemetry.getTracer('SequencerPublisher');
         this.l1TxUtils = deps.l1TxUtils;
+        this.getNextPublisher = deps.getNextPublisher;
         this.rollupContract = deps.rollupContract;
         this.govProposerContract = deps.governanceProposerContract;
         this.slashingProposerContract = deps.slashingProposerContract;
@@ -484,13 +488,30 @@ export class SequencerPublisher {
             const newSlashingProposer = await this.rollupContract.getSlashingProposer();
             this.slashingProposerContract = newSlashingProposer;
         });
-        this.slashFactoryContract = deps.slashFactoryContract;
         // Initialize L1 fee analyzer for fisherman mode
         if (config.fishermanMode) {
             this.l1FeeAnalyzer = new L1FeeAnalyzer(this.l1TxUtils.client, deps.dateProvider, createLogger('sequencer:publisher:fee-analyzer'));
         }
         // Initialize fee asset price oracle
         this.feeAssetPriceOracle = new FeeAssetPriceOracle(this.l1TxUtils.client, this.rollupContract, createLogger('sequencer:publisher:price-oracle'));
+        // Initialize failed L1 tx store (optional, for test networks)
+        this.failedTxStore = createL1TxFailedStore(config.l1TxFailedStore, this.log);
+    }
+    /**
+   * Backs up a failed L1 transaction to the configured store for debugging.
+   * Does nothing if no store is configured.
+   */ backupFailedTx(failedTx) {
+        if (!this.failedTxStore) {
+            return;
+        }
+        const tx = {
+            ...failedTx,
+            timestamp: Date.now()
+        };
+        // Fire and forget - don't block on backup
+        void this.failedTxStore.then((store)=>store?.saveFailedTx(tx)).catch((err)=>{
+            this.log.warn(`Failed to backup failed L1 tx to store`, err);
+        });
     }
     getRollupContract() {
         return this.rollupContract;
@@ -519,7 +540,7 @@ export class SequencerPublisher {
         this.requests.push(request);
     }
     getCurrentL2Slot() {
-        return this.epochCache.getEpochAndSlotNow().slot;
+        return this.epochCache.getSlotNow();
     }
     /**
    * Clears all pending requests without sending them.
@@ -636,12 +657,34 @@ export class SequencerPublisher {
         // This ensures the committee gets precomputed correctly
         validRequests.sort((a, b)=>compareActions(a.action, b.action));
         try {
+            // Capture context for failed tx backup before sending
+            const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
+            const multicallData = encodeFunctionData({
+                abi: multicall3Abi,
+                functionName: 'aggregate3',
+                args: [
+                    validRequests.map((r)=>({
+                            target: r.request.to,
+                            callData: r.request.data,
+                            allowFailure: true
+                        }))
+                ]
+            });
+            const blobDataHex = blobConfig?.blobs?.map((b)=>toHex(b));
+            const txContext = {
+                multicallData,
+                blobData: blobDataHex,
+                l1BlockNumber
+            };
             this.log.debug('Forwarding transactions', {
                 validRequests: validRequests.map((request)=>request.action),
                 txConfig
             });
-            const result = await Multicall3.forward(validRequests.map((request)=>request.request), this.l1TxUtils, txConfig, blobConfig, this.rollupContract.address, this.log);
-            const { successfulActions = [], failedActions = [] } = this.callbackBundledTransactions(validRequests, result);
+            const result = await this.forwardWithPublisherRotation(validRequests, txConfig, blobConfig);
+            if (result === undefined) {
+                return undefined;
+            }
+            const { successfulActions = [], failedActions = [] } = this.callbackBundledTransactions(validRequests, result, txContext);
             return {
                 result,
                 expiredActions,
@@ -661,10 +704,102 @@ export class SequencerPublisher {
             }
         }
     }
-    callbackBundledTransactions(requests, result) {
+    /**
+   * Forwards transactions via Multicall3, rotating to the next available publisher if a send
+   * failure occurs (i.e. the tx never reached the chain).
+   * On-chain reverts and simulation errors are returned as-is without rotation.
+   */ async forwardWithPublisherRotation(validRequests, txConfig, blobConfig) {
+        const triedAddresses = [];
+        let currentPublisher = this.l1TxUtils;
+        while(true){
+            triedAddresses.push(currentPublisher.getSenderAddress());
+            try {
+                const result = await Multicall3.forward(validRequests.map((r)=>r.request), currentPublisher, txConfig, blobConfig, this.rollupContract.address, this.log);
+                this.l1TxUtils = currentPublisher;
+                return result;
+            } catch (err) {
+                if (err instanceof TimeoutError) {
+                    throw err;
+                }
+                const viemError = formatViemError(err);
+                if (!this.getNextPublisher) {
+                    this.log.error('Failed to publish bundled transactions', viemError);
+                    return undefined;
+                }
+                this.log.warn(`Publisher ${currentPublisher.getSenderAddress()} failed to send, rotating to next publisher`, viemError);
+                const nextPublisher = await this.getNextPublisher([
+                    ...triedAddresses
+                ]);
+                if (!nextPublisher) {
+                    this.log.error('All available publishers exhausted, failed to publish bundled transactions');
+                    return undefined;
+                }
+                currentPublisher = nextPublisher;
+            }
+        }
+    }
+    /*
+   * Schedules sending all enqueued requests at (or after) the given timestamp.
+   * Uses InterruptibleSleep so it can be cancelled via interrupt().
+   * Returns the promise for the L1 response (caller should NOT await this in the work loop).
+   */ async sendRequestsAt(submitAfter) {
+        const ms = submitAfter.getTime() - this.dateProvider.now();
+        if (ms > 0) {
+            this.log.debug(`Sleeping ${ms}ms before sending requests`, {
+                submitAfter
+            });
+            await this.interruptibleSleep.sleep(ms);
+        }
+        if (this.interrupted) {
+            return undefined;
+        }
+        // Re-validate enqueued requests after the sleep (state may have changed, e.g. prune or L1 reorg)
+        const validRequests = [];
+        for (const request of this.requests){
+            if (!request.preCheck) {
+                validRequests.push(request);
+                continue;
+            }
+            try {
+                await request.preCheck();
+                validRequests.push(request);
+            } catch (err) {
+                this.log.warn(`Pre-send validation failed for ${request.action}, discarding request`, err);
+            }
+        }
+        this.requests = validRequests;
+        if (this.requests.length === 0) {
+            return undefined;
+        }
+        return this.sendRequests();
+    }
+    callbackBundledTransactions(requests, result, txContext) {
         const actionsListStr = requests.map((r)=>r.action).join(', ');
         if (result instanceof FormattedViemError) {
             this.log.error(`Failed to publish bundled transactions (${actionsListStr})`, result);
+            this.backupFailedTx({
+                id: keccak256(txContext.multicallData),
+                failureType: 'send-error',
+                request: {
+                    to: MULTI_CALL_3_ADDRESS,
+                    data: txContext.multicallData
+                },
+                blobData: txContext.blobData,
+                l1BlockNumber: txContext.l1BlockNumber.toString(),
+                error: {
+                    message: result.message,
+                    name: result.name
+                },
+                context: {
+                    actions: requests.map((r)=>r.action),
+                    requests: requests.map((r)=>({
+                            action: r.action,
+                            to: r.request.to,
+                            data: r.request.data
+                        })),
+                    sender: this.getSenderAddress().toString()
+                }
+            });
             return {
                 failedActions: requests.map((r)=>r.action)
             };
@@ -691,6 +826,37 @@ export class SequencerPublisher {
                     failedActions.push(request.action);
                 }
             }
+            // Single backup for the whole reverted tx
+            if (failedActions.length > 0 && result?.receipt?.status === 'reverted') {
+                this.backupFailedTx({
+                    id: result.receipt.transactionHash,
+                    failureType: 'revert',
+                    request: {
+                        to: MULTI_CALL_3_ADDRESS,
+                        data: txContext.multicallData
+                    },
+                    blobData: txContext.blobData,
+                    l1BlockNumber: result.receipt.blockNumber.toString(),
+                    receipt: {
+                        transactionHash: result.receipt.transactionHash,
+                        blockNumber: result.receipt.blockNumber.toString(),
+                        gasUsed: (result.receipt.gasUsed ?? 0n).toString(),
+                        status: 'reverted'
+                    },
+                    error: {
+                        message: result.errorMsg ?? 'Transaction reverted'
+                    },
+                    context: {
+                        actions: failedActions,
+                        requests: requests.filter((r)=>failedActions.includes(r.action)).map((r)=>({
+                                action: r.action,
+                                to: r.request.to,
+                                data: r.request.data
+                            })),
+                        sender: this.getSenderAddress().toString()
+                    }
+                });
+            }
             return {
                 successfulActions,
                 failedActions
@@ -701,19 +867,17 @@ export class SequencerPublisher {
    * @notice  Will call `canProposeAt` to make sure that it is possible to propose
    * @param tipArchive - The archive to check
    * @returns The slot and block number if it is possible to propose, undefined otherwise
-   */ canProposeAt(tipArchive, msgSender, opts = {}) {
+   */ async canProposeAt(tipArchive, msgSender, simulationOverridesPlan) {
         // TODO: #14291 - should loop through multiple keys to check if any of them can propose
         const ignoredErrors = [
             'SlotAlreadyInChain',
             'InvalidProposer',
             'InvalidArchive'
         ];
-        const pipelined = opts.pipelined ?? false;
+        const pipelined = this.epochCache.isProposerPipeliningEnabled();
         const slotOffset = pipelined ? this.aztecSlotDuration : 0n;
         const nextL1SlotTs = this.getNextL1SlotTimestamp() + slotOffset;
-        return this.rollupContract.canProposeAt(tipArchive.toBuffer(), msgSender.toString(), nextL1SlotTs, {
-            forcePendingCheckpointNumber: opts.forcePendingCheckpointNumber
-        }).catch((err)=>{
+        return this.rollupContract.canProposeAt(tipArchive.toBuffer(), msgSender.toString(), nextL1SlotTs, await buildSimulationOverridesStateOverride(this.rollupContract, simulationOverridesPlan)).catch((err)=>{
             if (err instanceof FormattedViemError && ignoredErrors.find((e)=>err.message.includes(e))) {
                 this.log.warn(`Failed canProposeAtTime check with ${ignoredErrors.find((e)=>err.message.includes(e))}`, {
                     error: err.message
@@ -729,7 +893,7 @@ export class SequencerPublisher {
    * @dev     This is a convenience function that can be used by the sequencer to validate a "partial" header.
    *          It will throw if the block header is invalid.
    * @param header - The block header to validate
-   */ async validateBlockHeader(header, opts) {
+   */ async validateBlockHeader(header, simulationOverridesPlan) {
         const flags = {
             ignoreDA: true,
             ignoreSignatures: true
@@ -744,7 +908,7 @@ export class SequencerPublisher {
             flags
         ];
         const ts = this.getSimulationTimestamp(header.slotNumber);
-        const stateOverrides = await this.rollupContract.makePendingCheckpointNumberOverride(opts?.forcePendingCheckpointNumber);
+        const stateOverrides = await buildSimulationOverridesStateOverride(this.rollupContract, simulationOverridesPlan);
         let balance = 0n;
         if (this.config.fishermanMode) {
             // In fisherman mode, we can't know where the proposer is publishing from
@@ -796,6 +960,7 @@ export class SequencerPublisher {
             ...logData,
             request
         });
+        const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
         try {
             const { gasUsed } = await this.l1TxUtils.simulate(request, undefined, undefined, mergeAbis([
                 request.abi ?? [],
@@ -811,6 +976,7 @@ export class SequencerPublisher {
                 gasUsed,
                 checkpointNumber,
                 forcePendingCheckpointNumber: CheckpointNumber(checkpointNumber - 1),
+                lastArchive: validationResult.checkpoint.lastArchive,
                 reason
             };
         } catch (err) {
@@ -823,8 +989,8 @@ export class SequencerPublisher {
                     request,
                     error: viemError.message
                 });
-                const latestPendingCheckpointNumber = await this.rollupContract.getCheckpointNumber();
-                if (latestPendingCheckpointNumber < checkpointNumber) {
+                const latestProposedCheckpointNumber = await this.rollupContract.getCheckpointNumber();
+                if (latestProposedCheckpointNumber < checkpointNumber) {
                     this.log.verbose(`Checkpoint ${checkpointNumber} has already been invalidated`, {
                         ...logData
                     });
@@ -838,6 +1004,27 @@ export class SequencerPublisher {
             }
             // Otherwise, throw. We cannot build the next checkpoint if we cannot invalidate the previous one.
             this.log.error(`Simulation for invalidate checkpoint ${checkpointNumber} failed`, viemError, logData);
+            this.backupFailedTx({
+                id: keccak256(request.data),
+                failureType: 'simulation',
+                request: {
+                    to: request.to,
+                    data: request.data,
+                    value: request.value?.toString()
+                },
+                l1BlockNumber: l1BlockNumber.toString(),
+                error: {
+                    message: viemError.message,
+                    name: viemError.name
+                },
+                context: {
+                    actions: [
+                        `invalidate-${reason}`
+                    ],
+                    checkpointNumber,
+                    sender: this.getSenderAddress().toString()
+                }
+            });
             throw new Error(`Failed to simulate invalidate checkpoint ${checkpointNumber}`, {
                 cause: viemError
             });
@@ -863,7 +1050,7 @@ export class SequencerPublisher {
             throw new Error(`Unknown reason for invalidation`);
         }
     }
-    /** Simulates `propose` to make sure that the checkpoint is valid for submission */ async validateCheckpointForSubmission(checkpoint, attestationsAndSigners, attestationsAndSignersSignature, options) {
+    /** Simulates `propose` to make sure that the checkpoint is valid for submission */ async validateCheckpointForSubmission(checkpoint, attestationsAndSigners, attestationsAndSignersSignature, simulationOverridesPlan) {
         const blobFields = checkpoint.toBlobFields();
         const blobs = await getBlobsPerL1Block(blobFields);
         const blobInput = getPrefixedEthBlobCommitments(blobs);
@@ -880,7 +1067,7 @@ export class SequencerPublisher {
             attestationsAndSignersSignature.toViemSignature(),
             blobInput
         ];
-        await this.simulateProposeTx(args, options);
+        await this.simulateProposeTx(args, simulationOverridesPlan);
     }
     async enqueueCastSignalHelper(slotNumber, signalType, payload, base, signerAddress, signer) {
         if (this.lastActions[signalType] && this.lastActions[signalType] === slotNumber) {
@@ -938,6 +1125,7 @@ export class SequencerPublisher {
             signer: this.l1TxUtils.client.account?.address,
             lastValidL2Slot: slotNumber
         });
+        const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
         const timestamp = this.getSimulationTimestamp(slotNumber);
         try {
             await this.l1TxUtils.simulate(request, {
@@ -952,7 +1140,29 @@ export class SequencerPublisher {
         } catch (err) {
             const viemError = formatViemError(err);
             this.log.error(`Failed simulation for ${action} at slot ${slotNumber} (enqueuing the action anyway)`, viemError, {
-                simulationTimestamp: timestamp
+                simulationTimestamp: timestamp,
+                l1BlockNumber
+            });
+            this.backupFailedTx({
+                id: keccak256(request.data),
+                failureType: 'simulation',
+                request: {
+                    to: request.to,
+                    data: request.data,
+                    value: request.value?.toString()
+                },
+                l1BlockNumber: l1BlockNumber.toString(),
+                error: {
+                    message: viemError.message,
+                    name: viemError.name
+                },
+                context: {
+                    actions: [
+                        action
+                    ],
+                    slot: slotNumber,
+                    sender: this.getSenderAddress().toString()
+                }
             });
         // Yes, we enqueue the request anyway, in case there was a bug with the simulation itself
         }
@@ -1008,43 +1218,6 @@ export class SequencerPublisher {
         }
         for (const action of actions){
             switch(action.type){
-                case 'vote-empire-payload':
-                    {
-                        if (this.slashingProposerContract?.type !== 'empire') {
-                            this.log.error('Cannot vote for empire payload on non-empire slashing contract');
-                            break;
-                        }
-                        this.log.debug(`Enqueuing slashing vote for payload ${action.payload} at slot ${slotNumber}`, {
-                            signerAddress
-                        });
-                        await this.enqueueCastSignalHelper(slotNumber, 'empire-slashing-signal', action.payload, this.slashingProposerContract, signerAddress, signer);
-                        break;
-                    }
-                case 'create-empire-payload':
-                    {
-                        this.log.debug(`Enqueuing slashing create payload at slot ${slotNumber}`, {
-                            slotNumber,
-                            signerAddress
-                        });
-                        const request = this.slashFactoryContract.buildCreatePayloadRequest(action.data);
-                        await this.simulateAndEnqueueRequest('create-empire-payload', request, (receipt)=>!!this.slashFactoryContract.tryExtractSlashPayloadCreatedEvent(receipt.logs), slotNumber);
-                        break;
-                    }
-                case 'execute-empire-payload':
-                    {
-                        this.log.debug(`Enqueuing slashing execute payload at slot ${slotNumber}`, {
-                            slotNumber,
-                            signerAddress
-                        });
-                        if (this.slashingProposerContract?.type !== 'empire') {
-                            this.log.error('Cannot execute slashing payload on non-empire slashing contract');
-                            return false;
-                        }
-                        const empireSlashingProposer = this.slashingProposerContract;
-                        const request = empireSlashingProposer.buildExecuteRoundRequest(action.round);
-                        await this.simulateAndEnqueueRequest('execute-empire-payload', request, (receipt)=>!!empireSlashingProposer.tryExtractPayloadSubmittedEvent(receipt.logs), slotNumber);
-                        break;
-                    }
                 case 'vote-offenses':
                     {
                         this.log.debug(`Enqueuing slashing vote for ${action.votes.length} votes at slot ${slotNumber}`, {
@@ -1053,14 +1226,13 @@ export class SequencerPublisher {
                             votesCount: action.votes.length,
                             signerAddress
                         });
-                        if (this.slashingProposerContract?.type !== 'tally') {
-                            this.log.error('Cannot vote for slashing offenses on non-tally slashing contract');
+                        if (!this.slashingProposerContract) {
+                            this.log.error('No slashing proposer contract available');
                             return false;
                         }
-                        const tallySlashingProposer = this.slashingProposerContract;
                         const votes = bufferToHex(encodeSlashConsensusVotes(action.votes));
-                        const request = await tallySlashingProposer.buildVoteRequestFromSigner(votes, slotNumber, signer);
-                        await this.simulateAndEnqueueRequest('vote-offenses', request, (receipt)=>!!tallySlashingProposer.tryExtractVoteCastEvent(receipt.logs), slotNumber);
+                        const request = await this.slashingProposerContract.buildVoteRequestFromSigner(votes, slotNumber, signer);
+                        await this.simulateAndEnqueueRequest('vote-offenses', request, (receipt)=>!!this.slashingProposerContract.tryExtractVoteCastEvent(receipt.logs), slotNumber);
                         break;
                     }
                 case 'execute-slash':
@@ -1070,13 +1242,12 @@ export class SequencerPublisher {
                             round: action.round,
                             signerAddress
                         });
-                        if (this.slashingProposerContract?.type !== 'tally') {
-                            this.log.error('Cannot execute slashing offenses on non-tally slashing contract');
+                        if (!this.slashingProposerContract) {
+                            this.log.error('No slashing proposer contract available');
                             return false;
                         }
-                        const tallySlashingProposer = this.slashingProposerContract;
-                        const request = tallySlashingProposer.buildExecuteRoundRequest(action.round, action.committees);
-                        await this.simulateAndEnqueueRequest('execute-slash', request, (receipt)=>!!tallySlashingProposer.tryExtractRoundExecutedEvent(receipt.logs), slotNumber);
+                        const executeRequest = this.slashingProposerContract.buildExecuteRoundRequest(action.round, action.committees);
+                        await this.simulateAndEnqueueRequest('execute-slash', executeRequest, (receipt)=>!!this.slashingProposerContract.tryExtractRoundExecutedEvent(receipt.logs), slotNumber);
                         break;
                     }
                 default:
@@ -1100,26 +1271,40 @@ export class SequencerPublisher {
             attestationsAndSignersSignature,
             feeAssetPriceModifier: checkpoint.feeAssetPriceModifier
         };
+        const simulationOverridesPlan = SimulationOverridesBuilder.from(opts.simulationOverridesPlan).withoutBlobCheck().build();
         try {
             // @note  This will make sure that we are passing the checks for our header ASSUMING that the data is also made available
             //        This means that we can avoid the simulation issues in later checks.
             //        By simulation issue, I mean the fact that the block.timestamp is equal to the last block, not the next, which
             //        make time consistency checks break.
             // TODO(palla): Check whether we're validating twice, once here and once within addProposeTx, since we call simulateProposeTx in both places.
-            await this.validateCheckpointForSubmission(checkpoint, attestationsAndSigners, attestationsAndSignersSignature, opts);
+            await this.validateCheckpointForSubmission(checkpoint, attestationsAndSigners, attestationsAndSignersSignature, simulationOverridesPlan);
         } catch (err) {
             this.log.error(`Checkpoint validation failed. ${err instanceof Error ? err.message : 'No error message'}`, err, {
                 ...checkpoint.getStats(),
                 slotNumber: checkpoint.header.slotNumber,
-                forcePendingCheckpointNumber: opts.forcePendingCheckpointNumber
+                simulationOverridesPlan
             });
             throw err;
         }
+        // Build a pre-check callback that re-validates the checkpoint before L1 submission.
+        // During pipelining this catches stale proposals due to prunes or L1 reorgs that occur during the pipeline sleep.
+        let preCheck = undefined;
+        if (this.epochCache.isProposerPipeliningEnabled()) {
+            preCheck = async ()=>{
+                this.log.debug(`Re-validating checkpoint ${checkpoint.number} before L1 submission`);
+                await this.validateCheckpointForSubmission(checkpoint, attestationsAndSigners, attestationsAndSignersSignature, simulationOverridesPlan);
+            };
+        }
         this.log.verbose(`Enqueuing checkpoint propose transaction`, {
             ...checkpoint.toCheckpointInfo(),
-            ...opts
+            txTimeoutAt: opts.txTimeoutAt,
+            simulationOverridesPlan
         });
-        await this.addProposeTx(checkpoint, proposeTxArgs, opts);
+        await this.addProposeTx(checkpoint, proposeTxArgs, {
+            txTimeoutAt: opts.txTimeoutAt,
+            simulationOverridesPlan
+        }, preCheck);
     }
     enqueueInvalidateCheckpoint(request, opts = {}) {
         if (!request) {
@@ -1174,6 +1359,7 @@ export class SequencerPublisher {
         const cachedLastActionSlot = this.lastActions[action];
         this.lastActions[action] = slotNumber;
         this.log.debug(`Simulating ${action} for slot ${slotNumber}`, logData);
+        const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
         let gasUsed;
         const simulateAbi = mergeAbis([
             request.abi ?? [],
@@ -1191,6 +1377,27 @@ export class SequencerPublisher {
         } catch (err) {
             const viemError = formatViemError(err, simulateAbi);
             this.log.error(`Simulation for ${action} at ${slotNumber} failed`, viemError, logData);
+            this.backupFailedTx({
+                id: keccak256(request.data),
+                failureType: 'simulation',
+                request: {
+                    to: request.to,
+                    data: request.data,
+                    value: request.value?.toString()
+                },
+                l1BlockNumber: l1BlockNumber.toString(),
+                error: {
+                    message: viemError.message,
+                    name: viemError.name
+                },
+                context: {
+                    actions: [
+                        action
+                    ],
+                    slot: slotNumber,
+                    sender: this.getSenderAddress().toString()
+                }
+            });
             return false;
         }
         // We issued the simulation against the rollup contract, so we need to account for the overhead of the multicall3
@@ -1236,13 +1443,14 @@ export class SequencerPublisher {
    * A call to `restart` is required before you can continue publishing.
    */ interrupt() {
         this.interrupted = true;
+        this.interruptibleSleep.interrupt();
         this.l1TxUtils.interrupt();
     }
     /** Restarts the publisher after calling `interrupt`. */ restart() {
         this.interrupted = false;
         this.l1TxUtils.restart();
     }
-    async prepareProposeTx(encodedData, options) {
+    async prepareProposeTx(encodedData, simulationOverridesPlan) {
         const kzg = Blob.getViemKzgInstance();
         const blobInput = getPrefixedEthBlobCommitments(encodedData.blobs);
         this.log.debug('Validating blob input', {
@@ -1269,10 +1477,38 @@ export class SequencerPublisher {
             }, {}, {
                 blobs: encodedData.blobs.map((b)=>b.data),
                 kzg
-            }).catch((err)=>{
-                const { message, metaMessages } = formatViemError(err);
-                this.log.error(`Failed to validate blobs`, message, {
-                    metaMessages
+            }).catch(async (err)=>{
+                const viemError = formatViemError(err);
+                this.log.error(`Failed to validate blobs`, viemError.message, {
+                    metaMessages: viemError.metaMessages
+                });
+                const validateBlobsData = encodeFunctionData({
+                    abi: RollupAbi,
+                    functionName: 'validateBlobs',
+                    args: [
+                        blobInput
+                    ]
+                });
+                const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
+                this.backupFailedTx({
+                    id: keccak256(validateBlobsData),
+                    failureType: 'simulation',
+                    request: {
+                        to: this.rollupContract.address,
+                        data: validateBlobsData
+                    },
+                    blobData: encodedData.blobs.map((b)=>toHex(b.data)),
+                    l1BlockNumber: l1BlockNumber.toString(),
+                    error: {
+                        message: viemError.message,
+                        name: viemError.name
+                    },
+                    context: {
+                        actions: [
+                            'validate-blobs'
+                        ],
+                        sender: this.getSenderAddress().toString()
+                    }
                 });
                 throw new Error('Failed to validate blobs');
             });
@@ -1291,7 +1527,7 @@ export class SequencerPublisher {
             encodedData.attestationsAndSignersSignature.toViemSignature(),
             blobInput
         ];
-        const { rollupData, simulationResult } = await this.simulateProposeTx(args, options);
+        const { rollupData, simulationResult } = await this.simulateProposeTx(args, simulationOverridesPlan);
         return {
             args,
             blobEvaluationGas,
@@ -1303,27 +1539,13 @@ export class SequencerPublisher {
    * Simulates the propose tx with eth_simulateV1
    * @param args - The propose tx args
    * @returns The simulation result
-   */ async simulateProposeTx(args, options) {
+   */ async simulateProposeTx(args, simulationOverridesPlan) {
         const rollupData = encodeFunctionData({
             abi: RollupAbi,
             functionName: 'propose',
             args
         });
-        // override the pending checkpoint number if requested
-        const forcePendingCheckpointNumberStateDiff = (options.forcePendingCheckpointNumber !== undefined ? await this.rollupContract.makePendingCheckpointNumberOverride(options.forcePendingCheckpointNumber) : []).flatMap((override)=>override.stateDiff ?? []);
-        const stateOverrides = [
-            {
-                address: this.rollupContract.address,
-                // @note we override checkBlob to false since blobs are not part simulate()
-                stateDiff: [
-                    {
-                        slot: toPaddedHex(RollupContract.checkBlobStorageSlot, true),
-                        value: toPaddedHex(0n, true)
-                    },
-                    ...forcePendingCheckpointNumberStateDiff
-                ]
-            }
-        ];
+        const stateOverrides = await buildSimulationOverridesStateOverride(this.rollupContract, simulationOverridesPlan);
         // In fisherman mode, simulate as the proposer but with sufficient balance
         if (this.proposerAddressForSimulation) {
             stateOverrides.push({
@@ -1331,6 +1553,7 @@ export class SequencerPublisher {
                 balance: 10n * WEI_CONST * WEI_CONST
             });
         }
+        const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
         const simTs = this.getSimulationTimestamp(SlotNumber.fromBigInt(args[0].header.slotNumber));
         const simulationResult = await this.l1TxUtils.simulate({
             to: this.rollupContract.address,
@@ -1360,6 +1583,26 @@ export class SequencerPublisher {
             this.log.error(`Failed to simulate propose tx`, viemError, {
                 simulationTimestamp: simTs
             });
+            this.backupFailedTx({
+                id: keccak256(rollupData),
+                failureType: 'simulation',
+                request: {
+                    to: this.rollupContract.address,
+                    data: rollupData
+                },
+                l1BlockNumber: l1BlockNumber.toString(),
+                error: {
+                    message: viemError.message,
+                    name: viemError.name
+                },
+                context: {
+                    actions: [
+                        'propose'
+                    ],
+                    slot: Number(args[0].header.slotNumber),
+                    sender: this.getSenderAddress().toString()
+                }
+            });
             throw err;
         });
         return {
@@ -1367,11 +1610,11 @@ export class SequencerPublisher {
             simulationResult
         };
     }
-    async addProposeTx(checkpoint, encodedData, opts = {}) {
+    async addProposeTx(checkpoint, encodedData, opts = {}, preCheck) {
         const slot = checkpoint.header.slotNumber;
         const timer = new Timer();
         const kzg = Blob.getViemKzgInstance();
-        const { rollupData, simulationResult, blobEvaluationGas } = await this.prepareProposeTx(encodedData, opts);
+        const { rollupData, simulationResult, blobEvaluationGas } = await this.prepareProposeTx(encodedData, opts.simulationOverridesPlan);
         const startBlock = await this.l1TxUtils.getBlockNumber();
         const gasLimit = this.l1TxUtils.bumpGasLimit(BigInt(Math.ceil(Number(simulationResult.gasUsed) * 64 / 63)) + blobEvaluationGas + SequencerPublisher.MULTICALL_OVERHEAD_GAS_GUESS);
         // Send the blobs to the blob client preemptively. This helps in tests where the sequencer mistakingly thinks that the propose
@@ -1387,9 +1630,10 @@ export class SequencerPublisher {
             },
             lastValidL2Slot: checkpoint.header.slotNumber,
             gasConfig: {
-                ...opts,
+                txTimeoutAt: opts.txTimeoutAt,
                 gasLimit
             },
+            preCheck,
             blobConfig: {
                 blobs: encodedData.blobs.map((b)=>b.data),
                 kzg