@aztec/sequencer-client 0.0.1-commit.1bb068fb5 → 0.0.1-commit.1dcfe2301

package/dest/publisher/sequencer-publisher.js

@@ -372,33 +372,35 @@ function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
 }
 var _dec, _dec1, _dec2, _initProto;
 import { Blob, getBlobsPerL1Block, getPrefixedEthBlobCommitments } from '@aztec/blob-lib';
-import { FeeAssetPriceOracle, MULTI_CALL_3_ADDRESS, Multicall3, RollupContract } from '@aztec/ethereum/contracts';
+import { FeeAssetPriceOracle, MULTI_CALL_3_ADDRESS, Multicall3, SimulationOverridesBuilder, buildSimulationOverridesStateOverride } from '@aztec/ethereum/contracts';
 import { L1FeeAnalyzer } from '@aztec/ethereum/l1-fee-analysis';
 import { MAX_L1_TX_LIMIT, WEI_CONST } from '@aztec/ethereum/l1-tx-utils';
 import { FormattedViemError, formatViemError, mergeAbis, tryExtractEvent } from '@aztec/ethereum/utils';
 import { sumBigint } from '@aztec/foundation/bigint';
-import { toHex as toPaddedHex } from '@aztec/foundation/bigint-buffer';
 import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types';
+import { trimmedBytesLength } from '@aztec/foundation/buffer';
 import { pick } from '@aztec/foundation/collection';
+import { TimeoutError } from '@aztec/foundation/error';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { Signature } from '@aztec/foundation/eth-signature';
 import { createLogger } from '@aztec/foundation/log';
+import { makeBackoff, retry } from '@aztec/foundation/retry';
+import { InterruptibleSleep } from '@aztec/foundation/sleep';
 import { bufferToHex } from '@aztec/foundation/string';
 import { Timer } from '@aztec/foundation/timer';
 import { EmpireBaseAbi, ErrorsAbi, RollupAbi } from '@aztec/l1-artifacts';
 import { encodeSlashConsensusVotes } from '@aztec/slasher';
 import { CommitteeAttestationsAndSigners } from '@aztec/stdlib/block';
+import { getLastL1SlotTimestampForL2Slot, getNextL1SlotTimestamp } from '@aztec/stdlib/epoch-helpers';
 import { getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
-import { encodeFunctionData, toHex } from 'viem';
+import { encodeFunctionData, keccak256, multicall3Abi, toHex } from 'viem';
+import { createL1TxFailedStore } from './l1_tx_failed_store/index.js';
 import { SequencerPublisherMetrics } from './sequencer-publisher-metrics.js';
 export const Actions = [
     'invalidate-by-invalid-attestation',
     'invalidate-by-insufficient-attestations',
     'propose',
     'governance-signal',
-    'empire-slashing-signal',
-    'create-empire-payload',
-    'execute-empire-payload',
     'vote-offenses',
     'execute-slash'
 ];
@@ -429,16 +431,22 @@ export class SequencerPublisher {
     interrupted;
     metrics;
     epochCache;
+    failedTxStore;
     governanceLog;
     slashingLog;
     lastActions;
     isPayloadEmptyCache;
+    payloadProposedCache;
     log;
     ethereumSlotDuration;
+    aztecSlotDuration;
+    /** Date provider for wall-clock time. */ dateProvider;
     blobClient;
     /** Address to use for simulations in fisherman mode (actual proposer's address) */ proposerAddressForSimulation;
+    /** Optional callback to obtain a replacement publisher when the current one fails to send. */ getNextPublisher;
     /** L1 fee analyzer for fisherman mode */ l1FeeAnalyzer;
     /** Fee asset price oracle for computing price modifiers from Uniswap V4 */ feeAssetPriceOracle;
+    /** Interruptible sleep used by sendRequestsAt to wait until a target timestamp. */ interruptibleSleep;
     // A CALL to a cold address is 2700 gas
     static MULTICALL_OVERHEAD_GAS_GUESS = 5000n;
     // Gas report for VotingWithSigTest shows a max gas of 100k, but we've seen it cost 700k+ in testnet
@@ -447,7 +455,6 @@ export class SequencerPublisher {
     rollupContract;
     govProposerContract;
     slashingProposerContract;
-    slashFactoryContract;
     tracer;
     requests;
     constructor(config, deps){
@@ -457,16 +464,22 @@ export class SequencerPublisher {
         this.slashingLog = createLogger('sequencer:publisher:slashing');
         this.lastActions = {};
         this.isPayloadEmptyCache = new Map();
+        this.payloadProposedCache = new Set();
+        this.interruptibleSleep = new InterruptibleSleep();
         this.requests = [];
         this.log = deps.log ?? createLogger('sequencer:publisher');
         this.ethereumSlotDuration = BigInt(config.ethereumSlotDuration);
+        this.aztecSlotDuration = BigInt(config.aztecSlotDuration);
+        this.dateProvider = deps.dateProvider;
         this.epochCache = deps.epochCache;
         this.lastActions = deps.lastActions;
         this.blobClient = deps.blobClient;
+        this.dateProvider = deps.dateProvider;
         const telemetry = deps.telemetry ?? getTelemetryClient();
         this.metrics = deps.metrics ?? new SequencerPublisherMetrics(telemetry, 'SequencerPublisher');
         this.tracer = telemetry.getTracer('SequencerPublisher');
         this.l1TxUtils = deps.l1TxUtils;
+        this.getNextPublisher = deps.getNextPublisher;
         this.rollupContract = deps.rollupContract;
         this.govProposerContract = deps.governanceProposerContract;
         this.slashingProposerContract = deps.slashingProposerContract;
@@ -475,13 +488,30 @@ export class SequencerPublisher {
             const newSlashingProposer = await this.rollupContract.getSlashingProposer();
             this.slashingProposerContract = newSlashingProposer;
         });
-        this.slashFactoryContract = deps.slashFactoryContract;
         // Initialize L1 fee analyzer for fisherman mode
         if (config.fishermanMode) {
             this.l1FeeAnalyzer = new L1FeeAnalyzer(this.l1TxUtils.client, deps.dateProvider, createLogger('sequencer:publisher:fee-analyzer'));
         }
         // Initialize fee asset price oracle
         this.feeAssetPriceOracle = new FeeAssetPriceOracle(this.l1TxUtils.client, this.rollupContract, createLogger('sequencer:publisher:price-oracle'));
+        // Initialize failed L1 tx store (optional, for test networks)
+        this.failedTxStore = createL1TxFailedStore(config.l1TxFailedStore, this.log);
+    }
+    /**
+   * Backs up a failed L1 transaction to the configured store for debugging.
+   * Does nothing if no store is configured.
+   */ backupFailedTx(failedTx) {
+        if (!this.failedTxStore) {
+            return;
+        }
+        const tx = {
+            ...failedTx,
+            timestamp: Date.now()
+        };
+        // Fire and forget - don't block on backup
+        void this.failedTxStore.then((store)=>store?.saveFailedTx(tx)).catch((err)=>{
+            this.log.warn(`Failed to backup failed L1 tx to store`, err);
+        });
     }
     getRollupContract() {
         return this.rollupContract;
@@ -510,7 +540,7 @@ export class SequencerPublisher {
         this.requests.push(request);
     }
     getCurrentL2Slot() {
-        return this.epochCache.getEpochAndSlotNow().slot;
+        return this.epochCache.getSlotNow();
     }
     /**
    * Clears all pending requests without sending them.
@@ -599,8 +629,8 @@ export class SequencerPublisher {
         // @note - we can only have one blob config per bundle
         // find requests with gas and blob configs
         // See https://github.com/AztecProtocol/aztec-packages/issues/11513
-        const gasConfigs = requestsToProcess.filter((request)=>request.gasConfig).map((request)=>request.gasConfig);
-        const blobConfigs = requestsToProcess.filter((request)=>request.blobConfig).map((request)=>request.blobConfig);
+        const gasConfigs = validRequests.filter((request)=>request.gasConfig).map((request)=>request.gasConfig);
+        const blobConfigs = validRequests.filter((request)=>request.blobConfig).map((request)=>request.blobConfig);
         if (blobConfigs.length > 1) {
             throw new Error('Multiple blob configs found');
         }
@@ -627,12 +657,34 @@ export class SequencerPublisher {
         // This ensures the committee gets precomputed correctly
         validRequests.sort((a, b)=>compareActions(a.action, b.action));
         try {
+            // Capture context for failed tx backup before sending
+            const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
+            const multicallData = encodeFunctionData({
+                abi: multicall3Abi,
+                functionName: 'aggregate3',
+                args: [
+                    validRequests.map((r)=>({
+                            target: r.request.to,
+                            callData: r.request.data,
+                            allowFailure: true
+                        }))
+                ]
+            });
+            const blobDataHex = blobConfig?.blobs?.map((b)=>toHex(b));
+            const txContext = {
+                multicallData,
+                blobData: blobDataHex,
+                l1BlockNumber
+            };
             this.log.debug('Forwarding transactions', {
                 validRequests: validRequests.map((request)=>request.action),
                 txConfig
             });
-            const result = await Multicall3.forward(validRequests.map((request)=>request.request), this.l1TxUtils, txConfig, blobConfig, this.rollupContract.address, this.log);
-            const { successfulActions = [], failedActions = [] } = this.callbackBundledTransactions(validRequests, result);
+            const result = await this.forwardWithPublisherRotation(validRequests, txConfig, blobConfig);
+            if (result === undefined) {
+                return undefined;
+            }
+            const { successfulActions = [], failedActions = [] } = this.callbackBundledTransactions(validRequests, result, txContext);
             return {
                 result,
                 expiredActions,
@@ -652,17 +704,118 @@ export class SequencerPublisher {
             }
         }
     }
-    callbackBundledTransactions(requests, result) {
+    /**
+   * Forwards transactions via Multicall3, rotating to the next available publisher if a send
+   * failure occurs (i.e. the tx never reached the chain).
+   * On-chain reverts and simulation errors are returned as-is without rotation.
+   */ async forwardWithPublisherRotation(validRequests, txConfig, blobConfig) {
+        const triedAddresses = [];
+        let currentPublisher = this.l1TxUtils;
+        while(true){
+            triedAddresses.push(currentPublisher.getSenderAddress());
+            try {
+                const result = await Multicall3.forward(validRequests.map((r)=>r.request), currentPublisher, txConfig, blobConfig, this.rollupContract.address, this.log);
+                this.l1TxUtils = currentPublisher;
+                return result;
+            } catch (err) {
+                if (err instanceof TimeoutError) {
+                    throw err;
+                }
+                const viemError = formatViemError(err);
+                if (!this.getNextPublisher) {
+                    this.log.error('Failed to publish bundled transactions', viemError);
+                    return undefined;
+                }
+                this.log.warn(`Publisher ${currentPublisher.getSenderAddress()} failed to send, rotating to next publisher`, viemError);
+                const nextPublisher = await this.getNextPublisher([
+                    ...triedAddresses
+                ]);
+                if (!nextPublisher) {
+                    this.log.error('All available publishers exhausted, failed to publish bundled transactions');
+                    return undefined;
+                }
+                currentPublisher = nextPublisher;
+            }
+        }
+    }
+    /*
+   * Schedules sending all enqueued requests at (or after) the given timestamp.
+   * Uses InterruptibleSleep so it can be cancelled via interrupt().
+   * Returns the promise for the L1 response (caller should NOT await this in the work loop).
+   */ async sendRequestsAt(submitAfter) {
+        const ms = submitAfter.getTime() - this.dateProvider.now();
+        if (ms > 0) {
+            this.log.debug(`Sleeping ${ms}ms before sending requests`, {
+                submitAfter
+            });
+            await this.interruptibleSleep.sleep(ms);
+        }
+        if (this.interrupted) {
+            return undefined;
+        }
+        // Re-validate enqueued requests after the sleep (state may have changed, e.g. prune or L1 reorg)
+        const validRequests = [];
+        for (const request of this.requests){
+            if (!request.preCheck) {
+                validRequests.push(request);
+                continue;
+            }
+            try {
+                await request.preCheck();
+                validRequests.push(request);
+            } catch (err) {
+                this.log.warn(`Pre-send validation failed for ${request.action}, discarding request`, err);
+            }
+        }
+        this.requests = validRequests;
+        if (this.requests.length === 0) {
+            return undefined;
+        }
+        return this.sendRequests();
+    }
+    callbackBundledTransactions(requests, result, txContext) {
         const actionsListStr = requests.map((r)=>r.action).join(', ');
         if (result instanceof FormattedViemError) {
             this.log.error(`Failed to publish bundled transactions (${actionsListStr})`, result);
+            this.backupFailedTx({
+                id: keccak256(txContext.multicallData),
+                failureType: 'send-error',
+                request: {
+                    to: MULTI_CALL_3_ADDRESS,
+                    data: txContext.multicallData
+                },
+                blobData: txContext.blobData,
+                l1BlockNumber: txContext.l1BlockNumber.toString(),
+                error: {
+                    message: result.message,
+                    name: result.name
+                },
+                context: {
+                    actions: requests.map((r)=>r.action),
+                    requests: requests.map((r)=>({
+                            action: r.action,
+                            to: r.request.to,
+                            data: r.request.data
+                        })),
+                    sender: this.getSenderAddress().toString()
+                }
+            });
             return {
                 failedActions: requests.map((r)=>r.action)
             };
         } else {
             this.log.verbose(`Published bundled transactions (${actionsListStr})`, {
                 result,
-                requests
+                requests: requests.map((r)=>({
+                        ...r,
+                        // Avoid logging large blob data
+                        blobConfig: r.blobConfig ? {
+                            ...r.blobConfig,
+                            blobs: r.blobConfig.blobs.map((b)=>({
+                                    size: trimmedBytesLength(b)
+                                }))
+                        } : undefined
+                    }))
             });
             const successfulActions = [];
             const failedActions = [];
@@ -673,6 +826,37 @@ export class SequencerPublisher {
                     failedActions.push(request.action);
                 }
             }
+            // Single backup for the whole reverted tx
+            if (failedActions.length > 0 && result?.receipt?.status === 'reverted') {
+                this.backupFailedTx({
+                    id: result.receipt.transactionHash,
+                    failureType: 'revert',
+                    request: {
+                        to: MULTI_CALL_3_ADDRESS,
+                        data: txContext.multicallData
+                    },
+                    blobData: txContext.blobData,
+                    l1BlockNumber: result.receipt.blockNumber.toString(),
+                    receipt: {
+                        transactionHash: result.receipt.transactionHash,
+                        blockNumber: result.receipt.blockNumber.toString(),
+                        gasUsed: (result.receipt.gasUsed ?? 0n).toString(),
+                        status: 'reverted'
+                    },
+                    error: {
+                        message: result.errorMsg ?? 'Transaction reverted'
+                    },
+                    context: {
+                        actions: failedActions,
+                        requests: requests.filter((r)=>failedActions.includes(r.action)).map((r)=>({
+                                action: r.action,
+                                to: r.request.to,
+                                data: r.request.data
+                            })),
+                        sender: this.getSenderAddress().toString()
+                    }
+                });
+            }
             return {
                 successfulActions,
                 failedActions
@@ -680,19 +864,20 @@ export class SequencerPublisher {
         }
     }
     /**
-   * @notice  Will call `canProposeAtNextEthBlock` to make sure that it is possible to propose
+   * @notice  Will call `canProposeAt` to make sure that it is possible to propose
    * @param tipArchive - The archive to check
    * @returns The slot and block number if it is possible to propose, undefined otherwise
-   */ canProposeAtNextEthBlock(tipArchive, msgSender, opts = {}) {
+   */ async canProposeAt(tipArchive, msgSender, simulationOverridesPlan) {
         // TODO: #14291 - should loop through multiple keys to check if any of them can propose
         const ignoredErrors = [
             'SlotAlreadyInChain',
             'InvalidProposer',
             'InvalidArchive'
         ];
-        return this.rollupContract.canProposeAtNextEthBlock(tipArchive.toBuffer(), msgSender.toString(), Number(this.ethereumSlotDuration), {
-            forcePendingCheckpointNumber: opts.forcePendingCheckpointNumber
-        }).catch((err)=>{
+        const pipelined = this.epochCache.isProposerPipeliningEnabled();
+        const slotOffset = pipelined ? this.aztecSlotDuration : 0n;
+        const nextL1SlotTs = this.getNextL1SlotTimestamp() + slotOffset;
+        return this.rollupContract.canProposeAt(tipArchive.toBuffer(), msgSender.toString(), nextL1SlotTs, await buildSimulationOverridesStateOverride(this.rollupContract, simulationOverridesPlan)).catch((err)=>{
             if (err instanceof FormattedViemError && ignoredErrors.find((e)=>err.message.includes(e))) {
                 this.log.warn(`Failed canProposeAtTime check with ${ignoredErrors.find((e)=>err.message.includes(e))}`, {
                     error: err.message
@@ -708,7 +893,7 @@ export class SequencerPublisher {
    * @dev     This is a convenience function that can be used by the sequencer to validate a "partial" header.
    *          It will throw if the block header is invalid.
    * @param header - The block header to validate
-   */ async validateBlockHeader(header, opts) {
+   */ async validateBlockHeader(header, simulationOverridesPlan) {
         const flags = {
             ignoreDA: true,
             ignoreSignatures: true
@@ -722,8 +907,8 @@ export class SequencerPublisher {
             header.blobsHash.toString(),
             flags
         ];
-        const ts = BigInt((await this.l1TxUtils.getBlock()).timestamp + this.ethereumSlotDuration);
-        const stateOverrides = await this.rollupContract.makePendingCheckpointNumberOverride(opts?.forcePendingCheckpointNumber);
+        const ts = this.getSimulationTimestamp(header.slotNumber);
+        const stateOverrides = await buildSimulationOverridesStateOverride(this.rollupContract, simulationOverridesPlan);
         let balance = 0n;
         if (this.config.fishermanMode) {
             // In fisherman mode, we can't know where the proposer is publishing from
@@ -745,7 +930,7 @@ export class SequencerPublisher {
             }),
             from: MULTI_CALL_3_ADDRESS
         }, {
-            time: ts + 1n
+            time: ts
         }, stateOverrides);
         this.log.debug(`Simulated validateHeader`);
     }
@@ -775,6 +960,7 @@ export class SequencerPublisher {
             ...logData,
             request
         });
+        const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
         try {
             const { gasUsed } = await this.l1TxUtils.simulate(request, undefined, undefined, mergeAbis([
                 request.abi ?? [],
@@ -790,6 +976,7 @@ export class SequencerPublisher {
                 gasUsed,
                 checkpointNumber,
                 forcePendingCheckpointNumber: CheckpointNumber(checkpointNumber - 1),
+                lastArchive: validationResult.checkpoint.lastArchive,
                 reason
             };
         } catch (err) {
@@ -802,8 +989,8 @@ export class SequencerPublisher {
                     request,
                     error: viemError.message
                 });
-                const latestPendingCheckpointNumber = await this.rollupContract.getCheckpointNumber();
-                if (latestPendingCheckpointNumber < checkpointNumber) {
+                const latestProposedCheckpointNumber = await this.rollupContract.getCheckpointNumber();
+                if (latestProposedCheckpointNumber < checkpointNumber) {
                     this.log.verbose(`Checkpoint ${checkpointNumber} has already been invalidated`, {
                         ...logData
                     });
@@ -817,6 +1004,27 @@ export class SequencerPublisher {
             }
             // Otherwise, throw. We cannot build the next checkpoint if we cannot invalidate the previous one.
             this.log.error(`Simulation for invalidate checkpoint ${checkpointNumber} failed`, viemError, logData);
+            this.backupFailedTx({
+                id: keccak256(request.data),
+                failureType: 'simulation',
+                request: {
+                    to: request.to,
+                    data: request.data,
+                    value: request.value?.toString()
+                },
+                l1BlockNumber: l1BlockNumber.toString(),
+                error: {
+                    message: viemError.message,
+                    name: viemError.name
+                },
+                context: {
+                    actions: [
+                        `invalidate-${reason}`
+                    ],
+                    checkpointNumber,
+                    sender: this.getSenderAddress().toString()
+                }
+            });
             throw new Error(`Failed to simulate invalidate checkpoint ${checkpointNumber}`, {
                 cause: viemError
             });
@@ -842,24 +1050,9 @@ export class SequencerPublisher {
             throw new Error(`Unknown reason for invalidation`);
         }
     }
-    /** Simulates `propose` to make sure that the checkpoint is valid for submission */ async validateCheckpointForSubmission(checkpoint, attestationsAndSigners, attestationsAndSignersSignature, options) {
-        const ts = BigInt((await this.l1TxUtils.getBlock()).timestamp + this.ethereumSlotDuration);
-        // TODO(palla/mbps): This should not be needed, there's no flow where we propose with zero attestations. Or is there?
-        // If we have no attestations, we still need to provide the empty attestations
-        // so that the committee is recalculated correctly
-        // const ignoreSignatures = attestationsAndSigners.attestations.length === 0;
-        // if (ignoreSignatures) {
-        //   const { committee } = await this.epochCache.getCommittee(block.header.globalVariables.slotNumber);
-        //   if (!committee) {
-        //     this.log.warn(`No committee found for slot ${block.header.globalVariables.slotNumber}`);
-        //     throw new Error(`No committee found for slot ${block.header.globalVariables.slotNumber}`);
-        //   }
-        //   attestationsAndSigners.attestations = committee.map(committeeMember =>
-        //     CommitteeAttestation.fromAddress(committeeMember),
-        //   );
-        // }
+    /** Simulates `propose` to make sure that the checkpoint is valid for submission */ async validateCheckpointForSubmission(checkpoint, attestationsAndSigners, attestationsAndSignersSignature, simulationOverridesPlan) {
         const blobFields = checkpoint.toBlobFields();
-        const blobs = getBlobsPerL1Block(blobFields);
+        const blobs = await getBlobsPerL1Block(blobFields);
         const blobInput = getPrefixedEthBlobCommitments(blobs);
         const args = [
             {
@@ -874,10 +1067,9 @@ export class SequencerPublisher {
             attestationsAndSignersSignature.toViemSignature(),
             blobInput
         ];
-        await this.simulateProposeTx(args, ts, options);
-        return ts;
+        await this.simulateProposeTx(args, simulationOverridesPlan);
     }
-    async enqueueCastSignalHelper(slotNumber, timestamp, signalType, payload, base, signerAddress, signer) {
+    async enqueueCastSignalHelper(slotNumber, signalType, payload, base, signerAddress, signer) {
         if (this.lastActions[signalType] && this.lastActions[signalType] === slotNumber) {
             this.log.debug(`Skipping duplicate vote cast signal ${signalType} for slot ${slotNumber}`);
             return false;
@@ -901,6 +1093,28 @@ export class SequencerPublisher {
             this.log.warn(`Skipping vote cast for payload with empty code`);
             return false;
         }
+        // Check if payload was already submitted to governance
+        const cacheKey = payload.toString();
+        if (!this.payloadProposedCache.has(cacheKey)) {
+            try {
+                const l1StartBlock = await this.rollupContract.getL1StartBlock();
+                const proposed = await retry(()=>base.hasPayloadBeenProposed(payload.toString(), l1StartBlock), 'Check if payload was proposed', makeBackoff([
+                    0,
+                    1,
+                    2
+                ]), this.log, true);
+                if (proposed) {
+                    this.payloadProposedCache.add(cacheKey);
+                }
+            } catch (err) {
+                this.log.warn(`Failed to check if payload ${payload} was proposed after retries, skipping signal`, err);
+                return false;
+            }
+        }
+        if (this.payloadProposedCache.has(cacheKey)) {
+            this.log.info(`Payload ${payload} was already proposed to governance, stopping signals`);
+            return false;
+        }
         const cachedLastVote = this.lastActions[signalType];
         this.lastActions[signalType] = slotNumber;
         const action = signalType;
@@ -911,6 +1125,8 @@ export class SequencerPublisher {
             signer: this.l1TxUtils.client.account?.address,
             lastValidL2Slot: slotNumber
         });
+        const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
+        const timestamp = this.getSimulationTimestamp(slotNumber);
         try {
             await this.l1TxUtils.simulate(request, {
                 time: timestamp
@@ -922,7 +1138,32 @@ export class SequencerPublisher {
                 request
             });
         } catch (err) {
-            this.log.error(`Failed simulation for ${action} at slot ${slotNumber} (enqueuing the action anyway)`, err);
+            const viemError = formatViemError(err);
+            this.log.error(`Failed simulation for ${action} at slot ${slotNumber} (enqueuing the action anyway)`, viemError, {
+                simulationTimestamp: timestamp,
+                l1BlockNumber
+            });
+            this.backupFailedTx({
+                id: keccak256(request.data),
+                failureType: 'simulation',
+                request: {
+                    to: request.to,
+                    data: request.data,
+                    value: request.value?.toString()
+                },
+                l1BlockNumber: l1BlockNumber.toString(),
+                error: {
+                    message: viemError.message,
+                    name: viemError.name
+                },
+                context: {
+                    actions: [
+                        action
+                    ],
+                    slot: slotNumber,
+                    sender: this.getSenderAddress().toString()
+                }
+            });
         // Yes, we enqueue the request anyway, in case there was a bug with the simulation itself
         }
         // TODO(palla/slash): All votes (governance and slashing) should txTimeoutAt at the end of the slot.
@@ -966,55 +1207,17 @@ export class SequencerPublisher {
     /**
    * Enqueues a governance castSignal transaction to cast a signal for a given slot number.
    * @param slotNumber - The slot number to cast a signal for.
-   * @param timestamp - The timestamp of the slot to cast a signal for.
    * @returns True if the signal was successfully enqueued, false otherwise.
-   */ enqueueGovernanceCastSignal(governancePayload, slotNumber, timestamp, signerAddress, signer) {
-        return this.enqueueCastSignalHelper(slotNumber, timestamp, 'governance-signal', governancePayload, this.govProposerContract, signerAddress, signer);
+   */ enqueueGovernanceCastSignal(governancePayload, slotNumber, signerAddress, signer) {
+        return this.enqueueCastSignalHelper(slotNumber, 'governance-signal', governancePayload, this.govProposerContract, signerAddress, signer);
     }
-    /** Enqueues all slashing actions as returned by the slasher client. */ async enqueueSlashingActions(actions, slotNumber, timestamp, signerAddress, signer) {
+    /** Enqueues all slashing actions as returned by the slasher client. */ async enqueueSlashingActions(actions, slotNumber, signerAddress, signer) {
         if (actions.length === 0) {
             this.log.debug(`No slashing actions to enqueue for slot ${slotNumber}`);
             return false;
         }
         for (const action of actions){
             switch(action.type){
-                case 'vote-empire-payload':
-                    {
-                        if (this.slashingProposerContract?.type !== 'empire') {
-                            this.log.error('Cannot vote for empire payload on non-empire slashing contract');
-                            break;
-                        }
-                        this.log.debug(`Enqueuing slashing vote for payload ${action.payload} at slot ${slotNumber}`, {
-                            signerAddress
-                        });
-                        await this.enqueueCastSignalHelper(slotNumber, timestamp, 'empire-slashing-signal', action.payload, this.slashingProposerContract, signerAddress, signer);
-                        break;
-                    }
-                case 'create-empire-payload':
-                    {
-                        this.log.debug(`Enqueuing slashing create payload at slot ${slotNumber}`, {
-                            slotNumber,
-                            signerAddress
-                        });
-                        const request = this.slashFactoryContract.buildCreatePayloadRequest(action.data);
-                        await this.simulateAndEnqueueRequest('create-empire-payload', request, (receipt)=>!!this.slashFactoryContract.tryExtractSlashPayloadCreatedEvent(receipt.logs), slotNumber, timestamp);
-                        break;
-                    }
-                case 'execute-empire-payload':
-                    {
-                        this.log.debug(`Enqueuing slashing execute payload at slot ${slotNumber}`, {
-                            slotNumber,
-                            signerAddress
-                        });
-                        if (this.slashingProposerContract?.type !== 'empire') {
-                            this.log.error('Cannot execute slashing payload on non-empire slashing contract');
-                            return false;
-                        }
-                        const empireSlashingProposer = this.slashingProposerContract;
-                        const request = empireSlashingProposer.buildExecuteRoundRequest(action.round);
-                        await this.simulateAndEnqueueRequest('execute-empire-payload', request, (receipt)=>!!empireSlashingProposer.tryExtractPayloadSubmittedEvent(receipt.logs), slotNumber, timestamp);
-                        break;
-                    }
                 case 'vote-offenses':
                     {
                         this.log.debug(`Enqueuing slashing vote for ${action.votes.length} votes at slot ${slotNumber}`, {
@@ -1023,14 +1226,13 @@ export class SequencerPublisher {
                             votesCount: action.votes.length,
                             signerAddress
                         });
-                        if (this.slashingProposerContract?.type !== 'tally') {
-                            this.log.error('Cannot vote for slashing offenses on non-tally slashing contract');
+                        if (!this.slashingProposerContract) {
+                            this.log.error('No slashing proposer contract available');
                             return false;
                         }
-                        const tallySlashingProposer = this.slashingProposerContract;
                         const votes = bufferToHex(encodeSlashConsensusVotes(action.votes));
-                        const request = await tallySlashingProposer.buildVoteRequestFromSigner(votes, slotNumber, signer);
-                        await this.simulateAndEnqueueRequest('vote-offenses', request, (receipt)=>!!tallySlashingProposer.tryExtractVoteCastEvent(receipt.logs), slotNumber, timestamp);
+                        const request = await this.slashingProposerContract.buildVoteRequestFromSigner(votes, slotNumber, signer);
+                        await this.simulateAndEnqueueRequest('vote-offenses', request, (receipt)=>!!this.slashingProposerContract.tryExtractVoteCastEvent(receipt.logs), slotNumber);
                         break;
                     }
                 case 'execute-slash':
@@ -1040,13 +1242,12 @@ export class SequencerPublisher {
                             round: action.round,
                             signerAddress
                         });
-                        if (this.slashingProposerContract?.type !== 'tally') {
-                            this.log.error('Cannot execute slashing offenses on non-tally slashing contract');
+                        if (!this.slashingProposerContract) {
+                            this.log.error('No slashing proposer contract available');
                             return false;
                         }
-                        const tallySlashingProposer = this.slashingProposerContract;
-                        const request = tallySlashingProposer.buildExecuteRoundRequest(action.round, action.committees);
-                        await this.simulateAndEnqueueRequest('execute-slash', request, (receipt)=>!!tallySlashingProposer.tryExtractRoundExecutedEvent(receipt.logs), slotNumber, timestamp);
+                        const executeRequest = this.slashingProposerContract.buildExecuteRoundRequest(action.round, action.committees);
+                        await this.simulateAndEnqueueRequest('execute-slash', executeRequest, (receipt)=>!!this.slashingProposerContract.tryExtractRoundExecutedEvent(receipt.logs), slotNumber);
                         break;
                     }
                 default:
@@ -1061,7 +1262,7 @@ export class SequencerPublisher {
     /** Simulates and enqueues a proposal for a checkpoint on L1 */ async enqueueProposeCheckpoint(checkpoint, attestationsAndSigners, attestationsAndSignersSignature, opts = {}) {
         const checkpointHeader = checkpoint.header;
         const blobFields = checkpoint.toBlobFields();
-        const blobs = getBlobsPerL1Block(blobFields);
+        const blobs = await getBlobsPerL1Block(blobFields);
         const proposeTxArgs = {
             header: checkpointHeader,
             archive: checkpoint.archive.root.toBuffer(),
@@ -1070,27 +1271,40 @@ export class SequencerPublisher {
             attestationsAndSignersSignature,
             feeAssetPriceModifier: checkpoint.feeAssetPriceModifier
         };
-        let ts;
+        const simulationOverridesPlan = SimulationOverridesBuilder.from(opts.simulationOverridesPlan).withoutBlobCheck().build();
         try {
             // @note  This will make sure that we are passing the checks for our header ASSUMING that the data is also made available
             //        This means that we can avoid the simulation issues in later checks.
             //        By simulation issue, I mean the fact that the block.timestamp is equal to the last block, not the next, which
             //        make time consistency checks break.
             // TODO(palla): Check whether we're validating twice, once here and once within addProposeTx, since we call simulateProposeTx in both places.
-            ts = await this.validateCheckpointForSubmission(checkpoint, attestationsAndSigners, attestationsAndSignersSignature, opts);
+            await this.validateCheckpointForSubmission(checkpoint, attestationsAndSigners, attestationsAndSignersSignature, simulationOverridesPlan);
         } catch (err) {
             this.log.error(`Checkpoint validation failed. ${err instanceof Error ? err.message : 'No error message'}`, err, {
                 ...checkpoint.getStats(),
                 slotNumber: checkpoint.header.slotNumber,
-                forcePendingCheckpointNumber: opts.forcePendingCheckpointNumber
+                simulationOverridesPlan
             });
             throw err;
         }
+        // Build a pre-check callback that re-validates the checkpoint before L1 submission.
+        // During pipelining this catches stale proposals due to prunes or L1 reorgs that occur during the pipeline sleep.
+        let preCheck = undefined;
+        if (this.epochCache.isProposerPipeliningEnabled()) {
+            preCheck = async ()=>{
+                this.log.debug(`Re-validating checkpoint ${checkpoint.number} before L1 submission`);
+                await this.validateCheckpointForSubmission(checkpoint, attestationsAndSigners, attestationsAndSignersSignature, simulationOverridesPlan);
+            };
+        }
         this.log.verbose(`Enqueuing checkpoint propose transaction`, {
             ...checkpoint.toCheckpointInfo(),
-            ...opts
+            txTimeoutAt: opts.txTimeoutAt,
+            simulationOverridesPlan
         });
-        await this.addProposeTx(checkpoint, proposeTxArgs, opts, ts);
+        await this.addProposeTx(checkpoint, proposeTxArgs, {
+            txTimeoutAt: opts.txTimeoutAt,
+            simulationOverridesPlan
+        }, preCheck);
     }
     enqueueInvalidateCheckpoint(request, opts = {}) {
         if (!request) {
@@ -1131,7 +1345,8 @@ export class SequencerPublisher {
             }
         });
     }
-    async simulateAndEnqueueRequest(action, request, checkSuccess, slotNumber, timestamp) {
+    async simulateAndEnqueueRequest(action, request, checkSuccess, slotNumber) {
+        const timestamp = this.getSimulationTimestamp(slotNumber);
         const logData = {
             slotNumber,
             timestamp,
@@ -1144,6 +1359,7 @@ export class SequencerPublisher {
         const cachedLastActionSlot = this.lastActions[action];
         this.lastActions[action] = slotNumber;
         this.log.debug(`Simulating ${action} for slot ${slotNumber}`, logData);
+        const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
         let gasUsed;
         const simulateAbi = mergeAbis([
             request.abi ?? [],
@@ -1152,7 +1368,7 @@ export class SequencerPublisher {
         try {
             ({ gasUsed } = await this.l1TxUtils.simulate(request, {
                 time: timestamp
-            }, [], simulateAbi)); // TODO(palla/slash): Check the timestamp logic
+            }, [], simulateAbi));
             this.log.verbose(`Simulation for ${action} succeeded`, {
                 ...logData,
                 request,
@@ -1161,6 +1377,27 @@ export class SequencerPublisher {
         } catch (err) {
             const viemError = formatViemError(err, simulateAbi);
             this.log.error(`Simulation for ${action} at ${slotNumber} failed`, viemError, logData);
+            this.backupFailedTx({
+                id: keccak256(request.data),
+                failureType: 'simulation',
+                request: {
+                    to: request.to,
+                    data: request.data,
+                    value: request.value?.toString()
+                },
+                l1BlockNumber: l1BlockNumber.toString(),
+                error: {
+                    message: viemError.message,
+                    name: viemError.name
+                },
+                context: {
+                    actions: [
+                        action
+                    ],
+                    slot: slotNumber,
+                    sender: this.getSenderAddress().toString()
+                }
+            });
             return false;
         }
         // We issued the simulation against the rollup contract, so we need to account for the overhead of the multicall3
@@ -1206,13 +1443,14 @@ export class SequencerPublisher {
    * A call to `restart` is required before you can continue publishing.
    */ interrupt() {
         this.interrupted = true;
+        this.interruptibleSleep.interrupt();
         this.l1TxUtils.interrupt();
     }
     /** Restarts the publisher after calling `interrupt`. */ restart() {
         this.interrupted = false;
         this.l1TxUtils.restart();
     }
-    async prepareProposeTx(encodedData, timestamp, options) {
+    async prepareProposeTx(encodedData, simulationOverridesPlan) {
         const kzg = Blob.getViemKzgInstance();
         const blobInput = getPrefixedEthBlobCommitments(encodedData.blobs);
         this.log.debug('Validating blob input', {
@@ -1239,10 +1477,38 @@ export class SequencerPublisher {
             }, {}, {
                 blobs: encodedData.blobs.map((b)=>b.data),
                 kzg
-            }).catch((err)=>{
-                const { message, metaMessages } = formatViemError(err);
-                this.log.error(`Failed to validate blobs`, message, {
-                    metaMessages
+            }).catch(async (err)=>{
+                const viemError = formatViemError(err);
+                this.log.error(`Failed to validate blobs`, viemError.message, {
+                    metaMessages: viemError.metaMessages
+                });
+                const validateBlobsData = encodeFunctionData({
+                    abi: RollupAbi,
+                    functionName: 'validateBlobs',
+                    args: [
+                        blobInput
+                    ]
+                });
+                const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
+                this.backupFailedTx({
+                    id: keccak256(validateBlobsData),
+                    failureType: 'simulation',
+                    request: {
+                        to: this.rollupContract.address,
+                        data: validateBlobsData
+                    },
+                    blobData: encodedData.blobs.map((b)=>toHex(b.data)),
+                    l1BlockNumber: l1BlockNumber.toString(),
+                    error: {
+                        message: viemError.message,
+                        name: viemError.name
+                    },
+                    context: {
+                        actions: [
+                            'validate-blobs'
+                        ],
+                        sender: this.getSenderAddress().toString()
+                    }
                 });
                 throw new Error('Failed to validate blobs');
             });
@@ -1261,7 +1527,7 @@ export class SequencerPublisher {
             encodedData.attestationsAndSignersSignature.toViemSignature(),
             blobInput
         ];
-        const { rollupData, simulationResult } = await this.simulateProposeTx(args, timestamp, options);
+        const { rollupData, simulationResult } = await this.simulateProposeTx(args, simulationOverridesPlan);
         return {
             args,
             blobEvaluationGas,
@@ -1272,29 +1538,14 @@ export class SequencerPublisher {
     /**
    * Simulates the propose tx with eth_simulateV1
    * @param args - The propose tx args
-   * @param timestamp - The timestamp to simulate proposal at
    * @returns The simulation result
-   */ async simulateProposeTx(args, timestamp, options) {
+   */ async simulateProposeTx(args, simulationOverridesPlan) {
         const rollupData = encodeFunctionData({
             abi: RollupAbi,
             functionName: 'propose',
             args
         });
-        // override the pending checkpoint number if requested
-        const forcePendingCheckpointNumberStateDiff = (options.forcePendingCheckpointNumber !== undefined ? await this.rollupContract.makePendingCheckpointNumberOverride(options.forcePendingCheckpointNumber) : []).flatMap((override)=>override.stateDiff ?? []);
-        const stateOverrides = [
-            {
-                address: this.rollupContract.address,
-                // @note we override checkBlob to false since blobs are not part simulate()
-                stateDiff: [
-                    {
-                        slot: toPaddedHex(RollupContract.checkBlobStorageSlot, true),
-                        value: toPaddedHex(0n, true)
-                    },
-                    ...forcePendingCheckpointNumberStateDiff
-                ]
-            }
-        ];
+        const stateOverrides = await buildSimulationOverridesStateOverride(this.rollupContract, simulationOverridesPlan);
         // In fisherman mode, simulate as the proposer but with sufficient balance
         if (this.proposerAddressForSimulation) {
             stateOverrides.push({
@@ -1302,6 +1553,8 @@ export class SequencerPublisher {
                 balance: 10n * WEI_CONST * WEI_CONST
             });
         }
+        const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
+        const simTs = this.getSimulationTimestamp(SlotNumber.fromBigInt(args[0].header.slotNumber));
         const simulationResult = await this.l1TxUtils.simulate({
             to: this.rollupContract.address,
             data: rollupData,
@@ -1310,8 +1563,7 @@ export class SequencerPublisher {
                 from: this.proposerAddressForSimulation.toString()
             }
         }, {
-            // @note we add 1n to the timestamp because geth implementation doesn't like simulation timestamp to be equal to the current block timestamp
-            time: timestamp + 1n,
+            time: simTs,
             // @note reth should have a 30m gas limit per block but throws errors that this tx is beyond limit so we increase here
             gasLimit: MAX_L1_TX_LIMIT * 2n
         }, stateOverrides, RollupAbi, {
@@ -1328,7 +1580,29 @@ export class SequencerPublisher {
                     logs: []
                 };
             }
-            this.log.error(`Failed to simulate propose tx`, viemError);
+            this.log.error(`Failed to simulate propose tx`, viemError, {
+                simulationTimestamp: simTs
+            });
+            this.backupFailedTx({
+                id: keccak256(rollupData),
+                failureType: 'simulation',
+                request: {
+                    to: this.rollupContract.address,
+                    data: rollupData
+                },
+                l1BlockNumber: l1BlockNumber.toString(),
+                error: {
+                    message: viemError.message,
+                    name: viemError.name
+                },
+                context: {
+                    actions: [
+                        'propose'
+                    ],
+                    slot: Number(args[0].header.slotNumber),
+                    sender: this.getSenderAddress().toString()
+                }
+            });
             throw err;
         });
         return {
@@ -1336,11 +1610,11 @@ export class SequencerPublisher {
             simulationResult
         };
     }
-    async addProposeTx(checkpoint, encodedData, opts = {}, timestamp) {
+    async addProposeTx(checkpoint, encodedData, opts = {}, preCheck) {
         const slot = checkpoint.header.slotNumber;
         const timer = new Timer();
         const kzg = Blob.getViemKzgInstance();
-        const { rollupData, simulationResult, blobEvaluationGas } = await this.prepareProposeTx(encodedData, timestamp, opts);
+        const { rollupData, simulationResult, blobEvaluationGas } = await this.prepareProposeTx(encodedData, opts.simulationOverridesPlan);
         const startBlock = await this.l1TxUtils.getBlockNumber();
         const gasLimit = this.l1TxUtils.bumpGasLimit(BigInt(Math.ceil(Number(simulationResult.gasUsed) * 64 / 63)) + blobEvaluationGas + SequencerPublisher.MULTICALL_OVERHEAD_GAS_GUESS);
         // Send the blobs to the blob client preemptively. This helps in tests where the sequencer mistakingly thinks that the propose
@@ -1356,9 +1630,10 @@ export class SequencerPublisher {
             },
             lastValidL2Slot: checkpoint.header.slotNumber,
             gasConfig: {
-                ...opts,
+                txTimeoutAt: opts.txTimeoutAt,
                 gasLimit
             },
+            preCheck,
             blobConfig: {
                 blobs: encodedData.blobs.map((b)=>b.data),
                 kzg
@@ -1405,4 +1680,13 @@ export class SequencerPublisher {
             }
         });
     }
+    /** Returns the timestamp of the last L1 slot within a given L2 slot. Used as the simulation timestamp
+   * for eth_simulateV1 calls, since it's guaranteed to be greater than any L1 block produced during the slot. */ getSimulationTimestamp(slot) {
+        const l1Constants = this.epochCache.getL1Constants();
+        return getLastL1SlotTimestampForL2Slot(slot, l1Constants);
+    }
+    /** Returns the timestamp of the next L1 slot boundary after now. */ getNextL1SlotTimestamp() {
+        const l1Constants = this.epochCache.getL1Constants();
+        return getNextL1SlotTimestamp(this.dateProvider.nowInSeconds(), l1Constants);
+    }
 }