@aztec/sequencer-client 0.0.1-commit.29c6b1a3 → 0.0.1-commit.2d9cb6034

package/dest/publisher/sequencer-publisher.js

@@ -372,24 +372,30 @@ function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
 }
 var _dec, _dec1, _dec2, _initProto;
 import { Blob, getBlobsPerL1Block, getPrefixedEthBlobCommitments } from '@aztec/blob-lib';
-import { MULTI_CALL_3_ADDRESS, Multicall3, RollupContract } from '@aztec/ethereum/contracts';
+import { FeeAssetPriceOracle, MULTI_CALL_3_ADDRESS, Multicall3, RollupContract } from '@aztec/ethereum/contracts';
 import { L1FeeAnalyzer } from '@aztec/ethereum/l1-fee-analysis';
-import { WEI_CONST } from '@aztec/ethereum/l1-tx-utils';
-import { FormattedViemError, formatViemError, tryExtractEvent } from '@aztec/ethereum/utils';
+import { MAX_L1_TX_LIMIT, WEI_CONST } from '@aztec/ethereum/l1-tx-utils';
+import { FormattedViemError, formatViemError, mergeAbis, tryExtractEvent } from '@aztec/ethereum/utils';
 import { sumBigint } from '@aztec/foundation/bigint';
 import { toHex as toPaddedHex } from '@aztec/foundation/bigint-buffer';
 import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types';
+import { trimmedBytesLength } from '@aztec/foundation/buffer';
 import { pick } from '@aztec/foundation/collection';
+import { TimeoutError } from '@aztec/foundation/error';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { Signature } from '@aztec/foundation/eth-signature';
 import { createLogger } from '@aztec/foundation/log';
+import { makeBackoff, retry } from '@aztec/foundation/retry';
+import { InterruptibleSleep } from '@aztec/foundation/sleep';
 import { bufferToHex } from '@aztec/foundation/string';
 import { Timer } from '@aztec/foundation/timer';
 import { EmpireBaseAbi, ErrorsAbi, RollupAbi } from '@aztec/l1-artifacts';
 import { encodeSlashConsensusVotes } from '@aztec/slasher';
 import { CommitteeAttestationsAndSigners } from '@aztec/stdlib/block';
+import { getLastL1SlotTimestampForL2Slot, getNextL1SlotTimestamp } from '@aztec/stdlib/epoch-helpers';
 import { getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
-import { encodeFunctionData, toHex } from 'viem';
+import { encodeFunctionData, keccak256, multicall3Abi, toHex } from 'viem';
+import { createL1TxFailedStore } from './l1_tx_failed_store/index.js';
 import { SequencerPublisherMetrics } from './sequencer-publisher-metrics.js';
 export const Actions = [
     'invalidate-by-invalid-attestation',
@@ -429,19 +435,22 @@ export class SequencerPublisher {
     interrupted;
     metrics;
     epochCache;
+    failedTxStore;
     governanceLog;
     slashingLog;
     lastActions;
     isPayloadEmptyCache;
+    payloadProposedCache;
     log;
     ethereumSlotDuration;
+    aztecSlotDuration;
+    /** Date provider for wall-clock time. */ dateProvider;
     blobClient;
     /** Address to use for simulations in fisherman mode (actual proposer's address) */ proposerAddressForSimulation;
+    /** Optional callback to obtain a replacement publisher when the current one fails to send. */ getNextPublisher;
     /** L1 fee analyzer for fisherman mode */ l1FeeAnalyzer;
-    // @note - with blobs, the below estimate seems too large.
-    // Total used for full block from int_l1_pub e2e test: 1m (of which 86k is 1x blob)
-    // Total used for emptier block from above test: 429k (of which 84k is 1x blob)
-    static PROPOSE_GAS_GUESS = 12_000_000n;
+    /** Fee asset price oracle for computing price modifiers from Uniswap V4 */ feeAssetPriceOracle;
+    /** Interruptible sleep used by sendRequestsAt to wait until a target timestamp. */ interruptibleSleep;
     // A CALL to a cold address is 2700 gas
     static MULTICALL_OVERHEAD_GAS_GUESS = 5000n;
     // Gas report for VotingWithSigTest shows a max gas of 100k, but we've seen it cost 700k+ in testnet
@@ -460,16 +469,22 @@ export class SequencerPublisher {
         this.slashingLog = createLogger('sequencer:publisher:slashing');
         this.lastActions = {};
         this.isPayloadEmptyCache = new Map();
+        this.payloadProposedCache = new Set();
+        this.interruptibleSleep = new InterruptibleSleep();
         this.requests = [];
         this.log = deps.log ?? createLogger('sequencer:publisher');
         this.ethereumSlotDuration = BigInt(config.ethereumSlotDuration);
+        this.aztecSlotDuration = BigInt(config.aztecSlotDuration);
+        this.dateProvider = deps.dateProvider;
         this.epochCache = deps.epochCache;
         this.lastActions = deps.lastActions;
         this.blobClient = deps.blobClient;
+        this.dateProvider = deps.dateProvider;
         const telemetry = deps.telemetry ?? getTelemetryClient();
         this.metrics = deps.metrics ?? new SequencerPublisherMetrics(telemetry, 'SequencerPublisher');
         this.tracer = telemetry.getTracer('SequencerPublisher');
         this.l1TxUtils = deps.l1TxUtils;
+        this.getNextPublisher = deps.getNextPublisher;
         this.rollupContract = deps.rollupContract;
         this.govProposerContract = deps.governanceProposerContract;
         this.slashingProposerContract = deps.slashingProposerContract;
@@ -483,10 +498,36 @@ export class SequencerPublisher {
         if (config.fishermanMode) {
             this.l1FeeAnalyzer = new L1FeeAnalyzer(this.l1TxUtils.client, deps.dateProvider, createLogger('sequencer:publisher:fee-analyzer'));
         }
+        // Initialize fee asset price oracle
+        this.feeAssetPriceOracle = new FeeAssetPriceOracle(this.l1TxUtils.client, this.rollupContract, createLogger('sequencer:publisher:price-oracle'));
+        // Initialize failed L1 tx store (optional, for test networks)
+        this.failedTxStore = createL1TxFailedStore(config.l1TxFailedStore, this.log);
+    }
+    /**
+   * Backs up a failed L1 transaction to the configured store for debugging.
+   * Does nothing if no store is configured.
+   */ backupFailedTx(failedTx) {
+        if (!this.failedTxStore) {
+            return;
+        }
+        const tx = {
+            ...failedTx,
+            timestamp: Date.now()
+        };
+        // Fire and forget - don't block on backup
+        void this.failedTxStore.then((store)=>store?.saveFailedTx(tx)).catch((err)=>{
+            this.log.warn(`Failed to backup failed L1 tx to store`, err);
+        });
     }
     getRollupContract() {
         return this.rollupContract;
     }
+    /**
+   * Gets the fee asset price modifier from the oracle.
+   * Returns 0n if the oracle query fails.
+   */ getFeeAssetPriceModifier() {
+        return this.feeAssetPriceOracle.computePriceModifier();
+    }
     getSenderAddress() {
         return this.l1TxUtils.getSenderAddress();
     }
@@ -505,7 +546,7 @@ export class SequencerPublisher {
         this.requests.push(request);
     }
     getCurrentL2Slot() {
-        return this.epochCache.getEpochAndSlotNow().slot;
+        return this.epochCache.getSlotNow();
     }
     /**
    * Clears all pending requests without sending them.
@@ -544,7 +585,7 @@ export class SequencerPublisher {
         // Get the transaction requests
         const l1Requests = requestsToAnalyze.map((r)=>r.request);
         // Start the analysis
-        const analysisId = await this.l1FeeAnalyzer.startAnalysis(l2SlotNumber, gasLimit > 0n ? gasLimit : SequencerPublisher.PROPOSE_GAS_GUESS, l1Requests, blobConfig, onComplete);
+        const analysisId = await this.l1FeeAnalyzer.startAnalysis(l2SlotNumber, gasLimit > 0n ? gasLimit : MAX_L1_TX_LIMIT, l1Requests, blobConfig, onComplete);
         this.log.info('Started L1 fee analysis', {
             analysisId,
             l2SlotNumber: l2SlotNumber.toString(),
@@ -594,15 +635,24 @@ export class SequencerPublisher {
         // @note - we can only have one blob config per bundle
         // find requests with gas and blob configs
         // See https://github.com/AztecProtocol/aztec-packages/issues/11513
-        const gasConfigs = requestsToProcess.filter((request)=>request.gasConfig).map((request)=>request.gasConfig);
-        const blobConfigs = requestsToProcess.filter((request)=>request.blobConfig).map((request)=>request.blobConfig);
+        const gasConfigs = validRequests.filter((request)=>request.gasConfig).map((request)=>request.gasConfig);
+        const blobConfigs = validRequests.filter((request)=>request.blobConfig).map((request)=>request.blobConfig);
         if (blobConfigs.length > 1) {
             throw new Error('Multiple blob configs found');
         }
         const blobConfig = blobConfigs[0];
         // Merge gasConfigs. Yields the sum of gasLimits, and the earliest txTimeoutAt, or undefined if no gasConfig sets them.
         const gasLimits = gasConfigs.map((g)=>g?.gasLimit).filter((g)=>g !== undefined);
-        const gasLimit = gasLimits.length > 0 ? sumBigint(gasLimits) : undefined; // sum
+        let gasLimit = gasLimits.length > 0 ? sumBigint(gasLimits) : undefined; // sum
+        // Cap at L1 block gas limit so the node accepts the tx ("gas limit too high" otherwise).
+        const maxGas = MAX_L1_TX_LIMIT;
+        if (gasLimit !== undefined && gasLimit > maxGas) {
+            this.log.debug('Capping bundled tx gas limit to L1 max', {
+                requested: gasLimit,
+                capped: maxGas
+            });
+            gasLimit = maxGas;
+        }
         const txTimeoutAts = gasConfigs.map((g)=>g?.txTimeoutAt).filter((g)=>g !== undefined);
         const txTimeoutAt = txTimeoutAts.length > 0 ? new Date(Math.min(...txTimeoutAts.map((g)=>g.getTime()))) : undefined; // earliest
         const txConfig = {
@@ -613,12 +663,34 @@ export class SequencerPublisher {
         // This ensures the committee gets precomputed correctly
         validRequests.sort((a, b)=>compareActions(a.action, b.action));
         try {
+            // Capture context for failed tx backup before sending
+            const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
+            const multicallData = encodeFunctionData({
+                abi: multicall3Abi,
+                functionName: 'aggregate3',
+                args: [
+                    validRequests.map((r)=>({
+                            target: r.request.to,
+                            callData: r.request.data,
+                            allowFailure: true
+                        }))
+                ]
+            });
+            const blobDataHex = blobConfig?.blobs?.map((b)=>toHex(b));
+            const txContext = {
+                multicallData,
+                blobData: blobDataHex,
+                l1BlockNumber
+            };
             this.log.debug('Forwarding transactions', {
                 validRequests: validRequests.map((request)=>request.action),
                 txConfig
             });
-            const result = await Multicall3.forward(validRequests.map((request)=>request.request), this.l1TxUtils, txConfig, blobConfig, this.rollupContract.address, this.log);
-            const { successfulActions = [], failedActions = [] } = this.callbackBundledTransactions(validRequests, result);
+            const result = await this.forwardWithPublisherRotation(validRequests, txConfig, blobConfig);
+            if (result === undefined) {
+                return undefined;
+            }
+            const { successfulActions = [], failedActions = [] } = this.callbackBundledTransactions(validRequests, result, txContext);
             return {
                 result,
                 expiredActions,
@@ -638,17 +710,100 @@ export class SequencerPublisher {
             }
         }
     }
-    callbackBundledTransactions(requests, result) {
+    /**
+   * Forwards transactions via Multicall3, rotating to the next available publisher if a send
+   * failure occurs (i.e. the tx never reached the chain).
+   * On-chain reverts and simulation errors are returned as-is without rotation.
+   */ async forwardWithPublisherRotation(validRequests, txConfig, blobConfig) {
+        const triedAddresses = [];
+        let currentPublisher = this.l1TxUtils;
+        while(true){
+            triedAddresses.push(currentPublisher.getSenderAddress());
+            try {
+                const result = await Multicall3.forward(validRequests.map((r)=>r.request), currentPublisher, txConfig, blobConfig, this.rollupContract.address, this.log);
+                this.l1TxUtils = currentPublisher;
+                return result;
+            } catch (err) {
+                if (err instanceof TimeoutError) {
+                    throw err;
+                }
+                const viemError = formatViemError(err);
+                if (!this.getNextPublisher) {
+                    this.log.error('Failed to publish bundled transactions', viemError);
+                    return undefined;
+                }
+                this.log.warn(`Publisher ${currentPublisher.getSenderAddress()} failed to send, rotating to next publisher`, viemError);
+                const nextPublisher = await this.getNextPublisher([
+                    ...triedAddresses
+                ]);
+                if (!nextPublisher) {
+                    this.log.error('All available publishers exhausted, failed to publish bundled transactions');
+                    return undefined;
+                }
+                currentPublisher = nextPublisher;
+            }
+        }
+    }
+    /*
+   * Schedules sending all enqueued requests at (or after) the given timestamp.
+   * Uses InterruptibleSleep so it can be cancelled via interrupt().
+   * Returns the promise for the L1 response (caller should NOT await this in the work loop).
+   */ async sendRequestsAt(submitAfter) {
+        const ms = submitAfter.getTime() - this.dateProvider.now();
+        if (ms > 0) {
+            this.log.debug(`Sleeping ${ms}ms before sending requests`, {
+                submitAfter
+            });
+            await this.interruptibleSleep.sleep(ms);
+        }
+        if (this.interrupted) {
+            return undefined;
+        }
+        return this.sendRequests();
+    }
+    callbackBundledTransactions(requests, result, txContext) {
         const actionsListStr = requests.map((r)=>r.action).join(', ');
         if (result instanceof FormattedViemError) {
             this.log.error(`Failed to publish bundled transactions (${actionsListStr})`, result);
+            this.backupFailedTx({
+                id: keccak256(txContext.multicallData),
+                failureType: 'send-error',
+                request: {
+                    to: MULTI_CALL_3_ADDRESS,
+                    data: txContext.multicallData
+                },
+                blobData: txContext.blobData,
+                l1BlockNumber: txContext.l1BlockNumber.toString(),
+                error: {
+                    message: result.message,
+                    name: result.name
+                },
+                context: {
+                    actions: requests.map((r)=>r.action),
+                    requests: requests.map((r)=>({
+                            action: r.action,
+                            to: r.request.to,
+                            data: r.request.data
+                        })),
+                    sender: this.getSenderAddress().toString()
+                }
+            });
             return {
                 failedActions: requests.map((r)=>r.action)
             };
         } else {
             this.log.verbose(`Published bundled transactions (${actionsListStr})`, {
                 result,
-                requests
+                requests: requests.map((r)=>({
+                        ...r,
+                        // Avoid logging large blob data
+                        blobConfig: r.blobConfig ? {
+                            ...r.blobConfig,
+                            blobs: r.blobConfig.blobs.map((b)=>({
+                                    size: trimmedBytesLength(b)
+                                }))
+                        } : undefined
+                    }))
             });
             const successfulActions = [];
             const failedActions = [];
@@ -659,6 +814,37 @@ export class SequencerPublisher {
                     failedActions.push(request.action);
                 }
             }
+            // Single backup for the whole reverted tx
+            if (failedActions.length > 0 && result?.receipt?.status === 'reverted') {
+                this.backupFailedTx({
+                    id: result.receipt.transactionHash,
+                    failureType: 'revert',
+                    request: {
+                        to: MULTI_CALL_3_ADDRESS,
+                        data: txContext.multicallData
+                    },
+                    blobData: txContext.blobData,
+                    l1BlockNumber: result.receipt.blockNumber.toString(),
+                    receipt: {
+                        transactionHash: result.receipt.transactionHash,
+                        blockNumber: result.receipt.blockNumber.toString(),
+                        gasUsed: (result.receipt.gasUsed ?? 0n).toString(),
+                        status: 'reverted'
+                    },
+                    error: {
+                        message: result.errorMsg ?? 'Transaction reverted'
+                    },
+                    context: {
+                        actions: failedActions,
+                        requests: requests.filter((r)=>failedActions.includes(r.action)).map((r)=>({
+                                action: r.action,
+                                to: r.request.to,
+                                data: r.request.data
+                            })),
+                        sender: this.getSenderAddress().toString()
+                    }
+                });
+            }
             return {
                 successfulActions,
                 failedActions
@@ -666,18 +852,22 @@ export class SequencerPublisher {
         }
     }
     /**
-   * @notice  Will call `canProposeAtNextEthBlock` to make sure that it is possible to propose
+   * @notice  Will call `canProposeAt` to make sure that it is possible to propose
    * @param tipArchive - The archive to check
    * @returns The slot and block number if it is possible to propose, undefined otherwise
-   */ canProposeAtNextEthBlock(tipArchive, msgSender, opts = {}) {
+   */ canProposeAt(tipArchive, msgSender, opts = {}) {
         // TODO: #14291 - should loop through multiple keys to check if any of them can propose
         const ignoredErrors = [
             'SlotAlreadyInChain',
             'InvalidProposer',
             'InvalidArchive'
         ];
-        return this.rollupContract.canProposeAtNextEthBlock(tipArchive.toBuffer(), msgSender.toString(), Number(this.ethereumSlotDuration), {
-            forcePendingCheckpointNumber: opts.forcePendingCheckpointNumber
+        const pipelined = opts.pipelined ?? this.epochCache.isProposerPipeliningEnabled();
+        const slotOffset = pipelined ? this.aztecSlotDuration : 0n;
+        const nextL1SlotTs = this.getNextL1SlotTimestamp() + slotOffset;
+        return this.rollupContract.canProposeAt(tipArchive.toBuffer(), msgSender.toString(), nextL1SlotTs, {
+            forcePendingCheckpointNumber: opts.forcePendingCheckpointNumber,
+            forceArchive: opts.forceArchive
         }).catch((err)=>{
             if (err instanceof FormattedViemError && ignoredErrors.find((e)=>err.message.includes(e))) {
                 this.log.warn(`Failed canProposeAtTime check with ${ignoredErrors.find((e)=>err.message.includes(e))}`, {
@@ -708,7 +898,7 @@ export class SequencerPublisher {
             header.blobsHash.toString(),
             flags
         ];
-        const ts = BigInt((await this.l1TxUtils.getBlock()).timestamp + this.ethereumSlotDuration);
+        const ts = this.getSimulationTimestamp(header.slotNumber);
         const stateOverrides = await this.rollupContract.makePendingCheckpointNumberOverride(opts?.forcePendingCheckpointNumber);
         let balance = 0n;
         if (this.config.fishermanMode) {
@@ -731,7 +921,7 @@ export class SequencerPublisher {
             }),
             from: MULTI_CALL_3_ADDRESS
         }, {
-            time: ts + 1n
+            time: ts
         }, stateOverrides);
         this.log.debug(`Simulated validateHeader`);
     }
@@ -761,8 +951,12 @@ export class SequencerPublisher {
             ...logData,
             request
         });
+        const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
         try {
-            const { gasUsed } = await this.l1TxUtils.simulate(request, undefined, undefined, ErrorsAbi);
+            const { gasUsed } = await this.l1TxUtils.simulate(request, undefined, undefined, mergeAbis([
+                request.abi ?? [],
+                ErrorsAbi
+            ]));
             this.log.verbose(`Simulation for invalidate checkpoint ${checkpointNumber} succeeded`, {
                 ...logData,
                 request,
@@ -773,20 +967,21 @@ export class SequencerPublisher {
                 gasUsed,
                 checkpointNumber,
                 forcePendingCheckpointNumber: CheckpointNumber(checkpointNumber - 1),
+                lastArchive: validationResult.checkpoint.lastArchive,
                 reason
             };
         } catch (err) {
             const viemError = formatViemError(err);
             // If the error is due to the checkpoint not being in the pending chain, and it was indeed removed by someone else,
             // we can safely ignore it and return undefined so we go ahead with checkpoint building.
-            if (viemError.message?.includes('Rollup__BlockNotInPendingChain')) {
+            if (viemError.message?.includes('Rollup__CheckpointNotInPendingChain')) {
                 this.log.verbose(`Simulation for invalidate checkpoint ${checkpointNumber} failed due to checkpoint not being in pending chain`, {
                     ...logData,
                     request,
                     error: viemError.message
                 });
-                const latestPendingCheckpointNumber = await this.rollupContract.getCheckpointNumber();
-                if (latestPendingCheckpointNumber < checkpointNumber) {
+                const latestProposedCheckpointNumber = await this.rollupContract.getCheckpointNumber();
+                if (latestProposedCheckpointNumber < checkpointNumber) {
                     this.log.verbose(`Checkpoint ${checkpointNumber} has already been invalidated`, {
                         ...logData
                     });
@@ -800,6 +995,27 @@ export class SequencerPublisher {
             }
             // Otherwise, throw. We cannot build the next checkpoint if we cannot invalidate the previous one.
             this.log.error(`Simulation for invalidate checkpoint ${checkpointNumber} failed`, viemError, logData);
+            this.backupFailedTx({
+                id: keccak256(request.data),
+                failureType: 'simulation',
+                request: {
+                    to: request.to,
+                    data: request.data,
+                    value: request.value?.toString()
+                },
+                l1BlockNumber: l1BlockNumber.toString(),
+                error: {
+                    message: viemError.message,
+                    name: viemError.name
+                },
+                context: {
+                    actions: [
+                        `invalidate-${reason}`
+                    ],
+                    checkpointNumber,
+                    sender: this.getSenderAddress().toString()
+                }
+            });
             throw new Error(`Failed to simulate invalidate checkpoint ${checkpointNumber}`, {
                 cause: viemError
             });
@@ -826,30 +1042,15 @@ export class SequencerPublisher {
         }
     }
     /** Simulates `propose` to make sure that the checkpoint is valid for submission */ async validateCheckpointForSubmission(checkpoint, attestationsAndSigners, attestationsAndSignersSignature, options) {
-        const ts = BigInt((await this.l1TxUtils.getBlock()).timestamp + this.ethereumSlotDuration);
-        // TODO(palla/mbps): This should not be needed, there's no flow where we propose with zero attestations. Or is there?
-        // If we have no attestations, we still need to provide the empty attestations
-        // so that the committee is recalculated correctly
-        // const ignoreSignatures = attestationsAndSigners.attestations.length === 0;
-        // if (ignoreSignatures) {
-        //   const { committee } = await this.epochCache.getCommittee(block.header.globalVariables.slotNumber);
-        //   if (!committee) {
-        //     this.log.warn(`No committee found for slot ${block.header.globalVariables.slotNumber}`);
-        //     throw new Error(`No committee found for slot ${block.header.globalVariables.slotNumber}`);
-        //   }
-        //   attestationsAndSigners.attestations = committee.map(committeeMember =>
-        //     CommitteeAttestation.fromAddress(committeeMember),
-        //   );
-        // }
         const blobFields = checkpoint.toBlobFields();
-        const blobs = getBlobsPerL1Block(blobFields);
+        const blobs = await getBlobsPerL1Block(blobFields);
         const blobInput = getPrefixedEthBlobCommitments(blobs);
         const args = [
             {
                 header: checkpoint.header.toViem(),
                 archive: toHex(checkpoint.archive.root.toBuffer()),
                 oracleInput: {
-                    feeAssetPriceModifier: 0n
+                    feeAssetPriceModifier: checkpoint.feeAssetPriceModifier
                 }
             },
             attestationsAndSigners.getPackedAttestations(),
@@ -857,10 +1058,9 @@ export class SequencerPublisher {
             attestationsAndSignersSignature.toViemSignature(),
             blobInput
         ];
-        await this.simulateProposeTx(args, ts, options);
-        return ts;
+        await this.simulateProposeTx(args, options);
     }
-    async enqueueCastSignalHelper(slotNumber, timestamp, signalType, payload, base, signerAddress, signer) {
+    async enqueueCastSignalHelper(slotNumber, signalType, payload, base, signerAddress, signer) {
         if (this.lastActions[signalType] && this.lastActions[signalType] === slotNumber) {
             this.log.debug(`Skipping duplicate vote cast signal ${signalType} for slot ${slotNumber}`);
             return false;
@@ -884,6 +1084,28 @@ export class SequencerPublisher {
             this.log.warn(`Skipping vote cast for payload with empty code`);
             return false;
         }
+        // Check if payload was already submitted to governance
+        const cacheKey = payload.toString();
+        if (!this.payloadProposedCache.has(cacheKey)) {
+            try {
+                const l1StartBlock = await this.rollupContract.getL1StartBlock();
+                const proposed = await retry(()=>base.hasPayloadBeenProposed(payload.toString(), l1StartBlock), 'Check if payload was proposed', makeBackoff([
+                    0,
+                    1,
+                    2
+                ]), this.log, true);
+                if (proposed) {
+                    this.payloadProposedCache.add(cacheKey);
+                }
+            } catch (err) {
+                this.log.warn(`Failed to check if payload ${payload} was proposed after retries, skipping signal`, err);
+                return false;
+            }
+        }
+        if (this.payloadProposedCache.has(cacheKey)) {
+            this.log.info(`Payload ${payload} was already proposed to governance, stopping signals`);
+            return false;
+        }
         const cachedLastVote = this.lastActions[signalType];
         this.lastActions[signalType] = slotNumber;
         const action = signalType;
@@ -894,15 +1116,45 @@ export class SequencerPublisher {
             signer: this.l1TxUtils.client.account?.address,
             lastValidL2Slot: slotNumber
         });
+        const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
+        const timestamp = this.getSimulationTimestamp(slotNumber);
         try {
             await this.l1TxUtils.simulate(request, {
                 time: timestamp
-            }, [], ErrorsAbi);
+            }, [], mergeAbis([
+                request.abi ?? [],
+                ErrorsAbi
+            ]));
             this.log.debug(`Simulation for ${action} at slot ${slotNumber} succeeded`, {
                 request
             });
         } catch (err) {
-            this.log.error(`Failed simulation for ${action} at slot ${slotNumber} (enqueuing the action anyway)`, err);
+            const viemError = formatViemError(err);
+            this.log.error(`Failed simulation for ${action} at slot ${slotNumber} (enqueuing the action anyway)`, viemError, {
+                simulationTimestamp: timestamp,
+                l1BlockNumber
+            });
+            this.backupFailedTx({
+                id: keccak256(request.data),
+                failureType: 'simulation',
+                request: {
+                    to: request.to,
+                    data: request.data,
+                    value: request.value?.toString()
+                },
+                l1BlockNumber: l1BlockNumber.toString(),
+                error: {
+                    message: viemError.message,
+                    name: viemError.name
+                },
+                context: {
+                    actions: [
+                        action
+                    ],
+                    slot: slotNumber,
+                    sender: this.getSenderAddress().toString()
+                }
+            });
         // Yes, we enqueue the request anyway, in case there was a bug with the simulation itself
         }
         // TODO(palla/slash): All votes (governance and slashing) should txTimeoutAt at the end of the slot.
@@ -946,12 +1198,11 @@ export class SequencerPublisher {
     /**
    * Enqueues a governance castSignal transaction to cast a signal for a given slot number.
    * @param slotNumber - The slot number to cast a signal for.
-   * @param timestamp - The timestamp of the slot to cast a signal for.
    * @returns True if the signal was successfully enqueued, false otherwise.
-   */ enqueueGovernanceCastSignal(governancePayload, slotNumber, timestamp, signerAddress, signer) {
-        return this.enqueueCastSignalHelper(slotNumber, timestamp, 'governance-signal', governancePayload, this.govProposerContract, signerAddress, signer);
+   */ enqueueGovernanceCastSignal(governancePayload, slotNumber, signerAddress, signer) {
+        return this.enqueueCastSignalHelper(slotNumber, 'governance-signal', governancePayload, this.govProposerContract, signerAddress, signer);
     }
-    /** Enqueues all slashing actions as returned by the slasher client. */ async enqueueSlashingActions(actions, slotNumber, timestamp, signerAddress, signer) {
+    /** Enqueues all slashing actions as returned by the slasher client. */ async enqueueSlashingActions(actions, slotNumber, signerAddress, signer) {
         if (actions.length === 0) {
             this.log.debug(`No slashing actions to enqueue for slot ${slotNumber}`);
             return false;
@@ -967,7 +1218,7 @@ export class SequencerPublisher {
                         this.log.debug(`Enqueuing slashing vote for payload ${action.payload} at slot ${slotNumber}`, {
                             signerAddress
                         });
-                        await this.enqueueCastSignalHelper(slotNumber, timestamp, 'empire-slashing-signal', action.payload, this.slashingProposerContract, signerAddress, signer);
+                        await this.enqueueCastSignalHelper(slotNumber, 'empire-slashing-signal', action.payload, this.slashingProposerContract, signerAddress, signer);
                         break;
                     }
                 case 'create-empire-payload':
@@ -977,7 +1228,7 @@ export class SequencerPublisher {
                             signerAddress
                         });
                         const request = this.slashFactoryContract.buildCreatePayloadRequest(action.data);
-                        await this.simulateAndEnqueueRequest('create-empire-payload', request, (receipt)=>!!this.slashFactoryContract.tryExtractSlashPayloadCreatedEvent(receipt.logs), slotNumber, timestamp);
+                        await this.simulateAndEnqueueRequest('create-empire-payload', request, (receipt)=>!!this.slashFactoryContract.tryExtractSlashPayloadCreatedEvent(receipt.logs), slotNumber);
                         break;
                     }
                 case 'execute-empire-payload':
@@ -992,7 +1243,7 @@ export class SequencerPublisher {
                         }
                         const empireSlashingProposer = this.slashingProposerContract;
                         const request = empireSlashingProposer.buildExecuteRoundRequest(action.round);
-                        await this.simulateAndEnqueueRequest('execute-empire-payload', request, (receipt)=>!!empireSlashingProposer.tryExtractPayloadSubmittedEvent(receipt.logs), slotNumber, timestamp);
+                        await this.simulateAndEnqueueRequest('execute-empire-payload', request, (receipt)=>!!empireSlashingProposer.tryExtractPayloadSubmittedEvent(receipt.logs), slotNumber);
                         break;
                     }
                 case 'vote-offenses':
@@ -1010,7 +1261,7 @@ export class SequencerPublisher {
                         const tallySlashingProposer = this.slashingProposerContract;
                         const votes = bufferToHex(encodeSlashConsensusVotes(action.votes));
                         const request = await tallySlashingProposer.buildVoteRequestFromSigner(votes, slotNumber, signer);
-                        await this.simulateAndEnqueueRequest('vote-offenses', request, (receipt)=>!!tallySlashingProposer.tryExtractVoteCastEvent(receipt.logs), slotNumber, timestamp);
+                        await this.simulateAndEnqueueRequest('vote-offenses', request, (receipt)=>!!tallySlashingProposer.tryExtractVoteCastEvent(receipt.logs), slotNumber);
                         break;
                     }
                 case 'execute-slash':
@@ -1026,7 +1277,7 @@ export class SequencerPublisher {
                         }
                         const tallySlashingProposer = this.slashingProposerContract;
                         const request = tallySlashingProposer.buildExecuteRoundRequest(action.round, action.committees);
-                        await this.simulateAndEnqueueRequest('execute-slash', request, (receipt)=>!!tallySlashingProposer.tryExtractRoundExecutedEvent(receipt.logs), slotNumber, timestamp);
+                        await this.simulateAndEnqueueRequest('execute-slash', request, (receipt)=>!!tallySlashingProposer.tryExtractRoundExecutedEvent(receipt.logs), slotNumber);
                         break;
                     }
                 default:
@@ -1041,22 +1292,22 @@ export class SequencerPublisher {
     /** Simulates and enqueues a proposal for a checkpoint on L1 */ async enqueueProposeCheckpoint(checkpoint, attestationsAndSigners, attestationsAndSignersSignature, opts = {}) {
         const checkpointHeader = checkpoint.header;
         const blobFields = checkpoint.toBlobFields();
-        const blobs = getBlobsPerL1Block(blobFields);
+        const blobs = await getBlobsPerL1Block(blobFields);
         const proposeTxArgs = {
             header: checkpointHeader,
             archive: checkpoint.archive.root.toBuffer(),
             blobs,
             attestationsAndSigners,
-            attestationsAndSignersSignature
+            attestationsAndSignersSignature,
+            feeAssetPriceModifier: checkpoint.feeAssetPriceModifier
         };
-        let ts;
         try {
             // @note  This will make sure that we are passing the checks for our header ASSUMING that the data is also made available
             //        This means that we can avoid the simulation issues in later checks.
             //        By simulation issue, I mean the fact that the block.timestamp is equal to the last block, not the next, which
             //        make time consistency checks break.
             // TODO(palla): Check whether we're validating twice, once here and once within addProposeTx, since we call simulateProposeTx in both places.
-            ts = await this.validateCheckpointForSubmission(checkpoint, attestationsAndSigners, attestationsAndSignersSignature, opts);
+            await this.validateCheckpointForSubmission(checkpoint, attestationsAndSigners, attestationsAndSignersSignature, opts);
         } catch (err) {
             this.log.error(`Checkpoint validation failed. ${err instanceof Error ? err.message : 'No error message'}`, err, {
                 ...checkpoint.getStats(),
@@ -1069,7 +1320,7 @@ export class SequencerPublisher {
             ...checkpoint.toCheckpointInfo(),
             ...opts
         });
-        await this.addProposeTx(checkpoint, proposeTxArgs, opts, ts);
+        await this.addProposeTx(checkpoint, proposeTxArgs, opts);
     }
     enqueueInvalidateCheckpoint(request, opts = {}) {
         if (!request) {
@@ -1110,7 +1361,8 @@ export class SequencerPublisher {
             }
         });
     }
-    async simulateAndEnqueueRequest(action, request, checkSuccess, slotNumber, timestamp) {
+    async simulateAndEnqueueRequest(action, request, checkSuccess, slotNumber) {
+        const timestamp = this.getSimulationTimestamp(slotNumber);
         const logData = {
             slotNumber,
             timestamp,
@@ -1123,28 +1375,60 @@ export class SequencerPublisher {
         const cachedLastActionSlot = this.lastActions[action];
         this.lastActions[action] = slotNumber;
         this.log.debug(`Simulating ${action} for slot ${slotNumber}`, logData);
+        const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
         let gasUsed;
+        const simulateAbi = mergeAbis([
+            request.abi ?? [],
+            ErrorsAbi
+        ]);
         try {
             ({ gasUsed } = await this.l1TxUtils.simulate(request, {
                 time: timestamp
-            }, [], ErrorsAbi)); // TODO(palla/slash): Check the timestamp logic
+            }, [], simulateAbi));
             this.log.verbose(`Simulation for ${action} succeeded`, {
                 ...logData,
                 request,
                 gasUsed
             });
         } catch (err) {
-            const viemError = formatViemError(err);
+            const viemError = formatViemError(err, simulateAbi);
             this.log.error(`Simulation for ${action} at ${slotNumber} failed`, viemError, logData);
+            this.backupFailedTx({
+                id: keccak256(request.data),
+                failureType: 'simulation',
+                request: {
+                    to: request.to,
+                    data: request.data,
+                    value: request.value?.toString()
+                },
+                l1BlockNumber: l1BlockNumber.toString(),
+                error: {
+                    message: viemError.message,
+                    name: viemError.name
+                },
+                context: {
+                    actions: [
+                        action
+                    ],
+                    slot: slotNumber,
+                    sender: this.getSenderAddress().toString()
+                }
+            });
             return false;
         }
         // We issued the simulation against the rollup contract, so we need to account for the overhead of the multicall3
         const gasLimit = this.l1TxUtils.bumpGasLimit(BigInt(Math.ceil(Number(gasUsed) * 64 / 63)));
         logData.gasLimit = gasLimit;
+        // Store the ABI used for simulation on the request so Multicall3.forward can decode errors
+        // when the tx is sent and a revert is diagnosed via simulation.
+        const requestWithAbi = {
+            ...request,
+            abi: simulateAbi
+        };
         this.log.debug(`Enqueuing ${action}`, logData);
         this.addRequest({
             action,
-            request,
+            request: requestWithAbi,
             gasConfig: {
                 gasLimit
             },
@@ -1175,13 +1459,14 @@ export class SequencerPublisher {
    * A call to `restart` is required before you can continue publishing.
    */ interrupt() {
         this.interrupted = true;
+        this.interruptibleSleep.interrupt();
         this.l1TxUtils.interrupt();
     }
     /** Restarts the publisher after calling `interrupt`. */ restart() {
         this.interrupted = false;
         this.l1TxUtils.restart();
     }
-    async prepareProposeTx(encodedData, timestamp, options) {
+    async prepareProposeTx(encodedData, options) {
         const kzg = Blob.getViemKzgInstance();
         const blobInput = getPrefixedEthBlobCommitments(encodedData.blobs);
         this.log.debug('Validating blob input', {
@@ -1208,10 +1493,38 @@ export class SequencerPublisher {
             }, {}, {
                 blobs: encodedData.blobs.map((b)=>b.data),
                 kzg
-            }).catch((err)=>{
-                const { message, metaMessages } = formatViemError(err);
-                this.log.error(`Failed to validate blobs`, message, {
-                    metaMessages
+            }).catch(async (err)=>{
+                const viemError = formatViemError(err);
+                this.log.error(`Failed to validate blobs`, viemError.message, {
+                    metaMessages: viemError.metaMessages
+                });
+                const validateBlobsData = encodeFunctionData({
+                    abi: RollupAbi,
+                    functionName: 'validateBlobs',
+                    args: [
+                        blobInput
+                    ]
+                });
+                const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
+                this.backupFailedTx({
+                    id: keccak256(validateBlobsData),
+                    failureType: 'simulation',
+                    request: {
+                        to: this.rollupContract.address,
+                        data: validateBlobsData
+                    },
+                    blobData: encodedData.blobs.map((b)=>toHex(b.data)),
+                    l1BlockNumber: l1BlockNumber.toString(),
+                    error: {
+                        message: viemError.message,
+                        name: viemError.name
+                    },
+                    context: {
+                        actions: [
+                            'validate-blobs'
+                        ],
+                        sender: this.getSenderAddress().toString()
+                    }
                 });
                 throw new Error('Failed to validate blobs');
             });
@@ -1222,8 +1535,7 @@ export class SequencerPublisher {
                 header: encodedData.header.toViem(),
                 archive: toHex(encodedData.archive),
                 oracleInput: {
-                    // We are currently not modifying these. See #9963
-                    feeAssetPriceModifier: 0n
+                    feeAssetPriceModifier: encodedData.feeAssetPriceModifier
                 }
             },
             encodedData.attestationsAndSigners.getPackedAttestations(),
@@ -1231,7 +1543,7 @@ export class SequencerPublisher {
             encodedData.attestationsAndSignersSignature.toViemSignature(),
             blobInput
         ];
-        const { rollupData, simulationResult } = await this.simulateProposeTx(args, timestamp, options);
+        const { rollupData, simulationResult } = await this.simulateProposeTx(args, options);
         return {
             args,
             blobEvaluationGas,
@@ -1242,16 +1554,17 @@ export class SequencerPublisher {
     /**
    * Simulates the propose tx with eth_simulateV1
    * @param args - The propose tx args
-   * @param timestamp - The timestamp to simulate proposal at
    * @returns The simulation result
-   */ async simulateProposeTx(args, timestamp, options) {
+   */ async simulateProposeTx(args, options) {
         const rollupData = encodeFunctionData({
             abi: RollupAbi,
             functionName: 'propose',
             args
         });
-        // override the pending checkpoint number if requested
+        // override the proposed checkpoint number if requested
         const forcePendingCheckpointNumberStateDiff = (options.forcePendingCheckpointNumber !== undefined ? await this.rollupContract.makePendingCheckpointNumberOverride(options.forcePendingCheckpointNumber) : []).flatMap((override)=>override.stateDiff ?? []);
+        // override the fee header for a specific checkpoint number if requested (used when pipelining)
+        const forceProposedFeeHeaderStateDiff = (options.forceProposedFeeHeader !== undefined ? await this.rollupContract.makeFeeHeaderOverride(options.forceProposedFeeHeader.checkpointNumber, options.forceProposedFeeHeader.feeHeader) : []).flatMap((override)=>override.stateDiff ?? []);
         const stateOverrides = [
             {
                 address: this.rollupContract.address,
@@ -1261,7 +1574,8 @@ export class SequencerPublisher {
                         slot: toPaddedHex(RollupContract.checkBlobStorageSlot, true),
                         value: toPaddedHex(0n, true)
                     },
-                    ...forcePendingCheckpointNumberStateDiff
+                    ...forcePendingCheckpointNumberStateDiff,
+                    ...forceProposedFeeHeaderStateDiff
                 ]
             }
         ];
@@ -1272,21 +1586,22 @@ export class SequencerPublisher {
                 balance: 10n * WEI_CONST * WEI_CONST
             });
         }
+        const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
+        const simTs = this.getSimulationTimestamp(SlotNumber.fromBigInt(args[0].header.slotNumber));
         const simulationResult = await this.l1TxUtils.simulate({
             to: this.rollupContract.address,
             data: rollupData,
-            gas: SequencerPublisher.PROPOSE_GAS_GUESS,
+            gas: MAX_L1_TX_LIMIT,
             ...this.proposerAddressForSimulation && {
                 from: this.proposerAddressForSimulation.toString()
             }
         }, {
-            // @note we add 1n to the timestamp because geth implementation doesn't like simulation timestamp to be equal to the current block timestamp
-            time: timestamp + 1n,
+            time: simTs,
             // @note reth should have a 30m gas limit per block but throws errors that this tx is beyond limit so we increase here
-            gasLimit: SequencerPublisher.PROPOSE_GAS_GUESS * 2n
+            gasLimit: MAX_L1_TX_LIMIT * 2n
         }, stateOverrides, RollupAbi, {
             // @note fallback gas estimate to use if the node doesn't support simulation API
-            fallbackGasEstimate: SequencerPublisher.PROPOSE_GAS_GUESS
+            fallbackGasEstimate: MAX_L1_TX_LIMIT
         }).catch((err)=>{
             // In fisherman mode, we expect ValidatorSelection__MissingProposerSignature since fisherman doesn't have proposer signature
             const viemError = formatViemError(err);
@@ -1294,11 +1609,33 @@ export class SequencerPublisher {
                 this.log.debug(`Ignoring expected ValidatorSelection__MissingProposerSignature error in fisherman mode`);
                 // Return a minimal simulation result with the fallback gas estimate
                 return {
-                    gasUsed: SequencerPublisher.PROPOSE_GAS_GUESS,
+                    gasUsed: MAX_L1_TX_LIMIT,
                     logs: []
                 };
             }
-            this.log.error(`Failed to simulate propose tx`, viemError);
+            this.log.error(`Failed to simulate propose tx`, viemError, {
+                simulationTimestamp: simTs
+            });
+            this.backupFailedTx({
+                id: keccak256(rollupData),
+                failureType: 'simulation',
+                request: {
+                    to: this.rollupContract.address,
+                    data: rollupData
+                },
+                l1BlockNumber: l1BlockNumber.toString(),
+                error: {
+                    message: viemError.message,
+                    name: viemError.name
+                },
+                context: {
+                    actions: [
+                        'propose'
+                    ],
+                    slot: Number(args[0].header.slotNumber),
+                    sender: this.getSenderAddress().toString()
+                }
+            });
             throw err;
         });
         return {
@@ -1306,11 +1643,11 @@ export class SequencerPublisher {
             simulationResult
         };
     }
-    async addProposeTx(checkpoint, encodedData, opts = {}, timestamp) {
+    async addProposeTx(checkpoint, encodedData, opts = {}) {
         const slot = checkpoint.header.slotNumber;
         const timer = new Timer();
         const kzg = Blob.getViemKzgInstance();
-        const { rollupData, simulationResult, blobEvaluationGas } = await this.prepareProposeTx(encodedData, timestamp, opts);
+        const { rollupData, simulationResult, blobEvaluationGas } = await this.prepareProposeTx(encodedData, opts);
         const startBlock = await this.l1TxUtils.getBlockNumber();
         const gasLimit = this.l1TxUtils.bumpGasLimit(BigInt(Math.ceil(Number(simulationResult.gasUsed) * 64 / 63)) + blobEvaluationGas + SequencerPublisher.MULTICALL_OVERHEAD_GAS_GUESS);
         // Send the blobs to the blob client preemptively. This helps in tests where the sequencer mistakingly thinks that the propose
@@ -1375,4 +1712,13 @@ export class SequencerPublisher {
             }
         });
     }
+    /** Returns the timestamp of the last L1 slot within a given L2 slot. Used as the simulation timestamp
+   * for eth_simulateV1 calls, since it's guaranteed to be greater than any L1 block produced during the slot. */ getSimulationTimestamp(slot) {
+        const l1Constants = this.epochCache.getL1Constants();
+        return getLastL1SlotTimestampForL2Slot(slot, l1Constants);
+    }
+    /** Returns the timestamp of the next L1 slot boundary after now. */ getNextL1SlotTimestamp() {
+        const l1Constants = this.epochCache.getL1Constants();
+        return getNextL1SlotTimestamp(this.dateProvider.nowInSeconds(), l1Constants);
+    }
 }