@aztec/sequencer-client 0.0.1-commit.03f7ef2 → 0.0.1-commit.04852196a

package/src/publisher/sequencer-publisher.ts

@@ -1,9 +1,10 @@
-import { type BlobClientInterface, createBlobClient } from '@aztec/blob-client/client';
+import type { BlobClientInterface } from '@aztec/blob-client/client';
 import { Blob, getBlobsPerL1Block, getPrefixedEthBlobCommitments } from '@aztec/blob-lib';
 import type { EpochCache } from '@aztec/epoch-cache';
 import type { L1ContractsConfig } from '@aztec/ethereum/config';
 import {
   type EmpireSlashingProposerContract,
+  FeeAssetPriceOracle,
   type GovernanceProposerContract,
   type IEmpireBase,
   MULTI_CALL_3_ADDRESS,
@@ -18,33 +19,46 @@ import {
   type L1BlobInputs,
   type L1TxConfig,
   type L1TxRequest,
+  type L1TxUtils,
+  MAX_L1_TX_LIMIT,
   type TransactionStats,
   WEI_CONST,
 } from '@aztec/ethereum/l1-tx-utils';
-import type { L1TxUtilsWithBlobs } from '@aztec/ethereum/l1-tx-utils-with-blobs';
-import { FormattedViemError, formatViemError, tryExtractEvent } from '@aztec/ethereum/utils';
+import { FormattedViemError, formatViemError, mergeAbis, tryExtractEvent } from '@aztec/ethereum/utils';
 import { sumBigint } from '@aztec/foundation/bigint';
 import { toHex as toPaddedHex } from '@aztec/foundation/bigint-buffer';
-import { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types';
+import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types';
 import { pick } from '@aztec/foundation/collection';
 import type { Fr } from '@aztec/foundation/curves/bn254';
+import { TimeoutError } from '@aztec/foundation/error';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { Signature, type ViemSignature } from '@aztec/foundation/eth-signature';
 import { type Logger, createLogger } from '@aztec/foundation/log';
+import { makeBackoff, retry } from '@aztec/foundation/retry';
 import { bufferToHex } from '@aztec/foundation/string';
 import { DateProvider, Timer } from '@aztec/foundation/timer';
 import { EmpireBaseAbi, ErrorsAbi, RollupAbi } from '@aztec/l1-artifacts';
 import { type ProposerSlashAction, encodeSlashConsensusVotes } from '@aztec/slasher';
-import { CommitteeAttestationsAndSigners, type ValidateBlockResult } from '@aztec/stdlib/block';
+import { CommitteeAttestationsAndSigners, type ValidateCheckpointResult } from '@aztec/stdlib/block';
 import type { Checkpoint } from '@aztec/stdlib/checkpoint';
 import { SlashFactoryContract } from '@aztec/stdlib/l1-contracts';
 import type { CheckpointHeader } from '@aztec/stdlib/rollup';
 import type { L1PublishCheckpointStats } from '@aztec/stdlib/stats';
-import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
+import { type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
 
-import { type StateOverride, type TransactionReceipt, type TypedDataDefinition, encodeFunctionData, toHex } from 'viem';
-
-import type { PublisherConfig, TxSenderConfig } from './config.js';
+import {
+  type Hex,
+  type StateOverride,
+  type TransactionReceipt,
+  type TypedDataDefinition,
+  encodeFunctionData,
+  keccak256,
+  multicall3Abi,
+  toHex,
+} from 'viem';
+
+import type { SequencerPublisherConfig } from './config.js';
+import { type FailedL1Tx, type L1TxFailedStore, createL1TxFailedStore } from './l1_tx_failed_store/index.js';
 import { SequencerPublisherMetrics } from './sequencer-publisher-metrics.js';
 
 /** Arguments to the process method of the rollup contract */
@@ -59,6 +73,8 @@ type L1ProcessArgs = {
   attestationsAndSigners: CommitteeAttestationsAndSigners;
   /** Attestations and signers signature */
   attestationsAndSignersSignature: Signature;
+  /** The fee asset price modifier in basis points (from oracle) */
+  feeAssetPriceModifier: bigint;
 };
 
 export const Actions = [
@@ -80,12 +96,12 @@ type GovernanceSignalAction = Extract<Action, 'governance-signal' | 'empire-slas
 // Sorting for actions such that invalidations go before proposals, and proposals go before votes
 export const compareActions = (a: Action, b: Action) => Actions.indexOf(a) - Actions.indexOf(b);
 
-export type InvalidateBlockRequest = {
+export type InvalidateCheckpointRequest = {
   request: L1TxRequest;
   reason: 'invalid-attestation' | 'insufficient-attestations';
   gasUsed: bigint;
-  blockNumber: BlockNumber;
-  forcePendingBlockNumber: BlockNumber;
+  checkpointNumber: CheckpointNumber;
+  forcePendingCheckpointNumber: CheckpointNumber;
 };
 
 interface RequestWithExpiry {
@@ -104,6 +120,7 @@ export class SequencerPublisher {
   private interrupted = false;
   private metrics: SequencerPublisherMetrics;
   public epochCache: EpochCache;
+  private failedTxStore?: Promise<L1TxFailedStore | undefined>;
 
   protected governanceLog = createLogger('sequencer:publisher:governance');
   protected slashingLog = createLogger('sequencer:publisher:slashing');
@@ -111,6 +128,7 @@ export class SequencerPublisher {
   protected lastActions: Partial<Record<Action, SlotNumber>> = {};
 
   private isPayloadEmptyCache: Map<string, boolean> = new Map<string, boolean>();
+  private payloadProposedCache: Set<string> = new Set<string>();
 
   protected log: Logger;
   protected ethereumSlotDuration: bigint;
@@ -120,12 +138,14 @@ export class SequencerPublisher {
   /** Address to use for simulations in fisherman mode (actual proposer's address) */
   private proposerAddressForSimulation?: EthAddress;
 
+  /** Optional callback to obtain a replacement publisher when the current one fails to send. */
+  private getNextPublisher?: (excludeAddresses: EthAddress[]) => Promise<L1TxUtils | undefined>;
+
   /** L1 fee analyzer for fisherman mode */
   private l1FeeAnalyzer?: L1FeeAnalyzer;
-  // @note - with blobs, the below estimate seems too large.
-  // Total used for full block from int_l1_pub e2e test: 1m (of which 86k is 1x blob)
-  // Total used for emptier block from above test: 429k (of which 84k is 1x blob)
-  public static PROPOSE_GAS_GUESS: bigint = 12_000_000n;
+
+  /** Fee asset price oracle for computing price modifiers from Uniswap V4 */
+  private feeAssetPriceOracle: FeeAssetPriceOracle;
 
   // A CALL to a cold address is 2700 gas
   public static MULTICALL_OVERHEAD_GAS_GUESS = 5000n;
@@ -133,20 +153,23 @@ export class SequencerPublisher {
   // Gas report for VotingWithSigTest shows a max gas of 100k, but we've seen it cost 700k+ in testnet
   public static VOTE_GAS_GUESS: bigint = 800_000n;
 
-  public l1TxUtils: L1TxUtilsWithBlobs;
+  public l1TxUtils: L1TxUtils;
   public rollupContract: RollupContract;
   public govProposerContract: GovernanceProposerContract;
   public slashingProposerContract: EmpireSlashingProposerContract | TallySlashingProposerContract | undefined;
   public slashFactoryContract: SlashFactoryContract;
 
+  public readonly tracer: Tracer;
+
   protected requests: RequestWithExpiry[] = [];
 
   constructor(
-    private config: TxSenderConfig & PublisherConfig & Pick<L1ContractsConfig, 'ethereumSlotDuration'>,
+    private config: Pick<SequencerPublisherConfig, 'fishermanMode' | 'l1TxFailedStore'> &
+      Pick<L1ContractsConfig, 'ethereumSlotDuration'> & { l1ChainId: number },
     deps: {
       telemetry?: TelemetryClient;
-      blobClient?: BlobClientInterface;
-      l1TxUtils: L1TxUtilsWithBlobs;
+      blobClient: BlobClientInterface;
+      l1TxUtils: L1TxUtils;
       rollupContract: RollupContract;
       slashingProposerContract: EmpireSlashingProposerContract | TallySlashingProposerContract | undefined;
       governanceProposerContract: GovernanceProposerContract;
@@ -156,6 +179,7 @@ export class SequencerPublisher {
       metrics: SequencerPublisherMetrics;
       lastActions: Partial<Record<Action, SlotNumber>>;
       log?: Logger;
+      getNextPublisher?: (excludeAddresses: EthAddress[]) => Promise<L1TxUtils | undefined>;
     },
   ) {
     this.log = deps.log ?? createLogger('sequencer:publisher');
@@ -163,12 +187,13 @@ export class SequencerPublisher {
     this.epochCache = deps.epochCache;
     this.lastActions = deps.lastActions;
 
-    this.blobClient =
-      deps.blobClient ?? createBlobClient(config, { logger: createLogger('sequencer:blob-client:client') });
+    this.blobClient = deps.blobClient;
 
     const telemetry = deps.telemetry ?? getTelemetryClient();
     this.metrics = deps.metrics ?? new SequencerPublisherMetrics(telemetry, 'SequencerPublisher');
+    this.tracer = telemetry.getTracer('SequencerPublisher');
     this.l1TxUtils = deps.l1TxUtils;
+    this.getNextPublisher = deps.getNextPublisher;
 
     this.rollupContract = deps.rollupContract;
 
@@ -190,12 +215,52 @@ export class SequencerPublisher {
         createLogger('sequencer:publisher:fee-analyzer'),
       );
     }
+
+    // Initialize fee asset price oracle
+    this.feeAssetPriceOracle = new FeeAssetPriceOracle(
+      this.l1TxUtils.client,
+      this.rollupContract,
+      createLogger('sequencer:publisher:price-oracle'),
+    );
+
+    // Initialize failed L1 tx store (optional, for test networks)
+    this.failedTxStore = createL1TxFailedStore(config.l1TxFailedStore, this.log);
+  }
+
+  /**
+   * Backs up a failed L1 transaction to the configured store for debugging.
+   * Does nothing if no store is configured.
+   */
+  private backupFailedTx(failedTx: Omit<FailedL1Tx, 'timestamp'>): void {
+    if (!this.failedTxStore) {
+      return;
+    }
+
+    const tx: FailedL1Tx = {
+      ...failedTx,
+      timestamp: Date.now(),
+    };
+
+    // Fire and forget - don't block on backup
+    void this.failedTxStore
+      .then(store => store?.saveFailedTx(tx))
+      .catch(err => {
+        this.log.warn(`Failed to backup failed L1 tx to store`, err);
+      });
   }
 
   public getRollupContract(): RollupContract {
     return this.rollupContract;
   }
 
+  /**
+   * Gets the fee asset price modifier from the oracle.
+   * Returns 0n if the oracle query fails.
+   */
+  public getFeeAssetPriceModifier(): Promise<bigint> {
+    return this.feeAssetPriceOracle.computePriceModifier();
+  }
+
   public getSenderAddress() {
     return this.l1TxUtils.getSenderAddress();
   }
@@ -271,7 +336,7 @@ export class SequencerPublisher {
     // Start the analysis
     const analysisId = await this.l1FeeAnalyzer.startAnalysis(
       l2SlotNumber,
-      gasLimit > 0n ? gasLimit : SequencerPublisher.PROPOSE_GAS_GUESS,
+      gasLimit > 0n ? gasLimit : MAX_L1_TX_LIMIT,
       l1Requests,
       blobConfig,
       onComplete,
@@ -297,6 +362,7 @@ export class SequencerPublisher {
    * - a receipt and errorMsg if it failed on L1
    * - undefined if no valid requests are found OR the tx failed to send.
    */
+  @trackSpan('SequencerPublisher.sendRequests')
   public async sendRequests() {
     const requestsToProcess = [...this.requests];
     this.requests = [];
@@ -343,7 +409,16 @@ export class SequencerPublisher {
 
     // Merge gasConfigs. Yields the sum of gasLimits, and the earliest txTimeoutAt, or undefined if no gasConfig sets them.
     const gasLimits = gasConfigs.map(g => g?.gasLimit).filter((g): g is bigint => g !== undefined);
-    const gasLimit = gasLimits.length > 0 ? sumBigint(gasLimits) : undefined; // sum
+    let gasLimit = gasLimits.length > 0 ? sumBigint(gasLimits) : undefined; // sum
+    // Cap at L1 block gas limit so the node accepts the tx ("gas limit too high" otherwise).
+    const maxGas = MAX_L1_TX_LIMIT;
+    if (gasLimit !== undefined && gasLimit > maxGas) {
+      this.log.debug('Capping bundled tx gas limit to L1 max', {
+        requested: gasLimit,
+        capped: maxGas,
+      });
+      gasLimit = maxGas;
+    }
     const txTimeoutAts = gasConfigs.map(g => g?.txTimeoutAt).filter((g): g is Date => g !== undefined);
     const txTimeoutAt = txTimeoutAts.length > 0 ? new Date(Math.min(...txTimeoutAts.map(g => g.getTime()))) : undefined; // earliest
     const txConfig: RequestWithExpiry['gasConfig'] = { gasLimit, txTimeoutAt };
@@ -353,19 +428,36 @@ export class SequencerPublisher {
     validRequests.sort((a, b) => compareActions(a.action, b.action));
 
     try {
+      // Capture context for failed tx backup before sending
+      const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
+      const multicallData = encodeFunctionData({
+        abi: multicall3Abi,
+        functionName: 'aggregate3',
+        args: [
+          validRequests.map(r => ({
+            target: r.request.to!,
+            callData: r.request.data!,
+            allowFailure: true,
+          })),
+        ],
+      });
+      const blobDataHex = blobConfig?.blobs?.map(b => toHex(b)) as Hex[] | undefined;
+
+      const txContext = { multicallData, blobData: blobDataHex, l1BlockNumber };
+
       this.log.debug('Forwarding transactions', {
         validRequests: validRequests.map(request => request.action),
         txConfig,
       });
-      const result = await Multicall3.forward(
-        validRequests.map(request => request.request),
-        this.l1TxUtils,
-        txConfig,
-        blobConfig,
-        this.rollupContract.address,
-        this.log,
+      const result = await this.forwardWithPublisherRotation(validRequests, txConfig, blobConfig);
+      if (result === undefined) {
+        return undefined;
+      }
+      const { successfulActions = [], failedActions = [] } = this.callbackBundledTransactions(
+        validRequests,
+        result,
+        txContext,
       );
-      const { successfulActions = [], failedActions = [] } = this.callbackBundledTransactions(validRequests, result);
       return { result, expiredActions, sentActions: validActions, successfulActions, failedActions };
     } catch (err) {
       const viemError = formatViemError(err);
@@ -383,13 +475,76 @@ export class SequencerPublisher {
     }
   }
 
+  /**
+   * Forwards transactions via Multicall3, rotating to the next available publisher if a send
+   * failure occurs (i.e. the tx never reached the chain).
+   * On-chain reverts and simulation errors are returned as-is without rotation.
+   */
+  private async forwardWithPublisherRotation(
+    validRequests: RequestWithExpiry[],
+    txConfig: RequestWithExpiry['gasConfig'],
+    blobConfig: L1BlobInputs | undefined,
+  ) {
+    const triedAddresses: EthAddress[] = [];
+    let currentPublisher = this.l1TxUtils;
+
+    while (true) {
+      triedAddresses.push(currentPublisher.getSenderAddress());
+      try {
+        const result = await Multicall3.forward(
+          validRequests.map(r => r.request),
+          currentPublisher,
+          txConfig,
+          blobConfig,
+          this.rollupContract.address,
+          this.log,
+        );
+        this.l1TxUtils = currentPublisher;
+        return result;
+      } catch (err) {
+        if (err instanceof TimeoutError) {
+          throw err;
+        }
+        const viemError = formatViemError(err);
+        if (!this.getNextPublisher) {
+          this.log.error('Failed to publish bundled transactions', viemError);
+          return undefined;
+        }
+        this.log.warn(
+          `Publisher ${currentPublisher.getSenderAddress()} failed to send, rotating to next publisher`,
+          viemError,
+        );
+        const nextPublisher = await this.getNextPublisher([...triedAddresses]);
+        if (!nextPublisher) {
+          this.log.error('All available publishers exhausted, failed to publish bundled transactions');
+          return undefined;
+        }
+        currentPublisher = nextPublisher;
+      }
+    }
+  }
+
   private callbackBundledTransactions(
     requests: RequestWithExpiry[],
-    result?: { receipt: TransactionReceipt } | FormattedViemError,
+    result: { receipt: TransactionReceipt; errorMsg?: string } | FormattedViemError | undefined,
+    txContext: { multicallData: Hex; blobData?: Hex[]; l1BlockNumber: bigint },
   ) {
     const actionsListStr = requests.map(r => r.action).join(', ');
     if (result instanceof FormattedViemError) {
       this.log.error(`Failed to publish bundled transactions (${actionsListStr})`, result);
+      this.backupFailedTx({
+        id: keccak256(txContext.multicallData),
+        failureType: 'send-error',
+        request: { to: MULTI_CALL_3_ADDRESS, data: txContext.multicallData },
+        blobData: txContext.blobData,
+        l1BlockNumber: txContext.l1BlockNumber.toString(),
+        error: { message: result.message, name: result.name },
+        context: {
+          actions: requests.map(r => r.action),
+          requests: requests.map(r => ({ action: r.action, to: r.request.to! as Hex, data: r.request.data! })),
+          sender: this.getSenderAddress().toString(),
+        },
+      });
       return { failedActions: requests.map(r => r.action) };
     } else {
       this.log.verbose(`Published bundled transactions (${actionsListStr})`, { result, requests });
@@ -402,6 +557,30 @@ export class SequencerPublisher {
           failedActions.push(request.action);
         }
       }
+      // Single backup for the whole reverted tx
+      if (failedActions.length > 0 && result?.receipt?.status === 'reverted') {
+        this.backupFailedTx({
+          id: result.receipt.transactionHash,
+          failureType: 'revert',
+          request: { to: MULTI_CALL_3_ADDRESS, data: txContext.multicallData },
+          blobData: txContext.blobData,
+          l1BlockNumber: result.receipt.blockNumber.toString(),
+          receipt: {
+            transactionHash: result.receipt.transactionHash,
+            blockNumber: result.receipt.blockNumber.toString(),
+            gasUsed: (result.receipt.gasUsed ?? 0n).toString(),
+            status: 'reverted',
+          },
+          error: { message: result.errorMsg ?? 'Transaction reverted' },
+          context: {
+            actions: failedActions,
+            requests: requests
+              .filter(r => failedActions.includes(r.action))
+              .map(r => ({ action: r.action, to: r.request.to! as Hex, data: r.request.data! })),
+            sender: this.getSenderAddress().toString(),
+          },
+        });
+      }
       return { successfulActions, failedActions };
     }
   }
@@ -414,17 +593,14 @@ export class SequencerPublisher {
   public canProposeAtNextEthBlock(
     tipArchive: Fr,
     msgSender: EthAddress,
-    opts: { forcePendingBlockNumber?: BlockNumber } = {},
+    opts: { forcePendingCheckpointNumber?: CheckpointNumber } = {},
   ) {
     // TODO: #14291 - should loop through multiple keys to check if any of them can propose
     const ignoredErrors = ['SlotAlreadyInChain', 'InvalidProposer', 'InvalidArchive'];
 
     return this.rollupContract
       .canProposeAtNextEthBlock(tipArchive.toBuffer(), msgSender.toString(), Number(this.ethereumSlotDuration), {
-        forcePendingCheckpointNumber:
-          opts.forcePendingBlockNumber !== undefined
-            ? CheckpointNumber.fromBlockNumber(opts.forcePendingBlockNumber)
-            : undefined,
+        forcePendingCheckpointNumber: opts.forcePendingCheckpointNumber,
       })
       .catch(err => {
         if (err instanceof FormattedViemError && ignoredErrors.find(e => err.message.includes(e))) {
@@ -443,10 +619,11 @@ export class SequencerPublisher {
    *          It will throw if the block header is invalid.
    * @param header - The block header to validate
    */
+  @trackSpan('SequencerPublisher.validateBlockHeader')
   public async validateBlockHeader(
     header: CheckpointHeader,
-    opts?: { forcePendingBlockNumber: BlockNumber | undefined },
-  ) {
+    opts?: { forcePendingCheckpointNumber: CheckpointNumber | undefined },
+  ): Promise<void> {
     const flags = { ignoreDA: true, ignoreSignatures: true };
 
     const args = [
@@ -455,17 +632,13 @@ export class SequencerPublisher {
       [], // no signers
       Signature.empty().toViemSignature(),
       `0x${'0'.repeat(64)}`, // 32 empty bytes
-      header.contentCommitment.blobsHash.toString(),
+      header.blobsHash.toString(),
       flags,
     ] as const;
 
     const ts = BigInt((await this.l1TxUtils.getBlock()).timestamp + this.ethereumSlotDuration);
-    const optsForcePendingCheckpointNumber =
-      opts?.forcePendingBlockNumber !== undefined
-        ? CheckpointNumber.fromBlockNumber(opts.forcePendingBlockNumber)
-        : undefined;
     const stateOverrides = await this.rollupContract.makePendingCheckpointNumberOverride(
-      optsForcePendingCheckpointNumber,
+      opts?.forcePendingCheckpointNumber,
     );
     let balance = 0n;
     if (this.config.fishermanMode) {
@@ -493,77 +666,109 @@ export class SequencerPublisher {
   }
 
   /**
-   * Simulate making a call to invalidate a block with invalid attestations. Returns undefined if no need to invalidate.
-   * @param block - The block to invalidate and the criteria for invalidation (as returned by the archiver)
+   * Simulate making a call to invalidate a checkpoint with invalid attestations. Returns undefined if no need to invalidate.
+   * @param validationResult - The validation result indicating which checkpoint to invalidate (as returned by the archiver)
    */
-  public async simulateInvalidateBlock(
-    validationResult: ValidateBlockResult,
-  ): Promise<InvalidateBlockRequest | undefined> {
+  public async simulateInvalidateCheckpoint(
+    validationResult: ValidateCheckpointResult,
+  ): Promise<InvalidateCheckpointRequest | undefined> {
     if (validationResult.valid) {
       return undefined;
     }
 
-    const { reason, block } = validationResult;
-    const blockNumber = block.blockNumber;
-    const logData = { ...block, reason };
+    const { reason, checkpoint } = validationResult;
+    const checkpointNumber = checkpoint.checkpointNumber;
+    const logData = { ...checkpoint, reason };
 
-    const currentBlockNumber = await this.rollupContract.getCheckpointNumber();
-    if (currentBlockNumber < validationResult.block.blockNumber) {
+    const currentCheckpointNumber = await this.rollupContract.getCheckpointNumber();
+    if (currentCheckpointNumber < checkpointNumber) {
       this.log.verbose(
-        `Skipping block ${blockNumber} invalidation since it has already been removed from the pending chain`,
-        { currentBlockNumber, ...logData },
+        `Skipping checkpoint ${checkpointNumber} invalidation since it has already been removed from the pending chain`,
+        { currentCheckpointNumber, ...logData },
       );
       return undefined;
     }
 
-    const request = this.buildInvalidateBlockRequest(validationResult);
-    this.log.debug(`Simulating invalidate block ${blockNumber}`, { ...logData, request });
+    const request = this.buildInvalidateCheckpointRequest(validationResult);
+    this.log.debug(`Simulating invalidate checkpoint ${checkpointNumber}`, { ...logData, request });
+
+    const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
 
     try {
-      const { gasUsed } = await this.l1TxUtils.simulate(request, undefined, undefined, ErrorsAbi);
-      this.log.verbose(`Simulation for invalidate block ${blockNumber} succeeded`, { ...logData, request, gasUsed });
+      const { gasUsed } = await this.l1TxUtils.simulate(
+        request,
+        undefined,
+        undefined,
+        mergeAbis([request.abi ?? [], ErrorsAbi]),
+      );
+      this.log.verbose(`Simulation for invalidate checkpoint ${checkpointNumber} succeeded`, {
+        ...logData,
+        request,
+        gasUsed,
+      });
 
-      return { request, gasUsed, blockNumber, forcePendingBlockNumber: BlockNumber(blockNumber - 1), reason };
+      return {
+        request,
+        gasUsed,
+        checkpointNumber,
+        forcePendingCheckpointNumber: CheckpointNumber(checkpointNumber - 1),
+        reason,
+      };
     } catch (err) {
       const viemError = formatViemError(err);
 
-      // If the error is due to the block not being in the pending chain, and it was indeed removed by someone else,
-      // we can safely ignore it and return undefined so we go ahead with block building.
-      if (viemError.message?.includes('Rollup__BlockNotInPendingChain')) {
+      // If the error is due to the checkpoint not being in the pending chain, and it was indeed removed by someone else,
+      // we can safely ignore it and return undefined so we go ahead with checkpoint building.
+      if (viemError.message?.includes('Rollup__CheckpointNotInPendingChain')) {
         this.log.verbose(
-          `Simulation for invalidate block ${blockNumber} failed due to block not being in pending chain`,
+          `Simulation for invalidate checkpoint ${checkpointNumber} failed due to checkpoint not being in pending chain`,
           { ...logData, request, error: viemError.message },
         );
-        const latestPendingBlockNumber = await this.rollupContract.getCheckpointNumber();
-        if (latestPendingBlockNumber < blockNumber) {
-          this.log.verbose(`Block number ${blockNumber} has already been invalidated`, { ...logData });
+        const latestPendingCheckpointNumber = await this.rollupContract.getCheckpointNumber();
+        if (latestPendingCheckpointNumber < checkpointNumber) {
+          this.log.verbose(`Checkpoint ${checkpointNumber} has already been invalidated`, { ...logData });
           return undefined;
         } else {
           this.log.error(
-            `Simulation for invalidate ${blockNumber} failed and it is still in pending chain`,
+            `Simulation for invalidate checkpoint ${checkpointNumber} failed and it is still in pending chain`,
             viemError,
             logData,
           );
-          throw new Error(`Failed to simulate invalidate block ${blockNumber} while it is still in pending chain`, {
-            cause: viemError,
-          });
+          throw new Error(
+            `Failed to simulate invalidate checkpoint ${checkpointNumber} while it is still in pending chain`,
+            {
+              cause: viemError,
+            },
+          );
         }
       }
 
-      // Otherwise, throw. We cannot build the next block if we cannot invalidate the previous one.
-      this.log.error(`Simulation for invalidate block ${blockNumber} failed`, viemError, logData);
-      throw new Error(`Failed to simulate invalidate block ${blockNumber}`, { cause: viemError });
+      // Otherwise, throw. We cannot build the next checkpoint if we cannot invalidate the previous one.
+      this.log.error(`Simulation for invalidate checkpoint ${checkpointNumber} failed`, viemError, logData);
+      this.backupFailedTx({
+        id: keccak256(request.data!),
+        failureType: 'simulation',
+        request: { to: request.to!, data: request.data!, value: request.value?.toString() },
+        l1BlockNumber: l1BlockNumber.toString(),
+        error: { message: viemError.message, name: viemError.name },
+        context: {
+          actions: [`invalidate-${reason}`],
+          checkpointNumber,
+          sender: this.getSenderAddress().toString(),
+        },
+      });
+      throw new Error(`Failed to simulate invalidate checkpoint ${checkpointNumber}`, { cause: viemError });
     }
   }
 
-  private buildInvalidateBlockRequest(validationResult: ValidateBlockResult) {
+  private buildInvalidateCheckpointRequest(validationResult: ValidateCheckpointResult) {
     if (validationResult.valid) {
-      throw new Error('Cannot invalidate a valid block');
+      throw new Error('Cannot invalidate a valid checkpoint');
     }
 
-    const { block, committee, reason } = validationResult;
-    const logData = { ...block, reason };
-    this.log.debug(`Simulating invalidate block ${block.blockNumber}`, logData);
+    const { checkpoint, committee, reason } = validationResult;
+    const logData = { ...checkpoint, reason };
+    this.log.debug(`Building invalidate checkpoint ${checkpoint.checkpointNumber} request`, logData);
 
     const attestationsAndSigners = new CommitteeAttestationsAndSigners(
       validationResult.attestations,
@@ -571,14 +776,14 @@ export class SequencerPublisher {
 
     if (reason === 'invalid-attestation') {
       return this.rollupContract.buildInvalidateBadAttestationRequest(
-        CheckpointNumber.fromBlockNumber(block.blockNumber),
+        checkpoint.checkpointNumber,
         attestationsAndSigners,
         committee,
         validationResult.invalidIndex,
       );
     } else if (reason === 'insufficient-attestations') {
       return this.rollupContract.buildInvalidateInsufficientAttestationsRequest(
-        CheckpointNumber.fromBlockNumber(block.blockNumber),
+        checkpoint.checkpointNumber,
         attestationsAndSigners,
         committee,
       );
@@ -589,31 +794,16 @@ export class SequencerPublisher {
   }
 
   /** Simulates `propose` to make sure that the checkpoint is valid for submission */
+  @trackSpan('SequencerPublisher.validateCheckpointForSubmission')
   public async validateCheckpointForSubmission(
     checkpoint: Checkpoint,
     attestationsAndSigners: CommitteeAttestationsAndSigners,
     attestationsAndSignersSignature: Signature,
-    options: { forcePendingBlockNumber?: BlockNumber }, // TODO(palla/mbps): Should this be forcePendingCheckpointNumber?
+    options: { forcePendingCheckpointNumber?: CheckpointNumber },
   ): Promise<bigint> {
     const ts = BigInt((await this.l1TxUtils.getBlock()).timestamp + this.ethereumSlotDuration);
-
-    // TODO(palla/mbps): This should not be needed, there's no flow where we propose with zero attestations. Or is there?
-    // If we have no attestations, we still need to provide the empty attestations
-    // so that the committee is recalculated correctly
-    // const ignoreSignatures = attestationsAndSigners.attestations.length === 0;
-    // if (ignoreSignatures) {
-    //   const { committee } = await this.epochCache.getCommittee(block.header.globalVariables.slotNumber);
-    //   if (!committee) {
-    //     this.log.warn(`No committee found for slot ${block.header.globalVariables.slotNumber}`);
-    //     throw new Error(`No committee found for slot ${block.header.globalVariables.slotNumber}`);
-    //   }
-    //   attestationsAndSigners.attestations = committee.map(committeeMember =>
-    //     CommitteeAttestation.fromAddress(committeeMember),
-    //   );
-    // }
-
     const blobFields = checkpoint.toBlobFields();
-    const blobs = getBlobsPerL1Block(blobFields);
+    const blobs = await getBlobsPerL1Block(blobFields);
     const blobInput = getPrefixedEthBlobCommitments(blobs);
 
     const args = [
@@ -621,7 +811,7 @@ export class SequencerPublisher {
         header: checkpoint.header.toViem(),
         archive: toHex(checkpoint.archive.root.toBuffer()),
         oracleInput: {
-          feeAssetPriceModifier: 0n,
+          feeAssetPriceModifier: checkpoint.feeAssetPriceModifier,
         },
       },
       attestationsAndSigners.getPackedAttestations(),
@@ -670,6 +860,32 @@ export class SequencerPublisher {
       return false;
     }
 
+    // Check if payload was already submitted to governance
+    const cacheKey = payload.toString();
+    if (!this.payloadProposedCache.has(cacheKey)) {
+      try {
+        const l1StartBlock = await this.rollupContract.getL1StartBlock();
+        const proposed = await retry(
+          () => base.hasPayloadBeenProposed(payload.toString(), l1StartBlock),
+          'Check if payload was proposed',
+          makeBackoff([0, 1, 2]),
+          this.log,
+          true,
+        );
+        if (proposed) {
+          this.payloadProposedCache.add(cacheKey);
+        }
+      } catch (err) {
+        this.log.warn(`Failed to check if payload ${payload} was proposed after retries, skipping signal`, err);
+        return false;
+      }
+    }
+
+    if (this.payloadProposedCache.has(cacheKey)) {
+      this.log.info(`Payload ${payload} was already proposed to governance, stopping signals`);
+      return false;
+    }
+
     const cachedLastVote = this.lastActions[signalType];
     this.lastActions[signalType] = slotNumber;
     const action = signalType;
@@ -688,11 +904,26 @@ export class SequencerPublisher {
       lastValidL2Slot: slotNumber,
     });
 
+    const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
+
     try {
-      await this.l1TxUtils.simulate(request, { time: timestamp }, [], ErrorsAbi);
+      await this.l1TxUtils.simulate(request, { time: timestamp }, [], mergeAbis([request.abi ?? [], ErrorsAbi]));
       this.log.debug(`Simulation for ${action} at slot ${slotNumber} succeeded`, { request });
     } catch (err) {
-      this.log.error(`Failed simulation for ${action} at slot ${slotNumber} (enqueuing the action anyway)`, err);
+      const viemError = formatViemError(err);
+      this.log.error(`Failed simulation for ${action} at slot ${slotNumber} (enqueuing the action anyway)`, viemError);
+      this.backupFailedTx({
+        id: keccak256(request.data!),
+        failureType: 'simulation',
+        request: { to: request.to!, data: request.data!, value: request.value?.toString() },
+        l1BlockNumber: l1BlockNumber.toString(),
+        error: { message: viemError.message, name: viemError.name },
+        context: {
+          actions: [action],
+          slot: slotNumber,
+          sender: this.getSenderAddress().toString(),
+        },
+      });
       // Yes, we enqueue the request anyway, in case there was a bug with the simulation itself
     }
 
@@ -892,19 +1123,20 @@ export class SequencerPublisher {
     checkpoint: Checkpoint,
     attestationsAndSigners: CommitteeAttestationsAndSigners,
     attestationsAndSignersSignature: Signature,
-    opts: { txTimeoutAt?: Date; forcePendingBlockNumber?: BlockNumber } = {},
+    opts: { txTimeoutAt?: Date; forcePendingCheckpointNumber?: CheckpointNumber } = {},
   ): Promise<void> {
     const checkpointHeader = checkpoint.header;
 
     const blobFields = checkpoint.toBlobFields();
-    const blobs = getBlobsPerL1Block(blobFields);
+    const blobs = await getBlobsPerL1Block(blobFields);
 
-    const proposeTxArgs = {
+    const proposeTxArgs: L1ProcessArgs = {
       header: checkpointHeader,
       archive: checkpoint.archive.root.toBuffer(),
       blobs,
       attestationsAndSigners,
       attestationsAndSignersSignature,
+      feeAssetPriceModifier: checkpoint.feeAssetPriceModifier,
     };
 
     let ts: bigint;
@@ -925,7 +1157,7 @@ export class SequencerPublisher {
       this.log.error(`Checkpoint validation failed. ${err instanceof Error ? err.message : 'No error message'}`, err, {
         ...checkpoint.getStats(),
         slotNumber: checkpoint.header.slotNumber,
-        forcePendingBlockNumber: opts.forcePendingBlockNumber,
+        forcePendingCheckpointNumber: opts.forcePendingCheckpointNumber,
       });
       throw err;
     }
@@ -934,7 +1166,10 @@ export class SequencerPublisher {
     await this.addProposeTx(checkpoint, proposeTxArgs, opts, ts);
   }
 
-  public enqueueInvalidateBlock(request: InvalidateBlockRequest | undefined, opts: { txTimeoutAt?: Date } = {}) {
+  public enqueueInvalidateCheckpoint(
+    request: InvalidateCheckpointRequest | undefined,
+    opts: { txTimeoutAt?: Date } = {},
+  ) {
     if (!request) {
       return;
     }
@@ -942,9 +1177,9 @@ export class SequencerPublisher {
     // We issued the simulation against the rollup contract, so we need to account for the overhead of the multicall3
     const gasLimit = this.l1TxUtils.bumpGasLimit(BigInt(Math.ceil((Number(request.gasUsed) * 64) / 63)));
 
-    const { gasUsed, blockNumber } = request;
-    const logData = { gasUsed, blockNumber, gasLimit, opts };
-    this.log.verbose(`Enqueuing invalidate block request`, logData);
+    const { gasUsed, checkpointNumber } = request;
+    const logData = { gasUsed, checkpointNumber, gasLimit, opts };
+    this.log.verbose(`Enqueuing invalidate checkpoint request`, logData);
     this.addRequest({
       action: `invalidate-by-${request.reason}`,
       request: request.request,
@@ -957,9 +1192,9 @@ export class SequencerPublisher {
           result.receipt.status === 'success' &&
           tryExtractEvent(result.receipt.logs, this.rollupContract.address, RollupAbi, 'CheckpointInvalidated');
         if (!success) {
-          this.log.warn(`Invalidate block ${request.blockNumber} failed`, { ...result, ...logData });
+          this.log.warn(`Invalidate checkpoint ${request.checkpointNumber} failed`, { ...result, ...logData });
         } else {
-          this.log.info(`Invalidate block ${request.blockNumber} succeeded`, { ...result, ...logData });
+          this.log.info(`Invalidate checkpoint ${request.checkpointNumber} succeeded`, { ...result, ...logData });
         }
         return !!success;
       },
@@ -984,13 +1219,30 @@ export class SequencerPublisher {
 
     this.log.debug(`Simulating ${action} for slot ${slotNumber}`, logData);
 
+    const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
+
     let gasUsed: bigint;
+    const simulateAbi = mergeAbis([request.abi ?? [], ErrorsAbi]);
     try {
-      ({ gasUsed } = await this.l1TxUtils.simulate(request, { time: timestamp }, [], ErrorsAbi)); // TODO(palla/slash): Check the timestamp logic
+      ({ gasUsed } = await this.l1TxUtils.simulate(request, { time: timestamp }, [], simulateAbi)); // TODO(palla/slash): Check the timestamp logic
       this.log.verbose(`Simulation for ${action} succeeded`, { ...logData, request, gasUsed });
     } catch (err) {
-      const viemError = formatViemError(err);
+      const viemError = formatViemError(err, simulateAbi);
       this.log.error(`Simulation for ${action} at ${slotNumber} failed`, viemError, logData);
+
+      this.backupFailedTx({
+        id: keccak256(request.data!),
+        failureType: 'simulation',
+        request: { to: request.to!, data: request.data!, value: request.value?.toString() },
+        l1BlockNumber: l1BlockNumber.toString(),
+        error: { message: viemError.message, name: viemError.name },
+        context: {
+          actions: [action],
+          slot: slotNumber,
+          sender: this.getSenderAddress().toString(),
+        },
+      });
+
       return false;
     }
 
@@ -998,10 +1250,14 @@ export class SequencerPublisher {
     const gasLimit = this.l1TxUtils.bumpGasLimit(BigInt(Math.ceil((Number(gasUsed) * 64) / 63)));
     logData.gasLimit = gasLimit;
 
+    // Store the ABI used for simulation on the request so Multicall3.forward can decode errors
+    // when the tx is sent and a revert is diagnosed via simulation.
+    const requestWithAbi = { ...request, abi: simulateAbi };
+
     this.log.debug(`Enqueuing ${action}`, logData);
     this.addRequest({
       action,
-      request,
+      request: requestWithAbi,
       gasConfig: { gasLimit },
       lastValidL2Slot: slotNumber,
       checkSuccess: (_req, result) => {
@@ -1038,7 +1294,7 @@ export class SequencerPublisher {
   private async prepareProposeTx(
     encodedData: L1ProcessArgs,
     timestamp: bigint,
-    options: { forcePendingBlockNumber?: BlockNumber },
+    options: { forcePendingCheckpointNumber?: CheckpointNumber },
   ) {
     const kzg = Blob.getViemKzgInstance();
     const blobInput = getPrefixedEthBlobCommitments(encodedData.blobs);
@@ -1070,9 +1326,27 @@ export class SequencerPublisher {
             kzg,
           },
         )
-        .catch(err => {
-          const { message, metaMessages } = formatViemError(err);
-          this.log.error(`Failed to validate blobs`, message, { metaMessages });
+        .catch(async err => {
+          const viemError = formatViemError(err);
+          this.log.error(`Failed to validate blobs`, viemError.message, { metaMessages: viemError.metaMessages });
+          const validateBlobsData = encodeFunctionData({
+            abi: RollupAbi,
+            functionName: 'validateBlobs',
+            args: [blobInput],
+          });
+          const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
+          this.backupFailedTx({
+            id: keccak256(validateBlobsData),
+            failureType: 'simulation',
+            request: { to: this.rollupContract.address as Hex, data: validateBlobsData },
+            blobData: encodedData.blobs.map(b => toHex(b.data)) as Hex[],
+            l1BlockNumber: l1BlockNumber.toString(),
+            error: { message: viemError.message, name: viemError.name },
+            context: {
+              actions: ['validate-blobs'],
+              sender: this.getSenderAddress().toString(),
+            },
+          });
           throw new Error('Failed to validate blobs');
         });
     }
@@ -1083,8 +1357,7 @@ export class SequencerPublisher {
         header: encodedData.header.toViem(),
         archive: toHex(encodedData.archive),
         oracleInput: {
-          // We are currently not modifying these. See #9963
-          feeAssetPriceModifier: 0n,
+          feeAssetPriceModifier: encodedData.feeAssetPriceModifier,
         },
       },
       encodedData.attestationsAndSigners.getPackedAttestations(),
@@ -1110,7 +1383,7 @@ export class SequencerPublisher {
         readonly header: ViemHeader;
         readonly archive: `0x${string}`;
         readonly oracleInput: {
-          readonly feeAssetPriceModifier: 0n;
+          readonly feeAssetPriceModifier: bigint;
         };
       },
       ViemCommitteeAttestations,
@@ -1119,7 +1392,7 @@ export class SequencerPublisher {
       `0x${string}`,
     ],
     timestamp: bigint,
-    options: { forcePendingBlockNumber?: BlockNumber },
+    options: { forcePendingCheckpointNumber?: CheckpointNumber },
   ) {
     const rollupData = encodeFunctionData({
       abi: RollupAbi,
@@ -1128,13 +1401,9 @@ export class SequencerPublisher {
     });
 
     // override the pending checkpoint number if requested
-    const optsForcePendingCheckpointNumber =
-      options.forcePendingBlockNumber !== undefined
-        ? CheckpointNumber.fromBlockNumber(options.forcePendingBlockNumber)
-        : undefined;
     const forcePendingCheckpointNumberStateDiff = (
-      optsForcePendingCheckpointNumber !== undefined
-        ? await this.rollupContract.makePendingCheckpointNumberOverride(optsForcePendingCheckpointNumber)
+      options.forcePendingCheckpointNumber !== undefined
+        ? await this.rollupContract.makePendingCheckpointNumberOverride(options.forcePendingCheckpointNumber)
         : []
     ).flatMap(override => override.stateDiff ?? []);
 
@@ -1156,25 +1425,27 @@ export class SequencerPublisher {
       });
     }
 
+    const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
+
     const simulationResult = await this.l1TxUtils
       .simulate(
         {
           to: this.rollupContract.address,
           data: rollupData,
-          gas: SequencerPublisher.PROPOSE_GAS_GUESS,
+          gas: MAX_L1_TX_LIMIT,
           ...(this.proposerAddressForSimulation && { from: this.proposerAddressForSimulation.toString() }),
         },
         {
           // @note we add 1n to the timestamp because geth implementation doesn't like simulation timestamp to be equal to the current block timestamp
           time: timestamp + 1n,
           // @note reth should have a 30m gas limit per block but throws errors that this tx is beyond limit so we increase here
-          gasLimit: SequencerPublisher.PROPOSE_GAS_GUESS * 2n,
+          gasLimit: MAX_L1_TX_LIMIT * 2n,
         },
         stateOverrides,
         RollupAbi,
         {
           // @note fallback gas estimate to use if the node doesn't support simulation API
-          fallbackGasEstimate: SequencerPublisher.PROPOSE_GAS_GUESS,
+          fallbackGasEstimate: MAX_L1_TX_LIMIT,
         },
       )
       .catch(err => {
@@ -1184,11 +1455,23 @@ export class SequencerPublisher {
           this.log.debug(`Ignoring expected ValidatorSelection__MissingProposerSignature error in fisherman mode`);
           // Return a minimal simulation result with the fallback gas estimate
           return {
-            gasUsed: SequencerPublisher.PROPOSE_GAS_GUESS,
+            gasUsed: MAX_L1_TX_LIMIT,
             logs: [],
           };
         }
         this.log.error(`Failed to simulate propose tx`, viemError);
+        this.backupFailedTx({
+          id: keccak256(rollupData),
+          failureType: 'simulation',
+          request: { to: this.rollupContract.address, data: rollupData },
+          l1BlockNumber: l1BlockNumber.toString(),
+          error: { message: viemError.message, name: viemError.name },
+          context: {
+            actions: ['propose'],
+            slot: Number(args[0].header.slotNumber),
+            sender: this.getSenderAddress().toString(),
+          },
+        });
         throw err;
       });
 
@@ -1198,7 +1481,7 @@ export class SequencerPublisher {
   private async addProposeTx(
     checkpoint: Checkpoint,
     encodedData: L1ProcessArgs,
-    opts: { txTimeoutAt?: Date; forcePendingBlockNumber?: BlockNumber } = {},
+    opts: { txTimeoutAt?: Date; forcePendingCheckpointNumber?: CheckpointNumber } = {},
     timestamp: bigint,
   ): Promise<void> {
     const slot = checkpoint.header.slotNumber;