@aztec/end-to-end 0.0.1-commit.64b6bbb → 0.0.1-commit.69c59a8b3

package/src/shared/uniswap_l1_l2.ts

@@ -17,12 +17,12 @@ import { InboxAbi, UniswapPortalAbi, UniswapPortalBytecode } from '@aztec/l1-art
 import { UniswapContract } from '@aztec/noir-contracts.js/Uniswap';
 import { computeL2ToL1MessageHash } from '@aztec/stdlib/hash';
 import { computeL2ToL1MembershipWitness } from '@aztec/stdlib/messaging';
-import type { TestWallet } from '@aztec/test-wallet/server';
 
 import { jest } from '@jest/globals';
 import { type GetContractReturnType, getContract, parseEther, toFunctionSelector } from 'viem';
 
 import { type EndToEndContext, ensureAccountContractsPublished } from '../fixtures/utils.js';
+import type { TestWallet } from '../test-wallet/test_wallet.js';
 import { CrossChainTestHarness } from './cross_chain_test_harness.js';
 
 // PSA: This tests works on forked mainnet. There is a dump of the data in `dumpedState` such that we
@@ -130,7 +130,9 @@ export const uniswapL1L2TestSuite = (
         client: l1Client,
       });
       // deploy l2 uniswap contract and attach to portal
-      uniswapL2Contract = await UniswapContract.deploy(wallet, uniswapPortalAddress).send({ from: ownerAddress });
+      ({ contract: uniswapL2Contract } = await UniswapContract.deploy(wallet, uniswapPortalAddress).send({
+        from: ownerAddress,
+      }));
 
       const registryAddress = (await aztecNode.getNodeInfo()).l1ContractAddresses.registryAddress;
 
@@ -195,7 +197,7 @@ export const uniswapL1L2TestSuite = (
       logger.info('Withdrawing weth to L1 and sending message to swap to dai');
       const [secretForDepositingSwappedDai, secretHashForDepositingSwappedDai] = await generateClaimSecret();
 
-      const l2UniswapInteractionReceipt = await uniswapL2Contract.methods
+      const { receipt: l2UniswapInteractionReceipt } = await uniswapL2Contract.methods
         .swap_private(
           wethCrossChainHarness.l2Token.address,
           wethCrossChainHarness.l2Bridge.address,
@@ -250,8 +252,12 @@ export const uniswapL1L2TestSuite = (
       await wethCrossChainHarness.expectPublicBalanceOnL2(uniswapL2Contract.address, 0n);
 
       // Since the outbox is only consumable when the epoch is proven, we need to advance to the next epoch.
-      const block = await aztecNode.getBlock(l2UniswapInteractionReceipt.blockNumber!);
-      const epoch = await rollup.getEpochNumberForCheckpoint(block!.checkpointNumber);
+      const swapResult = (await computeL2ToL1MembershipWitness(
+        aztecNode,
+        swapPrivateLeaf,
+        l2UniswapInteractionReceipt.txHash,
+      ))!;
+      const { epochNumber: epoch } = swapResult;
       await cheatCodes.rollup.advanceToEpoch(EpochNumber(epoch + 1));
       await waitForProven(aztecNode, l2UniswapInteractionReceipt, { provenTimeout: 300 });
 
@@ -260,14 +266,17 @@ export const uniswapL1L2TestSuite = (
       const daiL1BalanceOfPortalBeforeSwap = await daiCrossChainHarness.getL1BalanceOf(
         daiCrossChainHarness.tokenPortalAddress,
       );
-      const swapResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, swapPrivateLeaf);
-      const withdrawResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, withdrawLeaf);
+      const withdrawResult = (await computeL2ToL1MembershipWitness(
+        aztecNode,
+        withdrawLeaf,
+        l2UniswapInteractionReceipt.txHash,
+      ))!;
 
-      const swapPrivateL2MessageIndex = swapResult!.leafIndex;
-      const swapPrivateSiblingPath = swapResult!.siblingPath;
+      const swapPrivateL2MessageIndex = swapResult.leafIndex;
+      const swapPrivateSiblingPath = swapResult.siblingPath;
 
-      const withdrawL2MessageIndex = withdrawResult!.leafIndex;
-      const withdrawSiblingPath = withdrawResult!.siblingPath;
+      const withdrawL2MessageIndex = withdrawResult.leafIndex;
+      const withdrawSiblingPath = withdrawResult.siblingPath;
 
       const withdrawMessageMetadata = {
         _epoch: BigInt(epoch),
@@ -787,7 +796,7 @@ export const uniswapL1L2TestSuite = (
       logger.info('Withdrawing weth to L1 and sending message to swap to dai');
 
       const [, secretHashForDepositingSwappedDai] = await generateClaimSecret();
-      const withdrawReceipt = await uniswapL2Contract.methods
+      const { receipt: withdrawReceipt } = await uniswapL2Contract.methods
         .swap_private(
           wethCrossChainHarness.l2Token.address,
           wethCrossChainHarness.l2Bridge.address,
@@ -838,16 +847,15 @@ export const uniswapL1L2TestSuite = (
         chainId: new Fr(l1Client.chain.id),
       });
 
-      const block = await aztecNode.getBlock(withdrawReceipt.blockNumber!);
-      const epoch = await rollup.getEpochNumberForCheckpoint(block!.checkpointNumber);
-      const swapResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, swapPrivateLeaf);
-      const withdrawResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, withdrawLeaf);
+      const swapResult = (await computeL2ToL1MembershipWitness(aztecNode, swapPrivateLeaf, withdrawReceipt.txHash))!;
+      const { epochNumber: epoch } = swapResult;
+      const withdrawResult = (await computeL2ToL1MembershipWitness(aztecNode, withdrawLeaf, withdrawReceipt.txHash))!;
 
-      const swapPrivateL2MessageIndex = swapResult!.leafIndex;
-      const swapPrivateSiblingPath = swapResult!.siblingPath;
+      const swapPrivateL2MessageIndex = swapResult.leafIndex;
+      const swapPrivateSiblingPath = swapResult.siblingPath;
 
-      const withdrawL2MessageIndex = withdrawResult!.leafIndex;
-      const withdrawSiblingPath = withdrawResult!.siblingPath;
+      const withdrawL2MessageIndex = withdrawResult.leafIndex;
+      const withdrawSiblingPath = withdrawResult.siblingPath;
 
       const withdrawMessageMetadata = {
         _epoch: BigInt(epoch),
@@ -915,7 +923,7 @@ export const uniswapL1L2TestSuite = (
 
       // Call swap_public on L2
       const secretHashForDepositingSwappedDai = Fr.random();
-      const withdrawReceipt = await uniswapL2Contract.methods
+      const { receipt: withdrawReceipt } = await uniswapL2Contract.methods
         .swap_public(
           ownerAddress,
           wethCrossChainHarness.l2Bridge.address,
@@ -971,16 +979,15 @@ export const uniswapL1L2TestSuite = (
         chainId: new Fr(l1Client.chain.id),
       });
 
-      const block = await aztecNode.getBlock(withdrawReceipt.blockNumber!);
-      const epoch = await rollup.getEpochNumberForCheckpoint(block!.checkpointNumber);
-      const swapResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, swapPublicLeaf);
-      const withdrawResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, withdrawLeaf);
+      const swapResult = (await computeL2ToL1MembershipWitness(aztecNode, swapPublicLeaf, withdrawReceipt.txHash))!;
+      const { epochNumber: epoch } = swapResult;
+      const withdrawResult = (await computeL2ToL1MembershipWitness(aztecNode, withdrawLeaf, withdrawReceipt.txHash))!;
 
-      const swapPublicL2MessageIndex = swapResult!.leafIndex;
-      const swapPublicSiblingPath = swapResult!.siblingPath;
+      const swapPublicL2MessageIndex = swapResult.leafIndex;
+      const swapPublicSiblingPath = swapResult.siblingPath;
 
-      const withdrawL2MessageIndex = withdrawResult!.leafIndex;
-      const withdrawSiblingPath = withdrawResult!.siblingPath;
+      const withdrawL2MessageIndex = withdrawResult.leafIndex;
+      const withdrawSiblingPath = withdrawResult.siblingPath;
 
       const withdrawMessageMetadata = {
         _epoch: BigInt(epoch),

package/src/simulators/lending_simulator.ts

@@ -94,7 +94,9 @@ export class LendingSimulator {
 
   async prepare() {
     this.accumulator = BASE;
-    const slot = await this.rollup.getSlotAt(BigInt(await this.cc.eth.timestamp()) + BigInt(this.ethereumSlotDuration));
+    const slot = await this.rollup.getSlotAt(
+      BigInt(await this.cc.eth.lastBlockTimestamp()) + BigInt(this.ethereumSlotDuration),
+    );
     this.time = Number(await this.rollup.getTimestampForSlot(slot));
   }
 
@@ -103,7 +105,7 @@ export class LendingSimulator {
       return;
     }
 
-    const slot = await this.rollup.getSlotAt(BigInt(await this.cc.eth.timestamp()));
+    const slot = await this.rollup.getSlotAt(BigInt(await this.cc.eth.lastBlockTimestamp()));
     const targetSlot = SlotNumber(slot + diff);
     const ts = Number(await this.rollup.getTimestampForSlot(targetSlot));
     const timeDiff = ts - this.time;
@@ -186,14 +188,16 @@ export class LendingSimulator {
 
     expect(this.borrowed).toEqual(this.stableCoin.totalSupply - this.mintedOutside);
 
-    const asset = await this.lendingContract.methods.get_asset(0).simulate({ from: this.account.address });
+    const { result: asset } = await this.lendingContract.methods.get_asset(0).simulate({ from: this.account.address });
 
     const interestAccumulator = asset['interest_accumulator'];
     expect(interestAccumulator).toEqual(this.accumulator);
     expect(asset['last_updated_ts']).toEqual(BigInt(this.time));
 
     for (const key of [this.account.address, AztecAddress.fromField(await this.account.key())]) {
-      const privatePos = await this.lendingContract.methods.get_position(key).simulate({ from: this.account.address });
+      const { result: privatePos } = await this.lendingContract.methods
+        .get_position(key)
+        .simulate({ from: this.account.address });
       expect(new Fr(privatePos['collateral'])).toEqual(this.collateral[key.toString()] ?? Fr.ZERO);
       expect(new Fr(privatePos['static_debt'])).toEqual(this.staticDebt[key.toString()] ?? Fr.ZERO);
       expect(privatePos['debt']).toEqual(

package/src/simulators/token_simulator.ts

@@ -109,7 +109,9 @@ export class TokenSimulator {
       await Promise.all(
         chunk(calls, 5).map(batch => new BatchCall(this.defaultWallet, batch).simulate({ from: this.defaultAddress })),
       )
-    ).flat();
+    )
+      .flatMap(r => r.result)
+      .map(r => r.result);
     expect(results[0]).toEqual(this.totalSupply);
 
     // Check that all our balances match
@@ -119,39 +121,13 @@ export class TokenSimulator {
   }
 
   async checkPrivate() {
-    // Private calls
-    const defaultLookups = [];
-    const nonDefaultLookups = [];
-
     for (const address of this.accounts) {
-      if (this.lookupProvider.has(address.toString())) {
-        nonDefaultLookups.push(address);
-      } else {
-        defaultLookups.push(address);
-      }
-    }
-
-    const defaultCalls = defaultLookups.map(address => this.token.methods.balance_of_private(address));
-    const results = (
-      await Promise.all(
-        chunk(defaultCalls, 4).map(batch =>
-          new BatchCall(this.defaultWallet, batch).simulate({ from: this.defaultAddress }),
-        ),
-      )
-    ).flat();
-    for (let i = 0; i < defaultLookups.length; i++) {
-      expect(results[i]).toEqual(this.balanceOfPrivate(defaultLookups[i]));
-    }
-
-    // We are just running individual calls for the non-default lookups
-    // @todo We should also batch these
-    for (const address of nonDefaultLookups) {
       const wallet = this.lookupProvider.get(address.toString());
       const asset = wallet ? this.token.withWallet(wallet) : this.token;
 
-      const actualPrivateBalance = await asset.methods
-        .balance_of_private({ address })
-        .simulate({ from: this.defaultAddress });
+      const { result: actualPrivateBalance } = await asset.methods
+        .balance_of_private(address)
+        .simulate({ from: address });
       expect(actualPrivateBalance).toEqual(this.balanceOfPrivate(address));
     }
   }

package/src/spartan/setup_test_wallets.ts

@@ -1,4 +1,5 @@
 import { generateSchnorrAccounts } from '@aztec/accounts/testing';
+import { NO_FROM } from '@aztec/aztec.js/account';
 import { AztecAddress } from '@aztec/aztec.js/addresses';
 import { NO_WAIT } from '@aztec/aztec.js/contracts';
 import { L1FeeJuicePortalManager } from '@aztec/aztec.js/ethereum';
@@ -10,14 +11,17 @@ import type { Wallet } from '@aztec/aztec.js/wallet';
 import { createEthereumChain } from '@aztec/ethereum/chain';
 import { createExtendedL1Client } from '@aztec/ethereum/client';
 import type { Logger } from '@aztec/foundation/log';
-import { retryUntil } from '@aztec/foundation/retry';
+import { makeBackoff, retry, retryUntil } from '@aztec/foundation/retry';
 import { TokenContract } from '@aztec/noir-contracts.js/Token';
 import type { AztecNodeAdmin } from '@aztec/stdlib/interfaces/client';
-import { TestWallet, proveInteraction, registerInitialLocalNetworkAccountsInWallet } from '@aztec/test-wallet/server';
+import { registerInitialLocalNetworkAccountsInWallet } from '@aztec/wallets/testing';
 
 import { getACVMConfig } from '../fixtures/get_acvm_config.js';
 import { getBBConfig } from '../fixtures/get_bb_config.js';
 import { getSponsoredFPCAddress, registerSponsoredFPC } from '../fixtures/utils.js';
+import { TestWallet } from '../test-wallet/test_wallet.js';
+import { proveInteraction } from '../test-wallet/utils.js';
+import { WorkerWallet } from '../test-wallet/worker_wallet.js';
 
 export interface TestAccounts {
   aztecNode: AztecNode;
@@ -85,11 +89,19 @@ export async function deploySponsoredTestAccountsWithTokens(
 
   const paymentMethod = new SponsoredFeePaymentMethod(await getSponsoredFPCAddress());
   const recipientDeployMethod = await recipientAccount.getDeployMethod();
-  await recipientDeployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod }, wait: { timeout: 2400 } });
+  await recipientDeployMethod.send({
+    from: NO_FROM,
+    fee: { paymentMethod },
+    wait: { timeout: 2400 },
+  });
   await Promise.all(
     fundedAccounts.map(async a => {
       const deployMethod = await a.getDeployMethod();
-      await deployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod }, wait: { timeout: 2400 } }); // increase timeout on purpose in order to account for two empty epochs
+      await deployMethod.send({
+        from: NO_FROM,
+        fee: { paymentMethod },
+        wait: { timeout: 2400 },
+      }); // increase timeout on purpose in order to account for two empty epochs
       logger.info(`Account deployed at ${a.address}`);
     }),
   );
@@ -118,49 +130,91 @@ export async function deploySponsoredTestAccountsWithTokens(
 }
 
 async function deployAccountWithDiagnostics(
-  account: { getDeployMethod: () => Promise<{ send: (opts: any) => any }>; address: any },
+  account: { getDeployMethod: () => Promise<{ simulate: (opts: any) => any; send: (opts: any) => any }>; address: any },
   paymentMethod: SponsoredFeePaymentMethod,
   aztecNode: AztecNode,
   logger: Logger,
   accountLabel: string,
+  estimateGas?: boolean,
 ): Promise<void> {
   const deployMethod = await account.getDeployMethod();
-  let txHash;
-  try {
-    txHash = await deployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod }, wait: NO_WAIT });
-    await waitForTx(aztecNode, txHash, { timeout: 2400 });
-    logger.info(`${accountLabel} deployed at ${account.address}`);
-  } catch (error) {
-    const blockNumber = await aztecNode.getBlockNumber();
-    let receipt;
-    try {
-      receipt = await aztecNode.getTxReceipt(txHash);
-    } catch {
-      receipt = 'unavailable';
-    }
-    logger.error(`${accountLabel} deployment failed`, {
-      txHash: txHash.toString(),
-      receipt: JSON.stringify(receipt),
-      currentBlockNumber: blockNumber,
-      error: String(error),
-    });
-    throw error;
+  let gasSettings: any;
+  if (estimateGas) {
+    const sim = await deployMethod.simulate({ from: NO_FROM, fee: { paymentMethod } });
+    gasSettings = sim.estimatedGas;
+    logger.info(`${accountLabel} estimated gas: DA=${gasSettings.gasLimits.daGas} L2=${gasSettings.gasLimits.l2Gas}`);
   }
+
+  // Track the tx hash across retries so we don't re-send when the previous tx is still pending.
+  let sentTxHash: { txHash: any } | undefined;
+
+  await retry(
+    async () => {
+      // Check if already deployed (handles case where previous attempt succeeded but waitForTx timed out)
+      const existing = await aztecNode.getContract(account.address);
+      if (existing) {
+        logger.info(`${accountLabel} already deployed at ${account.address}, skipping`);
+        return;
+      }
+
+      // If we already sent a tx, check if it was dropped before deciding to re-send.
+      if (sentTxHash) {
+        const prevReceipt = await aztecNode.getTxReceipt(sentTxHash.txHash);
+        if (prevReceipt.isDropped()) {
+          logger.info(`${accountLabel} previous tx ${sentTxHash.txHash} was dropped, re-sending`);
+          sentTxHash = undefined;
+        } else {
+          logger.info(`${accountLabel} previous tx ${sentTxHash.txHash} still pending, waiting again...`);
+        }
+      }
+
+      if (!sentTxHash) {
+        const deployResult = await deployMethod.send({
+          from: NO_FROM,
+          fee: { paymentMethod, gasSettings },
+          wait: NO_WAIT,
+        });
+        sentTxHash = { txHash: deployResult.txHash };
+        logger.info(`${accountLabel} tx sent`, { txHash: sentTxHash.txHash.toString() });
+      }
+
+      const receipt = await waitForTx(aztecNode, sentTxHash.txHash, { timeout: 600 });
+      if (receipt.isDropped()) {
+        sentTxHash = undefined;
+        throw new Error(`${accountLabel} tx was dropped, retrying...`);
+      }
+      logger.info(`${accountLabel} deployed at ${account.address}`);
+    },
+    `deploy ${accountLabel}`,
+    makeBackoff([1, 2, 4, 8, 16]),
+    logger,
+  );
 }
 
 async function deployAccountsInBatches(
-  accounts: { getDeployMethod: () => Promise<{ send: (opts: any) => any }>; address: any }[],
+  accounts: {
+    getDeployMethod: () => Promise<{ simulate: (opts: any) => any; send: (opts: any) => any }>;
+    address: any;
+  }[],
   paymentMethod: SponsoredFeePaymentMethod,
   aztecNode: AztecNode,
   logger: Logger,
   labelPrefix: string,
   batchSize = 2,
+  estimateGas?: boolean,
 ): Promise<void> {
   for (let i = 0; i < accounts.length; i += batchSize) {
     const batch = accounts.slice(i, i + batchSize);
     await Promise.all(
       batch.map((account, idx) =>
-        deployAccountWithDiagnostics(account, paymentMethod, aztecNode, logger, `${labelPrefix}${i + idx + 1}`),
+        deployAccountWithDiagnostics(
+          account,
+          paymentMethod,
+          aztecNode,
+          logger,
+          `${labelPrefix}${i + idx + 1}`,
+          estimateGas,
+        ),
       ),
     );
   }
@@ -171,6 +225,7 @@ export async function deploySponsoredTestAccounts(
   aztecNode: AztecNode,
   logger: Logger,
   numberOfFundedWallets = 1,
+  opts?: { estimateGas?: boolean },
 ): Promise<TestAccountsWithoutTokens> {
   const [recipient, ...funded] = await generateSchnorrAccounts(numberOfFundedWallets + 1);
   const recipientAccount = await wallet.createSchnorrAccount(recipient.secret, recipient.salt);
@@ -180,8 +235,23 @@ export async function deploySponsoredTestAccounts(
 
   const paymentMethod = new SponsoredFeePaymentMethod(await getSponsoredFPCAddress());
 
-  await deployAccountWithDiagnostics(recipientAccount, paymentMethod, aztecNode, logger, 'Recipient account');
-  await deployAccountsInBatches(fundedAccounts, paymentMethod, aztecNode, logger, 'Funded account ', 2);
+  await deployAccountWithDiagnostics(
+    recipientAccount,
+    paymentMethod,
+    aztecNode,
+    logger,
+    'Recipient account',
+    opts?.estimateGas,
+  );
+  await deployAccountsInBatches(
+    fundedAccounts,
+    paymentMethod,
+    aztecNode,
+    logger,
+    'Funded account ',
+    2,
+    opts?.estimateGas,
+  );
 
   return {
     aztecNode,
@@ -219,7 +289,7 @@ export async function deployTestAccountsWithTokens(
     fundedAccounts.map(async (a, i) => {
       const paymentMethod = new FeeJuicePaymentMethodWithClaim(a.address, claims[i]);
       const deployMethod = await a.getDeployMethod();
-      await deployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod } });
+      await deployMethod.send({ from: NO_FROM, fee: { paymentMethod } });
       logger.info(`Account deployed at ${a.address}`);
     }),
   );
@@ -263,7 +333,7 @@ async function bridgeL1FeeJuice(
   const claim = await portal.bridgeTokensPublic(recipient, amount, true /* mint */);
 
   const isSynced = async () =>
-    (await aztecNode.getL1ToL2MessageBlock(Fr.fromHexString(claim.messageHash))) !== undefined;
+    (await aztecNode.getL1ToL2MessageCheckpoint(Fr.fromHexString(claim.messageHash))) !== undefined;
   await retryUntil(isSynced, `message ${claim.messageHash} sync`, 24, 0.5);
 
   log.info(`Created a claim for ${amount} L1 fee juice to ${recipient}.`, claim);
@@ -306,7 +376,7 @@ async function deployTokenAndMint(
     fee: {
       paymentMethod,
     },
-    wait: { timeout: 600, returnReceipt: true },
+    wait: { timeout: 600 },
   });
 
   const tokenAddress = tokenContract.address;
@@ -395,3 +465,42 @@ export async function createWalletAndAztecNodeClient(
     },
   };
 }
+
+export type WorkerWalletWrapper = {
+  wallet: WorkerWallet;
+  aztecNode: AztecNode;
+  cleanup: () => Promise<void>;
+};
+
+export async function createWorkerWalletClient(
+  nodeUrl: string,
+  proverEnabled: boolean,
+  logger: Logger,
+): Promise<WorkerWalletWrapper> {
+  const aztecNode = createAztecNodeClient(nodeUrl);
+  const [bbConfig, acvmConfig] = await Promise.all([getBBConfig(logger), getACVMConfig(logger)]);
+
+  // Strip cleanup functions — they can't be structured-cloned for worker transfer
+  const { cleanup: bbCleanup, ...bbPaths } = bbConfig ?? {};
+  const { cleanup: acvmCleanup, ...acvmPaths } = acvmConfig ?? {};
+
+  const pxeConfig = {
+    dataDirectory: undefined,
+    dataStoreMapSizeKb: 1024 * 1024,
+    ...bbPaths,
+    ...acvmPaths,
+    proverEnabled,
+  };
+
+  const wallet = await WorkerWallet.create(nodeUrl, pxeConfig);
+
+  return {
+    wallet,
+    aztecNode,
+    async cleanup() {
+      await wallet.stop();
+      await bbCleanup?.();
+      await acvmCleanup?.();
+    },
+  };
+}

package/src/spartan/tx_metrics.ts

@@ -296,7 +296,7 @@ export class TxInclusionMetrics {
           value: stats.mean,
         },
         {
-          name: `${group}/median_inclusion`,
+          name: `${group}/p50_inclusion`,
           unit: 's',
           value: stats.median,
         },

package/src/spartan/utils/bot.ts

@@ -34,7 +34,8 @@ export async function installTransferBot({
   logger: log,
   replicas = 1,
   txIntervalSeconds = 10,
-  followChain = 'PENDING',
+  followChain = 'CHECKPOINTED',
+  pxeSyncChainTip = 'proposed',
   mnemonic = process.env.LABS_INFRA_MNEMONIC ?? 'test test test test test test test test test test test junk',
   mnemonicStartIndex,
   botPrivateKey = process.env.BOT_TRANSFERS_L2_PRIVATE_KEY ?? '0xcafe01',
@@ -49,6 +50,7 @@ export async function installTransferBot({
   replicas?: number;
   txIntervalSeconds?: number;
   followChain?: string;
+  pxeSyncChainTip?: string;
   mnemonic?: string;
   mnemonicStartIndex?: number | string;
   botPrivateKey?: string;
@@ -67,6 +69,7 @@ export async function installTransferBot({
     'bot.replicaCount': replicas,
     'bot.txIntervalSeconds': txIntervalSeconds,
     'bot.followChain': followChain,
+    'bot.pxeSyncChainTip': pxeSyncChainTip,
     'bot.botPrivateKey': botPrivateKey,
     'bot.nodeUrl': resolvedNodeUrl,
     'bot.mnemonic': mnemonic,

package/src/spartan/utils/config.ts

@@ -8,6 +8,7 @@ const logger = createLogger('e2e:k8s-utils');
 const testConfigSchema = z.object({
   NAMESPACE: z.string().default('scenario'),
   REAL_VERIFIER: schemas.Boolean.optional().default(true),
+  DEBUG_FORCE_TX_PROOF_VERIFICATION: schemas.Boolean.optional().default(true),
   CREATE_ETH_DEVNET: schemas.Boolean.optional().default(false),
   L1_RPC_URLS_JSON: z.string().optional(),
   L1_ACCOUNT_MNEMONIC: z.string().optional(),
@@ -16,6 +17,7 @@ const testConfigSchema = z.object({
   AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5),
   AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET: z.coerce.number().optional().default(2),
   FUNDING_PRIVATE_KEY: z.string().optional(),
+  AZTEC_ADMIN_API_KEY: z.string().optional(),
 });
 
 export type TestConfig = z.infer<typeof testConfigSchema>;

package/src/spartan/utils/index.ts

@@ -25,6 +25,7 @@ export {
   getRPCEndpoint,
   getEthereumEndpoint,
   createResilientPrometheusConnection,
+  scaleProverAgents,
 } from './k8s.js';
 
 // Chaos Mesh
@@ -40,6 +41,9 @@ export {
   applyNetworkShaping,
 } from './chaos.js';
 
+// Helm
+export { hasDeployedHelmRelease } from './helm.js';
+
 // Bot management
 export { restartBot, installTransferBot, uninstallTransferBot } from './bot.js';
 
@@ -62,3 +66,6 @@ export { getPublicViemClient, getL1DeploymentAddresses, getNodeClient } from './
 
 // Health checks
 export { ChainHealth, type ChainHealthSnapshot } from './health.js';
+
+// Pod log extraction
+export { type BlockBuiltLogEntry, fetchBlockBuiltLogs } from './pod_logs.js';

package/src/spartan/utils/k8s.ts

@@ -522,6 +522,14 @@ export function createResilientPrometheusConnection(
   return { connect, runAlertCheck };
 }
 
+/** Scales the prover-agent Deployment to the given number of replicas. */
+export async function scaleProverAgents(namespace: string, replicas: number, log: Logger): Promise<void> {
+  const label = 'app.kubernetes.io/component=prover-agent';
+  const command = `kubectl scale deployment -l ${label} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+  log.info(`Scaling prover agents to ${replicas}: ${command}`);
+  await execAsync(command);
+}
+
 export function getChartDir(spartanDir: string, chartName: string) {
   return path.join(spartanDir.trim(), chartName);
 }

package/src/spartan/utils/nodes.ts

@@ -173,7 +173,7 @@ export async function withSequencersAdmin<T>(env: TestConfig, fn: (node: AztecNo
           if (statusRes.status !== 200) {
             throw new Error(`Admin endpoint returned status ${statusRes.status}`);
           }
-          const client = createAztecNodeAdminClient(url);
+          const client = createAztecNodeAdminClient(url, {}, undefined, env.AZTEC_ADMIN_API_KEY);
           return { result: await fn(client), process };
         } catch (err) {
           // Kill the port-forward before retrying
@@ -255,21 +255,18 @@ export async function initHADb(namespace: string): Promise<void> {
 }
 
 /**
- * Enables or disables probabilistic transaction dropping on validators and waits for rollout.
- * Wired to env vars P2P_DROP_TX and P2P_DROP_TX_CHANCE via Helm values.
+ * Sets probabilistic transaction dropping on validators and waits for rollout.
+ * Use probability=0 to disable. Wired to env var P2P_DROP_TX_CHANCE via Helm values.
  */
 export async function setValidatorTxDrop({
   namespace,
-  enabled,
   probability,
   logger: log,
 }: {
   namespace: string;
-  enabled: boolean;
   probability: number;
   logger: Logger;
 }) {
-  const drop = enabled ? 'true' : 'false';
   const prob = String(probability);
 
   const selectors = ['app.kubernetes.io/name=validator', 'app.kubernetes.io/component=validator', 'app=validator'];
@@ -284,7 +281,7 @@ export async function setValidatorTxDrop({
       if (names.length === 0) {
         continue;
       }
-      const cmd = `kubectl set env statefulset -l ${selector} -n ${namespace} P2P_DROP_TX=${drop} P2P_DROP_TX_CHANCE=${prob}`;
+      const cmd = `kubectl set env statefulset -l ${selector} -n ${namespace} P2P_DROP_TX_CHANCE=${prob}`;
       log.info(`command: ${cmd}`);
       await execAsync(cmd);
       updated = true;
@@ -366,16 +363,24 @@ export async function enableValidatorDynamicBootNode(
  */
 export async function rollAztecPods(namespace: string, clearState: boolean = false) {
   // Pod components use 'validator', but StatefulSets and PVCs use 'sequencer-node' for validators
+  // RPC nodes have nodeType='rpc-node' in Helm values, so their component label is 'rpc-node' (not 'rpc')
   const podComponents = [
     'p2p-bootstrap',
     'prover-node',
     'prover-broker',
     'prover-agent',
     'sequencer-node',
-    'rpc',
+    'rpc-node',
+    'validator-ha-db',
+  ];
+  const pvcComponents = [
+    'p2p-bootstrap',
+    'prover-node',
+    'prover-broker',
+    'sequencer-node',
+    'rpc-node',
     'validator-ha-db',
   ];
-  const pvcComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'sequencer-node', 'rpc', 'validator-ha-db'];
   // StatefulSet components that need to be scaled down before PVC deletion
   // Note: validators use 'sequencer-node' as component label, not 'validator'
   const statefulSetComponents = [
@@ -383,7 +388,7 @@ export async function rollAztecPods(namespace: string, clearState: boolean = fal
     'prover-node',
     'prover-broker',
     'sequencer-node',
-    'rpc',
+    'rpc-node',
     'validator-ha-db',
   ];