@aztec/end-to-end 0.0.1-commit.2ed92850 → 0.0.1-commit.2f68f620

package/src/spartan/setup_test_wallets.ts

@@ -1,23 +1,29 @@
 import { generateSchnorrAccounts } from '@aztec/accounts/testing';
+import { NO_FROM } from '@aztec/aztec.js/account';
 import { AztecAddress } from '@aztec/aztec.js/addresses';
 import { NO_WAIT } from '@aztec/aztec.js/contracts';
 import { L1FeeJuicePortalManager } from '@aztec/aztec.js/ethereum';
 import { FeeJuicePaymentMethodWithClaim } from '@aztec/aztec.js/fee';
 import { type FeePaymentMethod, SponsoredFeePaymentMethod } from '@aztec/aztec.js/fee';
 import { Fr } from '@aztec/aztec.js/fields';
+import { waitForL1ToL2MessageReady } from '@aztec/aztec.js/messaging';
 import { type AztecNode, createAztecNodeClient, waitForTx } from '@aztec/aztec.js/node';
 import type { Wallet } from '@aztec/aztec.js/wallet';
 import { createEthereumChain } from '@aztec/ethereum/chain';
 import { createExtendedL1Client } from '@aztec/ethereum/client';
 import type { Logger } from '@aztec/foundation/log';
-import { retryUntil } from '@aztec/foundation/retry';
+import { makeBackoff, retry, retryUntil } from '@aztec/foundation/retry';
 import { TokenContract } from '@aztec/noir-contracts.js/Token';
 import type { AztecNodeAdmin } from '@aztec/stdlib/interfaces/client';
-import { TestWallet, proveInteraction, registerInitialLocalNetworkAccountsInWallet } from '@aztec/test-wallet/server';
+import { TxStatus } from '@aztec/stdlib/tx';
+import { registerInitialLocalNetworkAccountsInWallet } from '@aztec/wallets/testing';
 
 import { getACVMConfig } from '../fixtures/get_acvm_config.js';
 import { getBBConfig } from '../fixtures/get_bb_config.js';
 import { getSponsoredFPCAddress, registerSponsoredFPC } from '../fixtures/utils.js';
+import { TestWallet } from '../test-wallet/test_wallet.js';
+import { proveInteraction } from '../test-wallet/utils.js';
+import { WorkerWallet } from '../test-wallet/worker_wallet.js';
 
 export interface TestAccounts {
   aztecNode: AztecNode;
@@ -85,11 +91,19 @@ export async function deploySponsoredTestAccountsWithTokens(
 
   const paymentMethod = new SponsoredFeePaymentMethod(await getSponsoredFPCAddress());
   const recipientDeployMethod = await recipientAccount.getDeployMethod();
-  await recipientDeployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod }, wait: { timeout: 2400 } });
+  await recipientDeployMethod.send({
+    from: NO_FROM,
+    fee: { paymentMethod },
+    wait: { timeout: 2400 },
+  });
   await Promise.all(
     fundedAccounts.map(async a => {
       const deployMethod = await a.getDeployMethod();
-      await deployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod }, wait: { timeout: 2400 } }); // increase timeout on purpose in order to account for two empty epochs
+      await deployMethod.send({
+        from: NO_FROM,
+        fee: { paymentMethod },
+        wait: { timeout: 2400 },
+      }); // increase timeout on purpose in order to account for two empty epochs
       logger.info(`Account deployed at ${a.address}`);
     }),
   );
@@ -118,49 +132,91 @@ export async function deploySponsoredTestAccountsWithTokens(
 }
 
 async function deployAccountWithDiagnostics(
-  account: { getDeployMethod: () => Promise<{ send: (opts: any) => any }>; address: any },
+  account: { getDeployMethod: () => Promise<{ simulate: (opts: any) => any; send: (opts: any) => any }>; address: any },
   paymentMethod: SponsoredFeePaymentMethod,
   aztecNode: AztecNode,
   logger: Logger,
   accountLabel: string,
+  estimateGas?: boolean,
 ): Promise<void> {
   const deployMethod = await account.getDeployMethod();
-  let txHash;
-  try {
-    txHash = await deployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod }, wait: NO_WAIT });
-    await waitForTx(aztecNode, txHash, { timeout: 2400 });
-    logger.info(`${accountLabel} deployed at ${account.address}`);
-  } catch (error) {
-    const blockNumber = await aztecNode.getBlockNumber();
-    let receipt;
-    try {
-      receipt = await aztecNode.getTxReceipt(txHash);
-    } catch {
-      receipt = 'unavailable';
-    }
-    logger.error(`${accountLabel} deployment failed`, {
-      txHash: txHash.toString(),
-      receipt: JSON.stringify(receipt),
-      currentBlockNumber: blockNumber,
-      error: String(error),
-    });
-    throw error;
+  let gasSettings: any;
+  if (estimateGas) {
+    const sim = await deployMethod.simulate({ from: NO_FROM, fee: { paymentMethod } });
+    gasSettings = sim.estimatedGas;
+    logger.info(`${accountLabel} estimated gas: DA=${gasSettings.gasLimits.daGas} L2=${gasSettings.gasLimits.l2Gas}`);
   }
+
+  // Track the tx hash across retries so we don't re-send when the previous tx is still pending.
+  let sentTxHash: { txHash: any } | undefined;
+
+  await retry(
+    async () => {
+      // Check if already deployed (handles case where previous attempt succeeded but waitForTx timed out)
+      const existing = await aztecNode.getContract(account.address);
+      if (existing) {
+        logger.info(`${accountLabel} already deployed at ${account.address}, skipping`);
+        return;
+      }
+
+      // If we already sent a tx, check if it was dropped before deciding to re-send.
+      if (sentTxHash) {
+        const prevReceipt = await aztecNode.getTxReceipt(sentTxHash.txHash);
+        if (prevReceipt.isDropped()) {
+          logger.info(`${accountLabel} previous tx ${sentTxHash.txHash} was dropped, re-sending`);
+          sentTxHash = undefined;
+        } else {
+          logger.info(`${accountLabel} previous tx ${sentTxHash.txHash} still pending, waiting again...`);
+        }
+      }
+
+      if (!sentTxHash) {
+        const deployResult = await deployMethod.send({
+          from: NO_FROM,
+          fee: { paymentMethod, gasSettings },
+          wait: NO_WAIT,
+        });
+        sentTxHash = { txHash: deployResult.txHash };
+        logger.info(`${accountLabel} tx sent`, { txHash: sentTxHash.txHash.toString() });
+      }
+
+      const receipt = await waitForTx(aztecNode, sentTxHash.txHash, { timeout: 600 });
+      if (receipt.isDropped()) {
+        sentTxHash = undefined;
+        throw new Error(`${accountLabel} tx was dropped, retrying...`);
+      }
+      logger.info(`${accountLabel} deployed at ${account.address}`);
+    },
+    `deploy ${accountLabel}`,
+    makeBackoff([1, 2, 4, 8, 16]),
+    logger,
+  );
 }
 
 async function deployAccountsInBatches(
-  accounts: { getDeployMethod: () => Promise<{ send: (opts: any) => any }>; address: any }[],
+  accounts: {
+    getDeployMethod: () => Promise<{ simulate: (opts: any) => any; send: (opts: any) => any }>;
+    address: any;
+  }[],
   paymentMethod: SponsoredFeePaymentMethod,
   aztecNode: AztecNode,
   logger: Logger,
   labelPrefix: string,
   batchSize = 2,
+  estimateGas?: boolean,
 ): Promise<void> {
   for (let i = 0; i < accounts.length; i += batchSize) {
     const batch = accounts.slice(i, i + batchSize);
     await Promise.all(
       batch.map((account, idx) =>
-        deployAccountWithDiagnostics(account, paymentMethod, aztecNode, logger, `${labelPrefix}${i + idx + 1}`),
+        deployAccountWithDiagnostics(
+          account,
+          paymentMethod,
+          aztecNode,
+          logger,
+          `${labelPrefix}${i + idx + 1}`,
+          estimateGas,
+        ),
       ),
     );
   }
@@ -171,6 +227,7 @@ export async function deploySponsoredTestAccounts(
   aztecNode: AztecNode,
   logger: Logger,
   numberOfFundedWallets = 1,
+  opts?: { estimateGas?: boolean },
 ): Promise<TestAccountsWithoutTokens> {
   const [recipient, ...funded] = await generateSchnorrAccounts(numberOfFundedWallets + 1);
   const recipientAccount = await wallet.createSchnorrAccount(recipient.secret, recipient.salt);
@@ -180,8 +237,23 @@ export async function deploySponsoredTestAccounts(
 
   const paymentMethod = new SponsoredFeePaymentMethod(await getSponsoredFPCAddress());
 
-  await deployAccountWithDiagnostics(recipientAccount, paymentMethod, aztecNode, logger, 'Recipient account');
-  await deployAccountsInBatches(fundedAccounts, paymentMethod, aztecNode, logger, 'Funded account ', 2);
+  await deployAccountWithDiagnostics(
+    recipientAccount,
+    paymentMethod,
+    aztecNode,
+    logger,
+    'Recipient account',
+    opts?.estimateGas,
+  );
+  await deployAccountsInBatches(
+    fundedAccounts,
+    paymentMethod,
+    aztecNode,
+    logger,
+    'Funded account ',
+    2,
+    opts?.estimateGas,
+  );
 
   return {
     aztecNode,
@@ -219,7 +291,7 @@ export async function deployTestAccountsWithTokens(
     fundedAccounts.map(async (a, i) => {
       const paymentMethod = new FeeJuicePaymentMethodWithClaim(a.address, claims[i]);
       const deployMethod = await a.getDeployMethod();
-      await deployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod } });
+      await deployMethod.send({ from: NO_FROM, fee: { paymentMethod } });
       logger.info(`Account deployed at ${a.address}`);
     }),
   );
@@ -262,9 +334,7 @@ async function bridgeL1FeeJuice(
   const portal = await L1FeeJuicePortalManager.new(aztecNode, l1Client, log);
   const claim = await portal.bridgeTokensPublic(recipient, amount, true /* mint */);
 
-  const isSynced = async () =>
-    (await aztecNode.getL1ToL2MessageBlock(Fr.fromHexString(claim.messageHash))) !== undefined;
-  await retryUntil(isSynced, `message ${claim.messageHash} sync`, 24, 0.5);
+  await waitForL1ToL2MessageReady(aztecNode, Fr.fromHexString(claim.messageHash), { timeoutSeconds: 24 });
 
   log.info(`Created a claim for ${amount} L1 fee juice to ${recipient}.`, claim);
   return claim;
@@ -306,7 +376,7 @@ async function deployTokenAndMint(
     fee: {
       paymentMethod,
     },
-    wait: { timeout: 600, returnReceipt: true },
+    wait: { timeout: 600 },
   });
 
   const tokenAddress = tokenContract.address;
@@ -357,7 +427,9 @@ export async function performTransfers({
 
     const provenTxs = await Promise.all(txs);
 
-    await Promise.all(provenTxs.map(t => t.send({ wait: { timeout: 600 } })));
+    // Wait only for the txs to be proposed, not checkpointed. This is enough to keep the chain
+    // loaded for the reorg scenario, and avoids each round blocking on the (slower) checkpoint lag.
+    await Promise.all(provenTxs.map(t => t.send({ wait: { timeout: 600, waitForStatus: TxStatus.PROPOSED } })));
 
     logger.info(`Completed round ${i + 1} / ${rounds}`);
   }
@@ -395,3 +467,42 @@ export async function createWalletAndAztecNodeClient(
     },
   };
 }
+
+export type WorkerWalletWrapper = {
+  wallet: WorkerWallet;
+  aztecNode: AztecNode;
+  cleanup: () => Promise<void>;
+};
+
+export async function createWorkerWalletClient(
+  nodeUrl: string,
+  proverEnabled: boolean,
+  logger: Logger,
+): Promise<WorkerWalletWrapper> {
+  const aztecNode = createAztecNodeClient(nodeUrl);
+  const [bbConfig, acvmConfig] = await Promise.all([getBBConfig(logger), getACVMConfig(logger)]);
+
+  // Strip cleanup functions — they can't be structured-cloned for worker transfer
+  const { cleanup: bbCleanup, ...bbPaths } = bbConfig ?? {};
+  const { cleanup: acvmCleanup, ...acvmPaths } = acvmConfig ?? {};
+
+  const pxeConfig = {
+    dataDirectory: undefined,
+    dataStoreMapSizeKb: 1024 * 1024,
+    ...bbPaths,
+    ...acvmPaths,
+    proverEnabled,
+  };
+
+  const wallet = await WorkerWallet.create(nodeUrl, pxeConfig);
+
+  return {
+    wallet,
+    aztecNode,
+    async cleanup() {
+      await wallet.stop();
+      await bbCleanup?.();
+      await acvmCleanup?.();
+    },
+  };
+}

package/src/spartan/tx_metrics.ts

@@ -1,16 +1,145 @@
 import type { AztecNode } from '@aztec/aztec.js/node';
 import type { Logger } from '@aztec/foundation/log';
-import type { L2Block } from '@aztec/stdlib/block';
+import type { BlockResponse } from '@aztec/stdlib/interfaces/client';
 import type { TopicType } from '@aztec/stdlib/p2p';
 import { Tx, type TxReceipt } from '@aztec/stdlib/tx';
 
 import { createHistogram } from 'perf_hooks';
 
+/** Metrics class for proving-related benchmarks. */
+export class ProvingMetrics {
+  private successfulTxs: number | undefined;
+  private proofDuration: number | undefined;
+  private activeAgents: number | undefined;
+  private avgQueueTime: number | undefined;
+  private jobRetries: number | undefined;
+  private jobDuration: number | undefined;
+  private timedOutJobs: number | undefined;
+  private resolvedJobs: number | undefined;
+  private rejectedJobs: number | undefined;
+  private epochProvingDuration: number | undefined;
+  private provenTransactions: number | undefined;
+  private provenBlocks: number | undefined;
+
+  constructor(private prefix: string) {}
+
+  recordSuccessfulTxs(count: number): void {
+    this.successfulTxs = count;
+  }
+
+  recordProofDuration(seconds: number): void {
+    this.proofDuration = seconds;
+  }
+
+  recordActiveAgents(count: number): void {
+    this.activeAgents = count;
+  }
+
+  recordAvgQueueTime(ms: number): void {
+    this.avgQueueTime = ms;
+  }
+
+  recordJobRetries(count: number): void {
+    this.jobRetries = count;
+  }
+
+  recordJobDuration(ms: number): void {
+    this.jobDuration = ms;
+  }
+
+  recordTimedOutJobs(count: number): void {
+    this.timedOutJobs = count;
+  }
+
+  recordResolvedJobs(count: number): void {
+    this.resolvedJobs = count;
+  }
+
+  recordRejectedJobs(count: number): void {
+    this.rejectedJobs = count;
+  }
+
+  recordEpochProvingDuration(seconds: number): void {
+    this.epochProvingDuration = seconds;
+  }
+
+  recordProvenTransactions(count: number): void {
+    this.provenTransactions = count;
+  }
+
+  recordProvenBlocks(count: number): void {
+    this.provenBlocks = count;
+  }
+
+  toGithubActionBenchmarkJSON(): Array<{ name: string; unit: string; value: number }> {
+    const data: Array<{ name: string; unit: string; value: number }> = [];
+
+    if (this.successfulTxs !== undefined) {
+      data.push({ name: `${this.prefix}/successful_txs`, unit: 'count', value: this.successfulTxs });
+    }
+
+    if (this.proofDuration !== undefined) {
+      data.push({ name: `${this.prefix}/proof_duration`, unit: 's', value: this.proofDuration });
+    }
+
+    if (this.activeAgents !== undefined) {
+      data.push({ name: `${this.prefix}/active_agents`, unit: 'count', value: this.activeAgents });
+    }
+
+    if (this.avgQueueTime !== undefined) {
+      data.push({ name: `${this.prefix}/avg_queue_time`, unit: 'ms', value: this.avgQueueTime });
+    }
+
+    if (this.jobRetries !== undefined) {
+      data.push({ name: `${this.prefix}/job_retries`, unit: 'count', value: this.jobRetries });
+    }
+
+    if (this.jobDuration !== undefined) {
+      data.push({ name: `${this.prefix}/job_duration`, unit: 'ms', value: this.jobDuration });
+    }
+
+    if (this.timedOutJobs !== undefined) {
+      data.push({ name: `${this.prefix}/timed_out_jobs`, unit: 'count', value: this.timedOutJobs });
+    }
+
+    if (this.resolvedJobs !== undefined) {
+      data.push({ name: `${this.prefix}/resolved_jobs`, unit: 'count', value: this.resolvedJobs });
+    }
+
+    if (this.rejectedJobs !== undefined) {
+      data.push({ name: `${this.prefix}/rejected_jobs`, unit: 'count', value: this.rejectedJobs });
+    }
+
+    if (this.epochProvingDuration !== undefined) {
+      data.push({ name: `${this.prefix}/epoch_proving_duration`, unit: 's', value: this.epochProvingDuration });
+    }
+
+    if (this.provenTransactions !== undefined) {
+      data.push({ name: `${this.prefix}/proven_transactions`, unit: 'count', value: this.provenTransactions });
+    }
+
+    if (this.provenBlocks !== undefined) {
+      data.push({ name: `${this.prefix}/proven_blocks`, unit: 'count', value: this.provenBlocks });
+    }
+
+    const scenario = process.env.BENCH_SCENARIO?.trim();
+    if (!scenario) {
+      return data;
+    }
+
+    const scenarioPrefix = `scenario/${scenario}/`;
+    return data.map(entry => ({ ...entry, name: `${scenarioPrefix}${entry.name}` }));
+  }
+}
+
 export type TxInclusionData = {
   txHash: string;
-  sentAt: number;
-  minedAt: number;
-  attestedAt: number;
+  /** Wall-clock at client when the tx was submitted, in ms (Date.now()). */
+  sentAtMs: number;
+  /** Wall-clock at client when the block containing the tx first became visible, in ms (Date.now()). -1 if never observed. */
+  minedAtMs: number;
+  /** Reserved for future attestation-observed-at signal; -1 today. */
+  attestedAtMs: number;
   blocknumber: number;
   priorityFee: number;
   totalFee: number;
@@ -21,7 +150,7 @@ export type TxInclusionData = {
 export class TxInclusionMetrics {
   private data = new Map<string, TxInclusionData>();
   private groups = new Set<string>();
-  private blocks = new Map<number, Promise<L2Block | undefined>>();
+  private blocks = new Map<number, Promise<BlockResponse<{ includeTransactions: true }> | undefined>>();
 
   private p2pGossipLatencyByTopic: Partial<Record<TopicType, { p50: number; p95: number }>> = {};
 
@@ -48,9 +177,9 @@ export class TxInclusionMetrics {
 
     this.data.set(txHash, {
       txHash,
-      sentAt: Math.trunc(Date.now() / 1000),
-      minedAt: -1,
-      attestedAt: -1,
+      sentAtMs: Date.now(),
+      minedAtMs: -1,
+      attestedAtMs: -1,
       blocknumber: -1,
       priorityFee: Number(priorityFees.feePerDaGas + priorityFees.feePerL2Gas),
       totalFee: -1,
@@ -60,6 +189,28 @@ export class TxInclusionMetrics {
     this.groups.add(group);
   }
 
+  /**
+   * Stamp mined-at metadata for any tracked tx contained in this block, using
+   * `observedAtMs` (caller-supplied wall-clock at the moment they first saw the
+   * block). Idempotent: existing minedAtMs is preserved so the first observer
+   * wins (typically the block-watcher; recordMinedTx is a fallback).
+   */
+  observeBlockForMinedTxs(
+    blockNumber: number,
+    txHashes: ReadonlyArray<{ toString(): string }>,
+    observedAtMs: number,
+  ): void {
+    txHashes.forEach((txHash, position) => {
+      const data = this.data.get(txHash.toString());
+      if (!data || data.minedAtMs !== -1) {
+        return;
+      }
+      data.blocknumber = blockNumber;
+      data.minedAtMs = observedAtMs;
+      data.positionInBlock = position;
+    });
+  }
+
   async recordMinedTx(txReceipt: TxReceipt): Promise<void> {
     const { txHash, blockNumber } = txReceipt;
     if (!txReceipt.isMined() || !txReceipt.hasExecutionSucceeded() || !blockNumber) {
@@ -71,26 +222,43 @@ export class TxInclusionMetrics {
       return;
     }
 
-    if (!this.blocks.has(blockNumber)) {
-      this.blocks.set(blockNumber, this.aztecNode.getBlock(blockNumber));
-    }
-
-    const block = await this.blocks.get(blockNumber)!;
-    if (!block) {
-      this.logger?.warn('Failed to load block for mined tx receipt', { txHash: txHash.toString(), blockNumber });
-      return;
-    }
     const data = this.data.get(txHash.toString());
     if (!data) {
       const message = `Missing sent tx record for mined tx ${txHash.toString()}`;
       this.logger?.warn(message, { txHash: txHash.toString(), blockNumber });
       throw new Error(message);
     }
-    data.blocknumber = blockNumber;
-    data.minedAt = Number(block.header.globalVariables.timestamp);
-    data.attestedAt = -1;
     data.totalFee = Number(txReceipt.transactionFee ?? 0n);
-    data.positionInBlock = block.body.txEffects.findIndex(txEffect => txEffect.txHash.equals(txHash));
+
+    // Fallback path for txs the block-watcher missed (e.g. observed only after
+    // the watcher stopped). Stamp with the block's L2 slot timestamp; this is
+    // earlier than the true client-observed time by attestation+propagation
+    // lag, but it's the only deterministic timestamp available post-hoc.
+    if (data.minedAtMs === -1) {
+      if (!this.blocks.has(blockNumber)) {
+        this.blocks.set(blockNumber, this.aztecNode.getBlock(blockNumber, { includeTransactions: true }));
+      }
+      const block = await this.blocks.get(blockNumber)!;
+      if (!block) {
+        this.logger?.warn('Failed to load block for mined tx receipt', { txHash: txHash.toString(), blockNumber });
+        return;
+      }
+      data.blocknumber = blockNumber;
+      data.minedAtMs = Number(block.header.globalVariables.timestamp) * 1000;
+      data.positionInBlock = block.body.txEffects.findIndex(txEffect => txEffect.txHash.equals(txHash));
+    }
+  }
+
+  /** Per-tx inclusion records for a group. Used to serialise out for downstream tooling. */
+  getInclusionRecords(group?: string): TxInclusionData[] {
+    const out: TxInclusionData[] = [];
+    for (const tx of this.data.values()) {
+      if (group !== undefined && tx.group !== group) {
+        continue;
+      }
+      out.push({ ...tx });
+    }
+    return out;
   }
 
   public inclusionTimeInSeconds(group: string): {
@@ -103,12 +271,28 @@ export class TxInclusionMetrics {
     p99: number;
   } {
     const histogram = createHistogram({});
+    let nonPositive = 0;
     for (const tx of this.data.values()) {
-      if (!tx.blocknumber || tx.group !== group || tx.minedAt === -1) {
+      if (!tx.blocknumber || tx.group !== group || tx.minedAtMs === -1) {
         continue;
       }
 
-      histogram.record(tx.minedAt - tx.sentAt);
+      // Both timestamps are client wall-clock (ms). A negative delta should be
+      // impossible since the watcher stamps minedAtMs strictly after sentAtMs,
+      // but the fallback path (recordMinedTx via L2 slot timestamp) can stamp
+      // earlier than sentAtMs. perf_hooks.createHistogram rejects <=0; skip
+      // those instead of crashing.
+      const deltaMs = tx.minedAtMs - tx.sentAtMs;
+      if (deltaMs <= 0) {
+        nonPositive++;
+        continue;
+      }
+      // Histogram is recorded in seconds (rounded) to match the existing
+      // toGithubActionBenchmarkJSON output unit; per-tx records carry the raw ms.
+      histogram.record(Math.max(1, Math.round(deltaMs / 1000)));
+    }
+    if (nonPositive > 0) {
+      this.logger?.debug(`Dropped ${nonPositive} tx inclusion samples with non-positive delta`, { group });
     }
 
     if (histogram.count === 0) {
@@ -170,7 +354,7 @@ export class TxInclusionMetrics {
           value: stats.mean,
         },
         {
-          name: `${group}/median_inclusion`,
+          name: `${group}/p50_inclusion`,
           unit: 's',
           value: stats.median,
         },

package/src/spartan/utils/bot.ts

@@ -34,7 +34,8 @@ export async function installTransferBot({
   logger: log,
   replicas = 1,
   txIntervalSeconds = 10,
-  followChain = 'PENDING',
+  followChain = 'CHECKPOINTED',
+  pxeSyncChainTip = 'proposed',
   mnemonic = process.env.LABS_INFRA_MNEMONIC ?? 'test test test test test test test test test test test junk',
   mnemonicStartIndex,
   botPrivateKey = process.env.BOT_TRANSFERS_L2_PRIVATE_KEY ?? '0xcafe01',
@@ -49,6 +50,7 @@ export async function installTransferBot({
   replicas?: number;
   txIntervalSeconds?: number;
   followChain?: string;
+  pxeSyncChainTip?: string;
   mnemonic?: string;
   mnemonicStartIndex?: number | string;
   botPrivateKey?: string;
@@ -67,6 +69,7 @@ export async function installTransferBot({
     'bot.replicaCount': replicas,
     'bot.txIntervalSeconds': txIntervalSeconds,
     'bot.followChain': followChain,
+    'bot.pxeSyncChainTip': pxeSyncChainTip,
     'bot.botPrivateKey': botPrivateKey,
     'bot.nodeUrl': resolvedNodeUrl,
     'bot.mnemonic': mnemonic,

package/src/spartan/utils/config.ts

@@ -8,6 +8,7 @@ const logger = createLogger('e2e:k8s-utils');
 const testConfigSchema = z.object({
   NAMESPACE: z.string().default('scenario'),
   REAL_VERIFIER: schemas.Boolean.optional().default(true),
+  DEBUG_FORCE_TX_PROOF_VERIFICATION: schemas.Boolean.optional().default(true),
   CREATE_ETH_DEVNET: schemas.Boolean.optional().default(false),
   L1_RPC_URLS_JSON: z.string().optional(),
   L1_ACCOUNT_MNEMONIC: z.string().optional(),
@@ -15,6 +16,8 @@ const testConfigSchema = z.object({
   AZTEC_EPOCH_DURATION: z.coerce.number().optional().default(32),
   AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5),
   AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET: z.coerce.number().optional().default(2),
+  FUNDING_PRIVATE_KEY: z.string().optional(),
+  AZTEC_ADMIN_API_KEY: z.string().optional(),
 });
 
 export type TestConfig = z.infer<typeof testConfigSchema>;

package/src/spartan/utils/index.ts

@@ -5,7 +5,7 @@
 export { type TestConfig, setupEnvironment } from './config.js';
 
 // Scripts
-export { getGitProjectRoot, getAztecBin, runAztecBin, runProjectScript } from './scripts.js';
+export { type ScriptResult, getGitProjectRoot, getAztecBin, runAztecBin, runProjectScript } from './scripts.js';
 
 // K8s operations
 export {
@@ -24,6 +24,8 @@ export {
   getServiceEndpoint,
   getRPCEndpoint,
   getEthereumEndpoint,
+  createResilientPrometheusConnection,
+  scaleProverAgents,
 } from './k8s.js';
 
 // Chaos Mesh
@@ -39,12 +41,16 @@ export {
   applyNetworkShaping,
 } from './chaos.js';
 
+// Helm
+export { hasDeployedHelmRelease } from './helm.js';
+
 // Bot management
 export { restartBot, installTransferBot, uninstallTransferBot } from './bot.js';
 
 // Node operations (sequencers, validators, pods)
 export {
   awaitCheckpointNumber,
+  waitForProvenToAdvance,
   getSequencers,
   updateSequencersConfig,
   getSequencersConfig,
@@ -60,3 +66,6 @@ export { getPublicViemClient, getL1DeploymentAddresses, getNodeClient } from './
 
 // Health checks
 export { ChainHealth, type ChainHealthSnapshot } from './health.js';
+
+// Pod log extraction
+export { type BlockBuiltLogEntry, fetchBlockBuiltLogs } from './pod_logs.js';