@aztec/end-to-end 0.0.1-commit.c80b6263 → 0.0.1-commit.c949de6bc

package/src/shared/mock_state_view.ts

@@ -0,0 +1,188 @@
+import { EthAddress } from '@aztec/aztec.js/addresses';
+import { type Logger, createLogger } from '@aztec/aztec.js/log';
+import type { EthCheatCodes } from '@aztec/aztec/testing';
+import type { ExtendedViemWalletClient } from '@aztec/ethereum/types';
+
+import { getContract } from 'viem';
+
+/**
+ * Mock StateView contract for testing the Uniswap price oracle.
+ *
+ * Wraps a compiled Solidity contract that mimics the Uniswap V4 StateView's getSlot0 function.
+ * The mock allows setting return values dynamically for testing different price scenarios.
+ *
+ * Solidity source:
+ * ```solidity
+ * // SPDX-License-Identifier: Apache-2.0
+ * pragma solidity >=0.8.27;
+ *
+ * contract MockStateView {
+ *     uint160 public sqrtPriceX96;
+ *     int24 public tick;
+ *     uint24 public protocolFee;
+ *     uint24 public lpFee;
+ *
+ *     function setReturnValues(
+ *         uint160 _sqrtPriceX96,
+ *         int24 _tick,
+ *         uint24 _protocolFee,
+ *         uint24 _lpFee
+ *     ) external {
+ *         sqrtPriceX96 = _sqrtPriceX96;
+ *         tick = _tick;
+ *         protocolFee = _protocolFee;
+ *         lpFee = _lpFee;
+ *     }
+ *
+ *     function getSlot0(bytes32 poolId) external view returns (uint160, int24, uint24, uint24) {
+ *         return (sqrtPriceX96, tick, protocolFee, lpFee);
+ *     }
+ * }
+ * ```
+ */
+export class MockStateView {
+  private static readonly BYTECODE: `0x${string}` =
+    '0x608060405234801561000f575f5ffd5b5060043610610060575f3560e01c80633eaf5d9f14610064578063704ce43e146100825780638db791d2146100a0578063b0e21e8a146100be578063b52e4bdd146100dc578063c815641c146100f8575b5f5ffd5b61006c61012b565b60405161007991906102ab565b60405180910390f35b61008a61013d565b60405161009791906102e1565b60405180910390f35b6100a8610151565b6040516100b59190610328565b60405180910390f35b6100c6610175565b6040516100d391906102e1565b60405180910390f35b6100f660048036038101906100f191906103c3565b610189565b005b610112600480360381019061010d919061045a565b61022b565b6040516101229493929190610485565b60405180910390f35b5f60149054906101000a900460020b81565b5f601a9054906101000a900462ffffff1681565b5f5f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1681565b5f60179054906101000a900462ffffff1681565b835f5f6101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550825f60146101000a81548162ffffff021916908360020b62ffffff160217905550815f60176101000a81548162ffffff021916908362ffffff160217905550805f601a6101000a81548162ffffff021916908362ffffff16021790555050505050565b5f5f5f5f5f5f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff165f60149054906101000a900460020b5f60179054906101000a900462ffffff165f601a9054906101000a900462ffffff1693509350935093509193509193565b5f8160020b9050919050565b6102a581610290565b82525050565b5f6020820190506102be5f83018461029c565b92915050565b5f62ffffff82169050919050565b6102db816102c4565b82525050565b5f6020820190506102f45f8301846102d2565b92915050565b5f73ffffffffffffffffffffffffffffffffffffffff82169050919050565b610322816102fa565b82525050565b5f60208201905061033b5f830184610319565b92915050565b5f5ffd5b61034e816102fa565b8114610358575f5ffd5b50565b5f8135905061036981610345565b92915050565b61037881610290565b8114610382575f5ffd5b50565b5f813590506103938161036f565b92915050565b6103a2816102c4565b81146103ac575f5ffd5b50565b5f813590506103bd81610399565b92915050565b5f5f5f5f608085870312156103db576103da610341565b5b5f6103e88782880161035b565b94505060206103f987828801610385565b935050604061040a878288016103af565b925050606061041b878288016103af565b91505092959194509250565b5f819050919050565b61043981610427565b8114610443575f5ffd5b50565b5f8135905061045481610430565b92915050565b5f6020828403121561046f5761046e610341565b5b5f61047c84828501610446565b91505092915050565b5f6080820190506104985f830187610319565b6104a5602083018661029c565b6104b260408301856102d2565b6104bf60608301846102d2565b9594505050505056fea2646970667358221220f8b1bfff284535bc078368ed34bd5e78981644845f3c9c1f5a4b8448c976805464736f6c634300081f0033';
+  private static readonly ABI = [
+    {
+      type: 'function',
+      name: 'setReturnValues',
+      inputs: [
+        { name: '_sqrtPriceX96', type: 'uint160' },
+        { name: '_tick', type: 'int24' },
+        { name: '_protocolFee', type: 'uint24' },
+        { name: '_lpFee', type: 'uint24' },
+      ],
+      outputs: [],
+      stateMutability: 'nonpayable',
+    },
+    {
+      inputs: [
+        {
+          internalType: 'bytes32',
+          name: 'poolId',
+          type: 'bytes32',
+        },
+      ],
+      name: 'getSlot0',
+      outputs: [
+        {
+          internalType: 'uint160',
+          name: '',
+          type: 'uint160',
+        },
+        {
+          internalType: 'int24',
+          name: '',
+          type: 'int24',
+        },
+        {
+          internalType: 'uint24',
+          name: '',
+          type: 'uint24',
+        },
+        {
+          internalType: 'uint24',
+          name: '',
+          type: 'uint24',
+        },
+      ],
+      stateMutability: 'view',
+      type: 'function',
+    },
+  ] as const;
+
+  private constructor(
+    private readonly address: EthAddress,
+    private readonly walletClient: ExtendedViemWalletClient,
+    private readonly log: Logger = createLogger('mock-state-view'),
+  ) {}
+
+  /**
+   * Deploys the mock StateView contract at the specified address using etch.
+   * @param ethCheatCodes - Cheat codes for etching bytecode
+   * @param walletClient - Wallet client for sending transactions
+   * @param address - Address to deploy the mock at (typically the real StateView address)
+   */
+  static async deploy(
+    ethCheatCodes: EthCheatCodes,
+    walletClient: ExtendedViemWalletClient,
+    address: EthAddress,
+  ): Promise<MockStateView> {
+    await ethCheatCodes.etch(address, MockStateView.BYTECODE);
+    return new MockStateView(address, walletClient);
+  }
+
+  /**
+   * Sets the price using the ethPerFeeAssetE12 format (same as rollup contract).
+   * Computes the corresponding sqrtPriceX96 internally.
+   *
+   * Math (from fee_asset_price_oracle.ts):
+   *   ethPerFeeAssetE12 = 1e12 * 2^192 / sqrtPriceX96^2
+   *
+   * Inverted:
+   *   sqrtPriceX96^2 = 1e12 * 2^192 / ethPerFeeAssetE12
+   *   sqrtPriceX96 = sqrt(1e12 * 2^192 / ethPerFeeAssetE12)
+   *
+   * @param ethPerFeeAssetE12 - The price in ETH per fee asset, scaled by 1e12
+   */
+  async setEthPerFeeAsset(ethPerFeeAssetE12: bigint) {
+    const sqrtPriceX96 = this.ethPerFeeAssetE12ToSqrtPriceX96(ethPerFeeAssetE12);
+    return await this.setSqrtPriceX96(sqrtPriceX96);
+  }
+
+  /**
+   * Sets the sqrtPriceX96 value directly (Uniswap's price encoding).
+   * @param sqrtPriceX96 - The sqrtPriceX96 value
+   * @param tick - The tick value (default 10)
+   * @param protocolFee - The protocol fee (default 0)
+   * @param lpFee - The LP fee (default 500)
+   */
+  async setSqrtPriceX96(sqrtPriceX96: bigint, tick: number = 10, protocolFee: number = 0, lpFee: number = 500) {
+    const contract = getContract({
+      address: this.address.toString() as `0x${string}`,
+      abi: MockStateView.ABI,
+      client: this.walletClient,
+    });
+
+    const hash = await contract.write.setReturnValues([sqrtPriceX96, tick, protocolFee, lpFee]);
+    this.log.info(`Set sqrtPriceX96 to ${sqrtPriceX96}`);
+    return await this.walletClient.waitForTransactionReceipt({ hash });
+  }
+
+  /**
+   * Converts ethPerFeeAssetE12 to sqrtPriceX96 (inverse of sqrtPriceX96ToEthPerFeeAssetE12).
+   *
+   * Math:
+   *   sqrtPriceX96 = sqrt(1e12 * 2^192 / ethPerFeeAssetE12)
+   */
+  ethPerFeeAssetE12ToSqrtPriceX96(ethPerFeeAssetE12: bigint): bigint {
+    if (ethPerFeeAssetE12 === 0n) {
+      throw new Error('Cannot convert zero ethPerFeeAssetE12');
+    }
+    const Q192 = 2n ** 192n;
+    const sqrtPriceSquared = (10n ** 12n * Q192) / ethPerFeeAssetE12;
+    return this.bigintSqrt(sqrtPriceSquared);
+  }
+
+  /** Integer square root using Newton's method */
+  bigintSqrt(n: bigint): bigint {
+    if (n < 0n) {
+      throw new Error('Cannot compute sqrt of negative number');
+    }
+    if (n === 0n) {
+      return 0n;
+    }
+    let x = n;
+    let y = (x + 1n) / 2n;
+    while (y < x) {
+      x = y;
+      y = (x + n / x) / 2n;
+    }
+    return x;
+  }
+}
+
+export function diffInBps(a: bigint, b: bigint): bigint {
+  return ((a - b) * 10000n) / b;
+}

package/src/shared/submit-transactions.ts

@@ -4,7 +4,8 @@ import { Fr, GrumpkinScalar } from '@aztec/aztec.js/fields';
 import type { Logger } from '@aztec/aztec.js/log';
 import { TxHash, type TxReceipt, TxStatus } from '@aztec/aztec.js/tx';
 import { times } from '@aztec/foundation/collection';
-import type { TestWallet } from '@aztec/test-wallet/server';
+
+import type { TestWallet } from '../test-wallet/test_wallet.js';
 
 // submits a set of transactions to the provided Wallet
 export const submitTxsTo = async (
@@ -18,7 +19,10 @@ export const submitTxsTo = async (
     times(numTxs, async () => {
       const accountManager = await wallet.createSchnorrAccount(Fr.random(), Fr.random(), GrumpkinScalar.random());
       const deployMethod = await accountManager.getDeployMethod();
-      const txHash = await deployMethod.send({ from: submitter, wait: NO_WAIT });
+      const txHash = await deployMethod.send({
+        from: submitter,
+        wait: NO_WAIT,
+      });
 
       logger.info(`Tx sent with hash ${txHash}`);
       const receipt: TxReceipt = await wallet.getTxReceipt(txHash);

package/src/shared/uniswap_l1_l2.ts

@@ -11,18 +11,18 @@ import type { DeployAztecL1ContractsReturnType } from '@aztec/ethereum/deploy-az
 import { deployL1Contract } from '@aztec/ethereum/deploy-l1-contract';
 import type { ExtendedViemWalletClient } from '@aztec/ethereum/types';
 import { extractEvent } from '@aztec/ethereum/utils';
-import { CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
+import { EpochNumber } from '@aztec/foundation/branded-types';
 import { sha256ToField } from '@aztec/foundation/crypto/sha256';
 import { InboxAbi, UniswapPortalAbi, UniswapPortalBytecode } from '@aztec/l1-artifacts';
 import { UniswapContract } from '@aztec/noir-contracts.js/Uniswap';
 import { computeL2ToL1MessageHash } from '@aztec/stdlib/hash';
 import { computeL2ToL1MembershipWitness } from '@aztec/stdlib/messaging';
-import type { TestWallet } from '@aztec/test-wallet/server';
 
 import { jest } from '@jest/globals';
 import { type GetContractReturnType, getContract, parseEther, toFunctionSelector } from 'viem';
 
 import { type EndToEndContext, ensureAccountContractsPublished } from '../fixtures/utils.js';
+import type { TestWallet } from '../test-wallet/test_wallet.js';
 import { CrossChainTestHarness } from './cross_chain_test_harness.js';
 
 // PSA: This tests works on forked mainnet. There is a dump of the data in `dumpedState` such that we
@@ -84,7 +84,7 @@ export const uniswapL1L2TestSuite = (
 
       l1Client = deployL1ContractsValues.l1Client;
 
-      t.watcher?.setIsMarkingAsProven(false);
+      t.watcher.setIsMarkingAsProven(false);
 
       if (Number(await l1Client.getBlockNumber()) < expectedForkBlockNumber) {
         throw new Error('This test must be run on a fork of mainnet with the expected fork block');
@@ -250,8 +250,8 @@ export const uniswapL1L2TestSuite = (
       await wethCrossChainHarness.expectPublicBalanceOnL2(uniswapL2Contract.address, 0n);
 
       // Since the outbox is only consumable when the epoch is proven, we need to advance to the next epoch.
-      const checkpointNumber = CheckpointNumber.fromBlockNumber(l2UniswapInteractionReceipt.blockNumber!);
-      const epoch = await rollup.getEpochNumberForCheckpoint(checkpointNumber);
+      const block = await aztecNode.getBlock(l2UniswapInteractionReceipt.blockNumber!);
+      const epoch = await rollup.getEpochNumberForCheckpoint(block!.checkpointNumber);
       await cheatCodes.rollup.advanceToEpoch(EpochNumber(epoch + 1));
       await waitForProven(aztecNode, l2UniswapInteractionReceipt, { provenTimeout: 300 });
 
@@ -838,9 +838,8 @@ export const uniswapL1L2TestSuite = (
         chainId: new Fr(l1Client.chain.id),
       });
 
-      const epoch = await rollup.getEpochNumberForCheckpoint(
-        CheckpointNumber.fromBlockNumber(withdrawReceipt.blockNumber!),
-      );
+      const block = await aztecNode.getBlock(withdrawReceipt.blockNumber!);
+      const epoch = await rollup.getEpochNumberForCheckpoint(block!.checkpointNumber);
       const swapResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, swapPrivateLeaf);
       const withdrawResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, withdrawLeaf);
 
@@ -972,9 +971,8 @@ export const uniswapL1L2TestSuite = (
         chainId: new Fr(l1Client.chain.id),
       });
 
-      const epoch = await rollup.getEpochNumberForCheckpoint(
-        CheckpointNumber.fromBlockNumber(withdrawReceipt.blockNumber!),
-      );
+      const block = await aztecNode.getBlock(withdrawReceipt.blockNumber!);
+      const epoch = await rollup.getEpochNumberForCheckpoint(block!.checkpointNumber);
       const swapResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, swapPublicLeaf);
       const withdrawResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, withdrawLeaf);
 

package/src/simulators/token_simulator.ts

@@ -119,39 +119,11 @@ export class TokenSimulator {
   }
 
   async checkPrivate() {
-    // Private calls
-    const defaultLookups = [];
-    const nonDefaultLookups = [];
-
     for (const address of this.accounts) {
-      if (this.lookupProvider.has(address.toString())) {
-        nonDefaultLookups.push(address);
-      } else {
-        defaultLookups.push(address);
-      }
-    }
-
-    const defaultCalls = defaultLookups.map(address => this.token.methods.balance_of_private(address));
-    const results = (
-      await Promise.all(
-        chunk(defaultCalls, 4).map(batch =>
-          new BatchCall(this.defaultWallet, batch).simulate({ from: this.defaultAddress }),
-        ),
-      )
-    ).flat();
-    for (let i = 0; i < defaultLookups.length; i++) {
-      expect(results[i]).toEqual(this.balanceOfPrivate(defaultLookups[i]));
-    }
-
-    // We are just running individual calls for the non-default lookups
-    // @todo We should also batch these
-    for (const address of nonDefaultLookups) {
       const wallet = this.lookupProvider.get(address.toString());
       const asset = wallet ? this.token.withWallet(wallet) : this.token;
 
-      const actualPrivateBalance = await asset.methods
-        .balance_of_private({ address })
-        .simulate({ from: this.defaultAddress });
+      const actualPrivateBalance = await asset.methods.balance_of_private(address).simulate({ from: address });
       expect(actualPrivateBalance).toEqual(this.balanceOfPrivate(address));
     }
   }

package/src/spartan/setup_test_wallets.ts

@@ -13,11 +13,14 @@ import type { Logger } from '@aztec/foundation/log';
 import { retryUntil } from '@aztec/foundation/retry';
 import { TokenContract } from '@aztec/noir-contracts.js/Token';
 import type { AztecNodeAdmin } from '@aztec/stdlib/interfaces/client';
-import { TestWallet, proveInteraction, registerInitialLocalNetworkAccountsInWallet } from '@aztec/test-wallet/server';
+import { registerInitialLocalNetworkAccountsInWallet } from '@aztec/wallets/testing';
 
 import { getACVMConfig } from '../fixtures/get_acvm_config.js';
 import { getBBConfig } from '../fixtures/get_bb_config.js';
 import { getSponsoredFPCAddress, registerSponsoredFPC } from '../fixtures/utils.js';
+import { TestWallet } from '../test-wallet/test_wallet.js';
+import { proveInteraction } from '../test-wallet/utils.js';
+import { WorkerWallet } from '../test-wallet/worker_wallet.js';
 
 export interface TestAccounts {
   aztecNode: AztecNode;
@@ -85,11 +88,19 @@ export async function deploySponsoredTestAccountsWithTokens(
 
   const paymentMethod = new SponsoredFeePaymentMethod(await getSponsoredFPCAddress());
   const recipientDeployMethod = await recipientAccount.getDeployMethod();
-  await recipientDeployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod }, wait: { timeout: 2400 } });
+  await recipientDeployMethod.send({
+    from: AztecAddress.ZERO,
+    fee: { paymentMethod },
+    wait: { timeout: 2400 },
+  });
   await Promise.all(
     fundedAccounts.map(async a => {
       const deployMethod = await a.getDeployMethod();
-      await deployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod }, wait: { timeout: 2400 } }); // increase timeout on purpose in order to account for two empty epochs
+      await deployMethod.send({
+        from: AztecAddress.ZERO,
+        fee: { paymentMethod },
+        wait: { timeout: 2400 },
+      }); // increase timeout on purpose in order to account for two empty epochs
       logger.info(`Account deployed at ${a.address}`);
     }),
   );
@@ -127,7 +138,11 @@ async function deployAccountWithDiagnostics(
   const deployMethod = await account.getDeployMethod();
   let txHash;
   try {
-    txHash = await deployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod }, wait: NO_WAIT });
+    txHash = await deployMethod.send({
+      from: AztecAddress.ZERO,
+      fee: { paymentMethod },
+      wait: NO_WAIT,
+    });
     await waitForTx(aztecNode, txHash, { timeout: 2400 });
     logger.info(`${accountLabel} deployed at ${account.address}`);
   } catch (error) {
@@ -395,3 +410,42 @@ export async function createWalletAndAztecNodeClient(
     },
   };
 }
+
+export type WorkerWalletWrapper = {
+  wallet: WorkerWallet;
+  aztecNode: AztecNode;
+  cleanup: () => Promise<void>;
+};
+
+export async function createWorkerWalletClient(
+  nodeUrl: string,
+  proverEnabled: boolean,
+  logger: Logger,
+): Promise<WorkerWalletWrapper> {
+  const aztecNode = createAztecNodeClient(nodeUrl);
+  const [bbConfig, acvmConfig] = await Promise.all([getBBConfig(logger), getACVMConfig(logger)]);
+
+  // Strip cleanup functions — they can't be structured-cloned for worker transfer
+  const { cleanup: bbCleanup, ...bbPaths } = bbConfig ?? {};
+  const { cleanup: acvmCleanup, ...acvmPaths } = acvmConfig ?? {};
+
+  const pxeConfig = {
+    dataDirectory: undefined,
+    dataStoreMapSizeKb: 1024 * 1024,
+    ...bbPaths,
+    ...acvmPaths,
+    proverEnabled,
+  };
+
+  const wallet = await WorkerWallet.create(nodeUrl, pxeConfig);
+
+  return {
+    wallet,
+    aztecNode,
+    async cleanup() {
+      await wallet.stop();
+      await bbCleanup?.();
+      await acvmCleanup?.();
+    },
+  };
+}

package/src/spartan/utils/bot.ts

@@ -34,7 +34,8 @@ export async function installTransferBot({
   logger: log,
   replicas = 1,
   txIntervalSeconds = 10,
-  followChain = 'PENDING',
+  followChain = 'CHECKPOINTED',
+  pxeSyncChainTip = 'proposed',
   mnemonic = process.env.LABS_INFRA_MNEMONIC ?? 'test test test test test test test test test test test junk',
   mnemonicStartIndex,
   botPrivateKey = process.env.BOT_TRANSFERS_L2_PRIVATE_KEY ?? '0xcafe01',
@@ -49,6 +50,7 @@ export async function installTransferBot({
   replicas?: number;
   txIntervalSeconds?: number;
   followChain?: string;
+  pxeSyncChainTip?: string;
   mnemonic?: string;
   mnemonicStartIndex?: number | string;
   botPrivateKey?: string;
@@ -67,6 +69,7 @@ export async function installTransferBot({
     'bot.replicaCount': replicas,
     'bot.txIntervalSeconds': txIntervalSeconds,
     'bot.followChain': followChain,
+    'bot.pxeSyncChainTip': pxeSyncChainTip,
     'bot.botPrivateKey': botPrivateKey,
     'bot.nodeUrl': resolvedNodeUrl,
     'bot.mnemonic': mnemonic,

package/src/spartan/utils/config.ts

@@ -15,6 +15,8 @@ const testConfigSchema = z.object({
   AZTEC_EPOCH_DURATION: z.coerce.number().optional().default(32),
   AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5),
   AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET: z.coerce.number().optional().default(2),
+  FUNDING_PRIVATE_KEY: z.string().optional(),
+  AZTEC_ADMIN_API_KEY: z.string().optional(),
 });
 
 export type TestConfig = z.infer<typeof testConfigSchema>;

package/src/spartan/utils/index.ts

@@ -5,7 +5,7 @@
 export { type TestConfig, setupEnvironment } from './config.js';
 
 // Scripts
-export { getGitProjectRoot, getAztecBin, runAztecBin, runProjectScript } from './scripts.js';
+export { type ScriptResult, getGitProjectRoot, getAztecBin, runAztecBin, runProjectScript } from './scripts.js';
 
 // K8s operations
 export {
@@ -24,6 +24,8 @@ export {
   getServiceEndpoint,
   getRPCEndpoint,
   getEthereumEndpoint,
+  createResilientPrometheusConnection,
+  scaleProverAgents,
 } from './k8s.js';
 
 // Chaos Mesh
@@ -39,12 +41,16 @@ export {
   applyNetworkShaping,
 } from './chaos.js';
 
+// Helm
+export { hasDeployedHelmRelease } from './helm.js';
+
 // Bot management
 export { restartBot, installTransferBot, uninstallTransferBot } from './bot.js';
 
 // Node operations (sequencers, validators, pods)
 export {
   awaitCheckpointNumber,
+  waitForProvenToAdvance,
   getSequencers,
   updateSequencersConfig,
   getSequencersConfig,

package/src/spartan/utils/k8s.ts

@@ -1,4 +1,5 @@
 import { createLogger } from '@aztec/aztec.js/log';
+import type { Logger } from '@aztec/foundation/log';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { retryUntil } from '@aztec/foundation/retry';
 
@@ -6,6 +7,8 @@ import { type ChildProcess, exec, spawn } from 'child_process';
 import path from 'path';
 import { promisify } from 'util';
 
+import { AlertTriggeredError, GrafanaClient } from '../../quality_of_service/grafana_client.js';
+
 const execAsync = promisify(exec);
 
 const logger = createLogger('e2e:k8s-utils');
@@ -370,6 +373,163 @@ export async function waitForResourcesByName({
   );
 }
 
+/**
+ * Waits for all StatefulSets matching a label to have all their replicas ready.
+ *
+ * @param namespace - Kubernetes namespace
+ * @param label - Label selector for StatefulSets (e.g., "app.kubernetes.io/component=sequencer-node")
+ * @param timeoutSeconds - Maximum time to wait in seconds
+ * @param pollIntervalSeconds - How often to check status
+ */
+export async function waitForStatefulSetsReady({
+  namespace,
+  label,
+  timeoutSeconds = 600,
+  pollIntervalSeconds = 5,
+}: {
+  namespace: string;
+  label: string;
+  timeoutSeconds?: number;
+  pollIntervalSeconds?: number;
+}): Promise<void> {
+  logger.info(`Waiting for StatefulSets with label ${label} to have all replicas ready (timeout: ${timeoutSeconds}s)`);
+
+  await retryUntil(
+    async () => {
+      // Get all StatefulSets matching the label
+      const getCmd = `kubectl get statefulset -l ${label} -n ${namespace} -o json`;
+      const { stdout } = await execAsync(getCmd);
+      const result = JSON.parse(stdout);
+
+      if (!result.items || result.items.length === 0) {
+        logger.verbose(`No StatefulSets found with label ${label}`);
+        return false;
+      }
+
+      // Check each StatefulSet
+      for (const sts of result.items) {
+        const name = sts.metadata.name;
+        const desired = sts.spec.replicas ?? 0;
+        const ready = sts.status.readyReplicas ?? 0;
+        const updated = sts.status.updatedReplicas ?? 0;
+
+        if (ready < desired || updated < desired) {
+          logger.verbose(`StatefulSet ${name}: ${ready}/${desired} ready, ${updated}/${desired} updated`);
+          return false;
+        }
+      }
+
+      logger.info(`All StatefulSets with label ${label} are ready`);
+      return true;
+    },
+    `StatefulSets with label ${label} to be ready`,
+    timeoutSeconds,
+    pollIntervalSeconds,
+  );
+}
+
+/**
+ * Creates a Prometheus connection that can re-establish port-forward on failure.
+ * Returns functions to connect and run alert checks that automatically reconnect if needed.
+ *
+ * @param namespace - K8s namespace to fall back to if metrics namespace doesn't have Prometheus
+ * @param endpoints - Array to track created endpoints for cleanup
+ * @param log - Logger instance
+ */
+export function createResilientPrometheusConnection(
+  namespace: string,
+  endpoints: ServiceEndpoint[],
+  log: Logger,
+): {
+  connect: () => Promise<GrafanaClient>;
+  runAlertCheck: (alerts: Parameters<GrafanaClient['runAlertCheck']>[0]) => Promise<void>;
+} {
+  let alertChecker: GrafanaClient | undefined;
+  let currentEndpoint: ServiceEndpoint | undefined;
+
+  const connect = async (): Promise<GrafanaClient> => {
+    // Kill existing connection if any
+    if (currentEndpoint?.process) {
+      currentEndpoint.process.kill();
+    }
+
+    // Try metrics namespace first, then network namespace
+    let promPort = 0;
+    let promUrl = '';
+    let promProc: ChildProcess | undefined;
+
+    try {
+      const metricsResult = await startPortForward({
+        resource: `svc/metrics-prometheus-server`,
+        namespace: 'metrics',
+        containerPort: 80,
+      });
+      promProc = metricsResult.process;
+      promPort = metricsResult.port;
+      promUrl = `http://127.0.0.1:${promPort}/api/v1`;
+    } catch {
+      // Metrics namespace might not have Prometheus, try network namespace
+      log.verbose('Metrics namespace Prometheus not available, trying network namespace');
+    }
+
+    if (promPort === 0) {
+      const nsResult = await startPortForward({
+        resource: `svc/prometheus-server`,
+        namespace,
+        containerPort: 80,
+      });
+      promProc = nsResult.process;
+      promPort = nsResult.port;
+      promUrl = `http://127.0.0.1:${promPort}/api/v1`;
+    }
+
+    if (!promProc || promPort === 0) {
+      throw new Error('Unable to port-forward to Prometheus');
+    }
+
+    currentEndpoint = { url: promUrl, process: promProc };
+    endpoints.push(currentEndpoint);
+    alertChecker = new GrafanaClient(log, { grafanaEndpoint: promUrl, grafanaCredentials: '' });
+    log.info(`Established Prometheus connection at ${promUrl}`);
+    return alertChecker;
+  };
+
+  const runAlertCheck = async (alerts: Parameters<GrafanaClient['runAlertCheck']>[0]): Promise<void> => {
+    if (!alertChecker) {
+      alertChecker = await connect();
+    }
+
+    try {
+      await alertChecker.runAlertCheck(alerts);
+    } catch (err) {
+      // If it's an AlertTriggeredError (expected behavior)
+      if (err instanceof AlertTriggeredError) {
+        throw err;
+      }
+
+      // Check if it's a connection error (port-forward died)
+      const errorStr = String(err);
+      if (errorStr.includes('fetch failed') || errorStr.includes('ECONNREFUSED') || errorStr.includes('ECONNRESET')) {
+        log.warn(`Prometheus connection lost, re-establishing port-forward...`);
+        alertChecker = await connect();
+        await alertChecker.runAlertCheck(alerts);
+      } else {
+        throw err;
+      }
+    }
+  };
+
+  return { connect, runAlertCheck };
+}
+
+/** Scales the prover-agent Deployment to the given number of replicas. */
+export async function scaleProverAgents(namespace: string, replicas: number, log: Logger): Promise<void> {
+  const label = 'app.kubernetes.io/component=prover-agent';
+  const command = `kubectl scale deployment -l ${label} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+  log.info(`Scaling prover agents to ${replicas}: ${command}`);
+  await execAsync(command);
+}
+
 export function getChartDir(spartanDir: string, chartName: string) {
   return path.join(spartanDir.trim(), chartName);
 }