@aztec/end-to-end 0.0.1-commit.c7c42ec → 0.0.1-commit.f295ac2

package/src/quality_of_service/{alert_checker.ts → grafana_client.ts}

@@ -25,7 +25,7 @@ const DEFAULT_CONFIG: AlertCheckerConfig = {
   grafanaCredentials: 'admin:admin',
 };
 
-export class AlertChecker {
+export class GrafanaClient {
   private config: AlertCheckerConfig;
   private logger: Logger;
 

package/src/quality_of_service/prometheus_client.ts

@@ -0,0 +1,113 @@
+export type PromteheusClientOptions = {
+  server: URL;
+};
+
+export class PrometheusClient {
+  constructor(
+    private config: PromteheusClientOptions,
+    private httpClient: typeof fetch = fetch,
+  ) {}
+
+  public async querySingleValue(query: string, time = new Date()): Promise<number> {
+    const resp = await this.queryRaw(query, time);
+    if (resp.status === 'success') {
+      if (resp.data.resultType === 'vector') {
+        if (resp.data.result.length === 0) {
+          return 0;
+        }
+        const [_, value] = resp.data.result[0].value;
+        return parseFloat(value);
+      }
+    }
+
+    throw new TypeError('Unsupported response body', { cause: JSON.stringify(resp) });
+  }
+
+  public queryRaw(query: string, time = new Date()): Promise<PrometheusResponse> {
+    const searchParams = new URLSearchParams();
+    searchParams.set('query', query);
+    searchParams.set('time', String(Math.trunc(time.getTime() / 1000)));
+    searchParams.set('limit', '10');
+
+    return this.callPrometheus('query', searchParams);
+  }
+
+  public queryRangeRaw(
+    query: string,
+    step: PrometheusDuration,
+    start: Date,
+    end = new Date(),
+  ): Promise<PrometheusResponse> {
+    const searchParams = new URLSearchParams();
+    searchParams.set('query', query);
+    searchParams.set('step', step);
+    searchParams.set('start', String(Math.trunc(start.getTime() / 1000)));
+    searchParams.set('end', String(Math.trunc(end.getTime() / 1000)));
+    searchParams.set('limit', '10');
+
+    return this.callPrometheus('query_range', searchParams);
+  }
+
+  private async callPrometheus(api: string, searchParams: URLSearchParams): Promise<PrometheusResponse> {
+    const url = new URL('api/v1/' + api, this.config.server);
+    for (const [name, value] of searchParams) {
+      url.searchParams.append(name, value);
+    }
+
+    const resp = await this.httpClient(url, { method: 'GET' });
+    if (!resp.ok || resp.status !== 200) {
+      throw new Error('Invalid HTTP response from Prometheus', {
+        cause: {
+          url,
+          status: resp.status,
+          statusText: resp.statusText,
+        },
+      });
+    }
+
+    const body = await resp.json();
+    if ('status' in body && (body.status === 'error' || body.status === 'success')) {
+      return body;
+    }
+
+    throw new Error('Invalid response from Prometheus', {
+      cause: {
+        url,
+        body,
+      },
+    });
+  }
+}
+
+export type PrometheusDuration = `${number}s` | `${number}m` | `${number}h`;
+
+export type PrometheusData =
+  | {
+      resultType: 'vector';
+      result: Array<{
+        metric: unknown;
+        value: [unixTimestamp: number, value: string];
+      }>;
+    }
+  | {
+      resultType: 'matrix';
+      result: Array<{
+        metric: unknown;
+        values: [unixTimestamp: number, value: string];
+      }>;
+    }
+  | {
+      resultType: 'scalar' | 'string';
+      result: unknown;
+    };
+
+export type PrometheusResponse =
+  | {
+      status: 'error';
+      errorType: string;
+      error: string;
+    }
+  | {
+      status: 'success';
+      data: PrometheusData;
+    };

package/src/shared/cross_chain_test_harness.ts

@@ -16,7 +16,7 @@ import type { Wallet } from '@aztec/aztec.js/wallet';
 import { deployL1Contract } from '@aztec/ethereum/deploy-l1-contract';
 import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses';
 import type { ExtendedViemWalletClient } from '@aztec/ethereum/types';
-import { BlockNumber } from '@aztec/foundation/branded-types';
+import { EpochNumber } from '@aztec/foundation/branded-types';
 import { retryUntil } from '@aztec/foundation/retry';
 import type { FieldsOf } from '@aztec/foundation/types';
 import { TestERC20Abi, TokenPortalAbi, TokenPortalBytecode } from '@aztec/l1-artifacts';
@@ -324,17 +324,11 @@ export class CrossChainTestHarness {
 
   withdrawFundsFromBridgeOnL1(
     amount: bigint,
-    blockNumber: BlockNumber,
+    epochNumber: EpochNumber,
     messageIndex: bigint,
     siblingPath: SiblingPath<number>,
   ) {
-    return this.l1TokenPortalManager.withdrawFunds(
-      amount,
-      this.ethAccount,
-      BigInt(blockNumber),
-      messageIndex,
-      siblingPath,
-    );
+    return this.l1TokenPortalManager.withdrawFunds(amount, this.ethAccount, epochNumber, messageIndex, siblingPath);
   }
 
   async transferToPrivateOnL2(shieldAmount: bigint) {

package/src/shared/index.ts

@@ -1 +1 @@
-export { uniswapL1L2TestSuite, type UniswapSetupContext } from './uniswap_l1_l2.js';
+export { uniswapL1L2TestSuite } from './uniswap_l1_l2.js';

package/src/shared/uniswap_l1_l2.ts

@@ -1,5 +1,6 @@
 import { AztecAddress, EthAddress } from '@aztec/aztec.js/addresses';
 import { computeAuthWitMessageHash } from '@aztec/aztec.js/authorization';
+import { waitForProven } from '@aztec/aztec.js/contracts';
 import { generateClaimSecret } from '@aztec/aztec.js/ethereum';
 import { Fr } from '@aztec/aztec.js/fields';
 import type { Logger } from '@aztec/aztec.js/log';
@@ -10,6 +11,7 @@ import type { DeployAztecL1ContractsReturnType } from '@aztec/ethereum/deploy-az
 import { deployL1Contract } from '@aztec/ethereum/deploy-l1-contract';
 import type { ExtendedViemWalletClient } from '@aztec/ethereum/types';
 import { extractEvent } from '@aztec/ethereum/utils';
+import { CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
 import { sha256ToField } from '@aztec/foundation/crypto/sha256';
 import { InboxAbi, UniswapPortalAbi, UniswapPortalBytecode } from '@aztec/l1-artifacts';
 import { UniswapContract } from '@aztec/noir-contracts.js/Uniswap';
@@ -20,7 +22,7 @@ import type { TestWallet } from '@aztec/test-wallet/server';
 import { jest } from '@jest/globals';
 import { type GetContractReturnType, getContract, parseEther, toFunctionSelector } from 'viem';
 
-import { ensureAccountContractsPublished } from '../fixtures/utils.js';
+import { type EndToEndContext, ensureAccountContractsPublished } from '../fixtures/utils.js';
 import { CrossChainTestHarness } from './cross_chain_test_harness.js';
 
 // PSA: This tests works on forked mainnet. There is a dump of the data in `dumpedState` such that we
@@ -32,28 +34,8 @@ import { CrossChainTestHarness } from './cross_chain_test_harness.js';
 
 const TIMEOUT = 360_000;
 
-/** Objects to be returned by the uniswap setup function */
-export type UniswapSetupContext = {
-  /** Aztec Node instance */
-  aztecNode: AztecNode;
-  /** Logger instance named as the current test. */
-  logger: Logger;
-  /** The L1 wallet client, extended with public actions. */
-  l1Client: ExtendedViemWalletClient;
-  /** The wallet. */
-  wallet: TestWallet;
-  /** The owner address. */
-  ownerAddress: AztecAddress;
-  /** The sponsor wallet. */
-  sponsorAddress: AztecAddress;
-  /**  */
-  deployL1ContractsValues: DeployAztecL1ContractsReturnType;
-  /** Cheat codes instance. */
-  cheatCodes: CheatCodes;
-};
-
 export const uniswapL1L2TestSuite = (
-  setup: () => Promise<UniswapSetupContext>,
+  setup: () => Promise<EndToEndContext>,
   cleanup: () => Promise<void>,
   expectedForkBlockNumber = 17514288,
 ) => {
@@ -90,8 +72,19 @@ export const uniswapL1L2TestSuite = (
     let cheatCodes: CheatCodes;
     let version: number;
     beforeAll(async () => {
-      ({ aztecNode, logger, l1Client, wallet, ownerAddress, sponsorAddress, deployL1ContractsValues, cheatCodes } =
-        await setup());
+      const t = await setup();
+      ({
+        aztecNode,
+        logger,
+        deployL1ContractsValues,
+        cheatCodes,
+        wallet,
+        accounts: [ownerAddress, sponsorAddress],
+      } = t);
+
+      l1Client = deployL1ContractsValues.l1Client;
+
+      t.watcher?.setIsMarkingAsProven(false);
 
       if (Number(await l1Client.getBlockNumber()) < expectedForkBlockNumber) {
         throw new Error('This test must be run on a fork of mainnet with the expected fork block');
@@ -259,24 +252,19 @@ export const uniswapL1L2TestSuite = (
       // ensure that uniswap contract didn't eat the funds.
       await wethCrossChainHarness.expectPublicBalanceOnL2(uniswapL2Contract.address, 0n);
 
-      // Since the outbox is only consumable when the block is proven, we need to set the block to be proven
-      await cheatCodes.rollup.markAsProven(await rollup.getCheckpointNumber());
+      // Since the outbox is only consumable when the epoch is proven, we need to advance to the next epoch.
+      const checkpointNumber = CheckpointNumber.fromBlockNumber(l2UniswapInteractionReceipt.blockNumber!);
+      const epoch = await rollup.getEpochNumberForCheckpoint(checkpointNumber);
+      await cheatCodes.rollup.advanceToEpoch(EpochNumber(epoch + 1));
+      await waitForProven(aztecNode, l2UniswapInteractionReceipt, { provenTimeout: 300 });
 
       // 5. Consume L2 to L1 message by calling uniswapPortal.swap_private()
       logger.info('Execute withdraw and swap on the uniswapPortal!');
       const daiL1BalanceOfPortalBeforeSwap = await daiCrossChainHarness.getL1BalanceOf(
         daiCrossChainHarness.tokenPortalAddress,
       );
-      const swapResult = await computeL2ToL1MembershipWitness(
-        aztecNode,
-        l2UniswapInteractionReceipt.blockNumber!,
-        swapPrivateLeaf,
-      );
-      const withdrawResult = await computeL2ToL1MembershipWitness(
-        aztecNode,
-        l2UniswapInteractionReceipt.blockNumber!,
-        withdrawLeaf,
-      );
+      const swapResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, swapPrivateLeaf);
+      const withdrawResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, withdrawLeaf);
 
       const swapPrivateL2MessageIndex = swapResult!.leafIndex;
       const swapPrivateSiblingPath = swapResult!.siblingPath;
@@ -285,7 +273,7 @@ export const uniswapL1L2TestSuite = (
       const withdrawSiblingPath = withdrawResult!.siblingPath;
 
       const withdrawMessageMetadata = {
-        _checkpointNumber: BigInt(l2UniswapInteractionReceipt.blockNumber!),
+        _epoch: BigInt(epoch),
         _leafIndex: BigInt(withdrawL2MessageIndex),
         _path: withdrawSiblingPath
           .toBufferArray()
@@ -293,7 +281,7 @@ export const uniswapL1L2TestSuite = (
       };
 
       const swapPrivateMessageMetadata = {
-        _checkpointNumber: BigInt(l2UniswapInteractionReceipt.blockNumber!),
+        _epoch: BigInt(epoch),
         _leafIndex: BigInt(swapPrivateL2MessageIndex),
         _path: swapPrivateSiblingPath
           .toBufferArray()
@@ -504,7 +492,7 @@ export const uniswapL1L2TestSuite = (
     //   );
 
     //   const withdrawMessageMetadata = {
-    //     _checkpointNumber: BigInt(uniswapL2Interaction.blockNumber!),
+    //     _epoch: epoch,
     //     _leafIndex: BigInt(withdrawL2MessageIndex),
     //     _path: withdrawSiblingPath
     //       .toBufferArray()
@@ -512,7 +500,7 @@ export const uniswapL1L2TestSuite = (
     //   };
 
     //   const swapPrivateMessageMetadata = {
-    //     _checkpointNumber: BigInt(uniswapL2Interaction.blockNumber!),
+    //     _epoch: epoch,
     //     _leafIndex: BigInt(swapPrivateL2MessageIndex),
     //     _path: swapPrivateSiblingPath
     //       .toBufferArray()
@@ -856,12 +844,11 @@ export const uniswapL1L2TestSuite = (
         chainId: new Fr(l1Client.chain.id),
       });
 
-      const swapResult = await computeL2ToL1MembershipWitness(aztecNode, withdrawReceipt.blockNumber!, swapPrivateLeaf);
-      const withdrawResult = await computeL2ToL1MembershipWitness(
-        aztecNode,
-        withdrawReceipt.blockNumber!,
-        withdrawLeaf,
+      const epoch = await rollup.getEpochNumberForCheckpoint(
+        CheckpointNumber.fromBlockNumber(withdrawReceipt.blockNumber!),
       );
+      const swapResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, swapPrivateLeaf);
+      const withdrawResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, withdrawLeaf);
 
       const swapPrivateL2MessageIndex = swapResult!.leafIndex;
       const swapPrivateSiblingPath = swapResult!.siblingPath;
@@ -870,7 +857,7 @@ export const uniswapL1L2TestSuite = (
       const withdrawSiblingPath = withdrawResult!.siblingPath;
 
       const withdrawMessageMetadata = {
-        _checkpointNumber: BigInt(withdrawReceipt.blockNumber!),
+        _epoch: BigInt(epoch),
         _leafIndex: BigInt(withdrawL2MessageIndex),
         _path: withdrawSiblingPath
           .toBufferArray()
@@ -878,7 +865,7 @@ export const uniswapL1L2TestSuite = (
       };
 
       const swapPrivateMessageMetadata = {
-        _checkpointNumber: BigInt(withdrawReceipt.blockNumber!),
+        _epoch: BigInt(epoch),
         _leafIndex: BigInt(swapPrivateL2MessageIndex),
         _path: swapPrivateSiblingPath
           .toBufferArray()
@@ -888,8 +875,9 @@ export const uniswapL1L2TestSuite = (
       // ensure that user's funds were burnt
       await wethCrossChainHarness.expectPrivateBalanceOnL2(ownerAddress, wethL2BalanceBeforeSwap - wethAmountToBridge);
 
-      // Since the outbox is only consumable when the block is proven, we need to set the block to be proven
-      await cheatCodes.rollup.markAsProven(await rollup.getCheckpointNumber());
+      // Since the outbox is only consumable when the epoch is proven, we need to advance to the next epoch.
+      await cheatCodes.rollup.advanceToEpoch(EpochNumber(epoch + 1));
+      await waitForProven(aztecNode, withdrawReceipt, { provenTimeout: 300 });
 
       // On L1 call swap_public!
       logger.info('call swap_public on L1');
@@ -991,12 +979,11 @@ export const uniswapL1L2TestSuite = (
         chainId: new Fr(l1Client.chain.id),
       });
 
-      const swapResult = await computeL2ToL1MembershipWitness(aztecNode, withdrawReceipt.blockNumber!, swapPublicLeaf);
-      const withdrawResult = await computeL2ToL1MembershipWitness(
-        aztecNode,
-        withdrawReceipt.blockNumber!,
-        withdrawLeaf,
+      const epoch = await rollup.getEpochNumberForCheckpoint(
+        CheckpointNumber.fromBlockNumber(withdrawReceipt.blockNumber!),
       );
+      const swapResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, swapPublicLeaf);
+      const withdrawResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, withdrawLeaf);
 
       const swapPublicL2MessageIndex = swapResult!.leafIndex;
       const swapPublicSiblingPath = swapResult!.siblingPath;
@@ -1005,7 +992,7 @@ export const uniswapL1L2TestSuite = (
       const withdrawSiblingPath = withdrawResult!.siblingPath;
 
       const withdrawMessageMetadata = {
-        _checkpointNumber: BigInt(withdrawReceipt.blockNumber!),
+        _epoch: BigInt(epoch),
         _leafIndex: BigInt(withdrawL2MessageIndex),
         _path: withdrawSiblingPath
           .toBufferArray()
@@ -1013,7 +1000,7 @@ export const uniswapL1L2TestSuite = (
       };
 
       const swapPublicMessageMetadata = {
-        _checkpointNumber: BigInt(withdrawReceipt.blockNumber!),
+        _epoch: BigInt(epoch),
         _leafIndex: BigInt(swapPublicL2MessageIndex),
         _path: swapPublicSiblingPath
           .toBufferArray()
@@ -1023,8 +1010,9 @@ export const uniswapL1L2TestSuite = (
       // check weth balance of owner on L2 (we first bridged `wethAmountToBridge` into L2 and now withdrew it!)
       await wethCrossChainHarness.expectPublicBalanceOnL2(ownerAddress, 0n);
 
-      // Since the outbox is only consumable when the block is proven, we need to set the block to be proven
-      await cheatCodes.rollup.markAsProven(await rollup.getCheckpointNumber());
+      // Since the outbox is only consumable when the epoch is proven, we need to advance to the next epoch.
+      await cheatCodes.rollup.advanceToEpoch(EpochNumber(epoch + 1));
+      await waitForProven(aztecNode, withdrawReceipt, { provenTimeout: 300 });
 
       // Call swap_private on L1
       logger.info('Execute withdraw and swap on the uniswapPortal!');

package/src/spartan/setup_test_wallets.ts

@@ -315,11 +315,17 @@ export async function performTransfers({
   }
 }
 
+export type WalletWrapper = {
+  wallet: TestWallet;
+  aztecNode: AztecNode;
+  cleanup: () => Promise<void>;
+};
+
 export async function createWalletAndAztecNodeClient(
   nodeUrl: string,
   proverEnabled: boolean,
   logger: Logger,
-): Promise<{ wallet: TestWallet; aztecNode: AztecNode; cleanup: () => Promise<void> }> {
+): Promise<WalletWrapper> {
   const aztecNode = createAztecNodeClient(nodeUrl);
   const [bbConfig, acvmConfig] = await Promise.all([getBBConfig(logger), getACVMConfig(logger)]);
   const pxeConfig = {

package/src/spartan/tx_metrics.ts

@@ -1,5 +1,6 @@
 import type { AztecNode } from '@aztec/aztec.js/node';
-import type { L2Block } from '@aztec/stdlib/block';
+import type { L2BlockNew } from '@aztec/stdlib/block';
+import type { TopicType } from '@aztec/stdlib/p2p';
 import { Tx, type TxReceipt, TxStatus } from '@aztec/stdlib/tx';
 
 import { createHistogram } from 'perf_hooks';
@@ -19,7 +20,9 @@ export type TxInclusionData = {
 export class TxInclusionMetrics {
   private data = new Map<string, TxInclusionData>();
   private groups = new Set<string>();
-  private blocks = new Map<number, Promise<L2Block>>();
+  private blocks = new Map<number, Promise<L2BlockNew | undefined>>();
+
+  private p2pGossipLatencyByTopic: Partial<Record<TopicType, { p50: number; p95: number }>> = {};
 
   constructor(private aztecNode: AztecNode) {}
 
@@ -48,10 +51,13 @@ export class TxInclusionMetrics {
     }
 
     if (!this.blocks.has(blockNumber)) {
-      this.blocks.set(blockNumber, this.aztecNode.getBlock(blockNumber) as Promise<L2Block>);
+      this.blocks.set(blockNumber, this.aztecNode.getBlock(blockNumber));
     }
 
     const block = await this.blocks.get(blockNumber)!;
+    if (!block) {
+      return;
+    }
     const data = this.data.get(txHash.toString())!;
     data.blocknumber = blockNumber;
     data.minedAt = Number(block.header.globalVariables.timestamp);
@@ -71,7 +77,7 @@ export class TxInclusionMetrics {
   } {
     const histogram = createHistogram({});
     for (const tx of this.data.values()) {
-      if (!tx.blocknumber || tx.group !== group) {
+      if (!tx.blocknumber || tx.group !== group || tx.minedAt === -1) {
         continue;
       }
 
@@ -101,6 +107,10 @@ export class TxInclusionMetrics {
     };
   }
 
+  public recordP2PGossipLatency(topicName: TopicType, p50: number, p95: number): void {
+    this.p2pGossipLatencyByTopic[topicName] = { p50, p95 };
+  }
+
   toGithubActionBenchmarkJSON(): Array<{ name: string; unit: string; value: number; range?: number; extra?: string }> {
     const data: Array<{ name: string; unit: string; value: number; range?: number; extra?: string }> = [];
     for (const group of this.groups) {
@@ -125,6 +135,19 @@ export class TxInclusionMetrics {
       );
     }
 
+    for (const [topic, { p50, p95 }] of Object.entries(this.p2pGossipLatencyByTopic)) {
+      data.push({
+        name: `p2p_gossip_latency/${topic}/p50`,
+        unit: 'ms',
+        value: p50,
+      });
+      data.push({
+        name: `p2p_gossip_latency/${topic}/p95`,
+        unit: 'ms',
+        value: p95,
+      });
+    }
+
     return data;
   }
 }

package/src/spartan/utils.ts

@@ -34,6 +34,7 @@ const testConfigSchema = z.object({
   AZTEC_SLOT_DURATION: z.coerce.number().optional().default(24),
   AZTEC_EPOCH_DURATION: z.coerce.number().optional().default(32),
   AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5),
+  AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET: z.coerce.number().optional().default(2),
 });
 
 export type TestConfig = z.infer<typeof testConfigSchema>;
@@ -117,7 +118,7 @@ export async function startPortForward({
   );
 
   let isResolved = false;
-  const connected = new Promise<number>(resolve => {
+  const connected = new Promise<number>((resolve, reject) => {
     process.stdout?.on('data', data => {
       const str = data.toString() as string;
       if (!isResolved && str.includes('Forwarding from')) {
@@ -125,7 +126,8 @@ export async function startPortForward({
         logger.debug(`Port forward for ${resource}: ${str}`);
         const port = str.search(/:\d+/);
         if (port === -1) {
-          throw new Error('Port not found in port forward output');
+          reject(new Error('Port not found in port forward output'));
+          return;
         }
         const portNumber = parseInt(str.slice(port + 1));
         logger.verbose(`Port forwarded for ${resource} at ${portNumber}:${containerPort}`);
@@ -145,17 +147,26 @@ export async function startPortForward({
     process.on('close', () => {
       if (!isResolved) {
         isResolved = true;
-        logger.warn(`Port forward for ${resource} closed before connection established`);
-        resolve(0);
+        const msg = `Port forward for ${resource} closed before connection established`;
+        logger.warn(msg);
+        reject(new Error(msg));
       }
     });
     process.on('error', error => {
-      logger.error(`Port forward for ${resource} error: ${error}`);
-      resolve(0);
+      if (!isResolved) {
+        isResolved = true;
+        const msg = `Port forward for ${resource} error: ${error}`;
+        logger.error(msg);
+        reject(new Error(msg));
+      }
     });
     process.on('exit', code => {
-      logger.verbose(`Port forward for ${resource} exited with code ${code}`);
-      resolve(0);
+      if (!isResolved) {
+        isResolved = true;
+        const msg = `Port forward for ${resource} exited with code ${code}`;
+        logger.verbose(msg);
+        reject(new Error(msg));
+      }
     });
   });
 
@@ -197,6 +208,14 @@ export function getExternalIP(namespace: string, serviceName: string): Promise<s
   return promise;
 }
 
+export function startPortForwardForPrometeheus(namespace: string) {
+  return startPortForward({
+    resource: `svc/${namespace}-prometheus-server`,
+    namespace,
+    containerPort: 80,
+  });
+}
+
 export function startPortForwardForRPC(namespace: string, index = 0) {
   return startPortForward({
     resource: `pod/${namespace}-rpc-aztec-node-${index}`,
@@ -1089,24 +1108,93 @@ export async function getL1DeploymentAddresses(env: TestConfig): Promise<L1Contr
 /**
  * Rolls the Aztec pods in the given namespace.
  * @param namespace - The namespace to roll the Aztec pods in.
- * @dev - IMPORTANT: This function DOES NOT delete the underlying PVCs.
- *        This means that the pods will be restarted with the same persistent storage.
- *        This is useful for testing, but you should be aware of the implications.
+ * @param clearState - If true, also deletes the underlying PVCs to clear persistent storage.
+ *        This is required for rollup upgrades where the old state is incompatible with the new rollup.
+ *        Defaults to false, which preserves the existing storage.
  */
-export async function rollAztecPods(namespace: string) {
-  await deleteResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=boot-node' });
-  await deleteResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=prover-node' });
-  await deleteResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=prover-broker' });
-  await deleteResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=prover-agent' });
-  await deleteResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=validator' });
-  await deleteResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=pxe' });
+export async function rollAztecPods(namespace: string, clearState: boolean = false) {
+  // Pod components use 'validator', but StatefulSets and PVCs use 'sequencer-node' for validators
+  const podComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'prover-agent', 'sequencer-node', 'rpc'];
+  const pvcComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'sequencer-node', 'rpc'];
+  // StatefulSet components that need to be scaled down before PVC deletion
+  // Note: validators use 'sequencer-node' as component label, not 'validator'
+  const statefulSetComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'sequencer-node', 'rpc'];
+
+  if (clearState) {
+    // To delete PVCs, we must first scale down StatefulSets so pods release the volumes
+    // Otherwise PVC deletion will hang waiting for pods to terminate
+
+    // First, save original replica counts
+    const originalReplicas: Map<string, number> = new Map();
+    for (const component of statefulSetComponents) {
+      try {
+        const getCmd = `kubectl get statefulset -l app.kubernetes.io/component=${component} -n ${namespace} -o jsonpath='{.items[0].spec.replicas}'`;
+        const { stdout } = await execAsync(getCmd);
+        const replicas = parseInt(stdout.replace(/'/g, '').trim(), 10);
+        if (!isNaN(replicas) && replicas > 0) {
+          originalReplicas.set(component, replicas);
+        }
+      } catch {
+        // Component might not exist, continue
+      }
+    }
+
+    // Scale down to 0
+    for (const component of statefulSetComponents) {
+      try {
+        const scaleCmd = `kubectl scale statefulset -l app.kubernetes.io/component=${component} -n ${namespace} --replicas=0 --timeout=2m`;
+        logger.info(`command: ${scaleCmd}`);
+        await execAsync(scaleCmd);
+      } catch (e) {
+        // Component might not exist or might be a Deployment, continue
+        logger.verbose(`Scale down ${component} skipped: ${e}`);
+      }
+    }
+
+    // Wait for pods to terminate
+    await sleep(15 * 1000);
+
+    // Now delete PVCs (they should no longer be in use)
+    for (const component of pvcComponents) {
+      await deleteResourceByLabel({
+        resource: 'persistentvolumeclaims',
+        namespace: namespace,
+        label: `app.kubernetes.io/component=${component}`,
+      });
+    }
+
+    // Scale StatefulSets back up to original replica counts
+    for (const component of statefulSetComponents) {
+      const replicas = originalReplicas.get(component) ?? 1;
+      try {
+        const scaleCmd = `kubectl scale statefulset -l app.kubernetes.io/component=${component} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+        logger.info(`command: ${scaleCmd}`);
+        await execAsync(scaleCmd);
+      } catch (e) {
+        logger.verbose(`Scale up ${component} skipped: ${e}`);
+      }
+    }
+  } else {
+    // Just delete pods (no state clearing)
+    for (const component of podComponents) {
+      await deleteResourceByLabel({
+        resource: 'pods',
+        namespace: namespace,
+        label: `app.kubernetes.io/component=${component}`,
+      });
+    }
+  }
+
   await sleep(10 * 1000);
-  await waitForResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=boot-node' });
-  await waitForResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=prover-node' });
-  await waitForResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=prover-broker' });
-  await waitForResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=prover-agent' });
-  await waitForResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=validator' });
-  await waitForResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=pxe' });
+
+  // Wait for pods to come back
+  for (const component of podComponents) {
+    await waitForResourceByLabel({
+      resource: 'pods',
+      namespace: namespace,
+      label: `app.kubernetes.io/component=${component}`,
+    });
+  }
 }
 
 /**