@aztec/end-to-end 0.0.1-commit.d3ec352c → 0.0.1-commit.f295ac2

package/src/spartan/utils.ts

@@ -1,6 +1,7 @@
 import { createLogger } from '@aztec/aztec.js/log';
 import type { RollupCheatCodes } from '@aztec/aztec/testing';
-import type { L1ContractAddresses, ViemPublicClient } from '@aztec/ethereum';
+import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses';
+import type { ViemPublicClient } from '@aztec/ethereum/types';
 import type { CheckpointNumber } from '@aztec/foundation/branded-types';
 import type { Logger } from '@aztec/foundation/log';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
@@ -31,7 +32,9 @@ const testConfigSchema = z.object({
   L1_RPC_URLS_JSON: z.string().optional(),
   L1_ACCOUNT_MNEMONIC: z.string().optional(),
   AZTEC_SLOT_DURATION: z.coerce.number().optional().default(24),
+  AZTEC_EPOCH_DURATION: z.coerce.number().optional().default(32),
   AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5),
+  AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET: z.coerce.number().optional().default(2),
 });
 
 export type TestConfig = z.infer<typeof testConfigSchema>;
@@ -115,7 +118,7 @@ export async function startPortForward({
   );
 
   let isResolved = false;
-  const connected = new Promise<number>(resolve => {
+  const connected = new Promise<number>((resolve, reject) => {
     process.stdout?.on('data', data => {
       const str = data.toString() as string;
       if (!isResolved && str.includes('Forwarding from')) {
@@ -123,7 +126,8 @@ export async function startPortForward({
         logger.debug(`Port forward for ${resource}: ${str}`);
         const port = str.search(/:\d+/);
         if (port === -1) {
-          throw new Error('Port not found in port forward output');
+          reject(new Error('Port not found in port forward output'));
+          return;
         }
         const portNumber = parseInt(str.slice(port + 1));
         logger.verbose(`Port forwarded for ${resource} at ${portNumber}:${containerPort}`);
@@ -143,17 +147,26 @@ export async function startPortForward({
     process.on('close', () => {
       if (!isResolved) {
         isResolved = true;
-        logger.warn(`Port forward for ${resource} closed before connection established`);
-        resolve(0);
+        const msg = `Port forward for ${resource} closed before connection established`;
+        logger.warn(msg);
+        reject(new Error(msg));
       }
     });
     process.on('error', error => {
-      logger.error(`Port forward for ${resource} error: ${error}`);
-      resolve(0);
+      if (!isResolved) {
+        isResolved = true;
+        const msg = `Port forward for ${resource} error: ${error}`;
+        logger.error(msg);
+        reject(new Error(msg));
+      }
     });
     process.on('exit', code => {
-      logger.verbose(`Port forward for ${resource} exited with code ${code}`);
-      resolve(0);
+      if (!isResolved) {
+        isResolved = true;
+        const msg = `Port forward for ${resource} exited with code ${code}`;
+        logger.verbose(msg);
+        reject(new Error(msg));
+      }
     });
   });
 
@@ -195,6 +208,14 @@ export function getExternalIP(namespace: string, serviceName: string): Promise<s
   return promise;
 }
 
+export function startPortForwardForPrometeheus(namespace: string) {
+  return startPortForward({
+    resource: `svc/${namespace}-prometheus-server`,
+    namespace,
+    containerPort: 80,
+  });
+}
+
 export function startPortForwardForRPC(namespace: string, index = 0) {
   return startPortForward({
     resource: `pod/${namespace}-rpc-aztec-node-${index}`,
@@ -243,11 +264,11 @@ export async function deleteResourceByLabel({
   timeout?: string;
   force?: boolean;
 }) {
-  // Check if the resource type exists before attempting to delete
   try {
-    await execAsync(
-      `kubectl api-resources --api-group="" --no-headers -o name | grep -q "^${resource}$" || kubectl api-resources --no-headers -o name | grep -q "^${resource}$"`,
-    );
+    // Match both plain and group-qualified names (e.g., "podchaos" or "podchaos.chaos-mesh.org")
+    const escaped = resource.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
+    const regex = `(^|\\.)${escaped}(\\.|$)`;
+    await execAsync(`kubectl api-resources --no-headers -o name | grep -Eq '${regex}'`);
   } catch (error) {
     logger.warn(`Resource type '${resource}' not found in cluster, skipping deletion ${error}`);
     return '';
@@ -280,6 +301,58 @@ export async function waitForResourceByLabel({
   return stdout;
 }
 
+export async function waitForResourceByName({
+  resource,
+  name,
+  namespace,
+  condition = 'Ready',
+  timeout = '10m',
+}: {
+  resource: string;
+  name: string;
+  namespace: string;
+  condition?: string;
+  timeout?: string;
+}) {
+  const command = `kubectl wait ${resource}/${name} --for=condition=${condition} -n ${namespace} --timeout=${timeout}`;
+  logger.info(`command: ${command}`);
+  const { stdout } = await execAsync(command);
+  return stdout;
+}
+
+export async function waitForResourcesByName({
+  resource,
+  names,
+  namespace,
+  condition = 'Ready',
+  timeout = '10m',
+}: {
+  resource: string;
+  names: string[];
+  namespace: string;
+  condition?: string;
+  timeout?: string;
+}) {
+  if (!names.length) {
+    throw new Error(`No ${resource} names provided to waitForResourcesByName`);
+  }
+
+  // Wait all in parallel; if any fails, surface which one.
+  await Promise.all(
+    names.map(async name => {
+      try {
+        await waitForResourceByName({ resource, name, namespace, condition, timeout });
+      } catch (err) {
+        throw new Error(
+          `Failed waiting for ${resource}/${name} condition=${condition} timeout=${timeout} namespace=${namespace}: ${String(
+            err,
+          )}`,
+        );
+      }
+    }),
+  );
+}
+
 export function getChartDir(spartanDir: string, chartName: string) {
   return path.join(spartanDir.trim(), chartName);
 }
@@ -330,7 +403,36 @@ async function execHelmCommand(args: Parameters<typeof createHelmCommand>[0]) {
   return stdout;
 }
 
-export async function cleanHelm(instanceName: string, namespace: string, logger: Logger) {
+async function getHelmReleaseStatus(instanceName: string, namespace: string): Promise<string | undefined> {
+  try {
+    const { stdout } = await execAsync(
+      `helm list --namespace ${namespace} --all --filter '^${instanceName}$' --output json | cat`,
+    );
+    const parsed = JSON.parse(stdout) as Array<{ name?: string; status?: string }>;
+    const row = parsed.find(r => r.name === instanceName);
+    return row?.status;
+  } catch {
+    return undefined;
+  }
+}
+
+async function forceDeleteHelmReleaseRecord(instanceName: string, namespace: string, logger: Logger) {
+  const labelSelector = `owner=helm,name=${instanceName}`;
+  const cmd = `kubectl delete secret -n ${namespace} -l ${labelSelector} --ignore-not-found=true`;
+  logger.warn(`Force deleting Helm release record: ${cmd}`);
+  await execAsync(cmd).catch(() => undefined);
+}
+
+async function hasDeployedHelmRelease(instanceName: string, namespace: string): Promise<boolean> {
+  try {
+    const status = await getHelmReleaseStatus(instanceName, namespace);
+    return status?.toLowerCase() === 'deployed';
+  } catch {
+    return false;
+  }
+}
+
+export async function uninstallChaosMesh(instanceName: string, namespace: string, logger: Logger) {
   // uninstall the helm chart if it exists
   logger.info(`Uninstalling helm chart ${instanceName}`);
   await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`);
@@ -394,7 +496,7 @@ export async function installChaosMeshChart({
   logger: Logger;
 }) {
   if (clean) {
-    await cleanHelm(instanceName, targetNamespace, logger);
+    await uninstallChaosMesh(instanceName, targetNamespace, logger);
   }
 
   return execHelmCommand({
@@ -430,22 +532,49 @@ export function applyProverFailure({
   });
 }
 
+export function applyValidatorFailure({
+  namespace,
+  spartanDir,
+  logger,
+  values,
+  instanceName,
+}: {
+  namespace: string;
+  spartanDir: string;
+  logger: Logger;
+  values?: Record<string, string | number>;
+  instanceName?: string;
+}) {
+  return installChaosMeshChart({
+    instanceName: instanceName ?? 'validator-failure',
+    targetNamespace: namespace,
+    valuesFile: 'validator-failure.yaml',
+    helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+    values,
+    logger,
+  });
+}
+
 export function applyProverKill({
   namespace,
   spartanDir,
   logger,
+  values,
 }: {
   namespace: string;
   spartanDir: string;
   logger: Logger;
+  values?: Record<string, string | number>;
 }) {
   return installChaosMeshChart({
     instanceName: 'prover-kill',
     targetNamespace: namespace,
     valuesFile: 'prover-kill.yaml',
     helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+    chaosMeshNamespace: namespace,
     clean: true,
     logger,
+    values,
   });
 }
 
@@ -453,10 +582,12 @@ export function applyProverBrokerKill({
   namespace,
   spartanDir,
   logger,
+  values,
 }: {
   namespace: string;
   spartanDir: string;
   logger: Logger;
+  values?: Record<string, string | number>;
 }) {
   return installChaosMeshChart({
     instanceName: 'prover-broker-kill',
@@ -465,66 +596,79 @@ export function applyProverBrokerKill({
     helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
     clean: true,
     logger,
+    values,
   });
 }
 
 export function applyBootNodeFailure({
+  instanceName = 'boot-node-failure',
   namespace,
   spartanDir,
   durationSeconds,
   logger,
+  values,
 }: {
+  instanceName?: string;
   namespace: string;
   spartanDir: string;
   durationSeconds: number;
   logger: Logger;
+  values?: Record<string, string | number>;
 }) {
   return installChaosMeshChart({
-    instanceName: 'boot-node-failure',
+    instanceName,
     targetNamespace: namespace,
     valuesFile: 'boot-node-failure.yaml',
     helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
     values: {
       'bootNodeFailure.duration': `${durationSeconds}s`,
+      ...(values ?? {}),
     },
     logger,
   });
 }
 
 export function applyValidatorKill({
+  instanceName = 'validator-kill',
   namespace,
   spartanDir,
   logger,
   values,
+  clean = true,
 }: {
+  instanceName?: string;
   namespace: string;
   spartanDir: string;
   logger: Logger;
   values?: Record<string, string | number>;
+  clean?: boolean;
 }) {
   return installChaosMeshChart({
-    instanceName: 'validator-kill',
+    instanceName: instanceName ?? 'validator-kill',
     targetNamespace: namespace,
     valuesFile: 'validator-kill.yaml',
     helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+    clean,
     logger,
     values,
   });
 }
 
 export function applyNetworkShaping({
+  instanceName = 'network-shaping',
   valuesFile,
   namespace,
   spartanDir,
   logger,
 }: {
+  instanceName?: string;
   valuesFile: string;
   namespace: string;
   spartanDir: string;
   logger: Logger;
 }) {
   return installChaosMeshChart({
-    instanceName: 'network-shaping',
+    instanceName,
     targetNamespace: namespace,
     valuesFile,
     helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
@@ -623,6 +767,12 @@ export async function installTransferBot({
     'bot.node.env.ETHEREUM_HOSTS': `http://${namespace}-eth-execution.${namespace}.svc.cluster.local:8545`,
     // Provide L1 mnemonic for bridging (falls back to labs mnemonic)
     'bot.node.env.BOT_L1_MNEMONIC': mnemonic,
+
+    // The bot does not need Kubernetes API access. Disable RBAC + ServiceAccount creation so the chart
+    // can be installed by users without cluster-scoped RBAC permissions.
+    'bot.rbac.create': false,
+    'bot.serviceAccount.create': false,
+    'bot.serviceAccount.name': 'default',
   };
   // Ensure we derive a funded L1 key (index 0 is funded on anvil default mnemonic)
   if (mnemonicStartIndex === undefined) {
@@ -647,7 +797,7 @@ export async function installTransferBot({
   if (!repository || !tag) {
     try {
       const { stdout } = await execAsync(
-        `kubectl get pods -l app.kubernetes.io/component=validator -n ${namespace} -o jsonpath='{.items[0].spec.containers[?(@.name=="aztec")].image}' | cat`,
+        `kubectl get pods -l app.kubernetes.io/name=validator -n ${namespace} -o jsonpath='{.items[0].spec.containers[?(@.name=="aztec")].image}' | cat`,
       );
       const image = stdout.trim().replace(/^'|'$/g, '');
       if (image && image.includes(':')) {
@@ -668,6 +818,26 @@ export async function installTransferBot({
       typeof mnemonicStartIndex === 'string' ? mnemonicStartIndex : Number(mnemonicStartIndex);
   }
 
+  // If a previous install attempt left the release in a non-deployed state (e.g. FAILED),
+  // `helm upgrade --install` can error with "has no deployed releases".
+  // In that case, clear the release record and do a clean install.
+  const existingStatus = await getHelmReleaseStatus(instanceName, namespace);
+  if (existingStatus && existingStatus.toLowerCase() !== 'deployed') {
+    logger.warn(`Transfer bot release ${instanceName} is in status '${existingStatus}'. Reinstalling cleanly.`);
+    await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`).catch(
+      () => undefined,
+    );
+    // If helm left the release in `uninstalling`, force-delete the record so we can reinstall.
+    const afterUninstallStatus = await getHelmReleaseStatus(instanceName, namespace);
+    if (afterUninstallStatus?.toLowerCase() === 'uninstalling') {
+      await forceDeleteHelmReleaseRecord(instanceName, namespace, logger);
+    }
+  }
+
+  // `--reuse-values` fails if the release has never successfully deployed (e.g. first install, or a previous failed install).
+  // Only reuse values when we have a deployed release to reuse from.
+  const effectiveReuseValues = reuseValues && (await hasDeployedHelmRelease(instanceName, namespace));
+
   await execHelmCommand({
     instanceName,
     helmChartDir,
@@ -675,7 +845,7 @@ export async function installTransferBot({
     valuesFile: undefined,
     timeout,
     values: values as unknown as Record<string, string | number | boolean>,
-    reuseValues,
+    reuseValues: effectiveReuseValues,
   });
 
   if (replicas > 0) {
@@ -720,7 +890,7 @@ export async function setValidatorTxDrop({
   const drop = enabled ? 'true' : 'false';
   const prob = String(probability);
 
-  const selectors = ['app=validator', 'app.kubernetes.io/component=validator'];
+  const selectors = ['app.kubernetes.io/name=validator', 'app.kubernetes.io/component=validator', 'app=validator'];
   let updated = false;
   for (const selector of selectors) {
     try {
@@ -751,7 +921,7 @@ export async function setValidatorTxDrop({
 }
 
 export async function restartValidators(namespace: string, logger: Logger) {
-  const selectors = ['app=validator', 'app.kubernetes.io/component=validator'];
+  const selectors = ['app.kubernetes.io/name=validator', 'app.kubernetes.io/component=validator', 'app=validator'];
   let any = false;
   for (const selector of selectors) {
     try {
@@ -806,11 +976,33 @@ export async function enableValidatorDynamicBootNode(
 }
 
 export async function getSequencers(namespace: string) {
-  const command = `kubectl get pods -l app.kubernetes.io/component=validator -n ${namespace} -o jsonpath='{.items[*].metadata.name}'`;
-  const { stdout } = await execAsync(command);
-  const sequencers = stdout.split(' ');
-  logger.verbose(`Found sequencer pods ${sequencers.join(', ')}`);
-  return sequencers;
+  const selectors = [
+    'app.kubernetes.io/name=validator',
+    'app.kubernetes.io/component=validator',
+    'app.kubernetes.io/component=sequencer-node',
+    'app=validator',
+  ];
+  for (const selector of selectors) {
+    try {
+      const command = `kubectl get pods -l ${selector} -n ${namespace} -o jsonpath='{.items[*].metadata.name}'`;
+      const { stdout } = await execAsync(command);
+      const sequencers = stdout
+        .split(' ')
+        .map(s => s.trim())
+        .filter(Boolean);
+      if (sequencers.length > 0) {
+        logger.verbose(`Found sequencer pods ${sequencers.join(', ')} (selector=${selector})`);
+        return sequencers;
+      }
+    } catch {
+      // try next selector
+    }
+  }
+
+  // Fail fast instead of returning [''] which leads to attempts to port-forward `pod/`.
+  throw new Error(
+    `No sequencer/validator pods found in namespace ${namespace}. Tried selectors: ${selectors.join(', ')}`,
+  );
 }
 
 export function updateSequencersConfig(env: TestConfig, config: Partial<AztecNodeAdminConfig>) {
@@ -871,7 +1063,7 @@ export async function getPublicViemClient(
       containerPort: 8545,
     });
     const url = `http://127.0.0.1:${port}`;
-    const client: ViemPublicClient = createPublicClient({ transport: fallback([http(url)]) });
+    const client: ViemPublicClient = createPublicClient({ transport: fallback([http(url, { batch: false })]) });
     if (processes) {
       processes.push(process);
     }
@@ -881,7 +1073,9 @@ export async function getPublicViemClient(
     if (!L1_RPC_URLS_JSON) {
       throw new Error(`L1_RPC_URLS_JSON is not defined`);
     }
-    const client: ViemPublicClient = createPublicClient({ transport: fallback([http(L1_RPC_URLS_JSON)]) });
+    const client: ViemPublicClient = createPublicClient({
+      transport: fallback([http(L1_RPC_URLS_JSON, { batch: false })]),
+    });
     return { url: L1_RPC_URLS_JSON, client };
   }
 }
@@ -914,24 +1108,93 @@ export async function getL1DeploymentAddresses(env: TestConfig): Promise<L1Contr
 /**
  * Rolls the Aztec pods in the given namespace.
  * @param namespace - The namespace to roll the Aztec pods in.
- * @dev - IMPORTANT: This function DOES NOT delete the underlying PVCs.
- *        This means that the pods will be restarted with the same persistent storage.
- *        This is useful for testing, but you should be aware of the implications.
+ * @param clearState - If true, also deletes the underlying PVCs to clear persistent storage.
+ *        This is required for rollup upgrades where the old state is incompatible with the new rollup.
+ *        Defaults to false, which preserves the existing storage.
  */
-export async function rollAztecPods(namespace: string) {
-  await deleteResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=boot-node' });
-  await deleteResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=prover-node' });
-  await deleteResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=prover-broker' });
-  await deleteResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=prover-agent' });
-  await deleteResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=validator' });
-  await deleteResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=pxe' });
+export async function rollAztecPods(namespace: string, clearState: boolean = false) {
+  // Pod components use 'validator', but StatefulSets and PVCs use 'sequencer-node' for validators
+  const podComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'prover-agent', 'sequencer-node', 'rpc'];
+  const pvcComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'sequencer-node', 'rpc'];
+  // StatefulSet components that need to be scaled down before PVC deletion
+  // Note: validators use 'sequencer-node' as component label, not 'validator'
+  const statefulSetComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'sequencer-node', 'rpc'];
+
+  if (clearState) {
+    // To delete PVCs, we must first scale down StatefulSets so pods release the volumes
+    // Otherwise PVC deletion will hang waiting for pods to terminate
+
+    // First, save original replica counts
+    const originalReplicas: Map<string, number> = new Map();
+    for (const component of statefulSetComponents) {
+      try {
+        const getCmd = `kubectl get statefulset -l app.kubernetes.io/component=${component} -n ${namespace} -o jsonpath='{.items[0].spec.replicas}'`;
+        const { stdout } = await execAsync(getCmd);
+        const replicas = parseInt(stdout.replace(/'/g, '').trim(), 10);
+        if (!isNaN(replicas) && replicas > 0) {
+          originalReplicas.set(component, replicas);
+        }
+      } catch {
+        // Component might not exist, continue
+      }
+    }
+
+    // Scale down to 0
+    for (const component of statefulSetComponents) {
+      try {
+        const scaleCmd = `kubectl scale statefulset -l app.kubernetes.io/component=${component} -n ${namespace} --replicas=0 --timeout=2m`;
+        logger.info(`command: ${scaleCmd}`);
+        await execAsync(scaleCmd);
+      } catch (e) {
+        // Component might not exist or might be a Deployment, continue
+        logger.verbose(`Scale down ${component} skipped: ${e}`);
+      }
+    }
+
+    // Wait for pods to terminate
+    await sleep(15 * 1000);
+
+    // Now delete PVCs (they should no longer be in use)
+    for (const component of pvcComponents) {
+      await deleteResourceByLabel({
+        resource: 'persistentvolumeclaims',
+        namespace: namespace,
+        label: `app.kubernetes.io/component=${component}`,
+      });
+    }
+
+    // Scale StatefulSets back up to original replica counts
+    for (const component of statefulSetComponents) {
+      const replicas = originalReplicas.get(component) ?? 1;
+      try {
+        const scaleCmd = `kubectl scale statefulset -l app.kubernetes.io/component=${component} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+        logger.info(`command: ${scaleCmd}`);
+        await execAsync(scaleCmd);
+      } catch (e) {
+        logger.verbose(`Scale up ${component} skipped: ${e}`);
+      }
+    }
+  } else {
+    // Just delete pods (no state clearing)
+    for (const component of podComponents) {
+      await deleteResourceByLabel({
+        resource: 'pods',
+        namespace: namespace,
+        label: `app.kubernetes.io/component=${component}`,
+      });
+    }
+  }
+
   await sleep(10 * 1000);
-  await waitForResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=boot-node' });
-  await waitForResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=prover-node' });
-  await waitForResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=prover-broker' });
-  await waitForResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=prover-agent' });
-  await waitForResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=validator' });
-  await waitForResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=pxe' });
+
+  // Wait for pods to come back
+  for (const component of podComponents) {
+    await waitForResourceByLabel({
+      resource: 'pods',
+      namespace: namespace,
+      label: `app.kubernetes.io/component=${component}`,
+    });
+  }
 }
 
 /**