@aztec/end-to-end 0.0.1-commit.24de95ac → 0.0.1-commit.3469e52

package/dest/spartan/utils.js

@@ -1,4 +1,5 @@
 import { createLogger } from '@aztec/aztec.js/log';
+import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { makeBackoff, retry } from '@aztec/foundation/retry';
 import { schemas } from '@aztec/foundation/schemas';
 import { sleep } from '@aztec/foundation/sleep';
@@ -17,7 +18,9 @@ const testConfigSchema = z.object({
     L1_RPC_URLS_JSON: z.string().optional(),
     L1_ACCOUNT_MNEMONIC: z.string().optional(),
     AZTEC_SLOT_DURATION: z.coerce.number().optional().default(24),
-    AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5)
+    AZTEC_EPOCH_DURATION: z.coerce.number().optional().default(32),
+    AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5),
+    AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET: z.coerce.number().optional().default(2)
 });
 export function setupEnvironment(env) {
     const config = testConfigSchema.parse(env);
@@ -90,7 +93,7 @@ export async function startPortForward({ resource, namespace, containerPort, hos
         ]
     });
     let isResolved = false;
-    const connected = new Promise((resolve)=>{
+    const connected = new Promise((resolve, reject)=>{
         process1.stdout?.on('data', (data)=>{
             const str = data.toString();
             if (!isResolved && str.includes('Forwarding from')) {
@@ -98,7 +101,8 @@ export async function startPortForward({ resource, namespace, containerPort, hos
                 logger.debug(`Port forward for ${resource}: ${str}`);
                 const port = str.search(/:\d+/);
                 if (port === -1) {
-                    throw new Error('Port not found in port forward output');
+                    reject(new Error('Port not found in port forward output'));
+                    return;
                 }
                 const portNumber = parseInt(str.slice(port + 1));
                 logger.verbose(`Port forwarded for ${resource} at ${portNumber}:${containerPort}`);
@@ -118,17 +122,26 @@ export async function startPortForward({ resource, namespace, containerPort, hos
         process1.on('close', ()=>{
             if (!isResolved) {
                 isResolved = true;
-                logger.warn(`Port forward for ${resource} closed before connection established`);
-                resolve(0);
+                const msg = `Port forward for ${resource} closed before connection established`;
+                logger.warn(msg);
+                reject(new Error(msg));
             }
         });
         process1.on('error', (error)=>{
-            logger.error(`Port forward for ${resource} error: ${error}`);
-            resolve(0);
+            if (!isResolved) {
+                isResolved = true;
+                const msg = `Port forward for ${resource} error: ${error}`;
+                logger.error(msg);
+                reject(new Error(msg));
+            }
         });
         process1.on('exit', (code)=>{
-            logger.verbose(`Port forward for ${resource} exited with code ${code}`);
-            resolve(0);
+            if (!isResolved) {
+                isResolved = true;
+                const msg = `Port forward for ${resource} exited with code ${code}`;
+                logger.verbose(msg);
+                reject(new Error(msg));
+            }
         });
     });
     const port = await connected;
@@ -137,9 +150,42 @@ export async function startPortForward({ resource, namespace, containerPort, hos
         port
     };
 }
-export function startPortForwardForRPC(namespace) {
+export function getExternalIP(namespace, serviceName) {
+    const { promise, resolve, reject } = promiseWithResolvers();
+    const process1 = spawn('kubectl', [
+        'get',
+        'service',
+        '-n',
+        namespace,
+        `${namespace}-${serviceName}`,
+        '--output',
+        "jsonpath='{.status.loadBalancer.ingress[0].ip}'"
+    ], {
+        stdio: 'pipe'
+    });
+    let ip = '';
+    process1.stdout.on('data', (data)=>{
+        ip += data;
+    });
+    process1.on('error', (err)=>{
+        reject(err);
+    });
+    process1.on('exit', ()=>{
+        // kubectl prints JSON. Remove the quotes
+        resolve(ip.replace(/"|'/g, ''));
+    });
+    return promise;
+}
+export function startPortForwardForPrometeheus(namespace) {
     return startPortForward({
-        resource: `services/${namespace}-rpc-aztec-node`,
+        resource: `svc/${namespace}-prometheus-server`,
+        namespace,
+        containerPort: 80
+    });
+}
+export function startPortForwardForRPC(namespace, index = 0) {
+    return startPortForward({
+        resource: `pod/${namespace}-rpc-aztec-node-${index}`,
         namespace,
         containerPort: 8080
     });
@@ -158,9 +204,11 @@ export async function deleteResourceByName({ resource, namespace, name, force =
     return stdout;
 }
 export async function deleteResourceByLabel({ resource, namespace, label, timeout = '5m', force = false }) {
-    // Check if the resource type exists before attempting to delete
     try {
-        await execAsync(`kubectl api-resources --api-group="" --no-headers -o name | grep -q "^${resource}$" || kubectl api-resources --no-headers -o name | grep -q "^${resource}$"`);
+        // Match both plain and group-qualified names (e.g., "podchaos" or "podchaos.chaos-mesh.org")
+        const escaped = resource.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
+        const regex = `(^|\\.)${escaped}(\\.|$)`;
+        await execAsync(`kubectl api-resources --no-headers -o name | grep -Eq '${regex}'`);
     } catch (error) {
         logger.warn(`Resource type '${resource}' not found in cluster, skipping deletion ${error}`);
         return '';
@@ -176,6 +224,31 @@ export async function waitForResourceByLabel({ resource, label, namespace, condi
     const { stdout } = await execAsync(command);
     return stdout;
 }
+export async function waitForResourceByName({ resource, name, namespace, condition = 'Ready', timeout = '10m' }) {
+    const command = `kubectl wait ${resource}/${name} --for=condition=${condition} -n ${namespace} --timeout=${timeout}`;
+    logger.info(`command: ${command}`);
+    const { stdout } = await execAsync(command);
+    return stdout;
+}
+export async function waitForResourcesByName({ resource, names, namespace, condition = 'Ready', timeout = '10m' }) {
+    if (!names.length) {
+        throw new Error(`No ${resource} names provided to waitForResourcesByName`);
+    }
+    // Wait all in parallel; if any fails, surface which one.
+    await Promise.all(names.map(async (name)=>{
+        try {
+            await waitForResourceByName({
+                resource,
+                name,
+                namespace,
+                condition,
+                timeout
+            });
+        } catch (err) {
+            throw new Error(`Failed waiting for ${resource}/${name} condition=${condition} timeout=${timeout} namespace=${namespace}: ${String(err)}`);
+        }
+    }));
+}
 export function getChartDir(spartanDir, chartName) {
     return path.join(spartanDir.trim(), chartName);
 }
@@ -197,6 +270,57 @@ async function execHelmCommand(args) {
     const { stdout } = await execAsync(helmCommand);
     return stdout;
 }
+async function getHelmReleaseStatus(instanceName, namespace) {
+    try {
+        const { stdout } = await execAsync(`helm list --namespace ${namespace} --all --filter '^${instanceName}$' --output json | cat`);
+        const parsed = JSON.parse(stdout);
+        const row = parsed.find((r)=>r.name === instanceName);
+        return row?.status;
+    } catch  {
+        return undefined;
+    }
+}
+async function forceDeleteHelmReleaseRecord(instanceName, namespace, logger) {
+    const labelSelector = `owner=helm,name=${instanceName}`;
+    const cmd = `kubectl delete secret -n ${namespace} -l ${labelSelector} --ignore-not-found=true`;
+    logger.warn(`Force deleting Helm release record: ${cmd}`);
+    await execAsync(cmd).catch(()=>undefined);
+}
+async function hasDeployedHelmRelease(instanceName, namespace) {
+    try {
+        const status = await getHelmReleaseStatus(instanceName, namespace);
+        return status?.toLowerCase() === 'deployed';
+    } catch  {
+        return false;
+    }
+}
+export async function uninstallChaosMesh(instanceName, namespace, logger) {
+    // uninstall the helm chart if it exists
+    logger.info(`Uninstalling helm chart ${instanceName}`);
+    await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`);
+    // and delete the chaos-mesh resources created by this release
+    const deleteByLabel = async (resource)=>{
+        const args = {
+            resource,
+            namespace: namespace,
+            label: `app.kubernetes.io/instance=${instanceName}`
+        };
+        logger.info(`Deleting ${resource} resources for release ${instanceName}`);
+        await deleteResourceByLabel(args).catch((e)=>{
+            logger.error(`Error deleting ${resource}: ${e}`);
+            logger.info(`Force deleting ${resource}`);
+            return deleteResourceByLabel({
+                ...args,
+                force: true
+            });
+        });
+    };
+    await deleteByLabel('podchaos');
+    await deleteByLabel('networkchaos');
+    await deleteByLabel('podnetworkchaos');
+    await deleteByLabel('workflows');
+    await deleteByLabel('workflownodes');
+}
 /**
  * Installs a Helm chart with the given parameters.
  * @param instanceName - The name of the Helm chart instance.
@@ -213,35 +337,14 @@ async function execHelmCommand(args) {
  * const stdout = await installChaosMeshChart({ instanceName: 'force-reorg', targetNamespace: 'smoke', valuesFile: 'prover-failure.yaml'});
  * console.log(stdout);
  * ```
- */ export async function installChaosMeshChart({ instanceName, targetNamespace, valuesFile, helmChartDir, chaosMeshNamespace = 'chaos-mesh', timeout = '10m', clean = true, values = {}, logger }) {
+ */ export async function installChaosMeshChart({ instanceName, targetNamespace, valuesFile, helmChartDir, timeout = '10m', clean = true, values = {}, logger }) {
     if (clean) {
-        // uninstall the helm chart if it exists
-        logger.info(`Uninstalling helm chart ${instanceName}`);
-        await execAsync(`helm uninstall ${instanceName} --namespace ${chaosMeshNamespace} --wait --ignore-not-found`);
-        // and delete the chaos-mesh resources created by this release
-        const deleteByLabel = async (resource)=>{
-            const args = {
-                resource,
-                namespace: chaosMeshNamespace,
-                label: `app.kubernetes.io/instance=${instanceName}`
-            };
-            logger.info(`Deleting ${resource} resources for release ${instanceName}`);
-            await deleteResourceByLabel(args).catch((e)=>{
-                logger.error(`Error deleting ${resource}: ${e}`);
-                logger.info(`Force deleting ${resource}`);
-                return deleteResourceByLabel({
-                    ...args,
-                    force: true
-                });
-            });
-        };
-        await deleteByLabel('podchaos');
-        await deleteByLabel('networkchaos');
+        await uninstallChaosMesh(instanceName, targetNamespace, logger);
     }
     return execHelmCommand({
         instanceName,
         helmChartDir,
-        namespace: chaosMeshNamespace,
+        namespace: targetNamespace,
         valuesFile,
         timeout,
         values: {
@@ -262,70 +365,85 @@ export function applyProverFailure({ namespace, spartanDir, durationSeconds, log
         logger
     });
 }
-export function applyProverKill({ namespace, spartanDir, logger }) {
+export function applyValidatorFailure({ namespace, spartanDir, logger, values, instanceName }) {
+    return installChaosMeshChart({
+        instanceName: instanceName ?? 'validator-failure',
+        targetNamespace: namespace,
+        valuesFile: 'validator-failure.yaml',
+        helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+        values,
+        logger
+    });
+}
+export function applyProverKill({ namespace, spartanDir, logger, values }) {
     return installChaosMeshChart({
         instanceName: 'prover-kill',
         targetNamespace: namespace,
         valuesFile: 'prover-kill.yaml',
         helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+        chaosMeshNamespace: namespace,
         clean: true,
-        logger
+        logger,
+        values
     });
 }
-export function applyProverBrokerKill({ namespace, spartanDir, logger }) {
+export function applyProverBrokerKill({ namespace, spartanDir, logger, values }) {
     return installChaosMeshChart({
         instanceName: 'prover-broker-kill',
         targetNamespace: namespace,
         valuesFile: 'prover-broker-kill.yaml',
         helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
         clean: true,
-        logger
+        logger,
+        values
     });
 }
-export function applyBootNodeFailure({ namespace, spartanDir, durationSeconds, logger }) {
+export function applyBootNodeFailure({ instanceName = 'boot-node-failure', namespace, spartanDir, durationSeconds, logger, values }) {
     return installChaosMeshChart({
-        instanceName: 'boot-node-failure',
+        instanceName,
         targetNamespace: namespace,
         valuesFile: 'boot-node-failure.yaml',
         helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
         values: {
-            'bootNodeFailure.duration': `${durationSeconds}s`
+            'bootNodeFailure.duration': `${durationSeconds}s`,
+            ...values ?? {}
         },
         logger
     });
 }
-export function applyValidatorKill({ namespace, spartanDir, logger, values }) {
+export function applyValidatorKill({ instanceName = 'validator-kill', namespace, spartanDir, logger, values, clean = true }) {
     return installChaosMeshChart({
-        instanceName: 'validator-kill',
+        instanceName: instanceName ?? 'validator-kill',
         targetNamespace: namespace,
         valuesFile: 'validator-kill.yaml',
         helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+        clean,
         logger,
         values
     });
 }
-export function applyNetworkShaping({ valuesFile, namespace, spartanDir, logger }) {
+export function applyNetworkShaping({ instanceName = 'network-shaping', valuesFile, namespace, spartanDir, logger }) {
     return installChaosMeshChart({
-        instanceName: 'network-shaping',
+        instanceName,
         targetNamespace: namespace,
         valuesFile,
         helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
         logger
     });
 }
-export async function awaitL2BlockNumber(rollupCheatCodes, blockNumber, timeoutSeconds, logger) {
-    logger.info(`Waiting for L2 Block ${blockNumber}`);
+export async function awaitCheckpointNumber(rollupCheatCodes, checkpointNumber, timeoutSeconds, logger) {
+    logger.info(`Waiting for checkpoint ${checkpointNumber}`);
     let tips = await rollupCheatCodes.getTips();
     const endTime = Date.now() + timeoutSeconds * 1000;
-    while(tips.pending < blockNumber && Date.now() < endTime){
-        logger.info(`At L2 Block ${tips.pending}`);
+    while(tips.pending < checkpointNumber && Date.now() < endTime){
+        logger.info(`At checkpoint ${tips.pending}`);
         await sleep(1000);
         tips = await rollupCheatCodes.getTips();
     }
-    if (tips.pending < blockNumber) {
-        throw new Error(`Timeout waiting for L2 Block ${blockNumber}, only reached ${tips.pending}`);
+    if (tips.pending < checkpointNumber) {
+        throw new Error(`Timeout waiting for checkpoint ${checkpointNumber}, only reached ${tips.pending}`);
     } else {
-        logger.info(`Reached L2 Block ${tips.pending}`);
+        logger.info(`Reached checkpoint ${tips.pending}`);
     }
 }
 export async function restartBot(namespace, logger) {
@@ -369,7 +487,12 @@ export async function restartBot(namespace, logger) {
         // Provide L1 execution RPC for bridging fee juice
         'bot.node.env.ETHEREUM_HOSTS': `http://${namespace}-eth-execution.${namespace}.svc.cluster.local:8545`,
         // Provide L1 mnemonic for bridging (falls back to labs mnemonic)
-        'bot.node.env.BOT_L1_MNEMONIC': mnemonic
+        'bot.node.env.BOT_L1_MNEMONIC': mnemonic,
+        // The bot does not need Kubernetes API access. Disable RBAC + ServiceAccount creation so the chart
+        // can be installed by users without cluster-scoped RBAC permissions.
+        'bot.rbac.create': false,
+        'bot.serviceAccount.create': false,
+        'bot.serviceAccount.name': 'default'
     };
     // Ensure we derive a funded L1 key (index 0 is funded on anvil default mnemonic)
     if (mnemonicStartIndex === undefined) {
@@ -392,7 +515,7 @@ export async function restartBot(namespace, logger) {
     let tag = tagFromEnv;
     if (!repository || !tag) {
         try {
-            const { stdout } = await execAsync(`kubectl get pods -l app.kubernetes.io/component=validator -n ${namespace} -o jsonpath='{.items[0].spec.containers[?(@.name=="aztec")].image}' | cat`);
+            const { stdout } = await execAsync(`kubectl get pods -l app.kubernetes.io/name=validator -n ${namespace} -o jsonpath='{.items[0].spec.containers[?(@.name=="aztec")].image}' | cat`);
             const image = stdout.trim().replace(/^'|'$/g, '');
             if (image && image.includes(':')) {
                 const lastColon = image.lastIndexOf(':');
@@ -410,6 +533,22 @@ export async function restartBot(namespace, logger) {
     if (mnemonicStartIndex !== undefined) {
         values['bot.mnemonicStartIndex'] = typeof mnemonicStartIndex === 'string' ? mnemonicStartIndex : Number(mnemonicStartIndex);
     }
+    // If a previous install attempt left the release in a non-deployed state (e.g. FAILED),
+    // `helm upgrade --install` can error with "has no deployed releases".
+    // In that case, clear the release record and do a clean install.
+    const existingStatus = await getHelmReleaseStatus(instanceName, namespace);
+    if (existingStatus && existingStatus.toLowerCase() !== 'deployed') {
+        logger.warn(`Transfer bot release ${instanceName} is in status '${existingStatus}'. Reinstalling cleanly.`);
+        await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`).catch(()=>undefined);
+        // If helm left the release in `uninstalling`, force-delete the record so we can reinstall.
+        const afterUninstallStatus = await getHelmReleaseStatus(instanceName, namespace);
+        if (afterUninstallStatus?.toLowerCase() === 'uninstalling') {
+            await forceDeleteHelmReleaseRecord(instanceName, namespace, logger);
+        }
+    }
+    // `--reuse-values` fails if the release has never successfully deployed (e.g. first install, or a previous failed install).
+    // Only reuse values when we have a deployed release to reuse from.
+    const effectiveReuseValues = reuseValues && await hasDeployedHelmRelease(instanceName, namespace);
     await execHelmCommand({
         instanceName,
         helmChartDir,
@@ -417,7 +556,7 @@ export async function restartBot(namespace, logger) {
         valuesFile: undefined,
         timeout,
         values: values,
-        reuseValues
+        reuseValues: effectiveReuseValues
     });
     if (replicas > 0) {
         await waitForResourceByLabel({
@@ -449,8 +588,9 @@ export async function restartBot(namespace, logger) {
     const drop = enabled ? 'true' : 'false';
     const prob = String(probability);
     const selectors = [
-        'app=validator',
-        'app.kubernetes.io/component=validator'
+        'app.kubernetes.io/name=validator',
+        'app.kubernetes.io/component=validator',
+        'app=validator'
     ];
     let updated = false;
     for (const selector of selectors){
@@ -477,8 +617,9 @@ export async function restartBot(namespace, logger) {
 }
 export async function restartValidators(namespace, logger) {
     const selectors = [
-        'app=validator',
-        'app.kubernetes.io/component=validator'
+        'app.kubernetes.io/name=validator',
+        'app.kubernetes.io/component=validator',
+        'app=validator'
     ];
     let any = false;
     for (const selector of selectors){
@@ -532,11 +673,27 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
     logger.info(`Validator dynamic boot node enabled`);
 }
 export async function getSequencers(namespace) {
-    const command = `kubectl get pods -l app.kubernetes.io/component=validator -n ${namespace} -o jsonpath='{.items[*].metadata.name}'`;
-    const { stdout } = await execAsync(command);
-    const sequencers = stdout.split(' ');
-    logger.verbose(`Found sequencer pods ${sequencers.join(', ')}`);
-    return sequencers;
+    const selectors = [
+        'app.kubernetes.io/name=validator',
+        'app.kubernetes.io/component=validator',
+        'app.kubernetes.io/component=sequencer-node',
+        'app=validator'
+    ];
+    for (const selector of selectors){
+        try {
+            const command = `kubectl get pods -l ${selector} -n ${namespace} -o jsonpath='{.items[*].metadata.name}'`;
+            const { stdout } = await execAsync(command);
+            const sequencers = stdout.split(' ').map((s)=>s.trim()).filter(Boolean);
+            if (sequencers.length > 0) {
+                logger.verbose(`Found sequencer pods ${sequencers.join(', ')} (selector=${selector})`);
+                return sequencers;
+            }
+        } catch  {
+        // try next selector
+        }
+    }
+    // Fail fast instead of returning [''] which leads to attempts to port-forward `pod/`.
+    throw new Error(`No sequencer/validator pods found in namespace ${namespace}. Tried selectors: ${selectors.join(', ')}`);
 }
 export function updateSequencersConfig(env, config) {
     return withSequencersAdmin(env, async (client)=>{
@@ -586,7 +743,9 @@ export async function withSequencersAdmin(env, fn) {
         const url = `http://127.0.0.1:${port}`;
         const client = createPublicClient({
             transport: fallback([
-                http(url)
+                http(url, {
+                    batch: false
+                })
             ])
         });
         if (processes) {
@@ -604,7 +763,9 @@ export async function withSequencersAdmin(env, fn) {
         }
         const client = createPublicClient({
             transport: fallback([
-                http(L1_RPC_URLS_JSON)
+                http(L1_RPC_URLS_JSON, {
+                    batch: false
+                })
             ])
         });
         return {
@@ -637,71 +798,103 @@ export async function withSequencersAdmin(env, fn) {
 /**
  * Rolls the Aztec pods in the given namespace.
  * @param namespace - The namespace to roll the Aztec pods in.
- * @dev - IMPORTANT: This function DOES NOT delete the underlying PVCs.
- *        This means that the pods will be restarted with the same persistent storage.
- *        This is useful for testing, but you should be aware of the implications.
- */ export async function rollAztecPods(namespace) {
-    await deleteResourceByLabel({
-        resource: 'pods',
-        namespace: namespace,
-        label: 'app=boot-node'
-    });
-    await deleteResourceByLabel({
-        resource: 'pods',
-        namespace: namespace,
-        label: 'app=prover-node'
-    });
-    await deleteResourceByLabel({
-        resource: 'pods',
-        namespace: namespace,
-        label: 'app=prover-broker'
-    });
-    await deleteResourceByLabel({
-        resource: 'pods',
-        namespace: namespace,
-        label: 'app=prover-agent'
-    });
-    await deleteResourceByLabel({
-        resource: 'pods',
-        namespace: namespace,
-        label: 'app=validator'
-    });
-    await deleteResourceByLabel({
-        resource: 'pods',
-        namespace: namespace,
-        label: 'app=pxe'
-    });
+ * @param clearState - If true, also deletes the underlying PVCs to clear persistent storage.
+ *        This is required for rollup upgrades where the old state is incompatible with the new rollup.
+ *        Defaults to false, which preserves the existing storage.
+ */ export async function rollAztecPods(namespace, clearState = false) {
+    // Pod components use 'validator', but StatefulSets and PVCs use 'sequencer-node' for validators
+    const podComponents = [
+        'p2p-bootstrap',
+        'prover-node',
+        'prover-broker',
+        'prover-agent',
+        'sequencer-node',
+        'rpc'
+    ];
+    const pvcComponents = [
+        'p2p-bootstrap',
+        'prover-node',
+        'prover-broker',
+        'sequencer-node',
+        'rpc'
+    ];
+    // StatefulSet components that need to be scaled down before PVC deletion
+    // Note: validators use 'sequencer-node' as component label, not 'validator'
+    const statefulSetComponents = [
+        'p2p-bootstrap',
+        'prover-node',
+        'prover-broker',
+        'sequencer-node',
+        'rpc'
+    ];
+    if (clearState) {
+        // To delete PVCs, we must first scale down StatefulSets so pods release the volumes
+        // Otherwise PVC deletion will hang waiting for pods to terminate
+        // First, save original replica counts
+        const originalReplicas = new Map();
+        for (const component of statefulSetComponents){
+            try {
+                const getCmd = `kubectl get statefulset -l app.kubernetes.io/component=${component} -n ${namespace} -o jsonpath='{.items[0].spec.replicas}'`;
+                const { stdout } = await execAsync(getCmd);
+                const replicas = parseInt(stdout.replace(/'/g, '').trim(), 10);
+                if (!isNaN(replicas) && replicas > 0) {
+                    originalReplicas.set(component, replicas);
+                }
+            } catch  {
+            // Component might not exist, continue
+            }
+        }
+        // Scale down to 0
+        for (const component of statefulSetComponents){
+            try {
+                const scaleCmd = `kubectl scale statefulset -l app.kubernetes.io/component=${component} -n ${namespace} --replicas=0 --timeout=2m`;
+                logger.info(`command: ${scaleCmd}`);
+                await execAsync(scaleCmd);
+            } catch (e) {
+                // Component might not exist or might be a Deployment, continue
+                logger.verbose(`Scale down ${component} skipped: ${e}`);
+            }
+        }
+        // Wait for pods to terminate
+        await sleep(15 * 1000);
+        // Now delete PVCs (they should no longer be in use)
+        for (const component of pvcComponents){
+            await deleteResourceByLabel({
+                resource: 'persistentvolumeclaims',
+                namespace: namespace,
+                label: `app.kubernetes.io/component=${component}`
+            });
+        }
+        // Scale StatefulSets back up to original replica counts
+        for (const component of statefulSetComponents){
+            const replicas = originalReplicas.get(component) ?? 1;
+            try {
+                const scaleCmd = `kubectl scale statefulset -l app.kubernetes.io/component=${component} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+                logger.info(`command: ${scaleCmd}`);
+                await execAsync(scaleCmd);
+            } catch (e) {
+                logger.verbose(`Scale up ${component} skipped: ${e}`);
+            }
+        }
+    } else {
+        // Just delete pods (no state clearing)
+        for (const component of podComponents){
+            await deleteResourceByLabel({
+                resource: 'pods',
+                namespace: namespace,
+                label: `app.kubernetes.io/component=${component}`
+            });
+        }
+    }
     await sleep(10 * 1000);
-    await waitForResourceByLabel({
-        resource: 'pods',
-        namespace: namespace,
-        label: 'app=boot-node'
-    });
-    await waitForResourceByLabel({
-        resource: 'pods',
-        namespace: namespace,
-        label: 'app=prover-node'
-    });
-    await waitForResourceByLabel({
-        resource: 'pods',
-        namespace: namespace,
-        label: 'app=prover-broker'
-    });
-    await waitForResourceByLabel({
-        resource: 'pods',
-        namespace: namespace,
-        label: 'app=prover-agent'
-    });
-    await waitForResourceByLabel({
-        resource: 'pods',
-        namespace: namespace,
-        label: 'app=validator'
-    });
-    await waitForResourceByLabel({
-        resource: 'pods',
-        namespace: namespace,
-        label: 'app=pxe'
-    });
+    // Wait for pods to come back
+    for (const component of podComponents){
+        await waitForResourceByLabel({
+            resource: 'pods',
+            namespace: namespace,
+            label: `app.kubernetes.io/component=${component}`
+        });
+    }
 }
 /**
  * Returns the absolute path to the git repository root