@aztec/end-to-end 0.0.1-commit.e61ad554 → 0.0.1-commit.f146247c

package/src/spartan/tx_metrics.ts

@@ -6,6 +6,132 @@ import { Tx, type TxReceipt } from '@aztec/stdlib/tx';
 
 import { createHistogram } from 'perf_hooks';
 
+/** Metrics class for proving-related benchmarks. */
+export class ProvingMetrics {
+  private successfulTxs: number | undefined;
+  private proofDuration: number | undefined;
+  private activeAgents: number | undefined;
+  private avgQueueTime: number | undefined;
+  private jobRetries: number | undefined;
+  private jobDuration: number | undefined;
+  private timedOutJobs: number | undefined;
+  private resolvedJobs: number | undefined;
+  private rejectedJobs: number | undefined;
+  private epochProvingDuration: number | undefined;
+  private provenTransactions: number | undefined;
+  private provenBlocks: number | undefined;
+
+  constructor(private prefix: string) {}
+
+  recordSuccessfulTxs(count: number): void {
+    this.successfulTxs = count;
+  }
+
+  recordProofDuration(seconds: number): void {
+    this.proofDuration = seconds;
+  }
+
+  recordActiveAgents(count: number): void {
+    this.activeAgents = count;
+  }
+
+  recordAvgQueueTime(ms: number): void {
+    this.avgQueueTime = ms;
+  }
+
+  recordJobRetries(count: number): void {
+    this.jobRetries = count;
+  }
+
+  recordJobDuration(ms: number): void {
+    this.jobDuration = ms;
+  }
+
+  recordTimedOutJobs(count: number): void {
+    this.timedOutJobs = count;
+  }
+
+  recordResolvedJobs(count: number): void {
+    this.resolvedJobs = count;
+  }
+
+  recordRejectedJobs(count: number): void {
+    this.rejectedJobs = count;
+  }
+
+  recordEpochProvingDuration(seconds: number): void {
+    this.epochProvingDuration = seconds;
+  }
+
+  recordProvenTransactions(count: number): void {
+    this.provenTransactions = count;
+  }
+
+  recordProvenBlocks(count: number): void {
+    this.provenBlocks = count;
+  }
+
+  toGithubActionBenchmarkJSON(): Array<{ name: string; unit: string; value: number }> {
+    const data: Array<{ name: string; unit: string; value: number }> = [];
+
+    if (this.successfulTxs !== undefined) {
+      data.push({ name: `${this.prefix}/successful_txs`, unit: 'count', value: this.successfulTxs });
+    }
+
+    if (this.proofDuration !== undefined) {
+      data.push({ name: `${this.prefix}/proof_duration`, unit: 's', value: this.proofDuration });
+    }
+
+    if (this.activeAgents !== undefined) {
+      data.push({ name: `${this.prefix}/active_agents`, unit: 'count', value: this.activeAgents });
+    }
+
+    if (this.avgQueueTime !== undefined) {
+      data.push({ name: `${this.prefix}/avg_queue_time`, unit: 'ms', value: this.avgQueueTime });
+    }
+
+    if (this.jobRetries !== undefined) {
+      data.push({ name: `${this.prefix}/job_retries`, unit: 'count', value: this.jobRetries });
+    }
+
+    if (this.jobDuration !== undefined) {
+      data.push({ name: `${this.prefix}/job_duration`, unit: 'ms', value: this.jobDuration });
+    }
+
+    if (this.timedOutJobs !== undefined) {
+      data.push({ name: `${this.prefix}/timed_out_jobs`, unit: 'count', value: this.timedOutJobs });
+    }
+
+    if (this.resolvedJobs !== undefined) {
+      data.push({ name: `${this.prefix}/resolved_jobs`, unit: 'count', value: this.resolvedJobs });
+    }
+
+    if (this.rejectedJobs !== undefined) {
+      data.push({ name: `${this.prefix}/rejected_jobs`, unit: 'count', value: this.rejectedJobs });
+    }
+
+    if (this.epochProvingDuration !== undefined) {
+      data.push({ name: `${this.prefix}/epoch_proving_duration`, unit: 's', value: this.epochProvingDuration });
+    }
+
+    if (this.provenTransactions !== undefined) {
+      data.push({ name: `${this.prefix}/proven_transactions`, unit: 'count', value: this.provenTransactions });
+    }
+
+    if (this.provenBlocks !== undefined) {
+      data.push({ name: `${this.prefix}/proven_blocks`, unit: 'count', value: this.provenBlocks });
+    }
+
+    const scenario = process.env.BENCH_SCENARIO?.trim();
+    if (!scenario) {
+      return data;
+    }
+
+    const scenarioPrefix = `scenario/${scenario}/`;
+    return data.map(entry => ({ ...entry, name: `${scenarioPrefix}${entry.name}` }));
+  }
+}
+
 export type TxInclusionData = {
   txHash: string;
   sentAt: number;

package/src/spartan/utils/index.ts

@@ -24,6 +24,7 @@ export {
   getServiceEndpoint,
   getRPCEndpoint,
   getEthereumEndpoint,
+  createResilientPrometheusConnection,
 } from './k8s.js';
 
 // Chaos Mesh
@@ -45,6 +46,7 @@ export { restartBot, installTransferBot, uninstallTransferBot } from './bot.js';
 // Node operations (sequencers, validators, pods)
 export {
   awaitCheckpointNumber,
+  waitForProvenToAdvance,
   getSequencers,
   updateSequencersConfig,
   getSequencersConfig,

package/src/spartan/utils/k8s.ts

@@ -1,4 +1,5 @@
 import { createLogger } from '@aztec/aztec.js/log';
+import type { Logger } from '@aztec/foundation/log';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { retryUntil } from '@aztec/foundation/retry';
 
@@ -6,6 +7,8 @@ import { type ChildProcess, exec, spawn } from 'child_process';
 import path from 'path';
 import { promisify } from 'util';
 
+import { AlertTriggeredError, GrafanaClient } from '../../quality_of_service/grafana_client.js';
+
 const execAsync = promisify(exec);
 
 const logger = createLogger('e2e:k8s-utils');
@@ -370,6 +373,155 @@ export async function waitForResourcesByName({
   );
 }
 
+/**
+ * Waits for all StatefulSets matching a label to have all their replicas ready.
+ *
+ * @param namespace - Kubernetes namespace
+ * @param label - Label selector for StatefulSets (e.g., "app.kubernetes.io/component=sequencer-node")
+ * @param timeoutSeconds - Maximum time to wait in seconds
+ * @param pollIntervalSeconds - How often to check status
+ */
+export async function waitForStatefulSetsReady({
+  namespace,
+  label,
+  timeoutSeconds = 600,
+  pollIntervalSeconds = 5,
+}: {
+  namespace: string;
+  label: string;
+  timeoutSeconds?: number;
+  pollIntervalSeconds?: number;
+}): Promise<void> {
+  logger.info(`Waiting for StatefulSets with label ${label} to have all replicas ready (timeout: ${timeoutSeconds}s)`);
+
+  await retryUntil(
+    async () => {
+      // Get all StatefulSets matching the label
+      const getCmd = `kubectl get statefulset -l ${label} -n ${namespace} -o json`;
+      const { stdout } = await execAsync(getCmd);
+      const result = JSON.parse(stdout);
+
+      if (!result.items || result.items.length === 0) {
+        logger.verbose(`No StatefulSets found with label ${label}`);
+        return false;
+      }
+
+      // Check each StatefulSet
+      for (const sts of result.items) {
+        const name = sts.metadata.name;
+        const desired = sts.spec.replicas ?? 0;
+        const ready = sts.status.readyReplicas ?? 0;
+        const updated = sts.status.updatedReplicas ?? 0;
+
+        if (ready < desired || updated < desired) {
+          logger.verbose(`StatefulSet ${name}: ${ready}/${desired} ready, ${updated}/${desired} updated`);
+          return false;
+        }
+      }
+
+      logger.info(`All StatefulSets with label ${label} are ready`);
+      return true;
+    },
+    `StatefulSets with label ${label} to be ready`,
+    timeoutSeconds,
+    pollIntervalSeconds,
+  );
+}
+
+/**
+ * Creates a Prometheus connection that can re-establish port-forward on failure.
+ * Returns functions to connect and run alert checks that automatically reconnect if needed.
+ *
+ * @param namespace - K8s namespace to fall back to if metrics namespace doesn't have Prometheus
+ * @param endpoints - Array to track created endpoints for cleanup
+ * @param log - Logger instance
+ */
+export function createResilientPrometheusConnection(
+  namespace: string,
+  endpoints: ServiceEndpoint[],
+  log: Logger,
+): {
+  connect: () => Promise<GrafanaClient>;
+  runAlertCheck: (alerts: Parameters<GrafanaClient['runAlertCheck']>[0]) => Promise<void>;
+} {
+  let alertChecker: GrafanaClient | undefined;
+  let currentEndpoint: ServiceEndpoint | undefined;
+
+  const connect = async (): Promise<GrafanaClient> => {
+    // Kill existing connection if any
+    if (currentEndpoint?.process) {
+      currentEndpoint.process.kill();
+    }
+
+    // Try metrics namespace first, then network namespace
+    let promPort = 0;
+    let promUrl = '';
+    let promProc: ChildProcess | undefined;
+
+    try {
+      const metricsResult = await startPortForward({
+        resource: `svc/metrics-prometheus-server`,
+        namespace: 'metrics',
+        containerPort: 80,
+      });
+      promProc = metricsResult.process;
+      promPort = metricsResult.port;
+      promUrl = `http://127.0.0.1:${promPort}/api/v1`;
+    } catch {
+      // Metrics namespace might not have Prometheus, try network namespace
+      log.verbose('Metrics namespace Prometheus not available, trying network namespace');
+    }
+
+    if (promPort === 0) {
+      const nsResult = await startPortForward({
+        resource: `svc/prometheus-server`,
+        namespace,
+        containerPort: 80,
+      });
+      promProc = nsResult.process;
+      promPort = nsResult.port;
+      promUrl = `http://127.0.0.1:${promPort}/api/v1`;
+    }
+
+    if (!promProc || promPort === 0) {
+      throw new Error('Unable to port-forward to Prometheus');
+    }
+
+    currentEndpoint = { url: promUrl, process: promProc };
+    endpoints.push(currentEndpoint);
+    alertChecker = new GrafanaClient(log, { grafanaEndpoint: promUrl, grafanaCredentials: '' });
+    log.info(`Established Prometheus connection at ${promUrl}`);
+    return alertChecker;
+  };
+
+  const runAlertCheck = async (alerts: Parameters<GrafanaClient['runAlertCheck']>[0]): Promise<void> => {
+    if (!alertChecker) {
+      alertChecker = await connect();
+    }
+
+    try {
+      await alertChecker.runAlertCheck(alerts);
+    } catch (err) {
+      // If it's an AlertTriggeredError (expected behavior)
+      if (err instanceof AlertTriggeredError) {
+        throw err;
+      }
+
+      // Check if it's a connection error (port-forward died)
+      const errorStr = String(err);
+      if (errorStr.includes('fetch failed') || errorStr.includes('ECONNREFUSED') || errorStr.includes('ECONNRESET')) {
+        log.warn(`Prometheus connection lost, re-establishing port-forward...`);
+        alertChecker = await connect();
+        await alertChecker.runAlertCheck(alerts);
+      } else {
+        throw err;
+      }
+    }
+  };
+
+  return { connect, runAlertCheck };
+}
+
 export function getChartDir(spartanDir: string, chartName: string) {
   return path.join(spartanDir.trim(), chartName);
 }

package/src/spartan/utils/nodes.ts

@@ -1,8 +1,9 @@
 import { createLogger } from '@aztec/aztec.js/log';
+import { createAztecNodeClient } from '@aztec/aztec.js/node';
 import type { RollupCheatCodes } from '@aztec/aztec/testing';
 import type { CheckpointNumber } from '@aztec/foundation/branded-types';
 import type { Logger } from '@aztec/foundation/log';
-import { makeBackoff, retry } from '@aztec/foundation/retry';
+import { makeBackoff, retry, retryUntil } from '@aztec/foundation/retry';
 import { sleep } from '@aztec/foundation/sleep';
 import {
   type AztecNodeAdmin,
@@ -15,7 +16,14 @@ import { promisify } from 'util';
 
 import type { TestConfig } from './config.js';
 import { execHelmCommand } from './helm.js';
-import { deleteResourceByLabel, getChartDir, startPortForward, waitForResourceByLabel } from './k8s.js';
+import {
+  deleteResourceByLabel,
+  getChartDir,
+  startPortForward,
+  waitForResourceByLabel,
+  waitForResourceByName,
+  waitForStatefulSetsReady,
+} from './k8s.js';
 
 const execAsync = promisify(exec);
 
@@ -42,6 +50,63 @@ export async function awaitCheckpointNumber(
   }
 }
 
+/**
+ * Waits until the proven block number increases.
+ *
+ * @param rpcUrl - URL of an Aztec RPC node to query
+ * @param log - Logger instance
+ * @param timeoutSeconds - Maximum time to wait
+ * @param pollIntervalSeconds - How often to check
+ */
+export async function waitForProvenToAdvance(
+  rpcUrl: string,
+  log: Logger,
+  timeoutSeconds: number = 300,
+  pollIntervalSeconds: number = 12, // slot duration
+): Promise<void> {
+  const node = createAztecNodeClient(rpcUrl);
+
+  log.info('Waiting for proven block to advance (indicating epoch proof just submitted)...');
+
+  // Get current proven block number
+  let initialProvenBlock: number;
+  try {
+    const tips = await node.getL2Tips();
+    initialProvenBlock = Number(tips.proven.block.number);
+    log.info(`Current proven block: ${initialProvenBlock}. Waiting for it to increase...`);
+  } catch (err) {
+    log.warn(`Error getting initial tips: ${err}. Will poll until successful.`);
+    initialProvenBlock = 0;
+  }
+
+  await retryUntil(
+    async () => {
+      try {
+        const tips = await node.getL2Tips();
+        const currentProvenBlock = Number(tips.proven.block.number);
+        const proposedBlock = Number(tips.proposed.number);
+
+        log.verbose(
+          `Chain state: proposed=${proposedBlock}, proven=${currentProvenBlock} (waiting for > ${initialProvenBlock})`,
+        );
+
+        if (currentProvenBlock > initialProvenBlock) {
+          log.info(`Proven block advanced from ${initialProvenBlock} to ${currentProvenBlock}.`);
+          return true;
+        }
+
+        return false;
+      } catch (err) {
+        log.verbose(`Error checking tips: ${err}`);
+        return false;
+      }
+    },
+    'proven block to advance',
+    timeoutSeconds,
+    pollIntervalSeconds,
+  );
+}
+
 export async function getSequencers(namespace: string) {
   const selectors = [
     'app.kubernetes.io/name=validator',
@@ -90,6 +155,8 @@ export async function withSequencersAdmin<T>(env: TestConfig, fn: (node: AztecNo
   const results = [];
 
   for (const sequencer of sequencers) {
+    // Ensure pod is Ready before attempting port-forward.
+    await waitForResourceByName({ resource: 'pods', name: sequencer, namespace });
     // Wrap port-forward + fetch in a retry to handle flaky port-forwards
     const result = await retry(
       async () => {
@@ -127,6 +194,66 @@ export async function withSequencersAdmin<T>(env: TestConfig, fn: (node: AztecNo
   return results;
 }
 
+async function getAztecImageForMigrations(namespace: string): Promise<string> {
+  const aztecDockerImage = process.env.AZTEC_DOCKER_IMAGE;
+  if (aztecDockerImage) {
+    return aztecDockerImage;
+  }
+
+  const { stdout } = await execAsync(
+    `kubectl get pods -l app.kubernetes.io/name=validator -n ${namespace} -o jsonpath='{.items[0].spec.containers[?(@.name=="aztec")].image}' | cat`,
+  );
+  const image = stdout.trim().replace(/^'|'$/g, '');
+  if (!image) {
+    throw new Error(`Could not detect aztec image from validator pod in namespace ${namespace}`);
+  }
+  return image;
+}
+
+async function getHaDbConnectionUrl(namespace: string): Promise<string> {
+  const secretName = `${namespace}-validator-ha-db-postgres`;
+  const { stdout } = await execAsync(`kubectl get secret ${secretName} -n ${namespace} -o json`);
+  const secret = JSON.parse(stdout);
+  const data = secret?.data ?? {};
+  const decode = (value?: string) => (value ? Buffer.from(value, 'base64').toString('utf8') : '');
+  const user = decode(data.POSTGRES_USER);
+  const password = decode(data.POSTGRES_PASSWORD);
+  const database = decode(data.POSTGRES_DB);
+  if (!user || !password || !database) {
+    throw new Error(`Missing HA DB credentials in secret ${secretName}`);
+  }
+  const host = `${namespace}-validator-ha-db-postgres.${namespace}.svc.cluster.local`;
+  return `postgresql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${host}:5432/${database}`;
+}
+
+export async function initHADb(namespace: string): Promise<void> {
+  const databaseUrl = await getHaDbConnectionUrl(namespace);
+  const image = await getAztecImageForMigrations(namespace);
+  const jobName = `${namespace}-validator-ha-db-migrate`;
+  await execAsync(`kubectl delete pod ${jobName} -n ${namespace} --ignore-not-found=true`).catch(() => undefined);
+
+  const migrateCmd = [
+    `kubectl run ${jobName} -n ${namespace}`,
+    '--rm -i',
+    '--restart=Never',
+    `--image=${image}`,
+    `--env=DATABASE_URL=${databaseUrl}`,
+    '--command -- node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js migrate-ha-db up',
+  ].join(' ');
+  const migrateCmdForLog = migrateCmd.replace(/--env=DATABASE_URL=\S+/, '--env=DATABASE_URL=<redacted>');
+
+  await retry(
+    async () => {
+      logger.info(`command: ${migrateCmdForLog}`);
+      await execAsync(migrateCmd);
+    },
+    'run HA DB migrations',
+    makeBackoff([1, 2, 4, 8, 16]),
+    logger,
+    true,
+  );
+}
+
 /**
  * Enables or disables probabilistic transaction dropping on validators and waits for rollout.
  * Wired to env vars P2P_DROP_TX and P2P_DROP_TX_CHANCE via Helm values.
@@ -239,25 +366,46 @@ export async function enableValidatorDynamicBootNode(
  */
 export async function rollAztecPods(namespace: string, clearState: boolean = false) {
   // Pod components use 'validator', but StatefulSets and PVCs use 'sequencer-node' for validators
-  const podComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'prover-agent', 'sequencer-node', 'rpc'];
-  const pvcComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'sequencer-node', 'rpc'];
+  const podComponents = [
+    'p2p-bootstrap',
+    'prover-node',
+    'prover-broker',
+    'prover-agent',
+    'sequencer-node',
+    'rpc',
+    'validator-ha-db',
+  ];
+  const pvcComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'sequencer-node', 'rpc', 'validator-ha-db'];
   // StatefulSet components that need to be scaled down before PVC deletion
   // Note: validators use 'sequencer-node' as component label, not 'validator'
-  const statefulSetComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'sequencer-node', 'rpc'];
+  const statefulSetComponents = [
+    'p2p-bootstrap',
+    'prover-node',
+    'prover-broker',
+    'sequencer-node',
+    'rpc',
+    'validator-ha-db',
+  ];
 
   if (clearState) {
     // To delete PVCs, we must first scale down StatefulSets so pods release the volumes
     // Otherwise PVC deletion will hang waiting for pods to terminate
 
-    // First, save original replica counts
+    // Save original replica counts for all StatefulSets
     const originalReplicas: Map<string, number> = new Map();
     for (const component of statefulSetComponents) {
       try {
-        const getCmd = `kubectl get statefulset -l app.kubernetes.io/component=${component} -n ${namespace} -o jsonpath='{.items[0].spec.replicas}'`;
+        // Get all StatefulSets that match the component label
+        const getCmd = `kubectl get statefulset -l app.kubernetes.io/component=${component} -n ${namespace} -o json`;
         const { stdout } = await execAsync(getCmd);
-        const replicas = parseInt(stdout.replace(/'/g, '').trim(), 10);
-        if (!isNaN(replicas) && replicas > 0) {
-          originalReplicas.set(component, replicas);
+        const result = JSON.parse(stdout);
+        for (const sts of result.items || []) {
+          const name = sts.metadata.name;
+          const replicas = sts.spec.replicas ?? 1;
+          if (replicas > 0) {
+            originalReplicas.set(name, replicas);
+            logger.debug(`Saved replica count for StatefulSet ${name}: ${replicas}`);
+          }
         }
       } catch {
         // Component might not exist, continue
@@ -276,27 +424,81 @@ export async function rollAztecPods(namespace: string, clearState: boolean = fal
       }
     }
 
-    // Wait for pods to terminate
-    await sleep(15 * 1000);
+    // Wait for all pods to fully terminate before deleting PVCs.
+    // terminationGracePeriodSeconds default is 30s.
+    logger.info('Waiting for pods to fully terminate before deleting PVCs...');
+    for (const component of statefulSetComponents) {
+      try {
+        // Wait for all pods with this component label to be deleted
+        const waitCmd = `kubectl wait pods -l app.kubernetes.io/component=${component} --for=delete -n ${namespace} --timeout=2m`;
+        logger.info(`command: ${waitCmd}`);
+        await execAsync(waitCmd);
+      } catch (e) {
+        logger.verbose(`Wait for pod deletion ${component} skipped: ${e}`);
+      }
+    }
+    // Extra buffer to ensure PVC protection finalizers are cleared
+    await sleep(5 * 1000);
 
     // Now delete PVCs (they should no longer be in use)
     for (const component of pvcComponents) {
-      await deleteResourceByLabel({
-        resource: 'persistentvolumeclaims',
-        namespace: namespace,
-        label: `app.kubernetes.io/component=${component}`,
-      });
+      try {
+        await deleteResourceByLabel({
+          resource: 'persistentvolumeclaims',
+          namespace: namespace,
+          label: `app.kubernetes.io/component=${component}`,
+        });
+      } catch (e) {
+        logger.warn(`Failed to delete PVCs for ${component}: ${e}`);
+      }
     }
 
-    // Scale StatefulSets back up to original replica counts
-    for (const component of statefulSetComponents) {
-      const replicas = originalReplicas.get(component) ?? 1;
+    // Verify PVCs are deleted
+    for (const component of pvcComponents) {
       try {
-        const scaleCmd = `kubectl scale statefulset -l app.kubernetes.io/component=${component} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+        const waitCmd = `kubectl wait pvc -l app.kubernetes.io/component=${component} --for=delete -n ${namespace} --timeout=2m`;
+        logger.info(`command: ${waitCmd}`);
+        await execAsync(waitCmd);
+      } catch (e) {
+        logger.verbose(`Wait for PVC deletion ${component} skipped: ${e}`);
+      }
+    }
+
+    const haDbStatefulSets = [...originalReplicas.entries()].filter(([name]) => name.includes('validator-ha-db'));
+    const otherStatefulSets = [...originalReplicas.entries()].filter(([name]) => !name.includes('validator-ha-db'));
+
+    // Bring up HA DB first so we can run migrations before validators start
+    for (const [stsName, replicas] of haDbStatefulSets) {
+      try {
+        const scaleCmd = `kubectl scale statefulset ${stsName} -n ${namespace} --replicas=${replicas} --timeout=2m`;
         logger.info(`command: ${scaleCmd}`);
         await execAsync(scaleCmd);
       } catch (e) {
-        logger.verbose(`Scale up ${component} skipped: ${e}`);
+        logger.verbose(`Scale up ${stsName} skipped: ${e}`);
+      }
+    }
+
+    if (haDbStatefulSets.length > 0) {
+      try {
+        await waitForStatefulSetsReady({
+          namespace,
+          label: 'app.kubernetes.io/component=validator-ha-db',
+          timeoutSeconds: 600,
+        });
+        await initHADb(namespace);
+      } catch (e) {
+        logger.warn(`HA DB migration step skipped or failed: ${e}`);
+      }
+    }
+
+    // Scale remaining StatefulSets back up to original replica counts (by name, not label)
+    for (const [stsName, replicas] of otherStatefulSets) {
+      try {
+        const scaleCmd = `kubectl scale statefulset ${stsName} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+        logger.info(`command: ${scaleCmd}`);
+        await execAsync(scaleCmd);
+      } catch (e) {
+        logger.verbose(`Scale up ${stsName} skipped: ${e}`);
       }
     }
   } else {
@@ -312,8 +514,21 @@ export async function rollAztecPods(namespace: string, clearState: boolean = fal
 
   await sleep(10 * 1000);
 
-  // Wait for pods to come back
-  for (const component of podComponents) {
+  // Wait for StatefulSets to have all replicas ready.
+  for (const component of statefulSetComponents) {
+    try {
+      await waitForStatefulSetsReady({
+        namespace,
+        label: `app.kubernetes.io/component=${component}`,
+        timeoutSeconds: 600, // 10 minutes
+      });
+    } catch (e) {
+      logger.warn(`StatefulSet component ${component} may not be fully ready: ${e}`);
+    }
+  }
+
+  const nonStatefulSetComponents = podComponents.filter(c => !statefulSetComponents.includes(c));
+  for (const component of nonStatefulSetComponents) {
     await waitForResourceByLabel({
       resource: 'pods',
       namespace: namespace,