@aztec/end-to-end 0.0.1-commit.1bea0213 → 0.0.1-commit.217f559981

package/dest/spartan/utils/k8s.js

@@ -4,6 +4,7 @@ import { retryUntil } from '@aztec/foundation/retry';
 import { exec, spawn } from 'child_process';
 import path from 'path';
 import { promisify } from 'util';
+import { AlertTriggeredError, GrafanaClient } from '../../quality_of_service/grafana_client.js';
 const execAsync = promisify(exec);
 const logger = createLogger('e2e:k8s-utils');
 export async function startPortForward({ resource, namespace, containerPort, hostPort }) {
@@ -252,6 +253,129 @@ export async function waitForResourcesByName({ resource, names, namespace, condi
         }
     }));
 }
+/**
+ * Waits for all StatefulSets matching a label to have all their replicas ready.
+ *
+ * @param namespace - Kubernetes namespace
+ * @param label - Label selector for StatefulSets (e.g., "app.kubernetes.io/component=sequencer-node")
+ * @param timeoutSeconds - Maximum time to wait in seconds
+ * @param pollIntervalSeconds - How often to check status
+ */ export async function waitForStatefulSetsReady({ namespace, label, timeoutSeconds = 600, pollIntervalSeconds = 5 }) {
+    logger.info(`Waiting for StatefulSets with label ${label} to have all replicas ready (timeout: ${timeoutSeconds}s)`);
+    await retryUntil(async ()=>{
+        // Get all StatefulSets matching the label
+        const getCmd = `kubectl get statefulset -l ${label} -n ${namespace} -o json`;
+        const { stdout } = await execAsync(getCmd);
+        const result = JSON.parse(stdout);
+        if (!result.items || result.items.length === 0) {
+            logger.verbose(`No StatefulSets found with label ${label}`);
+            return false;
+        }
+        // Check each StatefulSet
+        for (const sts of result.items){
+            const name = sts.metadata.name;
+            const desired = sts.spec.replicas ?? 0;
+            const ready = sts.status.readyReplicas ?? 0;
+            const updated = sts.status.updatedReplicas ?? 0;
+            if (ready < desired || updated < desired) {
+                logger.verbose(`StatefulSet ${name}: ${ready}/${desired} ready, ${updated}/${desired} updated`);
+                return false;
+            }
+        }
+        logger.info(`All StatefulSets with label ${label} are ready`);
+        return true;
+    }, `StatefulSets with label ${label} to be ready`, timeoutSeconds, pollIntervalSeconds);
+}
+/**
+ * Creates a Prometheus connection that can re-establish port-forward on failure.
+ * Returns functions to connect and run alert checks that automatically reconnect if needed.
+ *
+ * @param namespace - K8s namespace to fall back to if metrics namespace doesn't have Prometheus
+ * @param endpoints - Array to track created endpoints for cleanup
+ * @param log - Logger instance
+ */ export function createResilientPrometheusConnection(namespace, endpoints, log) {
+    let alertChecker;
+    let currentEndpoint;
+    const connect = async ()=>{
+        // Kill existing connection if any
+        if (currentEndpoint?.process) {
+            currentEndpoint.process.kill();
+        }
+        // Try metrics namespace first, then network namespace
+        let promPort = 0;
+        let promUrl = '';
+        let promProc;
+        try {
+            const metricsResult = await startPortForward({
+                resource: `svc/metrics-prometheus-server`,
+                namespace: 'metrics',
+                containerPort: 80
+            });
+            promProc = metricsResult.process;
+            promPort = metricsResult.port;
+            promUrl = `http://127.0.0.1:${promPort}/api/v1`;
+        } catch  {
+            // Metrics namespace might not have Prometheus, try network namespace
+            log.verbose('Metrics namespace Prometheus not available, trying network namespace');
+        }
+        if (promPort === 0) {
+            const nsResult = await startPortForward({
+                resource: `svc/prometheus-server`,
+                namespace,
+                containerPort: 80
+            });
+            promProc = nsResult.process;
+            promPort = nsResult.port;
+            promUrl = `http://127.0.0.1:${promPort}/api/v1`;
+        }
+        if (!promProc || promPort === 0) {
+            throw new Error('Unable to port-forward to Prometheus');
+        }
+        currentEndpoint = {
+            url: promUrl,
+            process: promProc
+        };
+        endpoints.push(currentEndpoint);
+        alertChecker = new GrafanaClient(log, {
+            grafanaEndpoint: promUrl,
+            grafanaCredentials: ''
+        });
+        log.info(`Established Prometheus connection at ${promUrl}`);
+        return alertChecker;
+    };
+    const runAlertCheck = async (alerts)=>{
+        if (!alertChecker) {
+            alertChecker = await connect();
+        }
+        try {
+            await alertChecker.runAlertCheck(alerts);
+        } catch (err) {
+            // If it's an AlertTriggeredError (expected behavior)
+            if (err instanceof AlertTriggeredError) {
+                throw err;
+            }
+            // Check if it's a connection error (port-forward died)
+            const errorStr = String(err);
+            if (errorStr.includes('fetch failed') || errorStr.includes('ECONNREFUSED') || errorStr.includes('ECONNRESET')) {
+                log.warn(`Prometheus connection lost, re-establishing port-forward...`);
+                alertChecker = await connect();
+                await alertChecker.runAlertCheck(alerts);
+            } else {
+                throw err;
+            }
+        }
+    };
+    return {
+        connect,
+        runAlertCheck
+    };
+}
+/** Scales the prover-agent Deployment to the given number of replicas. */ export async function scaleProverAgents(namespace, replicas, log) {
+    const label = 'app.kubernetes.io/component=prover-agent';
+    const command = `kubectl scale deployment -l ${label} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+    log.info(`Scaling prover agents to ${replicas}: ${command}`);
+    await execAsync(command);
+}
 export function getChartDir(spartanDir, chartName) {
     return path.join(spartanDir.trim(), chartName);
 }

package/dest/spartan/utils/nodes.d.ts

@@ -4,17 +4,26 @@ import type { Logger } from '@aztec/foundation/log';
 import { type AztecNodeAdmin, type AztecNodeAdminConfig } from '@aztec/stdlib/interfaces/client';
 import type { TestConfig } from './config.js';
 export declare function awaitCheckpointNumber(rollupCheatCodes: RollupCheatCodes, checkpointNumber: CheckpointNumber, timeoutSeconds: number, log: Logger): Promise<void>;
+/**
+ * Waits until the proven block number increases.
+ *
+ * @param rpcUrl - URL of an Aztec RPC node to query
+ * @param log - Logger instance
+ * @param timeoutSeconds - Maximum time to wait
+ * @param pollIntervalSeconds - How often to check
+ */
+export declare function waitForProvenToAdvance(rpcUrl: string, log: Logger, timeoutSeconds?: number, pollIntervalSeconds?: number): Promise<void>;
 export declare function getSequencers(namespace: string): Promise<string[]>;
 export declare function updateSequencersConfig(env: TestConfig, config: Partial<AztecNodeAdminConfig>): Promise<AztecNodeAdminConfig[]>;
 export declare function getSequencersConfig(env: TestConfig): Promise<AztecNodeAdminConfig[]>;
 export declare function withSequencersAdmin<T>(env: TestConfig, fn: (node: AztecNodeAdmin) => Promise<T>): Promise<T[]>;
+export declare function initHADb(namespace: string): Promise<void>;
 /**
- * Enables or disables probabilistic transaction dropping on validators and waits for rollout.
- * Wired to env vars P2P_DROP_TX and P2P_DROP_TX_CHANCE via Helm values.
+ * Sets probabilistic transaction dropping on validators and waits for rollout.
+ * Use probability=0 to disable. Wired to env var P2P_DROP_TX_CHANCE via Helm values.
  */
-export declare function setValidatorTxDrop({ namespace, enabled, probability, logger: log }: {
+export declare function setValidatorTxDrop({ namespace, probability, logger: log }: {
     namespace: string;
-    enabled: boolean;
     probability: number;
     logger: Logger;
 }): Promise<void>;
@@ -28,4 +37,4 @@ export declare function enableValidatorDynamicBootNode(instanceName: string, nam
  *        Defaults to false, which preserves the existing storage.
  */
 export declare function rollAztecPods(namespace: string, clearState?: boolean): Promise<void>;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibm9kZXMuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zcGFydGFuL3V0aWxzL25vZGVzLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFDN0QsT0FBTyxLQUFLLEVBQUUsZ0JBQWdCLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUN4RSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUdwRCxPQUFPLEVBQ0wsS0FBSyxjQUFjLEVBQ25CLEtBQUssb0JBQW9CLEVBRTFCLE1BQU0saUNBQWlDLENBQUM7QUFLekMsT0FBTyxLQUFLLEVBQUUsVUFBVSxFQUFFLE1BQU0sYUFBYSxDQUFDO0FBUTlDLHdCQUFzQixxQkFBcUIsQ0FDekMsZ0JBQWdCLEVBQUUsZ0JBQWdCLEVBQ2xDLGdCQUFnQixFQUFFLGdCQUFnQixFQUNsQyxjQUFjLEVBQUUsTUFBTSxFQUN0QixHQUFHLEVBQUUsTUFBTSxpQkFlWjtBQUVELHdCQUFzQixhQUFhLENBQUMsU0FBUyxFQUFFLE1BQU0scUJBNEJwRDtBQUVELHdCQUFnQixzQkFBc0IsQ0FBQyxHQUFHLEVBQUUsVUFBVSxFQUFFLE1BQU0sRUFBRSxPQUFPLENBQUMsb0JBQW9CLENBQUMsbUNBSzVGO0FBRUQsd0JBQWdCLG1CQUFtQixDQUFDLEdBQUcsRUFBRSxVQUFVLG1DQUVsRDtBQUVELHdCQUFzQixtQkFBbUIsQ0FBQyxDQUFDLEVBQUUsR0FBRyxFQUFFLFVBQVUsRUFBRSxFQUFFLEVBQUUsQ0FBQyxJQUFJLEVBQUUsY0FBYyxLQUFLLE9BQU8sQ0FBQyxDQUFDLENBQUMsR0FBRyxPQUFPLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0EwQ3BIO0FBRUQ7OztHQUdHO0FBQ0gsd0JBQXNCLGtCQUFrQixDQUFDLEVBQ3ZDLFNBQVMsRUFDVCxPQUFPLEVBQ1AsV0FBVyxFQUNYLE1BQU0sRUFBRSxHQUFHLEVBQ1osRUFBRTtJQUNELFNBQVMsRUFBRSxNQUFNLENBQUM7SUFDbEIsT0FBTyxFQUFFLE9BQU8sQ0FBQztJQUNqQixXQUFXLEVBQUUsTUFBTSxDQUFDO0lBQ3BCLE1BQU0sRUFBRSxNQUFNLENBQUM7Q0FDaEIsaUJBZ0NBO0FBRUQsd0JBQXNCLGlCQUFpQixDQUFDLFNBQVMsRUFBRSxNQUFNLEVBQUUsR0FBRyxFQUFFLE1BQU0saUJBK0JyRTtBQUVELHdCQUFzQiw4QkFBOEIsQ0FDbEQsWUFBWSxFQUFFLE1BQU0sRUFDcEIsU0FBUyxFQUFFLE1BQU0sRUFDakIsVUFBVSxFQUFFLE1BQU0sRUFDbEIsR0FBRyxFQUFFLE1BQU0saUJBZ0JaO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsd0JBQXNCLGFBQWEsQ0FBQyxTQUFTLEVBQUUsTUFBTSxFQUFFLFVBQVUsR0FBRSxPQUFlLGlCQW1GakYifQ==
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibm9kZXMuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zcGFydGFuL3V0aWxzL25vZGVzLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUVBLE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFDN0QsT0FBTyxLQUFLLEVBQUUsZ0JBQWdCLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUN4RSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUdwRCxPQUFPLEVBQ0wsS0FBSyxjQUFjLEVBQ25CLEtBQUssb0JBQW9CLEVBRTFCLE1BQU0saUNBQWlDLENBQUM7QUFLekMsT0FBTyxLQUFLLEVBQUUsVUFBVSxFQUFFLE1BQU0sYUFBYSxDQUFDO0FBZTlDLHdCQUFzQixxQkFBcUIsQ0FDekMsZ0JBQWdCLEVBQUUsZ0JBQWdCLEVBQ2xDLGdCQUFnQixFQUFFLGdCQUFnQixFQUNsQyxjQUFjLEVBQUUsTUFBTSxFQUN0QixHQUFHLEVBQUUsTUFBTSxpQkFlWjtBQUVEOzs7Ozs7O0dBT0c7QUFDSCx3QkFBc0Isc0JBQXNCLENBQzFDLE1BQU0sRUFBRSxNQUFNLEVBQ2QsR0FBRyxFQUFFLE1BQU0sRUFDWCxjQUFjLEdBQUUsTUFBWSxFQUM1QixtQkFBbUIsR0FBRSxNQUFXLEdBQy9CLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0EwQ2Y7QUFFRCx3QkFBc0IsYUFBYSxDQUFDLFNBQVMsRUFBRSxNQUFNLHFCQTRCcEQ7QUFFRCx3QkFBZ0Isc0JBQXNCLENBQUMsR0FBRyxFQUFFLFVBQVUsRUFBRSxNQUFNLEVBQUUsT0FBTyxDQUFDLG9CQUFvQixDQUFDLG1DQUs1RjtBQUVELHdCQUFnQixtQkFBbUIsQ0FBQyxHQUFHLEVBQUUsVUFBVSxtQ0FFbEQ7QUFFRCx3QkFBc0IsbUJBQW1CLENBQUMsQ0FBQyxFQUFFLEdBQUcsRUFBRSxVQUFVLEVBQUUsRUFBRSxFQUFFLENBQUMsSUFBSSxFQUFFLGNBQWMsS0FBSyxPQUFPLENBQUMsQ0FBQyxDQUFDLEdBQUcsT0FBTyxDQUFDLENBQUMsRUFBRSxDQUFDLENBNENwSDtBQWtDRCx3QkFBc0IsUUFBUSxDQUFDLFNBQVMsRUFBRSxNQUFNLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQTBCL0Q7QUFFRDs7O0dBR0c7QUFDSCx3QkFBc0Isa0JBQWtCLENBQUMsRUFDdkMsU0FBUyxFQUNULFdBQVcsRUFDWCxNQUFNLEVBQUUsR0FBRyxFQUNaLEVBQUU7SUFDRCxTQUFTLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLFdBQVcsRUFBRSxNQUFNLENBQUM7SUFDcEIsTUFBTSxFQUFFLE1BQU0sQ0FBQztDQUNoQixpQkErQkE7QUFFRCx3QkFBc0IsaUJBQWlCLENBQUMsU0FBUyxFQUFFLE1BQU0sRUFBRSxHQUFHLEVBQUUsTUFBTSxpQkErQnJFO0FBRUQsd0JBQXNCLDhCQUE4QixDQUNsRCxZQUFZLEVBQUUsTUFBTSxFQUNwQixTQUFTLEVBQUUsTUFBTSxFQUNqQixVQUFVLEVBQUUsTUFBTSxFQUNsQixHQUFHLEVBQUUsTUFBTSxpQkFnQlo7QUFFRDs7Ozs7O0dBTUc7QUFDSCx3QkFBc0IsYUFBYSxDQUFDLFNBQVMsRUFBRSxNQUFNLEVBQUUsVUFBVSxHQUFFLE9BQWUsaUJBMktqRiJ9

package/dest/spartan/utils/nodes.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"nodes.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/nodes.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACxE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAGpD,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,oBAAoB,EAE1B,MAAM,iCAAiC,CAAC;AAKzC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAQ9C,wBAAsB,qBAAqB,CACzC,gBAAgB,EAAE,gBAAgB,EAClC,gBAAgB,EAAE,gBAAgB,EAClC,cAAc,EAAE,MAAM,EACtB,GAAG,EAAE,MAAM,iBAeZ;AAED,wBAAsB,aAAa,CAAC,SAAS,EAAE,MAAM,qBA4BpD;AAED,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,oBAAoB,CAAC,mCAK5F;AAED,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,UAAU,mCAElD;AAED,wBAAsB,mBAAmB,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CA0CpH;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,EACvC,SAAS,EACT,OAAO,EACP,WAAW,EACX,MAAM,EAAE,GAAG,EACZ,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB,iBAgCA;AAED,wBAAsB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,iBA+BrE;AAED,wBAAsB,8BAA8B,CAClD,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,MAAM,iBAgBZ;AAED;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,GAAE,OAAe,iBAmFjF"}
+{"version":3,"file":"nodes.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/nodes.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACxE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAGpD,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,oBAAoB,EAE1B,MAAM,iCAAiC,CAAC;AAKzC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAe9C,wBAAsB,qBAAqB,CACzC,gBAAgB,EAAE,gBAAgB,EAClC,gBAAgB,EAAE,gBAAgB,EAClC,cAAc,EAAE,MAAM,EACtB,GAAG,EAAE,MAAM,iBAeZ;AAED;;;;;;;GAOG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,EACX,cAAc,GAAE,MAAY,EAC5B,mBAAmB,GAAE,MAAW,GAC/B,OAAO,CAAC,IAAI,CAAC,CA0Cf;AAED,wBAAsB,aAAa,CAAC,SAAS,EAAE,MAAM,qBA4BpD;AAED,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,oBAAoB,CAAC,mCAK5F;AAED,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,UAAU,mCAElD;AAED,wBAAsB,mBAAmB,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CA4CpH;AAkCD,wBAAsB,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA0B/D;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,EACvC,SAAS,EACT,WAAW,EACX,MAAM,EAAE,GAAG,EACZ,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB,iBA+BA;AAED,wBAAsB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,iBA+BrE;AAED,wBAAsB,8BAA8B,CAClD,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,MAAM,iBAgBZ;AAED;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,GAAE,OAAe,iBA2KjF"}

package/dest/spartan/utils/nodes.js

@@ -1,11 +1,12 @@
 import { createLogger } from '@aztec/aztec.js/log';
-import { makeBackoff, retry } from '@aztec/foundation/retry';
+import { createAztecNodeClient } from '@aztec/aztec.js/node';
+import { makeBackoff, retry, retryUntil } from '@aztec/foundation/retry';
 import { sleep } from '@aztec/foundation/sleep';
 import { createAztecNodeAdminClient } from '@aztec/stdlib/interfaces/client';
 import { exec } from 'child_process';
 import { promisify } from 'util';
 import { execHelmCommand } from './helm.js';
-import { deleteResourceByLabel, getChartDir, startPortForward, waitForResourceByLabel } from './k8s.js';
+import { deleteResourceByLabel, getChartDir, startPortForward, waitForResourceByLabel, waitForResourceByName, waitForStatefulSetsReady } from './k8s.js';
 const execAsync = promisify(exec);
 const logger = createLogger('e2e:k8s-utils');
 export async function awaitCheckpointNumber(rollupCheatCodes, checkpointNumber, timeoutSeconds, log) {
@@ -23,6 +24,43 @@ export async function awaitCheckpointNumber(rollupCheatCodes, checkpointNumber,
         log.info(`Reached checkpoint ${tips.pending}`);
     }
 }
+/**
+ * Waits until the proven block number increases.
+ *
+ * @param rpcUrl - URL of an Aztec RPC node to query
+ * @param log - Logger instance
+ * @param timeoutSeconds - Maximum time to wait
+ * @param pollIntervalSeconds - How often to check
+ */ export async function waitForProvenToAdvance(rpcUrl, log, timeoutSeconds = 300, pollIntervalSeconds = 12) {
+    const node = createAztecNodeClient(rpcUrl);
+    log.info('Waiting for proven block to advance (indicating epoch proof just submitted)...');
+    // Get current proven block number
+    let initialProvenBlock;
+    try {
+        const tips = await node.getL2Tips();
+        initialProvenBlock = Number(tips.proven.block.number);
+        log.info(`Current proven block: ${initialProvenBlock}. Waiting for it to increase...`);
+    } catch (err) {
+        log.warn(`Error getting initial tips: ${err}. Will poll until successful.`);
+        initialProvenBlock = 0;
+    }
+    await retryUntil(async ()=>{
+        try {
+            const tips = await node.getL2Tips();
+            const currentProvenBlock = Number(tips.proven.block.number);
+            const proposedBlock = Number(tips.proposed.number);
+            log.verbose(`Chain state: proposed=${proposedBlock}, proven=${currentProvenBlock} (waiting for > ${initialProvenBlock})`);
+            if (currentProvenBlock > initialProvenBlock) {
+                log.info(`Proven block advanced from ${initialProvenBlock} to ${currentProvenBlock}.`);
+                return true;
+            }
+            return false;
+        } catch (err) {
+            log.verbose(`Error checking tips: ${err}`);
+            return false;
+        }
+    }, 'proven block to advance', timeoutSeconds, pollIntervalSeconds);
+}
 export async function getSequencers(namespace) {
     const selectors = [
         'app.kubernetes.io/name=validator',
@@ -61,9 +99,15 @@ export async function withSequencersAdmin(env, fn) {
     const sequencers = await getSequencers(namespace);
     const results = [];
     for (const sequencer of sequencers){
+        // Ensure pod is Ready before attempting port-forward.
+        await waitForResourceByName({
+            resource: 'pods',
+            name: sequencer,
+            namespace
+        });
         // Wrap port-forward + fetch in a retry to handle flaky port-forwards
         const result = await retry(async ()=>{
-            const { process, port } = await startPortForward({
+            const { process: process1, port } = await startPortForward({
                 resource: `pod/${sequencer}`,
                 namespace,
                 containerPort: adminContainerPort
@@ -78,11 +122,11 @@ export async function withSequencersAdmin(env, fn) {
                 const client = createAztecNodeAdminClient(url);
                 return {
                     result: await fn(client),
-                    process
+                    process: process1
                 };
             } catch (err) {
                 // Kill the port-forward before retrying
-                process.kill();
+                process1.kill();
                 throw err;
             }
         }, 'connect to node admin', makeBackoff([
@@ -96,11 +140,62 @@ export async function withSequencersAdmin(env, fn) {
     }
     return results;
 }
+async function getAztecImageForMigrations(namespace) {
+    const aztecDockerImage = process.env.AZTEC_DOCKER_IMAGE;
+    if (aztecDockerImage) {
+        return aztecDockerImage;
+    }
+    const { stdout } = await execAsync(`kubectl get pods -l app.kubernetes.io/name=validator -n ${namespace} -o jsonpath='{.items[0].spec.containers[?(@.name=="aztec")].image}' | cat`);
+    const image = stdout.trim().replace(/^'|'$/g, '');
+    if (!image) {
+        throw new Error(`Could not detect aztec image from validator pod in namespace ${namespace}`);
+    }
+    return image;
+}
+async function getHaDbConnectionUrl(namespace) {
+    const secretName = `${namespace}-validator-ha-db-postgres`;
+    const { stdout } = await execAsync(`kubectl get secret ${secretName} -n ${namespace} -o json`);
+    const secret = JSON.parse(stdout);
+    const data = secret?.data ?? {};
+    const decode = (value)=>value ? Buffer.from(value, 'base64').toString('utf8') : '';
+    const user = decode(data.POSTGRES_USER);
+    const password = decode(data.POSTGRES_PASSWORD);
+    const database = decode(data.POSTGRES_DB);
+    if (!user || !password || !database) {
+        throw new Error(`Missing HA DB credentials in secret ${secretName}`);
+    }
+    const host = `${namespace}-validator-ha-db-postgres.${namespace}.svc.cluster.local`;
+    return `postgresql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${host}:5432/${database}`;
+}
+export async function initHADb(namespace) {
+    const databaseUrl = await getHaDbConnectionUrl(namespace);
+    const image = await getAztecImageForMigrations(namespace);
+    const jobName = `${namespace}-validator-ha-db-migrate`;
+    await execAsync(`kubectl delete pod ${jobName} -n ${namespace} --ignore-not-found=true`).catch(()=>undefined);
+    const migrateCmd = [
+        `kubectl run ${jobName} -n ${namespace}`,
+        '--rm -i',
+        '--restart=Never',
+        `--image=${image}`,
+        `--env=DATABASE_URL=${databaseUrl}`,
+        '--command -- node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js migrate-ha-db up'
+    ].join(' ');
+    const migrateCmdForLog = migrateCmd.replace(/--env=DATABASE_URL=\S+/, '--env=DATABASE_URL=<redacted>');
+    await retry(async ()=>{
+        logger.info(`command: ${migrateCmdForLog}`);
+        await execAsync(migrateCmd);
+    }, 'run HA DB migrations', makeBackoff([
+        1,
+        2,
+        4,
+        8,
+        16
+    ]), logger, true);
+}
 /**
- * Enables or disables probabilistic transaction dropping on validators and waits for rollout.
- * Wired to env vars P2P_DROP_TX and P2P_DROP_TX_CHANCE via Helm values.
- */ export async function setValidatorTxDrop({ namespace, enabled, probability, logger: log }) {
-    const drop = enabled ? 'true' : 'false';
+ * Sets probabilistic transaction dropping on validators and waits for rollout.
+ * Use probability=0 to disable. Wired to env var P2P_DROP_TX_CHANCE via Helm values.
+ */ export async function setValidatorTxDrop({ namespace, probability, logger: log }) {
     const prob = String(probability);
     const selectors = [
         'app.kubernetes.io/name=validator',
@@ -115,7 +210,7 @@ export async function withSequencersAdmin(env, fn) {
             if (names.length === 0) {
                 continue;
             }
-            const cmd = `kubectl set env statefulset -l ${selector} -n ${namespace} P2P_DROP_TX=${drop} P2P_DROP_TX_CHANCE=${prob}`;
+            const cmd = `kubectl set env statefulset -l ${selector} -n ${namespace} P2P_DROP_TX_CHANCE=${prob}`;
             log.info(`command: ${cmd}`);
             await execAsync(cmd);
             updated = true;
@@ -201,14 +296,16 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
         'prover-broker',
         'prover-agent',
         'sequencer-node',
-        'rpc'
+        'rpc',
+        'validator-ha-db'
     ];
     const pvcComponents = [
         'p2p-bootstrap',
         'prover-node',
         'prover-broker',
         'sequencer-node',
-        'rpc'
+        'rpc',
+        'validator-ha-db'
     ];
     // StatefulSet components that need to be scaled down before PVC deletion
     // Note: validators use 'sequencer-node' as component label, not 'validator'
@@ -217,20 +314,27 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
         'prover-node',
         'prover-broker',
         'sequencer-node',
-        'rpc'
+        'rpc',
+        'validator-ha-db'
     ];
     if (clearState) {
         // To delete PVCs, we must first scale down StatefulSets so pods release the volumes
         // Otherwise PVC deletion will hang waiting for pods to terminate
-        // First, save original replica counts
+        // Save original replica counts for all StatefulSets
         const originalReplicas = new Map();
         for (const component of statefulSetComponents){
             try {
-                const getCmd = `kubectl get statefulset -l app.kubernetes.io/component=${component} -n ${namespace} -o jsonpath='{.items[0].spec.replicas}'`;
+                // Get all StatefulSets that match the component label
+                const getCmd = `kubectl get statefulset -l app.kubernetes.io/component=${component} -n ${namespace} -o json`;
                 const { stdout } = await execAsync(getCmd);
-                const replicas = parseInt(stdout.replace(/'/g, '').trim(), 10);
-                if (!isNaN(replicas) && replicas > 0) {
-                    originalReplicas.set(component, replicas);
+                const result = JSON.parse(stdout);
+                for (const sts of result.items || []){
+                    const name = sts.metadata.name;
+                    const replicas = sts.spec.replicas ?? 1;
+                    if (replicas > 0) {
+                        originalReplicas.set(name, replicas);
+                        logger.debug(`Saved replica count for StatefulSet ${name}: ${replicas}`);
+                    }
                 }
             } catch  {
             // Component might not exist, continue
@@ -247,25 +351,79 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
                 logger.verbose(`Scale down ${component} skipped: ${e}`);
             }
         }
-        // Wait for pods to terminate
-        await sleep(15 * 1000);
+        // Wait for all pods to fully terminate before deleting PVCs.
+        // terminationGracePeriodSeconds default is 30s.
+        logger.info('Waiting for pods to fully terminate before deleting PVCs...');
+        for (const component of statefulSetComponents){
+            try {
+                // Wait for all pods with this component label to be deleted
+                const waitCmd = `kubectl wait pods -l app.kubernetes.io/component=${component} --for=delete -n ${namespace} --timeout=2m`;
+                logger.info(`command: ${waitCmd}`);
+                await execAsync(waitCmd);
+            } catch (e) {
+                logger.verbose(`Wait for pod deletion ${component} skipped: ${e}`);
+            }
+        }
+        // Extra buffer to ensure PVC protection finalizers are cleared
+        await sleep(5 * 1000);
         // Now delete PVCs (they should no longer be in use)
         for (const component of pvcComponents){
-            await deleteResourceByLabel({
-                resource: 'persistentvolumeclaims',
-                namespace: namespace,
-                label: `app.kubernetes.io/component=${component}`
-            });
+            try {
+                await deleteResourceByLabel({
+                    resource: 'persistentvolumeclaims',
+                    namespace: namespace,
+                    label: `app.kubernetes.io/component=${component}`
+                });
+            } catch (e) {
+                logger.warn(`Failed to delete PVCs for ${component}: ${e}`);
+            }
         }
-        // Scale StatefulSets back up to original replica counts
-        for (const component of statefulSetComponents){
-            const replicas = originalReplicas.get(component) ?? 1;
+        // Verify PVCs are deleted
+        for (const component of pvcComponents){
+            try {
+                const waitCmd = `kubectl wait pvc -l app.kubernetes.io/component=${component} --for=delete -n ${namespace} --timeout=2m`;
+                logger.info(`command: ${waitCmd}`);
+                await execAsync(waitCmd);
+            } catch (e) {
+                logger.verbose(`Wait for PVC deletion ${component} skipped: ${e}`);
+            }
+        }
+        const haDbStatefulSets = [
+            ...originalReplicas.entries()
+        ].filter(([name])=>name.includes('validator-ha-db'));
+        const otherStatefulSets = [
+            ...originalReplicas.entries()
+        ].filter(([name])=>!name.includes('validator-ha-db'));
+        // Bring up HA DB first so we can run migrations before validators start
+        for (const [stsName, replicas] of haDbStatefulSets){
+            try {
+                const scaleCmd = `kubectl scale statefulset ${stsName} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+                logger.info(`command: ${scaleCmd}`);
+                await execAsync(scaleCmd);
+            } catch (e) {
+                logger.verbose(`Scale up ${stsName} skipped: ${e}`);
+            }
+        }
+        if (haDbStatefulSets.length > 0) {
+            try {
+                await waitForStatefulSetsReady({
+                    namespace,
+                    label: 'app.kubernetes.io/component=validator-ha-db',
+                    timeoutSeconds: 600
+                });
+                await initHADb(namespace);
+            } catch (e) {
+                logger.warn(`HA DB migration step skipped or failed: ${e}`);
+            }
+        }
+        // Scale remaining StatefulSets back up to original replica counts (by name, not label)
+        for (const [stsName, replicas] of otherStatefulSets){
             try {
-                const scaleCmd = `kubectl scale statefulset -l app.kubernetes.io/component=${component} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+                const scaleCmd = `kubectl scale statefulset ${stsName} -n ${namespace} --replicas=${replicas} --timeout=2m`;
                 logger.info(`command: ${scaleCmd}`);
                 await execAsync(scaleCmd);
             } catch (e) {
-                logger.verbose(`Scale up ${component} skipped: ${e}`);
+                logger.verbose(`Scale up ${stsName} skipped: ${e}`);
             }
         }
     } else {
@@ -279,8 +437,20 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
         }
     }
     await sleep(10 * 1000);
-    // Wait for pods to come back
-    for (const component of podComponents){
+    // Wait for StatefulSets to have all replicas ready.
+    for (const component of statefulSetComponents){
+        try {
+            await waitForStatefulSetsReady({
+                namespace,
+                label: `app.kubernetes.io/component=${component}`,
+                timeoutSeconds: 600
+            });
+        } catch (e) {
+            logger.warn(`StatefulSet component ${component} may not be fully ready: ${e}`);
+        }
+    }
+    const nonStatefulSetComponents = podComponents.filter((c)=>!statefulSetComponents.includes(c));
+    for (const component of nonStatefulSetComponents){
         await waitForResourceByLabel({
             resource: 'pods',
             namespace: namespace,

package/dest/spartan/utils/scripts.d.ts

@@ -1,4 +1,10 @@
 import type { Logger } from '@aztec/foundation/log';
+/** Result from running a script */
+export type ScriptResult = {
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+};
 /**
  * Returns the absolute path to the git repository root
  */
@@ -9,8 +15,16 @@ export declare function getAztecBin(): string;
  * @param args - The arguments to pass to the Aztec binary
  * @param logger - The logger to use
  * @param env - Optional environment variables to set for the process
- * @returns The exit code of the Aztec binary
+ * @returns The exit code, stdout, and stderr of the Aztec binary
  */
-export declare function runAztecBin(args: string[], logger: Logger, env?: Record<string, string>): Promise<number>;
-export declare function runProjectScript(script: string, args: string[], logger: Logger, env?: Record<string, string>): Promise<number>;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2NyaXB0cy5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL3NwYXJ0YW4vdXRpbHMvc2NyaXB0cy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQTRCcEQ7O0dBRUc7QUFDSCx3QkFBZ0IsaUJBQWlCLElBQUksTUFBTSxDQVcxQztBQUVELHdCQUFnQixXQUFXLFdBRTFCO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsd0JBQWdCLFdBQVcsQ0FBQyxJQUFJLEVBQUUsTUFBTSxFQUFFLEVBQUUsTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLENBQUMsRUFBRSxNQUFNLENBQUMsTUFBTSxFQUFFLE1BQU0sQ0FBQyxtQkFFdkY7QUFFRCx3QkFBZ0IsZ0JBQWdCLENBQUMsTUFBTSxFQUFFLE1BQU0sRUFBRSxJQUFJLEVBQUUsTUFBTSxFQUFFLEVBQUUsTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLENBQUMsRUFBRSxNQUFNLENBQUMsTUFBTSxFQUFFLE1BQU0sQ0FBQyxtQkFHNUcifQ==
+export declare function runAztecBin(args: string[], logger: Logger, env?: Record<string, string>): Promise<ScriptResult>;
+/**
+ * Runs a script from the project root
+ * @param script - The path to the script, relative to the project root
+ * @param args - The arguments to pass to the script
+ * @param logger - The logger to use
+ * @param env - Optional environment variables to set for the process
+ * @returns The exit code, stdout, and stderr of the script
+ */
+export declare function runProjectScript(script: string, args: string[], logger: Logger, env?: Record<string, string>): Promise<ScriptResult>;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2NyaXB0cy5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL3NwYXJ0YW4vdXRpbHMvc2NyaXB0cy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUtwRCxtQ0FBbUM7QUFDbkMsTUFBTSxNQUFNLFlBQVksR0FBRztJQUN6QixRQUFRLEVBQUUsTUFBTSxDQUFDO0lBQ2pCLE1BQU0sRUFBRSxNQUFNLENBQUM7SUFDZixNQUFNLEVBQUUsTUFBTSxDQUFDO0NBQ2hCLENBQUM7QUF5Q0Y7O0dBRUc7QUFDSCx3QkFBZ0IsaUJBQWlCLElBQUksTUFBTSxDQVcxQztBQUVELHdCQUFnQixXQUFXLFdBRTFCO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsd0JBQWdCLFdBQVcsQ0FBQyxJQUFJLEVBQUUsTUFBTSxFQUFFLEVBQUUsTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLENBQUMsRUFBRSxNQUFNLENBQUMsTUFBTSxFQUFFLE1BQU0sQ0FBQyxHQUFHLE9BQU8sQ0FBQyxZQUFZLENBQUMsQ0FFL0c7QUFFRDs7Ozs7OztHQU9HO0FBQ0gsd0JBQWdCLGdCQUFnQixDQUM5QixNQUFNLEVBQUUsTUFBTSxFQUNkLElBQUksRUFBRSxNQUFNLEVBQUUsRUFDZCxNQUFNLEVBQUUsTUFBTSxFQUNkLEdBQUcsQ0FBQyxFQUFFLE1BQU0sQ0FBQyxNQUFNLEVBQUUsTUFBTSxDQUFDLEdBQzNCLE9BQU8sQ0FBQyxZQUFZLENBQUMsQ0FHdkIifQ==

package/dest/spartan/utils/scripts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scripts.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/scripts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AA4BpD;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,MAAM,CAW1C;AAED,wBAAgB,WAAW,WAE1B;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,mBAEvF;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,mBAG5G"}
+{"version":3,"file":"scripts.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/scripts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAKpD,mCAAmC;AACnC,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAyCF;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,MAAM,CAW1C;AAED,wBAAgB,WAAW,WAE1B;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC,CAE/G;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,MAAM,EACd,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC3B,OAAO,CAAC,YAAY,CAAC,CAGvB"}

package/dest/spartan/utils/scripts.js

@@ -4,7 +4,7 @@ import path from 'path';
  * @param scriptPath - The path to the script, relative to the project root
  * @param args - The arguments to pass to the script
  * @param logger - The logger to use
- * @returns The exit code of the script
+ * @returns The exit code, stdout, and stderr of the script
  */ function runScript(scriptPath, args, logger, env) {
     const childProcess = spawn(scriptPath, args, {
         stdio: [
@@ -17,13 +17,21 @@ import path from 'path';
             ...env
         } : process.env
     });
+    const stdoutChunks = [];
+    const stderrChunks = [];
     return new Promise((resolve, reject)=>{
-        childProcess.on('close', (code)=>resolve(code ?? 0));
+        childProcess.on('close', (code)=>resolve({
+                exitCode: code ?? 0,
+                stdout: Buffer.concat(stdoutChunks).toString(),
+                stderr: Buffer.concat(stderrChunks).toString()
+            }));
         childProcess.on('error', reject);
         childProcess.stdout?.on('data', (data)=>{
+            stdoutChunks.push(data);
             logger.info(data.toString());
         });
         childProcess.stderr?.on('data', (data)=>{
+            stderrChunks.push(data);
             logger.error(data.toString());
         });
     });
@@ -53,14 +61,21 @@ export function getAztecBin() {
  * @param args - The arguments to pass to the Aztec binary
  * @param logger - The logger to use
  * @param env - Optional environment variables to set for the process
- * @returns The exit code of the Aztec binary
+ * @returns The exit code, stdout, and stderr of the Aztec binary
  */ export function runAztecBin(args, logger, env) {
     return runScript('node', [
         getAztecBin(),
         ...args
     ], logger, env);
 }
-export function runProjectScript(script, args, logger, env) {
+/**
+ * Runs a script from the project root
+ * @param script - The path to the script, relative to the project root
+ * @param args - The arguments to pass to the script
+ * @param logger - The logger to use
+ * @param env - Optional environment variables to set for the process
+ * @returns The exit code, stdout, and stderr of the script
+ */ export function runProjectScript(script, args, logger, env) {
     const scriptPath = script.startsWith('/') ? script : path.join(getGitProjectRoot(), script);
     return runScript(scriptPath, args, logger, env);
 }