@aztec/end-to-end 4.0.0-nightly.20260121 → 4.0.0-nightly.20260123

package/dest/spartan/utils/helm.js

@@ -0,0 +1,47 @@
+import { createLogger } from '@aztec/aztec.js/log';
+import { exec } from 'child_process';
+import { promisify } from 'util';
+const execAsync = promisify(exec);
+const logger = createLogger('e2e:k8s-utils');
+function shellQuote(value) {
+    // Single-quote safe shell escaping: ' -> '\''
+    return `'${value.replace(/'/g, "'\\''")}'`;
+}
+function valuesToArgs(values) {
+    return Object.entries(values).map(([key, value])=>typeof value === 'number' || typeof value === 'boolean' ? `--set ${key}=${value}` : `--set-string ${key}=${shellQuote(String(value))}`).join(' ');
+}
+export function createHelmCommand({ instanceName, helmChartDir, namespace, valuesFile, timeout, values, reuseValues = false }) {
+    const valuesFileArgs = valuesFile ? `--values ${helmChartDir}/values/${valuesFile}` : '';
+    const reuseValuesArgs = reuseValues ? '--reuse-values' : '';
+    return `helm upgrade --install ${instanceName} ${helmChartDir} --namespace ${namespace} ${valuesFileArgs} ${reuseValuesArgs} --wait --timeout=${timeout} ${valuesToArgs(values)}`;
+}
+export async function execHelmCommand(args) {
+    const helmCommand = createHelmCommand(args);
+    logger.info(`helm command: ${helmCommand}`);
+    const { stdout } = await execAsync(helmCommand);
+    return stdout;
+}
+export async function getHelmReleaseStatus(instanceName, namespace) {
+    try {
+        const { stdout } = await execAsync(`helm list --namespace ${namespace} --all --filter '^${instanceName}$' --output json | cat`);
+        const parsed = JSON.parse(stdout);
+        const row = parsed.find((r)=>r.name === instanceName);
+        return row?.status;
+    } catch  {
+        return undefined;
+    }
+}
+export async function forceDeleteHelmReleaseRecord(instanceName, namespace, log) {
+    const labelSelector = `owner=helm,name=${instanceName}`;
+    const cmd = `kubectl delete secret -n ${namespace} -l ${labelSelector} --ignore-not-found=true`;
+    (log ?? logger).warn(`Force deleting Helm release record: ${cmd}`);
+    await execAsync(cmd).catch(()=>undefined);
+}
+export async function hasDeployedHelmRelease(instanceName, namespace) {
+    try {
+        const status = await getHelmReleaseStatus(instanceName, namespace);
+        return status?.toLowerCase() === 'deployed';
+    } catch  {
+        return false;
+    }
+}

package/dest/spartan/utils/index.d.ts

@@ -0,0 +1,9 @@
+export { type TestConfig, setupEnvironment } from './config.js';
+export { getGitProjectRoot, getAztecBin, runAztecBin, runProjectScript } from './scripts.js';
+export { startPortForward, getExternalIP, startPortForwardForPrometeheus, startPortForwardForRPC, startPortForwardForEthereum, deleteResourceByName, deleteResourceByLabel, waitForResourceByLabel, waitForResourceByName, waitForResourcesByName, getChartDir, } from './k8s.js';
+export { uninstallChaosMesh, installChaosMeshChart, applyProverFailure, applyValidatorFailure, applyProverKill, applyProverBrokerKill, applyBootNodeFailure, applyValidatorKill, applyNetworkShaping, } from './chaos.js';
+export { restartBot, installTransferBot, uninstallTransferBot } from './bot.js';
+export { awaitCheckpointNumber, getSequencers, updateSequencersConfig, getSequencersConfig, withSequencersAdmin, setValidatorTxDrop, restartValidators, enableValidatorDynamicBootNode, rollAztecPods, } from './nodes.js';
+export { getPublicViemClient, getL1DeploymentAddresses, getNodeClient } from './clients.js';
+export { ChainHealth, type ChainHealthSnapshot } from './health.js';
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zcGFydGFuL3V0aWxzL2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUlBLE9BQU8sRUFBRSxLQUFLLFVBQVUsRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLGFBQWEsQ0FBQztBQUdoRSxPQUFPLEVBQUUsaUJBQWlCLEVBQUUsV0FBVyxFQUFFLFdBQVcsRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLGNBQWMsQ0FBQztBQUc3RixPQUFPLEVBQ0wsZ0JBQWdCLEVBQ2hCLGFBQWEsRUFDYiw4QkFBOEIsRUFDOUIsc0JBQXNCLEVBQ3RCLDJCQUEyQixFQUMzQixvQkFBb0IsRUFDcEIscUJBQXFCLEVBQ3JCLHNCQUFzQixFQUN0QixxQkFBcUIsRUFDckIsc0JBQXNCLEVBQ3RCLFdBQVcsR0FDWixNQUFNLFVBQVUsQ0FBQztBQUdsQixPQUFPLEVBQ0wsa0JBQWtCLEVBQ2xCLHFCQUFxQixFQUNyQixrQkFBa0IsRUFDbEIscUJBQXFCLEVBQ3JCLGVBQWUsRUFDZixxQkFBcUIsRUFDckIsb0JBQW9CLEVBQ3BCLGtCQUFrQixFQUNsQixtQkFBbUIsR0FDcEIsTUFBTSxZQUFZLENBQUM7QUFHcEIsT0FBTyxFQUFFLFVBQVUsRUFBRSxrQkFBa0IsRUFBRSxvQkFBb0IsRUFBRSxNQUFNLFVBQVUsQ0FBQztBQUdoRixPQUFPLEVBQ0wscUJBQXFCLEVBQ3JCLGFBQWEsRUFDYixzQkFBc0IsRUFDdEIsbUJBQW1CLEVBQ25CLG1CQUFtQixFQUNuQixrQkFBa0IsRUFDbEIsaUJBQWlCLEVBQ2pCLDhCQUE4QixFQUM5QixhQUFhLEdBQ2QsTUFBTSxZQUFZLENBQUM7QUFHcEIsT0FBTyxFQUFFLG1CQUFtQixFQUFFLHdCQUF3QixFQUFFLGFBQWEsRUFBRSxNQUFNLGNBQWMsQ0FBQztBQUc1RixPQUFPLEVBQUUsV0FBVyxFQUFFLEtBQUssbUJBQW1CLEVBQUUsTUFBTSxhQUFhLENBQUMifQ==

package/dest/spartan/utils/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,UAAU,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAGhE,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAG7F,OAAO,EACL,gBAAgB,EAChB,aAAa,EACb,8BAA8B,EAC9B,sBAAsB,EACtB,2BAA2B,EAC3B,oBAAoB,EACpB,qBAAqB,EACrB,sBAAsB,EACtB,qBAAqB,EACrB,sBAAsB,EACtB,WAAW,GACZ,MAAM,UAAU,CAAC;AAGlB,OAAO,EACL,kBAAkB,EAClB,qBAAqB,EACrB,kBAAkB,EAClB,qBAAqB,EACrB,eAAe,EACf,qBAAqB,EACrB,oBAAoB,EACpB,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAGhF,OAAO,EACL,qBAAqB,EACrB,aAAa,EACb,sBAAsB,EACtB,mBAAmB,EACnB,mBAAmB,EACnB,kBAAkB,EAClB,iBAAiB,EACjB,8BAA8B,EAC9B,aAAa,GACd,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAG5F,OAAO,EAAE,WAAW,EAAE,KAAK,mBAAmB,EAAE,MAAM,aAAa,CAAC"}

package/dest/spartan/utils/index.js

@@ -0,0 +1,18 @@
+// Re-export all utilities from individual modules
+// Note: Internal helpers (helm commands, etc.) are not re-exported to maintain the original public API
+// Config
+export { setupEnvironment } from './config.js';
+// Scripts
+export { getGitProjectRoot, getAztecBin, runAztecBin, runProjectScript } from './scripts.js';
+// K8s operations
+export { startPortForward, getExternalIP, startPortForwardForPrometeheus, startPortForwardForRPC, startPortForwardForEthereum, deleteResourceByName, deleteResourceByLabel, waitForResourceByLabel, waitForResourceByName, waitForResourcesByName, getChartDir } from './k8s.js';
+// Chaos Mesh
+export { uninstallChaosMesh, installChaosMeshChart, applyProverFailure, applyValidatorFailure, applyProverKill, applyProverBrokerKill, applyBootNodeFailure, applyValidatorKill, applyNetworkShaping } from './chaos.js';
+// Bot management
+export { restartBot, installTransferBot, uninstallTransferBot } from './bot.js';
+// Node operations (sequencers, validators, pods)
+export { awaitCheckpointNumber, getSequencers, updateSequencersConfig, getSequencersConfig, withSequencersAdmin, setValidatorTxDrop, restartValidators, enableValidatorDynamicBootNode, rollAztecPods } from './nodes.js';
+// Client utilities
+export { getPublicViemClient, getL1DeploymentAddresses, getNodeClient } from './clients.js';
+// Health checks
+export { ChainHealth } from './health.js';

package/dest/spartan/utils/k8s.d.ts

@@ -0,0 +1,59 @@
+import { type ChildProcess } from 'child_process';
+export declare function startPortForward({ resource, namespace, containerPort, hostPort }: {
+    resource: string;
+    namespace: string;
+    containerPort: number;
+    hostPort?: number;
+}): Promise<{
+    process: ChildProcess;
+    port: number;
+}>;
+export declare function getExternalIP(namespace: string, serviceName: string): Promise<string>;
+export declare function startPortForwardForPrometeheus(namespace: string): Promise<{
+    process: ChildProcess;
+    port: number;
+}>;
+export declare function startPortForwardForRPC(namespace: string, index?: number): Promise<{
+    process: ChildProcess;
+    port: number;
+}>;
+export declare function startPortForwardForEthereum(namespace: string): Promise<{
+    process: ChildProcess;
+    port: number;
+}>;
+export declare function deleteResourceByName({ resource, namespace, name, force }: {
+    resource: string;
+    namespace: string;
+    name: string;
+    force?: boolean;
+}): Promise<string>;
+export declare function deleteResourceByLabel({ resource, namespace, label, timeout, force }: {
+    resource: string;
+    namespace: string;
+    label: string;
+    timeout?: string;
+    force?: boolean;
+}): Promise<string>;
+export declare function waitForResourceByLabel({ resource, label, namespace, condition, timeout }: {
+    resource: string;
+    label: string;
+    namespace: string;
+    condition?: string;
+    timeout?: string;
+}): Promise<string>;
+export declare function waitForResourceByName({ resource, name, namespace, condition, timeout }: {
+    resource: string;
+    name: string;
+    namespace: string;
+    condition?: string;
+    timeout?: string;
+}): Promise<string>;
+export declare function waitForResourcesByName({ resource, names, namespace, condition, timeout }: {
+    resource: string;
+    names: string[];
+    namespace: string;
+    condition?: string;
+    timeout?: string;
+}): Promise<void>;
+export declare function getChartDir(spartanDir: string, chartName: string): string;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiazhzLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc3BhcnRhbi91dGlscy9rOHMudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBR0EsT0FBTyxFQUFFLEtBQUssWUFBWSxFQUFlLE1BQU0sZUFBZSxDQUFDO0FBUS9ELHdCQUFzQixnQkFBZ0IsQ0FBQyxFQUNyQyxRQUFRLEVBQ1IsU0FBUyxFQUNULGFBQWEsRUFDYixRQUFRLEVBQ1QsRUFBRTtJQUNELFFBQVEsRUFBRSxNQUFNLENBQUM7SUFDakIsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixhQUFhLEVBQUUsTUFBTSxDQUFDO0lBRXRCLFFBQVEsQ0FBQyxFQUFFLE1BQU0sQ0FBQztDQUNuQixHQUFHLE9BQU8sQ0FBQztJQUNWLE9BQU8sRUFBRSxZQUFZLENBQUM7SUFDdEIsSUFBSSxFQUFFLE1BQU0sQ0FBQztDQUNkLENBQUMsQ0F1RUQ7QUFFRCx3QkFBZ0IsYUFBYSxDQUFDLFNBQVMsRUFBRSxNQUFNLEVBQUUsV0FBVyxFQUFFLE1BQU0sR0FBRyxPQUFPLENBQUMsTUFBTSxDQUFDLENBK0JyRjtBQUVELHdCQUFnQiw4QkFBOEIsQ0FBQyxTQUFTLEVBQUUsTUFBTTs7O0dBTS9EO0FBRUQsd0JBQWdCLHNCQUFzQixDQUFDLFNBQVMsRUFBRSxNQUFNLEVBQUUsS0FBSyxTQUFJOzs7R0FNbEU7QUFFRCx3QkFBZ0IsMkJBQTJCLENBQUMsU0FBUyxFQUFFLE1BQU07OztHQU01RDtBQUVELHdCQUFzQixvQkFBb0IsQ0FBQyxFQUN6QyxRQUFRLEVBQ1IsU0FBUyxFQUNULElBQUksRUFDSixLQUFhLEVBQ2QsRUFBRTtJQUNELFFBQVEsRUFBRSxNQUFNLENBQUM7SUFDakIsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixJQUFJLEVBQUUsTUFBTSxDQUFDO0lBQ2IsS0FBSyxDQUFDLEVBQUUsT0FBTyxDQUFDO0NBQ2pCLG1CQU9BO0FBRUQsd0JBQXNCLHFCQUFxQixDQUFDLEVBQzFDLFFBQVEsRUFDUixTQUFTLEVBQ1QsS0FBSyxFQUNMLE9BQWMsRUFDZCxLQUFhLEVBQ2QsRUFBRTtJQUNELFFBQVEsRUFBRSxNQUFNLENBQUM7SUFDakIsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixLQUFLLEVBQUUsTUFBTSxDQUFDO0lBQ2QsT0FBTyxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ2pCLEtBQUssQ0FBQyxFQUFFLE9BQU8sQ0FBQztDQUNqQixtQkFpQkE7QUFFRCx3QkFBc0Isc0JBQXNCLENBQUMsRUFDM0MsUUFBUSxFQUNSLEtBQUssRUFDTCxTQUFTLEVBQ1QsU0FBbUIsRUFDbkIsT0FBZSxFQUNoQixFQUFFO0lBQ0QsUUFBUSxFQUFFLE1BQU0sQ0FBQztJQUNqQixLQUFLLEVBQUUsTUFBTSxDQUFDO0lBQ2QsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixTQUFTLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDbkIsT0FBTyxDQUFDLEVBQUUsTUFBTSxDQUFDO0NBQ2xCLG1CQUtBO0FBRUQsd0JBQXNCLHFCQUFxQixDQUFDLEVBQzFDLFFBQVEsRUFDUixJQUFJLEVBQ0osU0FBUyxFQUNULFNBQW1CLEVBQ25CLE9BQWUsRUFDaEIsRUFBRTtJQUNELFFBQVEsRUFBRSxNQUFNLENBQUM7SUFDakIsSUFBSSxFQUFFLE1BQU0sQ0FBQztJQUNiLFNBQVMsRUFBRSxNQUFNLENBQUM7SUFDbEIsU0FBUyxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ25CLE9BQU8sQ0FBQyxFQUFFLE1BQU0sQ0FBQztDQUNsQixtQkFLQTtBQUVELHdCQUFzQixzQkFBc0IsQ0FBQyxFQUMzQyxRQUFRLEVBQ1IsS0FBSyxFQUNMLFNBQVMsRUFDVCxTQUFtQixFQUNuQixPQUFlLEVBQ2hCLEVBQUU7SUFDRCxRQUFRLEVBQUUsTUFBTSxDQUFDO0lBQ2pCLEtBQUssRUFBRSxNQUFNLEVBQUUsQ0FBQztJQUNoQixTQUFTLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLFNBQVMsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUNuQixPQUFPLENBQUMsRUFBRSxNQUFNLENBQUM7Q0FDbEIsaUJBbUJBO0FBRUQsd0JBQWdCLFdBQVcsQ0FBQyxVQUFVLEVBQUUsTUFBTSxFQUFFLFNBQVMsRUFBRSxNQUFNLFVBRWhFIn0=

package/dest/spartan/utils/k8s.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"k8s.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/k8s.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,YAAY,EAAe,MAAM,eAAe,CAAC;AAQ/D,wBAAsB,gBAAgB,CAAC,EACrC,QAAQ,EACR,SAAS,EACT,aAAa,EACb,QAAQ,EACT,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IAEtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,OAAO,CAAC;IACV,OAAO,EAAE,YAAY,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC,CAuED;AAED,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA+BrF;AAED,wBAAgB,8BAA8B,CAAC,SAAS,EAAE,MAAM;;;GAM/D;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,SAAI;;;GAMlE;AAED,wBAAgB,2BAA2B,CAAC,SAAS,EAAE,MAAM;;;GAM5D;AAED,wBAAsB,oBAAoB,CAAC,EACzC,QAAQ,EACR,SAAS,EACT,IAAI,EACJ,KAAa,EACd,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,mBAOA;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,SAAS,EACT,KAAK,EACL,OAAc,EACd,KAAa,EACd,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,mBAiBA;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,QAAQ,EACR,KAAK,EACL,SAAS,EACT,SAAmB,EACnB,OAAe,EAChB,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,mBAKA;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,IAAI,EACJ,SAAS,EACT,SAAmB,EACnB,OAAe,EAChB,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,mBAKA;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,QAAQ,EACR,KAAK,EACL,SAAS,EACT,SAAmB,EACnB,OAAe,EAChB,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,iBAmBA;AAED,wBAAgB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,UAEhE"}

package/dest/spartan/utils/k8s.js

@@ -0,0 +1,185 @@
+import { createLogger } from '@aztec/aztec.js/log';
+import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { exec, spawn } from 'child_process';
+import path from 'path';
+import { promisify } from 'util';
+const execAsync = promisify(exec);
+const logger = createLogger('e2e:k8s-utils');
+export async function startPortForward({ resource, namespace, containerPort, hostPort }) {
+    const hostPortAsString = hostPort ? hostPort.toString() : '';
+    logger.debug(`kubectl port-forward -n ${namespace} ${resource} ${hostPortAsString}:${containerPort}`);
+    const process = spawn('kubectl', [
+        'port-forward',
+        '-n',
+        namespace,
+        resource,
+        `${hostPortAsString}:${containerPort}`
+    ], {
+        detached: true,
+        windowsHide: true,
+        stdio: [
+            'ignore',
+            'pipe',
+            'pipe'
+        ]
+    });
+    let isResolved = false;
+    const connected = new Promise((resolve, reject)=>{
+        process.stdout?.on('data', (data)=>{
+            const str = data.toString();
+            if (!isResolved && str.includes('Forwarding from')) {
+                isResolved = true;
+                logger.debug(`Port forward for ${resource}: ${str}`);
+                const port = str.search(/:\d+/);
+                if (port === -1) {
+                    reject(new Error('Port not found in port forward output'));
+                    return;
+                }
+                const portNumber = parseInt(str.slice(port + 1));
+                logger.verbose(`Port forwarded for ${resource} at ${portNumber}:${containerPort}`);
+                resolve(portNumber);
+            } else {
+                logger.silent(str);
+            }
+        });
+        process.stderr?.on('data', (data)=>{
+            logger.verbose(`Port forward for ${resource}: ${data.toString()}`);
+            // It's a strange thing:
+            // If we don't pipe stderr, then the port forwarding does not work.
+            // Log to silent because this doesn't actually report errors,
+            // just extremely verbose debug logs.
+            logger.silent(data.toString());
+        });
+        process.on('close', ()=>{
+            if (!isResolved) {
+                isResolved = true;
+                const msg = `Port forward for ${resource} closed before connection established`;
+                logger.warn(msg);
+                reject(new Error(msg));
+            }
+        });
+        process.on('error', (error)=>{
+            if (!isResolved) {
+                isResolved = true;
+                const msg = `Port forward for ${resource} error: ${error}`;
+                logger.error(msg);
+                reject(new Error(msg));
+            }
+        });
+        process.on('exit', (code)=>{
+            if (!isResolved) {
+                isResolved = true;
+                const msg = `Port forward for ${resource} exited with code ${code}`;
+                logger.verbose(msg);
+                reject(new Error(msg));
+            }
+        });
+    });
+    const port = await connected;
+    return {
+        process,
+        port
+    };
+}
+export function getExternalIP(namespace, serviceName) {
+    const { promise, resolve, reject } = promiseWithResolvers();
+    const process = spawn('kubectl', [
+        'get',
+        'service',
+        '-n',
+        namespace,
+        `${namespace}-${serviceName}`,
+        '--output',
+        "jsonpath='{.status.loadBalancer.ingress[0].ip}'"
+    ], {
+        stdio: 'pipe'
+    });
+    let ip = '';
+    process.stdout.on('data', (data)=>{
+        ip += data;
+    });
+    process.on('error', (err)=>{
+        reject(err);
+    });
+    process.on('exit', ()=>{
+        // kubectl prints JSON. Remove the quotes
+        resolve(ip.replace(/"|'/g, ''));
+    });
+    return promise;
+}
+export function startPortForwardForPrometeheus(namespace) {
+    return startPortForward({
+        resource: `svc/${namespace}-prometheus-server`,
+        namespace,
+        containerPort: 80
+    });
+}
+export function startPortForwardForRPC(namespace, index = 0) {
+    return startPortForward({
+        resource: `pod/${namespace}-rpc-aztec-node-${index}`,
+        namespace,
+        containerPort: 8080
+    });
+}
+export function startPortForwardForEthereum(namespace) {
+    return startPortForward({
+        resource: `services/${namespace}-eth-execution`,
+        namespace,
+        containerPort: 8545
+    });
+}
+export async function deleteResourceByName({ resource, namespace, name, force = false }) {
+    const command = `kubectl delete ${resource} ${name} -n ${namespace} --ignore-not-found=true --wait=true ${force ? '--force' : ''}`;
+    logger.info(`command: ${command}`);
+    const { stdout } = await execAsync(command);
+    return stdout;
+}
+export async function deleteResourceByLabel({ resource, namespace, label, timeout = '5m', force = false }) {
+    try {
+        // Match both plain and group-qualified names (e.g., "podchaos" or "podchaos.chaos-mesh.org")
+        const escaped = resource.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
+        const regex = `(^|\\.)${escaped}(\\.|$)`;
+        await execAsync(`kubectl api-resources --no-headers -o name | grep -Eq '${regex}'`);
+    } catch (error) {
+        logger.warn(`Resource type '${resource}' not found in cluster, skipping deletion ${error}`);
+        return '';
+    }
+    const command = `kubectl delete ${resource} -l ${label} -n ${namespace} --ignore-not-found=true --wait=true --timeout=${timeout} ${force ? '--force' : ''}`;
+    logger.info(`command: ${command}`);
+    const { stdout } = await execAsync(command);
+    return stdout;
+}
+export async function waitForResourceByLabel({ resource, label, namespace, condition = 'Ready', timeout = '10m' }) {
+    const command = `kubectl wait ${resource} -l ${label} --for=condition=${condition} -n ${namespace} --timeout=${timeout}`;
+    logger.info(`command: ${command}`);
+    const { stdout } = await execAsync(command);
+    return stdout;
+}
+export async function waitForResourceByName({ resource, name, namespace, condition = 'Ready', timeout = '10m' }) {
+    const command = `kubectl wait ${resource}/${name} --for=condition=${condition} -n ${namespace} --timeout=${timeout}`;
+    logger.info(`command: ${command}`);
+    const { stdout } = await execAsync(command);
+    return stdout;
+}
+export async function waitForResourcesByName({ resource, names, namespace, condition = 'Ready', timeout = '10m' }) {
+    if (!names.length) {
+        throw new Error(`No ${resource} names provided to waitForResourcesByName`);
+    }
+    // Wait all in parallel; if any fails, surface which one.
+    await Promise.all(names.map(async (name)=>{
+        try {
+            await waitForResourceByName({
+                resource,
+                name,
+                namespace,
+                condition,
+                timeout
+            });
+        } catch (err) {
+            throw new Error(`Failed waiting for ${resource}/${name} condition=${condition} timeout=${timeout} namespace=${namespace}: ${String(err)}`);
+        }
+    }));
+}
+export function getChartDir(spartanDir, chartName) {
+    return path.join(spartanDir.trim(), chartName);
+}

package/dest/spartan/utils/nodes.d.ts

@@ -0,0 +1,31 @@
+import type { RollupCheatCodes } from '@aztec/aztec/testing';
+import type { CheckpointNumber } from '@aztec/foundation/branded-types';
+import type { Logger } from '@aztec/foundation/log';
+import { type AztecNodeAdmin, type AztecNodeAdminConfig } from '@aztec/stdlib/interfaces/client';
+import type { TestConfig } from './config.js';
+export declare function awaitCheckpointNumber(rollupCheatCodes: RollupCheatCodes, checkpointNumber: CheckpointNumber, timeoutSeconds: number, log: Logger): Promise<void>;
+export declare function getSequencers(namespace: string): Promise<string[]>;
+export declare function updateSequencersConfig(env: TestConfig, config: Partial<AztecNodeAdminConfig>): Promise<AztecNodeAdminConfig[]>;
+export declare function getSequencersConfig(env: TestConfig): Promise<AztecNodeAdminConfig[]>;
+export declare function withSequencersAdmin<T>(env: TestConfig, fn: (node: AztecNodeAdmin) => Promise<T>): Promise<T[]>;
+/**
+ * Enables or disables probabilistic transaction dropping on validators and waits for rollout.
+ * Wired to env vars P2P_DROP_TX and P2P_DROP_TX_CHANCE via Helm values.
+ */
+export declare function setValidatorTxDrop({ namespace, enabled, probability, logger: log }: {
+    namespace: string;
+    enabled: boolean;
+    probability: number;
+    logger: Logger;
+}): Promise<void>;
+export declare function restartValidators(namespace: string, log: Logger): Promise<void>;
+export declare function enableValidatorDynamicBootNode(instanceName: string, namespace: string, spartanDir: string, log: Logger): Promise<void>;
+/**
+ * Rolls the Aztec pods in the given namespace.
+ * @param namespace - The namespace to roll the Aztec pods in.
+ * @param clearState - If true, also deletes the underlying PVCs to clear persistent storage.
+ *        This is required for rollup upgrades where the old state is incompatible with the new rollup.
+ *        Defaults to false, which preserves the existing storage.
+ */
+export declare function rollAztecPods(namespace: string, clearState?: boolean): Promise<void>;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibm9kZXMuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zcGFydGFuL3V0aWxzL25vZGVzLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFDN0QsT0FBTyxLQUFLLEVBQUUsZ0JBQWdCLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUN4RSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUdwRCxPQUFPLEVBQ0wsS0FBSyxjQUFjLEVBQ25CLEtBQUssb0JBQW9CLEVBRTFCLE1BQU0saUNBQWlDLENBQUM7QUFLekMsT0FBTyxLQUFLLEVBQUUsVUFBVSxFQUFFLE1BQU0sYUFBYSxDQUFDO0FBUTlDLHdCQUFzQixxQkFBcUIsQ0FDekMsZ0JBQWdCLEVBQUUsZ0JBQWdCLEVBQ2xDLGdCQUFnQixFQUFFLGdCQUFnQixFQUNsQyxjQUFjLEVBQUUsTUFBTSxFQUN0QixHQUFHLEVBQUUsTUFBTSxpQkFlWjtBQUVELHdCQUFzQixhQUFhLENBQUMsU0FBUyxFQUFFLE1BQU0scUJBNEJwRDtBQUVELHdCQUFnQixzQkFBc0IsQ0FBQyxHQUFHLEVBQUUsVUFBVSxFQUFFLE1BQU0sRUFBRSxPQUFPLENBQUMsb0JBQW9CLENBQUMsbUNBSzVGO0FBRUQsd0JBQWdCLG1CQUFtQixDQUFDLEdBQUcsRUFBRSxVQUFVLG1DQUVsRDtBQUVELHdCQUFzQixtQkFBbUIsQ0FBQyxDQUFDLEVBQUUsR0FBRyxFQUFFLFVBQVUsRUFBRSxFQUFFLEVBQUUsQ0FBQyxJQUFJLEVBQUUsY0FBYyxLQUFLLE9BQU8sQ0FBQyxDQUFDLENBQUMsR0FBRyxPQUFPLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0EyQnBIO0FBRUQ7OztHQUdHO0FBQ0gsd0JBQXNCLGtCQUFrQixDQUFDLEVBQ3ZDLFNBQVMsRUFDVCxPQUFPLEVBQ1AsV0FBVyxFQUNYLE1BQU0sRUFBRSxHQUFHLEVBQ1osRUFBRTtJQUNELFNBQVMsRUFBRSxNQUFNLENBQUM7SUFDbEIsT0FBTyxFQUFFLE9BQU8sQ0FBQztJQUNqQixXQUFXLEVBQUUsTUFBTSxDQUFDO0lBQ3BCLE1BQU0sRUFBRSxNQUFNLENBQUM7Q0FDaEIsaUJBZ0NBO0FBRUQsd0JBQXNCLGlCQUFpQixDQUFDLFNBQVMsRUFBRSxNQUFNLEVBQUUsR0FBRyxFQUFFLE1BQU0saUJBK0JyRTtBQUVELHdCQUFzQiw4QkFBOEIsQ0FDbEQsWUFBWSxFQUFFLE1BQU0sRUFDcEIsU0FBUyxFQUFFLE1BQU0sRUFDakIsVUFBVSxFQUFFLE1BQU0sRUFDbEIsR0FBRyxFQUFFLE1BQU0saUJBZ0JaO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsd0JBQXNCLGFBQWEsQ0FBQyxTQUFTLEVBQUUsTUFBTSxFQUFFLFVBQVUsR0FBRSxPQUFlLGlCQW1GakYifQ==

package/dest/spartan/utils/nodes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"nodes.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/nodes.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACxE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAGpD,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,oBAAoB,EAE1B,MAAM,iCAAiC,CAAC;AAKzC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAQ9C,wBAAsB,qBAAqB,CACzC,gBAAgB,EAAE,gBAAgB,EAClC,gBAAgB,EAAE,gBAAgB,EAClC,cAAc,EAAE,MAAM,EACtB,GAAG,EAAE,MAAM,iBAeZ;AAED,wBAAsB,aAAa,CAAC,SAAS,EAAE,MAAM,qBA4BpD;AAED,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,oBAAoB,CAAC,mCAK5F;AAED,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,UAAU,mCAElD;AAED,wBAAsB,mBAAmB,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CA2BpH;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,EACvC,SAAS,EACT,OAAO,EACP,WAAW,EACX,MAAM,EAAE,GAAG,EACZ,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB,iBAgCA;AAED,wBAAsB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,iBA+BrE;AAED,wBAAsB,8BAA8B,CAClD,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,MAAM,iBAgBZ;AAED;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,GAAE,OAAe,iBAmFjF"}

package/dest/spartan/utils/nodes.js

@@ -0,0 +1,273 @@
+import { createLogger } from '@aztec/aztec.js/log';
+import { makeBackoff, retry } from '@aztec/foundation/retry';
+import { sleep } from '@aztec/foundation/sleep';
+import { createAztecNodeAdminClient } from '@aztec/stdlib/interfaces/client';
+import { exec } from 'child_process';
+import { promisify } from 'util';
+import { execHelmCommand } from './helm.js';
+import { deleteResourceByLabel, getChartDir, startPortForward, waitForResourceByLabel } from './k8s.js';
+const execAsync = promisify(exec);
+const logger = createLogger('e2e:k8s-utils');
+export async function awaitCheckpointNumber(rollupCheatCodes, checkpointNumber, timeoutSeconds, log) {
+    log.info(`Waiting for checkpoint ${checkpointNumber}`);
+    let tips = await rollupCheatCodes.getTips();
+    const endTime = Date.now() + timeoutSeconds * 1000;
+    while(tips.pending < checkpointNumber && Date.now() < endTime){
+        log.info(`At checkpoint ${tips.pending}`);
+        await sleep(1000);
+        tips = await rollupCheatCodes.getTips();
+    }
+    if (tips.pending < checkpointNumber) {
+        throw new Error(`Timeout waiting for checkpoint ${checkpointNumber}, only reached ${tips.pending}`);
+    } else {
+        log.info(`Reached checkpoint ${tips.pending}`);
+    }
+}
+export async function getSequencers(namespace) {
+    const selectors = [
+        'app.kubernetes.io/name=validator',
+        'app.kubernetes.io/component=validator',
+        'app.kubernetes.io/component=sequencer-node',
+        'app=validator'
+    ];
+    for (const selector of selectors){
+        try {
+            const command = `kubectl get pods -l ${selector} -n ${namespace} -o jsonpath='{.items[*].metadata.name}'`;
+            const { stdout } = await execAsync(command);
+            const sequencers = stdout.split(' ').map((s)=>s.trim()).filter(Boolean);
+            if (sequencers.length > 0) {
+                logger.verbose(`Found sequencer pods ${sequencers.join(', ')} (selector=${selector})`);
+                return sequencers;
+            }
+        } catch  {
+        // try next selector
+        }
+    }
+    // Fail fast instead of returning [''] which leads to attempts to port-forward `pod/`.
+    throw new Error(`No sequencer/validator pods found in namespace ${namespace}. Tried selectors: ${selectors.join(', ')}`);
+}
+export function updateSequencersConfig(env, config) {
+    return withSequencersAdmin(env, async (client)=>{
+        await client.setConfig(config);
+        return client.getConfig();
+    });
+}
+export function getSequencersConfig(env) {
+    return withSequencersAdmin(env, (client)=>client.getConfig());
+}
+export async function withSequencersAdmin(env, fn) {
+    const adminContainerPort = 8880;
+    const namespace = env.NAMESPACE;
+    const sequencers = await getSequencers(namespace);
+    const results = [];
+    for (const sequencer of sequencers){
+        const { process, port } = await startPortForward({
+            resource: `pod/${sequencer}`,
+            namespace,
+            containerPort: adminContainerPort
+        });
+        const url = `http://localhost:${port}`;
+        await retry(()=>fetch(`${url}/status`).then((res)=>res.status === 200), 'forward node admin port', makeBackoff([
+            1,
+            1,
+            2,
+            6
+        ]), logger, true);
+        const client = createAztecNodeAdminClient(url);
+        results.push(await fn(client));
+        process.kill();
+    }
+    return results;
+}
+/**
+ * Enables or disables probabilistic transaction dropping on validators and waits for rollout.
+ * Wired to env vars P2P_DROP_TX and P2P_DROP_TX_CHANCE via Helm values.
+ */ export async function setValidatorTxDrop({ namespace, enabled, probability, logger: log }) {
+    const drop = enabled ? 'true' : 'false';
+    const prob = String(probability);
+    const selectors = [
+        'app.kubernetes.io/name=validator',
+        'app.kubernetes.io/component=validator',
+        'app=validator'
+    ];
+    let updated = false;
+    for (const selector of selectors){
+        try {
+            const list = await execAsync(`kubectl get statefulset -l ${selector} -n ${namespace} --no-headers -o name | cat`);
+            const names = list.stdout.split('\n').map((s)=>s.trim()).filter(Boolean);
+            if (names.length === 0) {
+                continue;
+            }
+            const cmd = `kubectl set env statefulset -l ${selector} -n ${namespace} P2P_DROP_TX=${drop} P2P_DROP_TX_CHANCE=${prob}`;
+            log.info(`command: ${cmd}`);
+            await execAsync(cmd);
+            updated = true;
+        } catch (e) {
+            log.warn(`Failed to update validators with selector ${selector}: ${String(e)}`);
+        }
+    }
+    if (!updated) {
+        log.warn(`No validator StatefulSets found in ${namespace}. Skipping tx drop toggle.`);
+        return;
+    }
+    // Restart validator pods to ensure env vars take effect and wait for readiness
+    await restartValidators(namespace, log);
+}
+export async function restartValidators(namespace, log) {
+    const selectors = [
+        'app.kubernetes.io/name=validator',
+        'app.kubernetes.io/component=validator',
+        'app=validator'
+    ];
+    let any = false;
+    for (const selector of selectors){
+        try {
+            const { stdout } = await execAsync(`kubectl get pods -l ${selector} -n ${namespace} --no-headers -o name | cat`);
+            if (!stdout || stdout.trim().length === 0) {
+                continue;
+            }
+            any = true;
+            await deleteResourceByLabel({
+                resource: 'pods',
+                namespace,
+                label: selector
+            });
+        } catch (e) {
+            log.warn(`Error restarting validator pods with selector ${selector}: ${String(e)}`);
+        }
+    }
+    if (!any) {
+        log.warn(`No validator pods found to restart in ${namespace}.`);
+        return;
+    }
+    // Wait for either label to be Ready
+    for (const selector of selectors){
+        try {
+            await waitForResourceByLabel({
+                resource: 'pods',
+                namespace,
+                label: selector
+            });
+            return;
+        } catch  {
+        // try next
+        }
+    }
+    log.warn(`Validator pods did not report Ready; continuing.`);
+}
+export async function enableValidatorDynamicBootNode(instanceName, namespace, spartanDir, log) {
+    log.info(`Enabling validator dynamic boot node`);
+    await execHelmCommand({
+        instanceName,
+        namespace,
+        helmChartDir: getChartDir(spartanDir, 'aztec-network'),
+        values: {
+            'validator.dynamicBootNode': 'true'
+        },
+        valuesFile: undefined,
+        timeout: '15m',
+        reuseValues: true
+    });
+    log.info(`Validator dynamic boot node enabled`);
+}
+/**
+ * Rolls the Aztec pods in the given namespace.
+ * @param namespace - The namespace to roll the Aztec pods in.
+ * @param clearState - If true, also deletes the underlying PVCs to clear persistent storage.
+ *        This is required for rollup upgrades where the old state is incompatible with the new rollup.
+ *        Defaults to false, which preserves the existing storage.
+ */ export async function rollAztecPods(namespace, clearState = false) {
+    // Pod components use 'validator', but StatefulSets and PVCs use 'sequencer-node' for validators
+    const podComponents = [
+        'p2p-bootstrap',
+        'prover-node',
+        'prover-broker',
+        'prover-agent',
+        'sequencer-node',
+        'rpc'
+    ];
+    const pvcComponents = [
+        'p2p-bootstrap',
+        'prover-node',
+        'prover-broker',
+        'sequencer-node',
+        'rpc'
+    ];
+    // StatefulSet components that need to be scaled down before PVC deletion
+    // Note: validators use 'sequencer-node' as component label, not 'validator'
+    const statefulSetComponents = [
+        'p2p-bootstrap',
+        'prover-node',
+        'prover-broker',
+        'sequencer-node',
+        'rpc'
+    ];
+    if (clearState) {
+        // To delete PVCs, we must first scale down StatefulSets so pods release the volumes
+        // Otherwise PVC deletion will hang waiting for pods to terminate
+        // First, save original replica counts
+        const originalReplicas = new Map();
+        for (const component of statefulSetComponents){
+            try {
+                const getCmd = `kubectl get statefulset -l app.kubernetes.io/component=${component} -n ${namespace} -o jsonpath='{.items[0].spec.replicas}'`;
+                const { stdout } = await execAsync(getCmd);
+                const replicas = parseInt(stdout.replace(/'/g, '').trim(), 10);
+                if (!isNaN(replicas) && replicas > 0) {
+                    originalReplicas.set(component, replicas);
+                }
+            } catch  {
+            // Component might not exist, continue
+            }
+        }
+        // Scale down to 0
+        for (const component of statefulSetComponents){
+            try {
+                const scaleCmd = `kubectl scale statefulset -l app.kubernetes.io/component=${component} -n ${namespace} --replicas=0 --timeout=2m`;
+                logger.info(`command: ${scaleCmd}`);
+                await execAsync(scaleCmd);
+            } catch (e) {
+                // Component might not exist or might be a Deployment, continue
+                logger.verbose(`Scale down ${component} skipped: ${e}`);
+            }
+        }
+        // Wait for pods to terminate
+        await sleep(15 * 1000);
+        // Now delete PVCs (they should no longer be in use)
+        for (const component of pvcComponents){
+            await deleteResourceByLabel({
+                resource: 'persistentvolumeclaims',
+                namespace: namespace,
+                label: `app.kubernetes.io/component=${component}`
+            });
+        }
+        // Scale StatefulSets back up to original replica counts
+        for (const component of statefulSetComponents){
+            const replicas = originalReplicas.get(component) ?? 1;
+            try {
+                const scaleCmd = `kubectl scale statefulset -l app.kubernetes.io/component=${component} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+                logger.info(`command: ${scaleCmd}`);
+                await execAsync(scaleCmd);
+            } catch (e) {
+                logger.verbose(`Scale up ${component} skipped: ${e}`);
+            }
+        }
+    } else {
+        // Just delete pods (no state clearing)
+        for (const component of podComponents){
+            await deleteResourceByLabel({
+                resource: 'pods',
+                namespace: namespace,
+                label: `app.kubernetes.io/component=${component}`
+            });
+        }
+    }
+    await sleep(10 * 1000);
+    // Wait for pods to come back
+    for (const component of podComponents){
+        await waitForResourceByLabel({
+            resource: 'pods',
+            namespace: namespace,
+            label: `app.kubernetes.io/component=${component}`
+        });
+    }
+}