@aztec/end-to-end 4.0.0-nightly.20260120 → 4.0.0-nightly.20260122

package/src/spartan/utils.ts

@@ -1,1246 +1 @@
-import { createLogger } from '@aztec/aztec.js/log';
-import type { RollupCheatCodes } from '@aztec/aztec/testing';
-import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses';
-import type { ViemPublicClient } from '@aztec/ethereum/types';
-import type { CheckpointNumber } from '@aztec/foundation/branded-types';
-import type { Logger } from '@aztec/foundation/log';
-import { promiseWithResolvers } from '@aztec/foundation/promise';
-import { makeBackoff, retry } from '@aztec/foundation/retry';
-import { schemas } from '@aztec/foundation/schemas';
-import { sleep } from '@aztec/foundation/sleep';
-import {
-  type AztecNodeAdmin,
-  type AztecNodeAdminConfig,
-  createAztecNodeAdminClient,
-  createAztecNodeClient,
-} from '@aztec/stdlib/interfaces/client';
-
-import { ChildProcess, exec, execSync, spawn } from 'child_process';
-import path from 'path';
-import { promisify } from 'util';
-import { createPublicClient, fallback, http } from 'viem';
-import { z } from 'zod';
-
-const execAsync = promisify(exec);
-
-const logger = createLogger('e2e:k8s-utils');
-
-const testConfigSchema = z.object({
-  NAMESPACE: z.string().default('scenario'),
-  REAL_VERIFIER: schemas.Boolean.optional().default(true),
-  CREATE_ETH_DEVNET: schemas.Boolean.optional().default(false),
-  L1_RPC_URLS_JSON: z.string().optional(),
-  L1_ACCOUNT_MNEMONIC: z.string().optional(),
-  AZTEC_SLOT_DURATION: z.coerce.number().optional().default(24),
-  AZTEC_EPOCH_DURATION: z.coerce.number().optional().default(32),
-  AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5),
-  AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET: z.coerce.number().optional().default(2),
-});
-
-export type TestConfig = z.infer<typeof testConfigSchema>;
-
-export function setupEnvironment(env: unknown): TestConfig {
-  const config = testConfigSchema.parse(env);
-  logger.warn(`Loaded env config`, config);
-  return config;
-}
-
-/**
- * @param path - The path to the script, relative to the project root
- * @param args - The arguments to pass to the script
- * @param logger - The logger to use
- * @returns The exit code of the script
- */
-function runScript(path: string, args: string[], logger: Logger, env?: Record<string, string>) {
-  const childProcess = spawn(path, args, {
-    stdio: ['ignore', 'pipe', 'pipe'],
-    env: env ? { ...process.env, ...env } : process.env,
-  });
-  return new Promise<number>((resolve, reject) => {
-    childProcess.on('close', (code: number | null) => resolve(code ?? 0));
-    childProcess.on('error', reject);
-    childProcess.stdout?.on('data', (data: Buffer) => {
-      logger.info(data.toString());
-    });
-    childProcess.stderr?.on('data', (data: Buffer) => {
-      logger.error(data.toString());
-    });
-  });
-}
-
-export function getAztecBin() {
-  return path.join(getGitProjectRoot(), 'yarn-project/aztec/dest/bin/index.js');
-}
-
-/**
- * Runs the Aztec binary
- * @param args - The arguments to pass to the Aztec binary
- * @param logger - The logger to use
- * @param env - Optional environment variables to set for the process
- * @returns The exit code of the Aztec binary
- */
-export function runAztecBin(args: string[], logger: Logger, env?: Record<string, string>) {
-  return runScript('node', [getAztecBin(), ...args], logger, env);
-}
-
-export function runProjectScript(script: string, args: string[], logger: Logger, env?: Record<string, string>) {
-  const scriptPath = script.startsWith('/') ? script : path.join(getGitProjectRoot(), script);
-  return runScript(scriptPath, args, logger, env);
-}
-
-export async function startPortForward({
-  resource,
-  namespace,
-  containerPort,
-  hostPort,
-}: {
-  resource: string;
-  namespace: string;
-  containerPort: number;
-  // If not provided, the port will be chosen automatically
-  hostPort?: number;
-}): Promise<{
-  process: ChildProcess;
-  port: number;
-}> {
-  const hostPortAsString = hostPort ? hostPort.toString() : '';
-
-  logger.debug(`kubectl port-forward -n ${namespace} ${resource} ${hostPortAsString}:${containerPort}`);
-
-  const process = spawn(
-    'kubectl',
-    ['port-forward', '-n', namespace, resource, `${hostPortAsString}:${containerPort}`],
-    {
-      detached: true,
-      windowsHide: true,
-      stdio: ['ignore', 'pipe', 'pipe'],
-    },
-  );
-
-  let isResolved = false;
-  const connected = new Promise<number>((resolve, reject) => {
-    process.stdout?.on('data', data => {
-      const str = data.toString() as string;
-      if (!isResolved && str.includes('Forwarding from')) {
-        isResolved = true;
-        logger.debug(`Port forward for ${resource}: ${str}`);
-        const port = str.search(/:\d+/);
-        if (port === -1) {
-          reject(new Error('Port not found in port forward output'));
-          return;
-        }
-        const portNumber = parseInt(str.slice(port + 1));
-        logger.verbose(`Port forwarded for ${resource} at ${portNumber}:${containerPort}`);
-        resolve(portNumber);
-      } else {
-        logger.silent(str);
-      }
-    });
-    process.stderr?.on('data', data => {
-      logger.verbose(`Port forward for ${resource}: ${data.toString()}`);
-      // It's a strange thing:
-      // If we don't pipe stderr, then the port forwarding does not work.
-      // Log to silent because this doesn't actually report errors,
-      // just extremely verbose debug logs.
-      logger.silent(data.toString());
-    });
-    process.on('close', () => {
-      if (!isResolved) {
-        isResolved = true;
-        const msg = `Port forward for ${resource} closed before connection established`;
-        logger.warn(msg);
-        reject(new Error(msg));
-      }
-    });
-    process.on('error', error => {
-      if (!isResolved) {
-        isResolved = true;
-        const msg = `Port forward for ${resource} error: ${error}`;
-        logger.error(msg);
-        reject(new Error(msg));
-      }
-    });
-    process.on('exit', code => {
-      if (!isResolved) {
-        isResolved = true;
-        const msg = `Port forward for ${resource} exited with code ${code}`;
-        logger.verbose(msg);
-        reject(new Error(msg));
-      }
-    });
-  });
-
-  const port = await connected;
-
-  return { process, port };
-}
-
-export function getExternalIP(namespace: string, serviceName: string): Promise<string> {
-  const { promise, resolve, reject } = promiseWithResolvers<string>();
-  const process = spawn(
-    'kubectl',
-    [
-      'get',
-      'service',
-      '-n',
-      namespace,
-      `${namespace}-${serviceName}`,
-      '--output',
-      "jsonpath='{.status.loadBalancer.ingress[0].ip}'",
-    ],
-    {
-      stdio: 'pipe',
-    },
-  );
-
-  let ip = '';
-  process.stdout.on('data', data => {
-    ip += data;
-  });
-  process.on('error', err => {
-    reject(err);
-  });
-  process.on('exit', () => {
-    // kubectl prints JSON. Remove the quotes
-    resolve(ip.replace(/"|'/g, ''));
-  });
-
-  return promise;
-}
-
-export function startPortForwardForPrometeheus(namespace: string) {
-  return startPortForward({
-    resource: `svc/${namespace}-prometheus-server`,
-    namespace,
-    containerPort: 80,
-  });
-}
-
-export function startPortForwardForRPC(namespace: string, index = 0) {
-  return startPortForward({
-    resource: `pod/${namespace}-rpc-aztec-node-${index}`,
-    namespace,
-    containerPort: 8080,
-  });
-}
-
-export function startPortForwardForEthereum(namespace: string) {
-  return startPortForward({
-    resource: `services/${namespace}-eth-execution`,
-    namespace,
-    containerPort: 8545,
-  });
-}
-
-export async function deleteResourceByName({
-  resource,
-  namespace,
-  name,
-  force = false,
-}: {
-  resource: string;
-  namespace: string;
-  name: string;
-  force?: boolean;
-}) {
-  const command = `kubectl delete ${resource} ${name} -n ${namespace} --ignore-not-found=true --wait=true ${
-    force ? '--force' : ''
-  }`;
-  logger.info(`command: ${command}`);
-  const { stdout } = await execAsync(command);
-  return stdout;
-}
-
-export async function deleteResourceByLabel({
-  resource,
-  namespace,
-  label,
-  timeout = '5m',
-  force = false,
-}: {
-  resource: string;
-  namespace: string;
-  label: string;
-  timeout?: string;
-  force?: boolean;
-}) {
-  try {
-    // Match both plain and group-qualified names (e.g., "podchaos" or "podchaos.chaos-mesh.org")
-    const escaped = resource.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
-    const regex = `(^|\\.)${escaped}(\\.|$)`;
-    await execAsync(`kubectl api-resources --no-headers -o name | grep -Eq '${regex}'`);
-  } catch (error) {
-    logger.warn(`Resource type '${resource}' not found in cluster, skipping deletion ${error}`);
-    return '';
-  }
-
-  const command = `kubectl delete ${resource} -l ${label} -n ${namespace} --ignore-not-found=true --wait=true --timeout=${timeout} ${
-    force ? '--force' : ''
-  }`;
-  logger.info(`command: ${command}`);
-  const { stdout } = await execAsync(command);
-  return stdout;
-}
-
-export async function waitForResourceByLabel({
-  resource,
-  label,
-  namespace,
-  condition = 'Ready',
-  timeout = '10m',
-}: {
-  resource: string;
-  label: string;
-  namespace: string;
-  condition?: string;
-  timeout?: string;
-}) {
-  const command = `kubectl wait ${resource} -l ${label} --for=condition=${condition} -n ${namespace} --timeout=${timeout}`;
-  logger.info(`command: ${command}`);
-  const { stdout } = await execAsync(command);
-  return stdout;
-}
-
-export async function waitForResourceByName({
-  resource,
-  name,
-  namespace,
-  condition = 'Ready',
-  timeout = '10m',
-}: {
-  resource: string;
-  name: string;
-  namespace: string;
-  condition?: string;
-  timeout?: string;
-}) {
-  const command = `kubectl wait ${resource}/${name} --for=condition=${condition} -n ${namespace} --timeout=${timeout}`;
-  logger.info(`command: ${command}`);
-  const { stdout } = await execAsync(command);
-  return stdout;
-}
-
-export async function waitForResourcesByName({
-  resource,
-  names,
-  namespace,
-  condition = 'Ready',
-  timeout = '10m',
-}: {
-  resource: string;
-  names: string[];
-  namespace: string;
-  condition?: string;
-  timeout?: string;
-}) {
-  if (!names.length) {
-    throw new Error(`No ${resource} names provided to waitForResourcesByName`);
-  }
-
-  // Wait all in parallel; if any fails, surface which one.
-  await Promise.all(
-    names.map(async name => {
-      try {
-        await waitForResourceByName({ resource, name, namespace, condition, timeout });
-      } catch (err) {
-        throw new Error(
-          `Failed waiting for ${resource}/${name} condition=${condition} timeout=${timeout} namespace=${namespace}: ${String(
-            err,
-          )}`,
-        );
-      }
-    }),
-  );
-}
-
-export function getChartDir(spartanDir: string, chartName: string) {
-  return path.join(spartanDir.trim(), chartName);
-}
-
-function shellQuote(value: string) {
-  // Single-quote safe shell escaping: ' -> '\''
-  return `'${value.replace(/'/g, "'\\''")}'`;
-}
-
-function valuesToArgs(values: Record<string, string | number | boolean>) {
-  return Object.entries(values)
-    .map(([key, value]) =>
-      typeof value === 'number' || typeof value === 'boolean'
-        ? `--set ${key}=${value}`
-        : `--set-string ${key}=${shellQuote(String(value))}`,
-    )
-    .join(' ');
-}
-
-function createHelmCommand({
-  instanceName,
-  helmChartDir,
-  namespace,
-  valuesFile,
-  timeout,
-  values,
-  reuseValues = false,
-}: {
-  instanceName: string;
-  helmChartDir: string;
-  namespace: string;
-  valuesFile: string | undefined;
-  timeout: string;
-  values: Record<string, string | number | boolean>;
-  reuseValues?: boolean;
-}) {
-  const valuesFileArgs = valuesFile ? `--values ${helmChartDir}/values/${valuesFile}` : '';
-  const reuseValuesArgs = reuseValues ? '--reuse-values' : '';
-  return `helm upgrade --install ${instanceName} ${helmChartDir} --namespace ${namespace} ${valuesFileArgs} ${reuseValuesArgs} --wait --timeout=${timeout} ${valuesToArgs(
-    values,
-  )}`;
-}
-
-async function execHelmCommand(args: Parameters<typeof createHelmCommand>[0]) {
-  const helmCommand = createHelmCommand(args);
-  logger.info(`helm command: ${helmCommand}`);
-  const { stdout } = await execAsync(helmCommand);
-  return stdout;
-}
-
-async function getHelmReleaseStatus(instanceName: string, namespace: string): Promise<string | undefined> {
-  try {
-    const { stdout } = await execAsync(
-      `helm list --namespace ${namespace} --all --filter '^${instanceName}$' --output json | cat`,
-    );
-    const parsed = JSON.parse(stdout) as Array<{ name?: string; status?: string }>;
-    const row = parsed.find(r => r.name === instanceName);
-    return row?.status;
-  } catch {
-    return undefined;
-  }
-}
-
-async function forceDeleteHelmReleaseRecord(instanceName: string, namespace: string, logger: Logger) {
-  const labelSelector = `owner=helm,name=${instanceName}`;
-  const cmd = `kubectl delete secret -n ${namespace} -l ${labelSelector} --ignore-not-found=true`;
-  logger.warn(`Force deleting Helm release record: ${cmd}`);
-  await execAsync(cmd).catch(() => undefined);
-}
-
-async function hasDeployedHelmRelease(instanceName: string, namespace: string): Promise<boolean> {
-  try {
-    const status = await getHelmReleaseStatus(instanceName, namespace);
-    return status?.toLowerCase() === 'deployed';
-  } catch {
-    return false;
-  }
-}
-
-export async function uninstallChaosMesh(instanceName: string, namespace: string, logger: Logger) {
-  // uninstall the helm chart if it exists
-  logger.info(`Uninstalling helm chart ${instanceName}`);
-  await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`);
-  // and delete the chaos-mesh resources created by this release
-  const deleteByLabel = async (resource: string) => {
-    const args = {
-      resource,
-      namespace: namespace,
-      label: `app.kubernetes.io/instance=${instanceName}`,
-    } as const;
-    logger.info(`Deleting ${resource} resources for release ${instanceName}`);
-    await deleteResourceByLabel(args).catch(e => {
-      logger.error(`Error deleting ${resource}: ${e}`);
-      logger.info(`Force deleting ${resource}`);
-      return deleteResourceByLabel({ ...args, force: true });
-    });
-  };
-
-  await deleteByLabel('podchaos');
-  await deleteByLabel('networkchaos');
-  await deleteByLabel('podnetworkchaos');
-  await deleteByLabel('workflows');
-  await deleteByLabel('workflownodes');
-}
-
-/**
- * Installs a Helm chart with the given parameters.
- * @param instanceName - The name of the Helm chart instance.
- * @param targetNamespace - The namespace with the resources to be affected by the Helm chart.
- * @param valuesFile - The values file to use for the Helm chart.
- * @param chaosMeshNamespace - The namespace to install the Helm chart in.
- * @param timeout - The timeout for the Helm command.
- * @param clean - Whether to clean up the Helm chart before installing it.
- * @returns The stdout of the Helm command.
- * @throws If the Helm command fails.
- *
- * Example usage:
- * ```typescript
- * const stdout = await installChaosMeshChart({ instanceName: 'force-reorg', targetNamespace: 'smoke', valuesFile: 'prover-failure.yaml'});
- * console.log(stdout);
- * ```
- */
-export async function installChaosMeshChart({
-  instanceName,
-  targetNamespace,
-  valuesFile,
-  helmChartDir,
-  timeout = '10m',
-  clean = true,
-  values = {},
-  logger,
-}: {
-  instanceName: string;
-  targetNamespace: string;
-  valuesFile: string;
-  helmChartDir: string;
-  chaosMeshNamespace?: string;
-  timeout?: string;
-  clean?: boolean;
-  values?: Record<string, string | number>;
-  logger: Logger;
-}) {
-  if (clean) {
-    await uninstallChaosMesh(instanceName, targetNamespace, logger);
-  }
-
-  return execHelmCommand({
-    instanceName,
-    helmChartDir,
-    namespace: targetNamespace,
-    valuesFile,
-    timeout,
-    values: { ...values, 'global.targetNamespace': targetNamespace },
-  });
-}
-
-export function applyProverFailure({
-  namespace,
-  spartanDir,
-  durationSeconds,
-  logger,
-}: {
-  namespace: string;
-  spartanDir: string;
-  durationSeconds: number;
-  logger: Logger;
-}) {
-  return installChaosMeshChart({
-    instanceName: 'prover-failure',
-    targetNamespace: namespace,
-    valuesFile: 'prover-failure.yaml',
-    helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
-    values: {
-      'proverFailure.duration': `${durationSeconds}s`,
-    },
-    logger,
-  });
-}
-
-export function applyValidatorFailure({
-  namespace,
-  spartanDir,
-  logger,
-  values,
-  instanceName,
-}: {
-  namespace: string;
-  spartanDir: string;
-  logger: Logger;
-  values?: Record<string, string | number>;
-  instanceName?: string;
-}) {
-  return installChaosMeshChart({
-    instanceName: instanceName ?? 'validator-failure',
-    targetNamespace: namespace,
-    valuesFile: 'validator-failure.yaml',
-    helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
-    values,
-    logger,
-  });
-}
-
-export function applyProverKill({
-  namespace,
-  spartanDir,
-  logger,
-  values,
-}: {
-  namespace: string;
-  spartanDir: string;
-  logger: Logger;
-  values?: Record<string, string | number>;
-}) {
-  return installChaosMeshChart({
-    instanceName: 'prover-kill',
-    targetNamespace: namespace,
-    valuesFile: 'prover-kill.yaml',
-    helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
-    chaosMeshNamespace: namespace,
-    clean: true,
-    logger,
-    values,
-  });
-}
-
-export function applyProverBrokerKill({
-  namespace,
-  spartanDir,
-  logger,
-  values,
-}: {
-  namespace: string;
-  spartanDir: string;
-  logger: Logger;
-  values?: Record<string, string | number>;
-}) {
-  return installChaosMeshChart({
-    instanceName: 'prover-broker-kill',
-    targetNamespace: namespace,
-    valuesFile: 'prover-broker-kill.yaml',
-    helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
-    clean: true,
-    logger,
-    values,
-  });
-}
-
-export function applyBootNodeFailure({
-  instanceName = 'boot-node-failure',
-  namespace,
-  spartanDir,
-  durationSeconds,
-  logger,
-  values,
-}: {
-  instanceName?: string;
-  namespace: string;
-  spartanDir: string;
-  durationSeconds: number;
-  logger: Logger;
-  values?: Record<string, string | number>;
-}) {
-  return installChaosMeshChart({
-    instanceName,
-    targetNamespace: namespace,
-    valuesFile: 'boot-node-failure.yaml',
-    helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
-    values: {
-      'bootNodeFailure.duration': `${durationSeconds}s`,
-      ...(values ?? {}),
-    },
-    logger,
-  });
-}
-
-export function applyValidatorKill({
-  instanceName = 'validator-kill',
-  namespace,
-  spartanDir,
-  logger,
-  values,
-  clean = true,
-}: {
-  instanceName?: string;
-  namespace: string;
-  spartanDir: string;
-  logger: Logger;
-  values?: Record<string, string | number>;
-  clean?: boolean;
-}) {
-  return installChaosMeshChart({
-    instanceName: instanceName ?? 'validator-kill',
-    targetNamespace: namespace,
-    valuesFile: 'validator-kill.yaml',
-    helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
-    clean,
-    logger,
-    values,
-  });
-}
-
-export function applyNetworkShaping({
-  instanceName = 'network-shaping',
-  valuesFile,
-  namespace,
-  spartanDir,
-  logger,
-}: {
-  instanceName?: string;
-  valuesFile: string;
-  namespace: string;
-  spartanDir: string;
-  logger: Logger;
-}) {
-  return installChaosMeshChart({
-    instanceName,
-    targetNamespace: namespace,
-    valuesFile,
-    helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
-    logger,
-  });
-}
-
-export async function awaitCheckpointNumber(
-  rollupCheatCodes: RollupCheatCodes,
-  checkpointNumber: CheckpointNumber,
-  timeoutSeconds: number,
-  logger: Logger,
-) {
-  logger.info(`Waiting for checkpoint ${checkpointNumber}`);
-  let tips = await rollupCheatCodes.getTips();
-  const endTime = Date.now() + timeoutSeconds * 1000;
-  while (tips.pending < checkpointNumber && Date.now() < endTime) {
-    logger.info(`At checkpoint ${tips.pending}`);
-    await sleep(1000);
-    tips = await rollupCheatCodes.getTips();
-  }
-  if (tips.pending < checkpointNumber) {
-    throw new Error(`Timeout waiting for checkpoint ${checkpointNumber}, only reached ${tips.pending}`);
-  } else {
-    logger.info(`Reached checkpoint ${tips.pending}`);
-  }
-}
-
-export async function restartBot(namespace: string, logger: Logger) {
-  logger.info(`Restarting bot`);
-  await deleteResourceByLabel({ resource: 'pods', namespace, label: 'app.kubernetes.io/name=bot' });
-  await sleep(10 * 1000);
-  // Some bot images may take time to report Ready due to heavy boot-time proving.
-  // Waiting for PodReadyToStartContainers ensures the pod is scheduled and starting without blocking on full readiness.
-  await waitForResourceByLabel({
-    resource: 'pods',
-    namespace,
-    label: 'app.kubernetes.io/name=bot',
-    condition: 'PodReadyToStartContainers',
-  });
-  logger.info(`Bot restarted`);
-}
-
-/**
- * Installs or upgrades the transfer bot Helm release for the given namespace.
- * Intended for test setup to enable L2 traffic generation only when needed.
- */
-export async function installTransferBot({
-  namespace,
-  spartanDir,
-  logger,
-  replicas = 1,
-  txIntervalSeconds = 10,
-  followChain = 'PENDING',
-  mnemonic = process.env.LABS_INFRA_MNEMONIC ?? 'test test test test test test test test test test test junk',
-  mnemonicStartIndex,
-  botPrivateKey = process.env.BOT_TRANSFERS_L2_PRIVATE_KEY ?? '0xcafe01',
-  nodeUrl,
-  timeout = '15m',
-  reuseValues = true,
-  aztecSlotDuration = Number(process.env.AZTEC_SLOT_DURATION ?? 12),
-}: {
-  namespace: string;
-  spartanDir: string;
-  logger: Logger;
-  replicas?: number;
-  txIntervalSeconds?: number;
-  followChain?: string;
-  mnemonic?: string;
-  mnemonicStartIndex?: number | string;
-  botPrivateKey?: string;
-  nodeUrl?: string;
-  timeout?: string;
-  reuseValues?: boolean;
-  aztecSlotDuration?: number;
-}) {
-  const instanceName = `${namespace}-bot-transfers`;
-  const helmChartDir = getChartDir(spartanDir, 'aztec-bot');
-  const resolvedNodeUrl = nodeUrl ?? `http://${namespace}-rpc-aztec-node.${namespace}.svc.cluster.local:8080`;
-
-  logger.info(`Installing/upgrading transfer bot: replicas=${replicas}, followChain=${followChain}`);
-
-  const values: Record<string, string | number | boolean> = {
-    'bot.replicaCount': replicas,
-    'bot.txIntervalSeconds': txIntervalSeconds,
-    'bot.followChain': followChain,
-    'bot.botPrivateKey': botPrivateKey,
-    'bot.nodeUrl': resolvedNodeUrl,
-    'bot.mnemonic': mnemonic,
-    'bot.feePaymentMethod': 'fee_juice',
-    'aztec.slotDuration': aztecSlotDuration,
-    // Ensure bot can reach its own PXE started in-process (default rpc.port is 8080)
-    // Note: since aztec-bot depends on aztec-node with alias `bot`, env vars go under `bot.node.env`.
-    'bot.node.env.BOT_PXE_URL': 'http://127.0.0.1:8080',
-    // Provide L1 execution RPC for bridging fee juice
-    'bot.node.env.ETHEREUM_HOSTS': `http://${namespace}-eth-execution.${namespace}.svc.cluster.local:8545`,
-    // Provide L1 mnemonic for bridging (falls back to labs mnemonic)
-    'bot.node.env.BOT_L1_MNEMONIC': mnemonic,
-
-    // The bot does not need Kubernetes API access. Disable RBAC + ServiceAccount creation so the chart
-    // can be installed by users without cluster-scoped RBAC permissions.
-    'bot.rbac.create': false,
-    'bot.serviceAccount.create': false,
-    'bot.serviceAccount.name': 'default',
-  };
-  // Ensure we derive a funded L1 key (index 0 is funded on anvil default mnemonic)
-  if (mnemonicStartIndex === undefined) {
-    values['bot.mnemonicStartIndex'] = 0;
-  }
-  // Also pass a funded private key directly if available
-  if (process.env.FUNDING_PRIVATE_KEY) {
-    values['bot.node.env.BOT_L1_PRIVATE_KEY'] = process.env.FUNDING_PRIVATE_KEY;
-  }
-  // Align bot image with the running network image: prefer env var, else detect from a validator pod
-  let repositoryFromEnv: string | undefined;
-  let tagFromEnv: string | undefined;
-  const aztecDockerImage = process.env.AZTEC_DOCKER_IMAGE;
-  if (aztecDockerImage && aztecDockerImage.includes(':')) {
-    const lastColon = aztecDockerImage.lastIndexOf(':');
-    repositoryFromEnv = aztecDockerImage.slice(0, lastColon);
-    tagFromEnv = aztecDockerImage.slice(lastColon + 1);
-  }
-
-  let repository = repositoryFromEnv;
-  let tag = tagFromEnv;
-  if (!repository || !tag) {
-    try {
-      const { stdout } = await execAsync(
-        `kubectl get pods -l app.kubernetes.io/name=validator -n ${namespace} -o jsonpath='{.items[0].spec.containers[?(@.name=="aztec")].image}' | cat`,
-      );
-      const image = stdout.trim().replace(/^'|'$/g, '');
-      if (image && image.includes(':')) {
-        const lastColon = image.lastIndexOf(':');
-        repository = image.slice(0, lastColon);
-        tag = image.slice(lastColon + 1);
-      }
-    } catch (err) {
-      logger.warn(`Could not detect aztec image from validator pod: ${String(err)}`);
-    }
-  }
-  if (repository && tag) {
-    values['global.aztecImage.repository'] = repository;
-    values['global.aztecImage.tag'] = tag;
-  }
-  if (mnemonicStartIndex !== undefined) {
-    values['bot.mnemonicStartIndex'] =
-      typeof mnemonicStartIndex === 'string' ? mnemonicStartIndex : Number(mnemonicStartIndex);
-  }
-
-  // If a previous install attempt left the release in a non-deployed state (e.g. FAILED),
-  // `helm upgrade --install` can error with "has no deployed releases".
-  // In that case, clear the release record and do a clean install.
-  const existingStatus = await getHelmReleaseStatus(instanceName, namespace);
-  if (existingStatus && existingStatus.toLowerCase() !== 'deployed') {
-    logger.warn(`Transfer bot release ${instanceName} is in status '${existingStatus}'. Reinstalling cleanly.`);
-    await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`).catch(
-      () => undefined,
-    );
-    // If helm left the release in `uninstalling`, force-delete the record so we can reinstall.
-    const afterUninstallStatus = await getHelmReleaseStatus(instanceName, namespace);
-    if (afterUninstallStatus?.toLowerCase() === 'uninstalling') {
-      await forceDeleteHelmReleaseRecord(instanceName, namespace, logger);
-    }
-  }
-
-  // `--reuse-values` fails if the release has never successfully deployed (e.g. first install, or a previous failed install).
-  // Only reuse values when we have a deployed release to reuse from.
-  const effectiveReuseValues = reuseValues && (await hasDeployedHelmRelease(instanceName, namespace));
-
-  await execHelmCommand({
-    instanceName,
-    helmChartDir,
-    namespace,
-    valuesFile: undefined,
-    timeout,
-    values: values as unknown as Record<string, string | number | boolean>,
-    reuseValues: effectiveReuseValues,
-  });
-
-  if (replicas > 0) {
-    await waitForResourceByLabel({
-      resource: 'pods',
-      namespace,
-      label: 'app.kubernetes.io/name=bot',
-      condition: 'PodReadyToStartContainers',
-    });
-  }
-}
-
-/**
- * Uninstalls the transfer bot Helm release from the given namespace.
- * Intended for test teardown to clean up bot resources.
- */
-export async function uninstallTransferBot(namespace: string, logger: Logger) {
-  const instanceName = `${namespace}-bot-transfers`;
-  logger.info(`Uninstalling transfer bot release ${instanceName}`);
-  await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`);
-  // Ensure any leftover pods are removed
-  await deleteResourceByLabel({ resource: 'pods', namespace, label: 'app.kubernetes.io/name=bot' }).catch(
-    () => undefined,
-  );
-}
-
-/**
- * Enables or disables probabilistic transaction dropping on validators and waits for rollout.
- * Wired to env vars P2P_DROP_TX and P2P_DROP_TX_CHANCE via Helm values.
- */
-export async function setValidatorTxDrop({
-  namespace,
-  enabled,
-  probability,
-  logger,
-}: {
-  namespace: string;
-  enabled: boolean;
-  probability: number;
-  logger: Logger;
-}) {
-  const drop = enabled ? 'true' : 'false';
-  const prob = String(probability);
-
-  const selectors = ['app.kubernetes.io/name=validator', 'app.kubernetes.io/component=validator', 'app=validator'];
-  let updated = false;
-  for (const selector of selectors) {
-    try {
-      const list = await execAsync(`kubectl get statefulset -l ${selector} -n ${namespace} --no-headers -o name | cat`);
-      const names = list.stdout
-        .split('\n')
-        .map(s => s.trim())
-        .filter(Boolean);
-      if (names.length === 0) {
-        continue;
-      }
-      const cmd = `kubectl set env statefulset -l ${selector} -n ${namespace} P2P_DROP_TX=${drop} P2P_DROP_TX_CHANCE=${prob}`;
-      logger.info(`command: ${cmd}`);
-      await execAsync(cmd);
-      updated = true;
-    } catch (e) {
-      logger.warn(`Failed to update validators with selector ${selector}: ${String(e)}`);
-    }
-  }
-
-  if (!updated) {
-    logger.warn(`No validator StatefulSets found in ${namespace}. Skipping tx drop toggle.`);
-    return;
-  }
-
-  // Restart validator pods to ensure env vars take effect and wait for readiness
-  await restartValidators(namespace, logger);
-}
-
-export async function restartValidators(namespace: string, logger: Logger) {
-  const selectors = ['app.kubernetes.io/name=validator', 'app.kubernetes.io/component=validator', 'app=validator'];
-  let any = false;
-  for (const selector of selectors) {
-    try {
-      const { stdout } = await execAsync(`kubectl get pods -l ${selector} -n ${namespace} --no-headers -o name | cat`);
-      if (!stdout || stdout.trim().length === 0) {
-        continue;
-      }
-      any = true;
-      await deleteResourceByLabel({ resource: 'pods', namespace, label: selector });
-    } catch (e) {
-      logger.warn(`Error restarting validator pods with selector ${selector}: ${String(e)}`);
-    }
-  }
-
-  if (!any) {
-    logger.warn(`No validator pods found to restart in ${namespace}.`);
-    return;
-  }
-
-  // Wait for either label to be Ready
-  for (const selector of selectors) {
-    try {
-      await waitForResourceByLabel({ resource: 'pods', namespace, label: selector });
-      return;
-    } catch {
-      // try next
-    }
-  }
-  logger.warn(`Validator pods did not report Ready; continuing.`);
-}
-
-export async function enableValidatorDynamicBootNode(
-  instanceName: string,
-  namespace: string,
-  spartanDir: string,
-  logger: Logger,
-) {
-  logger.info(`Enabling validator dynamic boot node`);
-  await execHelmCommand({
-    instanceName,
-    namespace,
-    helmChartDir: getChartDir(spartanDir, 'aztec-network'),
-    values: {
-      'validator.dynamicBootNode': 'true',
-    },
-    valuesFile: undefined,
-    timeout: '15m',
-    reuseValues: true,
-  });
-
-  logger.info(`Validator dynamic boot node enabled`);
-}
-
-export async function getSequencers(namespace: string) {
-  const selectors = [
-    'app.kubernetes.io/name=validator',
-    'app.kubernetes.io/component=validator',
-    'app.kubernetes.io/component=sequencer-node',
-    'app=validator',
-  ];
-  for (const selector of selectors) {
-    try {
-      const command = `kubectl get pods -l ${selector} -n ${namespace} -o jsonpath='{.items[*].metadata.name}'`;
-      const { stdout } = await execAsync(command);
-      const sequencers = stdout
-        .split(' ')
-        .map(s => s.trim())
-        .filter(Boolean);
-      if (sequencers.length > 0) {
-        logger.verbose(`Found sequencer pods ${sequencers.join(', ')} (selector=${selector})`);
-        return sequencers;
-      }
-    } catch {
-      // try next selector
-    }
-  }
-
-  // Fail fast instead of returning [''] which leads to attempts to port-forward `pod/`.
-  throw new Error(
-    `No sequencer/validator pods found in namespace ${namespace}. Tried selectors: ${selectors.join(', ')}`,
-  );
-}
-
-export function updateSequencersConfig(env: TestConfig, config: Partial<AztecNodeAdminConfig>) {
-  return withSequencersAdmin(env, async client => {
-    await client.setConfig(config);
-    return client.getConfig();
-  });
-}
-
-export function getSequencersConfig(env: TestConfig) {
-  return withSequencersAdmin(env, client => client.getConfig());
-}
-
-export async function withSequencersAdmin<T>(env: TestConfig, fn: (node: AztecNodeAdmin) => Promise<T>): Promise<T[]> {
-  const adminContainerPort = 8880;
-  const namespace = env.NAMESPACE;
-  const sequencers = await getSequencers(namespace);
-  const results = [];
-
-  for (const sequencer of sequencers) {
-    const { process, port } = await startPortForward({
-      resource: `pod/${sequencer}`,
-      namespace,
-      containerPort: adminContainerPort,
-    });
-
-    const url = `http://localhost:${port}`;
-    await retry(
-      () => fetch(`${url}/status`).then(res => res.status === 200),
-      'forward node admin port',
-      makeBackoff([1, 1, 2, 6]),
-      logger,
-      true,
-    );
-    const client = createAztecNodeAdminClient(url);
-    results.push(await fn(client));
-    process.kill();
-  }
-
-  return results;
-}
-
-/**
- * Returns a public viem client to the eth execution node. If it was part of a local eth devnet,
- * it first port-forwards the service and points to it. Otherwise, just uses the external RPC url.
- */
-export async function getPublicViemClient(
-  env: TestConfig,
-  /** If set, will push the new process into it */
-  processes?: ChildProcess[],
-): Promise<{ url: string; client: ViemPublicClient; process?: ChildProcess }> {
-  const { NAMESPACE, CREATE_ETH_DEVNET, L1_RPC_URLS_JSON } = env;
-  if (CREATE_ETH_DEVNET) {
-    logger.info(`Creating port forward to eth execution node`);
-    const { process, port } = await startPortForward({
-      resource: `svc/${NAMESPACE}-eth-execution`,
-      namespace: NAMESPACE,
-      containerPort: 8545,
-    });
-    const url = `http://127.0.0.1:${port}`;
-    const client: ViemPublicClient = createPublicClient({ transport: fallback([http(url, { batch: false })]) });
-    if (processes) {
-      processes.push(process);
-    }
-    return { url, client, process };
-  } else {
-    logger.info(`Connecting to the eth execution node at ${L1_RPC_URLS_JSON}`);
-    if (!L1_RPC_URLS_JSON) {
-      throw new Error(`L1_RPC_URLS_JSON is not defined`);
-    }
-    const client: ViemPublicClient = createPublicClient({
-      transport: fallback([http(L1_RPC_URLS_JSON, { batch: false })]),
-    });
-    return { url: L1_RPC_URLS_JSON, client };
-  }
-}
-
-/** Queries an Aztec node for the L1 deployment addresses */
-export async function getL1DeploymentAddresses(env: TestConfig): Promise<L1ContractAddresses> {
-  let forwardProcess: ChildProcess | undefined;
-  try {
-    const [sequencer] = await getSequencers(env.NAMESPACE);
-    const { process, port } = await startPortForward({
-      resource: `pod/${sequencer}`,
-      namespace: env.NAMESPACE,
-      containerPort: 8080,
-    });
-
-    forwardProcess = process;
-    const url = `http://127.0.0.1:${port}`;
-    const node = createAztecNodeClient(url);
-    return await retry(
-      () => node.getNodeInfo().then(i => i.l1ContractAddresses),
-      'get node info',
-      makeBackoff([1, 3, 6]),
-      logger,
-    );
-  } finally {
-    forwardProcess?.kill();
-  }
-}
-
-/**
- * Rolls the Aztec pods in the given namespace.
- * @param namespace - The namespace to roll the Aztec pods in.
- * @param clearState - If true, also deletes the underlying PVCs to clear persistent storage.
- *        This is required for rollup upgrades where the old state is incompatible with the new rollup.
- *        Defaults to false, which preserves the existing storage.
- */
-export async function rollAztecPods(namespace: string, clearState: boolean = false) {
-  // Pod components use 'validator', but StatefulSets and PVCs use 'sequencer-node' for validators
-  const podComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'prover-agent', 'sequencer-node', 'rpc'];
-  const pvcComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'sequencer-node', 'rpc'];
-  // StatefulSet components that need to be scaled down before PVC deletion
-  // Note: validators use 'sequencer-node' as component label, not 'validator'
-  const statefulSetComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'sequencer-node', 'rpc'];
-
-  if (clearState) {
-    // To delete PVCs, we must first scale down StatefulSets so pods release the volumes
-    // Otherwise PVC deletion will hang waiting for pods to terminate
-
-    // First, save original replica counts
-    const originalReplicas: Map<string, number> = new Map();
-    for (const component of statefulSetComponents) {
-      try {
-        const getCmd = `kubectl get statefulset -l app.kubernetes.io/component=${component} -n ${namespace} -o jsonpath='{.items[0].spec.replicas}'`;
-        const { stdout } = await execAsync(getCmd);
-        const replicas = parseInt(stdout.replace(/'/g, '').trim(), 10);
-        if (!isNaN(replicas) && replicas > 0) {
-          originalReplicas.set(component, replicas);
-        }
-      } catch {
-        // Component might not exist, continue
-      }
-    }
-
-    // Scale down to 0
-    for (const component of statefulSetComponents) {
-      try {
-        const scaleCmd = `kubectl scale statefulset -l app.kubernetes.io/component=${component} -n ${namespace} --replicas=0 --timeout=2m`;
-        logger.info(`command: ${scaleCmd}`);
-        await execAsync(scaleCmd);
-      } catch (e) {
-        // Component might not exist or might be a Deployment, continue
-        logger.verbose(`Scale down ${component} skipped: ${e}`);
-      }
-    }
-
-    // Wait for pods to terminate
-    await sleep(15 * 1000);
-
-    // Now delete PVCs (they should no longer be in use)
-    for (const component of pvcComponents) {
-      await deleteResourceByLabel({
-        resource: 'persistentvolumeclaims',
-        namespace: namespace,
-        label: `app.kubernetes.io/component=${component}`,
-      });
-    }
-
-    // Scale StatefulSets back up to original replica counts
-    for (const component of statefulSetComponents) {
-      const replicas = originalReplicas.get(component) ?? 1;
-      try {
-        const scaleCmd = `kubectl scale statefulset -l app.kubernetes.io/component=${component} -n ${namespace} --replicas=${replicas} --timeout=2m`;
-        logger.info(`command: ${scaleCmd}`);
-        await execAsync(scaleCmd);
-      } catch (e) {
-        logger.verbose(`Scale up ${component} skipped: ${e}`);
-      }
-    }
-  } else {
-    // Just delete pods (no state clearing)
-    for (const component of podComponents) {
-      await deleteResourceByLabel({
-        resource: 'pods',
-        namespace: namespace,
-        label: `app.kubernetes.io/component=${component}`,
-      });
-    }
-  }
-
-  await sleep(10 * 1000);
-
-  // Wait for pods to come back
-  for (const component of podComponents) {
-    await waitForResourceByLabel({
-      resource: 'pods',
-      namespace: namespace,
-      label: `app.kubernetes.io/component=${component}`,
-    });
-  }
-}
-
-/**
- * Returns the absolute path to the git repository root
- */
-export function getGitProjectRoot(): string {
-  try {
-    const rootDir = execSync('git rev-parse --show-toplevel', {
-      encoding: 'utf-8',
-      stdio: ['ignore', 'pipe', 'ignore'],
-    }).trim();
-
-    return rootDir;
-  } catch (error) {
-    throw new Error(`Failed to determine git project root: ${error}`);
-  }
-}
-
-/** Returns a client to the RPC of the given sequencer (defaults to first) */
-export async function getNodeClient(
-  env: TestConfig,
-  index: number = 0,
-): Promise<{ node: ReturnType<typeof createAztecNodeClient>; port: number; process: ChildProcess }> {
-  const namespace = env.NAMESPACE;
-  const containerPort = 8080;
-  const sequencers = await getSequencers(namespace);
-  const sequencer = sequencers[index];
-  if (!sequencer) {
-    throw new Error(`No sequencer found at index ${index} in namespace ${namespace}`);
-  }
-
-  const { process, port } = await startPortForward({
-    resource: `pod/${sequencer}`,
-    namespace,
-    containerPort,
-  });
-
-  const url = `http://localhost:${port}`;
-  await retry(
-    () => fetch(`${url}/status`).then(res => res.status === 200),
-    'forward port',
-    makeBackoff([1, 1, 2, 6]),
-    logger,
-    true,
-  );
-
-  const client = createAztecNodeClient(url);
-  return { node: client, port, process };
-}
+export * from './utils/index.js';