@aztec/end-to-end 0.0.0-test.1 → 0.0.1-commit.1142ef1

package/src/spartan/utils.ts

@@ -1,89 +1,90 @@
-import { createAztecNodeClient, createLogger, sleep } from '@aztec/aztec.js';
-import type { RollupCheatCodes } from '@aztec/aztec.js/ethereum';
+import { createLogger } from '@aztec/aztec.js/log';
+import type { RollupCheatCodes } from '@aztec/aztec/testing';
+import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses';
+import type { ViemPublicClient } from '@aztec/ethereum/types';
+import type { CheckpointNumber } from '@aztec/foundation/branded-types';
 import type { Logger } from '@aztec/foundation/log';
-import type { SequencerConfig } from '@aztec/sequencer-client';
+import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { makeBackoff, retry } from '@aztec/foundation/retry';
+import { schemas } from '@aztec/foundation/schemas';
+import { sleep } from '@aztec/foundation/sleep';
+import {
+  type AztecNodeAdmin,
+  type AztecNodeAdminConfig,
+  createAztecNodeAdminClient,
+  createAztecNodeClient,
+} from '@aztec/stdlib/interfaces/client';
 
 import { ChildProcess, exec, execSync, spawn } from 'child_process';
 import path from 'path';
 import { promisify } from 'util';
+import { createPublicClient, fallback, http } from 'viem';
 import { z } from 'zod';
 
-import { AlertChecker, type AlertConfig } from '../quality_of_service/alert_checker.js';
-
 const execAsync = promisify(exec);
 
 const logger = createLogger('e2e:k8s-utils');
 
-const ethereumHostsSchema = z.string().refine(
-  str =>
-    str.split(',').every(url => {
-      try {
-        new URL(url.trim());
-        return true;
-      } catch {
-        return false;
-      }
-    }),
-  'ETHEREUM_HOSTS must be a comma-separated list of valid URLs',
-);
-
-const k8sLocalConfigSchema = z.object({
-  ETHEREUM_SLOT_DURATION: z.coerce.number().min(1, 'ETHEREUM_SLOT_DURATION env variable must be set'),
-  AZTEC_SLOT_DURATION: z.coerce.number().min(1, 'AZTEC_SLOT_DURATION env variable must be set'),
-  AZTEC_EPOCH_DURATION: z.coerce.number().min(1, 'AZTEC_EPOCH_DURATION env variable must be set'),
-  AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().min(1, 'AZTEC_PROOF_SUBMISSION_WINDOW env variable must be set'),
-  INSTANCE_NAME: z.string().min(1, 'INSTANCE_NAME env variable must be set'),
-  NAMESPACE: z.string().min(1, 'NAMESPACE env variable must be set'),
-  CONTAINER_NODE_PORT: z.coerce.number().default(8080),
-  CONTAINER_SEQUENCER_PORT: z.coerce.number().default(8080),
-  CONTAINER_PROVER_NODE_PORT: z.coerce.number().default(8080),
-  CONTAINER_PXE_PORT: z.coerce.number().default(8080),
-  CONTAINER_ETHEREUM_PORT: z.coerce.number().default(8545),
-  CONTAINER_METRICS_PORT: z.coerce.number().default(80),
-  GRAFANA_PASSWORD: z.string().optional(),
-  METRICS_API_PATH: z.string().default('/api/datasources/proxy/uid/spartan-metrics-prometheus/api/v1'),
-  SPARTAN_DIR: z.string().min(1, 'SPARTAN_DIR env variable must be set'),
-  ETHEREUM_HOSTS: ethereumHostsSchema.optional(),
-  L1_ACCOUNT_MNEMONIC: z.string().default('test test test test test test test test test test test junk'),
-  SEPOLIA_RUN: z.string().default('false'),
-  K8S: z.literal('local'),
-});
-
-const k8sGCloudConfigSchema = k8sLocalConfigSchema.extend({
-  K8S: z.literal('gcloud'),
-  CLUSTER_NAME: z.string().min(1, 'CLUSTER_NAME env variable must be set'),
-  REGION: z.string().min(1, 'REGION env variable must be set'),
+const testConfigSchema = z.object({
+  NAMESPACE: z.string().default('scenario'),
+  REAL_VERIFIER: schemas.Boolean.optional().default(true),
+  CREATE_ETH_DEVNET: schemas.Boolean.optional().default(false),
+  L1_RPC_URLS_JSON: z.string().optional(),
+  L1_ACCOUNT_MNEMONIC: z.string().optional(),
+  AZTEC_SLOT_DURATION: z.coerce.number().optional().default(24),
+  AZTEC_EPOCH_DURATION: z.coerce.number().optional().default(32),
+  AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5),
 });
 
-const directConfigSchema = z.object({
-  PXE_URL: z.string().url('PXE_URL must be a valid URL'),
-  NODE_URL: z.string().url('NODE_URL must be a valid URL'),
-  ETHEREUM_HOSTS: ethereumHostsSchema,
-  K8S: z.literal('false'),
-});
+export type TestConfig = z.infer<typeof testConfigSchema>;
 
-const envSchema = z.discriminatedUnion('K8S', [k8sLocalConfigSchema, k8sGCloudConfigSchema, directConfigSchema]);
+export function setupEnvironment(env: unknown): TestConfig {
+  const config = testConfigSchema.parse(env);
+  logger.warn(`Loaded env config`, config);
+  return config;
+}
 
-export type K8sLocalConfig = z.infer<typeof k8sLocalConfigSchema>;
-export type K8sGCloudConfig = z.infer<typeof k8sGCloudConfigSchema>;
-export type DirectConfig = z.infer<typeof directConfigSchema>;
-export type EnvConfig = z.infer<typeof envSchema>;
+/**
+ * @param path - The path to the script, relative to the project root
+ * @param args - The arguments to pass to the script
+ * @param logger - The logger to use
+ * @returns The exit code of the script
+ */
+function runScript(path: string, args: string[], logger: Logger, env?: Record<string, string>) {
+  const childProcess = spawn(path, args, {
+    stdio: ['ignore', 'pipe', 'pipe'],
+    env: env ? { ...process.env, ...env } : process.env,
+  });
+  return new Promise<number>((resolve, reject) => {
+    childProcess.on('close', (code: number | null) => resolve(code ?? 0));
+    childProcess.on('error', reject);
+    childProcess.stdout?.on('data', (data: Buffer) => {
+      logger.info(data.toString());
+    });
+    childProcess.stderr?.on('data', (data: Buffer) => {
+      logger.error(data.toString());
+    });
+  });
+}
 
-export function isK8sConfig(config: EnvConfig): config is K8sLocalConfig | K8sGCloudConfig {
-  return config.K8S === 'local' || config.K8S === 'gcloud';
+export function getAztecBin() {
+  return path.join(getGitProjectRoot(), 'yarn-project/aztec/dest/bin/index.js');
 }
 
-export function isGCloudConfig(config: EnvConfig): config is K8sGCloudConfig {
-  return config.K8S === 'gcloud';
+/**
+ * Runs the Aztec binary
+ * @param args - The arguments to pass to the Aztec binary
+ * @param logger - The logger to use
+ * @param env - Optional environment variables to set for the process
+ * @returns The exit code of the Aztec binary
+ */
+export function runAztecBin(args: string[], logger: Logger, env?: Record<string, string>) {
+  return runScript('node', [getAztecBin(), ...args], logger, env);
 }
 
-export function setupEnvironment(env: unknown): EnvConfig {
-  const config = envSchema.parse(env);
-  if (isGCloudConfig(config)) {
-    const command = `gcloud container clusters get-credentials ${config.CLUSTER_NAME} --region=${config.REGION}`;
-    execSync(command);
-  }
-  return config;
+export function runProjectScript(script: string, args: string[], logger: Logger, env?: Record<string, string>) {
+  const scriptPath = script.startsWith('/') ? script : path.join(getGitProjectRoot(), script);
+  return runScript(scriptPath, args, logger, env);
 }
 
 export async function startPortForward({
@@ -103,7 +104,7 @@ export async function startPortForward({
 }> {
   const hostPortAsString = hostPort ? hostPort.toString() : '';
 
-  logger.info(`kubectl port-forward -n ${namespace} ${resource} ${hostPortAsString}:${containerPort}`);
+  logger.debug(`kubectl port-forward -n ${namespace} ${resource} ${hostPortAsString}:${containerPort}`);
 
   const process = spawn(
     'kubectl',
@@ -121,21 +122,20 @@ export async function startPortForward({
       const str = data.toString() as string;
       if (!isResolved && str.includes('Forwarding from')) {
         isResolved = true;
-        logger.info(str);
+        logger.debug(`Port forward for ${resource}: ${str}`);
         const port = str.search(/:\d+/);
         if (port === -1) {
           throw new Error('Port not found in port forward output');
         }
         const portNumber = parseInt(str.slice(port + 1));
-        logger.info(`Port forward connected: ${portNumber}`);
-        logger.info(`Port forward connected: ${portNumber}`);
+        logger.verbose(`Port forwarded for ${resource} at ${portNumber}:${containerPort}`);
         resolve(portNumber);
       } else {
         logger.silent(str);
       }
     });
     process.stderr?.on('data', data => {
-      logger.info(data.toString());
+      logger.verbose(`Port forward for ${resource}: ${data.toString()}`);
       // It's a strange thing:
       // If we don't pipe stderr, then the port forwarding does not work.
       // Log to silent because this doesn't actually report errors,
@@ -145,16 +145,16 @@ export async function startPortForward({
     process.on('close', () => {
       if (!isResolved) {
         isResolved = true;
-        logger.warn('Port forward closed before connection established');
+        logger.warn(`Port forward for ${resource} closed before connection established`);
         resolve(0);
       }
     });
     process.on('error', error => {
-      logger.error(`Port forward error: ${error}`);
+      logger.error(`Port forward for ${resource} error: ${error}`);
       resolve(0);
     });
     process.on('exit', code => {
-      logger.info(`Port forward exited with code ${code}`);
+      logger.verbose(`Port forward for ${resource} exited with code ${code}`);
       resolve(0);
     });
   });
@@ -164,6 +164,55 @@ export async function startPortForward({
   return { process, port };
 }
 
+export function getExternalIP(namespace: string, serviceName: string): Promise<string> {
+  const { promise, resolve, reject } = promiseWithResolvers<string>();
+  const process = spawn(
+    'kubectl',
+    [
+      'get',
+      'service',
+      '-n',
+      namespace,
+      `${namespace}-${serviceName}`,
+      '--output',
+      "jsonpath='{.status.loadBalancer.ingress[0].ip}'",
+    ],
+    {
+      stdio: 'pipe',
+    },
+  );
+
+  let ip = '';
+  process.stdout.on('data', data => {
+    ip += data;
+  });
+  process.on('error', err => {
+    reject(err);
+  });
+  process.on('exit', () => {
+    // kubectl prints JSON. Remove the quotes
+    resolve(ip.replace(/"|'/g, ''));
+  });
+
+  return promise;
+}
+
+export function startPortForwardForRPC(namespace: string, index = 0) {
+  return startPortForward({
+    resource: `pod/${namespace}-rpc-aztec-node-${index}`,
+    namespace,
+    containerPort: 8080,
+  });
+}
+
+export function startPortForwardForEthereum(namespace: string) {
+  return startPortForward({
+    resource: `services/${namespace}-eth-execution`,
+    namespace,
+    containerPort: 8545,
+  });
+}
+
 export async function deleteResourceByName({
   resource,
   namespace,
@@ -187,12 +236,28 @@ export async function deleteResourceByLabel({
   resource,
   namespace,
   label,
+  timeout = '5m',
+  force = false,
 }: {
   resource: string;
   namespace: string;
   label: string;
+  timeout?: string;
+  force?: boolean;
 }) {
-  const command = `kubectl delete ${resource} -l ${label} -n ${namespace} --ignore-not-found=true --wait=true`;
+  try {
+    // Match both plain and group-qualified names (e.g., "podchaos" or "podchaos.chaos-mesh.org")
+    const escaped = resource.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
+    const regex = `(^|\\.)${escaped}(\\.|$)`;
+    await execAsync(`kubectl api-resources --no-headers -o name | grep -Eq '${regex}'`);
+  } catch (error) {
+    logger.warn(`Resource type '${resource}' not found in cluster, skipping deletion ${error}`);
+    return '';
+  }
+
+  const command = `kubectl delete ${resource} -l ${label} -n ${namespace} --ignore-not-found=true --wait=true --timeout=${timeout} ${
+    force ? '--force' : ''
+  }`;
   logger.info(`command: ${command}`);
   const { stdout } = await execAsync(command);
   return stdout;
@@ -217,13 +282,74 @@ export async function waitForResourceByLabel({
   return stdout;
 }
 
+export async function waitForResourceByName({
+  resource,
+  name,
+  namespace,
+  condition = 'Ready',
+  timeout = '10m',
+}: {
+  resource: string;
+  name: string;
+  namespace: string;
+  condition?: string;
+  timeout?: string;
+}) {
+  const command = `kubectl wait ${resource}/${name} --for=condition=${condition} -n ${namespace} --timeout=${timeout}`;
+  logger.info(`command: ${command}`);
+  const { stdout } = await execAsync(command);
+  return stdout;
+}
+
+export async function waitForResourcesByName({
+  resource,
+  names,
+  namespace,
+  condition = 'Ready',
+  timeout = '10m',
+}: {
+  resource: string;
+  names: string[];
+  namespace: string;
+  condition?: string;
+  timeout?: string;
+}) {
+  if (!names.length) {
+    throw new Error(`No ${resource} names provided to waitForResourcesByName`);
+  }
+
+  // Wait all in parallel; if any fails, surface which one.
+  await Promise.all(
+    names.map(async name => {
+      try {
+        await waitForResourceByName({ resource, name, namespace, condition, timeout });
+      } catch (err) {
+        throw new Error(
+          `Failed waiting for ${resource}/${name} condition=${condition} timeout=${timeout} namespace=${namespace}: ${String(
+            err,
+          )}`,
+        );
+      }
+    }),
+  );
+}
+
 export function getChartDir(spartanDir: string, chartName: string) {
   return path.join(spartanDir.trim(), chartName);
 }
 
-function valuesToArgs(values: Record<string, string | number>) {
+function shellQuote(value: string) {
+  // Single-quote safe shell escaping: ' -> '\''
+  return `'${value.replace(/'/g, "'\\''")}'`;
+}
+
+function valuesToArgs(values: Record<string, string | number | boolean>) {
   return Object.entries(values)
-    .map(([key, value]) => `--set ${key}=${value}`)
+    .map(([key, value]) =>
+      typeof value === 'number' || typeof value === 'boolean'
+        ? `--set ${key}=${value}`
+        : `--set-string ${key}=${shellQuote(String(value))}`,
+    )
     .join(' ');
 }
 
@@ -241,7 +367,7 @@ function createHelmCommand({
   namespace: string;
   valuesFile: string | undefined;
   timeout: string;
-  values: Record<string, string | number>;
+  values: Record<string, string | number | boolean>;
   reuseValues?: boolean;
 }) {
   const valuesFileArgs = valuesFile ? `--values ${helmChartDir}/values/${valuesFile}` : '';
@@ -258,6 +384,61 @@ async function execHelmCommand(args: Parameters<typeof createHelmCommand>[0]) {
   return stdout;
 }
 
+async function getHelmReleaseStatus(instanceName: string, namespace: string): Promise<string | undefined> {
+  try {
+    const { stdout } = await execAsync(
+      `helm list --namespace ${namespace} --all --filter '^${instanceName}$' --output json | cat`,
+    );
+    const parsed = JSON.parse(stdout) as Array<{ name?: string; status?: string }>;
+    const row = parsed.find(r => r.name === instanceName);
+    return row?.status;
+  } catch {
+    return undefined;
+  }
+}
+
+async function forceDeleteHelmReleaseRecord(instanceName: string, namespace: string, logger: Logger) {
+  const labelSelector = `owner=helm,name=${instanceName}`;
+  const cmd = `kubectl delete secret -n ${namespace} -l ${labelSelector} --ignore-not-found=true`;
+  logger.warn(`Force deleting Helm release record: ${cmd}`);
+  await execAsync(cmd).catch(() => undefined);
+}
+
+async function hasDeployedHelmRelease(instanceName: string, namespace: string): Promise<boolean> {
+  try {
+    const status = await getHelmReleaseStatus(instanceName, namespace);
+    return status?.toLowerCase() === 'deployed';
+  } catch {
+    return false;
+  }
+}
+
+export async function uninstallChaosMesh(instanceName: string, namespace: string, logger: Logger) {
+  // uninstall the helm chart if it exists
+  logger.info(`Uninstalling helm chart ${instanceName}`);
+  await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`);
+  // and delete the chaos-mesh resources created by this release
+  const deleteByLabel = async (resource: string) => {
+    const args = {
+      resource,
+      namespace: namespace,
+      label: `app.kubernetes.io/instance=${instanceName}`,
+    } as const;
+    logger.info(`Deleting ${resource} resources for release ${instanceName}`);
+    await deleteResourceByLabel(args).catch(e => {
+      logger.error(`Error deleting ${resource}: ${e}`);
+      logger.info(`Force deleting ${resource}`);
+      return deleteResourceByLabel({ ...args, force: true });
+    });
+  };
+
+  await deleteByLabel('podchaos');
+  await deleteByLabel('networkchaos');
+  await deleteByLabel('podnetworkchaos');
+  await deleteByLabel('workflows');
+  await deleteByLabel('workflownodes');
+}
+
 /**
  * Installs a Helm chart with the given parameters.
  * @param instanceName - The name of the Helm chart instance.
@@ -280,8 +461,7 @@ export async function installChaosMeshChart({
   targetNamespace,
   valuesFile,
   helmChartDir,
-  chaosMeshNamespace = 'chaos-mesh',
-  timeout = '5m',
+  timeout = '10m',
   clean = true,
   values = {},
   logger,
@@ -297,27 +477,13 @@ export async function installChaosMeshChart({
   logger: Logger;
 }) {
   if (clean) {
-    // uninstall the helm chart if it exists
-    logger.info(`Uninstalling helm chart ${instanceName}`);
-    await execAsync(`helm uninstall ${instanceName} --namespace ${chaosMeshNamespace} --wait --ignore-not-found`);
-    // and delete the podchaos resource
-    const deleteArgs = {
-      resource: 'podchaos',
-      namespace: chaosMeshNamespace,
-      name: `${targetNamespace}-${instanceName}`,
-    };
-    logger.info(`Deleting podchaos resource`);
-    await deleteResourceByName(deleteArgs).catch(e => {
-      logger.error(`Error deleting podchaos resource: ${e}`);
-      logger.info(`Force deleting podchaos resource`);
-      return deleteResourceByName({ ...deleteArgs, force: true });
-    });
+    await uninstallChaosMesh(instanceName, targetNamespace, logger);
   }
 
   return execHelmCommand({
     instanceName,
     helmChartDir,
-    namespace: chaosMeshNamespace,
+    namespace: targetNamespace,
     valuesFile,
     timeout,
     values: { ...values, 'global.targetNamespace': targetNamespace },
@@ -347,22 +513,49 @@ export function applyProverFailure({
   });
 }
 
+export function applyValidatorFailure({
+  namespace,
+  spartanDir,
+  logger,
+  values,
+  instanceName,
+}: {
+  namespace: string;
+  spartanDir: string;
+  logger: Logger;
+  values?: Record<string, string | number>;
+  instanceName?: string;
+}) {
+  return installChaosMeshChart({
+    instanceName: instanceName ?? 'validator-failure',
+    targetNamespace: namespace,
+    valuesFile: 'validator-failure.yaml',
+    helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+    values,
+    logger,
+  });
+}
+
 export function applyProverKill({
   namespace,
   spartanDir,
   logger,
+  values,
 }: {
   namespace: string;
   spartanDir: string;
   logger: Logger;
+  values?: Record<string, string | number>;
 }) {
   return installChaosMeshChart({
     instanceName: 'prover-kill',
     targetNamespace: namespace,
     valuesFile: 'prover-kill.yaml',
     helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+    chaosMeshNamespace: namespace,
     clean: true,
     logger,
+    values,
   });
 }
 
@@ -370,10 +563,12 @@ export function applyProverBrokerKill({
   namespace,
   spartanDir,
   logger,
+  values,
 }: {
   namespace: string;
   spartanDir: string;
   logger: Logger;
+  values?: Record<string, string | number>;
 }) {
   return installChaosMeshChart({
     instanceName: 'prover-broker-kill',
@@ -382,63 +577,79 @@ export function applyProverBrokerKill({
     helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
     clean: true,
     logger,
+    values,
   });
 }
 
 export function applyBootNodeFailure({
+  instanceName = 'boot-node-failure',
   namespace,
   spartanDir,
   durationSeconds,
   logger,
+  values,
 }: {
+  instanceName?: string;
   namespace: string;
   spartanDir: string;
   durationSeconds: number;
   logger: Logger;
+  values?: Record<string, string | number>;
 }) {
   return installChaosMeshChart({
-    instanceName: 'boot-node-failure',
+    instanceName,
     targetNamespace: namespace,
     valuesFile: 'boot-node-failure.yaml',
     helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
     values: {
       'bootNodeFailure.duration': `${durationSeconds}s`,
+      ...(values ?? {}),
     },
     logger,
   });
 }
 
 export function applyValidatorKill({
+  instanceName = 'validator-kill',
   namespace,
   spartanDir,
   logger,
+  values,
+  clean = true,
 }: {
+  instanceName?: string;
   namespace: string;
   spartanDir: string;
   logger: Logger;
+  values?: Record<string, string | number>;
+  clean?: boolean;
 }) {
   return installChaosMeshChart({
-    instanceName: 'validator-kill',
+    instanceName: instanceName ?? 'validator-kill',
     targetNamespace: namespace,
     valuesFile: 'validator-kill.yaml',
     helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+    clean,
     logger,
+    values,
   });
 }
 
 export function applyNetworkShaping({
+  instanceName = 'network-shaping',
   valuesFile,
   namespace,
   spartanDir,
   logger,
 }: {
+  instanceName?: string;
   valuesFile: string;
   namespace: string;
   spartanDir: string;
   logger: Logger;
 }) {
   return installChaosMeshChart({
-    instanceName: 'network-shaping',
+    instanceName,
     targetNamespace: namespace,
     valuesFile,
     helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
@@ -446,35 +657,283 @@ export function applyNetworkShaping({
   });
 }
 
-export async function awaitL2BlockNumber(
+export async function awaitCheckpointNumber(
   rollupCheatCodes: RollupCheatCodes,
-  blockNumber: bigint,
+  checkpointNumber: CheckpointNumber,
   timeoutSeconds: number,
   logger: Logger,
 ) {
-  logger.info(`Waiting for L2 Block ${blockNumber}`);
+  logger.info(`Waiting for checkpoint ${checkpointNumber}`);
   let tips = await rollupCheatCodes.getTips();
   const endTime = Date.now() + timeoutSeconds * 1000;
-  while (tips.pending < blockNumber && Date.now() < endTime) {
-    logger.info(`At L2 Block ${tips.pending}`);
+  while (tips.pending < checkpointNumber && Date.now() < endTime) {
+    logger.info(`At checkpoint ${tips.pending}`);
     await sleep(1000);
     tips = await rollupCheatCodes.getTips();
   }
-  if (tips.pending < blockNumber) {
-    throw new Error(`Timeout waiting for L2 Block ${blockNumber}, only reached ${tips.pending}`);
+  if (tips.pending < checkpointNumber) {
+    throw new Error(`Timeout waiting for checkpoint ${checkpointNumber}, only reached ${tips.pending}`);
   } else {
-    logger.info(`Reached L2 Block ${tips.pending}`);
+    logger.info(`Reached checkpoint ${tips.pending}`);
   }
 }
 
 export async function restartBot(namespace: string, logger: Logger) {
   logger.info(`Restarting bot`);
-  await deleteResourceByLabel({ resource: 'pods', namespace, label: 'app=bot' });
+  await deleteResourceByLabel({ resource: 'pods', namespace, label: 'app.kubernetes.io/name=bot' });
   await sleep(10 * 1000);
-  await waitForResourceByLabel({ resource: 'pods', namespace, label: 'app=bot' });
+  // Some bot images may take time to report Ready due to heavy boot-time proving.
+  // Waiting for PodReadyToStartContainers ensures the pod is scheduled and starting without blocking on full readiness.
+  await waitForResourceByLabel({
+    resource: 'pods',
+    namespace,
+    label: 'app.kubernetes.io/name=bot',
+    condition: 'PodReadyToStartContainers',
+  });
   logger.info(`Bot restarted`);
 }
 
+/**
+ * Installs or upgrades the transfer bot Helm release for the given namespace.
+ * Intended for test setup to enable L2 traffic generation only when needed.
+ */
+export async function installTransferBot({
+  namespace,
+  spartanDir,
+  logger,
+  replicas = 1,
+  txIntervalSeconds = 10,
+  followChain = 'PENDING',
+  mnemonic = process.env.LABS_INFRA_MNEMONIC ?? 'test test test test test test test test test test test junk',
+  mnemonicStartIndex,
+  botPrivateKey = process.env.BOT_TRANSFERS_L2_PRIVATE_KEY ?? '0xcafe01',
+  nodeUrl,
+  timeout = '15m',
+  reuseValues = true,
+  aztecSlotDuration = Number(process.env.AZTEC_SLOT_DURATION ?? 12),
+}: {
+  namespace: string;
+  spartanDir: string;
+  logger: Logger;
+  replicas?: number;
+  txIntervalSeconds?: number;
+  followChain?: string;
+  mnemonic?: string;
+  mnemonicStartIndex?: number | string;
+  botPrivateKey?: string;
+  nodeUrl?: string;
+  timeout?: string;
+  reuseValues?: boolean;
+  aztecSlotDuration?: number;
+}) {
+  const instanceName = `${namespace}-bot-transfers`;
+  const helmChartDir = getChartDir(spartanDir, 'aztec-bot');
+  const resolvedNodeUrl = nodeUrl ?? `http://${namespace}-rpc-aztec-node.${namespace}.svc.cluster.local:8080`;
+
+  logger.info(`Installing/upgrading transfer bot: replicas=${replicas}, followChain=${followChain}`);
+
+  const values: Record<string, string | number | boolean> = {
+    'bot.replicaCount': replicas,
+    'bot.txIntervalSeconds': txIntervalSeconds,
+    'bot.followChain': followChain,
+    'bot.botPrivateKey': botPrivateKey,
+    'bot.nodeUrl': resolvedNodeUrl,
+    'bot.mnemonic': mnemonic,
+    'bot.feePaymentMethod': 'fee_juice',
+    'aztec.slotDuration': aztecSlotDuration,
+    // Ensure bot can reach its own PXE started in-process (default rpc.port is 8080)
+    // Note: since aztec-bot depends on aztec-node with alias `bot`, env vars go under `bot.node.env`.
+    'bot.node.env.BOT_PXE_URL': 'http://127.0.0.1:8080',
+    // Provide L1 execution RPC for bridging fee juice
+    'bot.node.env.ETHEREUM_HOSTS': `http://${namespace}-eth-execution.${namespace}.svc.cluster.local:8545`,
+    // Provide L1 mnemonic for bridging (falls back to labs mnemonic)
+    'bot.node.env.BOT_L1_MNEMONIC': mnemonic,
+
+    // The bot does not need Kubernetes API access. Disable RBAC + ServiceAccount creation so the chart
+    // can be installed by users without cluster-scoped RBAC permissions.
+    'bot.rbac.create': false,
+    'bot.serviceAccount.create': false,
+    'bot.serviceAccount.name': 'default',
+  };
+  // Ensure we derive a funded L1 key (index 0 is funded on anvil default mnemonic)
+  if (mnemonicStartIndex === undefined) {
+    values['bot.mnemonicStartIndex'] = 0;
+  }
+  // Also pass a funded private key directly if available
+  if (process.env.FUNDING_PRIVATE_KEY) {
+    values['bot.node.env.BOT_L1_PRIVATE_KEY'] = process.env.FUNDING_PRIVATE_KEY;
+  }
+  // Align bot image with the running network image: prefer env var, else detect from a validator pod
+  let repositoryFromEnv: string | undefined;
+  let tagFromEnv: string | undefined;
+  const aztecDockerImage = process.env.AZTEC_DOCKER_IMAGE;
+  if (aztecDockerImage && aztecDockerImage.includes(':')) {
+    const lastColon = aztecDockerImage.lastIndexOf(':');
+    repositoryFromEnv = aztecDockerImage.slice(0, lastColon);
+    tagFromEnv = aztecDockerImage.slice(lastColon + 1);
+  }
+
+  let repository = repositoryFromEnv;
+  let tag = tagFromEnv;
+  if (!repository || !tag) {
+    try {
+      const { stdout } = await execAsync(
+        `kubectl get pods -l app.kubernetes.io/name=validator -n ${namespace} -o jsonpath='{.items[0].spec.containers[?(@.name=="aztec")].image}' | cat`,
+      );
+      const image = stdout.trim().replace(/^'|'$/g, '');
+      if (image && image.includes(':')) {
+        const lastColon = image.lastIndexOf(':');
+        repository = image.slice(0, lastColon);
+        tag = image.slice(lastColon + 1);
+      }
+    } catch (err) {
+      logger.warn(`Could not detect aztec image from validator pod: ${String(err)}`);
+    }
+  }
+  if (repository && tag) {
+    values['global.aztecImage.repository'] = repository;
+    values['global.aztecImage.tag'] = tag;
+  }
+  if (mnemonicStartIndex !== undefined) {
+    values['bot.mnemonicStartIndex'] =
+      typeof mnemonicStartIndex === 'string' ? mnemonicStartIndex : Number(mnemonicStartIndex);
+  }
+
+  // If a previous install attempt left the release in a non-deployed state (e.g. FAILED),
+  // `helm upgrade --install` can error with "has no deployed releases".
+  // In that case, clear the release record and do a clean install.
+  const existingStatus = await getHelmReleaseStatus(instanceName, namespace);
+  if (existingStatus && existingStatus.toLowerCase() !== 'deployed') {
+    logger.warn(`Transfer bot release ${instanceName} is in status '${existingStatus}'. Reinstalling cleanly.`);
+    await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`).catch(
+      () => undefined,
+    );
+    // If helm left the release in `uninstalling`, force-delete the record so we can reinstall.
+    const afterUninstallStatus = await getHelmReleaseStatus(instanceName, namespace);
+    if (afterUninstallStatus?.toLowerCase() === 'uninstalling') {
+      await forceDeleteHelmReleaseRecord(instanceName, namespace, logger);
+    }
+  }
+
+  // `--reuse-values` fails if the release has never successfully deployed (e.g. first install, or a previous failed install).
+  // Only reuse values when we have a deployed release to reuse from.
+  const effectiveReuseValues = reuseValues && (await hasDeployedHelmRelease(instanceName, namespace));
+
+  await execHelmCommand({
+    instanceName,
+    helmChartDir,
+    namespace,
+    valuesFile: undefined,
+    timeout,
+    values: values as unknown as Record<string, string | number | boolean>,
+    reuseValues: effectiveReuseValues,
+  });
+
+  if (replicas > 0) {
+    await waitForResourceByLabel({
+      resource: 'pods',
+      namespace,
+      label: 'app.kubernetes.io/name=bot',
+      condition: 'PodReadyToStartContainers',
+    });
+  }
+}
+
+/**
+ * Uninstalls the transfer bot Helm release from the given namespace.
+ * Intended for test teardown to clean up bot resources.
+ */
+export async function uninstallTransferBot(namespace: string, logger: Logger) {
+  const instanceName = `${namespace}-bot-transfers`;
+  logger.info(`Uninstalling transfer bot release ${instanceName}`);
+  await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`);
+  // Ensure any leftover pods are removed
+  await deleteResourceByLabel({ resource: 'pods', namespace, label: 'app.kubernetes.io/name=bot' }).catch(
+    () => undefined,
+  );
+}
+
+/**
+ * Enables or disables probabilistic transaction dropping on validators and waits for rollout.
+ * Wired to env vars P2P_DROP_TX and P2P_DROP_TX_CHANCE via Helm values.
+ */
+export async function setValidatorTxDrop({
+  namespace,
+  enabled,
+  probability,
+  logger,
+}: {
+  namespace: string;
+  enabled: boolean;
+  probability: number;
+  logger: Logger;
+}) {
+  const drop = enabled ? 'true' : 'false';
+  const prob = String(probability);
+
+  const selectors = ['app.kubernetes.io/name=validator', 'app.kubernetes.io/component=validator', 'app=validator'];
+  let updated = false;
+  for (const selector of selectors) {
+    try {
+      const list = await execAsync(`kubectl get statefulset -l ${selector} -n ${namespace} --no-headers -o name | cat`);
+      const names = list.stdout
+        .split('\n')
+        .map(s => s.trim())
+        .filter(Boolean);
+      if (names.length === 0) {
+        continue;
+      }
+      const cmd = `kubectl set env statefulset -l ${selector} -n ${namespace} P2P_DROP_TX=${drop} P2P_DROP_TX_CHANCE=${prob}`;
+      logger.info(`command: ${cmd}`);
+      await execAsync(cmd);
+      updated = true;
+    } catch (e) {
+      logger.warn(`Failed to update validators with selector ${selector}: ${String(e)}`);
+    }
+  }
+
+  if (!updated) {
+    logger.warn(`No validator StatefulSets found in ${namespace}. Skipping tx drop toggle.`);
+    return;
+  }
+
+  // Restart validator pods to ensure env vars take effect and wait for readiness
+  await restartValidators(namespace, logger);
+}
+
+export async function restartValidators(namespace: string, logger: Logger) {
+  const selectors = ['app.kubernetes.io/name=validator', 'app.kubernetes.io/component=validator', 'app=validator'];
+  let any = false;
+  for (const selector of selectors) {
+    try {
+      const { stdout } = await execAsync(`kubectl get pods -l ${selector} -n ${namespace} --no-headers -o name | cat`);
+      if (!stdout || stdout.trim().length === 0) {
+        continue;
+      }
+      any = true;
+      await deleteResourceByLabel({ resource: 'pods', namespace, label: selector });
+    } catch (e) {
+      logger.warn(`Error restarting validator pods with selector ${selector}: ${String(e)}`);
+    }
+  }
+
+  if (!any) {
+    logger.warn(`No validator pods found to restart in ${namespace}.`);
+    return;
+  }
+
+  // Wait for either label to be Ready
+  for (const selector of selectors) {
+    try {
+      await waitForResourceByLabel({ resource: 'pods', namespace, label: selector });
+      return;
+    } catch {
+      // try next
+    }
+  }
+  logger.warn(`Validator pods did not report Ready; continuing.`);
+}
+
 export async function enableValidatorDynamicBootNode(
   instanceName: string,
   namespace: string,
@@ -497,64 +956,133 @@ export async function enableValidatorDynamicBootNode(
   logger.info(`Validator dynamic boot node enabled`);
 }
 
-export async function runAlertCheck(config: EnvConfig, alerts: AlertConfig[], logger: Logger) {
-  if (isK8sConfig(config)) {
-    const { process, port } = await startPortForward({
-      resource: `svc/metrics-grafana`,
-      namespace: 'metrics',
-      containerPort: config.CONTAINER_METRICS_PORT,
-    });
-    const alertChecker = new AlertChecker(logger, {
-      grafanaEndpoint: `http://localhost:${port}${config.METRICS_API_PATH}`,
-      grafanaCredentials: `admin:${config.GRAFANA_PASSWORD}`,
-    });
-    await alertChecker.runAlertCheck(alerts);
-    process.kill();
-  } else {
-    logger.info('Not running alert check in non-k8s environment');
+export async function getSequencers(namespace: string) {
+  const selectors = [
+    'app.kubernetes.io/name=validator',
+    'app.kubernetes.io/component=validator',
+    'app.kubernetes.io/component=sequencer-node',
+    'app=validator',
+  ];
+  for (const selector of selectors) {
+    try {
+      const command = `kubectl get pods -l ${selector} -n ${namespace} -o jsonpath='{.items[*].metadata.name}'`;
+      const { stdout } = await execAsync(command);
+      const sequencers = stdout
+        .split(' ')
+        .map(s => s.trim())
+        .filter(Boolean);
+      if (sequencers.length > 0) {
+        logger.verbose(`Found sequencer pods ${sequencers.join(', ')} (selector=${selector})`);
+        return sequencers;
+      }
+    } catch {
+      // try next selector
+    }
   }
+
+  // Fail fast instead of returning [''] which leads to attempts to port-forward `pod/`.
+  throw new Error(
+    `No sequencer/validator pods found in namespace ${namespace}. Tried selectors: ${selectors.join(', ')}`,
+  );
 }
 
-export async function updateSequencerConfig(url: string, config: Partial<SequencerConfig>) {
-  const node = createAztecNodeClient(url);
-  await node.setConfig(config);
+export function updateSequencersConfig(env: TestConfig, config: Partial<AztecNodeAdminConfig>) {
+  return withSequencersAdmin(env, async client => {
+    await client.setConfig(config);
+    return client.getConfig();
+  });
 }
 
-export async function getSequencers(namespace: string) {
-  const command = `kubectl get pods -l app=validator -n ${namespace} -o jsonpath='{.items[*].metadata.name}'`;
-  const { stdout } = await execAsync(command);
-  return stdout.split(' ');
+export function getSequencersConfig(env: TestConfig) {
+  return withSequencersAdmin(env, client => client.getConfig());
 }
 
-export async function updateK8sSequencersConfig(args: {
-  containerPort: number;
-  namespace: string;
-  config: Partial<SequencerConfig>;
-}) {
-  const { containerPort, namespace, config } = args;
+export async function withSequencersAdmin<T>(env: TestConfig, fn: (node: AztecNodeAdmin) => Promise<T>): Promise<T[]> {
+  const adminContainerPort = 8880;
+  const namespace = env.NAMESPACE;
   const sequencers = await getSequencers(namespace);
+  const results = [];
+
   for (const sequencer of sequencers) {
     const { process, port } = await startPortForward({
       resource: `pod/${sequencer}`,
       namespace,
-      containerPort,
+      containerPort: adminContainerPort,
     });
 
     const url = `http://localhost:${port}`;
-    await updateSequencerConfig(url, config);
+    await retry(
+      () => fetch(`${url}/status`).then(res => res.status === 200),
+      'forward node admin port',
+      makeBackoff([1, 1, 2, 6]),
+      logger,
+      true,
+    );
+    const client = createAztecNodeAdminClient(url);
+    results.push(await fn(client));
     process.kill();
   }
+
+  return results;
 }
 
-export async function updateSequencersConfig(env: EnvConfig, config: Partial<SequencerConfig>) {
-  if (isK8sConfig(env)) {
-    await updateK8sSequencersConfig({
-      containerPort: env.CONTAINER_NODE_PORT,
-      namespace: env.NAMESPACE,
-      config,
+/**
+ * Returns a public viem client to the eth execution node. If it was part of a local eth devnet,
+ * it first port-forwards the service and points to it. Otherwise, just uses the external RPC url.
+ */
+export async function getPublicViemClient(
+  env: TestConfig,
+  /** If set, will push the new process into it */
+  processes?: ChildProcess[],
+): Promise<{ url: string; client: ViemPublicClient; process?: ChildProcess }> {
+  const { NAMESPACE, CREATE_ETH_DEVNET, L1_RPC_URLS_JSON } = env;
+  if (CREATE_ETH_DEVNET) {
+    logger.info(`Creating port forward to eth execution node`);
+    const { process, port } = await startPortForward({
+      resource: `svc/${NAMESPACE}-eth-execution`,
+      namespace: NAMESPACE,
+      containerPort: 8545,
     });
+    const url = `http://127.0.0.1:${port}`;
+    const client: ViemPublicClient = createPublicClient({ transport: fallback([http(url, { batch: false })]) });
+    if (processes) {
+      processes.push(process);
+    }
+    return { url, client, process };
   } else {
-    await updateSequencerConfig(env.NODE_URL, config);
+    logger.info(`Connecting to the eth execution node at ${L1_RPC_URLS_JSON}`);
+    if (!L1_RPC_URLS_JSON) {
+      throw new Error(`L1_RPC_URLS_JSON is not defined`);
+    }
+    const client: ViemPublicClient = createPublicClient({
+      transport: fallback([http(L1_RPC_URLS_JSON, { batch: false })]),
+    });
+    return { url: L1_RPC_URLS_JSON, client };
+  }
+}
+
+/** Queries an Aztec node for the L1 deployment addresses */
+export async function getL1DeploymentAddresses(env: TestConfig): Promise<L1ContractAddresses> {
+  let forwardProcess: ChildProcess | undefined;
+  try {
+    const [sequencer] = await getSequencers(env.NAMESPACE);
+    const { process, port } = await startPortForward({
+      resource: `pod/${sequencer}`,
+      namespace: env.NAMESPACE,
+      containerPort: 8080,
+    });
+
+    forwardProcess = process;
+    const url = `http://127.0.0.1:${port}`;
+    const node = createAztecNodeClient(url);
+    return await retry(
+      () => node.getNodeInfo().then(i => i.l1ContractAddresses),
+      'get node info',
+      makeBackoff([1, 3, 6]),
+      logger,
+    );
+  } finally {
+    forwardProcess?.kill();
   }
 }
 
@@ -580,3 +1108,51 @@ export async function rollAztecPods(namespace: string) {
   await waitForResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=validator' });
   await waitForResourceByLabel({ resource: 'pods', namespace: namespace, label: 'app=pxe' });
 }
+
+/**
+ * Returns the absolute path to the git repository root
+ */
+export function getGitProjectRoot(): string {
+  try {
+    const rootDir = execSync('git rev-parse --show-toplevel', {
+      encoding: 'utf-8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    }).trim();
+
+    return rootDir;
+  } catch (error) {
+    throw new Error(`Failed to determine git project root: ${error}`);
+  }
+}
+
+/** Returns a client to the RPC of the given sequencer (defaults to first) */
+export async function getNodeClient(
+  env: TestConfig,
+  index: number = 0,
+): Promise<{ node: ReturnType<typeof createAztecNodeClient>; port: number; process: ChildProcess }> {
+  const namespace = env.NAMESPACE;
+  const containerPort = 8080;
+  const sequencers = await getSequencers(namespace);
+  const sequencer = sequencers[index];
+  if (!sequencer) {
+    throw new Error(`No sequencer found at index ${index} in namespace ${namespace}`);
+  }
+
+  const { process, port } = await startPortForward({
+    resource: `pod/${sequencer}`,
+    namespace,
+    containerPort,
+  });
+
+  const url = `http://localhost:${port}`;
+  await retry(
+    () => fetch(`${url}/status`).then(res => res.status === 200),
+    'forward port',
+    makeBackoff([1, 1, 2, 6]),
+    logger,
+    true,
+  );
+
+  const client = createAztecNodeClient(url);
+  return { node: client, port, process };
+}