@aztec/end-to-end 0.0.0-test.1 → 0.0.1-commit.03f7ef2

package/dest/spartan/utils.js

@@ -1,74 +1,82 @@
-import { createAztecNodeClient, createLogger, sleep } from '@aztec/aztec.js';
+import { createLogger } from '@aztec/aztec.js/log';
+import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { makeBackoff, retry } from '@aztec/foundation/retry';
+import { schemas } from '@aztec/foundation/schemas';
+import { sleep } from '@aztec/foundation/sleep';
+import { createAztecNodeAdminClient, createAztecNodeClient } from '@aztec/stdlib/interfaces/client';
 import { exec, execSync, spawn } from 'child_process';
 import path from 'path';
 import { promisify } from 'util';
+import { createPublicClient, fallback, http } from 'viem';
 import { z } from 'zod';
-import { AlertChecker } from '../quality_of_service/alert_checker.js';
 const execAsync = promisify(exec);
 const logger = createLogger('e2e:k8s-utils');
-const ethereumHostsSchema = z.string().refine((str)=>str.split(',').every((url)=>{
-        try {
-            new URL(url.trim());
-            return true;
-        } catch  {
-            return false;
-        }
-    }), 'ETHEREUM_HOSTS must be a comma-separated list of valid URLs');
-const k8sLocalConfigSchema = z.object({
-    ETHEREUM_SLOT_DURATION: z.coerce.number().min(1, 'ETHEREUM_SLOT_DURATION env variable must be set'),
-    AZTEC_SLOT_DURATION: z.coerce.number().min(1, 'AZTEC_SLOT_DURATION env variable must be set'),
-    AZTEC_EPOCH_DURATION: z.coerce.number().min(1, 'AZTEC_EPOCH_DURATION env variable must be set'),
-    AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().min(1, 'AZTEC_PROOF_SUBMISSION_WINDOW env variable must be set'),
-    INSTANCE_NAME: z.string().min(1, 'INSTANCE_NAME env variable must be set'),
-    NAMESPACE: z.string().min(1, 'NAMESPACE env variable must be set'),
-    CONTAINER_NODE_PORT: z.coerce.number().default(8080),
-    CONTAINER_SEQUENCER_PORT: z.coerce.number().default(8080),
-    CONTAINER_PROVER_NODE_PORT: z.coerce.number().default(8080),
-    CONTAINER_PXE_PORT: z.coerce.number().default(8080),
-    CONTAINER_ETHEREUM_PORT: z.coerce.number().default(8545),
-    CONTAINER_METRICS_PORT: z.coerce.number().default(80),
-    GRAFANA_PASSWORD: z.string().optional(),
-    METRICS_API_PATH: z.string().default('/api/datasources/proxy/uid/spartan-metrics-prometheus/api/v1'),
-    SPARTAN_DIR: z.string().min(1, 'SPARTAN_DIR env variable must be set'),
-    ETHEREUM_HOSTS: ethereumHostsSchema.optional(),
-    L1_ACCOUNT_MNEMONIC: z.string().default('test test test test test test test test test test test junk'),
-    SEPOLIA_RUN: z.string().default('false'),
-    K8S: z.literal('local')
-});
-const k8sGCloudConfigSchema = k8sLocalConfigSchema.extend({
-    K8S: z.literal('gcloud'),
-    CLUSTER_NAME: z.string().min(1, 'CLUSTER_NAME env variable must be set'),
-    REGION: z.string().min(1, 'REGION env variable must be set')
-});
-const directConfigSchema = z.object({
-    PXE_URL: z.string().url('PXE_URL must be a valid URL'),
-    NODE_URL: z.string().url('NODE_URL must be a valid URL'),
-    ETHEREUM_HOSTS: ethereumHostsSchema,
-    K8S: z.literal('false')
+const testConfigSchema = z.object({
+    NAMESPACE: z.string().default('scenario'),
+    REAL_VERIFIER: schemas.Boolean.optional().default(true),
+    CREATE_ETH_DEVNET: schemas.Boolean.optional().default(false),
+    L1_RPC_URLS_JSON: z.string().optional(),
+    L1_ACCOUNT_MNEMONIC: z.string().optional(),
+    AZTEC_SLOT_DURATION: z.coerce.number().optional().default(24),
+    AZTEC_EPOCH_DURATION: z.coerce.number().optional().default(32),
+    AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5)
 });
-const envSchema = z.discriminatedUnion('K8S', [
-    k8sLocalConfigSchema,
-    k8sGCloudConfigSchema,
-    directConfigSchema
-]);
-export function isK8sConfig(config) {
-    return config.K8S === 'local' || config.K8S === 'gcloud';
-}
-export function isGCloudConfig(config) {
-    return config.K8S === 'gcloud';
-}
 export function setupEnvironment(env) {
-    const config = envSchema.parse(env);
-    if (isGCloudConfig(config)) {
-        const command = `gcloud container clusters get-credentials ${config.CLUSTER_NAME} --region=${config.REGION}`;
-        execSync(command);
-    }
+    const config = testConfigSchema.parse(env);
+    logger.warn(`Loaded env config`, config);
     return config;
 }
+/**
+ * @param path - The path to the script, relative to the project root
+ * @param args - The arguments to pass to the script
+ * @param logger - The logger to use
+ * @returns The exit code of the script
+ */ function runScript(path, args, logger, env) {
+    const childProcess = spawn(path, args, {
+        stdio: [
+            'ignore',
+            'pipe',
+            'pipe'
+        ],
+        env: env ? {
+            ...process.env,
+            ...env
+        } : process.env
+    });
+    return new Promise((resolve, reject)=>{
+        childProcess.on('close', (code)=>resolve(code ?? 0));
+        childProcess.on('error', reject);
+        childProcess.stdout?.on('data', (data)=>{
+            logger.info(data.toString());
+        });
+        childProcess.stderr?.on('data', (data)=>{
+            logger.error(data.toString());
+        });
+    });
+}
+export function getAztecBin() {
+    return path.join(getGitProjectRoot(), 'yarn-project/aztec/dest/bin/index.js');
+}
+/**
+ * Runs the Aztec binary
+ * @param args - The arguments to pass to the Aztec binary
+ * @param logger - The logger to use
+ * @param env - Optional environment variables to set for the process
+ * @returns The exit code of the Aztec binary
+ */ export function runAztecBin(args, logger, env) {
+    return runScript('node', [
+        getAztecBin(),
+        ...args
+    ], logger, env);
+}
+export function runProjectScript(script, args, logger, env) {
+    const scriptPath = script.startsWith('/') ? script : path.join(getGitProjectRoot(), script);
+    return runScript(scriptPath, args, logger, env);
+}
 export async function startPortForward({ resource, namespace, containerPort, hostPort }) {
     const hostPortAsString = hostPort ? hostPort.toString() : '';
-    logger.info(`kubectl port-forward -n ${namespace} ${resource} ${hostPortAsString}:${containerPort}`);
-    const process = spawn('kubectl', [
+    logger.debug(`kubectl port-forward -n ${namespace} ${resource} ${hostPortAsString}:${containerPort}`);
+    const process1 = spawn('kubectl', [
         'port-forward',
         '-n',
         namespace,
@@ -85,61 +93,106 @@ export async function startPortForward({ resource, namespace, containerPort, hos
     });
     let isResolved = false;
     const connected = new Promise((resolve)=>{
-        process.stdout?.on('data', (data)=>{
+        process1.stdout?.on('data', (data)=>{
             const str = data.toString();
             if (!isResolved && str.includes('Forwarding from')) {
                 isResolved = true;
-                logger.info(str);
+                logger.debug(`Port forward for ${resource}: ${str}`);
                 const port = str.search(/:\d+/);
                 if (port === -1) {
                     throw new Error('Port not found in port forward output');
                 }
                 const portNumber = parseInt(str.slice(port + 1));
-                logger.info(`Port forward connected: ${portNumber}`);
-                logger.info(`Port forward connected: ${portNumber}`);
+                logger.verbose(`Port forwarded for ${resource} at ${portNumber}:${containerPort}`);
                 resolve(portNumber);
             } else {
                 logger.silent(str);
             }
         });
-        process.stderr?.on('data', (data)=>{
-            logger.info(data.toString());
+        process1.stderr?.on('data', (data)=>{
+            logger.verbose(`Port forward for ${resource}: ${data.toString()}`);
             // It's a strange thing:
             // If we don't pipe stderr, then the port forwarding does not work.
             // Log to silent because this doesn't actually report errors,
             // just extremely verbose debug logs.
             logger.silent(data.toString());
         });
-        process.on('close', ()=>{
+        process1.on('close', ()=>{
             if (!isResolved) {
                 isResolved = true;
-                logger.warn('Port forward closed before connection established');
+                logger.warn(`Port forward for ${resource} closed before connection established`);
                 resolve(0);
             }
         });
-        process.on('error', (error)=>{
-            logger.error(`Port forward error: ${error}`);
+        process1.on('error', (error)=>{
+            logger.error(`Port forward for ${resource} error: ${error}`);
             resolve(0);
         });
-        process.on('exit', (code)=>{
-            logger.info(`Port forward exited with code ${code}`);
+        process1.on('exit', (code)=>{
+            logger.verbose(`Port forward for ${resource} exited with code ${code}`);
             resolve(0);
         });
     });
     const port = await connected;
     return {
-        process,
+        process: process1,
         port
     };
 }
+export function getExternalIP(namespace, serviceName) {
+    const { promise, resolve, reject } = promiseWithResolvers();
+    const process1 = spawn('kubectl', [
+        'get',
+        'service',
+        '-n',
+        namespace,
+        `${namespace}-${serviceName}`,
+        '--output',
+        "jsonpath='{.status.loadBalancer.ingress[0].ip}'"
+    ], {
+        stdio: 'pipe'
+    });
+    let ip = '';
+    process1.stdout.on('data', (data)=>{
+        ip += data;
+    });
+    process1.on('error', (err)=>{
+        reject(err);
+    });
+    process1.on('exit', ()=>{
+        // kubectl prints JSON. Remove the quotes
+        resolve(ip.replace(/"|'/g, ''));
+    });
+    return promise;
+}
+export function startPortForwardForRPC(namespace, index = 0) {
+    return startPortForward({
+        resource: `pod/${namespace}-rpc-aztec-node-${index}`,
+        namespace,
+        containerPort: 8080
+    });
+}
+export function startPortForwardForEthereum(namespace) {
+    return startPortForward({
+        resource: `services/${namespace}-eth-execution`,
+        namespace,
+        containerPort: 8545
+    });
+}
 export async function deleteResourceByName({ resource, namespace, name, force = false }) {
     const command = `kubectl delete ${resource} ${name} -n ${namespace} --ignore-not-found=true --wait=true ${force ? '--force' : ''}`;
     logger.info(`command: ${command}`);
     const { stdout } = await execAsync(command);
     return stdout;
 }
-export async function deleteResourceByLabel({ resource, namespace, label }) {
-    const command = `kubectl delete ${resource} -l ${label} -n ${namespace} --ignore-not-found=true --wait=true`;
+export async function deleteResourceByLabel({ resource, namespace, label, timeout = '5m', force = false }) {
+    try {
+        await execAsync(`kubectl api-resources --no-headers -o name | grep -Eq "^${resource}(\\\\..+)?$"`);
+    } catch (error) {
+        logger.warn(`Resource type '${resource}' not found in cluster, skipping deletion ${error}`);
+        return '';
+    }
+    const command = `kubectl delete ${resource} -l ${label} -n ${namespace} --ignore-not-found=true --wait=true --timeout=${timeout} ${force ? '--force' : ''}`;
     logger.info(`command: ${command}`);
     const { stdout } = await execAsync(command);
     return stdout;
@@ -153,8 +206,12 @@ export async function waitForResourceByLabel({ resource, label, namespace, condi
 export function getChartDir(spartanDir, chartName) {
     return path.join(spartanDir.trim(), chartName);
 }
+function shellQuote(value) {
+    // Single-quote safe shell escaping: ' -> '\''
+    return `'${value.replace(/'/g, "'\\''")}'`;
+}
 function valuesToArgs(values) {
-    return Object.entries(values).map(([key, value])=>`--set ${key}=${value}`).join(' ');
+    return Object.entries(values).map(([key, value])=>typeof value === 'number' || typeof value === 'boolean' ? `--set ${key}=${value}` : `--set-string ${key}=${shellQuote(String(value))}`).join(' ');
 }
 function createHelmCommand({ instanceName, helmChartDir, namespace, valuesFile, timeout, values, reuseValues = false }) {
     const valuesFileArgs = valuesFile ? `--values ${helmChartDir}/values/${valuesFile}` : '';
@@ -167,6 +224,57 @@ async function execHelmCommand(args) {
     const { stdout } = await execAsync(helmCommand);
     return stdout;
 }
+async function getHelmReleaseStatus(instanceName, namespace) {
+    try {
+        const { stdout } = await execAsync(`helm list --namespace ${namespace} --all --filter '^${instanceName}$' --output json | cat`);
+        const parsed = JSON.parse(stdout);
+        const row = parsed.find((r)=>r.name === instanceName);
+        return row?.status;
+    } catch  {
+        return undefined;
+    }
+}
+async function forceDeleteHelmReleaseRecord(instanceName, namespace, logger) {
+    const labelSelector = `owner=helm,name=${instanceName}`;
+    const cmd = `kubectl delete secret -n ${namespace} -l ${labelSelector} --ignore-not-found=true`;
+    logger.warn(`Force deleting Helm release record: ${cmd}`);
+    await execAsync(cmd).catch(()=>undefined);
+}
+async function hasDeployedHelmRelease(instanceName, namespace) {
+    try {
+        const status = await getHelmReleaseStatus(instanceName, namespace);
+        return status?.toLowerCase() === 'deployed';
+    } catch  {
+        return false;
+    }
+}
+export async function uninstallChaosMesh(instanceName, namespace, logger) {
+    // uninstall the helm chart if it exists
+    logger.info(`Uninstalling helm chart ${instanceName}`);
+    await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`);
+    // and delete the chaos-mesh resources created by this release
+    const deleteByLabel = async (resource)=>{
+        const args = {
+            resource,
+            namespace: namespace,
+            label: `app.kubernetes.io/instance=${instanceName}`
+        };
+        logger.info(`Deleting ${resource} resources for release ${instanceName}`);
+        await deleteResourceByLabel(args).catch((e)=>{
+            logger.error(`Error deleting ${resource}: ${e}`);
+            logger.info(`Force deleting ${resource}`);
+            return deleteResourceByLabel({
+                ...args,
+                force: true
+            });
+        });
+    };
+    await deleteByLabel('podchaos');
+    await deleteByLabel('networkchaos');
+    await deleteByLabel('podnetworkchaos');
+    await deleteByLabel('workflows');
+    await deleteByLabel('workflownodes');
+}
 /**
  * Installs a Helm chart with the given parameters.
  * @param instanceName - The name of the Helm chart instance.
@@ -183,31 +291,14 @@ async function execHelmCommand(args) {
  * const stdout = await installChaosMeshChart({ instanceName: 'force-reorg', targetNamespace: 'smoke', valuesFile: 'prover-failure.yaml'});
  * console.log(stdout);
  * ```
- */ export async function installChaosMeshChart({ instanceName, targetNamespace, valuesFile, helmChartDir, chaosMeshNamespace = 'chaos-mesh', timeout = '5m', clean = true, values = {}, logger }) {
+ */ export async function installChaosMeshChart({ instanceName, targetNamespace, valuesFile, helmChartDir, timeout = '10m', clean = true, values = {}, logger }) {
     if (clean) {
-        // uninstall the helm chart if it exists
-        logger.info(`Uninstalling helm chart ${instanceName}`);
-        await execAsync(`helm uninstall ${instanceName} --namespace ${chaosMeshNamespace} --wait --ignore-not-found`);
-        // and delete the podchaos resource
-        const deleteArgs = {
-            resource: 'podchaos',
-            namespace: chaosMeshNamespace,
-            name: `${targetNamespace}-${instanceName}`
-        };
-        logger.info(`Deleting podchaos resource`);
-        await deleteResourceByName(deleteArgs).catch((e)=>{
-            logger.error(`Error deleting podchaos resource: ${e}`);
-            logger.info(`Force deleting podchaos resource`);
-            return deleteResourceByName({
-                ...deleteArgs,
-                force: true
-            });
-        });
+        await uninstallChaosMesh(instanceName, targetNamespace, logger);
     }
     return execHelmCommand({
         instanceName,
         helmChartDir,
-        namespace: chaosMeshNamespace,
+        namespace: targetNamespace,
         valuesFile,
         timeout,
         values: {
@@ -228,69 +319,74 @@ export function applyProverFailure({ namespace, spartanDir, durationSeconds, log
         logger
     });
 }
-export function applyProverKill({ namespace, spartanDir, logger }) {
+export function applyProverKill({ namespace, spartanDir, logger, values }) {
     return installChaosMeshChart({
         instanceName: 'prover-kill',
         targetNamespace: namespace,
         valuesFile: 'prover-kill.yaml',
         helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+        chaosMeshNamespace: namespace,
         clean: true,
-        logger
+        logger,
+        values
     });
 }
-export function applyProverBrokerKill({ namespace, spartanDir, logger }) {
+export function applyProverBrokerKill({ namespace, spartanDir, logger, values }) {
     return installChaosMeshChart({
         instanceName: 'prover-broker-kill',
         targetNamespace: namespace,
         valuesFile: 'prover-broker-kill.yaml',
         helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
         clean: true,
-        logger
+        logger,
+        values
     });
 }
-export function applyBootNodeFailure({ namespace, spartanDir, durationSeconds, logger }) {
+export function applyBootNodeFailure({ instanceName = 'boot-node-failure', namespace, spartanDir, durationSeconds, logger, values }) {
     return installChaosMeshChart({
-        instanceName: 'boot-node-failure',
+        instanceName,
         targetNamespace: namespace,
         valuesFile: 'boot-node-failure.yaml',
         helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
         values: {
-            'bootNodeFailure.duration': `${durationSeconds}s`
+            'bootNodeFailure.duration': `${durationSeconds}s`,
+            ...values ?? {}
         },
         logger
     });
 }
-export function applyValidatorKill({ namespace, spartanDir, logger }) {
+export function applyValidatorKill({ instanceName = 'validator-kill', namespace, spartanDir, logger, values }) {
     return installChaosMeshChart({
-        instanceName: 'validator-kill',
+        instanceName,
         targetNamespace: namespace,
         valuesFile: 'validator-kill.yaml',
         helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
-        logger
+        logger,
+        values
     });
 }
-export function applyNetworkShaping({ valuesFile, namespace, spartanDir, logger }) {
+export function applyNetworkShaping({ instanceName = 'network-shaping', valuesFile, namespace, spartanDir, logger }) {
     return installChaosMeshChart({
-        instanceName: 'network-shaping',
+        instanceName,
         targetNamespace: namespace,
         valuesFile,
         helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
         logger
     });
 }
-export async function awaitL2BlockNumber(rollupCheatCodes, blockNumber, timeoutSeconds, logger) {
-    logger.info(`Waiting for L2 Block ${blockNumber}`);
+export async function awaitCheckpointNumber(rollupCheatCodes, checkpointNumber, timeoutSeconds, logger) {
+    logger.info(`Waiting for checkpoint ${checkpointNumber}`);
     let tips = await rollupCheatCodes.getTips();
     const endTime = Date.now() + timeoutSeconds * 1000;
-    while(tips.pending < blockNumber && Date.now() < endTime){
-        logger.info(`At L2 Block ${tips.pending}`);
+    while(tips.pending < checkpointNumber && Date.now() < endTime){
+        logger.info(`At checkpoint ${tips.pending}`);
         await sleep(1000);
         tips = await rollupCheatCodes.getTips();
     }
-    if (tips.pending < blockNumber) {
-        throw new Error(`Timeout waiting for L2 Block ${blockNumber}, only reached ${tips.pending}`);
+    if (tips.pending < checkpointNumber) {
+        throw new Error(`Timeout waiting for checkpoint ${checkpointNumber}, only reached ${tips.pending}`);
     } else {
-        logger.info(`Reached L2 Block ${tips.pending}`);
+        logger.info(`Reached checkpoint ${tips.pending}`);
     }
 }
 export async function restartBot(namespace, logger) {
@@ -298,16 +394,212 @@ export async function restartBot(namespace, logger) {
     await deleteResourceByLabel({
         resource: 'pods',
         namespace,
-        label: 'app=bot'
+        label: 'app.kubernetes.io/name=bot'
     });
     await sleep(10 * 1000);
+    // Some bot images may take time to report Ready due to heavy boot-time proving.
+    // Waiting for PodReadyToStartContainers ensures the pod is scheduled and starting without blocking on full readiness.
     await waitForResourceByLabel({
         resource: 'pods',
         namespace,
-        label: 'app=bot'
+        label: 'app.kubernetes.io/name=bot',
+        condition: 'PodReadyToStartContainers'
     });
     logger.info(`Bot restarted`);
 }
+/**
+ * Installs or upgrades the transfer bot Helm release for the given namespace.
+ * Intended for test setup to enable L2 traffic generation only when needed.
+ */ export async function installTransferBot({ namespace, spartanDir, logger, replicas = 1, txIntervalSeconds = 10, followChain = 'PENDING', mnemonic = process.env.LABS_INFRA_MNEMONIC ?? 'test test test test test test test test test test test junk', mnemonicStartIndex, botPrivateKey = process.env.BOT_TRANSFERS_L2_PRIVATE_KEY ?? '0xcafe01', nodeUrl, timeout = '15m', reuseValues = true, aztecSlotDuration = Number(process.env.AZTEC_SLOT_DURATION ?? 12) }) {
+    const instanceName = `${namespace}-bot-transfers`;
+    const helmChartDir = getChartDir(spartanDir, 'aztec-bot');
+    const resolvedNodeUrl = nodeUrl ?? `http://${namespace}-rpc-aztec-node.${namespace}.svc.cluster.local:8080`;
+    logger.info(`Installing/upgrading transfer bot: replicas=${replicas}, followChain=${followChain}`);
+    const values = {
+        'bot.replicaCount': replicas,
+        'bot.txIntervalSeconds': txIntervalSeconds,
+        'bot.followChain': followChain,
+        'bot.botPrivateKey': botPrivateKey,
+        'bot.nodeUrl': resolvedNodeUrl,
+        'bot.mnemonic': mnemonic,
+        'bot.feePaymentMethod': 'fee_juice',
+        'aztec.slotDuration': aztecSlotDuration,
+        // Ensure bot can reach its own PXE started in-process (default rpc.port is 8080)
+        // Note: since aztec-bot depends on aztec-node with alias `bot`, env vars go under `bot.node.env`.
+        'bot.node.env.BOT_PXE_URL': 'http://127.0.0.1:8080',
+        // Provide L1 execution RPC for bridging fee juice
+        'bot.node.env.ETHEREUM_HOSTS': `http://${namespace}-eth-execution.${namespace}.svc.cluster.local:8545`,
+        // Provide L1 mnemonic for bridging (falls back to labs mnemonic)
+        'bot.node.env.BOT_L1_MNEMONIC': mnemonic,
+        // The bot does not need Kubernetes API access. Disable RBAC + ServiceAccount creation so the chart
+        // can be installed by users without cluster-scoped RBAC permissions.
+        'bot.rbac.create': false,
+        'bot.serviceAccount.create': false,
+        'bot.serviceAccount.name': 'default'
+    };
+    // Ensure we derive a funded L1 key (index 0 is funded on anvil default mnemonic)
+    if (mnemonicStartIndex === undefined) {
+        values['bot.mnemonicStartIndex'] = 0;
+    }
+    // Also pass a funded private key directly if available
+    if (process.env.FUNDING_PRIVATE_KEY) {
+        values['bot.node.env.BOT_L1_PRIVATE_KEY'] = process.env.FUNDING_PRIVATE_KEY;
+    }
+    // Align bot image with the running network image: prefer env var, else detect from a validator pod
+    let repositoryFromEnv;
+    let tagFromEnv;
+    const aztecDockerImage = process.env.AZTEC_DOCKER_IMAGE;
+    if (aztecDockerImage && aztecDockerImage.includes(':')) {
+        const lastColon = aztecDockerImage.lastIndexOf(':');
+        repositoryFromEnv = aztecDockerImage.slice(0, lastColon);
+        tagFromEnv = aztecDockerImage.slice(lastColon + 1);
+    }
+    let repository = repositoryFromEnv;
+    let tag = tagFromEnv;
+    if (!repository || !tag) {
+        try {
+            const { stdout } = await execAsync(`kubectl get pods -l app.kubernetes.io/name=validator -n ${namespace} -o jsonpath='{.items[0].spec.containers[?(@.name=="aztec")].image}' | cat`);
+            const image = stdout.trim().replace(/^'|'$/g, '');
+            if (image && image.includes(':')) {
+                const lastColon = image.lastIndexOf(':');
+                repository = image.slice(0, lastColon);
+                tag = image.slice(lastColon + 1);
+            }
+        } catch (err) {
+            logger.warn(`Could not detect aztec image from validator pod: ${String(err)}`);
+        }
+    }
+    if (repository && tag) {
+        values['global.aztecImage.repository'] = repository;
+        values['global.aztecImage.tag'] = tag;
+    }
+    if (mnemonicStartIndex !== undefined) {
+        values['bot.mnemonicStartIndex'] = typeof mnemonicStartIndex === 'string' ? mnemonicStartIndex : Number(mnemonicStartIndex);
+    }
+    // If a previous install attempt left the release in a non-deployed state (e.g. FAILED),
+    // `helm upgrade --install` can error with "has no deployed releases".
+    // In that case, clear the release record and do a clean install.
+    const existingStatus = await getHelmReleaseStatus(instanceName, namespace);
+    if (existingStatus && existingStatus.toLowerCase() !== 'deployed') {
+        logger.warn(`Transfer bot release ${instanceName} is in status '${existingStatus}'. Reinstalling cleanly.`);
+        await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`).catch(()=>undefined);
+        // If helm left the release in `uninstalling`, force-delete the record so we can reinstall.
+        const afterUninstallStatus = await getHelmReleaseStatus(instanceName, namespace);
+        if (afterUninstallStatus?.toLowerCase() === 'uninstalling') {
+            await forceDeleteHelmReleaseRecord(instanceName, namespace, logger);
+        }
+    }
+    // `--reuse-values` fails if the release has never successfully deployed (e.g. first install, or a previous failed install).
+    // Only reuse values when we have a deployed release to reuse from.
+    const effectiveReuseValues = reuseValues && await hasDeployedHelmRelease(instanceName, namespace);
+    await execHelmCommand({
+        instanceName,
+        helmChartDir,
+        namespace,
+        valuesFile: undefined,
+        timeout,
+        values: values,
+        reuseValues: effectiveReuseValues
+    });
+    if (replicas > 0) {
+        await waitForResourceByLabel({
+            resource: 'pods',
+            namespace,
+            label: 'app.kubernetes.io/name=bot',
+            condition: 'PodReadyToStartContainers'
+        });
+    }
+}
+/**
+ * Uninstalls the transfer bot Helm release from the given namespace.
+ * Intended for test teardown to clean up bot resources.
+ */ export async function uninstallTransferBot(namespace, logger) {
+    const instanceName = `${namespace}-bot-transfers`;
+    logger.info(`Uninstalling transfer bot release ${instanceName}`);
+    await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`);
+    // Ensure any leftover pods are removed
+    await deleteResourceByLabel({
+        resource: 'pods',
+        namespace,
+        label: 'app.kubernetes.io/name=bot'
+    }).catch(()=>undefined);
+}
+/**
+ * Enables or disables probabilistic transaction dropping on validators and waits for rollout.
+ * Wired to env vars P2P_DROP_TX and P2P_DROP_TX_CHANCE via Helm values.
+ */ export async function setValidatorTxDrop({ namespace, enabled, probability, logger }) {
+    const drop = enabled ? 'true' : 'false';
+    const prob = String(probability);
+    const selectors = [
+        'app.kubernetes.io/name=validator',
+        'app.kubernetes.io/component=validator',
+        'app=validator'
+    ];
+    let updated = false;
+    for (const selector of selectors){
+        try {
+            const list = await execAsync(`kubectl get statefulset -l ${selector} -n ${namespace} --no-headers -o name | cat`);
+            const names = list.stdout.split('\n').map((s)=>s.trim()).filter(Boolean);
+            if (names.length === 0) {
+                continue;
+            }
+            const cmd = `kubectl set env statefulset -l ${selector} -n ${namespace} P2P_DROP_TX=${drop} P2P_DROP_TX_CHANCE=${prob}`;
+            logger.info(`command: ${cmd}`);
+            await execAsync(cmd);
+            updated = true;
+        } catch (e) {
+            logger.warn(`Failed to update validators with selector ${selector}: ${String(e)}`);
+        }
+    }
+    if (!updated) {
+        logger.warn(`No validator StatefulSets found in ${namespace}. Skipping tx drop toggle.`);
+        return;
+    }
+    // Restart validator pods to ensure env vars take effect and wait for readiness
+    await restartValidators(namespace, logger);
+}
+export async function restartValidators(namespace, logger) {
+    const selectors = [
+        'app.kubernetes.io/name=validator',
+        'app.kubernetes.io/component=validator',
+        'app=validator'
+    ];
+    let any = false;
+    for (const selector of selectors){
+        try {
+            const { stdout } = await execAsync(`kubectl get pods -l ${selector} -n ${namespace} --no-headers -o name | cat`);
+            if (!stdout || stdout.trim().length === 0) {
+                continue;
+            }
+            any = true;
+            await deleteResourceByLabel({
+                resource: 'pods',
+                namespace,
+                label: selector
+            });
+        } catch (e) {
+            logger.warn(`Error restarting validator pods with selector ${selector}: ${String(e)}`);
+        }
+    }
+    if (!any) {
+        logger.warn(`No validator pods found to restart in ${namespace}.`);
+        return;
+    }
+    // Wait for either label to be Ready
+    for (const selector of selectors){
+        try {
+            await waitForResourceByLabel({
+                resource: 'pods',
+                namespace,
+                label: selector
+            });
+            return;
+        } catch  {
+        // try next
+        }
+    }
+    logger.warn(`Validator pods did not report Ready; continuing.`);
+}
 export async function enableValidatorDynamicBootNode(instanceName, namespace, spartanDir, logger) {
     logger.info(`Enabling validator dynamic boot node`);
     await execHelmCommand({
@@ -323,55 +615,127 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
     });
     logger.info(`Validator dynamic boot node enabled`);
 }
-export async function runAlertCheck(config, alerts, logger) {
-    if (isK8sConfig(config)) {
-        const { process, port } = await startPortForward({
-            resource: `svc/metrics-grafana`,
-            namespace: 'metrics',
-            containerPort: config.CONTAINER_METRICS_PORT
-        });
-        const alertChecker = new AlertChecker(logger, {
-            grafanaEndpoint: `http://localhost:${port}${config.METRICS_API_PATH}`,
-            grafanaCredentials: `admin:${config.GRAFANA_PASSWORD}`
-        });
-        await alertChecker.runAlertCheck(alerts);
-        process.kill();
-    } else {
-        logger.info('Not running alert check in non-k8s environment');
+export async function getSequencers(namespace) {
+    const selectors = [
+        'app.kubernetes.io/name=validator',
+        'app.kubernetes.io/component=validator',
+        'app.kubernetes.io/component=sequencer-node',
+        'app=validator'
+    ];
+    for (const selector of selectors){
+        try {
+            const command = `kubectl get pods -l ${selector} -n ${namespace} -o jsonpath='{.items[*].metadata.name}'`;
+            const { stdout } = await execAsync(command);
+            const sequencers = stdout.split(' ').map((s)=>s.trim()).filter(Boolean);
+            if (sequencers.length > 0) {
+                logger.verbose(`Found sequencer pods ${sequencers.join(', ')} (selector=${selector})`);
+                return sequencers;
+            }
+        } catch  {
+        // try next selector
+        }
     }
+    // Fail fast instead of returning [''] which leads to attempts to port-forward `pod/`.
+    throw new Error(`No sequencer/validator pods found in namespace ${namespace}. Tried selectors: ${selectors.join(', ')}`);
 }
-export async function updateSequencerConfig(url, config) {
-    const node = createAztecNodeClient(url);
-    await node.setConfig(config);
+export function updateSequencersConfig(env, config) {
+    return withSequencersAdmin(env, async (client)=>{
+        await client.setConfig(config);
+        return client.getConfig();
+    });
 }
-export async function getSequencers(namespace) {
-    const command = `kubectl get pods -l app=validator -n ${namespace} -o jsonpath='{.items[*].metadata.name}'`;
-    const { stdout } = await execAsync(command);
-    return stdout.split(' ');
+export function getSequencersConfig(env) {
+    return withSequencersAdmin(env, (client)=>client.getConfig());
 }
-export async function updateK8sSequencersConfig(args) {
-    const { containerPort, namespace, config } = args;
+export async function withSequencersAdmin(env, fn) {
+    const adminContainerPort = 8880;
+    const namespace = env.NAMESPACE;
     const sequencers = await getSequencers(namespace);
+    const results = [];
     for (const sequencer of sequencers){
-        const { process, port } = await startPortForward({
+        const { process: process1, port } = await startPortForward({
             resource: `pod/${sequencer}`,
             namespace,
-            containerPort
+            containerPort: adminContainerPort
         });
         const url = `http://localhost:${port}`;
-        await updateSequencerConfig(url, config);
-        process.kill();
+        await retry(()=>fetch(`${url}/status`).then((res)=>res.status === 200), 'forward node admin port', makeBackoff([
+            1,
+            1,
+            2,
+            6
+        ]), logger, true);
+        const client = createAztecNodeAdminClient(url);
+        results.push(await fn(client));
+        process1.kill();
     }
+    return results;
 }
-export async function updateSequencersConfig(env, config) {
-    if (isK8sConfig(env)) {
-        await updateK8sSequencersConfig({
-            containerPort: env.CONTAINER_NODE_PORT,
-            namespace: env.NAMESPACE,
-            config
+/**
+ * Returns a public viem client to the eth execution node. If it was part of a local eth devnet,
+ * it first port-forwards the service and points to it. Otherwise, just uses the external RPC url.
+ */ export async function getPublicViemClient(env, /** If set, will push the new process into it */ processes) {
+    const { NAMESPACE, CREATE_ETH_DEVNET, L1_RPC_URLS_JSON } = env;
+    if (CREATE_ETH_DEVNET) {
+        logger.info(`Creating port forward to eth execution node`);
+        const { process: process1, port } = await startPortForward({
+            resource: `svc/${NAMESPACE}-eth-execution`,
+            namespace: NAMESPACE,
+            containerPort: 8545
         });
+        const url = `http://127.0.0.1:${port}`;
+        const client = createPublicClient({
+            transport: fallback([
+                http(url, {
+                    batch: false
+                })
+            ])
+        });
+        if (processes) {
+            processes.push(process1);
+        }
+        return {
+            url,
+            client,
+            process: process1
+        };
     } else {
-        await updateSequencerConfig(env.NODE_URL, config);
+        logger.info(`Connecting to the eth execution node at ${L1_RPC_URLS_JSON}`);
+        if (!L1_RPC_URLS_JSON) {
+            throw new Error(`L1_RPC_URLS_JSON is not defined`);
+        }
+        const client = createPublicClient({
+            transport: fallback([
+                http(L1_RPC_URLS_JSON, {
+                    batch: false
+                })
+            ])
+        });
+        return {
+            url: L1_RPC_URLS_JSON,
+            client
+        };
+    }
+}
+/** Queries an Aztec node for the L1 deployment addresses */ export async function getL1DeploymentAddresses(env) {
+    let forwardProcess;
+    try {
+        const [sequencer] = await getSequencers(env.NAMESPACE);
+        const { process: process1, port } = await startPortForward({
+            resource: `pod/${sequencer}`,
+            namespace: env.NAMESPACE,
+            containerPort: 8080
+        });
+        forwardProcess = process1;
+        const url = `http://127.0.0.1:${port}`;
+        const node = createAztecNodeClient(url);
+        return await retry(()=>node.getNodeInfo().then((i)=>i.l1ContractAddresses), 'get node info', makeBackoff([
+            1,
+            3,
+            6
+        ]), logger);
+    } finally{
+        forwardProcess?.kill();
     }
 }
 /**
@@ -443,3 +807,47 @@ export async function updateSequencersConfig(env, config) {
         label: 'app=pxe'
     });
 }
+/**
+ * Returns the absolute path to the git repository root
+ */ export function getGitProjectRoot() {
+    try {
+        const rootDir = execSync('git rev-parse --show-toplevel', {
+            encoding: 'utf-8',
+            stdio: [
+                'ignore',
+                'pipe',
+                'ignore'
+            ]
+        }).trim();
+        return rootDir;
+    } catch (error) {
+        throw new Error(`Failed to determine git project root: ${error}`);
+    }
+}
+/** Returns a client to the RPC of the given sequencer (defaults to first) */ export async function getNodeClient(env, index = 0) {
+    const namespace = env.NAMESPACE;
+    const containerPort = 8080;
+    const sequencers = await getSequencers(namespace);
+    const sequencer = sequencers[index];
+    if (!sequencer) {
+        throw new Error(`No sequencer found at index ${index} in namespace ${namespace}`);
+    }
+    const { process: process1, port } = await startPortForward({
+        resource: `pod/${sequencer}`,
+        namespace,
+        containerPort
+    });
+    const url = `http://localhost:${port}`;
+    await retry(()=>fetch(`${url}/status`).then((res)=>res.status === 200), 'forward port', makeBackoff([
+        1,
+        1,
+        2,
+        6
+    ]), logger, true);
+    const client = createAztecNodeClient(url);
+    return {
+        node: client,
+        port,
+        process: process1
+    };
+}