@aztec/end-to-end 4.0.0-nightly.20250907 → 4.0.0-nightly.20260108

package/dest/spartan/utils.js

@@ -1,6 +1,8 @@
-import { createLogger, sleep } from '@aztec/aztec.js';
+import { createLogger } from '@aztec/aztec.js/log';
+import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { makeBackoff, retry } from '@aztec/foundation/retry';
 import { schemas } from '@aztec/foundation/schemas';
+import { sleep } from '@aztec/foundation/sleep';
 import { createAztecNodeAdminClient, createAztecNodeClient } from '@aztec/stdlib/interfaces/client';
 import { exec, execSync, spawn } from 'child_process';
 import path from 'path';
@@ -13,7 +15,11 @@ const testConfigSchema = z.object({
     NAMESPACE: z.string().default('scenario'),
     REAL_VERIFIER: schemas.Boolean.optional().default(true),
     CREATE_ETH_DEVNET: schemas.Boolean.optional().default(false),
-    L1_RPC_URLS_JSON: z.string().optional()
+    L1_RPC_URLS_JSON: z.string().optional(),
+    L1_ACCOUNT_MNEMONIC: z.string().optional(),
+    AZTEC_SLOT_DURATION: z.coerce.number().optional().default(24),
+    AZTEC_EPOCH_DURATION: z.coerce.number().optional().default(32),
+    AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5)
 });
 export function setupEnvironment(env) {
     const config = testConfigSchema.parse(env);
@@ -133,9 +139,35 @@ export async function startPortForward({ resource, namespace, containerPort, hos
         port
     };
 }
-export function startPortForwardForRPC(namespace) {
+export function getExternalIP(namespace, serviceName) {
+    const { promise, resolve, reject } = promiseWithResolvers();
+    const process1 = spawn('kubectl', [
+        'get',
+        'service',
+        '-n',
+        namespace,
+        `${namespace}-${serviceName}`,
+        '--output',
+        "jsonpath='{.status.loadBalancer.ingress[0].ip}'"
+    ], {
+        stdio: 'pipe'
+    });
+    let ip = '';
+    process1.stdout.on('data', (data)=>{
+        ip += data;
+    });
+    process1.on('error', (err)=>{
+        reject(err);
+    });
+    process1.on('exit', ()=>{
+        // kubectl prints JSON. Remove the quotes
+        resolve(ip.replace(/"|'/g, ''));
+    });
+    return promise;
+}
+export function startPortForwardForRPC(namespace, index = 0) {
     return startPortForward({
-        resource: `services/${namespace}-rpc-aztec-node`,
+        resource: `pod/${namespace}-rpc-aztec-node-${index}`,
         namespace,
         containerPort: 8080
     });
@@ -154,6 +186,15 @@ export async function deleteResourceByName({ resource, namespace, name, force =
     return stdout;
 }
 export async function deleteResourceByLabel({ resource, namespace, label, timeout = '5m', force = false }) {
+    try {
+        // Match both plain and group-qualified names (e.g., "podchaos" or "podchaos.chaos-mesh.org")
+        const escaped = resource.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
+        const regex = `(^|\\.)${escaped}(\\.|$)`;
+        await execAsync(`kubectl api-resources --no-headers -o name | grep -Eq '${regex}'`);
+    } catch (error) {
+        logger.warn(`Resource type '${resource}' not found in cluster, skipping deletion ${error}`);
+        return '';
+    }
     const command = `kubectl delete ${resource} -l ${label} -n ${namespace} --ignore-not-found=true --wait=true --timeout=${timeout} ${force ? '--force' : ''}`;
     logger.info(`command: ${command}`);
     const { stdout } = await execAsync(command);
@@ -165,11 +206,40 @@ export async function waitForResourceByLabel({ resource, label, namespace, condi
     const { stdout } = await execAsync(command);
     return stdout;
 }
+export async function waitForResourceByName({ resource, name, namespace, condition = 'Ready', timeout = '10m' }) {
+    const command = `kubectl wait ${resource}/${name} --for=condition=${condition} -n ${namespace} --timeout=${timeout}`;
+    logger.info(`command: ${command}`);
+    const { stdout } = await execAsync(command);
+    return stdout;
+}
+export async function waitForResourcesByName({ resource, names, namespace, condition = 'Ready', timeout = '10m' }) {
+    if (!names.length) {
+        throw new Error(`No ${resource} names provided to waitForResourcesByName`);
+    }
+    // Wait all in parallel; if any fails, surface which one.
+    await Promise.all(names.map(async (name)=>{
+        try {
+            await waitForResourceByName({
+                resource,
+                name,
+                namespace,
+                condition,
+                timeout
+            });
+        } catch (err) {
+            throw new Error(`Failed waiting for ${resource}/${name} condition=${condition} timeout=${timeout} namespace=${namespace}: ${String(err)}`);
+        }
+    }));
+}
 export function getChartDir(spartanDir, chartName) {
     return path.join(spartanDir.trim(), chartName);
 }
+function shellQuote(value) {
+    // Single-quote safe shell escaping: ' -> '\''
+    return `'${value.replace(/'/g, "'\\''")}'`;
+}
 function valuesToArgs(values) {
-    return Object.entries(values).map(([key, value])=>`--set ${key}=${value}`).join(' ');
+    return Object.entries(values).map(([key, value])=>typeof value === 'number' || typeof value === 'boolean' ? `--set ${key}=${value}` : `--set-string ${key}=${shellQuote(String(value))}`).join(' ');
 }
 function createHelmCommand({ instanceName, helmChartDir, namespace, valuesFile, timeout, values, reuseValues = false }) {
     const valuesFileArgs = valuesFile ? `--values ${helmChartDir}/values/${valuesFile}` : '';
@@ -182,6 +252,57 @@ async function execHelmCommand(args) {
     const { stdout } = await execAsync(helmCommand);
     return stdout;
 }
+async function getHelmReleaseStatus(instanceName, namespace) {
+    try {
+        const { stdout } = await execAsync(`helm list --namespace ${namespace} --all --filter '^${instanceName}$' --output json | cat`);
+        const parsed = JSON.parse(stdout);
+        const row = parsed.find((r)=>r.name === instanceName);
+        return row?.status;
+    } catch  {
+        return undefined;
+    }
+}
+async function forceDeleteHelmReleaseRecord(instanceName, namespace, logger) {
+    const labelSelector = `owner=helm,name=${instanceName}`;
+    const cmd = `kubectl delete secret -n ${namespace} -l ${labelSelector} --ignore-not-found=true`;
+    logger.warn(`Force deleting Helm release record: ${cmd}`);
+    await execAsync(cmd).catch(()=>undefined);
+}
+async function hasDeployedHelmRelease(instanceName, namespace) {
+    try {
+        const status = await getHelmReleaseStatus(instanceName, namespace);
+        return status?.toLowerCase() === 'deployed';
+    } catch  {
+        return false;
+    }
+}
+export async function uninstallChaosMesh(instanceName, namespace, logger) {
+    // uninstall the helm chart if it exists
+    logger.info(`Uninstalling helm chart ${instanceName}`);
+    await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`);
+    // and delete the chaos-mesh resources created by this release
+    const deleteByLabel = async (resource)=>{
+        const args = {
+            resource,
+            namespace: namespace,
+            label: `app.kubernetes.io/instance=${instanceName}`
+        };
+        logger.info(`Deleting ${resource} resources for release ${instanceName}`);
+        await deleteResourceByLabel(args).catch((e)=>{
+            logger.error(`Error deleting ${resource}: ${e}`);
+            logger.info(`Force deleting ${resource}`);
+            return deleteResourceByLabel({
+                ...args,
+                force: true
+            });
+        });
+    };
+    await deleteByLabel('podchaos');
+    await deleteByLabel('networkchaos');
+    await deleteByLabel('podnetworkchaos');
+    await deleteByLabel('workflows');
+    await deleteByLabel('workflownodes');
+}
 /**
  * Installs a Helm chart with the given parameters.
  * @param instanceName - The name of the Helm chart instance.
@@ -198,31 +319,14 @@ async function execHelmCommand(args) {
  * const stdout = await installChaosMeshChart({ instanceName: 'force-reorg', targetNamespace: 'smoke', valuesFile: 'prover-failure.yaml'});
  * console.log(stdout);
  * ```
- */ export async function installChaosMeshChart({ instanceName, targetNamespace, valuesFile, helmChartDir, chaosMeshNamespace = 'chaos-mesh', timeout = '5m', clean = true, values = {}, logger }) {
+ */ export async function installChaosMeshChart({ instanceName, targetNamespace, valuesFile, helmChartDir, timeout = '10m', clean = true, values = {}, logger }) {
     if (clean) {
-        // uninstall the helm chart if it exists
-        logger.info(`Uninstalling helm chart ${instanceName}`);
-        await execAsync(`helm uninstall ${instanceName} --namespace ${chaosMeshNamespace} --wait --ignore-not-found`);
-        // and delete the podchaos resource
-        const deleteArgs = {
-            resource: 'podchaos',
-            namespace: chaosMeshNamespace,
-            label: `app.kubernetes.io/instance=${instanceName}`
-        };
-        logger.info(`Deleting podchaos resource`);
-        await deleteResourceByLabel(deleteArgs).catch((e)=>{
-            logger.error(`Error deleting podchaos resource: ${e}`);
-            logger.info(`Force deleting podchaos resource`);
-            return deleteResourceByLabel({
-                ...deleteArgs,
-                force: true
-            });
-        });
+        await uninstallChaosMesh(instanceName, targetNamespace, logger);
     }
     return execHelmCommand({
         instanceName,
         helmChartDir,
-        namespace: chaosMeshNamespace,
+        namespace: targetNamespace,
         valuesFile,
         timeout,
         values: {
@@ -243,70 +347,85 @@ export function applyProverFailure({ namespace, spartanDir, durationSeconds, log
         logger
     });
 }
-export function applyProverKill({ namespace, spartanDir, logger }) {
+export function applyValidatorFailure({ namespace, spartanDir, logger, values, instanceName }) {
+    return installChaosMeshChart({
+        instanceName: instanceName ?? 'validator-failure',
+        targetNamespace: namespace,
+        valuesFile: 'validator-failure.yaml',
+        helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+        values,
+        logger
+    });
+}
+export function applyProverKill({ namespace, spartanDir, logger, values }) {
     return installChaosMeshChart({
         instanceName: 'prover-kill',
         targetNamespace: namespace,
         valuesFile: 'prover-kill.yaml',
         helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+        chaosMeshNamespace: namespace,
         clean: true,
-        logger
+        logger,
+        values
     });
 }
-export function applyProverBrokerKill({ namespace, spartanDir, logger }) {
+export function applyProverBrokerKill({ namespace, spartanDir, logger, values }) {
     return installChaosMeshChart({
         instanceName: 'prover-broker-kill',
         targetNamespace: namespace,
         valuesFile: 'prover-broker-kill.yaml',
         helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
         clean: true,
-        logger
+        logger,
+        values
     });
 }
-export function applyBootNodeFailure({ namespace, spartanDir, durationSeconds, logger }) {
+export function applyBootNodeFailure({ instanceName = 'boot-node-failure', namespace, spartanDir, durationSeconds, logger, values }) {
     return installChaosMeshChart({
-        instanceName: 'boot-node-failure',
+        instanceName,
         targetNamespace: namespace,
         valuesFile: 'boot-node-failure.yaml',
         helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
         values: {
-            'bootNodeFailure.duration': `${durationSeconds}s`
+            'bootNodeFailure.duration': `${durationSeconds}s`,
+            ...values ?? {}
         },
         logger
     });
 }
-export function applyValidatorKill({ namespace, spartanDir, logger, values }) {
+export function applyValidatorKill({ instanceName = 'validator-kill', namespace, spartanDir, logger, values, clean = true }) {
     return installChaosMeshChart({
-        instanceName: 'validator-kill',
+        instanceName: instanceName ?? 'validator-kill',
         targetNamespace: namespace,
         valuesFile: 'validator-kill.yaml',
         helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+        clean,
         logger,
         values
     });
 }
-export function applyNetworkShaping({ valuesFile, namespace, spartanDir, logger }) {
+export function applyNetworkShaping({ instanceName = 'network-shaping', valuesFile, namespace, spartanDir, logger }) {
     return installChaosMeshChart({
-        instanceName: 'network-shaping',
+        instanceName,
         targetNamespace: namespace,
         valuesFile,
         helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
         logger
     });
 }
-export async function awaitL2BlockNumber(rollupCheatCodes, blockNumber, timeoutSeconds, logger) {
-    logger.info(`Waiting for L2 Block ${blockNumber}`);
+export async function awaitCheckpointNumber(rollupCheatCodes, checkpointNumber, timeoutSeconds, logger) {
+    logger.info(`Waiting for checkpoint ${checkpointNumber}`);
     let tips = await rollupCheatCodes.getTips();
     const endTime = Date.now() + timeoutSeconds * 1000;
-    while(tips.pending < blockNumber && Date.now() < endTime){
-        logger.info(`At L2 Block ${tips.pending}`);
+    while(tips.pending < checkpointNumber && Date.now() < endTime){
+        logger.info(`At checkpoint ${tips.pending}`);
         await sleep(1000);
         tips = await rollupCheatCodes.getTips();
     }
-    if (tips.pending < blockNumber) {
-        throw new Error(`Timeout waiting for L2 Block ${blockNumber}, only reached ${tips.pending}`);
+    if (tips.pending < checkpointNumber) {
+        throw new Error(`Timeout waiting for checkpoint ${checkpointNumber}, only reached ${tips.pending}`);
     } else {
-        logger.info(`Reached L2 Block ${tips.pending}`);
+        logger.info(`Reached checkpoint ${tips.pending}`);
     }
 }
 export async function restartBot(namespace, logger) {
@@ -314,16 +433,212 @@ export async function restartBot(namespace, logger) {
     await deleteResourceByLabel({
         resource: 'pods',
         namespace,
-        label: 'app=bot'
+        label: 'app.kubernetes.io/name=bot'
     });
     await sleep(10 * 1000);
+    // Some bot images may take time to report Ready due to heavy boot-time proving.
+    // Waiting for PodReadyToStartContainers ensures the pod is scheduled and starting without blocking on full readiness.
     await waitForResourceByLabel({
         resource: 'pods',
         namespace,
-        label: 'app=bot'
+        label: 'app.kubernetes.io/name=bot',
+        condition: 'PodReadyToStartContainers'
     });
     logger.info(`Bot restarted`);
 }
+/**
+ * Installs or upgrades the transfer bot Helm release for the given namespace.
+ * Intended for test setup to enable L2 traffic generation only when needed.
+ */ export async function installTransferBot({ namespace, spartanDir, logger, replicas = 1, txIntervalSeconds = 10, followChain = 'PENDING', mnemonic = process.env.LABS_INFRA_MNEMONIC ?? 'test test test test test test test test test test test junk', mnemonicStartIndex, botPrivateKey = process.env.BOT_TRANSFERS_L2_PRIVATE_KEY ?? '0xcafe01', nodeUrl, timeout = '15m', reuseValues = true, aztecSlotDuration = Number(process.env.AZTEC_SLOT_DURATION ?? 12) }) {
+    const instanceName = `${namespace}-bot-transfers`;
+    const helmChartDir = getChartDir(spartanDir, 'aztec-bot');
+    const resolvedNodeUrl = nodeUrl ?? `http://${namespace}-rpc-aztec-node.${namespace}.svc.cluster.local:8080`;
+    logger.info(`Installing/upgrading transfer bot: replicas=${replicas}, followChain=${followChain}`);
+    const values = {
+        'bot.replicaCount': replicas,
+        'bot.txIntervalSeconds': txIntervalSeconds,
+        'bot.followChain': followChain,
+        'bot.botPrivateKey': botPrivateKey,
+        'bot.nodeUrl': resolvedNodeUrl,
+        'bot.mnemonic': mnemonic,
+        'bot.feePaymentMethod': 'fee_juice',
+        'aztec.slotDuration': aztecSlotDuration,
+        // Ensure bot can reach its own PXE started in-process (default rpc.port is 8080)
+        // Note: since aztec-bot depends on aztec-node with alias `bot`, env vars go under `bot.node.env`.
+        'bot.node.env.BOT_PXE_URL': 'http://127.0.0.1:8080',
+        // Provide L1 execution RPC for bridging fee juice
+        'bot.node.env.ETHEREUM_HOSTS': `http://${namespace}-eth-execution.${namespace}.svc.cluster.local:8545`,
+        // Provide L1 mnemonic for bridging (falls back to labs mnemonic)
+        'bot.node.env.BOT_L1_MNEMONIC': mnemonic,
+        // The bot does not need Kubernetes API access. Disable RBAC + ServiceAccount creation so the chart
+        // can be installed by users without cluster-scoped RBAC permissions.
+        'bot.rbac.create': false,
+        'bot.serviceAccount.create': false,
+        'bot.serviceAccount.name': 'default'
+    };
+    // Ensure we derive a funded L1 key (index 0 is funded on anvil default mnemonic)
+    if (mnemonicStartIndex === undefined) {
+        values['bot.mnemonicStartIndex'] = 0;
+    }
+    // Also pass a funded private key directly if available
+    if (process.env.FUNDING_PRIVATE_KEY) {
+        values['bot.node.env.BOT_L1_PRIVATE_KEY'] = process.env.FUNDING_PRIVATE_KEY;
+    }
+    // Align bot image with the running network image: prefer env var, else detect from a validator pod
+    let repositoryFromEnv;
+    let tagFromEnv;
+    const aztecDockerImage = process.env.AZTEC_DOCKER_IMAGE;
+    if (aztecDockerImage && aztecDockerImage.includes(':')) {
+        const lastColon = aztecDockerImage.lastIndexOf(':');
+        repositoryFromEnv = aztecDockerImage.slice(0, lastColon);
+        tagFromEnv = aztecDockerImage.slice(lastColon + 1);
+    }
+    let repository = repositoryFromEnv;
+    let tag = tagFromEnv;
+    if (!repository || !tag) {
+        try {
+            const { stdout } = await execAsync(`kubectl get pods -l app.kubernetes.io/name=validator -n ${namespace} -o jsonpath='{.items[0].spec.containers[?(@.name=="aztec")].image}' | cat`);
+            const image = stdout.trim().replace(/^'|'$/g, '');
+            if (image && image.includes(':')) {
+                const lastColon = image.lastIndexOf(':');
+                repository = image.slice(0, lastColon);
+                tag = image.slice(lastColon + 1);
+            }
+        } catch (err) {
+            logger.warn(`Could not detect aztec image from validator pod: ${String(err)}`);
+        }
+    }
+    if (repository && tag) {
+        values['global.aztecImage.repository'] = repository;
+        values['global.aztecImage.tag'] = tag;
+    }
+    if (mnemonicStartIndex !== undefined) {
+        values['bot.mnemonicStartIndex'] = typeof mnemonicStartIndex === 'string' ? mnemonicStartIndex : Number(mnemonicStartIndex);
+    }
+    // If a previous install attempt left the release in a non-deployed state (e.g. FAILED),
+    // `helm upgrade --install` can error with "has no deployed releases".
+    // In that case, clear the release record and do a clean install.
+    const existingStatus = await getHelmReleaseStatus(instanceName, namespace);
+    if (existingStatus && existingStatus.toLowerCase() !== 'deployed') {
+        logger.warn(`Transfer bot release ${instanceName} is in status '${existingStatus}'. Reinstalling cleanly.`);
+        await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`).catch(()=>undefined);
+        // If helm left the release in `uninstalling`, force-delete the record so we can reinstall.
+        const afterUninstallStatus = await getHelmReleaseStatus(instanceName, namespace);
+        if (afterUninstallStatus?.toLowerCase() === 'uninstalling') {
+            await forceDeleteHelmReleaseRecord(instanceName, namespace, logger);
+        }
+    }
+    // `--reuse-values` fails if the release has never successfully deployed (e.g. first install, or a previous failed install).
+    // Only reuse values when we have a deployed release to reuse from.
+    const effectiveReuseValues = reuseValues && await hasDeployedHelmRelease(instanceName, namespace);
+    await execHelmCommand({
+        instanceName,
+        helmChartDir,
+        namespace,
+        valuesFile: undefined,
+        timeout,
+        values: values,
+        reuseValues: effectiveReuseValues
+    });
+    if (replicas > 0) {
+        await waitForResourceByLabel({
+            resource: 'pods',
+            namespace,
+            label: 'app.kubernetes.io/name=bot',
+            condition: 'PodReadyToStartContainers'
+        });
+    }
+}
+/**
+ * Uninstalls the transfer bot Helm release from the given namespace.
+ * Intended for test teardown to clean up bot resources.
+ */ export async function uninstallTransferBot(namespace, logger) {
+    const instanceName = `${namespace}-bot-transfers`;
+    logger.info(`Uninstalling transfer bot release ${instanceName}`);
+    await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`);
+    // Ensure any leftover pods are removed
+    await deleteResourceByLabel({
+        resource: 'pods',
+        namespace,
+        label: 'app.kubernetes.io/name=bot'
+    }).catch(()=>undefined);
+}
+/**
+ * Enables or disables probabilistic transaction dropping on validators and waits for rollout.
+ * Wired to env vars P2P_DROP_TX and P2P_DROP_TX_CHANCE via Helm values.
+ */ export async function setValidatorTxDrop({ namespace, enabled, probability, logger }) {
+    const drop = enabled ? 'true' : 'false';
+    const prob = String(probability);
+    const selectors = [
+        'app.kubernetes.io/name=validator',
+        'app.kubernetes.io/component=validator',
+        'app=validator'
+    ];
+    let updated = false;
+    for (const selector of selectors){
+        try {
+            const list = await execAsync(`kubectl get statefulset -l ${selector} -n ${namespace} --no-headers -o name | cat`);
+            const names = list.stdout.split('\n').map((s)=>s.trim()).filter(Boolean);
+            if (names.length === 0) {
+                continue;
+            }
+            const cmd = `kubectl set env statefulset -l ${selector} -n ${namespace} P2P_DROP_TX=${drop} P2P_DROP_TX_CHANCE=${prob}`;
+            logger.info(`command: ${cmd}`);
+            await execAsync(cmd);
+            updated = true;
+        } catch (e) {
+            logger.warn(`Failed to update validators with selector ${selector}: ${String(e)}`);
+        }
+    }
+    if (!updated) {
+        logger.warn(`No validator StatefulSets found in ${namespace}. Skipping tx drop toggle.`);
+        return;
+    }
+    // Restart validator pods to ensure env vars take effect and wait for readiness
+    await restartValidators(namespace, logger);
+}
+export async function restartValidators(namespace, logger) {
+    const selectors = [
+        'app.kubernetes.io/name=validator',
+        'app.kubernetes.io/component=validator',
+        'app=validator'
+    ];
+    let any = false;
+    for (const selector of selectors){
+        try {
+            const { stdout } = await execAsync(`kubectl get pods -l ${selector} -n ${namespace} --no-headers -o name | cat`);
+            if (!stdout || stdout.trim().length === 0) {
+                continue;
+            }
+            any = true;
+            await deleteResourceByLabel({
+                resource: 'pods',
+                namespace,
+                label: selector
+            });
+        } catch (e) {
+            logger.warn(`Error restarting validator pods with selector ${selector}: ${String(e)}`);
+        }
+    }
+    if (!any) {
+        logger.warn(`No validator pods found to restart in ${namespace}.`);
+        return;
+    }
+    // Wait for either label to be Ready
+    for (const selector of selectors){
+        try {
+            await waitForResourceByLabel({
+                resource: 'pods',
+                namespace,
+                label: selector
+            });
+            return;
+        } catch  {
+        // try next
+        }
+    }
+    logger.warn(`Validator pods did not report Ready; continuing.`);
+}
 export async function enableValidatorDynamicBootNode(instanceName, namespace, spartanDir, logger) {
     logger.info(`Enabling validator dynamic boot node`);
     await execHelmCommand({
@@ -340,11 +655,27 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
     logger.info(`Validator dynamic boot node enabled`);
 }
 export async function getSequencers(namespace) {
-    const command = `kubectl get pods -l app.kubernetes.io/component=validator -n ${namespace} -o jsonpath='{.items[*].metadata.name}'`;
-    const { stdout } = await execAsync(command);
-    const sequencers = stdout.split(' ');
-    logger.verbose(`Found sequencer pods ${sequencers.join(', ')}`);
-    return sequencers;
+    const selectors = [
+        'app.kubernetes.io/name=validator',
+        'app.kubernetes.io/component=validator',
+        'app.kubernetes.io/component=sequencer-node',
+        'app=validator'
+    ];
+    for (const selector of selectors){
+        try {
+            const command = `kubectl get pods -l ${selector} -n ${namespace} -o jsonpath='{.items[*].metadata.name}'`;
+            const { stdout } = await execAsync(command);
+            const sequencers = stdout.split(' ').map((s)=>s.trim()).filter(Boolean);
+            if (sequencers.length > 0) {
+                logger.verbose(`Found sequencer pods ${sequencers.join(', ')} (selector=${selector})`);
+                return sequencers;
+            }
+        } catch  {
+        // try next selector
+        }
+    }
+    // Fail fast instead of returning [''] which leads to attempts to port-forward `pod/`.
+    throw new Error(`No sequencer/validator pods found in namespace ${namespace}. Tried selectors: ${selectors.join(', ')}`);
 }
 export function updateSequencersConfig(env, config) {
     return withSequencersAdmin(env, async (client)=>{
@@ -394,7 +725,9 @@ export async function withSequencersAdmin(env, fn) {
         const url = `http://127.0.0.1:${port}`;
         const client = createPublicClient({
             transport: fallback([
-                http(url)
+                http(url, {
+                    batch: false
+                })
             ])
         });
         if (processes) {
@@ -412,7 +745,9 @@ export async function withSequencersAdmin(env, fn) {
         }
         const client = createPublicClient({
             transport: fallback([
-                http(L1_RPC_URLS_JSON)
+                http(L1_RPC_URLS_JSON, {
+                    batch: false
+                })
             ])
         });
         return {
@@ -528,3 +863,30 @@ export async function withSequencersAdmin(env, fn) {
         throw new Error(`Failed to determine git project root: ${error}`);
     }
 }
+/** Returns a client to the RPC of the given sequencer (defaults to first) */ export async function getNodeClient(env, index = 0) {
+    const namespace = env.NAMESPACE;
+    const containerPort = 8080;
+    const sequencers = await getSequencers(namespace);
+    const sequencer = sequencers[index];
+    if (!sequencer) {
+        throw new Error(`No sequencer found at index ${index} in namespace ${namespace}`);
+    }
+    const { process: process1, port } = await startPortForward({
+        resource: `pod/${sequencer}`,
+        namespace,
+        containerPort
+    });
+    const url = `http://localhost:${port}`;
+    await retry(()=>fetch(`${url}/status`).then((res)=>res.status === 200), 'forward port', makeBackoff([
+        1,
+        1,
+        2,
+        6
+    ]), logger, true);
+    const client = createAztecNodeClient(url);
+    return {
+        node: client,
+        port,
+        process: process1
+    };
+}