@aztec/end-to-end 0.0.1-commit.d3ec352c → 0.0.1-commit.e6bd8901

package/src/spartan/utils/k8s.ts

@@ -0,0 +1,375 @@
+import { createLogger } from '@aztec/aztec.js/log';
+import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { retryUntil } from '@aztec/foundation/retry';
+
+import { type ChildProcess, exec, spawn } from 'child_process';
+import path from 'path';
+import { promisify } from 'util';
+
+const execAsync = promisify(exec);
+
+const logger = createLogger('e2e:k8s-utils');
+
+/**
+ * Represents an endpoint to reach a K8s service.
+ * May be a LoadBalancer external IP or a port-forward.
+ */
+export interface ServiceEndpoint {
+  url: string;
+  process?: ChildProcess;
+}
+
+export async function startPortForward({
+  resource,
+  namespace,
+  containerPort,
+  hostPort,
+}: {
+  resource: string;
+  namespace: string;
+  containerPort: number;
+  // If not provided, the port will be chosen automatically
+  hostPort?: number;
+}): Promise<{
+  process: ChildProcess;
+  port: number;
+}> {
+  const hostPortAsString = hostPort ? hostPort.toString() : '';
+
+  logger.debug(`kubectl port-forward -n ${namespace} ${resource} ${hostPortAsString}:${containerPort}`);
+
+  const process = spawn(
+    'kubectl',
+    ['port-forward', '-n', namespace, resource, `${hostPortAsString}:${containerPort}`],
+    {
+      detached: true,
+      windowsHide: true,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    },
+  );
+
+  let isResolved = false;
+  const connected = new Promise<number>((resolve, reject) => {
+    process.stdout?.on('data', data => {
+      const str = data.toString() as string;
+      if (!isResolved && str.includes('Forwarding from')) {
+        isResolved = true;
+        logger.debug(`Port forward for ${resource}: ${str}`);
+        const port = str.search(/:\d+/);
+        if (port === -1) {
+          reject(new Error('Port not found in port forward output'));
+          return;
+        }
+        const portNumber = parseInt(str.slice(port + 1));
+        logger.verbose(`Port forwarded for ${resource} at ${portNumber}:${containerPort}`);
+        resolve(portNumber);
+      } else {
+        logger.silent(str);
+      }
+    });
+    process.stderr?.on('data', data => {
+      logger.verbose(`Port forward for ${resource}: ${data.toString()}`);
+      // It's a strange thing:
+      // If we don't pipe stderr, then the port forwarding does not work.
+      // Log to silent because this doesn't actually report errors,
+      // just extremely verbose debug logs.
+      logger.silent(data.toString());
+    });
+    process.on('close', () => {
+      if (!isResolved) {
+        isResolved = true;
+        const msg = `Port forward for ${resource} closed before connection established`;
+        logger.warn(msg);
+        reject(new Error(msg));
+      }
+    });
+    process.on('error', error => {
+      if (!isResolved) {
+        isResolved = true;
+        const msg = `Port forward for ${resource} error: ${error}`;
+        logger.error(msg);
+        reject(new Error(msg));
+      }
+    });
+    process.on('exit', code => {
+      if (!isResolved) {
+        isResolved = true;
+        const msg = `Port forward for ${resource} exited with code ${code}`;
+        logger.verbose(msg);
+        reject(new Error(msg));
+      }
+    });
+  });
+
+  const port = await connected;
+
+  return { process, port };
+}
+
+export function getExternalIP(namespace: string, serviceName: string): Promise<string> {
+  const { promise, resolve, reject } = promiseWithResolvers<string>();
+  const process = spawn(
+    'kubectl',
+    [
+      'get',
+      'service',
+      '-n',
+      namespace,
+      `${namespace}-${serviceName}`,
+      '--output',
+      "jsonpath='{.status.loadBalancer.ingress[0].ip}'",
+    ],
+    {
+      stdio: 'pipe',
+    },
+  );
+
+  let ip = '';
+  process.stdout.on('data', data => {
+    ip += data;
+  });
+  process.on('error', err => {
+    reject(err);
+  });
+  process.on('exit', () => {
+    // kubectl prints JSON. Remove the quotes
+    resolve(ip.replace(/"|'/g, ''));
+  });
+
+  return promise;
+}
+
+/**
+ * Gets an endpoint for a K8s service.
+ * By default, tries to get the external IP first and falls back to port-forward if unavailable.
+ *
+ * @param opts.namespace - K8s namespace
+ * @param opts.serviceName - Service name suffix (e.g., 'rpc-aztec-node', 'eth-execution')
+ * @param opts.containerPort - Port the service exposes
+ * @param opts.usePortForward - If true, skip external IP check and always use port-forward
+ */
+export async function getServiceEndpoint(opts: {
+  namespace: string;
+  serviceName: string;
+  containerPort: number;
+  forcePortForward?: boolean;
+}): Promise<ServiceEndpoint> {
+  const { namespace, serviceName, containerPort, forcePortForward } = opts;
+
+  if (!forcePortForward) {
+    try {
+      const ip = await retryUntil(
+        async () => {
+          try {
+            const ip = await getExternalIP(namespace, serviceName);
+            if (ip && ip !== '' && ip !== '<pending>' && ip !== 'null') {
+              return ip;
+            }
+          } catch (err) {
+            logger.verbose(`Failed to get external IP for ${serviceName}: ${err}`);
+          }
+          return undefined;
+        },
+        `external IP for ${serviceName}`,
+        30,
+        5,
+      );
+      logger.info(`Using external IP for ${serviceName}: ${ip}:${containerPort}`);
+      return { url: `http://${ip}:${containerPort}` };
+    } catch {
+      logger.warn(`External IP not available for ${serviceName} after 5min, using port-forward`);
+    }
+  }
+
+  // Fallback to port-forward
+  const resource = `svc/${namespace}-${serviceName}`;
+
+  const { process, port } = await startPortForward({
+    resource,
+    namespace,
+    containerPort,
+  });
+
+  return { url: `http://127.0.0.1:${port}`, process };
+}
+
+/**
+ * Gets an endpoint for the RPC node service.
+ * Tries external IP first, falls back to port-forward.
+ *
+ * @param namespace - K8s namespace
+ * @param usePortForward - If true, skip external IP and use port-forward directly
+ */
+export async function getRPCEndpoint(namespace: string, forcePortForward?: boolean): Promise<ServiceEndpoint> {
+  return await getServiceEndpoint({
+    namespace,
+    serviceName: 'rpc-aztec-node',
+    containerPort: 8080,
+    forcePortForward,
+  });
+}
+
+/**
+ * Gets an endpoint for the Ethereum execution service.
+ * Tries external IP first, falls back to port-forward.
+ *
+ * @param namespace - K8s namespace
+ * @param usePortForward - If true, skip external IP and use port-forward directly
+ */
+export async function getEthereumEndpoint(namespace: string, forcePortForward?: boolean): Promise<ServiceEndpoint> {
+  return await getServiceEndpoint({
+    namespace,
+    serviceName: 'eth-execution',
+    containerPort: 8545,
+    forcePortForward,
+  });
+}
+
+export function startPortForwardForPrometeheus(namespace: string) {
+  return startPortForward({
+    resource: `svc/${namespace}-prometheus-server`,
+    namespace,
+    containerPort: 80,
+  });
+}
+
+export function startPortForwardForRPC(namespace: string, index = 0) {
+  return startPortForward({
+    resource: `pod/${namespace}-rpc-aztec-node-${index}`,
+    namespace,
+    containerPort: 8080,
+  });
+}
+
+export function startPortForwardForEthereum(namespace: string) {
+  return startPortForward({
+    resource: `services/${namespace}-eth-execution`,
+    namespace,
+    containerPort: 8545,
+  });
+}
+
+export async function deleteResourceByName({
+  resource,
+  namespace,
+  name,
+  force = false,
+}: {
+  resource: string;
+  namespace: string;
+  name: string;
+  force?: boolean;
+}) {
+  const command = `kubectl delete ${resource} ${name} -n ${namespace} --ignore-not-found=true --wait=true ${
+    force ? '--force' : ''
+  }`;
+  logger.info(`command: ${command}`);
+  const { stdout } = await execAsync(command);
+  return stdout;
+}
+
+export async function deleteResourceByLabel({
+  resource,
+  namespace,
+  label,
+  timeout = '5m',
+  force = false,
+}: {
+  resource: string;
+  namespace: string;
+  label: string;
+  timeout?: string;
+  force?: boolean;
+}) {
+  try {
+    // Match both plain and group-qualified names (e.g., "podchaos" or "podchaos.chaos-mesh.org")
+    const escaped = resource.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
+    const regex = `(^|\\.)${escaped}(\\.|$)`;
+    await execAsync(`kubectl api-resources --no-headers -o name | grep -Eq '${regex}'`);
+  } catch (error) {
+    logger.warn(`Resource type '${resource}' not found in cluster, skipping deletion ${error}`);
+    return '';
+  }
+
+  const command = `kubectl delete ${resource} -l ${label} -n ${namespace} --ignore-not-found=true --wait=true --timeout=${timeout} ${
+    force ? '--force' : ''
+  }`;
+  logger.info(`command: ${command}`);
+  const { stdout } = await execAsync(command);
+  return stdout;
+}
+
+export async function waitForResourceByLabel({
+  resource,
+  label,
+  namespace,
+  condition = 'Ready',
+  timeout = '10m',
+}: {
+  resource: string;
+  label: string;
+  namespace: string;
+  condition?: string;
+  timeout?: string;
+}) {
+  const command = `kubectl wait ${resource} -l ${label} --for=condition=${condition} -n ${namespace} --timeout=${timeout}`;
+  logger.info(`command: ${command}`);
+  const { stdout } = await execAsync(command);
+  return stdout;
+}
+
+export async function waitForResourceByName({
+  resource,
+  name,
+  namespace,
+  condition = 'Ready',
+  timeout = '10m',
+}: {
+  resource: string;
+  name: string;
+  namespace: string;
+  condition?: string;
+  timeout?: string;
+}) {
+  const command = `kubectl wait ${resource}/${name} --for=condition=${condition} -n ${namespace} --timeout=${timeout}`;
+  logger.info(`command: ${command}`);
+  const { stdout } = await execAsync(command);
+  return stdout;
+}
+
+export async function waitForResourcesByName({
+  resource,
+  names,
+  namespace,
+  condition = 'Ready',
+  timeout = '10m',
+}: {
+  resource: string;
+  names: string[];
+  namespace: string;
+  condition?: string;
+  timeout?: string;
+}) {
+  if (!names.length) {
+    throw new Error(`No ${resource} names provided to waitForResourcesByName`);
+  }
+
+  // Wait all in parallel; if any fails, surface which one.
+  await Promise.all(
+    names.map(async name => {
+      try {
+        await waitForResourceByName({ resource, name, namespace, condition, timeout });
+      } catch (err) {
+        throw new Error(
+          `Failed waiting for ${resource}/${name} condition=${condition} timeout=${timeout} namespace=${namespace}: ${String(
+            err,
+          )}`,
+        );
+      }
+    }),
+  );
+}
+
+export function getChartDir(spartanDir: string, chartName: string) {
+  return path.join(spartanDir.trim(), chartName);
+}

package/src/spartan/utils/nodes.ts

@@ -0,0 +1,323 @@
+import { createLogger } from '@aztec/aztec.js/log';
+import type { RollupCheatCodes } from '@aztec/aztec/testing';
+import type { CheckpointNumber } from '@aztec/foundation/branded-types';
+import type { Logger } from '@aztec/foundation/log';
+import { makeBackoff, retry } from '@aztec/foundation/retry';
+import { sleep } from '@aztec/foundation/sleep';
+import {
+  type AztecNodeAdmin,
+  type AztecNodeAdminConfig,
+  createAztecNodeAdminClient,
+} from '@aztec/stdlib/interfaces/client';
+
+import { exec } from 'child_process';
+import { promisify } from 'util';
+
+import type { TestConfig } from './config.js';
+import { execHelmCommand } from './helm.js';
+import { deleteResourceByLabel, getChartDir, startPortForward, waitForResourceByLabel } from './k8s.js';
+
+const execAsync = promisify(exec);
+
+const logger = createLogger('e2e:k8s-utils');
+
+export async function awaitCheckpointNumber(
+  rollupCheatCodes: RollupCheatCodes,
+  checkpointNumber: CheckpointNumber,
+  timeoutSeconds: number,
+  log: Logger,
+) {
+  log.info(`Waiting for checkpoint ${checkpointNumber}`);
+  let tips = await rollupCheatCodes.getTips();
+  const endTime = Date.now() + timeoutSeconds * 1000;
+  while (tips.pending < checkpointNumber && Date.now() < endTime) {
+    log.info(`At checkpoint ${tips.pending}`);
+    await sleep(1000);
+    tips = await rollupCheatCodes.getTips();
+  }
+  if (tips.pending < checkpointNumber) {
+    throw new Error(`Timeout waiting for checkpoint ${checkpointNumber}, only reached ${tips.pending}`);
+  } else {
+    log.info(`Reached checkpoint ${tips.pending}`);
+  }
+}
+
+export async function getSequencers(namespace: string) {
+  const selectors = [
+    'app.kubernetes.io/name=validator',
+    'app.kubernetes.io/component=validator',
+    'app.kubernetes.io/component=sequencer-node',
+    'app=validator',
+  ];
+  for (const selector of selectors) {
+    try {
+      const command = `kubectl get pods -l ${selector} -n ${namespace} -o jsonpath='{.items[*].metadata.name}'`;
+      const { stdout } = await execAsync(command);
+      const sequencers = stdout
+        .split(' ')
+        .map(s => s.trim())
+        .filter(Boolean);
+      if (sequencers.length > 0) {
+        logger.verbose(`Found sequencer pods ${sequencers.join(', ')} (selector=${selector})`);
+        return sequencers;
+      }
+    } catch {
+      // try next selector
+    }
+  }
+
+  // Fail fast instead of returning [''] which leads to attempts to port-forward `pod/`.
+  throw new Error(
+    `No sequencer/validator pods found in namespace ${namespace}. Tried selectors: ${selectors.join(', ')}`,
+  );
+}
+
+export function updateSequencersConfig(env: TestConfig, config: Partial<AztecNodeAdminConfig>) {
+  return withSequencersAdmin(env, async client => {
+    await client.setConfig(config);
+    return client.getConfig();
+  });
+}
+
+export function getSequencersConfig(env: TestConfig) {
+  return withSequencersAdmin(env, client => client.getConfig());
+}
+
+export async function withSequencersAdmin<T>(env: TestConfig, fn: (node: AztecNodeAdmin) => Promise<T>): Promise<T[]> {
+  const adminContainerPort = 8880;
+  const namespace = env.NAMESPACE;
+  const sequencers = await getSequencers(namespace);
+  const results = [];
+
+  for (const sequencer of sequencers) {
+    // Wrap port-forward + fetch in a retry to handle flaky port-forwards
+    const result = await retry(
+      async () => {
+        const { process, port } = await startPortForward({
+          resource: `pod/${sequencer}`,
+          namespace,
+          containerPort: adminContainerPort,
+        });
+
+        try {
+          const url = `http://localhost:${port}`;
+          // Quick health check before using the connection
+          const statusRes = await fetch(`${url}/status`);
+          if (statusRes.status !== 200) {
+            throw new Error(`Admin endpoint returned status ${statusRes.status}`);
+          }
+          const client = createAztecNodeAdminClient(url);
+          return { result: await fn(client), process };
+        } catch (err) {
+          // Kill the port-forward before retrying
+          process.kill();
+          throw err;
+        }
+      },
+      'connect to node admin',
+      makeBackoff([1, 2, 4, 8]),
+      logger,
+      true,
+    );
+
+    results.push(result.result);
+    result.process.kill();
+  }
+
+  return results;
+}
+
+/**
+ * Enables or disables probabilistic transaction dropping on validators and waits for rollout.
+ * Wired to env vars P2P_DROP_TX and P2P_DROP_TX_CHANCE via Helm values.
+ */
+export async function setValidatorTxDrop({
+  namespace,
+  enabled,
+  probability,
+  logger: log,
+}: {
+  namespace: string;
+  enabled: boolean;
+  probability: number;
+  logger: Logger;
+}) {
+  const drop = enabled ? 'true' : 'false';
+  const prob = String(probability);
+
+  const selectors = ['app.kubernetes.io/name=validator', 'app.kubernetes.io/component=validator', 'app=validator'];
+  let updated = false;
+  for (const selector of selectors) {
+    try {
+      const list = await execAsync(`kubectl get statefulset -l ${selector} -n ${namespace} --no-headers -o name | cat`);
+      const names = list.stdout
+        .split('\n')
+        .map(s => s.trim())
+        .filter(Boolean);
+      if (names.length === 0) {
+        continue;
+      }
+      const cmd = `kubectl set env statefulset -l ${selector} -n ${namespace} P2P_DROP_TX=${drop} P2P_DROP_TX_CHANCE=${prob}`;
+      log.info(`command: ${cmd}`);
+      await execAsync(cmd);
+      updated = true;
+    } catch (e) {
+      log.warn(`Failed to update validators with selector ${selector}: ${String(e)}`);
+    }
+  }
+
+  if (!updated) {
+    log.warn(`No validator StatefulSets found in ${namespace}. Skipping tx drop toggle.`);
+    return;
+  }
+
+  // Restart validator pods to ensure env vars take effect and wait for readiness
+  await restartValidators(namespace, log);
+}
+
+export async function restartValidators(namespace: string, log: Logger) {
+  const selectors = ['app.kubernetes.io/name=validator', 'app.kubernetes.io/component=validator', 'app=validator'];
+  let any = false;
+  for (const selector of selectors) {
+    try {
+      const { stdout } = await execAsync(`kubectl get pods -l ${selector} -n ${namespace} --no-headers -o name | cat`);
+      if (!stdout || stdout.trim().length === 0) {
+        continue;
+      }
+      any = true;
+      await deleteResourceByLabel({ resource: 'pods', namespace, label: selector });
+    } catch (e) {
+      log.warn(`Error restarting validator pods with selector ${selector}: ${String(e)}`);
+    }
+  }
+
+  if (!any) {
+    log.warn(`No validator pods found to restart in ${namespace}.`);
+    return;
+  }
+
+  // Wait for either label to be Ready
+  for (const selector of selectors) {
+    try {
+      await waitForResourceByLabel({ resource: 'pods', namespace, label: selector });
+      return;
+    } catch {
+      // try next
+    }
+  }
+  log.warn(`Validator pods did not report Ready; continuing.`);
+}
+
+export async function enableValidatorDynamicBootNode(
+  instanceName: string,
+  namespace: string,
+  spartanDir: string,
+  log: Logger,
+) {
+  log.info(`Enabling validator dynamic boot node`);
+  await execHelmCommand({
+    instanceName,
+    namespace,
+    helmChartDir: getChartDir(spartanDir, 'aztec-network'),
+    values: {
+      'validator.dynamicBootNode': 'true',
+    },
+    valuesFile: undefined,
+    timeout: '15m',
+    reuseValues: true,
+  });
+
+  log.info(`Validator dynamic boot node enabled`);
+}
+
+/**
+ * Rolls the Aztec pods in the given namespace.
+ * @param namespace - The namespace to roll the Aztec pods in.
+ * @param clearState - If true, also deletes the underlying PVCs to clear persistent storage.
+ *        This is required for rollup upgrades where the old state is incompatible with the new rollup.
+ *        Defaults to false, which preserves the existing storage.
+ */
+export async function rollAztecPods(namespace: string, clearState: boolean = false) {
+  // Pod components use 'validator', but StatefulSets and PVCs use 'sequencer-node' for validators
+  const podComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'prover-agent', 'sequencer-node', 'rpc'];
+  const pvcComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'sequencer-node', 'rpc'];
+  // StatefulSet components that need to be scaled down before PVC deletion
+  // Note: validators use 'sequencer-node' as component label, not 'validator'
+  const statefulSetComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'sequencer-node', 'rpc'];
+
+  if (clearState) {
+    // To delete PVCs, we must first scale down StatefulSets so pods release the volumes
+    // Otherwise PVC deletion will hang waiting for pods to terminate
+
+    // First, save original replica counts
+    const originalReplicas: Map<string, number> = new Map();
+    for (const component of statefulSetComponents) {
+      try {
+        const getCmd = `kubectl get statefulset -l app.kubernetes.io/component=${component} -n ${namespace} -o jsonpath='{.items[0].spec.replicas}'`;
+        const { stdout } = await execAsync(getCmd);
+        const replicas = parseInt(stdout.replace(/'/g, '').trim(), 10);
+        if (!isNaN(replicas) && replicas > 0) {
+          originalReplicas.set(component, replicas);
+        }
+      } catch {
+        // Component might not exist, continue
+      }
+    }
+
+    // Scale down to 0
+    for (const component of statefulSetComponents) {
+      try {
+        const scaleCmd = `kubectl scale statefulset -l app.kubernetes.io/component=${component} -n ${namespace} --replicas=0 --timeout=2m`;
+        logger.info(`command: ${scaleCmd}`);
+        await execAsync(scaleCmd);
+      } catch (e) {
+        // Component might not exist or might be a Deployment, continue
+        logger.verbose(`Scale down ${component} skipped: ${e}`);
+      }
+    }
+
+    // Wait for pods to terminate
+    await sleep(15 * 1000);
+
+    // Now delete PVCs (they should no longer be in use)
+    for (const component of pvcComponents) {
+      await deleteResourceByLabel({
+        resource: 'persistentvolumeclaims',
+        namespace: namespace,
+        label: `app.kubernetes.io/component=${component}`,
+      });
+    }
+
+    // Scale StatefulSets back up to original replica counts
+    for (const component of statefulSetComponents) {
+      const replicas = originalReplicas.get(component) ?? 1;
+      try {
+        const scaleCmd = `kubectl scale statefulset -l app.kubernetes.io/component=${component} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+        logger.info(`command: ${scaleCmd}`);
+        await execAsync(scaleCmd);
+      } catch (e) {
+        logger.verbose(`Scale up ${component} skipped: ${e}`);
+      }
+    }
+  } else {
+    // Just delete pods (no state clearing)
+    for (const component of podComponents) {
+      await deleteResourceByLabel({
+        resource: 'pods',
+        namespace: namespace,
+        label: `app.kubernetes.io/component=${component}`,
+      });
+    }
+  }
+
+  await sleep(10 * 1000);
+
+  // Wait for pods to come back
+  for (const component of podComponents) {
+    await waitForResourceByLabel({
+      resource: 'pods',
+      namespace: namespace,
+      label: `app.kubernetes.io/component=${component}`,
+    });
+  }
+}