@aztec/end-to-end 0.0.1-commit.96bb3f7 → 0.0.1-commit.a072138

package/src/spartan/utils/helm.ts

@@ -0,0 +1,84 @@
+import { createLogger } from '@aztec/aztec.js/log';
+import type { Logger } from '@aztec/foundation/log';
+
+import { exec } from 'child_process';
+import { promisify } from 'util';
+
+const execAsync = promisify(exec);
+
+const logger = createLogger('e2e:k8s-utils');
+
+function shellQuote(value: string) {
+  // Single-quote safe shell escaping: ' -> '\''
+  return `'${value.replace(/'/g, "'\\''")}'`;
+}
+
+function valuesToArgs(values: Record<string, string | number | boolean>) {
+  return Object.entries(values)
+    .map(([key, value]) =>
+      typeof value === 'number' || typeof value === 'boolean'
+        ? `--set ${key}=${value}`
+        : `--set-string ${key}=${shellQuote(String(value))}`,
+    )
+    .join(' ');
+}
+
+export function createHelmCommand({
+  instanceName,
+  helmChartDir,
+  namespace,
+  valuesFile,
+  timeout,
+  values,
+  reuseValues = false,
+}: {
+  instanceName: string;
+  helmChartDir: string;
+  namespace: string;
+  valuesFile: string | undefined;
+  timeout: string;
+  values: Record<string, string | number | boolean>;
+  reuseValues?: boolean;
+}) {
+  const valuesFileArgs = valuesFile ? `--values ${helmChartDir}/values/${valuesFile}` : '';
+  const reuseValuesArgs = reuseValues ? '--reuse-values' : '';
+  return `helm upgrade --install ${instanceName} ${helmChartDir} --namespace ${namespace} ${valuesFileArgs} ${reuseValuesArgs} --wait --timeout=${timeout} ${valuesToArgs(
+    values,
+  )}`;
+}
+
+export async function execHelmCommand(args: Parameters<typeof createHelmCommand>[0]) {
+  const helmCommand = createHelmCommand(args);
+  logger.info(`helm command: ${helmCommand}`);
+  const { stdout } = await execAsync(helmCommand);
+  return stdout;
+}
+
+export async function getHelmReleaseStatus(instanceName: string, namespace: string): Promise<string | undefined> {
+  try {
+    const { stdout } = await execAsync(
+      `helm list --namespace ${namespace} --all --filter '^${instanceName}$' --output json | cat`,
+    );
+    const parsed = JSON.parse(stdout) as Array<{ name?: string; status?: string }>;
+    const row = parsed.find(r => r.name === instanceName);
+    return row?.status;
+  } catch {
+    return undefined;
+  }
+}
+
+export async function forceDeleteHelmReleaseRecord(instanceName: string, namespace: string, log?: Logger) {
+  const labelSelector = `owner=helm,name=${instanceName}`;
+  const cmd = `kubectl delete secret -n ${namespace} -l ${labelSelector} --ignore-not-found=true`;
+  (log ?? logger).warn(`Force deleting Helm release record: ${cmd}`);
+  await execAsync(cmd).catch(() => undefined);
+}
+
+export async function hasDeployedHelmRelease(instanceName: string, namespace: string): Promise<boolean> {
+  try {
+    const status = await getHelmReleaseStatus(instanceName, namespace);
+    return status?.toLowerCase() === 'deployed';
+  } catch {
+    return false;
+  }
+}

package/src/spartan/utils/index.ts

@@ -0,0 +1,64 @@
+// Re-export all utilities from individual modules
+// Note: Internal helpers (helm commands, etc.) are not re-exported to maintain the original public API
+
+// Config
+export { type TestConfig, setupEnvironment } from './config.js';
+
+// Scripts
+export { getGitProjectRoot, getAztecBin, runAztecBin, runProjectScript } from './scripts.js';
+
+// K8s operations
+export {
+  startPortForward,
+  getExternalIP,
+  startPortForwardForPrometeheus,
+  startPortForwardForRPC,
+  startPortForwardForEthereum,
+  deleteResourceByName,
+  deleteResourceByLabel,
+  waitForResourceByLabel,
+  waitForResourceByName,
+  waitForResourcesByName,
+  getChartDir,
+  type ServiceEndpoint,
+  getServiceEndpoint,
+  getRPCEndpoint,
+  getEthereumEndpoint,
+  createResilientPrometheusConnection,
+} from './k8s.js';
+
+// Chaos Mesh
+export {
+  uninstallChaosMesh,
+  installChaosMeshChart,
+  applyProverFailure,
+  applyValidatorFailure,
+  applyProverKill,
+  applyProverBrokerKill,
+  applyBootNodeFailure,
+  applyValidatorKill,
+  applyNetworkShaping,
+} from './chaos.js';
+
+// Bot management
+export { restartBot, installTransferBot, uninstallTransferBot } from './bot.js';
+
+// Node operations (sequencers, validators, pods)
+export {
+  awaitCheckpointNumber,
+  waitForProvenToAdvance,
+  getSequencers,
+  updateSequencersConfig,
+  getSequencersConfig,
+  withSequencersAdmin,
+  setValidatorTxDrop,
+  restartValidators,
+  enableValidatorDynamicBootNode,
+  rollAztecPods,
+} from './nodes.js';
+
+// Client utilities
+export { getPublicViemClient, getL1DeploymentAddresses, getNodeClient } from './clients.js';
+
+// Health checks
+export { ChainHealth, type ChainHealthSnapshot } from './health.js';

package/src/spartan/utils/k8s.ts

@@ -0,0 +1,527 @@
+import { createLogger } from '@aztec/aztec.js/log';
+import type { Logger } from '@aztec/foundation/log';
+import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { retryUntil } from '@aztec/foundation/retry';
+
+import { type ChildProcess, exec, spawn } from 'child_process';
+import path from 'path';
+import { promisify } from 'util';
+
+import { AlertTriggeredError, GrafanaClient } from '../../quality_of_service/grafana_client.js';
+
+const execAsync = promisify(exec);
+
+const logger = createLogger('e2e:k8s-utils');
+
+/**
+ * Represents an endpoint to reach a K8s service.
+ * May be a LoadBalancer external IP or a port-forward.
+ */
+export interface ServiceEndpoint {
+  url: string;
+  process?: ChildProcess;
+}
+
+export async function startPortForward({
+  resource,
+  namespace,
+  containerPort,
+  hostPort,
+}: {
+  resource: string;
+  namespace: string;
+  containerPort: number;
+  // If not provided, the port will be chosen automatically
+  hostPort?: number;
+}): Promise<{
+  process: ChildProcess;
+  port: number;
+}> {
+  const hostPortAsString = hostPort ? hostPort.toString() : '';
+
+  logger.debug(`kubectl port-forward -n ${namespace} ${resource} ${hostPortAsString}:${containerPort}`);
+
+  const process = spawn(
+    'kubectl',
+    ['port-forward', '-n', namespace, resource, `${hostPortAsString}:${containerPort}`],
+    {
+      detached: true,
+      windowsHide: true,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    },
+  );
+
+  let isResolved = false;
+  const connected = new Promise<number>((resolve, reject) => {
+    process.stdout?.on('data', data => {
+      const str = data.toString() as string;
+      if (!isResolved && str.includes('Forwarding from')) {
+        isResolved = true;
+        logger.debug(`Port forward for ${resource}: ${str}`);
+        const port = str.search(/:\d+/);
+        if (port === -1) {
+          reject(new Error('Port not found in port forward output'));
+          return;
+        }
+        const portNumber = parseInt(str.slice(port + 1));
+        logger.verbose(`Port forwarded for ${resource} at ${portNumber}:${containerPort}`);
+        resolve(portNumber);
+      } else {
+        logger.silent(str);
+      }
+    });
+    process.stderr?.on('data', data => {
+      logger.verbose(`Port forward for ${resource}: ${data.toString()}`);
+      // It's a strange thing:
+      // If we don't pipe stderr, then the port forwarding does not work.
+      // Log to silent because this doesn't actually report errors,
+      // just extremely verbose debug logs.
+      logger.silent(data.toString());
+    });
+    process.on('close', () => {
+      if (!isResolved) {
+        isResolved = true;
+        const msg = `Port forward for ${resource} closed before connection established`;
+        logger.warn(msg);
+        reject(new Error(msg));
+      }
+    });
+    process.on('error', error => {
+      if (!isResolved) {
+        isResolved = true;
+        const msg = `Port forward for ${resource} error: ${error}`;
+        logger.error(msg);
+        reject(new Error(msg));
+      }
+    });
+    process.on('exit', code => {
+      if (!isResolved) {
+        isResolved = true;
+        const msg = `Port forward for ${resource} exited with code ${code}`;
+        logger.verbose(msg);
+        reject(new Error(msg));
+      }
+    });
+  });
+
+  const port = await connected;
+
+  return { process, port };
+}
+
+export function getExternalIP(namespace: string, serviceName: string): Promise<string> {
+  const { promise, resolve, reject } = promiseWithResolvers<string>();
+  const process = spawn(
+    'kubectl',
+    [
+      'get',
+      'service',
+      '-n',
+      namespace,
+      `${namespace}-${serviceName}`,
+      '--output',
+      "jsonpath='{.status.loadBalancer.ingress[0].ip}'",
+    ],
+    {
+      stdio: 'pipe',
+    },
+  );
+
+  let ip = '';
+  process.stdout.on('data', data => {
+    ip += data;
+  });
+  process.on('error', err => {
+    reject(err);
+  });
+  process.on('exit', () => {
+    // kubectl prints JSON. Remove the quotes
+    resolve(ip.replace(/"|'/g, ''));
+  });
+
+  return promise;
+}
+
+/**
+ * Gets an endpoint for a K8s service.
+ * By default, tries to get the external IP first and falls back to port-forward if unavailable.
+ *
+ * @param opts.namespace - K8s namespace
+ * @param opts.serviceName - Service name suffix (e.g., 'rpc-aztec-node', 'eth-execution')
+ * @param opts.containerPort - Port the service exposes
+ * @param opts.usePortForward - If true, skip external IP check and always use port-forward
+ */
+export async function getServiceEndpoint(opts: {
+  namespace: string;
+  serviceName: string;
+  containerPort: number;
+  forcePortForward?: boolean;
+}): Promise<ServiceEndpoint> {
+  const { namespace, serviceName, containerPort, forcePortForward } = opts;
+
+  if (!forcePortForward) {
+    try {
+      const ip = await retryUntil(
+        async () => {
+          try {
+            const ip = await getExternalIP(namespace, serviceName);
+            if (ip && ip !== '' && ip !== '<pending>' && ip !== 'null') {
+              return ip;
+            }
+          } catch (err) {
+            logger.verbose(`Failed to get external IP for ${serviceName}: ${err}`);
+          }
+          return undefined;
+        },
+        `external IP for ${serviceName}`,
+        30,
+        5,
+      );
+      logger.info(`Using external IP for ${serviceName}: ${ip}:${containerPort}`);
+      return { url: `http://${ip}:${containerPort}` };
+    } catch {
+      logger.warn(`External IP not available for ${serviceName} after 5min, using port-forward`);
+    }
+  }
+
+  // Fallback to port-forward
+  const resource = `svc/${namespace}-${serviceName}`;
+
+  const { process, port } = await startPortForward({
+    resource,
+    namespace,
+    containerPort,
+  });
+
+  return { url: `http://127.0.0.1:${port}`, process };
+}
+
+/**
+ * Gets an endpoint for the RPC node service.
+ * Tries external IP first, falls back to port-forward.
+ *
+ * @param namespace - K8s namespace
+ * @param usePortForward - If true, skip external IP and use port-forward directly
+ */
+export async function getRPCEndpoint(namespace: string, forcePortForward?: boolean): Promise<ServiceEndpoint> {
+  return await getServiceEndpoint({
+    namespace,
+    serviceName: 'rpc-aztec-node',
+    containerPort: 8080,
+    forcePortForward,
+  });
+}
+
+/**
+ * Gets an endpoint for the Ethereum execution service.
+ * Tries external IP first, falls back to port-forward.
+ *
+ * @param namespace - K8s namespace
+ * @param usePortForward - If true, skip external IP and use port-forward directly
+ */
+export async function getEthereumEndpoint(namespace: string, forcePortForward?: boolean): Promise<ServiceEndpoint> {
+  return await getServiceEndpoint({
+    namespace,
+    serviceName: 'eth-execution',
+    containerPort: 8545,
+    forcePortForward,
+  });
+}
+
+export function startPortForwardForPrometeheus(namespace: string) {
+  return startPortForward({
+    resource: `svc/${namespace}-prometheus-server`,
+    namespace,
+    containerPort: 80,
+  });
+}
+
+export function startPortForwardForRPC(namespace: string, index = 0) {
+  return startPortForward({
+    resource: `pod/${namespace}-rpc-aztec-node-${index}`,
+    namespace,
+    containerPort: 8080,
+  });
+}
+
+export function startPortForwardForEthereum(namespace: string) {
+  return startPortForward({
+    resource: `services/${namespace}-eth-execution`,
+    namespace,
+    containerPort: 8545,
+  });
+}
+
+export async function deleteResourceByName({
+  resource,
+  namespace,
+  name,
+  force = false,
+}: {
+  resource: string;
+  namespace: string;
+  name: string;
+  force?: boolean;
+}) {
+  const command = `kubectl delete ${resource} ${name} -n ${namespace} --ignore-not-found=true --wait=true ${
+    force ? '--force' : ''
+  }`;
+  logger.info(`command: ${command}`);
+  const { stdout } = await execAsync(command);
+  return stdout;
+}
+
+export async function deleteResourceByLabel({
+  resource,
+  namespace,
+  label,
+  timeout = '5m',
+  force = false,
+}: {
+  resource: string;
+  namespace: string;
+  label: string;
+  timeout?: string;
+  force?: boolean;
+}) {
+  try {
+    // Match both plain and group-qualified names (e.g., "podchaos" or "podchaos.chaos-mesh.org")
+    const escaped = resource.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
+    const regex = `(^|\\.)${escaped}(\\.|$)`;
+    await execAsync(`kubectl api-resources --no-headers -o name | grep -Eq '${regex}'`);
+  } catch (error) {
+    logger.warn(`Resource type '${resource}' not found in cluster, skipping deletion ${error}`);
+    return '';
+  }
+
+  const command = `kubectl delete ${resource} -l ${label} -n ${namespace} --ignore-not-found=true --wait=true --timeout=${timeout} ${
+    force ? '--force' : ''
+  }`;
+  logger.info(`command: ${command}`);
+  const { stdout } = await execAsync(command);
+  return stdout;
+}
+
+export async function waitForResourceByLabel({
+  resource,
+  label,
+  namespace,
+  condition = 'Ready',
+  timeout = '10m',
+}: {
+  resource: string;
+  label: string;
+  namespace: string;
+  condition?: string;
+  timeout?: string;
+}) {
+  const command = `kubectl wait ${resource} -l ${label} --for=condition=${condition} -n ${namespace} --timeout=${timeout}`;
+  logger.info(`command: ${command}`);
+  const { stdout } = await execAsync(command);
+  return stdout;
+}
+
+export async function waitForResourceByName({
+  resource,
+  name,
+  namespace,
+  condition = 'Ready',
+  timeout = '10m',
+}: {
+  resource: string;
+  name: string;
+  namespace: string;
+  condition?: string;
+  timeout?: string;
+}) {
+  const command = `kubectl wait ${resource}/${name} --for=condition=${condition} -n ${namespace} --timeout=${timeout}`;
+  logger.info(`command: ${command}`);
+  const { stdout } = await execAsync(command);
+  return stdout;
+}
+
+export async function waitForResourcesByName({
+  resource,
+  names,
+  namespace,
+  condition = 'Ready',
+  timeout = '10m',
+}: {
+  resource: string;
+  names: string[];
+  namespace: string;
+  condition?: string;
+  timeout?: string;
+}) {
+  if (!names.length) {
+    throw new Error(`No ${resource} names provided to waitForResourcesByName`);
+  }
+
+  // Wait all in parallel; if any fails, surface which one.
+  await Promise.all(
+    names.map(async name => {
+      try {
+        await waitForResourceByName({ resource, name, namespace, condition, timeout });
+      } catch (err) {
+        throw new Error(
+          `Failed waiting for ${resource}/${name} condition=${condition} timeout=${timeout} namespace=${namespace}: ${String(
+            err,
+          )}`,
+        );
+      }
+    }),
+  );
+}
+
+/**
+ * Waits for all StatefulSets matching a label to have all their replicas ready.
+ *
+ * @param namespace - Kubernetes namespace
+ * @param label - Label selector for StatefulSets (e.g., "app.kubernetes.io/component=sequencer-node")
+ * @param timeoutSeconds - Maximum time to wait in seconds
+ * @param pollIntervalSeconds - How often to check status
+ */
+export async function waitForStatefulSetsReady({
+  namespace,
+  label,
+  timeoutSeconds = 600,
+  pollIntervalSeconds = 5,
+}: {
+  namespace: string;
+  label: string;
+  timeoutSeconds?: number;
+  pollIntervalSeconds?: number;
+}): Promise<void> {
+  logger.info(`Waiting for StatefulSets with label ${label} to have all replicas ready (timeout: ${timeoutSeconds}s)`);
+
+  await retryUntil(
+    async () => {
+      // Get all StatefulSets matching the label
+      const getCmd = `kubectl get statefulset -l ${label} -n ${namespace} -o json`;
+      const { stdout } = await execAsync(getCmd);
+      const result = JSON.parse(stdout);
+
+      if (!result.items || result.items.length === 0) {
+        logger.verbose(`No StatefulSets found with label ${label}`);
+        return false;
+      }
+
+      // Check each StatefulSet
+      for (const sts of result.items) {
+        const name = sts.metadata.name;
+        const desired = sts.spec.replicas ?? 0;
+        const ready = sts.status.readyReplicas ?? 0;
+        const updated = sts.status.updatedReplicas ?? 0;
+
+        if (ready < desired || updated < desired) {
+          logger.verbose(`StatefulSet ${name}: ${ready}/${desired} ready, ${updated}/${desired} updated`);
+          return false;
+        }
+      }
+
+      logger.info(`All StatefulSets with label ${label} are ready`);
+      return true;
+    },
+    `StatefulSets with label ${label} to be ready`,
+    timeoutSeconds,
+    pollIntervalSeconds,
+  );
+}
+
+/**
+ * Creates a Prometheus connection that can re-establish port-forward on failure.
+ * Returns functions to connect and run alert checks that automatically reconnect if needed.
+ *
+ * @param namespace - K8s namespace to fall back to if metrics namespace doesn't have Prometheus
+ * @param endpoints - Array to track created endpoints for cleanup
+ * @param log - Logger instance
+ */
+export function createResilientPrometheusConnection(
+  namespace: string,
+  endpoints: ServiceEndpoint[],
+  log: Logger,
+): {
+  connect: () => Promise<GrafanaClient>;
+  runAlertCheck: (alerts: Parameters<GrafanaClient['runAlertCheck']>[0]) => Promise<void>;
+} {
+  let alertChecker: GrafanaClient | undefined;
+  let currentEndpoint: ServiceEndpoint | undefined;
+
+  const connect = async (): Promise<GrafanaClient> => {
+    // Kill existing connection if any
+    if (currentEndpoint?.process) {
+      currentEndpoint.process.kill();
+    }
+
+    // Try metrics namespace first, then network namespace
+    let promPort = 0;
+    let promUrl = '';
+    let promProc: ChildProcess | undefined;
+
+    try {
+      const metricsResult = await startPortForward({
+        resource: `svc/metrics-prometheus-server`,
+        namespace: 'metrics',
+        containerPort: 80,
+      });
+      promProc = metricsResult.process;
+      promPort = metricsResult.port;
+      promUrl = `http://127.0.0.1:${promPort}/api/v1`;
+    } catch {
+      // Metrics namespace might not have Prometheus, try network namespace
+      log.verbose('Metrics namespace Prometheus not available, trying network namespace');
+    }
+
+    if (promPort === 0) {
+      const nsResult = await startPortForward({
+        resource: `svc/prometheus-server`,
+        namespace,
+        containerPort: 80,
+      });
+      promProc = nsResult.process;
+      promPort = nsResult.port;
+      promUrl = `http://127.0.0.1:${promPort}/api/v1`;
+    }
+
+    if (!promProc || promPort === 0) {
+      throw new Error('Unable to port-forward to Prometheus');
+    }
+
+    currentEndpoint = { url: promUrl, process: promProc };
+    endpoints.push(currentEndpoint);
+    alertChecker = new GrafanaClient(log, { grafanaEndpoint: promUrl, grafanaCredentials: '' });
+    log.info(`Established Prometheus connection at ${promUrl}`);
+    return alertChecker;
+  };
+
+  const runAlertCheck = async (alerts: Parameters<GrafanaClient['runAlertCheck']>[0]): Promise<void> => {
+    if (!alertChecker) {
+      alertChecker = await connect();
+    }
+
+    try {
+      await alertChecker.runAlertCheck(alerts);
+    } catch (err) {
+      // If it's an AlertTriggeredError (expected behavior)
+      if (err instanceof AlertTriggeredError) {
+        throw err;
+      }
+
+      // Check if it's a connection error (port-forward died)
+      const errorStr = String(err);
+      if (errorStr.includes('fetch failed') || errorStr.includes('ECONNREFUSED') || errorStr.includes('ECONNRESET')) {
+        log.warn(`Prometheus connection lost, re-establishing port-forward...`);
+        alertChecker = await connect();
+        await alertChecker.runAlertCheck(alerts);
+      } else {
+        throw err;
+      }
+    }
+  };
+
+  return { connect, runAlertCheck };
+}
+
+export function getChartDir(spartanDir: string, chartName: string) {
+  return path.join(spartanDir.trim(), chartName);
+}