@aztec/end-to-end 4.0.0-nightly.20260120 → 4.0.0-nightly.20260122

package/src/spartan/utils/chaos.ts

@@ -0,0 +1,253 @@
+import type { Logger } from '@aztec/foundation/log';
+
+import { exec } from 'child_process';
+import { promisify } from 'util';
+
+import { execHelmCommand } from './helm.js';
+import { deleteResourceByLabel, getChartDir } from './k8s.js';
+
+const execAsync = promisify(exec);
+
+export async function uninstallChaosMesh(instanceName: string, namespace: string, log: Logger) {
+  // uninstall the helm chart if it exists
+  log.info(`Uninstalling helm chart ${instanceName}`);
+  await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`);
+  // and delete the chaos-mesh resources created by this release
+  const deleteByLabel = async (resource: string) => {
+    const args = {
+      resource,
+      namespace: namespace,
+      label: `app.kubernetes.io/instance=${instanceName}`,
+    } as const;
+    log.info(`Deleting ${resource} resources for release ${instanceName}`);
+    await deleteResourceByLabel(args).catch(e => {
+      log.error(`Error deleting ${resource}: ${e}`);
+      log.info(`Force deleting ${resource}`);
+      return deleteResourceByLabel({ ...args, force: true });
+    });
+  };
+
+  await deleteByLabel('podchaos');
+  await deleteByLabel('networkchaos');
+  await deleteByLabel('podnetworkchaos');
+  await deleteByLabel('workflows');
+  await deleteByLabel('workflownodes');
+}
+
+/**
+ * Installs a Helm chart with the given parameters.
+ * @param instanceName - The name of the Helm chart instance.
+ * @param targetNamespace - The namespace with the resources to be affected by the Helm chart.
+ * @param valuesFile - The values file to use for the Helm chart.
+ * @param chaosMeshNamespace - The namespace to install the Helm chart in.
+ * @param timeout - The timeout for the Helm command.
+ * @param clean - Whether to clean up the Helm chart before installing it.
+ * @returns The stdout of the Helm command.
+ * @throws If the Helm command fails.
+ *
+ * Example usage:
+ * ```typescript
+ * const stdout = await installChaosMeshChart({ instanceName: 'force-reorg', targetNamespace: 'smoke', valuesFile: 'prover-failure.yaml'});
+ * console.log(stdout);
+ * ```
+ */
+export async function installChaosMeshChart({
+  instanceName,
+  targetNamespace,
+  valuesFile,
+  helmChartDir,
+  timeout = '10m',
+  clean = true,
+  values = {},
+  logger: log,
+}: {
+  instanceName: string;
+  targetNamespace: string;
+  valuesFile: string;
+  helmChartDir: string;
+  chaosMeshNamespace?: string;
+  timeout?: string;
+  clean?: boolean;
+  values?: Record<string, string | number>;
+  logger: Logger;
+}) {
+  if (clean) {
+    await uninstallChaosMesh(instanceName, targetNamespace, log);
+  }
+
+  return execHelmCommand({
+    instanceName,
+    helmChartDir,
+    namespace: targetNamespace,
+    valuesFile,
+    timeout,
+    values: { ...values, 'global.targetNamespace': targetNamespace },
+  });
+}
+
+export function applyProverFailure({
+  namespace,
+  spartanDir,
+  durationSeconds,
+  logger: log,
+}: {
+  namespace: string;
+  spartanDir: string;
+  durationSeconds: number;
+  logger: Logger;
+}) {
+  return installChaosMeshChart({
+    instanceName: 'prover-failure',
+    targetNamespace: namespace,
+    valuesFile: 'prover-failure.yaml',
+    helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+    values: {
+      'proverFailure.duration': `${durationSeconds}s`,
+    },
+    logger: log,
+  });
+}
+
+export function applyValidatorFailure({
+  namespace,
+  spartanDir,
+  logger: log,
+  values,
+  instanceName,
+}: {
+  namespace: string;
+  spartanDir: string;
+  logger: Logger;
+  values?: Record<string, string | number>;
+  instanceName?: string;
+}) {
+  return installChaosMeshChart({
+    instanceName: instanceName ?? 'validator-failure',
+    targetNamespace: namespace,
+    valuesFile: 'validator-failure.yaml',
+    helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+    values,
+    logger: log,
+  });
+}
+
+export function applyProverKill({
+  namespace,
+  spartanDir,
+  logger: log,
+  values,
+}: {
+  namespace: string;
+  spartanDir: string;
+  logger: Logger;
+  values?: Record<string, string | number>;
+}) {
+  return installChaosMeshChart({
+    instanceName: 'prover-kill',
+    targetNamespace: namespace,
+    valuesFile: 'prover-kill.yaml',
+    helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+    chaosMeshNamespace: namespace,
+    clean: true,
+    logger: log,
+    values,
+  });
+}
+
+export function applyProverBrokerKill({
+  namespace,
+  spartanDir,
+  logger: log,
+  values,
+}: {
+  namespace: string;
+  spartanDir: string;
+  logger: Logger;
+  values?: Record<string, string | number>;
+}) {
+  return installChaosMeshChart({
+    instanceName: 'prover-broker-kill',
+    targetNamespace: namespace,
+    valuesFile: 'prover-broker-kill.yaml',
+    helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+    clean: true,
+    logger: log,
+    values,
+  });
+}
+
+export function applyBootNodeFailure({
+  instanceName = 'boot-node-failure',
+  namespace,
+  spartanDir,
+  durationSeconds,
+  logger: log,
+  values,
+}: {
+  instanceName?: string;
+  namespace: string;
+  spartanDir: string;
+  durationSeconds: number;
+  logger: Logger;
+  values?: Record<string, string | number>;
+}) {
+  return installChaosMeshChart({
+    instanceName,
+    targetNamespace: namespace,
+    valuesFile: 'boot-node-failure.yaml',
+    helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+    values: {
+      'bootNodeFailure.duration': `${durationSeconds}s`,
+      ...(values ?? {}),
+    },
+    logger: log,
+  });
+}
+
+export function applyValidatorKill({
+  instanceName = 'validator-kill',
+  namespace,
+  spartanDir,
+  logger: log,
+  values,
+  clean = true,
+}: {
+  instanceName?: string;
+  namespace: string;
+  spartanDir: string;
+  logger: Logger;
+  values?: Record<string, string | number>;
+  clean?: boolean;
+}) {
+  return installChaosMeshChart({
+    instanceName: instanceName ?? 'validator-kill',
+    targetNamespace: namespace,
+    valuesFile: 'validator-kill.yaml',
+    helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+    clean,
+    logger: log,
+    values,
+  });
+}
+
+export function applyNetworkShaping({
+  instanceName = 'network-shaping',
+  valuesFile,
+  namespace,
+  spartanDir,
+  logger: log,
+}: {
+  instanceName?: string;
+  valuesFile: string;
+  namespace: string;
+  spartanDir: string;
+  logger: Logger;
+}) {
+  return installChaosMeshChart({
+    instanceName,
+    targetNamespace: namespace,
+    valuesFile,
+    helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'),
+    logger: log,
+  });
+}

package/src/spartan/utils/clients.ts

@@ -0,0 +1,106 @@
+import { createLogger } from '@aztec/aztec.js/log';
+import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses';
+import type { ViemPublicClient } from '@aztec/ethereum/types';
+import { makeBackoff, retry } from '@aztec/foundation/retry';
+import { createAztecNodeClient } from '@aztec/stdlib/interfaces/client';
+
+import type { ChildProcess } from 'child_process';
+import { createPublicClient, fallback, http } from 'viem';
+
+import type { TestConfig } from './config.js';
+import { startPortForward } from './k8s.js';
+import { getSequencers } from './nodes.js';
+
+const logger = createLogger('e2e:k8s-utils');
+
+/**
+ * Returns a public viem client to the eth execution node. If it was part of a local eth devnet,
+ * it first port-forwards the service and points to it. Otherwise, just uses the external RPC url.
+ */
+export async function getPublicViemClient(
+  env: TestConfig,
+  /** If set, will push the new process into it */
+  processes?: ChildProcess[],
+): Promise<{ url: string; client: ViemPublicClient; process?: ChildProcess }> {
+  const { NAMESPACE, CREATE_ETH_DEVNET, L1_RPC_URLS_JSON } = env;
+  if (CREATE_ETH_DEVNET) {
+    logger.info(`Creating port forward to eth execution node`);
+    const { process, port } = await startPortForward({
+      resource: `svc/${NAMESPACE}-eth-execution`,
+      namespace: NAMESPACE,
+      containerPort: 8545,
+    });
+    const url = `http://127.0.0.1:${port}`;
+    const client: ViemPublicClient = createPublicClient({ transport: fallback([http(url, { batch: false })]) });
+    if (processes) {
+      processes.push(process);
+    }
+    return { url, client, process };
+  } else {
+    logger.info(`Connecting to the eth execution node at ${L1_RPC_URLS_JSON}`);
+    if (!L1_RPC_URLS_JSON) {
+      throw new Error(`L1_RPC_URLS_JSON is not defined`);
+    }
+    const client: ViemPublicClient = createPublicClient({
+      transport: fallback([http(L1_RPC_URLS_JSON, { batch: false })]),
+    });
+    return { url: L1_RPC_URLS_JSON, client };
+  }
+}
+
+/** Queries an Aztec node for the L1 deployment addresses */
+export async function getL1DeploymentAddresses(env: TestConfig): Promise<L1ContractAddresses> {
+  let forwardProcess: ChildProcess | undefined;
+  try {
+    const [sequencer] = await getSequencers(env.NAMESPACE);
+    const { process, port } = await startPortForward({
+      resource: `pod/${sequencer}`,
+      namespace: env.NAMESPACE,
+      containerPort: 8080,
+    });
+
+    forwardProcess = process;
+    const url = `http://127.0.0.1:${port}`;
+    const node = createAztecNodeClient(url);
+    return await retry(
+      () => node.getNodeInfo().then(i => i.l1ContractAddresses),
+      'get node info',
+      makeBackoff([1, 3, 6]),
+      logger,
+    );
+  } finally {
+    forwardProcess?.kill();
+  }
+}
+
+/** Returns a client to the RPC of the given sequencer (defaults to first) */
+export async function getNodeClient(
+  env: TestConfig,
+  index: number = 0,
+): Promise<{ node: ReturnType<typeof createAztecNodeClient>; port: number; process: ChildProcess }> {
+  const namespace = env.NAMESPACE;
+  const containerPort = 8080;
+  const sequencers = await getSequencers(namespace);
+  const sequencer = sequencers[index];
+  if (!sequencer) {
+    throw new Error(`No sequencer found at index ${index} in namespace ${namespace}`);
+  }
+
+  const { process, port } = await startPortForward({
+    resource: `pod/${sequencer}`,
+    namespace,
+    containerPort,
+  });
+
+  const url = `http://localhost:${port}`;
+  await retry(
+    () => fetch(`${url}/status`).then(res => res.status === 200),
+    'forward port',
+    makeBackoff([1, 1, 2, 6]),
+    logger,
+    true,
+  );
+
+  const client = createAztecNodeClient(url);
+  return { node: client, port, process };
+}

package/src/spartan/utils/config.ts

@@ -0,0 +1,26 @@
+import { createLogger } from '@aztec/aztec.js/log';
+import { schemas } from '@aztec/foundation/schemas';
+
+import { z } from 'zod';
+
+const logger = createLogger('e2e:k8s-utils');
+
+const testConfigSchema = z.object({
+  NAMESPACE: z.string().default('scenario'),
+  REAL_VERIFIER: schemas.Boolean.optional().default(true),
+  CREATE_ETH_DEVNET: schemas.Boolean.optional().default(false),
+  L1_RPC_URLS_JSON: z.string().optional(),
+  L1_ACCOUNT_MNEMONIC: z.string().optional(),
+  AZTEC_SLOT_DURATION: z.coerce.number().optional().default(24),
+  AZTEC_EPOCH_DURATION: z.coerce.number().optional().default(32),
+  AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5),
+  AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET: z.coerce.number().optional().default(2),
+});
+
+export type TestConfig = z.infer<typeof testConfigSchema>;
+
+export function setupEnvironment(env: unknown): TestConfig {
+  const config = testConfigSchema.parse(env);
+  logger.warn(`Loaded env config`, config);
+  return config;
+}

package/src/spartan/utils/health.ts

@@ -0,0 +1,256 @@
+import { createAztecNodeClient } from '@aztec/aztec.js/node';
+import { createEthereumChain } from '@aztec/ethereum/chain';
+import { RollupContract } from '@aztec/ethereum/contracts';
+import type { ViemPublicClient } from '@aztec/ethereum/types';
+import { CheckpointNumber } from '@aztec/foundation/branded-types';
+import type { Logger } from '@aztec/foundation/log';
+
+import type { ChildProcess } from 'child_process';
+import { createPublicClient, fallback, http } from 'viem';
+
+import { startPortForwardForEthereum, startPortForwardForRPC } from './k8s.js';
+
+/**
+ * Snapshot of chain state captured during setup for comparison in teardown.
+ */
+export interface ChainHealthSnapshot {
+  blockNumber: number;
+  checkpointNumber: CheckpointNumber;
+  timestamp: number;
+}
+
+/**
+ * Pre-flight and post-flight health checks for the Aztec network.
+ *
+ * Use in beforeAll/afterAll to validate the chain is healthy before tests run
+ * and verify it continued progressing during the test.
+ *
+ * @example
+ * ```typescript
+ * const health = new ChainHealth(config.NAMESPACE, logger);
+ *
+ * beforeAll(async () => {
+ *   await health.setup();
+ * });
+ *
+ * afterAll(async () => {
+ *   await health.teardown();
+ * });
+ * ```
+ */
+export class ChainHealth {
+  private namespace: string;
+  private logger: Logger;
+  private snapshot?: ChainHealthSnapshot;
+
+  constructor(namespace: string, logger: Logger) {
+    this.namespace = namespace;
+    this.logger = logger;
+  }
+
+  /**
+   * Pre-flight health check. Validates chain is in a testable state and captures
+   * initial state for comparison in teardown.
+   *
+   * Checks performed:
+   * - Node is reachable and returns valid info
+   * - ENR exists
+   * - L1 is accessible
+   * - At least 1 L2 block has been mined
+   * - Committee exists
+   * - At least 1 checkpoint has been reached
+   *
+   * @throws Error if any health check fails
+   */
+  async setup(): Promise<void> {
+    const processes: ChildProcess[] = [];
+
+    try {
+      // Establish temporary connections
+      const { process: rpcProcess, port: rpcPort } = await startPortForwardForRPC(this.namespace);
+      processes.push(rpcProcess);
+
+      const { process: ethProcess, port: ethPort } = await startPortForwardForEthereum(this.namespace);
+      processes.push(ethProcess);
+
+      const nodeUrl = `http://127.0.0.1:${rpcPort}`;
+      const ethereumUrl = `http://127.0.0.1:${ethPort}`;
+
+      // Create clients
+      const node = createAztecNodeClient(nodeUrl);
+
+      // Check 1: Node is reachable
+      let nodeInfo;
+      try {
+        nodeInfo = await node.getNodeInfo();
+      } catch (err) {
+        throw new Error(`Health check failed: Node is not reachable at ${nodeUrl}. Error: ${err}`);
+      }
+
+      if (!nodeInfo) {
+        throw new Error('Health check failed: Node returned empty info');
+      }
+
+      // Check 2: ENR exists (P2P identity)
+      if (!nodeInfo.enr || !nodeInfo.enr.startsWith('enr:-')) {
+        throw new Error(`Health check failed: Invalid or missing ENR. Got: ${nodeInfo.enr}`);
+      }
+
+      // Check 3: L1 is accessible
+      const chain = createEthereumChain([ethereumUrl], nodeInfo.l1ChainId);
+      const ethereumClient: ViemPublicClient = createPublicClient({
+        chain: chain.chainInfo,
+        transport: fallback([http(ethereumUrl, { batch: false })]),
+      });
+
+      try {
+        await ethereumClient.getBlockNumber();
+      } catch (err) {
+        throw new Error(`Health check failed: L1 is not accessible at ${ethereumUrl}. Error: ${err}`);
+      }
+
+      // Check 4: At least 1 L2 block mined
+      let l2BlockNumber;
+      try {
+        l2BlockNumber = await node.getBlockNumber();
+      } catch (err) {
+        throw new Error(`Health check failed: Could not get L2 block number. Error: ${err}`);
+      }
+
+      if (l2BlockNumber < 1) {
+        throw new Error(`Health check failed: No L2 blocks mined yet. Block number: ${l2BlockNumber}`);
+      }
+
+      // Check 5: Committee exists
+      const rollup = new RollupContract(ethereumClient, nodeInfo.l1ContractAddresses.rollupAddress);
+
+      let committee;
+      try {
+        committee = await rollup.getCurrentEpochCommittee();
+      } catch (err) {
+        throw new Error(`Health check failed: Could not get committee. Error: ${err}`);
+      }
+
+      if (!committee || committee.length === 0) {
+        throw new Error('Health check failed: No committee exists. Validators may not be registered yet.');
+      }
+
+      // Check 6: At least 1 checkpoint reached
+      let checkpointNumber;
+      try {
+        checkpointNumber = await rollup.getCheckpointNumber();
+      } catch (err) {
+        throw new Error(`Health check failed: Could not get checkpoint number. Error: ${err}`);
+      }
+
+      if (checkpointNumber < CheckpointNumber(1)) {
+        throw new Error(
+          `Health check failed: No checkpoint reached yet. Checkpoint number: ${checkpointNumber}. ` +
+            'The proving pipeline may not have completed a proof yet.',
+        );
+      }
+
+      // Capture snapshot for teardown comparison
+      this.snapshot = {
+        blockNumber: l2BlockNumber,
+        checkpointNumber,
+        timestamp: Date.now(),
+      };
+
+      this.logger.info('Pre-flight health check passed');
+    } finally {
+      processes.forEach(p => p.kill());
+    }
+  }
+
+  /**
+   * Post-flight health check. Verifies the chain continued progressing during the test.
+   *
+   * For tests that ran longer than the threshold, checks:
+   * - Block number increased since setup
+   * - Checkpoint number increased since setup
+   *
+   * For shorter tests, skips the check.
+   *
+   * @throws Error if chain did not progress
+   */
+  async teardown(): Promise<void> {
+    if (!this.snapshot) {
+      this.logger.warn('Teardown called without setup - skipping chain progress check');
+      return;
+    }
+
+    const processes: ChildProcess[] = [];
+    // Minimum test duration to check chain progression
+    const PROGRESS_CHECK_THRESHOLD_SECONDS = 120;
+
+    try {
+      const elapsedSeconds = Math.round((Date.now() - this.snapshot.timestamp) / 1000);
+
+      // Skip progress check for short tests
+      if (elapsedSeconds <= PROGRESS_CHECK_THRESHOLD_SECONDS) {
+        this.logger.info('Post-flight health check passed (skipped progress check - test too short)');
+        return;
+      }
+
+      const { process: rpcProcess, port: rpcPort } = await startPortForwardForRPC(this.namespace);
+      processes.push(rpcProcess);
+
+      const { process: ethProcess, port: ethPort } = await startPortForwardForEthereum(this.namespace);
+      processes.push(ethProcess);
+
+      const nodeUrl = `http://127.0.0.1:${rpcPort}`;
+      const ethereumUrl = `http://127.0.0.1:${ethPort}`;
+      const node = createAztecNodeClient(nodeUrl);
+
+      // Check that block number increased
+      let currentBlockNumber;
+      try {
+        currentBlockNumber = await node.getBlockNumber();
+      } catch (err) {
+        throw new Error(`Teardown health check failed: Could not get block number. Error: ${err}`);
+      }
+
+      if (currentBlockNumber <= this.snapshot.blockNumber) {
+        throw new Error(
+          `Chain did not progress during test. ` +
+            `Block number at setup: ${this.snapshot.blockNumber}, ` +
+            `Block number at teardown: ${currentBlockNumber}, ` +
+            `Elapsed time: ${elapsedSeconds}s. ` +
+            `The chain may have stalled during the test.`,
+        );
+      }
+
+      // Check that checkpoint number increased
+      const nodeInfo = await node.getNodeInfo();
+      const chain = createEthereumChain([ethereumUrl], nodeInfo.l1ChainId);
+      const ethereumClient: ViemPublicClient = createPublicClient({
+        chain: chain.chainInfo,
+        transport: fallback([http(ethereumUrl, { batch: false })]),
+      });
+
+      const rollup = new RollupContract(ethereumClient, nodeInfo.l1ContractAddresses.rollupAddress);
+      let currentCheckpoint;
+      try {
+        currentCheckpoint = await rollup.getCheckpointNumber();
+      } catch (err) {
+        throw new Error(`Teardown health check failed: Could not get checkpoint number. Error: ${err}`);
+      }
+
+      if (currentCheckpoint <= this.snapshot.checkpointNumber) {
+        throw new Error(
+          `Proving pipeline did not progress during test. ` +
+            `Checkpoint at setup: ${this.snapshot.checkpointNumber}, ` +
+            `Checkpoint at teardown: ${currentCheckpoint}, ` +
+            `Elapsed time: ${elapsedSeconds}s. ` +
+            `The proving pipeline may have stalled during the test.`,
+        );
+      }
+
+      this.logger.info('Post-flight health check passed');
+    } finally {
+      processes.forEach(p => p.kill());
+      this.snapshot = undefined;
+    }
+  }
+}