@aztec/end-to-end 0.0.1-commit.d431d1c → 0.0.1-commit.e3c1de76

package/src/spartan/utils/health.ts

@@ -0,0 +1,255 @@
+import { createAztecNodeClient } from '@aztec/aztec.js/node';
+import { createEthereumChain } from '@aztec/ethereum/chain';
+import { RollupContract } from '@aztec/ethereum/contracts';
+import type { ViemPublicClient } from '@aztec/ethereum/types';
+import { CheckpointNumber } from '@aztec/foundation/branded-types';
+import type { Logger } from '@aztec/foundation/log';
+
+import { createPublicClient, fallback, http } from 'viem';
+
+import { type ServiceEndpoint, getEthereumEndpoint, getRPCEndpoint } from './k8s.js';
+
+/**
+ * Snapshot of chain state captured during setup for comparison in teardown.
+ */
+export interface ChainHealthSnapshot {
+  blockNumber: number;
+  checkpointNumber: CheckpointNumber;
+  timestamp: number;
+}
+
+/**
+ * Pre-flight and post-flight health checks for the Aztec network.
+ *
+ * Use in beforeAll/afterAll to validate the chain is healthy before tests run
+ * and verify it continued progressing during the test.
+ *
+ * @example
+ * ```typescript
+ * const health = new ChainHealth(config.NAMESPACE, logger);
+ *
+ * beforeAll(async () => {
+ *   await health.setup();
+ * });
+ *
+ * afterAll(async () => {
+ *   await health.teardown();
+ * });
+ * ```
+ */
+export class ChainHealth {
+  private namespace: string;
+  private logger: Logger;
+  private snapshot?: ChainHealthSnapshot;
+
+  constructor(namespace: string, logger: Logger) {
+    this.namespace = namespace;
+    this.logger = logger;
+  }
+
+  /**
+   * Pre-flight health check. Validates chain is in a testable state and captures
+   * initial state for comparison in teardown.
+   *
+   * Checks performed:
+   * - Node is reachable and returns valid info
+   * - ENR exists
+   * - L1 is accessible
+   * - At least 1 L2 block has been mined
+   * - Committee exists
+   * - At least 1 checkpoint has been reached
+   *
+   * @throws Error if any health check fails
+   */
+  async setup(): Promise<void> {
+    const endpoints: ServiceEndpoint[] = [];
+
+    try {
+      // Establish connections (LB or port-forward)
+      const rpcEndpoint = await getRPCEndpoint(this.namespace);
+      endpoints.push(rpcEndpoint);
+
+      const ethEndpoint = await getEthereumEndpoint(this.namespace);
+      endpoints.push(ethEndpoint);
+
+      const nodeUrl = rpcEndpoint.url;
+      const ethereumUrl = ethEndpoint.url;
+
+      // Create clients
+      const node = createAztecNodeClient(nodeUrl);
+
+      // Check 1: Node is reachable
+      let nodeInfo;
+      try {
+        nodeInfo = await node.getNodeInfo();
+      } catch (err) {
+        throw new Error(`Health check failed: Node is not reachable at ${nodeUrl}. Error: ${err}`);
+      }
+
+      if (!nodeInfo) {
+        throw new Error('Health check failed: Node returned empty info');
+      }
+
+      // Check 2: ENR exists (P2P identity)
+      if (!nodeInfo.enr || !nodeInfo.enr.startsWith('enr:-')) {
+        throw new Error(`Health check failed: Invalid or missing ENR. Got: ${nodeInfo.enr}`);
+      }
+
+      // Check 3: L1 is accessible
+      const chain = createEthereumChain([ethereumUrl], nodeInfo.l1ChainId);
+      const ethereumClient: ViemPublicClient = createPublicClient({
+        chain: chain.chainInfo,
+        transport: fallback([http(ethereumUrl, { batch: false })]),
+      });
+
+      try {
+        await ethereumClient.getBlockNumber();
+      } catch (err) {
+        throw new Error(`Health check failed: L1 is not accessible at ${ethereumUrl}. Error: ${err}`);
+      }
+
+      // Check 4: At least 1 L2 block mined
+      let l2BlockNumber;
+      try {
+        l2BlockNumber = await node.getBlockNumber();
+      } catch (err) {
+        throw new Error(`Health check failed: Could not get L2 block number. Error: ${err}`);
+      }
+
+      if (l2BlockNumber < 1) {
+        throw new Error(`Health check failed: No L2 blocks mined yet. Block number: ${l2BlockNumber}`);
+      }
+
+      // Check 5: Committee exists
+      const rollup = new RollupContract(ethereumClient, nodeInfo.l1ContractAddresses.rollupAddress);
+
+      let committee;
+      try {
+        committee = await rollup.getCurrentEpochCommittee();
+      } catch (err) {
+        throw new Error(`Health check failed: Could not get committee. Error: ${err}`);
+      }
+
+      if (!committee || committee.length === 0) {
+        throw new Error('Health check failed: No committee exists. Validators may not be registered yet.');
+      }
+
+      // Check 6: At least 1 checkpoint reached
+      let checkpointNumber;
+      try {
+        checkpointNumber = await rollup.getCheckpointNumber();
+      } catch (err) {
+        throw new Error(`Health check failed: Could not get checkpoint number. Error: ${err}`);
+      }
+
+      if (checkpointNumber < CheckpointNumber(1)) {
+        throw new Error(
+          `Health check failed: No checkpoint reached yet. Checkpoint number: ${checkpointNumber}. ` +
+            'The proving pipeline may not have completed a proof yet.',
+        );
+      }
+
+      // Capture snapshot for teardown comparison
+      this.snapshot = {
+        blockNumber: l2BlockNumber,
+        checkpointNumber,
+        timestamp: Date.now(),
+      };
+
+      this.logger.info('Pre-flight health check passed');
+    } finally {
+      endpoints.forEach(e => e.process?.kill());
+    }
+  }
+
+  /**
+   * Post-flight health check. Verifies the chain continued progressing during the test.
+   *
+   * For tests that ran longer than the threshold, checks:
+   * - Block number increased since setup
+   * - Checkpoint number increased since setup
+   *
+   * For shorter tests, skips the check.
+   *
+   * @throws Error if chain did not progress
+   */
+  async teardown(): Promise<void> {
+    if (!this.snapshot) {
+      this.logger.warn('Teardown called without setup - skipping chain progress check');
+      return;
+    }
+
+    const endpoints: ServiceEndpoint[] = [];
+    // Minimum test duration to check chain progression
+    const PROGRESS_CHECK_THRESHOLD_SECONDS = 120;
+
+    try {
+      const elapsedSeconds = Math.round((Date.now() - this.snapshot.timestamp) / 1000);
+
+      // Skip progress check for short tests
+      if (elapsedSeconds <= PROGRESS_CHECK_THRESHOLD_SECONDS) {
+        this.logger.info('Post-flight health check passed (skipped progress check - test too short)');
+        return;
+      }
+
+      const rpcEndpoint = await getRPCEndpoint(this.namespace);
+      endpoints.push(rpcEndpoint);
+
+      const ethEndpoint = await getEthereumEndpoint(this.namespace);
+      endpoints.push(ethEndpoint);
+
+      const nodeUrl = rpcEndpoint.url;
+      const ethereumUrl = ethEndpoint.url;
+      const node = createAztecNodeClient(nodeUrl);
+
+      // Check that block number increased
+      let currentBlockNumber;
+      try {
+        currentBlockNumber = await node.getBlockNumber();
+      } catch (err) {
+        throw new Error(`Teardown health check failed: Could not get block number. Error: ${err}`);
+      }
+
+      if (currentBlockNumber <= this.snapshot.blockNumber) {
+        throw new Error(
+          `Chain did not progress during test. ` +
+            `Block number at setup: ${this.snapshot.blockNumber}, ` +
+            `Block number at teardown: ${currentBlockNumber}, ` +
+            `Elapsed time: ${elapsedSeconds}s. ` +
+            `The chain may have stalled during the test.`,
+        );
+      }
+
+      // Check that checkpoint number increased
+      const nodeInfo = await node.getNodeInfo();
+      const chain = createEthereumChain([ethereumUrl], nodeInfo.l1ChainId);
+      const ethereumClient: ViemPublicClient = createPublicClient({
+        chain: chain.chainInfo,
+        transport: fallback([http(ethereumUrl, { batch: false })]),
+      });
+
+      const rollup = new RollupContract(ethereumClient, nodeInfo.l1ContractAddresses.rollupAddress);
+      let currentCheckpoint;
+      try {
+        currentCheckpoint = await rollup.getCheckpointNumber();
+      } catch (err) {
+        throw new Error(`Teardown health check failed: Could not get checkpoint number. Error: ${err}`);
+      }
+
+      if (currentCheckpoint <= this.snapshot.checkpointNumber) {
+        throw new Error(
+          `Proving pipeline did not progress during test. ` +
+            `Checkpoint at setup: ${this.snapshot.checkpointNumber}, ` +
+            `Checkpoint at teardown: ${currentCheckpoint}, ` +
+            `Elapsed time: ${elapsedSeconds}s. ` +
+            `The proving pipeline may have stalled during the test.`,
+        );
+      }
+
+      this.logger.info('Post-flight health check passed');
+    } finally {
+      endpoints.forEach(e => e.process?.kill());
+      this.snapshot = undefined;
+    }
+  }
+}

package/src/spartan/utils/helm.ts

@@ -0,0 +1,84 @@
+import { createLogger } from '@aztec/aztec.js/log';
+import type { Logger } from '@aztec/foundation/log';
+
+import { exec } from 'child_process';
+import { promisify } from 'util';
+
+const execAsync = promisify(exec);
+
+const logger = createLogger('e2e:k8s-utils');
+
+function shellQuote(value: string) {
+  // Single-quote safe shell escaping: ' -> '\''
+  return `'${value.replace(/'/g, "'\\''")}'`;
+}
+
+function valuesToArgs(values: Record<string, string | number | boolean>) {
+  return Object.entries(values)
+    .map(([key, value]) =>
+      typeof value === 'number' || typeof value === 'boolean'
+        ? `--set ${key}=${value}`
+        : `--set-string ${key}=${shellQuote(String(value))}`,
+    )
+    .join(' ');
+}
+
+export function createHelmCommand({
+  instanceName,
+  helmChartDir,
+  namespace,
+  valuesFile,
+  timeout,
+  values,
+  reuseValues = false,
+}: {
+  instanceName: string;
+  helmChartDir: string;
+  namespace: string;
+  valuesFile: string | undefined;
+  timeout: string;
+  values: Record<string, string | number | boolean>;
+  reuseValues?: boolean;
+}) {
+  const valuesFileArgs = valuesFile ? `--values ${helmChartDir}/values/${valuesFile}` : '';
+  const reuseValuesArgs = reuseValues ? '--reuse-values' : '';
+  return `helm upgrade --install ${instanceName} ${helmChartDir} --namespace ${namespace} ${valuesFileArgs} ${reuseValuesArgs} --wait --timeout=${timeout} ${valuesToArgs(
+    values,
+  )}`;
+}
+
+export async function execHelmCommand(args: Parameters<typeof createHelmCommand>[0]) {
+  const helmCommand = createHelmCommand(args);
+  logger.info(`helm command: ${helmCommand}`);
+  const { stdout } = await execAsync(helmCommand);
+  return stdout;
+}
+
+export async function getHelmReleaseStatus(instanceName: string, namespace: string): Promise<string | undefined> {
+  try {
+    const { stdout } = await execAsync(
+      `helm list --namespace ${namespace} --all --filter '^${instanceName}$' --output json | cat`,
+    );
+    const parsed = JSON.parse(stdout) as Array<{ name?: string; status?: string }>;
+    const row = parsed.find(r => r.name === instanceName);
+    return row?.status;
+  } catch {
+    return undefined;
+  }
+}
+
+export async function forceDeleteHelmReleaseRecord(instanceName: string, namespace: string, log?: Logger) {
+  const labelSelector = `owner=helm,name=${instanceName}`;
+  const cmd = `kubectl delete secret -n ${namespace} -l ${labelSelector} --ignore-not-found=true`;
+  (log ?? logger).warn(`Force deleting Helm release record: ${cmd}`);
+  await execAsync(cmd).catch(() => undefined);
+}
+
+export async function hasDeployedHelmRelease(instanceName: string, namespace: string): Promise<boolean> {
+  try {
+    const status = await getHelmReleaseStatus(instanceName, namespace);
+    return status?.toLowerCase() === 'deployed';
+  } catch {
+    return false;
+  }
+}

package/src/spartan/utils/index.ts

@@ -0,0 +1,64 @@
+// Re-export all utilities from individual modules
+// Note: Internal helpers (helm commands, etc.) are not re-exported to maintain the original public API
+
+// Config
+export { type TestConfig, setupEnvironment } from './config.js';
+
+// Scripts
+export { getGitProjectRoot, getAztecBin, runAztecBin, runProjectScript } from './scripts.js';
+
+// K8s operations
+export {
+  startPortForward,
+  getExternalIP,
+  startPortForwardForPrometeheus,
+  startPortForwardForRPC,
+  startPortForwardForEthereum,
+  deleteResourceByName,
+  deleteResourceByLabel,
+  waitForResourceByLabel,
+  waitForResourceByName,
+  waitForResourcesByName,
+  getChartDir,
+  type ServiceEndpoint,
+  getServiceEndpoint,
+  getRPCEndpoint,
+  getEthereumEndpoint,
+  createResilientPrometheusConnection,
+} from './k8s.js';
+
+// Chaos Mesh
+export {
+  uninstallChaosMesh,
+  installChaosMeshChart,
+  applyProverFailure,
+  applyValidatorFailure,
+  applyProverKill,
+  applyProverBrokerKill,
+  applyBootNodeFailure,
+  applyValidatorKill,
+  applyNetworkShaping,
+} from './chaos.js';
+
+// Bot management
+export { restartBot, installTransferBot, uninstallTransferBot } from './bot.js';
+
+// Node operations (sequencers, validators, pods)
+export {
+  awaitCheckpointNumber,
+  waitForProvenToAdvance,
+  getSequencers,
+  updateSequencersConfig,
+  getSequencersConfig,
+  withSequencersAdmin,
+  setValidatorTxDrop,
+  restartValidators,
+  enableValidatorDynamicBootNode,
+  rollAztecPods,
+} from './nodes.js';
+
+// Client utilities
+export { getPublicViemClient, getL1DeploymentAddresses, getNodeClient } from './clients.js';
+
+// Health checks
+export { ChainHealth, type ChainHealthSnapshot } from './health.js';