@aztec/end-to-end 0.0.1-commit.bf2612ae → 0.0.1-commit.c2595eba

package/src/fixtures/setup_p2p_test.ts

@@ -4,14 +4,13 @@
 import { type AztecNodeConfig, AztecNodeService } from '@aztec/aztec-node';
 import { range } from '@aztec/foundation/array';
 import { SecretValue } from '@aztec/foundation/config';
-import { addLogNameHandler, removeLogNameHandler } from '@aztec/foundation/log';
+import { withLoggerBindings } from '@aztec/foundation/log/server';
 import { bufferToHex } from '@aztec/foundation/string';
 import type { DateProvider } from '@aztec/foundation/timer';
 import type { ProverNodeConfig, ProverNodeDeps } from '@aztec/prover-node';
 import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees';
 
 import getPort from 'get-port';
-import { AsyncLocalStorage } from 'node:async_hooks';
 
 import { TEST_PEER_CHECK_INTERVAL_MS } from './fixtures.js';
 import { createAndSyncProverNode, getPrivateKeyFromIndex } from './utils.js';
@@ -22,6 +21,11 @@ import { getEndToEndTestTelemetryClient } from './with_telemetry_utils.js';
 // to avoid running validators with the same key
 export const ATTESTER_PRIVATE_KEYS_START_INDEX = 3;
 
+// Global counters for actor naming (start at 1)
+let validatorCounter = 1;
+let nodeCounter = 1;
+let proverCounter = 1;
+
 export function generatePrivateKeys(startIndex: number, numberOfKeys: number): `0x${string}`[] {
   const privateKeys: `0x${string}`[] = [];
   // Do not start from 0 as it is used during setup
@@ -44,10 +48,6 @@ export async function createNodes(
   validatorsPerNode = 1,
 ): Promise<AztecNodeService[]> {
   const nodePromises: Promise<AztecNodeService>[] = [];
-  const loggerIdStorage = new AsyncLocalStorage<string>();
-  const logNameHandler = (module: string) =>
-    loggerIdStorage.getStore() ? `${module}:${loggerIdStorage.getStore()}` : module;
-  addLogNameHandler(logNameHandler);
 
   for (let i = 0; i < numNodes; i++) {
     const index = indexOffset + i;
@@ -69,7 +69,6 @@ export async function createNodes(
       prefilledPublicData,
       dataDir,
       metricsPort,
-      loggerIdStorage,
     );
     nodePromises.push(nodePromise);
   }
@@ -81,7 +80,6 @@ export async function createNodes(
     throw new Error('Sequencer not found');
   }
 
-  removeLogNameHandler(logNameHandler);
   return nodes;
 }
 
@@ -95,9 +93,9 @@ export async function createNode(
   prefilledPublicData?: PublicDataTreeLeaf[],
   dataDirectory?: string,
   metricsPort?: number,
-  loggerIdStorage?: AsyncLocalStorage<string>,
 ) {
-  const createNode = async () => {
+  const actorIndex = validatorCounter++;
+  return await withLoggerBindings({ actor: `validator-${actorIndex}` }, async () => {
     const validatorConfig = await createValidatorConfig(config, bootstrapNode, tcpPort, addressIndex, dataDirectory);
     const telemetry = await getEndToEndTestTelemetryClient(metricsPort);
     return await AztecNodeService.createAndSync(
@@ -105,8 +103,7 @@ export async function createNode(
       { telemetry, dateProvider },
       { prefilledPublicData, dontStartSequencer: config.dontStartSequencer },
     );
-  };
-  return loggerIdStorage ? await loggerIdStorage.run(tcpPort.toString(), createNode) : createNode();
+  });
 }
 
 /** Creates a P2P enabled instance of Aztec Node Service without a validator */
@@ -118,9 +115,9 @@ export async function createNonValidatorNode(
   prefilledPublicData?: PublicDataTreeLeaf[],
   dataDirectory?: string,
   metricsPort?: number,
-  loggerIdStorage?: AsyncLocalStorage<string>,
 ) {
-  const createNode = async () => {
+  const actorIndex = nodeCounter++;
+  return await withLoggerBindings({ actor: `node-${actorIndex}` }, async () => {
     const p2pConfig = await createP2PConfig(baseConfig, bootstrapNode, tcpPort, dataDirectory);
     const config: AztecNodeConfig = {
       ...p2pConfig,
@@ -130,8 +127,7 @@ export async function createNonValidatorNode(
     };
     const telemetry = await getEndToEndTestTelemetryClient(metricsPort);
     return await AztecNodeService.createAndSync(config, { telemetry, dateProvider }, { prefilledPublicData });
-  };
-  return loggerIdStorage ? await loggerIdStorage.run(tcpPort.toString(), createNode) : createNode();
+  });
 }
 
 export async function createProverNode(
@@ -143,9 +139,9 @@ export async function createProverNode(
   prefilledPublicData?: PublicDataTreeLeaf[],
   dataDirectory?: string,
   metricsPort?: number,
-  loggerIdStorage?: AsyncLocalStorage<string>,
 ) {
-  const createProverNode = async () => {
+  const actorIndex = proverCounter++;
+  return await withLoggerBindings({ actor: `prover-${actorIndex}` }, async () => {
     const proverNodePrivateKey = getPrivateKeyFromIndex(ATTESTER_PRIVATE_KEYS_START_INDEX + addressIndex)!;
     const telemetry = await getEndToEndTestTelemetryClient(metricsPort);
 
@@ -165,8 +161,7 @@ export async function createProverNode(
       prefilledPublicData,
       { ...proverNodeDeps, telemetry },
     );
-  };
-  return loggerIdStorage ? await loggerIdStorage.run(tcpPort.toString(), createProverNode) : createProverNode();
+  });
 }
 
 export async function createP2PConfig(

package/src/spartan/tx_metrics.ts

@@ -6,6 +6,132 @@ import { Tx, type TxReceipt } from '@aztec/stdlib/tx';
 
 import { createHistogram } from 'perf_hooks';
 
+/** Metrics class for proving-related benchmarks. */
+export class ProvingMetrics {
+  private successfulTxs: number | undefined;
+  private proofDuration: number | undefined;
+  private activeAgents: number | undefined;
+  private avgQueueTime: number | undefined;
+  private jobRetries: number | undefined;
+  private jobDuration: number | undefined;
+  private timedOutJobs: number | undefined;
+  private resolvedJobs: number | undefined;
+  private rejectedJobs: number | undefined;
+  private epochProvingDuration: number | undefined;
+  private provenTransactions: number | undefined;
+  private provenBlocks: number | undefined;
+
+  constructor(private prefix: string) {}
+
+  recordSuccessfulTxs(count: number): void {
+    this.successfulTxs = count;
+  }
+
+  recordProofDuration(seconds: number): void {
+    this.proofDuration = seconds;
+  }
+
+  recordActiveAgents(count: number): void {
+    this.activeAgents = count;
+  }
+
+  recordAvgQueueTime(ms: number): void {
+    this.avgQueueTime = ms;
+  }
+
+  recordJobRetries(count: number): void {
+    this.jobRetries = count;
+  }
+
+  recordJobDuration(ms: number): void {
+    this.jobDuration = ms;
+  }
+
+  recordTimedOutJobs(count: number): void {
+    this.timedOutJobs = count;
+  }
+
+  recordResolvedJobs(count: number): void {
+    this.resolvedJobs = count;
+  }
+
+  recordRejectedJobs(count: number): void {
+    this.rejectedJobs = count;
+  }
+
+  recordEpochProvingDuration(seconds: number): void {
+    this.epochProvingDuration = seconds;
+  }
+
+  recordProvenTransactions(count: number): void {
+    this.provenTransactions = count;
+  }
+
+  recordProvenBlocks(count: number): void {
+    this.provenBlocks = count;
+  }
+
+  toGithubActionBenchmarkJSON(): Array<{ name: string; unit: string; value: number }> {
+    const data: Array<{ name: string; unit: string; value: number }> = [];
+
+    if (this.successfulTxs !== undefined) {
+      data.push({ name: `${this.prefix}/successful_txs`, unit: 'count', value: this.successfulTxs });
+    }
+
+    if (this.proofDuration !== undefined) {
+      data.push({ name: `${this.prefix}/proof_duration`, unit: 's', value: this.proofDuration });
+    }
+
+    if (this.activeAgents !== undefined) {
+      data.push({ name: `${this.prefix}/active_agents`, unit: 'count', value: this.activeAgents });
+    }
+
+    if (this.avgQueueTime !== undefined) {
+      data.push({ name: `${this.prefix}/avg_queue_time`, unit: 'ms', value: this.avgQueueTime });
+    }
+
+    if (this.jobRetries !== undefined) {
+      data.push({ name: `${this.prefix}/job_retries`, unit: 'count', value: this.jobRetries });
+    }
+
+    if (this.jobDuration !== undefined) {
+      data.push({ name: `${this.prefix}/job_duration`, unit: 'ms', value: this.jobDuration });
+    }
+
+    if (this.timedOutJobs !== undefined) {
+      data.push({ name: `${this.prefix}/timed_out_jobs`, unit: 'count', value: this.timedOutJobs });
+    }
+
+    if (this.resolvedJobs !== undefined) {
+      data.push({ name: `${this.prefix}/resolved_jobs`, unit: 'count', value: this.resolvedJobs });
+    }
+
+    if (this.rejectedJobs !== undefined) {
+      data.push({ name: `${this.prefix}/rejected_jobs`, unit: 'count', value: this.rejectedJobs });
+    }
+
+    if (this.epochProvingDuration !== undefined) {
+      data.push({ name: `${this.prefix}/epoch_proving_duration`, unit: 's', value: this.epochProvingDuration });
+    }
+
+    if (this.provenTransactions !== undefined) {
+      data.push({ name: `${this.prefix}/proven_transactions`, unit: 'count', value: this.provenTransactions });
+    }
+
+    if (this.provenBlocks !== undefined) {
+      data.push({ name: `${this.prefix}/proven_blocks`, unit: 'count', value: this.provenBlocks });
+    }
+
+    const scenario = process.env.BENCH_SCENARIO?.trim();
+    if (!scenario) {
+      return data;
+    }
+
+    const scenarioPrefix = `scenario/${scenario}/`;
+    return data.map(entry => ({ ...entry, name: `${scenarioPrefix}${entry.name}` }));
+  }
+}
+
 export type TxInclusionData = {
   txHash: string;
   sentAt: number;

package/src/spartan/utils/index.ts

@@ -24,6 +24,7 @@ export {
   getServiceEndpoint,
   getRPCEndpoint,
   getEthereumEndpoint,
+  createResilientPrometheusConnection,
 } from './k8s.js';
 
 // Chaos Mesh
@@ -45,6 +46,7 @@ export { restartBot, installTransferBot, uninstallTransferBot } from './bot.js';
 // Node operations (sequencers, validators, pods)
 export {
   awaitCheckpointNumber,
+  waitForProvenToAdvance,
   getSequencers,
   updateSequencersConfig,
   getSequencersConfig,

package/src/spartan/utils/k8s.ts

@@ -1,4 +1,5 @@
 import { createLogger } from '@aztec/aztec.js/log';
+import type { Logger } from '@aztec/foundation/log';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { retryUntil } from '@aztec/foundation/retry';
 
@@ -6,6 +7,8 @@ import { type ChildProcess, exec, spawn } from 'child_process';
 import path from 'path';
 import { promisify } from 'util';
 
+import { AlertTriggeredError, GrafanaClient } from '../../quality_of_service/grafana_client.js';
+
 const execAsync = promisify(exec);
 
 const logger = createLogger('e2e:k8s-utils');
@@ -370,6 +373,155 @@ export async function waitForResourcesByName({
   );
 }
 
+/**
+ * Waits for all StatefulSets matching a label to have all their replicas ready.
+ *
+ * @param namespace - Kubernetes namespace
+ * @param label - Label selector for StatefulSets (e.g., "app.kubernetes.io/component=sequencer-node")
+ * @param timeoutSeconds - Maximum time to wait in seconds
+ * @param pollIntervalSeconds - How often to check status
+ */
+export async function waitForStatefulSetsReady({
+  namespace,
+  label,
+  timeoutSeconds = 600,
+  pollIntervalSeconds = 5,
+}: {
+  namespace: string;
+  label: string;
+  timeoutSeconds?: number;
+  pollIntervalSeconds?: number;
+}): Promise<void> {
+  logger.info(`Waiting for StatefulSets with label ${label} to have all replicas ready (timeout: ${timeoutSeconds}s)`);
+
+  await retryUntil(
+    async () => {
+      // Get all StatefulSets matching the label
+      const getCmd = `kubectl get statefulset -l ${label} -n ${namespace} -o json`;
+      const { stdout } = await execAsync(getCmd);
+      const result = JSON.parse(stdout);
+
+      if (!result.items || result.items.length === 0) {
+        logger.verbose(`No StatefulSets found with label ${label}`);
+        return false;
+      }
+
+      // Check each StatefulSet
+      for (const sts of result.items) {
+        const name = sts.metadata.name;
+        const desired = sts.spec.replicas ?? 0;
+        const ready = sts.status.readyReplicas ?? 0;
+        const updated = sts.status.updatedReplicas ?? 0;
+
+        if (ready < desired || updated < desired) {
+          logger.verbose(`StatefulSet ${name}: ${ready}/${desired} ready, ${updated}/${desired} updated`);
+          return false;
+        }
+      }
+
+      logger.info(`All StatefulSets with label ${label} are ready`);
+      return true;
+    },
+    `StatefulSets with label ${label} to be ready`,
+    timeoutSeconds,
+    pollIntervalSeconds,
+  );
+}
+
+/**
+ * Creates a Prometheus connection that can re-establish port-forward on failure.
+ * Returns functions to connect and run alert checks that automatically reconnect if needed.
+ *
+ * @param namespace - K8s namespace to fall back to if metrics namespace doesn't have Prometheus
+ * @param endpoints - Array to track created endpoints for cleanup
+ * @param log - Logger instance
+ */
+export function createResilientPrometheusConnection(
+  namespace: string,
+  endpoints: ServiceEndpoint[],
+  log: Logger,
+): {
+  connect: () => Promise<GrafanaClient>;
+  runAlertCheck: (alerts: Parameters<GrafanaClient['runAlertCheck']>[0]) => Promise<void>;
+} {
+  let alertChecker: GrafanaClient | undefined;
+  let currentEndpoint: ServiceEndpoint | undefined;
+
+  const connect = async (): Promise<GrafanaClient> => {
+    // Kill existing connection if any
+    if (currentEndpoint?.process) {
+      currentEndpoint.process.kill();
+    }
+
+    // Try metrics namespace first, then network namespace
+    let promPort = 0;
+    let promUrl = '';
+    let promProc: ChildProcess | undefined;
+
+    try {
+      const metricsResult = await startPortForward({
+        resource: `svc/metrics-prometheus-server`,
+        namespace: 'metrics',
+        containerPort: 80,
+      });
+      promProc = metricsResult.process;
+      promPort = metricsResult.port;
+      promUrl = `http://127.0.0.1:${promPort}/api/v1`;
+    } catch {
+      // Metrics namespace might not have Prometheus, try network namespace
+      log.verbose('Metrics namespace Prometheus not available, trying network namespace');
+    }
+
+    if (promPort === 0) {
+      const nsResult = await startPortForward({
+        resource: `svc/prometheus-server`,
+        namespace,
+        containerPort: 80,
+      });
+      promProc = nsResult.process;
+      promPort = nsResult.port;
+      promUrl = `http://127.0.0.1:${promPort}/api/v1`;
+    }
+
+    if (!promProc || promPort === 0) {
+      throw new Error('Unable to port-forward to Prometheus');
+    }
+
+    currentEndpoint = { url: promUrl, process: promProc };
+    endpoints.push(currentEndpoint);
+    alertChecker = new GrafanaClient(log, { grafanaEndpoint: promUrl, grafanaCredentials: '' });
+    log.info(`Established Prometheus connection at ${promUrl}`);
+    return alertChecker;
+  };
+
+  const runAlertCheck = async (alerts: Parameters<GrafanaClient['runAlertCheck']>[0]): Promise<void> => {
+    if (!alertChecker) {
+      alertChecker = await connect();
+    }
+
+    try {
+      await alertChecker.runAlertCheck(alerts);
+    } catch (err) {
+      // If it's an AlertTriggeredError (expected behavior)
+      if (err instanceof AlertTriggeredError) {
+        throw err;
+      }
+
+      // Check if it's a connection error (port-forward died)
+      const errorStr = String(err);
+      if (errorStr.includes('fetch failed') || errorStr.includes('ECONNREFUSED') || errorStr.includes('ECONNRESET')) {
+        log.warn(`Prometheus connection lost, re-establishing port-forward...`);
+        alertChecker = await connect();
+        await alertChecker.runAlertCheck(alerts);
+      } else {
+        throw err;
+      }
+    }
+  };
+
+  return { connect, runAlertCheck };
+}
+
 export function getChartDir(spartanDir: string, chartName: string) {
   return path.join(spartanDir.trim(), chartName);
 }