@aztec/end-to-end 0.0.1-commit.29c6b1a3 → 0.0.1-commit.2e2504e2

package/src/fixtures/e2e_prover_test.ts

@@ -198,7 +198,7 @@ export class FullProverTest {
       this.aztecNode,
       { proverEnabled: this.realProofs },
       undefined,
-      true,
+      'pxe-proven',
     );
     this.logger.debug(`Contract address ${this.fakeProofsAsset.address}`);
     await provenWallet.registerContract(this.fakeProofsAssetInstance, TokenContract.artifact);

package/src/fixtures/ha_setup.ts

@@ -0,0 +1,184 @@
+import { EthAddress } from '@aztec/aztec.js/addresses';
+import { Fr } from '@aztec/aztec.js/fields';
+import type { Logger } from '@aztec/aztec.js/log';
+import { SecretValue } from '@aztec/foundation/config';
+
+import { Pool } from 'pg';
+import { privateKeyToAccount } from 'viem/accounts';
+
+/**
+ * Configuration for HA database connection
+ */
+export interface HADatabaseConfig {
+  /** PostgreSQL connection URL */
+  databaseUrl: string;
+  /** Node ID for HA coordination */
+  nodeId: string;
+  /** Enable HA signing */
+  haSigningEnabled: boolean;
+  /** Polling interval in ms */
+  pollingIntervalMs: number;
+  /** Signing timeout in ms */
+  signingTimeoutMs: number;
+  /** Max stuck duties age in ms */
+  maxStuckDutiesAgeMs: number;
+}
+
+/**
+ * Get database configuration from environment variables
+ */
+export function createHADatabaseConfig(nodeId: string): HADatabaseConfig {
+  const databaseUrl = process.env.DATABASE_URL || 'postgresql://aztec:aztec@localhost:5432/aztec_ha_test';
+
+  return {
+    databaseUrl,
+    nodeId,
+    haSigningEnabled: true,
+    pollingIntervalMs: 100,
+    signingTimeoutMs: 3000,
+    maxStuckDutiesAgeMs: 72000,
+  };
+}
+
+/**
+ * Setup PostgreSQL database connection pool for HA tests
+ *
+ * Note: Database migrations should be run separately before starting tests,
+ * either via docker-compose entrypoint or manually with: aztec migrate-ha-db up
+ */
+export function setupHADatabase(databaseUrl: string, logger?: Logger): Pool {
+  try {
+    // Create connection pool for test usage
+    // Migrations are already run by docker-compose entrypoint before tests start
+    const pool = new Pool({ connectionString: databaseUrl });
+
+    logger?.info('Connected to HA database (migrations should already be applied)');
+
+    return pool;
+  } catch (error) {
+    logger?.error(`Failed to connect to HA database: ${error}`);
+    throw error;
+  }
+}
+
+/**
+ * Clean up HA database - drop all tables
+ * Use this between tests to ensure clean state
+ */
+export async function cleanupHADatabase(pool: Pool, logger?: Logger): Promise<void> {
+  try {
+    // Drop all HA tables
+    await pool.query('DROP TABLE IF EXISTS validator_duties CASCADE');
+    await pool.query('DROP TABLE IF EXISTS slashing_protection CASCADE');
+    await pool.query('DROP TABLE IF EXISTS schema_version CASCADE');
+
+    logger?.info('HA database cleaned up successfully');
+  } catch (error) {
+    logger?.error(`Failed to cleanup HA database: ${error}`);
+    throw error;
+  }
+}
+
+/**
+ * Query validator duties from the database
+ */
+export async function getValidatorDuties(
+  pool: Pool,
+  slot: bigint,
+  dutyType?: 'ATTESTATION' | 'BLOCK_PROPOSAL' | 'GOVERNANCE_VOTE' | 'SLASHING_VOTE',
+): Promise<
+  Array<{
+    slot: string;
+    dutyType: string;
+    validatorAddress: string;
+    nodeId: string;
+    startedAt: Date;
+    completedAt: Date | undefined;
+  }>
+> {
+  const query = dutyType
+    ? 'SELECT slot, duty_type, validator_address, node_id, started_at, completed_at FROM validator_duties WHERE slot = $1 AND duty_type = $2 ORDER BY started_at'
+    : 'SELECT slot, duty_type, validator_address, node_id, started_at, completed_at FROM validator_duties WHERE slot = $1 ORDER BY started_at';
+
+  const params = dutyType ? [slot.toString(), dutyType] : [slot.toString()];
+
+  const result = await pool.query<{
+    slot: string;
+    duty_type: string;
+    validator_address: string;
+    node_id: string;
+    started_at: Date;
+    completed_at: Date | undefined;
+  }>(query, params);
+
+  return result.rows.map(row => ({
+    slot: row.slot,
+    dutyType: row.duty_type,
+    validatorAddress: row.validator_address,
+    nodeId: row.node_id,
+    startedAt: row.started_at,
+    completedAt: row.completed_at,
+  }));
+}
+
+/**
+ * Convert private keys to Ethereum addresses
+ */
+export function getAddressesFromPrivateKeys(privateKeys: `0x${string}`[]): string[] {
+  return privateKeys.map(pk => {
+    const account = privateKeyToAccount(pk);
+    return account.address;
+  });
+}
+
+/**
+ * Create initial validators from private keys for L1 contract deployment
+ */
+export function createInitialValidatorsFromPrivateKeys(attesterPrivateKeys: `0x${string}`[]): Array<{
+  attester: EthAddress;
+  withdrawer: EthAddress;
+  privateKey: `0x${string}`;
+  bn254SecretKey: SecretValue<bigint>;
+}> {
+  return attesterPrivateKeys.map(pk => {
+    const account = privateKeyToAccount(pk);
+    return {
+      attester: EthAddress.fromString(account.address),
+      withdrawer: EthAddress.fromString(account.address),
+      privateKey: pk,
+      bn254SecretKey: new SecretValue(Fr.random().toBigInt()),
+    };
+  });
+}
+
+/**
+ * Verify no duplicate attestations per validator (HA coordination check)
+ * Groups duties by validator address and verifies each validator attested exactly once
+ */
+export function verifyNoDuplicateAttestations(
+  attestationDuties: Array<{
+    validatorAddress: string;
+    nodeId: string;
+    completedAt: Date | undefined;
+  }>,
+  logger?: Logger,
+): Map<string, typeof attestationDuties> {
+  const dutiesByValidator = new Map<string, typeof attestationDuties>();
+  for (const duty of attestationDuties) {
+    const existing = dutiesByValidator.get(duty.validatorAddress) || [];
+    existing.push(duty);
+    dutiesByValidator.set(duty.validatorAddress, existing);
+  }
+
+  for (const [validatorAddress, validatorDuties] of dutiesByValidator.entries()) {
+    if (validatorDuties.length !== 1) {
+      throw new Error(`Validator ${validatorAddress} attested ${validatorDuties.length} times (expected exactly once)`);
+    }
+    if (!validatorDuties[0].completedAt) {
+      throw new Error(`Validator ${validatorAddress} attestation duty not completed`);
+    }
+    logger?.info(`Validator ${validatorAddress} attested once via node ${validatorDuties[0].nodeId}`);
+  }
+
+  return dutiesByValidator;
+}

package/src/fixtures/index.ts

@@ -1,4 +1,5 @@
 export * from './fixtures.js';
+export * from './ha_setup.js';
 export * from './logging.js';
 export * from './utils.js';
 export * from './token_utils.js';

package/src/fixtures/setup.ts

@@ -41,7 +41,7 @@ import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
 import { SecretValue } from '@aztec/foundation/config';
 import { randomBytes } from '@aztec/foundation/crypto/random';
 import { tryRmDir } from '@aztec/foundation/fs';
-import { withLogNameSuffix } from '@aztec/foundation/log';
+import { withLoggerBindings } from '@aztec/foundation/log/server';
 import { retryUntil } from '@aztec/foundation/retry';
 import { sleep } from '@aztec/foundation/sleep';
 import { DateProvider, TestDateProvider } from '@aztec/foundation/timer';
@@ -125,14 +125,14 @@ export async function setupSharedBlobStorage(config: { dataDirectory?: string }
  * @param aztecNode - An instance of Aztec Node.
  * @param opts - Partial configuration for the PXE.
  * @param logger - The logger to be used.
- * @param useLogSuffix - Whether to add a randomly generated suffix to the PXE debug logs.
+ * @param actor - Actor label to include in log output (e.g., 'pxe-test').
  * @returns A test wallet, logger and teardown function.
  */
 export async function setupPXEAndGetWallet(
   aztecNode: AztecNode,
   opts: Partial<PXEConfig> = {},
   logger = getLogger(),
-  useLogSuffix = false,
+  actor?: string,
 ): Promise<{
   wallet: TestWallet;
   logger: Logger;
@@ -150,9 +150,7 @@ export async function setupPXEAndGetWallet(
 
   const teardown = configuredDataDirectory ? () => Promise.resolve() : () => tryRmDir(PXEConfig.dataDirectory!);
 
-  const wallet = await TestWallet.create(aztecNode, PXEConfig, {
-    useLogSuffix,
-  });
+  const wallet = await TestWallet.create(aztecNode, PXEConfig, { loggerActorLabel: actor });
 
   return {
     wallet,
@@ -574,10 +572,12 @@ export async function setup(
       }
     }
 
-    const aztecNodeService = await AztecNodeService.createAndSync(
-      config,
-      { dateProvider, telemetry: telemetryClient, p2pClientDeps, logger: createLogger('node:MAIN-aztec-node') },
-      { prefilledPublicData },
+    const aztecNodeService = await withLoggerBindings({ actor: 'node-0' }, () =>
+      AztecNodeService.createAndSync(
+        config,
+        { dateProvider, telemetry: telemetryClient, p2pClientDeps },
+        { prefilledPublicData },
+      ),
     );
     const sequencerClient = aztecNodeService.getSequencer();
 
@@ -611,7 +611,7 @@ export async function setup(
     pxeConfig.dataDirectory = path.join(directoryToCleanup, randomBytes(8).toString('hex'));
     // For tests we only want proving enabled if specifically requested
     pxeConfig.proverEnabled = !!pxeOpts.proverEnabled;
-    const wallet = await TestWallet.create(aztecNodeService, pxeConfig);
+    const wallet = await TestWallet.create(aztecNodeService, pxeConfig, { loggerActorLabel: 'pxe-0' });
 
     if (opts.walletMinFeePadding !== undefined) {
       wallet.setMinFeePadding(opts.walletMinFeePadding);
@@ -797,7 +797,7 @@ export function createAndSyncProverNode(
   prefilledPublicData: PublicDataTreeLeaf[] = [],
   proverNodeDeps: ProverNodeDeps = {},
 ) {
-  return withLogNameSuffix('prover-node', async () => {
+  return withLoggerBindings({ actor: 'prover-0' }, async () => {
     const aztecNodeTxProvider = aztecNode && {
       getTxByHash: aztecNode.getTxByHash.bind(aztecNode),
       getTxsByHash: aztecNode.getTxsByHash.bind(aztecNode),

package/src/fixtures/setup_p2p_test.ts

@@ -4,14 +4,13 @@
 import { type AztecNodeConfig, AztecNodeService } from '@aztec/aztec-node';
 import { range } from '@aztec/foundation/array';
 import { SecretValue } from '@aztec/foundation/config';
-import { addLogNameHandler, removeLogNameHandler } from '@aztec/foundation/log';
+import { withLoggerBindings } from '@aztec/foundation/log/server';
 import { bufferToHex } from '@aztec/foundation/string';
 import type { DateProvider } from '@aztec/foundation/timer';
 import type { ProverNodeConfig, ProverNodeDeps } from '@aztec/prover-node';
 import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees';
 
 import getPort from 'get-port';
-import { AsyncLocalStorage } from 'node:async_hooks';
 
 import { TEST_PEER_CHECK_INTERVAL_MS } from './fixtures.js';
 import { createAndSyncProverNode, getPrivateKeyFromIndex } from './utils.js';
@@ -22,6 +21,11 @@ import { getEndToEndTestTelemetryClient } from './with_telemetry_utils.js';
 // to avoid running validators with the same key
 export const ATTESTER_PRIVATE_KEYS_START_INDEX = 3;
 
+// Global counters for actor naming (start at 1)
+let validatorCounter = 1;
+let nodeCounter = 1;
+let proverCounter = 1;
+
 export function generatePrivateKeys(startIndex: number, numberOfKeys: number): `0x${string}`[] {
   const privateKeys: `0x${string}`[] = [];
   // Do not start from 0 as it is used during setup
@@ -44,10 +48,6 @@ export async function createNodes(
   validatorsPerNode = 1,
 ): Promise<AztecNodeService[]> {
   const nodePromises: Promise<AztecNodeService>[] = [];
-  const loggerIdStorage = new AsyncLocalStorage<string>();
-  const logNameHandler = (module: string) =>
-    loggerIdStorage.getStore() ? `${module}:${loggerIdStorage.getStore()}` : module;
-  addLogNameHandler(logNameHandler);
 
   for (let i = 0; i < numNodes; i++) {
     const index = indexOffset + i;
@@ -69,7 +69,6 @@ export async function createNodes(
       prefilledPublicData,
       dataDir,
       metricsPort,
-      loggerIdStorage,
     );
     nodePromises.push(nodePromise);
   }
@@ -81,7 +80,6 @@ export async function createNodes(
     throw new Error('Sequencer not found');
   }
 
-  removeLogNameHandler(logNameHandler);
   return nodes;
 }
 
@@ -95,9 +93,9 @@ export async function createNode(
   prefilledPublicData?: PublicDataTreeLeaf[],
   dataDirectory?: string,
   metricsPort?: number,
-  loggerIdStorage?: AsyncLocalStorage<string>,
 ) {
-  const createNode = async () => {
+  const actorIndex = validatorCounter++;
+  return await withLoggerBindings({ actor: `validator-${actorIndex}` }, async () => {
     const validatorConfig = await createValidatorConfig(config, bootstrapNode, tcpPort, addressIndex, dataDirectory);
     const telemetry = await getEndToEndTestTelemetryClient(metricsPort);
     return await AztecNodeService.createAndSync(
@@ -105,8 +103,7 @@ export async function createNode(
       { telemetry, dateProvider },
       { prefilledPublicData, dontStartSequencer: config.dontStartSequencer },
     );
-  };
-  return loggerIdStorage ? await loggerIdStorage.run(tcpPort.toString(), createNode) : createNode();
+  });
 }
 
 /** Creates a P2P enabled instance of Aztec Node Service without a validator */
@@ -118,9 +115,9 @@ export async function createNonValidatorNode(
   prefilledPublicData?: PublicDataTreeLeaf[],
   dataDirectory?: string,
   metricsPort?: number,
-  loggerIdStorage?: AsyncLocalStorage<string>,
 ) {
-  const createNode = async () => {
+  const actorIndex = nodeCounter++;
+  return await withLoggerBindings({ actor: `node-${actorIndex}` }, async () => {
     const p2pConfig = await createP2PConfig(baseConfig, bootstrapNode, tcpPort, dataDirectory);
     const config: AztecNodeConfig = {
       ...p2pConfig,
@@ -130,8 +127,7 @@ export async function createNonValidatorNode(
     };
     const telemetry = await getEndToEndTestTelemetryClient(metricsPort);
     return await AztecNodeService.createAndSync(config, { telemetry, dateProvider }, { prefilledPublicData });
-  };
-  return loggerIdStorage ? await loggerIdStorage.run(tcpPort.toString(), createNode) : createNode();
+  });
 }
 
 export async function createProverNode(
@@ -143,9 +139,9 @@ export async function createProverNode(
   prefilledPublicData?: PublicDataTreeLeaf[],
   dataDirectory?: string,
   metricsPort?: number,
-  loggerIdStorage?: AsyncLocalStorage<string>,
 ) {
-  const createProverNode = async () => {
+  const actorIndex = proverCounter++;
+  return await withLoggerBindings({ actor: `prover-${actorIndex}` }, async () => {
     const proverNodePrivateKey = getPrivateKeyFromIndex(ATTESTER_PRIVATE_KEYS_START_INDEX + addressIndex)!;
     const telemetry = await getEndToEndTestTelemetryClient(metricsPort);
 
@@ -165,8 +161,7 @@ export async function createProverNode(
       prefilledPublicData,
       { ...proverNodeDeps, telemetry },
     );
-  };
-  return loggerIdStorage ? await loggerIdStorage.run(tcpPort.toString(), createProverNode) : createProverNode();
+  });
 }
 
 export async function createP2PConfig(

package/src/shared/uniswap_l1_l2.ts

@@ -11,7 +11,7 @@ import type { DeployAztecL1ContractsReturnType } from '@aztec/ethereum/deploy-az
 import { deployL1Contract } from '@aztec/ethereum/deploy-l1-contract';
 import type { ExtendedViemWalletClient } from '@aztec/ethereum/types';
 import { extractEvent } from '@aztec/ethereum/utils';
-import { CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
+import { EpochNumber } from '@aztec/foundation/branded-types';
 import { sha256ToField } from '@aztec/foundation/crypto/sha256';
 import { InboxAbi, UniswapPortalAbi, UniswapPortalBytecode } from '@aztec/l1-artifacts';
 import { UniswapContract } from '@aztec/noir-contracts.js/Uniswap';
@@ -250,8 +250,8 @@ export const uniswapL1L2TestSuite = (
       await wethCrossChainHarness.expectPublicBalanceOnL2(uniswapL2Contract.address, 0n);
 
       // Since the outbox is only consumable when the epoch is proven, we need to advance to the next epoch.
-      const checkpointNumber = CheckpointNumber.fromBlockNumber(l2UniswapInteractionReceipt.blockNumber!);
-      const epoch = await rollup.getEpochNumberForCheckpoint(checkpointNumber);
+      const block = await aztecNode.getBlock(l2UniswapInteractionReceipt.blockNumber!);
+      const epoch = await rollup.getEpochNumberForCheckpoint(block!.checkpointNumber);
       await cheatCodes.rollup.advanceToEpoch(EpochNumber(epoch + 1));
       await waitForProven(aztecNode, l2UniswapInteractionReceipt, { provenTimeout: 300 });
 
@@ -838,9 +838,8 @@ export const uniswapL1L2TestSuite = (
         chainId: new Fr(l1Client.chain.id),
       });
 
-      const epoch = await rollup.getEpochNumberForCheckpoint(
-        CheckpointNumber.fromBlockNumber(withdrawReceipt.blockNumber!),
-      );
+      const block = await aztecNode.getBlock(withdrawReceipt.blockNumber!);
+      const epoch = await rollup.getEpochNumberForCheckpoint(block!.checkpointNumber);
       const swapResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, swapPrivateLeaf);
       const withdrawResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, withdrawLeaf);
 
@@ -972,9 +971,8 @@ export const uniswapL1L2TestSuite = (
         chainId: new Fr(l1Client.chain.id),
       });
 
-      const epoch = await rollup.getEpochNumberForCheckpoint(
-        CheckpointNumber.fromBlockNumber(withdrawReceipt.blockNumber!),
-      );
+      const block = await aztecNode.getBlock(withdrawReceipt.blockNumber!);
+      const epoch = await rollup.getEpochNumberForCheckpoint(block!.checkpointNumber);
       const swapResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, swapPublicLeaf);
       const withdrawResult = await computeL2ToL1MembershipWitness(aztecNode, epoch, withdrawLeaf);
 

package/src/spartan/utils/config.ts

@@ -15,6 +15,7 @@ const testConfigSchema = z.object({
   AZTEC_EPOCH_DURATION: z.coerce.number().optional().default(32),
   AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5),
   AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET: z.coerce.number().optional().default(2),
+  FUNDING_PRIVATE_KEY: z.string().optional(),
 });
 
 export type TestConfig = z.infer<typeof testConfigSchema>;

package/src/spartan/utils/index.ts

@@ -5,7 +5,7 @@
 export { type TestConfig, setupEnvironment } from './config.js';
 
 // Scripts
-export { getGitProjectRoot, getAztecBin, runAztecBin, runProjectScript } from './scripts.js';
+export { type ScriptResult, getGitProjectRoot, getAztecBin, runAztecBin, runProjectScript } from './scripts.js';
 
 // K8s operations
 export {
@@ -24,6 +24,7 @@ export {
   getServiceEndpoint,
   getRPCEndpoint,
   getEthereumEndpoint,
+  createResilientPrometheusConnection,
 } from './k8s.js';
 
 // Chaos Mesh
@@ -45,6 +46,7 @@ export { restartBot, installTransferBot, uninstallTransferBot } from './bot.js';
 // Node operations (sequencers, validators, pods)
 export {
   awaitCheckpointNumber,
+  waitForProvenToAdvance,
   getSequencers,
   updateSequencersConfig,
   getSequencersConfig,

package/src/spartan/utils/k8s.ts

@@ -1,4 +1,5 @@
 import { createLogger } from '@aztec/aztec.js/log';
+import type { Logger } from '@aztec/foundation/log';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { retryUntil } from '@aztec/foundation/retry';
 
@@ -6,6 +7,8 @@ import { type ChildProcess, exec, spawn } from 'child_process';
 import path from 'path';
 import { promisify } from 'util';
 
+import { AlertTriggeredError, GrafanaClient } from '../../quality_of_service/grafana_client.js';
+
 const execAsync = promisify(exec);
 
 const logger = createLogger('e2e:k8s-utils');
@@ -370,6 +373,155 @@ export async function waitForResourcesByName({
   );
 }
 
+/**
+ * Waits for all StatefulSets matching a label to have all their replicas ready.
+ *
+ * @param namespace - Kubernetes namespace
+ * @param label - Label selector for StatefulSets (e.g., "app.kubernetes.io/component=sequencer-node")
+ * @param timeoutSeconds - Maximum time to wait in seconds
+ * @param pollIntervalSeconds - How often to check status
+ */
+export async function waitForStatefulSetsReady({
+  namespace,
+  label,
+  timeoutSeconds = 600,
+  pollIntervalSeconds = 5,
+}: {
+  namespace: string;
+  label: string;
+  timeoutSeconds?: number;
+  pollIntervalSeconds?: number;
+}): Promise<void> {
+  logger.info(`Waiting for StatefulSets with label ${label} to have all replicas ready (timeout: ${timeoutSeconds}s)`);
+
+  await retryUntil(
+    async () => {
+      // Get all StatefulSets matching the label
+      const getCmd = `kubectl get statefulset -l ${label} -n ${namespace} -o json`;
+      const { stdout } = await execAsync(getCmd);
+      const result = JSON.parse(stdout);
+
+      if (!result.items || result.items.length === 0) {
+        logger.verbose(`No StatefulSets found with label ${label}`);
+        return false;
+      }
+
+      // Check each StatefulSet
+      for (const sts of result.items) {
+        const name = sts.metadata.name;
+        const desired = sts.spec.replicas ?? 0;
+        const ready = sts.status.readyReplicas ?? 0;
+        const updated = sts.status.updatedReplicas ?? 0;
+
+        if (ready < desired || updated < desired) {
+          logger.verbose(`StatefulSet ${name}: ${ready}/${desired} ready, ${updated}/${desired} updated`);
+          return false;
+        }
+      }
+
+      logger.info(`All StatefulSets with label ${label} are ready`);
+      return true;
+    },
+    `StatefulSets with label ${label} to be ready`,
+    timeoutSeconds,
+    pollIntervalSeconds,
+  );
+}
+
+/**
+ * Creates a Prometheus connection that can re-establish port-forward on failure.
+ * Returns functions to connect and run alert checks that automatically reconnect if needed.
+ *
+ * @param namespace - K8s namespace to fall back to if metrics namespace doesn't have Prometheus
+ * @param endpoints - Array to track created endpoints for cleanup
+ * @param log - Logger instance
+ */
+export function createResilientPrometheusConnection(
+  namespace: string,
+  endpoints: ServiceEndpoint[],
+  log: Logger,
+): {
+  connect: () => Promise<GrafanaClient>;
+  runAlertCheck: (alerts: Parameters<GrafanaClient['runAlertCheck']>[0]) => Promise<void>;
+} {
+  let alertChecker: GrafanaClient | undefined;
+  let currentEndpoint: ServiceEndpoint | undefined;
+
+  const connect = async (): Promise<GrafanaClient> => {
+    // Kill existing connection if any
+    if (currentEndpoint?.process) {
+      currentEndpoint.process.kill();
+    }
+
+    // Try metrics namespace first, then network namespace
+    let promPort = 0;
+    let promUrl = '';
+    let promProc: ChildProcess | undefined;
+
+    try {
+      const metricsResult = await startPortForward({
+        resource: `svc/metrics-prometheus-server`,
+        namespace: 'metrics',
+        containerPort: 80,
+      });
+      promProc = metricsResult.process;
+      promPort = metricsResult.port;
+      promUrl = `http://127.0.0.1:${promPort}/api/v1`;
+    } catch {
+      // Metrics namespace might not have Prometheus, try network namespace
+      log.verbose('Metrics namespace Prometheus not available, trying network namespace');
+    }
+
+    if (promPort === 0) {
+      const nsResult = await startPortForward({
+        resource: `svc/prometheus-server`,
+        namespace,
+        containerPort: 80,
+      });
+      promProc = nsResult.process;
+      promPort = nsResult.port;
+      promUrl = `http://127.0.0.1:${promPort}/api/v1`;
+    }
+
+    if (!promProc || promPort === 0) {
+      throw new Error('Unable to port-forward to Prometheus');
+    }
+
+    currentEndpoint = { url: promUrl, process: promProc };
+    endpoints.push(currentEndpoint);
+    alertChecker = new GrafanaClient(log, { grafanaEndpoint: promUrl, grafanaCredentials: '' });
+    log.info(`Established Prometheus connection at ${promUrl}`);
+    return alertChecker;
+  };
+
+  const runAlertCheck = async (alerts: Parameters<GrafanaClient['runAlertCheck']>[0]): Promise<void> => {
+    if (!alertChecker) {
+      alertChecker = await connect();
+    }
+
+    try {
+      await alertChecker.runAlertCheck(alerts);
+    } catch (err) {
+      // If it's an AlertTriggeredError (expected behavior)
+      if (err instanceof AlertTriggeredError) {
+        throw err;
+      }
+
+      // Check if it's a connection error (port-forward died)
+      const errorStr = String(err);
+      if (errorStr.includes('fetch failed') || errorStr.includes('ECONNREFUSED') || errorStr.includes('ECONNRESET')) {
+        log.warn(`Prometheus connection lost, re-establishing port-forward...`);
+        alertChecker = await connect();
+        await alertChecker.runAlertCheck(alerts);
+      } else {
+        throw err;
+      }
+    }
+  };
+
+  return { connect, runAlertCheck };
+}
+
 export function getChartDir(spartanDir: string, chartName: string) {
   return path.join(spartanDir.trim(), chartName);
 }