@aztec/end-to-end 0.0.1-commit.18ccd8f0 → 0.0.1-commit.1a421b1a1

package/src/spartan/setup_test_wallets.ts

@@ -13,11 +13,14 @@ import type { Logger } from '@aztec/foundation/log';
 import { retryUntil } from '@aztec/foundation/retry';
 import { TokenContract } from '@aztec/noir-contracts.js/Token';
 import type { AztecNodeAdmin } from '@aztec/stdlib/interfaces/client';
-import { TestWallet, proveInteraction, registerInitialLocalNetworkAccountsInWallet } from '@aztec/test-wallet/server';
+import { registerInitialLocalNetworkAccountsInWallet } from '@aztec/wallets/testing';
 
 import { getACVMConfig } from '../fixtures/get_acvm_config.js';
 import { getBBConfig } from '../fixtures/get_bb_config.js';
 import { getSponsoredFPCAddress, registerSponsoredFPC } from '../fixtures/utils.js';
+import { TestWallet } from '../test-wallet/test_wallet.js';
+import { proveInteraction } from '../test-wallet/utils.js';
+import { WorkerWallet } from '../test-wallet/worker_wallet.js';
 
 export interface TestAccounts {
   aztecNode: AztecNode;
@@ -85,11 +88,19 @@ export async function deploySponsoredTestAccountsWithTokens(
 
   const paymentMethod = new SponsoredFeePaymentMethod(await getSponsoredFPCAddress());
   const recipientDeployMethod = await recipientAccount.getDeployMethod();
-  await recipientDeployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod }, wait: { timeout: 2400 } });
+  await recipientDeployMethod.send({
+    from: AztecAddress.ZERO,
+    fee: { paymentMethod },
+    wait: { timeout: 2400 },
+  });
   await Promise.all(
     fundedAccounts.map(async a => {
       const deployMethod = await a.getDeployMethod();
-      await deployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod }, wait: { timeout: 2400 } }); // increase timeout on purpose in order to account for two empty epochs
+      await deployMethod.send({
+        from: AztecAddress.ZERO,
+        fee: { paymentMethod },
+        wait: { timeout: 2400 },
+      }); // increase timeout on purpose in order to account for two empty epochs
       logger.info(`Account deployed at ${a.address}`);
     }),
   );
@@ -118,16 +129,28 @@ export async function deploySponsoredTestAccountsWithTokens(
 }
 
 async function deployAccountWithDiagnostics(
-  account: { getDeployMethod: () => Promise<{ send: (opts: any) => any }>; address: any },
+  account: { getDeployMethod: () => Promise<{ simulate: (opts: any) => any; send: (opts: any) => any }>; address: any },
   paymentMethod: SponsoredFeePaymentMethod,
   aztecNode: AztecNode,
   logger: Logger,
   accountLabel: string,
+  estimateGas?: boolean,
 ): Promise<void> {
   const deployMethod = await account.getDeployMethod();
   let txHash;
   try {
-    txHash = await deployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod }, wait: NO_WAIT });
+    let gasSettings;
+    if (estimateGas) {
+      const sim = await deployMethod.simulate({ from: AztecAddress.ZERO, fee: { paymentMethod } });
+      gasSettings = sim.estimatedGas;
+      logger.info(`${accountLabel} estimated gas: DA=${gasSettings.gasLimits.daGas} L2=${gasSettings.gasLimits.l2Gas}`);
+    }
+    const deployResult = await deployMethod.send({
+      from: AztecAddress.ZERO,
+      fee: { paymentMethod, gasSettings },
+      wait: NO_WAIT,
+    });
+    txHash = deployResult.txHash;
     await waitForTx(aztecNode, txHash, { timeout: 2400 });
     logger.info(`${accountLabel} deployed at ${account.address}`);
   } catch (error) {
@@ -149,18 +172,29 @@ async function deployAccountWithDiagnostics(
 }
 
 async function deployAccountsInBatches(
-  accounts: { getDeployMethod: () => Promise<{ send: (opts: any) => any }>; address: any }[],
+  accounts: {
+    getDeployMethod: () => Promise<{ simulate: (opts: any) => any; send: (opts: any) => any }>;
+    address: any;
+  }[],
   paymentMethod: SponsoredFeePaymentMethod,
   aztecNode: AztecNode,
   logger: Logger,
   labelPrefix: string,
   batchSize = 2,
+  estimateGas?: boolean,
 ): Promise<void> {
   for (let i = 0; i < accounts.length; i += batchSize) {
     const batch = accounts.slice(i, i + batchSize);
     await Promise.all(
       batch.map((account, idx) =>
-        deployAccountWithDiagnostics(account, paymentMethod, aztecNode, logger, `${labelPrefix}${i + idx + 1}`),
+        deployAccountWithDiagnostics(
+          account,
+          paymentMethod,
+          aztecNode,
+          logger,
+          `${labelPrefix}${i + idx + 1}`,
+          estimateGas,
+        ),
       ),
     );
   }
@@ -171,6 +205,7 @@ export async function deploySponsoredTestAccounts(
   aztecNode: AztecNode,
   logger: Logger,
   numberOfFundedWallets = 1,
+  opts?: { estimateGas?: boolean },
 ): Promise<TestAccountsWithoutTokens> {
   const [recipient, ...funded] = await generateSchnorrAccounts(numberOfFundedWallets + 1);
   const recipientAccount = await wallet.createSchnorrAccount(recipient.secret, recipient.salt);
@@ -180,8 +215,23 @@ export async function deploySponsoredTestAccounts(
 
   const paymentMethod = new SponsoredFeePaymentMethod(await getSponsoredFPCAddress());
 
-  await deployAccountWithDiagnostics(recipientAccount, paymentMethod, aztecNode, logger, 'Recipient account');
-  await deployAccountsInBatches(fundedAccounts, paymentMethod, aztecNode, logger, 'Funded account ', 2);
+  await deployAccountWithDiagnostics(
+    recipientAccount,
+    paymentMethod,
+    aztecNode,
+    logger,
+    'Recipient account',
+    opts?.estimateGas,
+  );
+  await deployAccountsInBatches(
+    fundedAccounts,
+    paymentMethod,
+    aztecNode,
+    logger,
+    'Funded account ',
+    2,
+    opts?.estimateGas,
+  );
 
   return {
     aztecNode,
@@ -263,7 +313,7 @@ async function bridgeL1FeeJuice(
   const claim = await portal.bridgeTokensPublic(recipient, amount, true /* mint */);
 
   const isSynced = async () =>
-    (await aztecNode.getL1ToL2MessageBlock(Fr.fromHexString(claim.messageHash))) !== undefined;
+    (await aztecNode.getL1ToL2MessageCheckpoint(Fr.fromHexString(claim.messageHash))) !== undefined;
   await retryUntil(isSynced, `message ${claim.messageHash} sync`, 24, 0.5);
 
   log.info(`Created a claim for ${amount} L1 fee juice to ${recipient}.`, claim);
@@ -295,13 +345,9 @@ async function deployTokenAndMint(
   logger: Logger,
 ) {
   logger.verbose(`Deploying TokenContract...`);
-  const { contract: tokenContract } = await TokenContract.deploy(
-    wallet,
-    admin,
-    TOKEN_NAME,
-    TOKEN_SYMBOL,
-    TOKEN_DECIMALS,
-  ).send({
+  const {
+    receipt: { contract: tokenContract },
+  } = await TokenContract.deploy(wallet, admin, TOKEN_NAME, TOKEN_SYMBOL, TOKEN_DECIMALS).send({
     from: admin,
     fee: {
       paymentMethod,
@@ -395,3 +441,42 @@ export async function createWalletAndAztecNodeClient(
     },
   };
 }
+
+export type WorkerWalletWrapper = {
+  wallet: WorkerWallet;
+  aztecNode: AztecNode;
+  cleanup: () => Promise<void>;
+};
+
+export async function createWorkerWalletClient(
+  nodeUrl: string,
+  proverEnabled: boolean,
+  logger: Logger,
+): Promise<WorkerWalletWrapper> {
+  const aztecNode = createAztecNodeClient(nodeUrl);
+  const [bbConfig, acvmConfig] = await Promise.all([getBBConfig(logger), getACVMConfig(logger)]);
+
+  // Strip cleanup functions — they can't be structured-cloned for worker transfer
+  const { cleanup: bbCleanup, ...bbPaths } = bbConfig ?? {};
+  const { cleanup: acvmCleanup, ...acvmPaths } = acvmConfig ?? {};
+
+  const pxeConfig = {
+    dataDirectory: undefined,
+    dataStoreMapSizeKb: 1024 * 1024,
+    ...bbPaths,
+    ...acvmPaths,
+    proverEnabled,
+  };
+
+  const wallet = await WorkerWallet.create(nodeUrl, pxeConfig);
+
+  return {
+    wallet,
+    aztecNode,
+    async cleanup() {
+      await wallet.stop();
+      await bbCleanup?.();
+      await acvmCleanup?.();
+    },
+  };
+}

package/src/spartan/tx_metrics.ts

@@ -296,7 +296,7 @@ export class TxInclusionMetrics {
           value: stats.mean,
         },
         {
-          name: `${group}/median_inclusion`,
+          name: `${group}/p50_inclusion`,
           unit: 's',
           value: stats.median,
         },

package/src/spartan/utils/bot.ts

@@ -34,7 +34,8 @@ export async function installTransferBot({
   logger: log,
   replicas = 1,
   txIntervalSeconds = 10,
-  followChain = 'PENDING',
+  followChain = 'CHECKPOINTED',
+  pxeSyncChainTip = 'proposed',
   mnemonic = process.env.LABS_INFRA_MNEMONIC ?? 'test test test test test test test test test test test junk',
   mnemonicStartIndex,
   botPrivateKey = process.env.BOT_TRANSFERS_L2_PRIVATE_KEY ?? '0xcafe01',
@@ -49,6 +50,7 @@ export async function installTransferBot({
   replicas?: number;
   txIntervalSeconds?: number;
   followChain?: string;
+  pxeSyncChainTip?: string;
   mnemonic?: string;
   mnemonicStartIndex?: number | string;
   botPrivateKey?: string;
@@ -67,6 +69,7 @@ export async function installTransferBot({
     'bot.replicaCount': replicas,
     'bot.txIntervalSeconds': txIntervalSeconds,
     'bot.followChain': followChain,
+    'bot.pxeSyncChainTip': pxeSyncChainTip,
     'bot.botPrivateKey': botPrivateKey,
     'bot.nodeUrl': resolvedNodeUrl,
     'bot.mnemonic': mnemonic,

package/src/spartan/utils/config.ts

@@ -8,6 +8,7 @@ const logger = createLogger('e2e:k8s-utils');
 const testConfigSchema = z.object({
   NAMESPACE: z.string().default('scenario'),
   REAL_VERIFIER: schemas.Boolean.optional().default(true),
+  DEBUG_FORCE_TX_PROOF_VERIFICATION: schemas.Boolean.optional().default(true),
   CREATE_ETH_DEVNET: schemas.Boolean.optional().default(false),
   L1_RPC_URLS_JSON: z.string().optional(),
   L1_ACCOUNT_MNEMONIC: z.string().optional(),
@@ -15,6 +16,8 @@ const testConfigSchema = z.object({
   AZTEC_EPOCH_DURATION: z.coerce.number().optional().default(32),
   AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5),
   AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET: z.coerce.number().optional().default(2),
+  FUNDING_PRIVATE_KEY: z.string().optional(),
+  AZTEC_ADMIN_API_KEY: z.string().optional(),
 });
 
 export type TestConfig = z.infer<typeof testConfigSchema>;

package/src/spartan/utils/index.ts

@@ -5,7 +5,7 @@
 export { type TestConfig, setupEnvironment } from './config.js';
 
 // Scripts
-export { getGitProjectRoot, getAztecBin, runAztecBin, runProjectScript } from './scripts.js';
+export { type ScriptResult, getGitProjectRoot, getAztecBin, runAztecBin, runProjectScript } from './scripts.js';
 
 // K8s operations
 export {
@@ -24,6 +24,8 @@ export {
   getServiceEndpoint,
   getRPCEndpoint,
   getEthereumEndpoint,
+  createResilientPrometheusConnection,
+  scaleProverAgents,
 } from './k8s.js';
 
 // Chaos Mesh
@@ -39,12 +41,16 @@ export {
   applyNetworkShaping,
 } from './chaos.js';
 
+// Helm
+export { hasDeployedHelmRelease } from './helm.js';
+
 // Bot management
 export { restartBot, installTransferBot, uninstallTransferBot } from './bot.js';
 
 // Node operations (sequencers, validators, pods)
 export {
   awaitCheckpointNumber,
+  waitForProvenToAdvance,
   getSequencers,
   updateSequencersConfig,
   getSequencersConfig,
@@ -60,3 +66,6 @@ export { getPublicViemClient, getL1DeploymentAddresses, getNodeClient } from './
 
 // Health checks
 export { ChainHealth, type ChainHealthSnapshot } from './health.js';
+
+// Pod log extraction
+export { type BlockBuiltLogEntry, fetchBlockBuiltLogs } from './pod_logs.js';

package/src/spartan/utils/k8s.ts

@@ -1,4 +1,5 @@
 import { createLogger } from '@aztec/aztec.js/log';
+import type { Logger } from '@aztec/foundation/log';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { retryUntil } from '@aztec/foundation/retry';
 
@@ -6,6 +7,8 @@ import { type ChildProcess, exec, spawn } from 'child_process';
 import path from 'path';
 import { promisify } from 'util';
 
+import { AlertTriggeredError, GrafanaClient } from '../../quality_of_service/grafana_client.js';
+
 const execAsync = promisify(exec);
 
 const logger = createLogger('e2e:k8s-utils');
@@ -370,6 +373,163 @@ export async function waitForResourcesByName({
   );
 }
 
+/**
+ * Waits for all StatefulSets matching a label to have all their replicas ready.
+ *
+ * @param namespace - Kubernetes namespace
+ * @param label - Label selector for StatefulSets (e.g., "app.kubernetes.io/component=sequencer-node")
+ * @param timeoutSeconds - Maximum time to wait in seconds
+ * @param pollIntervalSeconds - How often to check status
+ */
+export async function waitForStatefulSetsReady({
+  namespace,
+  label,
+  timeoutSeconds = 600,
+  pollIntervalSeconds = 5,
+}: {
+  namespace: string;
+  label: string;
+  timeoutSeconds?: number;
+  pollIntervalSeconds?: number;
+}): Promise<void> {
+  logger.info(`Waiting for StatefulSets with label ${label} to have all replicas ready (timeout: ${timeoutSeconds}s)`);
+
+  await retryUntil(
+    async () => {
+      // Get all StatefulSets matching the label
+      const getCmd = `kubectl get statefulset -l ${label} -n ${namespace} -o json`;
+      const { stdout } = await execAsync(getCmd);
+      const result = JSON.parse(stdout);
+
+      if (!result.items || result.items.length === 0) {
+        logger.verbose(`No StatefulSets found with label ${label}`);
+        return false;
+      }
+
+      // Check each StatefulSet
+      for (const sts of result.items) {
+        const name = sts.metadata.name;
+        const desired = sts.spec.replicas ?? 0;
+        const ready = sts.status.readyReplicas ?? 0;
+        const updated = sts.status.updatedReplicas ?? 0;
+
+        if (ready < desired || updated < desired) {
+          logger.verbose(`StatefulSet ${name}: ${ready}/${desired} ready, ${updated}/${desired} updated`);
+          return false;
+        }
+      }
+
+      logger.info(`All StatefulSets with label ${label} are ready`);
+      return true;
+    },
+    `StatefulSets with label ${label} to be ready`,
+    timeoutSeconds,
+    pollIntervalSeconds,
+  );
+}
+
+/**
+ * Creates a Prometheus connection that can re-establish port-forward on failure.
+ * Returns functions to connect and run alert checks that automatically reconnect if needed.
+ *
+ * @param namespace - K8s namespace to fall back to if metrics namespace doesn't have Prometheus
+ * @param endpoints - Array to track created endpoints for cleanup
+ * @param log - Logger instance
+ */
+export function createResilientPrometheusConnection(
+  namespace: string,
+  endpoints: ServiceEndpoint[],
+  log: Logger,
+): {
+  connect: () => Promise<GrafanaClient>;
+  runAlertCheck: (alerts: Parameters<GrafanaClient['runAlertCheck']>[0]) => Promise<void>;
+} {
+  let alertChecker: GrafanaClient | undefined;
+  let currentEndpoint: ServiceEndpoint | undefined;
+
+  const connect = async (): Promise<GrafanaClient> => {
+    // Kill existing connection if any
+    if (currentEndpoint?.process) {
+      currentEndpoint.process.kill();
+    }
+
+    // Try metrics namespace first, then network namespace
+    let promPort = 0;
+    let promUrl = '';
+    let promProc: ChildProcess | undefined;
+
+    try {
+      const metricsResult = await startPortForward({
+        resource: `svc/metrics-prometheus-server`,
+        namespace: 'metrics',
+        containerPort: 80,
+      });
+      promProc = metricsResult.process;
+      promPort = metricsResult.port;
+      promUrl = `http://127.0.0.1:${promPort}/api/v1`;
+    } catch {
+      // Metrics namespace might not have Prometheus, try network namespace
+      log.verbose('Metrics namespace Prometheus not available, trying network namespace');
+    }
+
+    if (promPort === 0) {
+      const nsResult = await startPortForward({
+        resource: `svc/prometheus-server`,
+        namespace,
+        containerPort: 80,
+      });
+      promProc = nsResult.process;
+      promPort = nsResult.port;
+      promUrl = `http://127.0.0.1:${promPort}/api/v1`;
+    }
+
+    if (!promProc || promPort === 0) {
+      throw new Error('Unable to port-forward to Prometheus');
+    }
+
+    currentEndpoint = { url: promUrl, process: promProc };
+    endpoints.push(currentEndpoint);
+    alertChecker = new GrafanaClient(log, { grafanaEndpoint: promUrl, grafanaCredentials: '' });
+    log.info(`Established Prometheus connection at ${promUrl}`);
+    return alertChecker;
+  };
+
+  const runAlertCheck = async (alerts: Parameters<GrafanaClient['runAlertCheck']>[0]): Promise<void> => {
+    if (!alertChecker) {
+      alertChecker = await connect();
+    }
+
+    try {
+      await alertChecker.runAlertCheck(alerts);
+    } catch (err) {
+      // If it's an AlertTriggeredError (expected behavior)
+      if (err instanceof AlertTriggeredError) {
+        throw err;
+      }
+
+      // Check if it's a connection error (port-forward died)
+      const errorStr = String(err);
+      if (errorStr.includes('fetch failed') || errorStr.includes('ECONNREFUSED') || errorStr.includes('ECONNRESET')) {
+        log.warn(`Prometheus connection lost, re-establishing port-forward...`);
+        alertChecker = await connect();
+        await alertChecker.runAlertCheck(alerts);
+      } else {
+        throw err;
+      }
+    }
+  };
+
+  return { connect, runAlertCheck };
+}
+
+/** Scales the prover-agent Deployment to the given number of replicas. */
+export async function scaleProverAgents(namespace: string, replicas: number, log: Logger): Promise<void> {
+  const label = 'app.kubernetes.io/component=prover-agent';
+  const command = `kubectl scale deployment -l ${label} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+  log.info(`Scaling prover agents to ${replicas}: ${command}`);
+  await execAsync(command);
+}
+
 export function getChartDir(spartanDir: string, chartName: string) {
   return path.join(spartanDir.trim(), chartName);
 }