@aztec/end-to-end 0.0.1-commit.ff7989d6c → 0.0.1-commit.ffe5b04ea

package/src/shared/jest_setup.ts

@@ -1,8 +1,19 @@
 import { createLogger } from '@aztec/aztec.js/log';
 
-import { beforeEach, expect } from '@jest/globals';
+import { afterAll, afterEach, beforeEach, expect } from '@jest/globals';
+import { readlinkSync } from 'fs';
 import { basename } from 'path';
 
+import { EluMonitor } from '../fixtures/elu_monitor.js';
+
+const eluMonitor = process.env.ELU_MONITOR_FILE
+  ? new EluMonitor(process.env.ELU_MONITOR_FILE, Number(process.env.ELU_MONITOR_INTERVAL_MS) || undefined)
+  : undefined;
+
+if (eluMonitor) {
+  process.on('exit', () => eluMonitor.stop());
+}
+
 beforeEach(() => {
   const { testPath, currentTestName } = expect.getState();
   if (!testPath || !currentTestName) {
@@ -10,4 +21,43 @@ beforeEach(() => {
   }
   const logger = createLogger(`e2e:${basename(testPath).replace('.test.ts', '')}`);
   logger.info(`Running test: ${currentTestName}`);
+  eluMonitor?.startTest(currentTestName);
+});
+
+afterEach(() => {
+  eluMonitor?.stopTest();
+});
+
+// Log leaked handles after all tests complete. This runs after test-level afterAll hooks,
+// so any handles still alive at this point were not properly cleaned up during teardown.
+// This diagnostic helps identify the source of exit hangs without masking them.
+afterAll(() => {
+  const handles = (process as any)._getActiveHandles();
+  if (handles.length > 0) {
+    const details = handles.map((h: any) => {
+      const type = h?.constructor?.name ?? typeof h;
+      const fd = h?.fd ?? h?._handle?.fd ?? '?';
+      const destroyed = h?.destroyed ?? '?';
+      const hasRef = typeof h?.hasRef === 'function' ? h.hasRef() : '?';
+      const localAddr = h?.localAddress ?? '';
+      const remoteAddr = h?.remoteAddress ?? '';
+      const localPort = h?.localPort ?? '';
+      const remotePort = h?.remotePort ?? '';
+      const proto = Object.getPrototypeOf(h)?.constructor?.name ?? '?';
+      const keys = Object.keys(h).slice(0, 10).join(',');
+      let fdTarget = '';
+      if (typeof fd === 'number') {
+        try {
+          fdTarget = ` -> ${readlinkSync(`/proc/self/fd/${fd}`)}`;
+        } catch {
+          // ignore
+        }
+      }
+      return `  ${type}(fd=${fd}, destroyed=${destroyed}, hasRef=${hasRef}${fdTarget}) proto=${proto} addr=${localAddr}:${localPort}->${remoteAddr}:${remotePort} keys=[${keys}]`;
+    });
+    process.stderr.write(
+      `\n[jest_setup] WARNING: ${handles.length} handle(s) still active after teardown:\n${details.join('\n')}\n` +
+        `These may prevent Jest from exiting. Investigate and fix the leak.\n\n`,
+    );
+  }
 });

package/src/shared/submit-transactions.ts

@@ -19,10 +19,7 @@ export const submitTxsTo = async (
     times(numTxs, async () => {
       const accountManager = await wallet.createSchnorrAccount(Fr.random(), Fr.random(), GrumpkinScalar.random());
       const deployMethod = await accountManager.getDeployMethod();
-      const txHash = await deployMethod.send({
-        from: submitter,
-        wait: NO_WAIT,
-      });
+      const { txHash } = await deployMethod.send({ from: submitter, wait: NO_WAIT });
 
       logger.info(`Tx sent with hash ${txHash}`);
       const receipt: TxReceipt = await wallet.getTxReceipt(txHash);

package/src/shared/uniswap_l1_l2.ts

@@ -130,7 +130,9 @@ export const uniswapL1L2TestSuite = (
         client: l1Client,
       });
       // deploy l2 uniswap contract and attach to portal
-      uniswapL2Contract = await UniswapContract.deploy(wallet, uniswapPortalAddress).send({ from: ownerAddress });
+      ({ contract: uniswapL2Contract } = await UniswapContract.deploy(wallet, uniswapPortalAddress).send({
+        from: ownerAddress,
+      }));
 
       const registryAddress = (await aztecNode.getNodeInfo()).l1ContractAddresses.registryAddress;
 
@@ -195,7 +197,7 @@ export const uniswapL1L2TestSuite = (
       logger.info('Withdrawing weth to L1 and sending message to swap to dai');
       const [secretForDepositingSwappedDai, secretHashForDepositingSwappedDai] = await generateClaimSecret();
 
-      const l2UniswapInteractionReceipt = await uniswapL2Contract.methods
+      const { receipt: l2UniswapInteractionReceipt } = await uniswapL2Contract.methods
         .swap_private(
           wethCrossChainHarness.l2Token.address,
           wethCrossChainHarness.l2Bridge.address,
@@ -787,7 +789,7 @@ export const uniswapL1L2TestSuite = (
       logger.info('Withdrawing weth to L1 and sending message to swap to dai');
 
       const [, secretHashForDepositingSwappedDai] = await generateClaimSecret();
-      const withdrawReceipt = await uniswapL2Contract.methods
+      const { receipt: withdrawReceipt } = await uniswapL2Contract.methods
         .swap_private(
           wethCrossChainHarness.l2Token.address,
           wethCrossChainHarness.l2Bridge.address,
@@ -915,7 +917,7 @@ export const uniswapL1L2TestSuite = (
 
       // Call swap_public on L2
       const secretHashForDepositingSwappedDai = Fr.random();
-      const withdrawReceipt = await uniswapL2Contract.methods
+      const { receipt: withdrawReceipt } = await uniswapL2Contract.methods
         .swap_public(
           ownerAddress,
           wethCrossChainHarness.l2Bridge.address,

package/src/simulators/lending_simulator.ts

@@ -186,14 +186,16 @@ export class LendingSimulator {
 
     expect(this.borrowed).toEqual(this.stableCoin.totalSupply - this.mintedOutside);
 
-    const asset = await this.lendingContract.methods.get_asset(0).simulate({ from: this.account.address });
+    const { result: asset } = await this.lendingContract.methods.get_asset(0).simulate({ from: this.account.address });
 
     const interestAccumulator = asset['interest_accumulator'];
     expect(interestAccumulator).toEqual(this.accumulator);
     expect(asset['last_updated_ts']).toEqual(BigInt(this.time));
 
     for (const key of [this.account.address, AztecAddress.fromField(await this.account.key())]) {
-      const privatePos = await this.lendingContract.methods.get_position(key).simulate({ from: this.account.address });
+      const { result: privatePos } = await this.lendingContract.methods
+        .get_position(key)
+        .simulate({ from: this.account.address });
       expect(new Fr(privatePos['collateral'])).toEqual(this.collateral[key.toString()] ?? Fr.ZERO);
       expect(new Fr(privatePos['static_debt'])).toEqual(this.staticDebt[key.toString()] ?? Fr.ZERO);
       expect(privatePos['debt']).toEqual(

package/src/simulators/token_simulator.ts

@@ -109,7 +109,9 @@ export class TokenSimulator {
       await Promise.all(
         chunk(calls, 5).map(batch => new BatchCall(this.defaultWallet, batch).simulate({ from: this.defaultAddress })),
       )
-    ).flat();
+    )
+      .flat()
+      .map(r => r.result);
     expect(results[0]).toEqual(this.totalSupply);
 
     // Check that all our balances match
@@ -123,7 +125,9 @@ export class TokenSimulator {
       const wallet = this.lookupProvider.get(address.toString());
       const asset = wallet ? this.token.withWallet(wallet) : this.token;
 
-      const actualPrivateBalance = await asset.methods.balance_of_private(address).simulate({ from: address });
+      const { result: actualPrivateBalance } = await asset.methods
+        .balance_of_private(address)
+        .simulate({ from: address });
       expect(actualPrivateBalance).toEqual(this.balanceOfPrivate(address));
     }
   }

package/src/spartan/setup_test_wallets.ts

@@ -138,11 +138,8 @@ async function deployAccountWithDiagnostics(
   const deployMethod = await account.getDeployMethod();
   let txHash;
   try {
-    txHash = await deployMethod.send({
-      from: AztecAddress.ZERO,
-      fee: { paymentMethod },
-      wait: NO_WAIT,
-    });
+    const deployResult = await deployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod }, wait: NO_WAIT });
+    txHash = deployResult.txHash;
     await waitForTx(aztecNode, txHash, { timeout: 2400 });
     logger.info(`${accountLabel} deployed at ${account.address}`);
   } catch (error) {
@@ -278,7 +275,7 @@ async function bridgeL1FeeJuice(
   const claim = await portal.bridgeTokensPublic(recipient, amount, true /* mint */);
 
   const isSynced = async () =>
-    (await aztecNode.getL1ToL2MessageBlock(Fr.fromHexString(claim.messageHash))) !== undefined;
+    (await aztecNode.getL1ToL2MessageCheckpoint(Fr.fromHexString(claim.messageHash))) !== undefined;
   await retryUntil(isSynced, `message ${claim.messageHash} sync`, 24, 0.5);
 
   log.info(`Created a claim for ${amount} L1 fee juice to ${recipient}.`, claim);
@@ -310,13 +307,9 @@ async function deployTokenAndMint(
   logger: Logger,
 ) {
   logger.verbose(`Deploying TokenContract...`);
-  const { contract: tokenContract } = await TokenContract.deploy(
-    wallet,
-    admin,
-    TOKEN_NAME,
-    TOKEN_SYMBOL,
-    TOKEN_DECIMALS,
-  ).send({
+  const {
+    receipt: { contract: tokenContract },
+  } = await TokenContract.deploy(wallet, admin, TOKEN_NAME, TOKEN_SYMBOL, TOKEN_DECIMALS).send({
     from: admin,
     fee: {
       paymentMethod,

package/src/spartan/tx_metrics.ts

@@ -296,7 +296,7 @@ export class TxInclusionMetrics {
           value: stats.mean,
         },
         {
-          name: `${group}/median_inclusion`,
+          name: `${group}/p50_inclusion`,
           unit: 's',
           value: stats.median,
         },

package/src/spartan/utils/config.ts

@@ -16,6 +16,7 @@ const testConfigSchema = z.object({
   AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5),
   AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET: z.coerce.number().optional().default(2),
   FUNDING_PRIVATE_KEY: z.string().optional(),
+  AZTEC_ADMIN_API_KEY: z.string().optional(),
 });
 
 export type TestConfig = z.infer<typeof testConfigSchema>;

package/src/spartan/utils/index.ts

@@ -41,6 +41,9 @@ export {
   applyNetworkShaping,
 } from './chaos.js';
 
+// Helm
+export { hasDeployedHelmRelease } from './helm.js';
+
 // Bot management
 export { restartBot, installTransferBot, uninstallTransferBot } from './bot.js';
 

package/src/spartan/utils/nodes.ts

@@ -173,7 +173,7 @@ export async function withSequencersAdmin<T>(env: TestConfig, fn: (node: AztecNo
           if (statusRes.status !== 200) {
             throw new Error(`Admin endpoint returned status ${statusRes.status}`);
           }
-          const client = createAztecNodeAdminClient(url);
+          const client = createAztecNodeAdminClient(url, {}, undefined, env.AZTEC_ADMIN_API_KEY);
           return { result: await fn(client), process };
         } catch (err) {
           // Kill the port-forward before retrying
@@ -363,16 +363,24 @@ export async function enableValidatorDynamicBootNode(
  */
 export async function rollAztecPods(namespace: string, clearState: boolean = false) {
   // Pod components use 'validator', but StatefulSets and PVCs use 'sequencer-node' for validators
+  // RPC nodes have nodeType='rpc-node' in Helm values, so their component label is 'rpc-node' (not 'rpc')
   const podComponents = [
     'p2p-bootstrap',
     'prover-node',
     'prover-broker',
     'prover-agent',
     'sequencer-node',
-    'rpc',
+    'rpc-node',
+    'validator-ha-db',
+  ];
+  const pvcComponents = [
+    'p2p-bootstrap',
+    'prover-node',
+    'prover-broker',
+    'sequencer-node',
+    'rpc-node',
     'validator-ha-db',
   ];
-  const pvcComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'sequencer-node', 'rpc', 'validator-ha-db'];
   // StatefulSet components that need to be scaled down before PVC deletion
   // Note: validators use 'sequencer-node' as component label, not 'validator'
   const statefulSetComponents = [
@@ -380,7 +388,7 @@ export async function rollAztecPods(namespace: string, clearState: boolean = fal
     'prover-node',
     'prover-broker',
     'sequencer-node',
-    'rpc',
+    'rpc-node',
     'validator-ha-db',
   ];
 

package/src/test-wallet/wallet_worker_script.ts

@@ -1,43 +1,60 @@
 import { createAztecNodeClient } from '@aztec/aztec.js/node';
+import type { SendOptions } from '@aztec/aztec.js/wallet';
 import { jsonStringify } from '@aztec/foundation/json-rpc';
-import type { ApiSchema } from '@aztec/foundation/schemas';
+import { createLogger } from '@aztec/foundation/log';
+import type { ApiSchema, Fr } from '@aztec/foundation/schemas';
 import { parseWithOptionals, schemaHasMethod } from '@aztec/foundation/schemas';
 import { NodeListener, TransportServer } from '@aztec/foundation/transport';
+import { ExecutionPayload, Tx } from '@aztec/stdlib/tx';
 
 import { workerData } from 'worker_threads';
 
 import { TestWallet } from './test_wallet.js';
 import { WorkerWalletSchema } from './worker_wallet_schema.js';
 
-const { nodeUrl, pxeConfig } = workerData as { nodeUrl: string; pxeConfig?: Record<string, unknown> };
+const logger = createLogger('e2e:test-wallet:worker');
 
-const node = createAztecNodeClient(nodeUrl);
-const wallet = await TestWallet.create(node, pxeConfig);
+try {
+  const { nodeUrl, pxeConfig } = workerData as { nodeUrl: string; pxeConfig?: Record<string, unknown> };
 
-/** Handlers for methods that need custom implementation (not direct wallet passthrough). */
-const handlers: Record<string, (...args: any[]) => Promise<any>> = {
-  proveTx: async (exec, opts) => {
-    const provenTx = await wallet.proveTx(exec, opts);
-    // ProvenTx has non-serializable fields (node proxy, etc.) — extract only Tx-compatible fields
-    const { data, chonkProof, contractClassLogFields, publicFunctionCalldata } = provenTx;
-    return { data, chonkProof, contractClassLogFields, publicFunctionCalldata };
-  },
-  registerAccount: async (secret, salt) => {
-    const manager = await wallet.createSchnorrAccount(secret, salt);
-    return manager.address;
-  },
-};
+  logger.info('Initializing worker wallet', { nodeUrl });
+  const node = createAztecNodeClient(nodeUrl);
+  const wallet = await TestWallet.create(node, pxeConfig);
+  logger.info('Worker wallet initialized');
 
-const schema = WorkerWalletSchema as ApiSchema;
-const listener = new NodeListener();
-const server = new TransportServer<{ fn: string; args: string }>(listener, async msg => {
-  if (!schemaHasMethod(schema, msg.fn)) {
-    throw new Error(`Unknown method: ${msg.fn}`);
-  }
-  const jsonParams = JSON.parse(msg.args) as unknown[];
-  const args = await parseWithOptionals(jsonParams, schema[msg.fn].parameters());
-  const handler = handlers[msg.fn];
-  const result = handler ? await handler(...args) : await (wallet as any)[msg.fn](...args);
-  return jsonStringify(result);
-});
-server.start();
+  const customMethods = {
+    proveTx: async (exec: ExecutionPayload, opts: Omit<SendOptions, 'wait'>) => {
+      const provenTx = await wallet.proveTx(exec, opts);
+      return new Tx(
+        provenTx.getTxHash(),
+        provenTx.data,
+        provenTx.chonkProof,
+        provenTx.contractClassLogFields,
+        provenTx.publicFunctionCalldata,
+      );
+    },
+    registerAccount: async (secret: Fr, salt: Fr) => {
+      const manager = await wallet.createSchnorrAccount(secret, salt);
+      return manager.address;
+    },
+  };
+
+  const schema = WorkerWalletSchema as ApiSchema;
+  const listener = new NodeListener();
+  const server = new TransportServer<{ fn: string; args: string }>(listener, async msg => {
+    if (!schemaHasMethod(schema, msg.fn)) {
+      throw new Error(`Unknown method: ${msg.fn}`);
+    }
+    const jsonParams = JSON.parse(msg.args) as unknown[];
+    const args: any[] = await parseWithOptionals(jsonParams, schema[msg.fn].parameters());
+    // we have to erase the fn type in order to be able to spread ...args
+    const handler: ((...args: any[]) => Promise<any>) | undefined =
+      msg.fn in customMethods ? customMethods[msg.fn as keyof typeof customMethods] : undefined;
+    const result = handler ? await handler(...args) : await (wallet as any)[msg.fn](...args);
+    return jsonStringify(result);
+  });
+  server.start();
+} catch (err: unknown) {
+  logger.error('Worker wallet initialization failed', { error: err instanceof Error ? err.stack : String(err) });
+  process.exit(1);
+}

package/src/test-wallet/worker_wallet.ts

@@ -19,7 +19,10 @@ import type {
 import type { ChainInfo } from '@aztec/entrypoints/interfaces';
 import type { Fr } from '@aztec/foundation/curves/bn254';
 import { jsonStringify } from '@aztec/foundation/json-rpc';
+import { createLogger } from '@aztec/foundation/log';
+import { promiseWithResolvers } from '@aztec/foundation/promise';
 import type { ApiSchema } from '@aztec/foundation/schemas';
+import { sleep } from '@aztec/foundation/sleep';
 import { NodeConnector, TransportClient } from '@aztec/foundation/transport';
 import type { PXEConfig } from '@aztec/pxe/config';
 import type { ContractArtifact, EventMetadataDefinition, FunctionCall } from '@aztec/stdlib/abi';
@@ -35,6 +38,10 @@ import { WorkerWalletSchema } from './worker_wallet_schema.js';
 
 type WorkerMsg = { fn: string; args: string };
 
+const log = createLogger('e2e:test-wallet:worker-wallet');
+
+const WORKER_READY_TIMEOUT_MS = 120_000;
+
 /**
  * Wallet implementation that offloads all work to a worker thread.
  * Implements the Wallet interface by proxying calls over a transport layer
@@ -53,8 +60,18 @@ export class WorkerWallet implements Wallet {
    * @returns A WorkerWallet ready to use.
    */
   static async create(nodeUrl: string, pxeConfig?: Partial<PXEConfig>): Promise<WorkerWallet> {
-    const worker = new Worker(new URL('./wallet_worker_script.js', import.meta.url), {
+    // replace stc/ with dest/ so the wallet works in Jest tests
+    const workerUrl = new URL('./wallet_worker_script.js', import.meta.url);
+    workerUrl.pathname = workerUrl.pathname.replace('/src/', '/dest/');
+    // remove JEST_WORKER_ID so the worker uses pino-pretty transport instead of Jest's raw output.
+    const { JEST_WORKER_ID: _, ...parentEnv } = process.env;
+    const worker = new Worker(workerUrl, {
       workerData: { nodeUrl, pxeConfig },
+      env: {
+        ...parentEnv,
+        ...(process.stderr.isTTY || process.env.FORCE_COLOR ? { FORCE_COLOR: '1' } : {}),
+        LOG_LEVEL: process.env.WORKER_LOG_LEVEL ?? 'warn',
+      },
     });
 
     const connector = new NodeConnector(worker);
@@ -62,8 +79,39 @@ export class WorkerWallet implements Wallet {
     await client.open();
 
     const wallet = new WorkerWallet(worker, client);
-    // Warmup / readiness check — blocks until the worker has finished creating the TestWallet.
-    await wallet.getChainInfo();
+
+    const { promise: workerDied, reject: rejectWorkerDied } = promiseWithResolvers<void>();
+    // reject if the worker exits or errors before the warmup completes.
+    const onError = (err: Error): void => {
+      worker.off('exit', onExit!);
+      rejectWorkerDied(new Error(`Worker wallet thread error: ${err.message}`));
+    };
+
+    const onExit = (code: number): void => {
+      worker.off('error', onError!);
+      rejectWorkerDied(new Error(`Worker wallet thread exited with code ${code} before becoming ready`));
+    };
+
+    worker.once('error', onError);
+    worker.once('exit', onExit);
+
+    const timeout = sleep(WORKER_READY_TIMEOUT_MS).then(() => {
+      throw new Error(`Worker wallet creation timed out after ${WORKER_READY_TIMEOUT_MS / 1000}s`);
+    });
+
+    try {
+      // wait for worker wallet to start
+      await Promise.race([wallet.getChainInfo(), workerDied, timeout]);
+    } catch (err) {
+      log.error('Worker wallet creation failed, cleaning up', { error: String(err) });
+      client.close();
+      await worker.terminate();
+      throw err;
+    } finally {
+      worker.off('error', onError);
+      worker.off('exit', onExit);
+    }
+
     return wallet;
   }