@aztec/end-to-end 4.0.0-nightly.20260121 → 4.0.0-nightly.20260122

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aztec/end-to-end",
-  "version": "4.0.0-nightly.20260121",
+  "version": "4.0.0-nightly.20260122",
   "type": "module",
   "exports": "./dest/index.js",
   "inherits": [
@@ -25,44 +25,44 @@
     "formatting": "run -T prettier --check ./src && run -T eslint ./src"
   },
   "dependencies": {
-    "@aztec/accounts": "4.0.0-nightly.20260121",
-    "@aztec/archiver": "4.0.0-nightly.20260121",
-    "@aztec/aztec": "4.0.0-nightly.20260121",
-    "@aztec/aztec-node": "4.0.0-nightly.20260121",
-    "@aztec/aztec.js": "4.0.0-nightly.20260121",
-    "@aztec/bb-prover": "4.0.0-nightly.20260121",
-    "@aztec/bb.js": "4.0.0-nightly.20260121",
-    "@aztec/blob-client": "4.0.0-nightly.20260121",
-    "@aztec/blob-lib": "4.0.0-nightly.20260121",
-    "@aztec/bot": "4.0.0-nightly.20260121",
-    "@aztec/cli": "4.0.0-nightly.20260121",
-    "@aztec/constants": "4.0.0-nightly.20260121",
-    "@aztec/entrypoints": "4.0.0-nightly.20260121",
-    "@aztec/epoch-cache": "4.0.0-nightly.20260121",
-    "@aztec/ethereum": "4.0.0-nightly.20260121",
-    "@aztec/foundation": "4.0.0-nightly.20260121",
-    "@aztec/kv-store": "4.0.0-nightly.20260121",
-    "@aztec/l1-artifacts": "4.0.0-nightly.20260121",
-    "@aztec/merkle-tree": "4.0.0-nightly.20260121",
-    "@aztec/node-keystore": "4.0.0-nightly.20260121",
-    "@aztec/noir-contracts.js": "4.0.0-nightly.20260121",
-    "@aztec/noir-noirc_abi": "4.0.0-nightly.20260121",
-    "@aztec/noir-protocol-circuits-types": "4.0.0-nightly.20260121",
-    "@aztec/noir-test-contracts.js": "4.0.0-nightly.20260121",
-    "@aztec/p2p": "4.0.0-nightly.20260121",
-    "@aztec/protocol-contracts": "4.0.0-nightly.20260121",
-    "@aztec/prover-client": "4.0.0-nightly.20260121",
-    "@aztec/prover-node": "4.0.0-nightly.20260121",
-    "@aztec/pxe": "4.0.0-nightly.20260121",
-    "@aztec/sequencer-client": "4.0.0-nightly.20260121",
-    "@aztec/simulator": "4.0.0-nightly.20260121",
-    "@aztec/slasher": "4.0.0-nightly.20260121",
-    "@aztec/stdlib": "4.0.0-nightly.20260121",
-    "@aztec/telemetry-client": "4.0.0-nightly.20260121",
-    "@aztec/test-wallet": "4.0.0-nightly.20260121",
-    "@aztec/validator-client": "4.0.0-nightly.20260121",
-    "@aztec/validator-ha-signer": "4.0.0-nightly.20260121",
-    "@aztec/world-state": "4.0.0-nightly.20260121",
+    "@aztec/accounts": "4.0.0-nightly.20260122",
+    "@aztec/archiver": "4.0.0-nightly.20260122",
+    "@aztec/aztec": "4.0.0-nightly.20260122",
+    "@aztec/aztec-node": "4.0.0-nightly.20260122",
+    "@aztec/aztec.js": "4.0.0-nightly.20260122",
+    "@aztec/bb-prover": "4.0.0-nightly.20260122",
+    "@aztec/bb.js": "4.0.0-nightly.20260122",
+    "@aztec/blob-client": "4.0.0-nightly.20260122",
+    "@aztec/blob-lib": "4.0.0-nightly.20260122",
+    "@aztec/bot": "4.0.0-nightly.20260122",
+    "@aztec/cli": "4.0.0-nightly.20260122",
+    "@aztec/constants": "4.0.0-nightly.20260122",
+    "@aztec/entrypoints": "4.0.0-nightly.20260122",
+    "@aztec/epoch-cache": "4.0.0-nightly.20260122",
+    "@aztec/ethereum": "4.0.0-nightly.20260122",
+    "@aztec/foundation": "4.0.0-nightly.20260122",
+    "@aztec/kv-store": "4.0.0-nightly.20260122",
+    "@aztec/l1-artifacts": "4.0.0-nightly.20260122",
+    "@aztec/merkle-tree": "4.0.0-nightly.20260122",
+    "@aztec/node-keystore": "4.0.0-nightly.20260122",
+    "@aztec/noir-contracts.js": "4.0.0-nightly.20260122",
+    "@aztec/noir-noirc_abi": "4.0.0-nightly.20260122",
+    "@aztec/noir-protocol-circuits-types": "4.0.0-nightly.20260122",
+    "@aztec/noir-test-contracts.js": "4.0.0-nightly.20260122",
+    "@aztec/p2p": "4.0.0-nightly.20260122",
+    "@aztec/protocol-contracts": "4.0.0-nightly.20260122",
+    "@aztec/prover-client": "4.0.0-nightly.20260122",
+    "@aztec/prover-node": "4.0.0-nightly.20260122",
+    "@aztec/pxe": "4.0.0-nightly.20260122",
+    "@aztec/sequencer-client": "4.0.0-nightly.20260122",
+    "@aztec/simulator": "4.0.0-nightly.20260122",
+    "@aztec/slasher": "4.0.0-nightly.20260122",
+    "@aztec/stdlib": "4.0.0-nightly.20260122",
+    "@aztec/telemetry-client": "4.0.0-nightly.20260122",
+    "@aztec/test-wallet": "4.0.0-nightly.20260122",
+    "@aztec/validator-client": "4.0.0-nightly.20260122",
+    "@aztec/validator-ha-signer": "4.0.0-nightly.20260122",
+    "@aztec/world-state": "4.0.0-nightly.20260122",
     "@iarna/toml": "^2.2.5",
     "@jest/globals": "^30.0.0",
     "@noble/curves": "=1.0.0",

package/src/shared/cross_chain_test_harness.ts

@@ -18,7 +18,6 @@ import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses'
 import type { ExtendedViemWalletClient } from '@aztec/ethereum/types';
 import { EpochNumber } from '@aztec/foundation/branded-types';
 import { retryUntil } from '@aztec/foundation/retry';
-import type { FieldsOf } from '@aztec/foundation/types';
 import { TestERC20Abi, TokenPortalAbi, TokenPortalBytecode } from '@aztec/l1-artifacts';
 import { TokenContract } from '@aztec/noir-contracts.js/Token';
 import { TokenBridgeContract } from '@aztec/noir-contracts.js/TokenBridge';
@@ -276,7 +275,7 @@ export class CrossChainTestHarness {
     withdrawAmount: bigint,
     authwitNonce: Fr = Fr.ZERO,
     authWitness: AuthWitness,
-  ): Promise<FieldsOf<TxReceipt>> {
+  ): Promise<TxReceipt> {
     const withdrawReceipt = await this.l2Bridge.methods
       .exit_to_l1_private(this.l2Token.address, this.ethAccount, withdrawAmount, EthAddress.ZERO, authwitNonce)
       .send({ authWitnesses: [authWitness], from: this.ownerAddress })
@@ -285,7 +284,7 @@ export class CrossChainTestHarness {
     return withdrawReceipt;
   }
 
-  async withdrawPublicFromAztecToL1(withdrawAmount: bigint, authwitNonce: Fr = Fr.ZERO): Promise<FieldsOf<TxReceipt>> {
+  async withdrawPublicFromAztecToL1(withdrawAmount: bigint, authwitNonce: Fr = Fr.ZERO): Promise<TxReceipt> {
     const withdrawReceipt = await this.l2Bridge.methods
       .exit_to_l1_public(this.ethAccount, withdrawAmount, EthAddress.ZERO, authwitNonce)
       .send({ from: this.ownerAddress })

package/src/shared/submit-transactions.ts

@@ -23,12 +23,7 @@ export const submitTxsTo = async (
 
       logger.info(`Tx sent with hash ${txHash}`);
       const receipt = await tx.getReceipt();
-      expect(receipt).toEqual(
-        expect.objectContaining({
-          status: TxStatus.PENDING,
-          error: '',
-        }),
-      );
+      expect(receipt.status).toBe(TxStatus.PENDING);
       logger.info(`Receipt received for ${txHash}`);
       txs.push(tx);
     }),

package/src/spartan/tx_metrics.ts

@@ -1,7 +1,7 @@
 import type { AztecNode } from '@aztec/aztec.js/node';
 import type { L2BlockNew } from '@aztec/stdlib/block';
 import type { TopicType } from '@aztec/stdlib/p2p';
-import { Tx, type TxReceipt, TxStatus } from '@aztec/stdlib/tx';
+import { Tx, type TxReceipt } from '@aztec/stdlib/tx';
 
 import { createHistogram } from 'perf_hooks';
 
@@ -24,6 +24,14 @@ export class TxInclusionMetrics {
 
   private p2pGossipLatencyByTopic: Partial<Record<TopicType, { p50: number; p95: number }>> = {};
 
+  private attestationLatency: { p50: number; p95: number } | undefined;
+  private attestationCounts: { success: number; failedBad: number; failedNode: number } | undefined;
+  private reqRespStats: { fraction: number; delayP50: number; delayP95: number } | undefined;
+  private peerStats: { avgCount: number; connectionDurationP50: number; connectionDurationP95: number } | undefined;
+  private mempoolMinedDelay:
+    | { txP50: number; txP95: number; attestationP50: number; attestationP95: number }
+    | undefined;
+
   constructor(private aztecNode: AztecNode) {}
 
   recordSentTx(tx: Tx, group: string): void {
@@ -45,8 +53,8 @@ export class TxInclusionMetrics {
   }
 
   async recordMinedTx(txReceipt: TxReceipt): Promise<void> {
-    const { status, txHash, blockNumber } = txReceipt;
-    if (status !== TxStatus.SUCCESS || !blockNumber) {
+    const { txHash, blockNumber } = txReceipt;
+    if (!txReceipt.isMined() || !txReceipt.hasExecutionSucceeded() || !blockNumber) {
       return;
     }
 
@@ -111,6 +119,26 @@ export class TxInclusionMetrics {
     this.p2pGossipLatencyByTopic[topicName] = { p50, p95 };
   }
 
+  public recordAttestationLatency(p50: number, p95: number): void {
+    this.attestationLatency = { p50, p95 };
+  }
+
+  public recordAttestationCounts(success: number, failedBad: number, failedNode: number): void {
+    this.attestationCounts = { success, failedBad, failedNode };
+  }
+
+  public recordReqRespStats(fraction: number, delayP50: number, delayP95: number): void {
+    this.reqRespStats = { fraction, delayP50, delayP95 };
+  }
+
+  public recordPeerStats(avgCount: number, connectionDurationP50: number, connectionDurationP95: number): void {
+    this.peerStats = { avgCount, connectionDurationP50, connectionDurationP95 };
+  }
+
+  public recordMempoolMinedDelay(txP50: number, txP95: number, attestationP50: number, attestationP95: number): void {
+    this.mempoolMinedDelay = { txP50, txP95, attestationP50, attestationP95 };
+  }
+
   toGithubActionBenchmarkJSON(): Array<{ name: string; unit: string; value: number; range?: number; extra?: string }> {
     const data: Array<{ name: string; unit: string; value: number; range?: number; extra?: string }> = [];
     for (const group of this.groups) {
@@ -148,6 +176,56 @@ export class TxInclusionMetrics {
       });
     }
 
-    return data;
+    if (this.attestationLatency) {
+      data.push(
+        { name: 'attestation_latency/p50', unit: 'ms', value: this.attestationLatency.p50 },
+        { name: 'attestation_latency/p95', unit: 'ms', value: this.attestationLatency.p95 },
+      );
+    }
+
+    if (this.attestationCounts) {
+      const { success, failedBad, failedNode } = this.attestationCounts;
+      const total = success + failedBad + failedNode;
+      const ratio = total > 0 ? success / total : 0;
+      data.push(
+        { name: 'attestation/success_count', unit: 'count', value: success },
+        { name: 'attestation/failed_bad_proposal_count', unit: 'count', value: failedBad },
+        { name: 'attestation/failed_node_issue_count', unit: 'count', value: failedNode },
+        { name: 'attestation/success_ratio', unit: 'ratio', value: ratio },
+      );
+    }
+
+    if (this.reqRespStats) {
+      data.push(
+        { name: 'req_resp/txs_requested_fraction', unit: 'ratio', value: this.reqRespStats.fraction },
+        { name: 'req_resp/delay_p50', unit: 'ms', value: this.reqRespStats.delayP50 },
+        { name: 'req_resp/delay_p95', unit: 'ms', value: this.reqRespStats.delayP95 },
+      );
+    }
+
+    if (this.peerStats) {
+      data.push(
+        { name: 'peers/avg_count', unit: 'peers', value: this.peerStats.avgCount },
+        { name: 'peers/connection_duration_p50', unit: 'ms', value: this.peerStats.connectionDurationP50 },
+        { name: 'peers/connection_duration_p95', unit: 'ms', value: this.peerStats.connectionDurationP95 },
+      );
+    }
+
+    if (this.mempoolMinedDelay) {
+      data.push(
+        { name: 'mempool/tx_mined_delay_p50', unit: 'ms', value: this.mempoolMinedDelay.txP50 },
+        { name: 'mempool/tx_mined_delay_p95', unit: 'ms', value: this.mempoolMinedDelay.txP95 },
+        { name: 'mempool/attestation_mined_delay_p50', unit: 'ms', value: this.mempoolMinedDelay.attestationP50 },
+        { name: 'mempool/attestation_mined_delay_p95', unit: 'ms', value: this.mempoolMinedDelay.attestationP95 },
+      );
+    }
+
+    const scenario = process.env.BENCH_SCENARIO?.trim();
+    if (!scenario) {
+      return data;
+    }
+
+    const scenarioPrefix = `scenario/${scenario}/`;
+    return data.map(entry => ({ ...entry, name: `${scenarioPrefix}${entry.name}` }));
   }
 }

package/src/spartan/utils/bot.ts

@@ -0,0 +1,185 @@
+import type { Logger } from '@aztec/foundation/log';
+import { sleep } from '@aztec/foundation/sleep';
+
+import { exec } from 'child_process';
+import { promisify } from 'util';
+
+import { execHelmCommand, forceDeleteHelmReleaseRecord, getHelmReleaseStatus, hasDeployedHelmRelease } from './helm.js';
+import { deleteResourceByLabel, getChartDir, waitForResourceByLabel } from './k8s.js';
+
+const execAsync = promisify(exec);
+
+export async function restartBot(namespace: string, log: Logger) {
+  log.info(`Restarting bot`);
+  await deleteResourceByLabel({ resource: 'pods', namespace, label: 'app.kubernetes.io/name=bot' });
+  await sleep(10 * 1000);
+  // Some bot images may take time to report Ready due to heavy boot-time proving.
+  // Waiting for PodReadyToStartContainers ensures the pod is scheduled and starting without blocking on full readiness.
+  await waitForResourceByLabel({
+    resource: 'pods',
+    namespace,
+    label: 'app.kubernetes.io/name=bot',
+    condition: 'PodReadyToStartContainers',
+  });
+  log.info(`Bot restarted`);
+}
+
+/**
+ * Installs or upgrades the transfer bot Helm release for the given namespace.
+ * Intended for test setup to enable L2 traffic generation only when needed.
+ */
+export async function installTransferBot({
+  namespace,
+  spartanDir,
+  logger: log,
+  replicas = 1,
+  txIntervalSeconds = 10,
+  followChain = 'PENDING',
+  mnemonic = process.env.LABS_INFRA_MNEMONIC ?? 'test test test test test test test test test test test junk',
+  mnemonicStartIndex,
+  botPrivateKey = process.env.BOT_TRANSFERS_L2_PRIVATE_KEY ?? '0xcafe01',
+  nodeUrl,
+  timeout = '15m',
+  reuseValues = true,
+  aztecSlotDuration = Number(process.env.AZTEC_SLOT_DURATION ?? 12),
+}: {
+  namespace: string;
+  spartanDir: string;
+  logger: Logger;
+  replicas?: number;
+  txIntervalSeconds?: number;
+  followChain?: string;
+  mnemonic?: string;
+  mnemonicStartIndex?: number | string;
+  botPrivateKey?: string;
+  nodeUrl?: string;
+  timeout?: string;
+  reuseValues?: boolean;
+  aztecSlotDuration?: number;
+}) {
+  const instanceName = `${namespace}-bot-transfers`;
+  const helmChartDir = getChartDir(spartanDir, 'aztec-bot');
+  const resolvedNodeUrl = nodeUrl ?? `http://${namespace}-rpc-aztec-node.${namespace}.svc.cluster.local:8080`;
+
+  log.info(`Installing/upgrading transfer bot: replicas=${replicas}, followChain=${followChain}`);
+
+  const values: Record<string, string | number | boolean> = {
+    'bot.replicaCount': replicas,
+    'bot.txIntervalSeconds': txIntervalSeconds,
+    'bot.followChain': followChain,
+    'bot.botPrivateKey': botPrivateKey,
+    'bot.nodeUrl': resolvedNodeUrl,
+    'bot.mnemonic': mnemonic,
+    'bot.feePaymentMethod': 'fee_juice',
+    'aztec.slotDuration': aztecSlotDuration,
+    // Ensure bot can reach its own PXE started in-process (default rpc.port is 8080)
+    // Note: since aztec-bot depends on aztec-node with alias `bot`, env vars go under `bot.node.env`.
+    'bot.node.env.BOT_PXE_URL': 'http://127.0.0.1:8080',
+    // Provide L1 execution RPC for bridging fee juice
+    'bot.node.env.ETHEREUM_HOSTS': `http://${namespace}-eth-execution.${namespace}.svc.cluster.local:8545`,
+    // Provide L1 mnemonic for bridging (falls back to labs mnemonic)
+    'bot.node.env.BOT_L1_MNEMONIC': mnemonic,
+
+    // The bot does not need Kubernetes API access. Disable RBAC + ServiceAccount creation so the chart
+    // can be installed by users without cluster-scoped RBAC permissions.
+    'bot.rbac.create': false,
+    'bot.serviceAccount.create': false,
+    'bot.serviceAccount.name': 'default',
+  };
+  // Ensure we derive a funded L1 key (index 0 is funded on anvil default mnemonic)
+  if (mnemonicStartIndex === undefined) {
+    values['bot.mnemonicStartIndex'] = 0;
+  }
+  // Also pass a funded private key directly if available
+  if (process.env.FUNDING_PRIVATE_KEY) {
+    values['bot.node.env.BOT_L1_PRIVATE_KEY'] = process.env.FUNDING_PRIVATE_KEY;
+  }
+  // Align bot image with the running network image: prefer env var, else detect from a validator pod
+  let repositoryFromEnv: string | undefined;
+  let tagFromEnv: string | undefined;
+  const aztecDockerImage = process.env.AZTEC_DOCKER_IMAGE;
+  if (aztecDockerImage && aztecDockerImage.includes(':')) {
+    const lastColon = aztecDockerImage.lastIndexOf(':');
+    repositoryFromEnv = aztecDockerImage.slice(0, lastColon);
+    tagFromEnv = aztecDockerImage.slice(lastColon + 1);
+  }
+
+  let repository = repositoryFromEnv;
+  let tag = tagFromEnv;
+  if (!repository || !tag) {
+    try {
+      const { stdout } = await execAsync(
+        `kubectl get pods -l app.kubernetes.io/name=validator -n ${namespace} -o jsonpath='{.items[0].spec.containers[?(@.name=="aztec")].image}' | cat`,
+      );
+      const image = stdout.trim().replace(/^'|'$/g, '');
+      if (image && image.includes(':')) {
+        const lastColon = image.lastIndexOf(':');
+        repository = image.slice(0, lastColon);
+        tag = image.slice(lastColon + 1);
+      }
+    } catch (err) {
+      log.warn(`Could not detect aztec image from validator pod: ${String(err)}`);
+    }
+  }
+  if (repository && tag) {
+    values['global.aztecImage.repository'] = repository;
+    values['global.aztecImage.tag'] = tag;
+  }
+  if (mnemonicStartIndex !== undefined) {
+    values['bot.mnemonicStartIndex'] =
+      typeof mnemonicStartIndex === 'string' ? mnemonicStartIndex : Number(mnemonicStartIndex);
+  }
+
+  // If a previous install attempt left the release in a non-deployed state (e.g. FAILED),
+  // `helm upgrade --install` can error with "has no deployed releases".
+  // In that case, clear the release record and do a clean install.
+  const existingStatus = await getHelmReleaseStatus(instanceName, namespace);
+  if (existingStatus && existingStatus.toLowerCase() !== 'deployed') {
+    log.warn(`Transfer bot release ${instanceName} is in status '${existingStatus}'. Reinstalling cleanly.`);
+    await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`).catch(
+      () => undefined,
+    );
+    // If helm left the release in `uninstalling`, force-delete the record so we can reinstall.
+    const afterUninstallStatus = await getHelmReleaseStatus(instanceName, namespace);
+    if (afterUninstallStatus?.toLowerCase() === 'uninstalling') {
+      await forceDeleteHelmReleaseRecord(instanceName, namespace, log);
+    }
+  }
+
+  // `--reuse-values` fails if the release has never successfully deployed (e.g. first install, or a previous failed install).
+  // Only reuse values when we have a deployed release to reuse from.
+  const effectiveReuseValues = reuseValues && (await hasDeployedHelmRelease(instanceName, namespace));
+
+  await execHelmCommand({
+    instanceName,
+    helmChartDir,
+    namespace,
+    valuesFile: undefined,
+    timeout,
+    values: values as unknown as Record<string, string | number | boolean>,
+    reuseValues: effectiveReuseValues,
+  });
+
+  if (replicas > 0) {
+    await waitForResourceByLabel({
+      resource: 'pods',
+      namespace,
+      label: 'app.kubernetes.io/name=bot',
+      condition: 'PodReadyToStartContainers',
+    });
+  }
+}
+
+/**
+ * Uninstalls the transfer bot Helm release from the given namespace.
+ * Intended for test teardown to clean up bot resources.
+ */
+export async function uninstallTransferBot(namespace: string, log: Logger) {
+  const instanceName = `${namespace}-bot-transfers`;
+  log.info(`Uninstalling transfer bot release ${instanceName}`);
+  await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`);
+  // Ensure any leftover pods are removed
+  await deleteResourceByLabel({ resource: 'pods', namespace, label: 'app.kubernetes.io/name=bot' }).catch(
+    () => undefined,
+  );
+}