@aztec/end-to-end 5.0.0-rc.1 → 5.0.0

package/src/bench/client_flows/client_flows_benchmark.ts

@@ -13,6 +13,7 @@ import { deployL1Contract } from '@aztec/ethereum/deploy-l1-contract';
 import { ChainMonitor } from '@aztec/ethereum/test';
 import { randomBytes } from '@aztec/foundation/crypto/random';
 import { Fr } from '@aztec/foundation/curves/bn254';
+import { GrumpkinScalar } from '@aztec/foundation/curves/grumpkin';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { TestERC20Abi } from '@aztec/l1-artifacts/TestERC20Abi';
 import { TestERC20Bytecode } from '@aztec/l1-artifacts/TestERC20Bytecode';
@@ -20,15 +21,20 @@ import { AMMContract } from '@aztec/noir-contracts.js/AMM';
 import { FPCContract } from '@aztec/noir-contracts.js/FPC';
 import { FeeJuiceContract } from '@aztec/noir-contracts.js/FeeJuice';
 import { SponsoredFPCContract } from '@aztec/noir-contracts.js/SponsoredFPC';
-import { TokenContract as BananaCoin, TokenContract } from '@aztec/noir-contracts.js/Token';
+import { TestTokenContract as BananaCoin, TestTokenContract } from '@aztec/noir-test-contracts.js/TestToken';
 import { ProtocolContractAddress } from '@aztec/protocol-contracts';
 import { getCanonicalFeeJuice } from '@aztec/protocol-contracts/fee-juice';
 import { type PXEConfig, getPXEConfig } from '@aztec/pxe/server';
 import type { ContractInstanceWithAddress } from '@aztec/stdlib/contract';
 import { Gas, GasSettings } from '@aztec/stdlib/gas';
-import { deriveSigningKey } from '@aztec/stdlib/keys';
+import { AppTaggingSecretKind } from '@aztec/stdlib/logs';
 
-import { AUTOMINE_E2E_OPTS, MNEMONIC, getPaddedMaxFeesPerGas } from '../../fixtures/fixtures.js';
+import {
+  AUTOMINE_E2E_OPTS,
+  L1_DIRECT_WRITE_ACCOUNT_INDEX,
+  MNEMONIC,
+  getPaddedMaxFeesPerGas,
+} from '../../fixtures/fixtures.js';
 import { type EndToEndContext, type SetupOptions, setup, teardown } from '../../fixtures/setup.js';
 import { mintTokensToPrivate } from '../../fixtures/token_utils.js';
 import { setupSponsoredFPC } from '../../fixtures/utils.js';
@@ -71,13 +77,13 @@ export class ClientFlowsBenchmark {
   public bananaFPC!: FPCContract;
   public bananaFPCInstance!: ContractInstanceWithAddress;
   // Random asset we want to trade
-  public candyBarCoin!: TokenContract;
+  public candyBarCoin!: TestTokenContract;
   public candyBarCoinInstance!: ContractInstanceWithAddress;
   // AMM contract
   public amm!: AMMContract;
   public ammInstance!: ContractInstanceWithAddress;
   // Liquidity token for AMM
-  public liquidityToken!: TokenContract;
+  public liquidityToken!: TestTokenContract;
   public liquidityTokenInstance!: ContractInstanceWithAddress;
   // Sponsored FPC contract
   public sponsoredFPC!: SponsoredFPCContract;
@@ -134,7 +140,8 @@ export class ClientFlowsBenchmark {
   async setup() {
     this.logger.info('Setting up subsystems from fresh');
     // Token allowlist entries are test-only: FPC-based fee payment with custom tokens won't work on mainnet alpha.
-    const tokenAllowList = await getTokenAllowedSetupFunctions();
+    // BananaCoin is the codegen'd TestToken here, so the allowlist must key on its class, not canonical Token's.
+    const tokenAllowList = await getTokenAllowedSetupFunctions(BananaCoin.artifact);
     this.context = await setup(2, {
       ...this.setupOptions,
       fundSponsoredFPC: true,
@@ -185,7 +192,7 @@ export class ClientFlowsBenchmark {
 
     let benchysPrivateSigningKey;
     if (type === 'schnorr') {
-      benchysPrivateSigningKey = deriveSigningKey(benchysSecret);
+      benchysPrivateSigningKey = GrumpkinScalar.random();
       return wallet.createSchnorrAccount(benchysSecret, salt, benchysPrivateSigningKey);
     } else if (type === 'ecdsar1') {
       benchysPrivateSigningKey = randomBytes(32);
@@ -225,6 +232,19 @@ export class ClientFlowsBenchmark {
       loggers: {
         prover: this.proxyLogger.createLogger('pxe:bb:wasm:bundle:proxied'),
       },
+      // The benchmark measures steady-state app cost, not first-send discovery cost. Reproduce the pre-handshake-default
+      // behavior of unconstrained delivery: derive the tagging secret from the (sender, recipient) key pair via ECDH
+      // instead of taking the current default (a non-interactive handshake, which injects two extra private app
+      // executions and a nullifier per cold chain). Constrained delivery is unaffected: the Noir circuit rejects
+      // address-derived for constrained, so the hook falls through to a handshake there.
+      hooks: {
+        resolveTaggingSecretStrategy: ({ deliveryMode }) =>
+          Promise.resolve(
+            deliveryMode === AppTaggingSecretKind.UNCONSTRAINED
+              ? { type: 'address-derived' }
+              : { type: 'non-interactive-handshake' },
+          ),
+      },
     });
   }
 
@@ -235,7 +255,15 @@ export class ClientFlowsBenchmark {
     this.feeJuiceBridgeTestHarness = await FeeJuicePortalTestingHarnessFactory.create({
       aztecNode: this.context.aztecNodeService,
       aztecNodeAdmin: this.context.aztecNodeService,
-      l1Client: this.context.deployL1ContractsValues.l1Client,
+      // Bridge from a dedicated L1 account so its direct writes don't race the sequencer publisher's
+      // txs on the deployer account (see L1_DIRECT_WRITE_ACCOUNT_INDEX).
+      l1Client: createExtendedL1Client(
+        this.context.config.l1RpcUrls,
+        MNEMONIC,
+        undefined,
+        undefined,
+        L1_DIRECT_WRITE_ACCOUNT_INDEX,
+      ),
       wallet: this.adminWallet,
       logger: this.logger,
     });
@@ -259,7 +287,7 @@ export class ClientFlowsBenchmark {
 
   async applyDeployCandyBarToken() {
     this.logger.info('Applying candy bar token deployment');
-    const { contract: candyBarCoin, instance: candyBarCoinInstance } = await TokenContract.deploy(
+    const { contract: candyBarCoin, instance: candyBarCoinInstance } = await TestTokenContract.deploy(
       this.adminWallet,
       this.adminAddress,
       'CBC',
@@ -329,7 +357,6 @@ export class ClientFlowsBenchmark {
 
   public async createAndFundBenchmarkingAccountOnUserWallet(accountType: AccountType) {
     const benchysAccountManager = await this.createBenchmarkingAccountManager(this.adminWallet, accountType);
-    const benchysAccount = await benchysAccountManager.getAccount();
     const benchysAddress = benchysAccountManager.address;
     const claim = await this.feeJuiceBridgeTestHarness.prepareTokensOnL1(benchysAddress);
     const behchysDeployMethod = await benchysAccountManager.getDeployMethod();
@@ -339,7 +366,7 @@ export class ClientFlowsBenchmark {
     });
     // Register benchy on the user's Wallet, where we're going to be interacting from
     const accountManager = await this.userWallet.createAccount({
-      secret: benchysAccount.getSecretKey(),
+      secret: benchysAccountManager.getSecretKey(),
       salt: new Fr(benchysAccountManager.getInstance().salt),
       contract: benchysAccountManager.getAccountContract(),
     });
@@ -348,7 +375,7 @@ export class ClientFlowsBenchmark {
 
   public async applyDeployAmm() {
     this.logger.info('Applying AMM deployment');
-    const { contract: liquidityToken, instance: liquidityTokenInstance } = await TokenContract.deploy(
+    const { contract: liquidityToken, instance: liquidityTokenInstance } = await TestTokenContract.deploy(
       this.adminWallet,
       this.adminAddress,
       'LPT',

package/src/composed/ha/ha_full_setup.ts

@@ -0,0 +1,466 @@
+/**
+ * Shared setup for the docker-compose HA full suite.
+ *
+ * Stands up the complete HA cluster used by `e2e_ha_full.parallel.test.ts` and
+ * `e2e_ha_distribute_work.test.ts`: a bootstrap RPC/P2P node plus NODE_COUNT in-proc
+ * `AztecNodeService` HA peers that share one PostgreSQL slashing-protection DB and a Web3Signer keystore.
+ * Requires the docker-compose HA suite (run_test.sh ha): live Postgres (DATABASE_URL) and Web3Signer
+ * sidecar.
+ *
+ * The suite is split across two files because the "distribute work" test kills nodes as it runs, leaving
+ * the cluster unusable; giving it its own file (its own cluster) removes the previous "must run last"
+ * ordering contract without changing what any test asserts.
+ */
+import { type AztecNodeConfig, AztecNodeService, createAztecNodeService } from '@aztec/aztec-node';
+import { AztecAddress, EthAddress } from '@aztec/aztec.js/addresses';
+import { NO_WAIT, getContractInstanceFromInstantiationParams } from '@aztec/aztec.js/contracts';
+import { Fr } from '@aztec/aztec.js/fields';
+import type { Logger } from '@aztec/aztec.js/log';
+import { type AztecNode, waitForTx } from '@aztec/aztec.js/node';
+import { GovernanceProposerContract } from '@aztec/ethereum/contracts';
+import type { DeployAztecL1ContractsReturnType } from '@aztec/ethereum/deploy-aztec-l1-contracts';
+import { SecretValue } from '@aztec/foundation/config';
+import { withLoggerBindings } from '@aztec/foundation/log/server';
+import { retryUntil } from '@aztec/foundation/retry';
+import type { TestDateProvider } from '@aztec/foundation/timer';
+import { TestContract } from '@aztec/noir-test-contracts.js/Test';
+import { TopicType } from '@aztec/stdlib/p2p';
+import { TxHash, type TxReceipt, TxStatus } from '@aztec/stdlib/tx';
+import type { GenesisData } from '@aztec/stdlib/world-state';
+
+import getPort, { portNumbers } from 'get-port';
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Pool } from 'pg';
+
+import { PIPELINING_SETUP_OPTS } from '../../fixtures/fixtures.js';
+import {
+  type HADatabaseConfig,
+  cleanupHADatabase,
+  createHADatabaseConfig,
+  createInitialValidatorsFromPrivateKeys,
+  getAddressesFromPrivateKeys,
+  setupHADatabase,
+} from '../../fixtures/ha_setup.js';
+import { getPrivateKeyFromIndex, setup } from '../../fixtures/utils.js';
+import {
+  createWeb3SignerKeystore,
+  getWeb3SignerTestKeystoreDir,
+  getWeb3SignerUrl,
+  refreshWeb3Signer,
+} from '../../fixtures/web3signer.js';
+import type { TestWallet } from '../../test-wallet/test_wallet.js';
+import { proveInteraction } from '../../test-wallet/utils.js';
+
+export const NODE_COUNT = 5;
+export const VALIDATOR_COUNT = 4;
+export const COMMITTEE_SIZE = 4;
+
+// Allocate p2p listen ports from above the OS ephemeral range (Linux default tops out at 60999) so they
+// never collide with an ephemeral socket the OS may already have handed out -- e.g. the in-process prover
+// node (which listens on p2pPort 0) or any outbound connection. The previous fixed 4040x ports sat inside
+// the ephemeral range, so an ephemeral socket occasionally held a node's port at bind time, surfacing as
+// libp2p ERR_NO_VALID_ADDRESSES and aborting beforeAll. get-port also locks each returned port briefly, so
+// concurrent calls within this process never hand back the same one.
+const getFreeP2PPort = () => getPort({ port: portNumbers(61000, 65535) });
+
+export async function registerTestContract(wallet: TestWallet): Promise<TestContract> {
+  const instance = await getContractInstanceFromInstantiationParams(TestContract.artifact, {
+    constructorArgs: [],
+    constructorArtifact: undefined,
+    salt: Fr.ZERO,
+    publicKeys: undefined,
+    deployer: undefined,
+  });
+  await wallet.registerContract(instance, TestContract.artifact);
+  return TestContract.at(instance.address, wallet);
+}
+
+export async function submitTriggerTx(
+  wallet: TestWallet,
+  testContract: TestContract,
+  from: AztecAddress,
+): Promise<TxHash> {
+  const tx = await proveInteraction(wallet, testContract.methods.emit_nullifier(Fr.random()), { from });
+  return await tx.send({ wait: NO_WAIT });
+}
+
+export async function waitForTriggerTx(node: AztecNode, txHash: TxHash): Promise<TxReceipt> {
+  const receipt = await waitForTx(node, txHash, { waitForStatus: TxStatus.CHECKPOINTED });
+  if (!receipt.blockNumber) {
+    throw new Error('Trigger tx was checkpointed without a block number');
+  }
+  return receipt;
+}
+
+/**
+ * Owns the full HA cluster lifecycle and the shared helpers the two HA test files drive it with. Each
+ * test file constructs one instance and calls {@link setup} in `beforeAll` / {@link teardown} in
+ * `afterAll`; state fields are populated by {@link setup} and read directly by the tests.
+ */
+export class HaFullTestContext {
+  logger!: Logger;
+  wallet!: TestWallet;
+  ownerAddress!: AztecAddress;
+  testContract!: TestContract;
+  aztecNode!: AztecNode;
+  config!: AztecNodeConfig;
+  accounts!: AztecAddress[];
+  dateProvider!: TestDateProvider;
+  genesis: GenesisData | undefined;
+
+  haNodePools!: Pool[]; // Database pools for HA nodes (for cleanup)
+  haNodeServices!: AztecNodeService[]; // All N HA peer nodes
+  haKeystoreDirs!: string[];
+  mainPool!: Pool;
+  databaseConfig!: HADatabaseConfig;
+  attesterPrivateKeys!: `0x${string}`[];
+  attesterAddresses!: string[];
+  publisherPrivateKeys!: `0x${string}`[];
+  publisherAddresses!: string[];
+  web3SignerUrl!: string;
+  deployL1ContractsValues!: DeployAztecL1ContractsReturnType;
+  governanceProposer!: GovernanceProposerContract;
+  /** Per-node initial keystore JSON (all 4 attesters, node's own publisher) for restore after reload test */
+  initialKeystoreJsons!: string[];
+
+  private teardownBootstrap: () => Promise<void> = async () => {};
+  private haSequencersStarted = false;
+  private readonly stoppedHANodeIndexes = new Set<number>();
+
+  getSignatureContext = () => ({
+    chainId: this.config.l1ChainId,
+    rollupAddress: this.deployL1ContractsValues.l1ContractAddresses.rollupAddress,
+  });
+
+  startHASequencers = async () => {
+    if (this.haSequencersStarted) {
+      return;
+    }
+
+    await Promise.all(
+      this.haNodeServices.map(async (service, i) => {
+        this.logger.info(`Starting HA peer node ${i} sequencer`);
+        await service.getSequencer()?.start();
+      }),
+    );
+    this.haSequencersStarted = true;
+    this.logger.info('All HA peer sequencers started');
+  };
+
+  sendTriggerTx = async (): Promise<TxReceipt> => {
+    await this.startHASequencers();
+    const txHash = await submitTriggerTx(this.wallet, this.testContract, this.ownerAddress);
+    return await waitForTriggerTx(this.aztecNode, txHash);
+  };
+
+  stopHANode = async (nodeIndex: number) => {
+    if (this.stoppedHANodeIndexes.has(nodeIndex)) {
+      return;
+    }
+
+    this.logger.info(`Stopping HA peer node ${nodeIndex}`);
+    await this.haNodeServices[nodeIndex].stop();
+    this.stoppedHANodeIndexes.add(nodeIndex);
+  };
+
+  async setup(): Promise<void> {
+    // Check required environment variables
+    if (!process.env.DATABASE_URL) {
+      throw new Error('DATABASE_URL environment variable must be set for HA tests');
+    }
+
+    this.web3SignerUrl = getWeb3SignerUrl();
+    if (!this.web3SignerUrl) {
+      throw new Error('WEB3_SIGNER_URL environment variable must be set for HA tests');
+    }
+
+    // Setup database configuration
+    this.databaseConfig = createHADatabaseConfig('ha-full-test');
+
+    // Connect to database (migrations already run by docker-compose entrypoint)
+    this.mainPool = setupHADatabase(this.databaseConfig.databaseUrl.getValue()!);
+
+    this.attesterPrivateKeys = Array.from(
+      { length: VALIDATOR_COUNT },
+      (_, i) => `0x${getPrivateKeyFromIndex(i)!.toString('hex')}` as `0x${string}`,
+    );
+
+    this.publisherPrivateKeys = Array.from(
+      { length: NODE_COUNT },
+      (_, i) => `0x${getPrivateKeyFromIndex(i + VALIDATOR_COUNT)!.toString('hex')}` as `0x${string}`,
+    );
+
+    const web3SignerDir = getWeb3SignerTestKeystoreDir();
+    const allKeys = [...this.attesterPrivateKeys, ...this.publisherPrivateKeys];
+    for (const key of allKeys) {
+      await createWeb3SignerKeystore(web3SignerDir, key);
+    }
+
+    this.attesterAddresses = getAddressesFromPrivateKeys(this.attesterPrivateKeys);
+
+    this.publisherAddresses = getAddressesFromPrivateKeys(this.publisherPrivateKeys);
+
+    // Refresh Web3Signer to load all the keys (attesters + publishers)
+    await refreshWeb3Signer(this.web3SignerUrl, ...this.attesterAddresses, ...this.publisherAddresses);
+
+    // Create database pools for HA nodes
+    this.haNodePools = Array.from({ length: NODE_COUNT }, () => {
+      const pool = new Pool({ connectionString: this.databaseConfig.databaseUrl.getValue()! });
+      // pg-pool re-emits idle-client errors on the pool; with no listener the emit throws - in production this
+      // crashes the validator process.
+      pool.on('error', (err: Error) => this.logger?.warn(`HA node pool error: ${err.message}`));
+      return pool;
+    });
+
+    const initialValidators = createInitialValidatorsFromPrivateKeys(this.attesterPrivateKeys);
+
+    const bootstrapP2PPort = await getFreeP2PPort();
+
+    ({
+      teardown: this.teardownBootstrap,
+      logger: this.logger,
+      wallet: this.wallet,
+      aztecNode: this.aztecNode,
+      config: this.config,
+      accounts: this.accounts,
+      dateProvider: this.dateProvider,
+      deployL1ContractsValues: this.deployL1ContractsValues,
+      genesis: this.genesis,
+    } = await setup(
+      // A single default initializerless account, created/funded/registered by setup with no on-chain
+      // deploy tx -- the bootstrap node can't build blocks (disableValidator), so the owner must be usable
+      // without one.
+      1,
+      {
+        ...PIPELINING_SETUP_OPTS,
+        automineL1Setup: true,
+        initialValidators,
+        sequencerPublisherPrivateKeys: [new SecretValue(this.publisherPrivateKeys[0])],
+        aztecTargetCommitteeSize: COMMITTEE_SIZE,
+        // The full HA docker/Web3Signer stack can still be joining and syncing after the shared
+        // 12s pipelining preset's 2.5s start window has closed. Keep real sequencing, but give
+        // HA validators enough time to pass the enforced build-start gate in CI.
+        aztecSlotDuration: 16,
+        // This suite validates HA coordination on tx-bearing checkpoints. Requiring one tx avoids a startup empty
+        // checkpoint from occupying the shared HA publisher while the trigger tx is still being prepared.
+        minTxsPerBlock: 1,
+        archiverPollingIntervalMS: 200,
+        sequencerPollingIntervalMS: 200,
+        worldStateBlockCheckIntervalMS: 200,
+        blockCheckIntervalMS: 200,
+        startProverNode: true,
+        // The bootstrap node is only an RPC/P2P anchor. HA validators are the first block producers in this suite.
+        disableValidator: true,
+        // Enable P2P for transaction gossip
+        p2pEnabled: true,
+        // Bind the bootstrap node above the ephemeral range too (see getFreeP2PPort), so it can't lose
+        // its port to an ephemeral socket and abort the whole suite before any HA node is created. Set
+        // the broadcast port explicitly to the same value: discv5 otherwise defaults p2pBroadcastPort to
+        // p2pPort by mutating this config object in place, and that mutated value would then leak into the
+        // HA nodes' configs below (built by spreading `config`), making them advertise the wrong port.
+        p2pPort: bootstrapP2PPort,
+        p2pBroadcastPort: bootstrapP2PPort,
+        // Enable slashing for testing governance + slashing vote coordination
+        slasherEnabled: true,
+        slashingRoundSizeInEpochs: 1, // 32 slots (1 epoch)
+        slashingQuorum: 17, // >50% of 32 slots for tally quorum,
+      },
+      { syncChainTip: 'proven' },
+    ));
+
+    this.ownerAddress = this.accounts[0];
+    this.testContract = await registerTestContract(this.wallet);
+
+    if (!this.dateProvider) {
+      throw new Error('dateProvider must be provided by setup for HA tests');
+    }
+
+    this.logger.info(
+      'Bootstrap node setup complete; funded initializerless account and test contract registered locally',
+    );
+
+    // Get bootstrap node's P2P ENR for HA nodes to connect to
+    const bootstrapNodeEnr = await this.aztecNode.getEncodedEnr();
+    if (!bootstrapNodeEnr) {
+      throw new Error('Failed to get bootstrap node ENR - P2P may not be enabled');
+    }
+    this.logger.info(`Bootstrap node ENR: ${bootstrapNodeEnr}`);
+
+    // L1 contract wrappers for querying votes
+    this.governanceProposer = new GovernanceProposerContract(
+      this.deployL1ContractsValues.l1Client,
+      this.deployL1ContractsValues.l1ContractAddresses.governanceProposerAddress.toString(),
+    );
+    this.logger.info('L1 contract wrappers initialized');
+
+    this.haNodeServices = [];
+    this.haKeystoreDirs = [];
+    this.logger.info(`Starting ${NODE_COUNT} HA peer nodes...`);
+
+    // Per-node keystore: all attesters but only this node's publisher to avoid nonce conflicts.
+    // When keyStoreDirectory is set the node loads validators/publishers from file only, so we omit them from config.
+    this.initialKeystoreJsons = [];
+
+    for (let i = 0; i < NODE_COUNT; i++) {
+      const nodeId = `${this.databaseConfig.nodeId}-${i + 1}`;
+      this.logger.info(`Starting HA peer node ${i} with nodeId: ${nodeId}`);
+
+      const keystoreContent = {
+        schemaVersion: 1,
+        validators: [
+          {
+            attester: this.attesterAddresses,
+            feeRecipient: AztecAddress.ZERO.toString(),
+            coinbase: EthAddress.fromString(this.attesterAddresses[0]).toChecksumString(),
+            remoteSigner: this.web3SignerUrl,
+            publisher: [this.publisherAddresses[i]],
+          },
+        ],
+      };
+      const keystoreJson = JSON.stringify(keystoreContent, null, 2);
+      this.initialKeystoreJsons.push(keystoreJson);
+
+      const keystoreDir = await mkdtemp(join(tmpdir(), `ha-keystore-${i}-`));
+      this.haKeystoreDirs.push(keystoreDir);
+      await writeFile(join(keystoreDir, 'keystore.json'), keystoreJson);
+
+      const dataDirectory = this.config.dataDirectory ? `${this.config.dataDirectory}-${i}` : undefined;
+
+      const nodeP2PPort = await getFreeP2PPort();
+      const nodeConfig: AztecNodeConfig = {
+        ...this.config,
+        nodeId,
+        keyStoreDirectory: keystoreDir,
+        // Ensure txs are included in proposals to test full signing path
+        publishTxsWithProposals: true,
+        dataDirectory,
+        databaseUrl: this.databaseConfig.databaseUrl,
+        pollingIntervalMs: this.databaseConfig.pollingIntervalMs,
+        peerSigningTimeoutMs: this.databaseConfig.peerSigningTimeoutMs,
+        maxStuckDutiesAgeMs: this.databaseConfig.maxStuckDutiesAgeMs,
+        haSigningEnabled: true,
+        disableValidator: false,
+        // Enable P2P for transaction and block gossip
+        p2pEnabled: true,
+        // Each HA node gets its own free port above the ephemeral range. Override the broadcast port too:
+        // `...config` carries the bootstrap node's broadcast port (discv5 sets it in place), which would
+        // otherwise make every HA node advertise the bootstrap's port instead of its own.
+        p2pPort: nodeP2PPort,
+        p2pBroadcastPort: nodeP2PPort,
+        // Connect to bootstrap node for tx gossip
+        bootstrapNodes: [bootstrapNodeEnr],
+        web3SignerUrl: this.web3SignerUrl,
+      };
+
+      const nodeService = await withLoggerBindings({ actor: `HA-${i}` }, async () => {
+        return await createAztecNodeService(
+          nodeConfig,
+          { dateProvider: this.dateProvider },
+          { genesis: this.genesis, dontStartSequencer: true },
+        );
+      });
+
+      this.haNodeServices.push(nodeService);
+      this.logger.info(`HA peer node ${i} started successfully`);
+    }
+
+    this.logger.info(`All ${NODE_COUNT} HA peer nodes started and coordinating via PostgreSQL database`);
+    this.logger.info('Waiting for HA peer nodes to join the tx gossip mesh');
+    await retryUntil(
+      async () => {
+        const meshStates = await Promise.all(
+          this.haNodeServices.map(async (service, nodeIndex) => {
+            const p2p = service.getP2P();
+            const [peers, txMeshPeerCount] = await Promise.all([
+              p2p.getPeers(),
+              p2p.getGossipMeshPeerCount(TopicType.tx),
+            ]);
+
+            return { nodeIndex, peerCount: peers.length, txMeshPeerCount };
+          }),
+        );
+
+        this.logger.debug('HA tx gossip mesh status', { meshStates });
+        return meshStates.every(({ peerCount, txMeshPeerCount }) => peerCount > 0 && txMeshPeerCount > 0)
+          ? true
+          : undefined;
+      },
+      'HA tx gossip mesh readiness',
+      60,
+      1,
+    );
+
+    // The owner is an initializerless account, so it needs no deployment tx -- it was funded at genesis
+    // and registered during setup, and is ready to transact as soon as the HA nodes start building blocks.
+    this.logger.info(`Test account ready at ${this.ownerAddress}`);
+  }
+
+  async teardown(): Promise<void> {
+    // Stop all sequencers before tearing down the nodes: a sequencer stop awaits its in-flight
+    // iteration, which can spend tens of seconds finishing a vote or checkpoint publish on L1.
+    // Stops must be awaited fully — jest runs without forceExit, so a node abandoned mid-stop
+    // outlives the test environment and keeps the worker process alive until the CI job timeout.
+    // The dateProvider reset must wait until nodes are stopped: it rewinds the shared clock from
+    // chain time to wall time (minutes apart after the automine deploy burst), and any publisher
+    // deadline armed against the rewound clock would block shutdown until wall time catches up.
+    if (this.haNodeServices) {
+      await Promise.allSettled(
+        this.haNodeServices.map(async (service, i) => {
+          try {
+            await service.getSequencer()?.stop();
+          } catch (error) {
+            this.logger.error(`Failed to stop sequencer of HA peer node ${i}: ${error}`);
+          }
+        }),
+      );
+      await Promise.allSettled(
+        this.haNodeServices.map((_, i) =>
+          this.stopHANode(i).catch(error => {
+            this.logger.error(`Failed to stop HA peer node ${i}: ${error}`);
+          }),
+        ),
+      );
+    }
+
+    this.dateProvider?.reset();
+
+    // Cleanup HA keystore temp directories
+    if (this.haKeystoreDirs) {
+      for (let i = 0; i < this.haKeystoreDirs.length; i++) {
+        try {
+          await rm(this.haKeystoreDirs[i], { recursive: true });
+        } catch (error) {
+          this.logger.error(`Failed to remove HA keystore dir ${i}: ${error}`);
+        }
+      }
+    }
+
+    // Cleanup HA resources (database pools, etc.)
+    if (this.haNodePools) {
+      for (const pool of this.haNodePools) {
+        try {
+          await pool.end();
+        } catch (error) {
+          this.logger.error(`Failed to close HA node pool: ${error}`);
+        }
+      }
+    }
+    await cleanupHADatabase(this.mainPool, this.logger);
+    await this.mainPool.end();
+
+    // Cleanup bootstrap node and test infrastructure (this cleans up the shared data directory)
+    await this.teardownBootstrap();
+  }
+
+  /** Clean up database state between tests. */
+  async resetDutiesTable(): Promise<void> {
+    try {
+      await this.mainPool.query('DELETE FROM validator_duties');
+    } catch (error) {
+      // Ignore cleanup errors (table might not exist on first run failure)
+      this.logger?.warn(`Failed to clean up validator_duties: ${error}`);
+    }
+  }
+}