@aztec/end-to-end 0.0.1-commit.d3ec352c → 0.0.1-commit.d58ff9d0

package/src/p2p/reqresp/utils.ts

@@ -0,0 +1,263 @@
+import type { AztecNodeService } from '@aztec/aztec-node';
+import { createLogger } from '@aztec/aztec.js/log';
+import { waitForTx } from '@aztec/aztec.js/node';
+import { Tx } from '@aztec/aztec.js/tx';
+import { PROPOSER_PIPELINING_SLOT_OFFSET } from '@aztec/epoch-cache';
+import { RollupContract } from '@aztec/ethereum/contracts';
+import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types';
+import { timesAsync } from '@aztec/foundation/collection';
+import { retryUntil } from '@aztec/foundation/retry';
+
+import { expect, jest } from '@jest/globals';
+
+import { getBootNodeUdpPort, shouldCollectMetrics } from '../../fixtures/fixtures.js';
+import { createNodes } from '../../fixtures/setup_p2p_test.js';
+import { P2PNetworkTest } from '../p2p_network.js';
+import { prepareTransactions } from '../shared.js';
+
+// Don't set this to a higher value than 9 because each node will use a different L1 publisher account and anvil seeds
+export const NUM_VALIDATORS = 6;
+export const NUM_TXS_PER_NODE = 4;
+export const BOOT_NODE_UDP_PORT = getBootNodeUdpPort();
+
+type ReqrespOptions = {
+  disableStatusHandshake?: boolean;
+};
+
+export async function createReqrespTest(options: ReqrespOptions = {}): Promise<P2PNetworkTest> {
+  const { disableStatusHandshake = false } = options;
+  const t = await P2PNetworkTest.create({
+    testName: 'e2e_p2p_reqresp_tx',
+    numberOfNodes: 0,
+    numberOfValidators: NUM_VALIDATORS,
+    basePort: BOOT_NODE_UDP_PORT,
+    // To collect metrics - run in aztec-packages `docker compose --profile metrics up`
+    metricsPort: shouldCollectMetrics(),
+    initialConfig: {
+      ethereumSlotDuration: 12,
+      aztecSlotDuration: 36,
+      blockDurationMs: 6000,
+      minTxsPerBlock: 1,
+      maxTxsPerBlock: 2,
+      aztecProofSubmissionEpochs: 1024, // effectively do not reorg
+      ...(disableStatusHandshake ? { p2pDisableStatusHandshake: true } : {}),
+      listenAddress: '127.0.0.1',
+      aztecEpochDuration: 64, // stable committee
+      // Pipelining: target-slot is one ahead of build-slot; inboxLag sources L1->L2
+      // messages from the previous checkpoint to avoid L1ToL2MessagesNotReadyError.
+      inboxLag: 2,
+    },
+  });
+  await t.setup();
+  await t.applyBaseSetup();
+  return t;
+}
+
+export async function cleanupReqrespTest(params: { t: P2PNetworkTest; nodes?: AztecNodeService[] }) {
+  const { t, nodes } = params;
+  if (nodes) {
+    await t.stopNodes(nodes);
+  }
+  await t.teardown();
+}
+
+const getNodePort = (nodeIndex: number) => BOOT_NODE_UDP_PORT + 1 + nodeIndex;
+
+export async function runReqrespTxTest(params: {
+  t: P2PNetworkTest;
+  disableStatusHandshake?: boolean;
+}): Promise<AztecNodeService[]> {
+  const { t, disableStatusHandshake = false } = params;
+
+  if (!t.bootstrapNodeEnr) {
+    throw new Error('Bootstrap node ENR is not available');
+  }
+
+  t.logger.info('Creating nodes');
+  const aztecNodeConfig = disableStatusHandshake
+    ? { ...t.ctx.aztecNodeConfig, p2pDisableStatusHandshake: true }
+    : t.ctx.aztecNodeConfig;
+
+  const nodes = await createNodes(
+    aztecNodeConfig,
+    t.ctx.dateProvider,
+    t.bootstrapNodeEnr,
+    NUM_VALIDATORS,
+    BOOT_NODE_UDP_PORT,
+    t.genesis,
+    t.dataDirFor('validator'),
+    shouldCollectMetrics(),
+  );
+
+  t.logger.info('Waiting for nodes to connect');
+  await t.waitForP2PMeshConnectivity(nodes, NUM_VALIDATORS);
+
+  await t.setupAccount();
+
+  const targetBlockNumber = await t.ctx.aztecNodeService.getBlockNumber();
+  await retryUntil(
+    async () => {
+      const blockNumbers = await Promise.all(nodes.map(node => node.getBlockNumber()));
+      return blockNumbers.every(blockNumber => blockNumber >= targetBlockNumber) ? true : undefined;
+    },
+    `validators to sync to L2 block ${targetBlockNumber}`,
+    60,
+    0.5,
+  );
+
+  t.logger.info('Preparing transactions to send');
+  const txBatches = await timesAsync(2, () =>
+    prepareTransactions(t.logger, t.ctx.aztecNodeService, NUM_TXS_PER_NODE, t.fundedAccount),
+  );
+
+  t.logger.info('Removing initial node');
+  await t.removeInitialNode();
+
+  t.logger.info('Starting fresh slot');
+  const [timestamp] = await t.ctx.cheatCodes.rollup.advanceToNextSlot();
+  t.ctx.dateProvider.setTime(Number(timestamp) * 1000);
+  const startSlotTimestamp = BigInt(timestamp);
+
+  // Under pipelining the active builder during wallclock slot S targets slot S+1, so
+  // we must address the proposer of S+1 (not S) for batch 0. Shift the proposer lookup
+  // window by the pipelining offset so we always send to the currently-building proposer.
+  const proposerSlotOffset = PROPOSER_PIPELINING_SLOT_OFFSET;
+  const { proposerIndexes, nodesToTurnOffTxGossip } = await getProposerIndexes(
+    t,
+    startSlotTimestamp,
+    proposerSlotOffset,
+  );
+  t.logger.info(`Turning off tx gossip for nodes: ${nodesToTurnOffTxGossip.map(getNodePort)}`);
+  t.logger.info(`Sending txs to proposer nodes: ${proposerIndexes.map(getNodePort)}`);
+
+  // Replace the p2p node implementation of some of the nodes with a spy such that it does not store transactions that are gossiped to it
+  // Original implementation of `handleGossipedTx` will store received transactions in the tx pool.
+  // We chose the first 2 nodes that will be the proposers for the next few slots
+  for (const nodeIndex of nodesToTurnOffTxGossip) {
+    const logger = createLogger(`p2p:${getNodePort(nodeIndex)}`);
+    jest.spyOn((nodes[nodeIndex] as any).p2pClient.p2pService, 'handleGossipedTx').mockImplementation(((
+      payloadData: Buffer,
+    ) => {
+      const txHash = Tx.fromBuffer(payloadData).getTxHash();
+      logger.info(`Skipping storage of gossiped transaction ${txHash.toString()}`);
+      return Promise.resolve();
+    }) as any);
+  }
+
+  // We send the tx to the proposer nodes directly, ignoring the pxe and node in each context
+  // We cannot just call tx.send since they were created using a pxe wired to the first node which is now stopped
+  t.logger.info('Sending transactions through proposer nodes');
+  const submittedTxs = await Promise.all(
+    txBatches.map(async (batch, batchIndex) => {
+      const proposerNode = nodes[proposerIndexes[batchIndex]];
+      for (const tx of batch) {
+        t.logger.info(`Tx ${tx.getTxHash().toString()} base64: ${tx.toBuffer().toString('base64')}`);
+      }
+      const txHashes = batch.map(tx => tx.getTxHash().toString());
+      t.logger.info(
+        `Sending batch ${batchIndex} to proposer ${getNodePort(proposerIndexes[batchIndex])}: ${txHashes.join(', ')}`,
+      );
+      await Promise.all(
+        batch.map(async tx => {
+          try {
+            await proposerNode.sendTx(tx);
+          } catch (err) {
+            t.logger.error(`Error sending tx: ${err}`);
+            throw err;
+          }
+        }),
+      );
+      return batch.map(tx => ({ node: proposerNode, txHash: tx.getTxHash() }));
+    }),
+  );
+
+  // Log pool state per node after sending
+  for (let i = 0; i < NUM_VALIDATORS; i++) {
+    const count = await nodes[i].getPendingTxCount();
+    t.logger.info(`Node ${getNodePort(i)} pool has ${count} pending txs`);
+  }
+
+  // Use the test's own aztecSlotDuration (not the env default that p2p_network's
+  // WAIT_FOR_TX_TIMEOUT is derived from) so the timeout scales with this test's 36s slot.
+  // Under pipelining the round-trip is roughly build-slot + target-slot + L1 publish, so
+  // budget for >= 3 slots.
+  const waitForTxTimeout = t.ctx.aztecNodeConfig.aztecSlotDuration * 4.5;
+  t.logger.info('Waiting for all transactions to be mined');
+  await Promise.all(
+    submittedTxs.flatMap((batch, batchIndex) =>
+      batch.map(async (submittedTx, txIndex) => {
+        t.logger.info(`Waiting for tx ${batchIndex}-${txIndex} ${submittedTx.txHash.toString()} to be mined`);
+        await waitForTx(submittedTx.node, submittedTx.txHash, { timeout: waitForTxTimeout });
+        t.logger.info(`Tx ${batchIndex}-${txIndex} ${submittedTx.txHash.toString()} has been mined`);
+      }),
+    ),
+  );
+
+  t.logger.info('All transactions mined');
+
+  // Assert that multiple blocks were built for at least one slot
+  t.logger.info('Verifying multiple blocks for at least one checkpoint');
+  // Wait for L1 checkpoint sync, which may lag behind P2P block propagation.
+  const checkpoints = await retryUntil(
+    async () => {
+      const cps = await nodes[0].getCheckpoints(CheckpointNumber(1), 50, { includeBlocks: true });
+      return cps.length > 0 && cps.some(cp => (cp.blocks?.length ?? 0) >= 2) ? cps : undefined;
+    },
+    'waiting for multi-block checkpoint to sync from L1',
+    30,
+    1,
+  );
+
+  let mbpsFound = false;
+  let expectedBlockNumber = checkpoints[0].blocks![0].number;
+
+  for (const published of checkpoints) {
+    const blocks = published.blocks!;
+    const blockCount = blocks.length;
+    mbpsFound = mbpsFound || blockCount >= 2;
+
+    for (let i = 0; i < blockCount; i++) {
+      const block = blocks[i];
+      expect(block.indexWithinCheckpoint).toBe(i);
+      expect(block.checkpointNumber).toBe(published.number);
+      expect(block.number).toBe(expectedBlockNumber);
+      expectedBlockNumber++;
+    }
+  }
+
+  expect(mbpsFound).toBe(true);
+  return nodes;
+}
+
+async function getProposerIndexes(t: P2PNetworkTest, startSlotTimestamp: bigint, slotOffset = 0) {
+  // Get the nodes for the next set of slots
+  const rollupContract = new RollupContract(
+    t.ctx.deployL1ContractsValues.l1Client,
+    t.ctx.deployL1ContractsValues.l1ContractAddresses.rollupAddress,
+  );
+
+  const attesters = await rollupContract.getAttesters();
+  const startSlot = await rollupContract.getSlotAt(startSlotTimestamp);
+
+  const proposers = await Promise.all(
+    Array.from({ length: 3 }, async (_, i) => {
+      const slot = SlotNumber(startSlot + slotOffset + i);
+      const slotTimestamp = await rollupContract.getTimestampForSlot(slot);
+      return await rollupContract.getProposerAt(slotTimestamp);
+    }),
+  );
+  // Get the indexes of the nodes that are responsible for the next two slots
+  const proposerIndexes = proposers.map(proposer => attesters.findIndex(a => a.equals(proposer)));
+
+  if (proposerIndexes.some(i => i === -1)) {
+    throw new Error(
+      `Proposer index not found for proposer ` +
+        `(proposers=${proposers.map(p => p.toString()).join(',')}, indices=${proposerIndexes.join(',')})`,
+    );
+  }
+
+  const nodesToTurnOffTxGossip = Array.from({ length: NUM_VALIDATORS }, (_, i) => i).filter(
+    i => !proposerIndexes.includes(i),
+  );
+  return { proposerIndexes, nodesToTurnOffTxGossip };
+}

package/src/p2p/shared.ts

@@ -0,0 +1,329 @@
+import type { InitialAccountData } from '@aztec/accounts/testing';
+import type { Archiver } from '@aztec/archiver';
+import type { AztecNodeService } from '@aztec/aztec-node';
+import type { AztecAddress } from '@aztec/aztec.js/addresses';
+import { NO_WAIT, getContractInstanceFromInstantiationParams } from '@aztec/aztec.js/contracts';
+import { Fr } from '@aztec/aztec.js/fields';
+import type { Logger } from '@aztec/aztec.js/log';
+import { TxHash } from '@aztec/aztec.js/tx';
+import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
+import { timesAsync } from '@aztec/foundation/collection';
+import { Signature } from '@aztec/foundation/eth-signature';
+import { retryUntil } from '@aztec/foundation/retry';
+import type { SpamContract } from '@aztec/noir-test-contracts.js/Spam';
+import { TestContract, TestContractArtifact } from '@aztec/noir-test-contracts.js/Test';
+import { getPXEConfig, getPXEConfig as getRpcConfig } from '@aztec/pxe/server';
+import type { SequencerClient } from '@aztec/sequencer-client';
+import { CheckpointAttestation, ConsensusPayload, type TopicType } from '@aztec/stdlib/p2p';
+
+import { expect } from '@jest/globals';
+
+import { shouldCollectMetrics } from '../fixtures/fixtures.js';
+import { SchnorrHardcodedKeyAccountContract } from '../fixtures/schnorr_hardcoded_account_contract.js';
+import { createNodes } from '../fixtures/setup_p2p_test.js';
+import { waitForTxs } from '../fixtures/wait_helpers.js';
+import { type AlertConfig, GrafanaClient } from '../quality_of_service/grafana_client.js';
+import { submitTxsTo } from '../shared/submit-transactions.js';
+import { TestWallet } from '../test-wallet/test_wallet.js';
+import { type ProvenTx, proveInteraction } from '../test-wallet/utils.js';
+import { type P2PNetworkTest, WAIT_FOR_TX_TIMEOUT } from './p2p_network.js';
+
+/** One published checkpoint as returned by the archiver, including its attestations. */
+type PublishedCheckpoint = Awaited<ReturnType<Archiver['getCheckpoints']>>[number];
+
+// submits a set of transactions to the provided Private eXecution Environment (PXE)
+export const submitComplexTxsTo = async (
+  logger: Logger,
+  from: AztecAddress,
+  spamContract: SpamContract,
+  numTxs: number,
+  opts: { callPublic?: boolean } = {},
+) => {
+  const txs: TxHash[] = [];
+
+  const seed = 1234n;
+  const spamCount = 15;
+  for (let i = 0; i < numTxs; i++) {
+    const method = spamContract.methods.spam(seed + BigInt(i * spamCount), spamCount, !!opts.callPublic);
+    const { txHash } = await method.send({ from, wait: NO_WAIT });
+    logger.info(`Tx sent with hash ${txHash.toString()}`);
+    txs.push(txHash);
+  }
+  return txs;
+};
+
+// creates a wallet and submit a given number of transactions through it.
+export const submitTransactions = async (
+  logger: Logger,
+  node: AztecNodeService,
+  numTxs: number,
+  fundedAccount: InitialAccountData,
+): Promise<TxHash[]> => {
+  const rpcConfig = getRpcConfig();
+  rpcConfig.proverEnabled = false;
+  const wallet = await TestWallet.create(
+    node,
+    // Use checkpointed chain tip to avoid anchoring on provisional blocks that the archiver can prune
+    // when their slot ends without a checkpoint landing on L1.
+    { ...getPXEConfig(), proverEnabled: false, syncChainTip: 'checkpointed' },
+    { loggerActorLabel: 'pxe-tx' },
+  );
+  const contract = new SchnorrHardcodedKeyAccountContract();
+  const fundedAccountManager = await wallet.createAccount({
+    secret: fundedAccount.secret,
+    salt: fundedAccount.salt,
+    contract,
+  });
+  return submitTxsTo(wallet, fundedAccountManager.address, numTxs, logger);
+};
+
+export async function prepareTransactions(
+  logger: Logger,
+  node: AztecNodeService,
+  numTxs: number,
+  fundedAccount: InitialAccountData,
+): Promise<ProvenTx[]> {
+  const rpcConfig = getRpcConfig();
+  rpcConfig.proverEnabled = false;
+
+  const wallet = await TestWallet.create(
+    node,
+    { ...getPXEConfig(), proverEnabled: false, syncChainTip: 'checkpointed' },
+    { loggerActorLabel: 'pxe-tx' },
+  );
+  const accountContract = new SchnorrHardcodedKeyAccountContract();
+  const fundedAccountManager = await wallet.createAccount({
+    secret: fundedAccount.secret,
+    salt: fundedAccount.salt,
+    contract: accountContract,
+  });
+
+  const testContractInstance = await getContractInstanceFromInstantiationParams(TestContractArtifact, {
+    salt: Fr.random(),
+  });
+  await wallet.registerContract(testContractInstance, TestContractArtifact);
+  const contract = TestContract.at(testContractInstance.address, wallet);
+
+  return timesAsync(numTxs, async () => {
+    const tx = await proveInteraction(wallet, contract.methods.emit_nullifier(Fr.random()), {
+      from: fundedAccountManager.address,
+    });
+    logger.info(`Tx prepared with hash ${tx.getTxHash()}`);
+    return tx;
+  });
+}
+
+const CHECK_ALERTS = process.env.CHECK_ALERTS === 'true';
+
+const qosAlerts: AlertConfig[] = [
+  {
+    alert: 'SequencerTimeToCollectAttestations',
+    expr: 'aztec_sequencer_time_to_collect_attestations > 3500',
+    labels: { severity: 'error' },
+    for: '10m',
+    annotations: {},
+  },
+];
+
+/** Runs the shared p2p QoS Grafana alert check when CHECK_ALERTS=true; a no-op otherwise. */
+export async function maybeCheckQosAlerts(logger: Logger): Promise<void> {
+  if (CHECK_ALERTS) {
+    const checker = new GrafanaClient(logger);
+    await checker.runAlertCheck(qosAlerts);
+  }
+}
+
+/** Waits until every node's synced block number reaches the initial node's current tip. */
+export async function waitForNodesToSync(t: P2PNetworkTest, nodes: AztecNodeService[]): Promise<void> {
+  const targetBlock = await t.ctx.aztecNode.getBlockNumber();
+  t.logger.warn(`Waiting for all nodes to sync to block number ${targetBlock}`);
+  await retryUntil(
+    async () => {
+      const blockNumbers = await Promise.all(nodes.map(node => node.getBlockNumber()));
+      const checkpointNumber = (await t.monitor.run()).checkpointNumber;
+      t.logger.info(`Current block numbers ${blockNumbers} (checkpoint number on L1 is ${checkpointNumber})`);
+      return blockNumbers.every(bn => bn >= targetBlock);
+    },
+    `nodes to sync to block number ${targetBlock}`,
+    30,
+    0.5,
+  );
+}
+
+/** Reads the published checkpoint containing the block that mined the given tx. */
+export async function getPublishedCheckpointForTx(
+  node: AztecNodeService,
+  txHash: TxHash,
+): Promise<PublishedCheckpoint> {
+  const receipt = await node.getTxReceipt(txHash);
+  const blockNumber = receipt.blockNumber!;
+  const dataStore = node.getBlockSource() as Archiver;
+  const blockData = await dataStore.getBlockData({ number: BlockNumber(blockNumber) });
+  const [publishedCheckpoint] = await dataStore.getCheckpoints({ from: blockData!.checkpointNumber, limit: 1 });
+  return publishedCheckpoint;
+}
+
+/** Polls until the archiver has indexed the first published checkpoint, then returns it. */
+export async function waitForFirstPublishedCheckpoint(
+  t: P2PNetworkTest,
+  nodes: AztecNodeService[],
+  timeoutSeconds = 120,
+): Promise<PublishedCheckpoint> {
+  const dataStore = nodes[0].getBlockSource() as Archiver;
+  t.logger.warn('Waiting for first checkpoint to be published and indexed by the archiver');
+  return await retryUntil(
+    async () => {
+      const blockNumbers = await Promise.all(nodes.map(node => node.getBlockNumber()));
+      const checkpointNumber = (await t.monitor.run()).checkpointNumber;
+      t.logger.info(`Current block numbers ${blockNumbers} (checkpoint number on L1 is ${checkpointNumber})`);
+      const [checkpoint] = await dataStore.getCheckpoints({ from: CheckpointNumber(1), limit: 1 });
+      return checkpoint;
+    },
+    'published checkpoint to be indexed',
+    timeoutSeconds,
+    1,
+  );
+}
+
+/**
+ * Recovers the attestation signers from a published checkpoint and asserts each one belongs to the
+ * validator set formed by the given nodes. Returns the recovered signer addresses so callers can add
+ * scenario-specific assertions (e.g. an exact signer count).
+ */
+export async function verifyAttestationSigners(
+  t: P2PNetworkTest,
+  nodes: AztecNodeService[],
+  publishedCheckpoint: PublishedCheckpoint,
+): Promise<string[]> {
+  const signatureContext = {
+    chainId: t.ctx.aztecNodeConfig.l1ChainId,
+    rollupAddress: t.ctx.deployL1ContractsValues.l1ContractAddresses.rollupAddress,
+  };
+  const payload = ConsensusPayload.fromCheckpoint(publishedCheckpoint.checkpoint, signatureContext);
+  const attestations = publishedCheckpoint.attestations
+    .filter(a => !a.signature.isEmpty())
+    .map(a => new CheckpointAttestation(payload, a.signature, Signature.empty()));
+  const signers = await Promise.all(attestations.map(att => att.getSender()!.toString()));
+  t.logger.info(`Attestation signers`, { signers });
+
+  // Check that the signers found are part of the proposer nodes to ensure the archiver fetched them right
+  const validatorAddresses = nodes.flatMap(node =>
+    (node.getSequencer() as SequencerClient).validatorAddresses?.map(a => a.toString()),
+  );
+  t.logger.info(`Validator addresses`, { addresses: validatorAddresses });
+  for (const signer of signers) {
+    expect(validatorAddresses).toContain(signer);
+  }
+  return signers;
+}
+
+/** Options that parameterize {@link runGossipScenario}. */
+export interface GossipScenarioOptions {
+  /** The initialized P2P network test (setup() + validator registration already done). */
+  t: P2PNetworkTest;
+  /** Number of validator nodes forming the committee. */
+  numValidators: number;
+  /** Base UDP port for the validator nodes. */
+  bootNodePort: number;
+  /** Transactions submitted per validator node; 0 skips tx submission entirely (e.g. the oracle test). */
+  txsPerNode: number;
+  /** Submit tx batches sequentially instead of concurrently. */
+  submitSequentially?: boolean;
+  /** Overrides for the P2P mesh-connectivity wait; unset fields fall back to waitForP2PMeshConnectivity defaults. */
+  mesh?: {
+    expectedNodeCount?: number;
+    timeoutSeconds?: number;
+    checkIntervalSeconds?: number;
+    topics?: TopicType[];
+    minMeshPeerCount?: number;
+  };
+  /** Which published checkpoint the attestation signers are read from (defaults to the first tx's block). */
+  checkpointSource?: 'first-tx' | 'first-published';
+  /** Runs after validator registration but before the validator nodes are created. */
+  beforeCreateNodes?: () => Promise<void>;
+  /** Creates extra non-validator nodes (prover/monitor) once the validator nodes exist. */
+  createExtraNodes?: (nodes: AztecNodeService[]) => Promise<void>;
+  /** Runs after the account is registered but before txs are submitted (sync waits, checkpoint waits, sleeps). */
+  beforeSubmit?: (nodes: AztecNodeService[]) => Promise<void>;
+  /** Scenario-specific verification run after attestation-signer verification (proven block, price convergence). */
+  afterVerify?: (nodes: AztecNodeService[]) => Promise<void>;
+}
+
+/**
+ * Shared skeleton for the p2p gossip tests: create the validator nodes and any extra nodes, wait for
+ * the mesh to form, register the account, optionally submit and mine txs, then verify the attestation
+ * signers of a published checkpoint. Each varying part (validator registration, extra nodes, pre-submit
+ * waits, scenario-specific verification) is supplied via the callbacks in {@link GossipScenarioOptions}.
+ * Returns the validator nodes so the caller can track them for teardown.
+ */
+export async function runGossipScenario(opts: GossipScenarioOptions): Promise<AztecNodeService[]> {
+  const { t, numValidators, bootNodePort, txsPerNode } = opts;
+
+  if (!t.bootstrapNodeEnr) {
+    throw new Error('Bootstrap node ENR is not available');
+  }
+
+  await opts.beforeCreateNodes?.();
+
+  t.logger.info('Creating validator nodes');
+  const nodes = await createNodes(
+    t.ctx.aztecNodeConfig,
+    t.ctx.dateProvider,
+    t.bootstrapNodeEnr,
+    numValidators,
+    bootNodePort,
+    t.genesis,
+    t.dataDirFor('validator'),
+    shouldCollectMetrics(),
+  );
+
+  await opts.createExtraNodes?.(nodes);
+
+  t.logger.info('Waiting for nodes to connect');
+  await t.waitForP2PMeshConnectivity(
+    nodes,
+    opts.mesh?.expectedNodeCount ?? numValidators,
+    opts.mesh?.timeoutSeconds,
+    opts.mesh?.checkIntervalSeconds,
+    opts.mesh?.topics,
+    opts.mesh?.minMeshPeerCount,
+  );
+
+  // We need to create the nodes before we setup the account, because those nodes form the committee
+  // and blocks cannot be built without them (targetCommitteeSize is set to the number of nodes).
+  await t.setupAccount();
+
+  await opts.beforeSubmit?.(nodes);
+
+  let firstTxHash: TxHash | undefined;
+  if (txsPerNode > 0) {
+    t.logger.info('Submitting transactions');
+    const submitOne = (node: AztecNodeService) => submitTransactions(t.logger, node, txsPerNode, t.fundedAccount);
+    // Each submitTransactions call builds its own wallet/PXE, so submissions are independent. When run
+    // concurrently, Promise.all preserves node order so submitted[i] stays aligned with nodes[i].
+    const submitted: TxHash[][] = [];
+    if (opts.submitSequentially) {
+      for (const node of nodes) {
+        submitted.push(await submitOne(node));
+      }
+    } else {
+      submitted.push(...(await Promise.all(nodes.map(submitOne))));
+    }
+    firstTxHash = submitted[0][0];
+
+    t.logger.info('Waiting for transactions to be mined');
+    await waitForTxs(nodes[0], submitted.flat(), { timeout: WAIT_FOR_TX_TIMEOUT });
+    t.logger.info('All transactions mined');
+  }
+
+  const publishedCheckpoint =
+    opts.checkpointSource === 'first-published'
+      ? await waitForFirstPublishedCheckpoint(t, nodes)
+      : await getPublishedCheckpointForTx(nodes[0], firstTxHash!);
+
+  await verifyAttestationSigners(t, nodes, publishedCheckpoint);
+
+  await opts.afterVerify?.(nodes);
+
+  return nodes;
+}

package/src/quality_of_service/{alert_checker.ts → grafana_client.ts}

@@ -25,7 +25,7 @@ const DEFAULT_CONFIG: AlertCheckerConfig = {
   grafanaCredentials: 'admin:admin',
 };
 
-export class AlertChecker {
+export class GrafanaClient {
   private config: AlertCheckerConfig;
   private logger: Logger;