@aztec/end-to-end 0.0.1-commit.d3ec352c → 0.0.1-commit.d58ff9d0

package/src/legacy-jest-resolver.cjs

@@ -0,0 +1,112 @@
+// Custom Jest resolver. When CONTRACT_ARTIFACTS_VERSION is set, redirects *only* JSON artifact files under
+// @aztec/noir-contracts.js/artifacts/, @aztec/noir-test-contracts.js/artifacts/, and @aztec/accounts/artifacts/ to a local cache of the pinned
+// legacy versions. TypeScript wrapper classes (e.g. Token.ts) continue to load from the current workspace and use the
+// current @aztec/aztec.js — only the artifact JSON (the deployed-contract ABI / bytecode / notes surface) is swapped.
+//
+// Why JSON-only: the JSON artifact is the actual interchange surface a "deployed contract" exposes. The TS wrapper is
+// generated client-side ergonomics that's tightly coupled to the current @aztec/aztec.js API. Redirecting the wrapper
+// would couple this test to a moving aztec.js surface and break at import time on unrelated breaking changes; we want
+// to fail only on actual artifact-compat regressions.
+//
+// Cache population lives in install_legacy_contracts.cjs — invoked lazily here for local dev, and eagerly
+// by bootstrap.sh ci-compat-e2e before hermetic test containers (which run with --net=none) launch.
+//
+// Missing artifacts: legacy version directories are immutable, so an artifact missing from the cache means the
+// contract was added after the pinned release — there's nothing to compat-test. Rather than failing or silently
+// falling back to the workspace artifact (which would turn the compat run into a regular e2e run that always
+// passes), we log the miss and exit the process cleanly with code 0. The test never runs, but the per-test CI
+// log captures the explanatory line so the reason is auditable. This keeps the change scoped to this resolver,
+// avoiding a new exit-code contract in the shared ci3 test runner.
+//
+// Activated by env var; passthrough otherwise.
+/* eslint-disable @typescript-eslint/no-require-imports */
+
+const path = require('path');
+const fs = require('fs');
+const { installLegacyContracts, REDIRECTED, cacheRoot } = require('./install_legacy_contracts.cjs');
+
+const version = process.env.CONTRACT_ARTIFACTS_VERSION;
+const cacheDir = version ? cacheRoot(version) : null;
+
+function pkgJsonPath(name) {
+  return path.join(cacheDir, 'node_modules', name, 'package.json');
+}
+
+// Kept in a separate module (not inlined) because bootstrap.sh ci-compat-e2e also calls it directly
+// via `node .../install_legacy_contracts.cjs <version>` to pre-populate the cache on the host before
+// hermetic --net=none test containers launch. Inlining here would force us to duplicate the logic
+// in bash or re-run jest just to trigger the install.
+if (version) {
+  installLegacyContracts(version);
+}
+
+let bannerPrinted = false;
+const seen = new Set();
+
+function printBannerOnce() {
+  if (bannerPrinted || !version) {
+    return;
+  }
+  bannerPrinted = true;
+  const lines = ['='.repeat(60), `[legacy-contracts][jest] CONTRACT_ARTIFACTS_VERSION=${version}`];
+  for (const p of REDIRECTED) {
+    const v = JSON.parse(fs.readFileSync(pkgJsonPath(p), 'utf8')).version;
+    if (v !== version) {
+      throw new Error(`[legacy-contracts] ${p} on disk is ${v}, expected ${version}`);
+    }
+    lines.push(`[legacy-contracts][jest] redirecting ${p}/artifacts/*.json -> .legacy-contracts/${version}/...`);
+  }
+  lines.push('='.repeat(60));
+  process.stderr.write(lines.join('\n') + '\n');
+}
+
+// Match a resolved absolute path against the workspace artifacts dirs and return the legacy cache equivalent, or null
+// if it's not an artifact path we should redirect.
+function legacyArtifactPath(resolved) {
+  if (!resolved.endsWith('.json')) {
+    return null;
+  }
+  for (const pkg of REDIRECTED) {
+    // pkg = '@aztec/noir-contracts.js' -> match '/noir-contracts.js/artifacts/'
+    const dirName = pkg.split('/')[1];
+    const marker = `/${dirName}/artifacts/`;
+    const idx = resolved.indexOf(marker);
+    if (idx === -1) {
+      continue;
+    }
+    const basename = resolved.slice(idx + marker.length);
+    return path.join(cacheDir, 'node_modules', pkg, 'artifacts', basename);
+  }
+  return null;
+}
+
+module.exports = function legacyResolver(request, options) {
+  // Always run the default resolver first. We only inspect (and possibly rewrite) the *result*; this catches both
+  // bare-specifier imports of `@aztec/noir-contracts.js/artifacts/foo.json` and the relative `../artifacts/foo.json`
+  // imports inside the workspace TS wrapper classes — both resolve to the same workspace artifact path that we then
+  // redirect.
+  const resolved = options.defaultResolver(request, options);
+  if (!version) {
+    return resolved;
+  }
+  printBannerOnce();
+  const legacy = legacyArtifactPath(resolved);
+  if (!legacy) {
+    return resolved;
+  }
+  if (!fs.existsSync(legacy)) {
+    // Contract was added after this historical release, there is nothing to compat-test for it. Exit the process
+    // cleanly with code 0 so the test runner reports the run as passed.
+    fs.writeSync(
+      2,
+      `[legacy-contracts][jest] artifact ${path.basename(legacy)} not in legacy cache @${version}; ` +
+        `assumed added after this release. No compat coverage applicable for this version, treating as passed.\n`,
+    );
+    process.exit(0);
+  }
+  if (!seen.has(resolved)) {
+    seen.add(resolved);
+    process.stderr.write(`[legacy-contracts][jest] redirected ${path.basename(legacy)} -> ${legacy}\n`);
+  }
+  return legacy;
+};

package/src/multi-node/README.md

@@ -0,0 +1,95 @@
+# `multi-node` e2e test category
+
+Multi-node tests run N validator nodes sharing an in-memory `MockGossipSubNetwork` bus (no real
+libp2p). This is the category for any multi-node test whose subject — proposals, attestations,
+checkpointing, pruning/recovery, offense detection — is faithfully reproduced by the mock-gossip bus.
+Tests that need *real networking* (peer discovery / discv5, the req/resp protocol, gossip mesh
+formation, peer auth/scoring, transport behavior) belong in the `p2p` category instead.
+
+## Base class
+
+`MultiNodeTestContext` (`multi_node_test_context.ts`) extends `SingleNodeTestContext` (in
+`../single-node/`) with the N-validator topology:
+
+- `createValidatorNode` / `createValidatorNodeAt(index)` spawn nodes on the mock-gossip bus; passing
+  the same index to two calls (with different coinbases) models an equivocating proposer that shares
+  a key across two nodes. Validators with no spawned node stay registered-but-offline.
+- The per-validator registration accessors (`validatorAt` / `addressAt` / `privateKeyAt`) for the
+  validators registered at genesis via `initialValidators`.
+- `getSlashingContracts()` for the rollup / slasher / slashing-proposer L1 contracts.
+- The committee convergence helpers `waitForAllNodes*` and `findSlotsWithProposers`.
+
+The environment (in-process anvil + L1 deploy), the prover lifecycle, and the proving / reorg waiters
+all live on the parent and are inherited.
+
+## Shared presets and helpers
+
+These are exported from `multi_node_test_context.ts` and spread into a `setup(...)` call rather than
+copy-pasted:
+
+- `buildMockGossipValidators(n)` — the deterministic validator set (keys from
+  `getPrivateKeyFromIndex(i + 3)`), passed as `initialValidators`.
+- `MOCK_GOSSIP_MULTI_VALIDATOR_OPTS` — a tight committee on the mock bus with no prover
+  (`{ mockGossipSubNetwork, skipInitialSequencer, startProverNode: false, aztecProofSubmissionEpochs:
+  1024, numberOfAccounts: 0 }`). Tests that want a prover leave `startProverNode` explicit.
+- `SLASHER_ENABLED_MULTI_VALIDATOR_OPTS` — the same committee with the slasher turned on, used by the
+  offense-detection tests.
+- `defaultSlashingPenalties(unit?)` / `withOnlyOffense(offense, unit?)` — build the per-offense
+  `slash*Penalty` knobs; `withOnlyOffense` zeroes all but the named offense to isolate one offense.
+- `setupHaPairs(test, validators, { baseOpts, coinbases })` — stands up two HA pairs (nodes 0/1 share
+  keys pk1+pk2, nodes 2/3 share pk3+pk4), each pair on a shared slashing-protection DB with distinct
+  per-node coinbases. Used by the `high-availability/` tests.
+
+## Organizing principle
+
+The top level groups tests by node topology and setup model; the second level names the primary
+behavior under test, not the shared setup or a flag. Multiple-blocks-per-slot and proposer pipelining
+are default traits of block production here, not separate categories. Each file has a single top-level
+`describe` named to match its path, and a co-located `setup.ts` holds shared setup. A `.parallel`
+suffix marks files with more than one top-level `it`; CI splits each `it` into its own job.
+
+## Subfolders
+
+| Folder | Base / preset | Contents |
+|---|---|---|
+| `block-production/` | `MultiNodeTestContext` + wide-slot / block-production timing (`setup.ts`) | Happy-path committee block production with multiple blocks per slot and pipelining as the default cadence: `simple`, `high_tps`, `first_slot` (blocks on the first two slots of an epoch), `deploy_and_call_ordering` (a contract deployed and called in separate blocks of one slot), `cross_chain_messages.parallel` (multi-block slots carrying L2→L1 and L1→L2 messages), `proposed_chain.parallel` (txs anchored to proposed blocks; non-validators re-execute and sync multi-block slots), `proof_boundary.parallel` (the proof-submission deadline vs. the pipelining boundary slot, across five proof-landing scenarios), `redistribution.parallel` (checkpoint budget redistributed so a late tx burst fits across the last blocks), `blob_promotion` (a promotion-disabled node fetches blobs while peers skip them, and a high-block-count checkpoint still proves), `multi_validator_node.parallel` (one physical node hosting 5 validator keys builds blocks and collects quorum attestations; after withdrawing 2 validators, signing is filtered to the active committee only). |
+| `recovery/` | `MultiNodeTestContext` + wide-slot / block-production timing | The chain detects a bad/withheld/conflicting proposal and recovers: `proposal_failure_recovery.parallel` (all nodes prune and recover when a proposer fails to publish to L1), `pipeline_prune` (an uncheckpointed-blocks prune under pipelined MBPS, then recovery to a multi-block checkpoint), `equivocation_recovery` (an L1-confirmed checkpoint overrides a gossip-only equivocating proposal, the chain heals, and observers record the offense). |
+| `invalid-attestations/` | `MultiNodeTestContext` (slasher on) | Invalid checkpoints are detected, invalidated on L1, and the chain progresses: `invalidate_block.parallel`, a six-validator suite injecting insufficient/fake/high-s/unrecoverable/shuffled attestations and withheld blobs. |
+| `high-availability/` | `MultiNodeTestContext` + `setupHaPairs` | HA-pair sync and handoff between nodes that share validator keys: `ha_sync` (a peer that did not build a block syncs to the proposed chain tip over P2P), `ha_checkpoint_handoff` (a peer records and takes over a pipelined checkpoint when its partner proposes the previous slot). |
+| `slashing/` | `MultiNodeTestContext` + `SLASHER_ENABLED_MULTI_VALIDATOR_OPTS` (`setup.ts`) | Offense detection and the sentinel that drives it. Equivocation — both duplicate-proposal and duplicate-attestation — (`equivocation_offenses`), broadcast of an invalid block or checkpoint proposal (`broadcasted_invalid_block_proposal_slash`, `broadcasted_invalid_checkpoint_proposal_slash.parallel`), attesting to an invalid checkpoint (`attested_invalid_proposal.parallel`), data withholding (`data_withholding_slash`), inactivity (`inactivity_slash`, `inactivity_slash_with_consecutive_epochs`, sharing the `InactivityTest` fixture in `inactivity_setup.ts`), the slasher veto path (`slash_veto_demo`), and sentinel observability (`validators_sentinel.parallel`, `multiple_validators_sentinel.parallel`, `sentinel_status_slash.parallel`). `setup.ts` holds the two timing profiles (`baseSlashingOpts`, `SENTINEL_TIMING`), the shared offense lookup (`findSlashOffense`), and the slashing waiters (`awaitCommitteeExists`, `awaitOffenseDetected`, `advanceToEpochBeforeProposer`, `findUpcomingProposerSlot`, `awaitCommitteeKicked`, `awaitProposalExecution`). |
+| `governance/` | `MultiNodeTestContext` + `MOCK_GOSSIP_MULTI_VALIDATOR_OPTS` + `GOVERNANCE_TIMING` (`setup.ts`) | A validator committee drives an L1 governance upgrade. `upgrade_governance_proposer` (the committee signals a new governance-proposer payload, reaches quorum, and the proposal executes) and `add_rollup` (a new rollup version is registered and the nodes migrate to it, exercising L1↔L2 bridging on both rollups). |
+
+## Helper surface
+
+Prefer these named waiters over hand-rolled `retryUntil` / raw `.on` / `sleep` polling in test bodies.
+
+On `SingleNodeTestContext` (inherited):
+
+- `waitUntilEpochStarts(epoch)` / `waitUntilNextEpochStarts()` — epoch-boundary waiters.
+- `waitUntilCheckpointNumber(n)` / `waitUntilProvenCheckpointNumber(n)` — checkpoint waiters.
+- `waitUntilLastSlotOfProofSubmissionWindow(epoch)` — proof-window timing.
+- `waitForNodeToSync(blockNumber, type)` — single-node sync wait.
+- `watchSequencerEvents(sequencers, …)` accumulates state-changes and fail-events across sequencers;
+  `assertNoFailuresFromSequencers(failEvents)` asserts none fired.
+- `waitForSequencerEvent(sequencer, event, match?, opts)` — one-shot wait for a matching event, with
+  timeout and listener cleanup.
+- `assertMultipleBlocksPerSlot(n)` — asserts some checkpoint has at least `n` blocks (MBPS).
+
+Added by `MultiNodeTestContext`:
+
+- `waitForAllNodes(predicate, opts)` and the conveniences
+  `waitForAllNodesToReachProvenCheckpoint(target, opts)` and
+  `waitForAllNodesToReachBlockAtSlot(slot, tag, match?, opts)` — multi-node convergence.
+- `findSlotsWithProposers(count, predicate, opts)` — finds N consecutive slots whose proposers
+  satisfy `predicate`, warping the L1 clock forward and retrying on `EpochNotStable`.
+
+On `ChainMonitor` (`@aztec/ethereum/test`):
+
+- `waitUntilCheckpoint(n)` / `waitUntilCheckpointProven(n)`, `waitUntilL2Slot(slot)`,
+  `waitUntilL1Block` / `waitUntilL1Timestamp`.
+
+Node-only / wallet-only waits (no context dependency) live in `../fixtures/wait_helpers.ts`:
+
+- `waitForBlockNumber(node, target, { tag })` / `waitForProvenBlock(node, target)`.
+- `waitForNodeCheckpoint(node, target, opts)` / `waitForNodeProvenCheckpoint(node, target)`.
+- `waitForTxs(node, txHashes, opts)` — the plural form of `waitForTx`.

package/src/multi-node/block-production/setup.ts

@@ -0,0 +1,272 @@
+import type { Archiver } from '@aztec/archiver';
+import type { AztecNodeConfig, AztecNodeService } from '@aztec/aztec-node';
+import { AztecAddress, EthAddress } from '@aztec/aztec.js/addresses';
+import { NO_WAIT } from '@aztec/aztec.js/contracts';
+import { generateClaimSecret } from '@aztec/aztec.js/ethereum';
+import { Fr } from '@aztec/aztec.js/fields';
+import type { Logger } from '@aztec/aztec.js/log';
+import { isL1ToL2MessageReady } from '@aztec/aztec.js/messaging';
+import { waitForTx } from '@aztec/aztec.js/node';
+import { RollupContract } from '@aztec/ethereum/contracts';
+import { waitUntilL1Timestamp } from '@aztec/ethereum/l1-tx-utils';
+import { asyncMap } from '@aztec/foundation/async-map';
+import { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types';
+import { timesAsync } from '@aztec/foundation/collection';
+import { retryUntil } from '@aztec/foundation/retry';
+import { sleep } from '@aztec/foundation/sleep';
+import { executeTimeout } from '@aztec/foundation/timer';
+import { TestContract } from '@aztec/noir-test-contracts.js/Test';
+import type { SequencerEvents } from '@aztec/sequencer-client';
+import { getSlotAtTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
+import { GasFees } from '@aztec/stdlib/gas';
+import { TxStatus } from '@aztec/stdlib/tx';
+
+import { jest } from '@jest/globals';
+
+import { sendL1ToL2Message } from '../../fixtures/l1_to_l2_messaging.js';
+import { testSpan } from '../../fixtures/timing.js';
+import type { EndToEndContext } from '../../fixtures/utils.js';
+import { waitForBlockNumber, waitForTxs } from '../../fixtures/wait_helpers.js';
+import type { TestWallet } from '../../test-wallet/test_wallet.js';
+import { proveInteraction } from '../../test-wallet/utils.js';
+import {
+  type BlockProposedEvent,
+  MOCK_GOSSIP_MULTI_VALIDATOR_OPTS,
+  MULTI_VALIDATOR_BLOCK_PRODUCTION_TIMING,
+  MultiNodeTestContext,
+  type MultiNodeTestOpts,
+  type RegisteredValidator,
+  type TrackedSequencerEvent,
+  WIDE_SLOT_TIMING,
+  buildMockGossipValidators,
+} from '../multi_node_test_context.js';
+
+jest.setTimeout(1000 * 60 * 20);
+
+export const NODE_COUNT = 4;
+
+// Send enough transactions to trigger multiple blocks within a checkpoint assuming 2 txs per block.
+// If we start including txs at the 2nd block of a checkpoint, we can ensure a 3-block checkpoint
+// if we produce 10 txs:
+// - Checkpoint 1: Block 1 (0 txs), Block 2 (2 txs), Block 3 (2 txs)
+// - Checkpoint 2: Block 1 (2 txs), Block 2 (2 txs), Block 3 (2 txs)
+export const TX_COUNT = 10;
+
+/** The validator cluster + context produced by {@link setupSimpleBlockProduction}. */
+export type SimpleBlockProductionFixture = {
+  test: MultiNodeTestContext;
+  context: EndToEndContext;
+  logger: Logger;
+  validators: RegisteredValidator[];
+  nodes: AztecNodeService[];
+  from: AztecAddress;
+};
+
+/** State shared by the wide-slot `it`s (handles 4 validators + prover + a wallet pointed at node 0). */
+export type BlockProductionWithProverFixture = {
+  test: MultiNodeTestContext;
+  context: EndToEndContext;
+  logger: Logger;
+  rollup: RollupContract;
+  archiver: Archiver;
+  validators: RegisteredValidator[];
+  nodes: AztecNodeService[];
+  contract: TestContract;
+  wallet: TestWallet;
+  from: AztecAddress;
+  failEvents: TrackedSequencerEvent[];
+};
+
+/** Per-validator node config, or a function deriving it from the validator's 0-based index. */
+type ValidatorNodeOpts = Partial<AztecNodeConfig> & { dontStartSequencer?: boolean };
+
+/** Shared spine: builds N mock-gossip validators, sets up the context, spawns one node per validator. */
+async function buildValidatorCluster(opts: {
+  nodeCount: number;
+  setupOpts: Partial<MultiNodeTestOpts>;
+  nodeOpts?: ValidatorNodeOpts | ((index: number) => ValidatorNodeOpts);
+}): Promise<SimpleBlockProductionFixture> {
+  const validators = buildMockGossipValidators(opts.nodeCount);
+
+  const test = await MultiNodeTestContext.setup({
+    ...opts.setupOpts,
+    initialValidators: validators,
+  });
+
+  const { context, logger } = test;
+  const from = context.accounts[0];
+
+  logger.warn(`Initial setup complete. Starting ${opts.nodeCount} validator nodes.`);
+  const nodes = await asyncMap(validators, ({ privateKey }, i) =>
+    test.createValidatorNode(
+      [privateKey],
+      typeof opts.nodeOpts === 'function' ? opts.nodeOpts(i) : { ...opts.nodeOpts },
+    ),
+  );
+  logger.warn(`Started ${opts.nodeCount} validator nodes.`, { validators: validators.map(v => v.attester.toString()) });
+
+  return { test, context, logger, validators, nodes, from };
+}
+
+/**
+ * Stands up the `block-production` validator cluster shared by the `MULTI_VALIDATOR_BLOCK_PRODUCTION_TIMING` tests
+ * (`simple`, `high_tps`): builds `nodeCount` mock-gossip validators, sets up the context with the
+ * block-production timing profile, spawns one validator node per validator, and returns the cluster. The
+ * per-test divergence (`fakeProcessingDelayPerTxMs`, `txDelayerMaxInclusionTimeIntoSlot`,
+ * min/maxTxsPerBlock, whether sequencers start eagerly, contract type) passes through `opts`. Mirrors
+ * how {@link setupBlockProductionWithProver} factors out the prover-backed setup; the test still registers its own contract.
+ */
+export function setupSimpleBlockProduction(opts: {
+  nodeCount: number;
+  setupOpts?: Partial<MultiNodeTestOpts>;
+  nodeOpts?: Partial<AztecNodeConfig> & { dontStartSequencer?: boolean };
+}): Promise<SimpleBlockProductionFixture> {
+  return buildValidatorCluster({
+    nodeCount: opts.nodeCount,
+    setupOpts: { ...MOCK_GOSSIP_MULTI_VALIDATOR_OPTS, ...MULTI_VALIDATOR_BLOCK_PRODUCTION_TIMING, ...opts.setupOpts },
+    nodeOpts: opts.nodeOpts,
+  });
+}
+
+/**
+ * Creates validators and sets up a wide-slot test context with the pipelining timing profile and a prover
+ * node, then starts (paused) validator nodes and points the wallet at node 0. Mirrors the per-test
+ * setup from the dissolved `mbps.parallel` file. The blob-promotion and pipeline-prune suites layer their
+ * adverse-network shape on via `mockGossipSubNetworkLatency`, `maxTxsPerCheckpoint`, `clearInheritedCoinbase`,
+ * and `disableCheckpointPromotionOnFirstNode`.
+ */
+export async function setupBlockProductionWithProver(opts: {
+  syncChainTip: 'proposed' | 'checkpointed';
+  minTxsPerBlock?: number;
+  maxTxsPerBlock?: number;
+  maxTxsPerCheckpoint?: number;
+  buildCheckpointIfEmpty?: boolean;
+  skipPushProposedBlocksToArchiver?: boolean;
+  /** Injects artificial mock-gossip propagation latency (ms) to model adverse network conditions. */
+  mockGossipSubNetworkLatency?: number;
+  /** Clears each validator node's inherited coinbase so it derives one from its own attester key. */
+  clearInheritedCoinbase?: boolean;
+  /**
+   * Disables checkpoint promotion on node 0 (`skipPromoteProposedCheckpointDuringL1Sync`), so node 0 fetches
+   * blobs during L1 sync while its peers promote their own proposed checkpoints and skip the blob fetch.
+   */
+  disableCheckpointPromotionOnFirstNode?: boolean;
+}): Promise<BlockProductionWithProverFixture> {
+  const {
+    syncChainTip = 'checkpointed',
+    clearInheritedCoinbase = false,
+    disableCheckpointPromotionOnFirstNode = false,
+    ...setupOpts
+  } = opts;
+
+  // WIDE_SLOT_TIMING is the wide 72s/12s pipelining cadence (see A-914 on why the tighter 36s/4s breaks
+  // non-proposer nodes); the JSDoc on the profile carries the full rationale.
+  const { test, context, logger, validators, nodes, from } = await buildValidatorCluster({
+    nodeCount: NODE_COUNT,
+    setupOpts: {
+      ...WIDE_SLOT_TIMING,
+      numberOfAccounts: 0,
+      mockGossipSubNetwork: true,
+      startProverNode: true,
+      ...setupOpts,
+      pxeOpts: { syncChainTip },
+      skipInitialSequencer: true,
+      inboxLag: 2,
+    },
+    nodeOpts: (index: number) => ({
+      dontStartSequencer: true,
+      ...(clearInheritedCoinbase ? { coinbase: undefined } : {}),
+      ...(disableCheckpointPromotionOnFirstNode && index === 0
+        ? { skipPromoteProposedCheckpointDuringL1Sync: true }
+        : {}),
+    }),
+  });
+
+  const { rollup } = test;
+  const wallet = context.wallet as TestWallet;
+  const { failEvents } = test.watchNodeSequencerEvents(nodes);
+
+  // Point the wallet at a validator node. The initial node-0 has all validator keys in its config,
+  // so it rejects block proposals from validators thinking they come from itself. By redirecting
+  // the wallet to a validator node, the PXE correctly tracks proposed blocks.
+  wallet.updateNode(nodes[0]);
+  const archiver = nodes[0].getBlockSource() as Archiver;
+
+  // Register contract for sending txs.
+  const contract = await test.registerTestContract(wallet);
+  logger.warn(`Test setup completed.`, { validators: validators.map(v => v.attester.toString()) });
+
+  return { test, context, logger, rollup, archiver, validators, nodes, contract, wallet, from, failEvents };
+}
+
+/** Waits until a specific multi-block checkpoint is proven, verifying that proving succeeds with multiple-blocks-per-slot. */
+export async function waitForProvenCheckpoint(
+  fixture: BlockProductionWithProverFixture,
+  targetCheckpoint: CheckpointNumber,
+) {
+  const { test, nodes, logger, failEvents } = fixture;
+  test.assertNoFailuresFromSequencers(failEvents);
+
+  logger.warn(`Stopping validator sequencers before waiting for checkpoint ${targetCheckpoint} to be proven`);
+  await Promise.all(nodes.map(n => n.getSequencer()?.stop()));
+
+  // With the sequencers stopped, no further blocks are produced, so waiting out the rest of the epoch in
+  // wall-clock is dead time. Warp the L1 clock forward past the epoch boundary so the epoch containing
+  // targetCheckpoint closes and the fake prover can prove+submit it; the subsequent wait then only covers
+  // the (real-time) proving+submission. A single next-epoch jump stays inside the proof-submission window
+  // (proofSubmissionEpochs >= 1), so it never crosses the submission deadline. Skipped if already proven,
+  // and forward-only since advanceToNextEpoch never rewinds.
+  const { proven } = await test.context.cheatCodes.rollup.getTips();
+  if (proven < targetCheckpoint) {
+    await testSpan('warp:proven-checkpoint-epoch', () => test.context.cheatCodes.rollup.advanceToNextEpoch());
+  }
+
+  const provenTimeout = test.L2_SLOT_DURATION_IN_S * test.epochDuration * 4;
+  logger.warn(`Waiting for checkpoint ${targetCheckpoint} to be proven (timeout=${provenTimeout}s)`);
+  await test.waitUntilProvenCheckpointNumber(targetCheckpoint, provenTimeout);
+  logger.warn(`Proven checkpoint advanced to ${test.monitor.provenCheckpointNumber}`);
+}
+
+export {
+  type Archiver,
+  type AztecNodeConfig,
+  type AztecNodeService,
+  AztecAddress,
+  EthAddress,
+  NO_WAIT,
+  generateClaimSecret,
+  Fr,
+  type Logger,
+  isL1ToL2MessageReady,
+  waitForTx,
+  RollupContract,
+  waitUntilL1Timestamp,
+  asyncMap,
+  BlockNumber,
+  CheckpointNumber,
+  SlotNumber,
+  timesAsync,
+  retryUntil,
+  sleep,
+  executeTimeout,
+  TestContract,
+  type SequencerEvents,
+  getSlotAtTimestamp,
+  getTimestampForSlot,
+  GasFees,
+  TxStatus,
+  jest,
+  sendL1ToL2Message,
+  type EndToEndContext,
+  waitForBlockNumber,
+  waitForTxs,
+  type TestWallet,
+  proveInteraction,
+  type BlockProposedEvent,
+  WIDE_SLOT_TIMING,
+  MULTI_VALIDATOR_BLOCK_PRODUCTION_TIMING,
+  MultiNodeTestContext,
+  type RegisteredValidator,
+  type TrackedSequencerEvent,
+  buildMockGossipValidators,
+};

package/src/multi-node/governance/setup.ts

@@ -0,0 +1,172 @@
+import { RollupContract } from '@aztec/ethereum/contracts';
+import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils';
+import type { ExtendedViemWalletClient } from '@aztec/ethereum/types';
+import { SlotNumber } from '@aztec/foundation/branded-types';
+import { retryUntil } from '@aztec/foundation/retry';
+import { GovernanceAbi, GovernanceProposerAbi } from '@aztec/l1-artifacts';
+
+import { jest } from '@jest/globals';
+import { type GetContractReturnType, encodeFunctionData, getAddress, getContract } from 'viem';
+
+import type { MultiNodeTestContext } from '../multi_node_test_context.js';
+
+jest.setTimeout(1000 * 60 * 10);
+
+/**
+ * The shortened block-time timing the governance tests run on: a 12s L2 slot, 4s L1 slot, and a long
+ * proof-submission window so unproven blocks are never pruned (the committee runs no prover). Spread into
+ * a {@link MultiNodeTestContext.setup} call alongside {@link MOCK_GOSSIP_MULTI_VALIDATOR_OPTS} and
+ * `initialValidators`.
+ */
+export const GOVERNANCE_TIMING = {
+  aztecSlotDuration: 12,
+  ethereumSlotDuration: 4,
+  aztecProofSubmissionEpochs: 640,
+} as const;
+
+/** The current round leader's payload and its signal count, as observed by {@link GovernanceTestDriver.govInfo}. */
+type GovRoundState = { round: bigint; leaderVotes: bigint };
+
+/**
+ * Wraps the L1 governance-proposer / governance / rollup contracts and the round-driving mechanics shared by
+ * the governance suites (`add_rollup`, `upgrade_governance_proposer`): warping to round boundaries, polling
+ * round state, waiting for signal quorum, submitting the round winner, and voting a proposal through to
+ * executable. Scenario-specific payload construction, node signaling, and post-execution assertions stay in
+ * each test.
+ */
+export interface GovernanceTestDriver {
+  governanceProposer: GetContractReturnType<typeof GovernanceProposerAbi, ExtendedViemWalletClient>;
+  governance: GetContractReturnType<typeof GovernanceAbi, ExtendedViemWalletClient>;
+  rollup: RollupContract;
+  roundSize: bigint;
+  emperor: ExtendedViemWalletClient['account'];
+  /** Sends a 1-wei self-transfer to mine a fresh L1 block, so warped time takes effect. */
+  waitL1Block: () => Promise<void>;
+  /** Reads the current round's leading payload and its signal count. */
+  govInfo: () => Promise<GovRoundState>;
+  /** Warps L1 to the start of the next governance-proposer round. */
+  warpToNextRound: () => Promise<void>;
+  /** Polls once per L2 slot until the round leader reaches quorum, returning the round state at that point. */
+  waitForQuorum: (quorumSize: bigint, timeoutSeconds: number) => Promise<GovRoundState>;
+  /** Submits the winning payload of `round` to the governance proposer, creating the on-chain proposal. */
+  submitRoundWinner: (round: bigint) => Promise<void>;
+  /**
+   * Warps past the proposal's voting delay, casts a yes vote (asserting it lands), then warps past the
+   * voting duration and execution delay so the proposal becomes executable.
+   */
+  voteToExecutable: () => Promise<void>;
+}
+
+/** Builds a {@link GovernanceTestDriver} over the context's L1 governance contracts. */
+export async function createGovernanceTestDriver(
+  test: MultiNodeTestContext,
+  l1TxUtils: L1TxUtils,
+): Promise<GovernanceTestDriver> {
+  const { l1Client, l1ContractAddresses } = test.context.deployL1ContractsValues;
+  const rollupAddress = l1ContractAddresses.rollupAddress.toString();
+
+  const governanceProposer = getContract({
+    address: getAddress(l1ContractAddresses.governanceProposerAddress.toString()),
+    abi: GovernanceProposerAbi,
+    client: l1Client,
+  });
+  const governance = getContract({
+    address: getAddress(l1ContractAddresses.governanceAddress.toString()),
+    abi: GovernanceAbi,
+    client: l1Client,
+  });
+  const rollup = new RollupContract(l1Client, l1ContractAddresses.rollupAddress);
+  const roundSize = await governanceProposer.read.ROUND_SIZE();
+  const emperor = l1Client.account;
+
+  const waitL1Block = async () => {
+    await l1TxUtils.sendAndMonitorTransaction({ to: emperor.address, value: 1n });
+  };
+
+  const govInfo = async (): Promise<GovRoundState> => {
+    const bn = await test.context.cheatCodes.eth.blockNumber();
+    const slot = await rollup.getSlotNumber();
+    const round = await governanceProposer.read.computeRound([BigInt(slot)]);
+    const info = await governanceProposer.read.getRoundData([rollupAddress, round]);
+    const leaderVotes = await governanceProposer.read.signalCount([rollupAddress, round, info.payloadWithMostSignals]);
+    test.logger.info(
+      `Governance stats for round ${round} (Slot: ${slot}, BN: ${bn}). Leader: ${info.payloadWithMostSignals} have ${leaderVotes} signals`,
+    );
+    return { round, leaderVotes };
+  };
+
+  const warpToNextRound = async () => {
+    const currentSlot = await rollup.getSlotNumber();
+    const nextRoundSlot = SlotNumber.fromBigInt((BigInt(currentSlot) / roundSize) * roundSize + roundSize);
+    const nextRoundTimestamp = await rollup.getTimestampForSlot(nextRoundSlot);
+    await test.context.cheatCodes.eth.warp(Number(nextRoundTimestamp));
+  };
+
+  const waitForQuorum = (quorumSize: bigint, timeoutSeconds: number) =>
+    retryUntil(
+      async () => {
+        const data = await govInfo();
+        return data.leaderVotes >= quorumSize ? data : undefined;
+      },
+      'governance leader reaches quorum',
+      timeoutSeconds,
+      GOVERNANCE_TIMING.aztecSlotDuration,
+    );
+
+  const submitRoundWinner = async (round: bigint) => {
+    await l1TxUtils.sendAndMonitorTransaction({
+      to: governanceProposer.address,
+      data: encodeFunctionData({ abi: GovernanceProposerAbi, functionName: 'submitRoundWinner', args: [round] }),
+    });
+  };
+
+  const voteToExecutable = async () => {
+    const proposal = await governance.read.getProposal([0n]);
+
+    const timeToActive = proposal.creation + proposal.config.votingDelay;
+    await test.context.cheatCodes.eth.warp(Number(timeToActive + 1n));
+    await waitL1Block();
+
+    const voteTx = await rollup.vote(l1TxUtils, 0n);
+    expect(voteTx.receipt?.status).toBe('success');
+
+    const timeToExecutable = timeToActive + proposal.config.votingDuration + proposal.config.executionDelay + 1n;
+    await test.context.cheatCodes.eth.warp(Number(timeToExecutable));
+    await waitL1Block();
+  };
+
+  return {
+    governanceProposer,
+    governance,
+    rollup,
+    roundSize,
+    emperor,
+    waitL1Block,
+    govInfo,
+    warpToNextRound,
+    waitForQuorum,
+    submitRoundWinner,
+    voteToExecutable,
+  };
+}
+
+/**
+ * Drives one governance round to an executable proposal: waits for the signaled payload to reach quorum,
+ * warps to the next round, submits the round winner, then votes the resulting proposal through its voting
+ * and execution delays. Returns the round state observed at quorum; the caller performs the scenario-specific
+ * execution and assertions.
+ */
+export async function driveGovernanceRound(
+  driver: GovernanceTestDriver,
+  opts: { quorumTimeoutSeconds: number },
+): Promise<{ govData: GovRoundState }> {
+  const quorumSize = await driver.governanceProposer.read.QUORUM_SIZE();
+  const govData = await driver.waitForQuorum(quorumSize, opts.quorumTimeoutSeconds);
+  await driver.warpToNextRound();
+  await driver.waitL1Block();
+  await driver.submitRoundWinner(govData.round);
+  await driver.voteToExecutable();
+  return { govData };
+}
+
+export { jest };