@aztec/end-to-end 0.0.1-commit.e558bd1c → 0.0.1-commit.e57c76e

package/src/e2e_p2p/reqresp/utils.ts

@@ -2,25 +2,26 @@ import type { AztecNodeService } from '@aztec/aztec-node';
 import { createLogger } from '@aztec/aztec.js/log';
 import { waitForTx } from '@aztec/aztec.js/node';
 import { Tx } from '@aztec/aztec.js/tx';
+import { PROPOSER_PIPELINING_SLOT_OFFSET } from '@aztec/epoch-cache';
 import { RollupContract } from '@aztec/ethereum/contracts';
-import { SlotNumber } from '@aztec/foundation/branded-types';
+import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types';
 import { timesAsync } from '@aztec/foundation/collection';
 import { retryUntil } from '@aztec/foundation/retry';
 
-import { jest } from '@jest/globals';
+import { expect, jest } from '@jest/globals';
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
 
-import { shouldCollectMetrics } from '../../fixtures/fixtures.js';
+import { getBootNodeUdpPort, shouldCollectMetrics } from '../../fixtures/fixtures.js';
 import { createNodes } from '../../fixtures/setup_p2p_test.js';
-import { P2PNetworkTest, SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES, WAIT_FOR_TX_TIMEOUT } from '../p2p_network.js';
+import { P2PNetworkTest } from '../p2p_network.js';
 import { prepareTransactions } from '../shared.js';
 
 // Don't set this to a higher value than 9 because each node will use a different L1 publisher account and anvil seeds
 export const NUM_VALIDATORS = 6;
-export const NUM_TXS_PER_NODE = 2;
-export const BOOT_NODE_UDP_PORT = 4500;
+export const NUM_TXS_PER_NODE = 4;
+export const BOOT_NODE_UDP_PORT = getBootNodeUdpPort();
 
 export const createReqrespDataDir = () => fs.mkdtempSync(path.join(os.tmpdir(), 'reqresp-'));
 
@@ -38,11 +39,20 @@ export async function createReqrespTest(options: ReqrespOptions = {}): Promise<P
     // To collect metrics - run in aztec-packages `docker compose --profile metrics up`
     metricsPort: shouldCollectMetrics(),
     initialConfig: {
-      ...SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES,
-      aztecSlotDuration: 24,
+      ethereumSlotDuration: 8,
+      aztecSlotDuration: 36,
+      blockDurationMs: 6000,
+      l1PublishingTime: 8,
+      minTxsPerBlock: 1,
+      maxTxsPerBlock: 2,
+      enforceTimeTable: true,
+      aztecProofSubmissionEpochs: 1024, // effectively do not reorg
       ...(disableStatusHandshake ? { p2pDisableStatusHandshake: true } : {}),
       listenAddress: '127.0.0.1',
       aztecEpochDuration: 64, // stable committee
+      // Pipelining: target-slot is one ahead of build-slot; inboxLag sources L1->L2
+      // messages from the previous checkpoint to avoid L1ToL2MessagesNotReadyError.
+      inboxLag: 2,
     },
   });
   await t.setup();
@@ -81,11 +91,11 @@ export async function runReqrespTxTest(params: {
 
   const nodes = await createNodes(
     aztecNodeConfig,
-    t.ctx.dateProvider!,
+    t.ctx.dateProvider,
     t.bootstrapNodeEnr,
     NUM_VALIDATORS,
     BOOT_NODE_UDP_PORT,
-    t.prefilledPublicData,
+    t.genesis,
     dataDir,
     shouldCollectMetrics(),
   );
@@ -95,7 +105,7 @@ export async function runReqrespTxTest(params: {
 
   await t.setupAccount();
 
-  const targetBlockNumber = await t.ctx.aztecNodeService!.getBlockNumber();
+  const targetBlockNumber = await t.ctx.aztecNodeService.getBlockNumber();
   await retryUntil(
     async () => {
       const blockNumbers = await Promise.all(nodes.map(node => node.getBlockNumber()));
@@ -108,7 +118,7 @@ export async function runReqrespTxTest(params: {
 
   t.logger.info('Preparing transactions to send');
   const txBatches = await timesAsync(2, () =>
-    prepareTransactions(t.logger, t.ctx.aztecNodeService!, NUM_TXS_PER_NODE, t.fundedAccount),
+    prepareTransactions(t.logger, t.ctx.aztecNodeService, NUM_TXS_PER_NODE, t.fundedAccount),
   );
 
   t.logger.info('Removing initial node');
@@ -116,10 +126,18 @@ export async function runReqrespTxTest(params: {
 
   t.logger.info('Starting fresh slot');
   const [timestamp] = await t.ctx.cheatCodes.rollup.advanceToNextSlot();
-  t.ctx.dateProvider!.setTime(Number(timestamp) * 1000);
+  t.ctx.dateProvider.setTime(Number(timestamp) * 1000);
   const startSlotTimestamp = BigInt(timestamp);
 
-  const { proposerIndexes, nodesToTurnOffTxGossip } = await getProposerIndexes(t, startSlotTimestamp);
+  // Under pipelining the active builder during wallclock slot S targets slot S+1, so
+  // we must address the proposer of S+1 (not S) for batch 0. Shift the proposer lookup
+  // window by the pipelining offset so we always send to the currently-building proposer.
+  const proposerSlotOffset = PROPOSER_PIPELINING_SLOT_OFFSET;
+  const { proposerIndexes, nodesToTurnOffTxGossip } = await getProposerIndexes(
+    t,
+    startSlotTimestamp,
+    proposerSlotOffset,
+  );
   t.logger.info(`Turning off tx gossip for nodes: ${nodesToTurnOffTxGossip.map(getNodePort)}`);
   t.logger.info(`Sending txs to proposer nodes: ${proposerIndexes.map(getNodePort)}`);
 
@@ -143,6 +161,13 @@ export async function runReqrespTxTest(params: {
   const submittedTxs = await Promise.all(
     txBatches.map(async (batch, batchIndex) => {
       const proposerNode = nodes[proposerIndexes[batchIndex]];
+      for (const tx of batch) {
+        t.logger.info(`Tx ${tx.getTxHash().toString()} base64: ${tx.toBuffer().toString('base64')}`);
+      }
+      const txHashes = batch.map(tx => tx.getTxHash().toString());
+      t.logger.info(
+        `Sending batch ${batchIndex} to proposer ${getNodePort(proposerIndexes[batchIndex])}: ${txHashes.join(', ')}`,
+      );
       await Promise.all(
         batch.map(async tx => {
           try {
@@ -157,12 +182,23 @@ export async function runReqrespTxTest(params: {
     }),
   );
 
+  // Log pool state per node after sending
+  for (let i = 0; i < NUM_VALIDATORS; i++) {
+    const count = await nodes[i].getPendingTxCount();
+    t.logger.info(`Node ${getNodePort(i)} pool has ${count} pending txs`);
+  }
+
+  // Use the test's own aztecSlotDuration (not the env default that p2p_network's
+  // WAIT_FOR_TX_TIMEOUT is derived from) so the timeout scales with this test's 36s slot.
+  // Under pipelining the round-trip is roughly build-slot + target-slot + L1 publish, so
+  // budget for >= 3 slots.
+  const waitForTxTimeout = t.ctx.aztecNodeConfig.aztecSlotDuration * 4.5;
   t.logger.info('Waiting for all transactions to be mined');
   await Promise.all(
     submittedTxs.flatMap((batch, batchIndex) =>
       batch.map(async (submittedTx, txIndex) => {
         t.logger.info(`Waiting for tx ${batchIndex}-${txIndex} ${submittedTx.txHash.toString()} to be mined`);
-        await waitForTx(submittedTx.node, submittedTx.txHash, { timeout: WAIT_FOR_TX_TIMEOUT * 1.5 });
+        await waitForTx(submittedTx.node, submittedTx.txHash, { timeout: waitForTxTimeout });
         t.logger.info(`Tx ${batchIndex}-${txIndex} ${submittedTx.txHash.toString()} has been mined`);
       }),
     ),
@@ -170,10 +206,41 @@ export async function runReqrespTxTest(params: {
 
   t.logger.info('All transactions mined');
 
+  // Assert that multiple blocks were built for at least one slot
+  t.logger.info('Verifying multiple blocks for at least one checkpoint');
+  // Wait for L1 checkpoint sync, which may lag behind P2P block propagation.
+  const checkpoints = await retryUntil(
+    async () => {
+      const cps = await nodes[0].getCheckpoints(CheckpointNumber(1), 50, { includeBlocks: true });
+      return cps.length > 0 && cps.some(cp => (cp.blocks?.length ?? 0) >= 2) ? cps : undefined;
+    },
+    'waiting for multi-block checkpoint to sync from L1',
+    30,
+    1,
+  );
+
+  let mbpsFound = false;
+  let expectedBlockNumber = checkpoints[0].blocks![0].number;
+
+  for (const published of checkpoints) {
+    const blocks = published.blocks!;
+    const blockCount = blocks.length;
+    mbpsFound = mbpsFound || blockCount >= 2;
+
+    for (let i = 0; i < blockCount; i++) {
+      const block = blocks[i];
+      expect(block.indexWithinCheckpoint).toBe(i);
+      expect(block.checkpointNumber).toBe(published.number);
+      expect(block.number).toBe(expectedBlockNumber);
+      expectedBlockNumber++;
+    }
+  }
+
+  expect(mbpsFound).toBe(true);
   return nodes;
 }
 
-async function getProposerIndexes(t: P2PNetworkTest, startSlotTimestamp: bigint) {
+async function getProposerIndexes(t: P2PNetworkTest, startSlotTimestamp: bigint, slotOffset = 0) {
   // Get the nodes for the next set of slots
   const rollupContract = new RollupContract(
     t.ctx.deployL1ContractsValues.l1Client,
@@ -185,7 +252,7 @@ async function getProposerIndexes(t: P2PNetworkTest, startSlotTimestamp: bigint)
 
   const proposers = await Promise.all(
     Array.from({ length: 3 }, async (_, i) => {
-      const slot = SlotNumber(startSlot + i);
+      const slot = SlotNumber(startSlot + slotOffset + i);
       const slotTimestamp = await rollupContract.getTimestampForSlot(slot);
       return await rollupContract.getProposerAt(slotTimestamp);
     }),

package/src/e2e_p2p/shared.ts

@@ -6,12 +6,9 @@ import { Fr } from '@aztec/aztec.js/fields';
 import type { Logger } from '@aztec/aztec.js/log';
 import { TxHash } from '@aztec/aztec.js/tx';
 import type { RollupCheatCodes } from '@aztec/aztec/testing';
-import type {
-  EmpireSlashingProposerContract,
-  RollupContract,
-  TallySlashingProposerContract,
-} from '@aztec/ethereum/contracts';
-import { EpochNumber } from '@aztec/foundation/branded-types';
+import type { EpochCacheInterface } from '@aztec/epoch-cache';
+import type { RollupContract, SlashingProposerContract } from '@aztec/ethereum/contracts';
+import { EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
 import { timesAsync, unique } from '@aztec/foundation/collection';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { retryUntil } from '@aztec/foundation/retry';
@@ -21,10 +18,11 @@ import { TestContract, TestContractArtifact } from '@aztec/noir-test-contracts.j
 import { getPXEConfig, getPXEConfig as getRpcConfig } from '@aztec/pxe/server';
 import { getRoundForOffense } from '@aztec/slasher';
 import type { AztecNodeAdmin } from '@aztec/stdlib/interfaces/client';
-import type { SlashFactoryContract } from '@aztec/stdlib/l1-contracts';
-import { type ProvenTx, TestWallet, proveInteraction } from '@aztec/test-wallet/server';
 
+import { SchnorrHardcodedKeyAccountContract } from '../fixtures/schnorr_hardcoded_account_contract.js';
 import { submitTxsTo } from '../shared/submit-transactions.js';
+import { TestWallet } from '../test-wallet/test_wallet.js';
+import { type ProvenTx, proveInteraction } from '../test-wallet/utils.js';
 
 // submits a set of transactions to the provided Private eXecution Environment (PXE)
 export const submitComplexTxsTo = async (
@@ -40,7 +38,7 @@ export const submitComplexTxsTo = async (
   const spamCount = 15;
   for (let i = 0; i < numTxs; i++) {
     const method = spamContract.methods.spam(seed + BigInt(i * spamCount), spamCount, !!opts.callPublic);
-    const txHash = await method.send({ from, wait: NO_WAIT });
+    const { txHash } = await method.send({ from, wait: NO_WAIT });
     logger.info(`Tx sent with hash ${txHash.toString()}`);
     txs.push(txHash);
   }
@@ -58,10 +56,17 @@ export const submitTransactions = async (
   rpcConfig.proverEnabled = false;
   const wallet = await TestWallet.create(
     node,
-    { ...getPXEConfig(), proverEnabled: false },
+    // Use checkpointed chain tip to avoid anchoring on provisional blocks that the archiver can prune
+    // when their slot ends without a checkpoint landing on L1.
+    { ...getPXEConfig(), proverEnabled: false, syncChainTip: 'checkpointed' },
     { loggerActorLabel: 'pxe-tx' },
   );
-  const fundedAccountManager = await wallet.createSchnorrAccount(fundedAccount.secret, fundedAccount.salt);
+  const contract = new SchnorrHardcodedKeyAccountContract();
+  const fundedAccountManager = await wallet.createAccount({
+    secret: fundedAccount.secret,
+    salt: fundedAccount.salt,
+    contract,
+  });
   return submitTxsTo(wallet, fundedAccountManager.address, numTxs, logger);
 };
 
@@ -76,10 +81,15 @@ export async function prepareTransactions(
 
   const wallet = await TestWallet.create(
     node,
-    { ...getPXEConfig(), proverEnabled: false },
+    { ...getPXEConfig(), proverEnabled: false, syncChainTip: 'checkpointed' },
     { loggerActorLabel: 'pxe-tx' },
   );
-  const fundedAccountManager = await wallet.createSchnorrAccount(fundedAccount.secret, fundedAccount.salt);
+  const accountContract = new SchnorrHardcodedKeyAccountContract();
+  const fundedAccountManager = await wallet.createAccount({
+    secret: fundedAccount.secret,
+    salt: fundedAccount.salt,
+    contract: accountContract,
+  });
 
   const testContractInstance = await getContractInstanceFromInstantiationParams(TestContractArtifact, {
     salt: Fr.random(),
@@ -97,7 +107,7 @@ export async function prepareTransactions(
 }
 
 export function awaitProposalExecution(
-  slashingProposer: EmpireSlashingProposerContract | TallySlashingProposerContract,
+  slashingProposer: SlashingProposerContract,
   timeoutSeconds: number,
   logger: Logger,
 ): Promise<bigint> {
@@ -107,24 +117,12 @@ export function awaitProposalExecution(
       reject(new Error(`Timeout waiting for proposal execution after ${timeoutSeconds}s`));
     }, timeoutSeconds * 1000);
 
-    if (slashingProposer.type === 'empire') {
-      const unwatch = slashingProposer.listenToPayloadSubmitted(args => {
-        logger.warn(`Proposal ${args.payload} from round ${args.round} executed`);
-        clearTimeout(timeout);
-        unwatch();
-        resolve(args.round);
-      });
-    } else if (slashingProposer.type === 'tally') {
-      const unwatch = slashingProposer.listenToRoundExecuted(args => {
-        logger.warn(`Slash from round ${args.round} executed`);
-        clearTimeout(timeout);
-        unwatch();
-        resolve(args.round);
-      });
-    } else {
+    const unwatch = slashingProposer.listenToRoundExecuted(args => {
+      logger.warn(`Slash from round ${args.round} executed`);
       clearTimeout(timeout);
-      reject(new Error(`Unknown slashing proposer type: ${(slashingProposer as any).type}`));
-    }
+      unwatch();
+      resolve(args.round);
+    });
   });
 }
 
@@ -149,6 +147,72 @@ export async function awaitCommitteeExists({
   return committee!.map(c => c.toString() as `0x${string}`);
 }
 
+/**
+ * Advance epochs until we find one where the target proposer is selected for a slot at least
+ * `warmupSlots` into the epoch, then stop one epoch before it. This leaves time for the caller to
+ * start sequencers before warping to the target epoch, avoiding the race where the target epoch
+ * passes before sequencers are ready.
+ *
+ * The first `warmupSlots` slots of the epoch are skipped on purpose. Callers warp to one slot
+ * before the target epoch and, under proposer pipelining, the proposer begins building one slot
+ * before its proposal slot. If the proposer were in the first slot of the epoch, that build would
+ * begin at the exact instant of the warp, leaving the freshly-started sequencer no warm-up margin;
+ * it then serializes its (often AVM-heavy) proposal past the slot boundary and honest receivers
+ * reject it as late. Picking a slot at least `warmupSlots` into the epoch guarantees that many full
+ * slots of wall-clock between the warp and the start of the proposer's build.
+ *
+ * Returns the target epoch and the concrete target slot so the caller can warp to it after starting
+ * sequencers.
+ */
+export async function advanceToEpochBeforeProposer({
+  epochCache,
+  cheatCodes,
+  targetProposer,
+  logger,
+  maxAttempts = 20,
+  warmupSlots = 1,
+}: {
+  epochCache: EpochCacheInterface;
+  cheatCodes: RollupCheatCodes;
+  targetProposer: EthAddress;
+  logger: Logger;
+  maxAttempts?: number;
+  warmupSlots?: number;
+}): Promise<{ targetEpoch: EpochNumber; targetSlot: SlotNumber }> {
+  const { epochDuration } = await cheatCodes.getConfig();
+
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    const currentEpoch = await cheatCodes.getEpoch();
+    // Check the NEXT epoch's slots so we stay one epoch before the target,
+    // giving the caller time to start sequencers before the target epoch arrives.
+    const nextEpoch = Number(currentEpoch) + 1;
+    const epochStartSlot = nextEpoch * Number(epochDuration);
+    // Skip the first `warmupSlots` slots so the caller keeps a warm-up margin after warping to one
+    // slot before the epoch (see the doc comment above).
+    const startSlot = epochStartSlot + warmupSlots;
+    const endSlot = epochStartSlot + Number(epochDuration);
+
+    logger.info(
+      `Checking next epoch ${nextEpoch} (slots ${startSlot}-${endSlot - 1}) for proposer ${targetProposer} (current epoch: ${currentEpoch})`,
+    );
+
+    for (let s = startSlot; s < endSlot; s++) {
+      const proposer = await epochCache.getProposerAttesterAddressInSlot(SlotNumber(s));
+      if (proposer && proposer.equals(targetProposer)) {
+        logger.warn(
+          `Found target proposer ${targetProposer} in slot ${s} of epoch ${nextEpoch}. Staying at epoch ${currentEpoch} to allow sequencer startup.`,
+        );
+        return { targetEpoch: EpochNumber(nextEpoch), targetSlot: SlotNumber(s) };
+      }
+    }
+
+    logger.info(`Target proposer not found in epoch ${nextEpoch}, advancing to next epoch`);
+    await cheatCodes.advanceToNextEpoch();
+  }
+
+  throw new Error(`Target proposer ${targetProposer} not found in any slot after ${maxAttempts} epoch attempts`);
+}
+
 export async function awaitOffenseDetected({
   logger,
   nodeAdmin,
@@ -191,7 +255,6 @@ export async function awaitCommitteeKicked({
   rollup,
   cheatCodes,
   committee,
-  slashFactory,
   slashingProposer,
   slashingRoundSize,
   aztecSlotDuration,
@@ -202,8 +265,7 @@ export async function awaitCommitteeKicked({
   rollup: RollupContract;
   cheatCodes: RollupCheatCodes;
   committee: readonly `0x${string}`[];
-  slashFactory: SlashFactoryContract;
-  slashingProposer: EmpireSlashingProposerContract | TallySlashingProposerContract | undefined;
+  slashingProposer: SlashingProposerContract | undefined;
   slashingRoundSize: number;
   aztecSlotDuration: number;
   aztecEpochDuration: number;
@@ -216,36 +278,14 @@ export async function awaitCommitteeKicked({
 
   await cheatCodes.debugRollup();
 
-  if (slashingProposer.type === 'empire') {
-    // Await for the slash payload to be created if empire (no payload is created on tally until execution time)
-    const targetEpoch = EpochNumber((await cheatCodes.getEpoch()) + (await rollup.getLagInEpochsForValidatorSet()) + 1);
-    logger.info(`Advancing to epoch ${targetEpoch} so we start slashing`);
-    await cheatCodes.advanceToEpoch(targetEpoch);
-
-    const slashPayloadEvents = await retryUntil(
-      async () => {
-        const events = await slashFactory.getSlashPayloadCreatedEvents();
-        return events.length > 0 ? events : undefined;
-      },
-      'slash payload created',
-      120,
-      1,
-    );
-    expect(slashPayloadEvents.length).toBe(1);
-    // The uniqueness check is needed since a validator may be slashed more than once on the same round (eg because they let two epochs be pruned)
-    expect(unique(slashPayloadEvents[0].slashes.map(slash => slash.validator.toString()))).toHaveLength(
-      committee.length,
-    );
-  } else {
-    // Use the slash offset to ensure we are in the right epoch for tally
-    const slashOffsetInRounds = await slashingProposer.getSlashOffsetInRounds();
-    const slashingRoundSizeInEpochs = slashingRoundSize / aztecEpochDuration;
-    const slashingOffsetInEpochs = Number(slashOffsetInRounds) * slashingRoundSizeInEpochs;
-    const firstEpochInOffenseRound = offenseEpoch - (offenseEpoch % slashingRoundSizeInEpochs);
-    const targetEpoch = firstEpochInOffenseRound + slashingOffsetInEpochs;
-    logger.info(`Advancing to epoch ${targetEpoch} so we start slashing`);
-    await cheatCodes.advanceToEpoch(EpochNumber(targetEpoch), { offset: -aztecSlotDuration / 2 });
-  }
+  // Use the slash offset to ensure we are in the right epoch for tally
+  const slashOffsetInRounds = await slashingProposer.getSlashOffsetInRounds();
+  const slashingRoundSizeInEpochs = slashingRoundSize / aztecEpochDuration;
+  const slashingOffsetInEpochs = Number(slashOffsetInRounds) * slashingRoundSizeInEpochs;
+  const firstEpochInOffenseRound = offenseEpoch - (offenseEpoch % slashingRoundSizeInEpochs);
+  const targetEpoch = firstEpochInOffenseRound + slashingOffsetInEpochs;
+  logger.info(`Advancing to epoch ${targetEpoch} so we start slashing`);
+  await cheatCodes.advanceToEpoch(EpochNumber(targetEpoch), { offset: -aztecSlotDuration / 2 });
 
   const attestersPre = await rollup.getAttesters();
   expect(attestersPre.length).toBe(committee.length);
@@ -255,7 +295,10 @@ export async function awaitCommitteeKicked({
     expect(attesterInfo.status).toEqual(1); // Validating
   }
 
-  const timeout = slashingRoundSize * 2 * aztecSlotDuration + 30;
+  // Allow up to four round-lengths so that under proposer pipelining, where individual rounds
+  // sometimes fail to gather quorum because parts of the committee miss votes due to chain-state
+  // races, we still see a later round execute the slash.
+  const timeout = slashingRoundSize * 4 * aztecSlotDuration + 30;
   logger.info(`Waiting for slash to be executed (timeout ${timeout}s)`);
   await awaitProposalExecution(slashingProposer, timeout, logger);