@aztec/end-to-end 5.0.0-rc.1 → 5.0.0

package/src/shared/timing_env.mjs

@@ -0,0 +1,363 @@
+import { appendFileSync, writeFileSync } from 'node:fs';
+import { basename } from 'node:path';
+
+import CustomEnvironment from '../../../foundation/src/jest/env.mjs';
+
+// Per-test e2e timing environment. Gated entirely on the TEST_TIMING_FILE env var: when unset, this
+// behaves exactly like the base CustomEnvironment (it only delegates). When set, it records, per test
+// worker process, the time spent in jest before/after hooks and the test body, and folds in the named
+// spans recorded via the `testSpan()` wrapper (fixtures/timing.ts) through a collector shared on
+// `this.global`.
+//
+// Output is one JSONL file per worker process (the env runs once per worker). Each line carries a
+// `type` discriminator and is one of:
+//   - `type: 'test'` (`name` set): beforeEach hooks, the it() body, afterEach hooks for one test; or
+//   - `type: 'suite'` (`name: null`): the suite-scoped beforeAll/afterAll hooks for the whole file.
+//
+// Each line also gains a `spans` map: per `category:label` tag the test/suite touched, the collector
+// records `{ count, totalMs, busyMs, maxMs }`. `busyMs` is the duration of the union of the spans'
+// `[start, end)` intervals on the shared `performance.now()` clock, so concurrent spans (Promise.all
+// fan-out) do not inflate it the way `totalMs` does. The legacy `setupFnMs`/`teardownFnMs` fields are
+// kept for back-compat, derived from the `setup:env`/`teardown:env` spans.
+//
+// When `TEST_TIMING_SPANS=1`, the collector additionally retains every raw span occurrence and emits
+// one `type:"span"` line per occurrence (owner, name, ms) for deep dives, at the cost of a larger file.
+export default class TimingEnvironment extends CustomEnvironment {
+  constructor(config, context) {
+    super(config, context);
+
+    this.timingFile = process.env.TEST_TIMING_FILE;
+    if (!this.timingFile) {
+      return;
+    }
+
+    this.suite = basename(context?.testPath ?? config?.testPath ?? 'unknown').replace(/\.test\.[cm]?[jt]s$/, '');
+    this.meta = {
+      commit: process.env.COMMIT_HASH ?? null,
+      branch: process.env.TARGET_BRANCH ?? process.env.REF_NAME ?? null,
+      runId: process.env.RUN_ID ?? null,
+    };
+
+    // Opt-in per-occurrence emission for deep dives (off by default keeps the JSONL one line per test).
+    this.emitSpanLines = process.env.TEST_TIMING_SPANS === '1';
+
+    // Shared collector. The instrumented helpers (running in the sandbox realm) read
+    // `globalThis.__e2eTimings`; this env (host realm) reads `this.global.__e2eTimings` — they are the
+    // same object. `current` is the full name of the test currently running (null during
+    // beforeAll/afterAll), used to attribute each span to a test or to the suite-scoped line. `spans`
+    // accumulates every recorded span until flush. The `start`/`end` timestamps come from the sandbox
+    // realm's `performance.now()`; Node's perf clock is process-wide (one monotonic origin across all
+    // realms), so merging the intervals here for `busyMs` is valid.
+    this.collector = { current: null, spans: [] };
+    this.global.__e2eTimings = this.collector;
+
+    // Records to flush as JSONL on teardown. One per test plus one suite-scoped record.
+    this.records = [];
+    this.recordsByName = new Map();
+    // The suite-scoped record (beforeAll/afterAll); bodyMs/totalMs are computed at flush time.
+    this.suiteRecord = { name: null, status: 'passed', beforeHooksMs: 0, afterHooksMs: 0 };
+    // Start times keyed by hook type (beforeAll/beforeEach/afterAll/afterEach).
+    this.hookStarts = {};
+    this.testStarts = new Map();
+    // Guards against double-flushing (we flush on both the teardown event and the teardown method).
+    this.flushed = false;
+    // Raw per-occurrence spans retained only when TEST_TIMING_SPANS=1, emitted as `type:"span"` lines.
+    this.rawSpans = [];
+  }
+
+  // Flush on the teardown method too: the jest lifecycle always calls this, whereas the 'teardown'
+  // event is not reliably delivered when a test run is interrupted. Whichever fires first wins.
+  async teardown() {
+    this.finalizeAndFlush();
+    await super.teardown();
+  }
+
+  async handleTestEvent(event, state) {
+    // Run the base env first so its unhandledRejection patching for after-hooks stays intact.
+    await super.handleTestEvent(event, state);
+
+    if (!this.timingFile) {
+      return;
+    }
+
+    const now = Date.now();
+    switch (event.name) {
+      case 'hook_start': {
+        this.hookStarts[event.hook.type] = now;
+        break;
+      }
+      case 'hook_success':
+      case 'hook_failure': {
+        const start = this.hookStarts[event.hook.type];
+        if (start === undefined) {
+          break;
+        }
+        delete this.hookStarts[event.hook.type];
+        const ms = now - start;
+        if (event.hook.type === 'beforeEach') {
+          this.addToCurrent('beforeHooksMs', ms);
+        } else if (event.hook.type === 'afterEach') {
+          this.addToCurrent('afterHooksMs', ms);
+        } else if (event.hook.type === 'beforeAll') {
+          this.suiteRecord.beforeHooksMs += ms;
+        } else if (event.hook.type === 'afterAll') {
+          this.suiteRecord.afterHooksMs += ms;
+        }
+        break;
+      }
+      case 'test_start': {
+        const name = this.fullTestName(event.test);
+        const record = {
+          type: 'test',
+          name,
+          status: 'passed',
+          setupFnMs: 0,
+          beforeHooksMs: 0,
+          bodyMs: 0,
+          teardownFnMs: 0,
+          afterHooksMs: 0,
+          totalMs: 0,
+          startedAt: new Date(now).toISOString(),
+        };
+        this.records.push(record);
+        this.recordsByName.set(name, record);
+        this.testStarts.set(name, now);
+        this.collector.current = name;
+        break;
+      }
+      case 'test_fn_start': {
+        this.testStarts.set(`fn:${this.fullTestName(event.test)}`, now);
+        break;
+      }
+      case 'test_fn_success':
+      case 'test_fn_failure': {
+        const name = this.fullTestName(event.test);
+        const start = this.testStarts.get(`fn:${name}`);
+        const record = this.recordsByName.get(name);
+        if (record && start !== undefined) {
+          record.bodyMs = now - start;
+        }
+        if (record && event.name === 'test_fn_failure') {
+          record.status = 'failed';
+        }
+        break;
+      }
+      case 'test_done': {
+        const name = this.fullTestName(event.test);
+        const record = this.recordsByName.get(name);
+        const start = this.testStarts.get(name);
+        if (record && start !== undefined) {
+          record.totalMs = now - start;
+        }
+        if (record && event.test?.errors?.length) {
+          record.status = 'failed';
+        }
+        this.collector.current = null;
+        break;
+      }
+      case 'teardown': {
+        this.finalizeAndFlush();
+        break;
+      }
+      default:
+        break;
+    }
+  }
+
+  /** The record for the test currently running, if any. */
+  currentRecord() {
+    return this.collector.current ? this.recordsByName.get(this.collector.current) : undefined;
+  }
+
+  /** Adds `ms` to a hook bucket on the current per-test record, or the suite record during beforeAll/afterAll. */
+  addToCurrent(field, ms) {
+    const record = this.currentRecord();
+    if (record) {
+      record[field] += ms;
+    }
+  }
+
+  /** Builds the full test name by walking up the describe blocks, matching jest's currentTestName. */
+  fullTestName(test) {
+    if (!test) {
+      return null;
+    }
+    const parts = [];
+    let node = test;
+    while (node && node.name && node.name !== 'ROOT_DESCRIBE_BLOCK') {
+      parts.unshift(node.name);
+      node = node.parent;
+    }
+    return parts.join(' ');
+  }
+
+  /** Folds the collected spans into the per-test / suite records via {@link foldSpansInto}, then drains the collector. */
+  foldSpans() {
+    const rawSpans = foldSpansInto(this.collector.spans, {
+      recordsByName: this.recordsByName,
+      suiteRecord: this.suiteRecord,
+      emitSpanLines: this.emitSpanLines,
+    });
+    this.rawSpans.push(...rawSpans);
+    this.collector.spans = [];
+  }
+
+  /** Folds spans into records, finalizes the suite-scoped record, and writes all JSONL. */
+  finalizeAndFlush() {
+    if (this.flushed || !this.timingFile) {
+      return;
+    }
+    this.flushed = true;
+    this.foldSpans();
+
+    const lines = [];
+    for (const record of this.records) {
+      lines.push(this.toLine({ ...record }));
+    }
+    lines.push(
+      this.toLine({
+        type: 'suite',
+        name: null,
+        status: 'passed',
+        setupFnMs: this.suiteRecord.setupFnMs ?? 0,
+        beforeHooksMs: this.suiteRecord.beforeHooksMs,
+        teardownFnMs: this.suiteRecord.teardownFnMs ?? 0,
+        afterHooksMs: this.suiteRecord.afterHooksMs,
+        totalMs: this.suiteRecord.beforeHooksMs + this.suiteRecord.afterHooksMs,
+        spans: this.suiteRecord.spans,
+      }),
+    );
+    for (const raw of this.rawSpans) {
+      lines.push(this.toLine({ type: 'span', name: raw.owner, span: raw.name, ms: raw.ms }));
+    }
+
+    const payload = lines.join('\n') + '\n';
+    try {
+      // One file per worker process, but a worker may run several suites: append so we keep them all.
+      appendFileSync(this.timingFile, payload);
+    } catch {
+      try {
+        writeFileSync(this.timingFile, payload);
+      } catch {
+        // Timing is best-effort; never fail a test because we couldn't write the file.
+      }
+    }
+  }
+
+  /** Flattens metadata onto a record and serializes to a single JSON line, rounding all Ms fields. */
+  toLine(record) {
+    const obj = { suite: this.suite, ...record, ...this.meta };
+    for (const key of Object.keys(obj)) {
+      if (key.endsWith('Ms') && typeof obj[key] === 'number') {
+        obj[key] = Math.round(obj[key]);
+      }
+    }
+    // Round the nested span aggregates' Ms fields too (totalMs/busyMs/maxMs per tag).
+    if (obj.spans && typeof obj.spans === 'object') {
+      for (const agg of Object.values(obj.spans)) {
+        for (const key of Object.keys(agg)) {
+          if (key.endsWith('Ms') && typeof agg[key] === 'number') {
+            agg[key] = Math.round(agg[key]);
+          }
+        }
+      }
+    }
+    return JSON.stringify(obj);
+  }
+}
+
+/**
+ * Aggregates a list of `{ start, end }` spans of one tag into `{ count, totalMs, busyMs, maxMs }`.
+ * `totalMs` is the naive sum of per-occurrence durations (correct for serial repeats); `busyMs` is
+ * the duration of the union of the `[start, end)` intervals (concurrency-correct — N concurrent
+ * spans collapse to their wall-clock span instead of summing to ~N×); `maxMs` is the longest single
+ * occurrence. All durations are on the shared process-wide `performance.now()` clock. Does not mutate
+ * the input array.
+ */
+export function aggregateSpans(spans) {
+  let totalMs = 0;
+  let maxMs = 0;
+  for (const span of spans) {
+    const ms = span.end - span.start;
+    totalMs += ms;
+    if (ms > maxMs) {
+      maxMs = ms;
+    }
+  }
+  // busyMs: sort by start and merge overlapping intervals, summing the merged lengths.
+  const sorted = [...spans].sort((a, b) => a.start - b.start);
+  let busyMs = 0;
+  let mergeStart = null;
+  let mergeEnd = null;
+  for (const span of sorted) {
+    if (mergeStart === null) {
+      mergeStart = span.start;
+      mergeEnd = span.end;
+    } else if (span.start <= mergeEnd) {
+      if (span.end > mergeEnd) {
+        mergeEnd = span.end;
+      }
+    } else {
+      busyMs += mergeEnd - mergeStart;
+      mergeStart = span.start;
+      mergeEnd = span.end;
+    }
+  }
+  if (mergeStart !== null) {
+    busyMs += mergeEnd - mergeStart;
+  }
+  return { count: spans.length, totalMs, busyMs, maxMs };
+}
+
+/**
+ * Groups `spans` by owner, then by tag, and attaches a `spans` map of aggregates to the matching
+ * per-test record (looked up in `recordsByName`) or, for `null`-owner spans, to `suiteRecord`. Also
+ * derives the back-compat `setupFnMs` / `teardownFnMs` fields from the `setup:env:<mode>` /
+ * `teardown:env` aggregates (additive over any pre-existing value). Spans whose owner matches no
+ * record (e.g. the pinned `other:mempool-feeder` owner) are dropped. Returns the raw per-occurrence
+ * list (`{ owner, name, ms }`), which is empty unless `emitSpanLines` is set.
+ */
+export function foldSpansInto(spans, { recordsByName, suiteRecord, emitSpanLines = false }) {
+  const rawSpans = [];
+  // Group by owner (null owner → suite record), then by tag.
+  const byOwner = new Map();
+  for (const span of spans) {
+    const owner = span.owner ?? null;
+    let byName = byOwner.get(owner);
+    if (!byName) {
+      byName = new Map();
+      byOwner.set(owner, byName);
+    }
+    let list = byName.get(span.name);
+    if (!list) {
+      list = [];
+      byName.set(span.name, list);
+    }
+    list.push(span);
+    if (emitSpanLines) {
+      rawSpans.push({ owner, name: span.name, ms: span.end - span.start });
+    }
+  }
+
+  for (const [owner, byName] of byOwner) {
+    const record = owner === null ? suiteRecord : recordsByName.get(owner);
+    if (!record) {
+      continue;
+    }
+    const aggregates = {};
+    for (const [name, list] of byName) {
+      aggregates[name] = aggregateSpans(list);
+    }
+    record.spans = aggregates;
+    // Back-compat: the legacy fn-span buckets are derived from the top-level env setup/teardown tags.
+    // The top-level setup is tagged `setup:env:<proverMode>` (the mode is a closed set, distinct from
+    // the `:anvil`/`:l1-deploy`/... sub-phase tags), so sum those three exact tags.
+    const setupFnMs =
+      (aggregates['setup:env:none']?.totalMs ?? 0) +
+      (aggregates['setup:env:fake']?.totalMs ?? 0) +
+      (aggregates['setup:env:real']?.totalMs ?? 0);
+    record.setupFnMs = setupFnMs + (record.setupFnMs ?? 0);
+    record.teardownFnMs = (aggregates['teardown:env']?.totalMs ?? 0) + (record.teardownFnMs ?? 0);
+  }
+  return rawSpans;
+}

package/src/shared/wait_for_l1_to_l2_message.ts

@@ -18,6 +18,6 @@ export function waitForL1ToL2MessageSeen(
     async () => (await node.getL1ToL2MessageCheckpoint(l1ToL2MessageHash)) !== undefined,
     `L1 to L2 message ${l1ToL2MessageHash.toString()} seen`,
     opts.timeoutSeconds,
-    1,
+    0.25,
   );
 }

package/src/simulators/lending_simulator.ts

@@ -208,7 +208,7 @@ export class LendingSimulator {
     expect(interestAccumulator).toEqual(this.accumulator);
     expect(asset['last_updated_ts']).toEqual(BigInt(this.time));
 
-    for (const key of [this.account.address, AztecAddress.fromField(await this.account.key())]) {
+    for (const key of [this.account.address, AztecAddress.fromFieldUnsafe(await this.account.key())]) {
       const { result: privatePos } = await this.lendingContract.methods
         .get_position(key)
         .simulate({ from: this.account.address });

package/src/single-node/README.md

@@ -0,0 +1,66 @@
+# `single-node` e2e test category
+
+Single-node tests run one Aztec node with the production sequencer (and, where a test needs proving,
+a prover node). They cover behavior that only requires a single sequencer and no multi-validator
+committee: block building, sequencer config and governance signalling, world-state sync, the
+proving/epoch lifecycle, partial proofs, L1-reorg handling, and pending-chain recovery.
+
+## Base class
+
+All tests use `SingleNodeTestContext` (`single_node_test_context.ts`), which owns:
+
+- The environment: an in-process anvil plus the L1 contract deploy.
+- Node spawning: `createNonValidatorNode` and `createProverNode` (the latter wires the mock-gossip
+  `p2pServiceFactory`, which is harmless with a single node).
+- The `ChainMonitor`.
+- The epoch / checkpoint / proof-window / reorg waiters and assertion helpers (`waitUntilEpochStarts`,
+  `waitUntilProvenCheckpointNumber`, `waitForNodeToSync`, `verifyHistoricBlock`, …).
+
+`MultiNodeTestContext` (in `../multi-node/`) extends this base with the N-validator topology, so the
+multi-node category inherits the same environment and waiters.
+
+Prefer these named waiters — and the node-level ones in `../fixtures/wait_helpers.ts` — over hand-rolled
+`retryUntil` / `.on` / `sleep` polling in test bodies; the full catalog, including the helpers this base
+class defines, is listed under [Helper surface](../multi-node/README.md#helper-surface) in the multi-node
+README.
+
+## Setup factories
+
+`setup.ts` holds thin factories over `SingleNodeTestContext.setup`, named by the prover mode a test
+wants. Tests call the factory rather than the static method directly:
+
+- `setupWithProver(opts)` — a single sequencer plus the context's fake in-process prover node. This is
+  the default the proving / partial-proofs / l1-reorgs / recovery / misc suites use.
+- `setupBlockProducer(opts)` — a single production sequencer with **no prover node**, used by the
+  block-building / sequencer / sync suites. It raises `aztecProofSubmissionEpochs` to
+  `NO_REORG_SUBMISSION_EPOCHS` (1024) so unproven blocks are not pruned without a prover, and points the
+  PXE at `syncChainTip: 'proposed'` so tests can assert on freshly proposed blocks. Both are overridable
+  via `opts`.
+
+The `prover/` suite (real Barretenberg proofs) builds its environment through `FullProverTest`, which
+extends `SingleNodeTestContext` directly rather than going through a factory.
+
+## Organizing principle
+
+The top level groups tests by node topology and setup model; the second level names the primary
+behavior under test (the proving lifecycle, partial proofs, reorgs, recovery) rather than the shared
+setup. Each file has a single top-level `describe` named to match its path, and a co-located
+`setup.ts` holds any shared timeout/option wiring. A `.parallel` suffix marks files with more than one
+top-level `it`; CI splits each `it` into its own job.
+
+## Subfolders
+
+| Path | Contents |
+|---|---|
+| `block-building/` | Block assembly mechanics under the production sequencer with pipelining. `block_building` (multi-tx blocks, double-spend rejection, log ordering, regressions, and L1 reorgs), `debug_trace` (blocks proposed through a Forwarder proxy, including a failing-then-succeeding propose call), `multiple_blobs` (a block whose combined side effects span more than one EIP-4844 blob). |
+| `sequencer/` | Sequencer configuration, governance signalling, and publisher management on a single node. `gov_proposal.parallel` (a 16-validator committee proposes blocks while casting governance votes, and votes even when block building is disabled), `escape_hatch_vote_only` (governance signals advance while the escape hatch is closed), `reload_keystore` (the keystore is hot-reloaded to add a validator and pick up new coinbases), `multi_eoa` (publisher rotation when an L1 tx is withheld; exercises multi-EOA publisher failover), `publisher_funding_multi` (PublisherManager auto top-up of publisher EOAs when balances drop below threshold), `runtime_config` (the former `slasher_config` and `sequencer_config` admin-API checks on one node: slasher inactivity-config getters, plus runtime `maxL2BlockGas`/`manaTarget` reconfiguration enforced via a live Bot). |
+| `fees/` | Fee mechanics on a single node. `fee_asset_price_oracle` (on-chain fee-asset price-oracle convergence; starts its own Anvil instance with a MockStateView etched at the StateView address). The remaining files run on the `FeesTest` harness (`fees_test.ts`), which extends `SingleNodeTestContext` with the fee/gas domain setup (FPC funding, fee-juice bridging, banana token): `account_init` (fee payment during account-contract initialization), `failures` (fees still charged when txs revert in setup/app/teardown), `fee_juice_payments` (direct Fee Juice payment with and without initial funds), `fee_settings` (max-fee-per-gas handling, stale-fee-snapshot race, governance fee-config bump), `gas_estimation.parallel` (gas-estimation accuracy and FPC teardown gas prediction), `private_payments.parallel` (private fee payment via the BananaCoin FPC, plus the single-assertion public-FPC and sponsored-FPC payment checks folded onto the same fixture from the former `public_payments`/`sponsored_payments` files). |
+| `cross-chain/` | L1↔L2 messaging on a single node, on the `CrossChainMessagingTest` harness (`cross_chain_messaging_test.ts`), which extends `SingleNodeTestContext` and owns a `CrossChainTestHarness` plus the L1 inbox/outbox handles (it auto-proves via an `EpochTestSettler` when no prover node runs). The shared L1→L2 message helpers live in `message_test_helpers.ts` (`createL1ToL2MessageHelpers`, whose injected `markAsProven` lets each suite plug in its own proving policy). `l1_to_l2` (L1→L2 message readiness and duplicate messages, over private and public scope), `l1_to_l2_inbox_drift` (inbox checkpoint drift after a rollup reorg, over private and public scope), `l2_to_l1` (L2→L1 message inclusion across single/multi-message txs and multi-block checkpoints, subtree-root balancing, and a reorg-and-remine case), `token_bridge` (private and public L1→L2 deposits and L2→L1 withdrawals via the TokenBridge — including mint-on-behalf — plus the withdrawal/claim failure cases, merged from the former three `token_bridge_*` files). |
+| `bot/` | Transaction bot implementations. `bot` (transfer bot, AMM bot, and cross-chain bot; exercises fee-juice portal deposits, L2→L1 messages, and bot contract reuse). |
+| `sync/` | World-state sync stress and snapshot sync. `synching` builds fixture block data (env-gated, slow) and replays it for sync benchmarks and prune/reorg scenarios (only the outer `it.each` runs in CI); `snapshot_sync` exercises the node snapshot upload/download path, syncing fresh nodes from one or multiple snapshot URLs, including fallback from a corrupted snapshot. |
+| `proving/` | Epoch and proof lifecycle. `default_node` (basic proving/node coverage that shares one context-default node across three suites: consecutive epochs prove and finalized blocks are purged from world state beyond the checkpoint-history window, a proof is submitted even with no txs, and the node returns initial genesis-block data — the former `world_state_pruning`, `empty_blocks`, and `node_block_api` files), `long_proving_time` (a prover delay spanning multiple epochs), `multi_proof` (multiple prover nodes prove one epoch), `optimistic.parallel` (checkpoint-driven proving across the happy path and several mid-epoch / last-slot / during-proving reorg cases), `proof_fails.parallel` (proof not accepted after epoch end; proving aborts when the next epoch ends), `cross_chain_public_message` (an epoch with a public tx that consumes an L1→L2 message in the block it lands, guarding against a sequencer/prover state-root mismatch), `upload_failed_proof` (a failed proving job's state is uploaded and re-run on a fresh instance). |
+| `prover/` | Real-proof exercises on the `FullProverTest` harness (real Barretenberg when `FAKE_PROOFS=0`, fake otherwise), split by where the proving runs so CI can size their containers independently: `client/client` (client-side proof generation and `verifyProof` for private and public transfers, no on-chain submission) and `server/full` (the end-to-end pipeline: client proves, node builds blocks, prover node generates epoch proofs, L1 verifies them). |
+| `partial-proofs/` | Manually driven partial-proof submission. `single_root` (the prover node's `startProof` path on a single root) and `multi_root` (three partial-proof roots are staged and messages consume against any covering root, exercising the multi-root Outbox semantics). |
+| `l1-reorgs/` | Behavior under L1 reorgs, split by what reorgs. `blocks.parallel` (prune L2 blocks when a reorg drops a proof, hold when a replacement proof lands in the window, restore blocks when a proof reappears, prune pending-chain blocks, and see new blocks added by a reorg) and `messages.parallel` (L1→L2 messages updated by a reorg, and a missed message inserted by one). `setup.ts` holds the shared `FAST_REORG_TIMING` profile and delayer wiring. |
+| `recovery/` | Reorg and pending-chain recovery. `manual_rollback` (the `rollbackTo` admin API rolls back to an unfinalized block), `sync_after_reorg` (a fresh node syncs world state past an unpruned reorg window), `prune_when_cannot_build` (a solo sequencer prunes the pending chain via the fallback path when it cannot propose). |
+| `misc/` | Genuine single-node outliers. `missed_l1_slot` (the sequencer builds a block after missed L1 slots once the previous checkpoint is synced). |