@aztec/end-to-end 0.0.1-commit.381b1a9 → 0.0.1-commit.3a4ae741b

package/src/fixtures/elu_monitor.ts

@@ -0,0 +1,126 @@
+import { appendFileSync } from 'node:fs';
+import { type EventLoopUtilization, type IntervalHistogram, monitorEventLoopDelay, performance } from 'node:perf_hooks';
+
+const NANOS_PER_MS = 1_000_000;
+
+/** Samples event-loop utilization, delay histogram, and heap usage per test, writing columnar text to a file. */
+export class EluMonitor {
+  private filePath: string;
+  private intervalMs: number;
+  private timer: ReturnType<typeof setInterval> | undefined;
+  private lastELU: EventLoopUtilization | undefined;
+  private histogram: IntervalHistogram;
+  private testName: string | undefined;
+  private testStart: number | undefined;
+  private eluSamples: number[] = [];
+
+  constructor(filePath: string, intervalMs?: number) {
+    this.filePath = filePath;
+    this.intervalMs = intervalMs ?? 2000;
+    this.histogram = monitorEventLoopDelay({ resolution: 20 });
+  }
+
+  /** Begin sampling for a test. Writes a header line and starts the periodic sampler. */
+  startTest(testName: string): void {
+    this.stopTest();
+
+    this.testName = testName;
+    this.testStart = performance.now();
+    this.eluSamples = [];
+
+    appendFileSync(this.filePath, `\n=== Test: ${testName} ===\n`);
+    appendFileSync(
+      this.filePath,
+      padColumns('TIME', 'ELU', 'EL_DLY_P50', 'EL_DLY_P99', 'EL_DLY_MAX', 'HEAP_MB') + '\n',
+    );
+
+    this.lastELU = performance.eventLoopUtilization();
+    this.histogram.enable();
+
+    this.timer = setInterval(() => this.sample(), this.intervalMs);
+    // Allow the process to exit even if the timer is still running.
+    this.timer.unref();
+  }
+
+  /** Stop sampling and write a summary line. */
+  stopTest(): void {
+    if (!this.timer) {
+      return;
+    }
+
+    // Take a final sample before stopping.
+    this.sample();
+
+    clearInterval(this.timer);
+    this.timer = undefined;
+    this.histogram.disable();
+    this.histogram.reset();
+
+    this.writeSummary();
+
+    this.lastELU = undefined;
+    this.testName = undefined;
+    this.testStart = undefined;
+    this.eluSamples = [];
+  }
+
+  /** Alias for stopTest — call on process exit to flush any remaining data. */
+  stop(): void {
+    this.stopTest();
+  }
+
+  private sample(): void {
+    const newELU = performance.eventLoopUtilization();
+    const delta = performance.eventLoopUtilization(newELU, this.lastELU);
+    this.lastELU = newELU;
+
+    const elu = delta.utilization;
+    this.eluSamples.push(elu);
+
+    const p50 = this.histogram.percentile(50) / NANOS_PER_MS;
+    const p99 = this.histogram.percentile(99) / NANOS_PER_MS;
+    const max = this.histogram.max / NANOS_PER_MS;
+    const heapMb = Math.round(process.memoryUsage().heapUsed / (1024 * 1024));
+
+    const now = new Date();
+    const time = [now.getHours(), now.getMinutes(), now.getSeconds()].map(n => String(n).padStart(2, '0')).join(':');
+
+    const line = padColumns(
+      time,
+      elu.toFixed(2),
+      `${p50.toFixed(1)}ms`,
+      `${p99.toFixed(1)}ms`,
+      `${max.toFixed(1)}ms`,
+      String(heapMb),
+    );
+    appendFileSync(this.filePath, line + '\n');
+
+    // Reset histogram so next sample only reflects the new interval.
+    this.histogram.reset();
+  }
+
+  private writeSummary(): void {
+    if (this.eluSamples.length === 0 || this.testStart === undefined) {
+      return;
+    }
+
+    const mean = this.eluSamples.reduce((a, b) => a + b, 0) / this.eluSamples.length;
+    const maxElu = Math.max(...this.eluSamples);
+    const sorted = [...this.eluSamples].sort((a, b) => a - b);
+    const p90Elu = sorted[Math.floor(sorted.length * 0.9)] ?? maxElu;
+    const durationS = ((performance.now() - this.testStart) / 1000).toFixed(1);
+
+    let summary = `--- Summary: mean_elu=${mean.toFixed(2)} max_elu=${maxElu.toFixed(2)} p90_elu=${p90Elu.toFixed(2)} duration=${durationS}s`;
+    if (maxElu > 0.85) {
+      summary += ' WARNING:ELU>0.85';
+    }
+    summary += ' ---\n';
+
+    appendFileSync(this.filePath, summary);
+  }
+}
+
+function padColumns(...cols: string[]): string {
+  const widths = [11, 7, 12, 12, 12, 8];
+  return cols.map((col, i) => col.padEnd(widths[i] ?? 10)).join('');
+}

package/src/fixtures/get_bb_config.ts

@@ -13,8 +13,10 @@ const {
   BB_SKIP_CLEANUP = '',
   TEMP_DIR = tmpdir(),
   BB_WORKING_DIRECTORY = '',
-  BB_NUM_IVC_VERIFIERS = '1',
+  BB_NUM_IVC_VERIFIERS = '8',
   BB_IVC_CONCURRENCY = '1',
+  BB_CHONK_VERIFY_MAX_BATCH = '16',
+  BB_CHONK_VERIFY_BATCH_CONCURRENCY = '6',
 } = process.env;
 
 export const getBBConfig = async (
@@ -41,16 +43,15 @@ export const getBBConfig = async (
     const bbSkipCleanup = ['1', 'true'].includes(BB_SKIP_CLEANUP);
     const cleanup = bbSkipCleanup ? () => Promise.resolve() : () => tryRmDir(directoryToCleanup);
 
-    const numIvcVerifiers = Number(BB_NUM_IVC_VERIFIERS);
-    const ivcConcurrency = Number(BB_IVC_CONCURRENCY);
-
     return {
       bbSkipCleanup,
       bbBinaryPath,
       bbWorkingDirectory,
       cleanup,
-      numConcurrentIVCVerifiers: numIvcVerifiers,
-      bbIVCConcurrency: ivcConcurrency,
+      numConcurrentIVCVerifiers: Number(BB_NUM_IVC_VERIFIERS),
+      bbIVCConcurrency: Number(BB_IVC_CONCURRENCY),
+      bbChonkVerifyMaxBatch: Number(BB_CHONK_VERIFY_MAX_BATCH),
+      bbChonkVerifyConcurrency: Number(BB_CHONK_VERIFY_BATCH_CONCURRENCY),
     };
   } catch (err) {
     logger.error(`Native BB not available, error: ${err}`);

package/src/fixtures/setup.ts

@@ -1,6 +1,7 @@
 import { SchnorrAccountContractArtifact } from '@aztec/accounts/schnorr';
 import { type InitialAccountData, generateSchnorrAccounts } from '@aztec/accounts/testing';
 import { type AztecNodeConfig, AztecNodeService, getConfigEnvVars } from '@aztec/aztec-node';
+import { NO_FROM } from '@aztec/aztec.js/account';
 import { AztecAddress, EthAddress } from '@aztec/aztec.js/addresses';
 import {
   BatchCall,
@@ -29,7 +30,8 @@ import {
   deployAztecL1Contracts,
 } from '@aztec/ethereum/deploy-aztec-l1-contracts';
 import type { Delayer } from '@aztec/ethereum/l1-tx-utils';
-import { type Anvil, EthCheatCodes, EthCheatCodesWithState, startAnvil } from '@aztec/ethereum/test';
+import { EthCheatCodes, EthCheatCodesWithState, startAnvil } from '@aztec/ethereum/test';
+import type { Anvil } from '@aztec/ethereum/test';
 import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
 import { SecretValue } from '@aztec/foundation/config';
 import { randomBytes } from '@aztec/foundation/crypto/random';
@@ -301,6 +303,8 @@ export async function setup(
       config.dataDirectory = directoryToCleanup;
     }
 
+    const dateProvider = new TestDateProvider();
+
     if (!config.l1RpcUrls?.length) {
       if (!isAnvilTestChain(chain.id)) {
         throw new Error(`No ETHEREUM_HOSTS set but non anvil chain requested`);
@@ -310,6 +314,7 @@ export async function setup(
         accounts: opts.anvilAccounts,
         port: opts.anvilPort ?? (process.env.ANVIL_PORT ? parseInt(process.env.ANVIL_PORT) : undefined),
         slotsInAnEpoch: opts.anvilSlotsInAnEpoch,
+        dateProvider,
       });
       anvil = res.anvil;
       config.l1RpcUrls = [res.rpcUrl];
@@ -321,8 +326,6 @@ export async function setup(
       logger.info(`Logging metrics to ${filename}`);
       setupMetricsLogger(filename);
     }
-
-    const dateProvider = new TestDateProvider();
     const ethCheatCodes = new EthCheatCodesWithState(config.l1RpcUrls, dateProvider);
 
     if (opts.stateLoad) {
@@ -418,11 +421,12 @@ export async function setup(
       await ethCheatCodes.setIntervalMining(config.ethereumSlotDuration);
     }
 
-    // Always sync dateProvider to L1 time after deploying L1 contracts, regardless of mining mode.
-    // In compose mode, L1 time may have drifted ahead of system time due to the local-network watcher
-    // warping time forward on each filled slot. Without this sync, the sequencer computes the wrong
-    // slot from its dateProvider and cannot propose blocks.
-    dateProvider.setTime((await ethCheatCodes.timestamp()) * 1000);
+    // In compose mode (no local anvil), sync dateProvider to L1 time since it may have drifted
+    // ahead of system time due to the local-network watcher warping time forward on each filled slot.
+    // When running with a local anvil, the dateProvider is kept in sync via the stdout listener.
+    if (!anvil) {
+      dateProvider.setTime((await ethCheatCodes.lastBlockTimestamp()) * 1000);
+    }
 
     if (opts.l2StartTime) {
       await ethCheatCodes.warp(opts.l2StartTime, { resetBlockInterval: true });
@@ -846,7 +850,7 @@ export const deployAccounts =
       );
       const deployMethod = await accountManager.getDeployMethod();
       await deployMethod.send({
-        from: AztecAddress.ZERO,
+        from: NO_FROM,
         skipClassPublication: i !== 0, // Publish the contract class at most once.
       });
     }

package/src/fixtures/token_utils.ts

@@ -27,8 +27,9 @@ export async function mintTokensToPrivate(
   minter: AztecAddress,
   recipient: AztecAddress,
   amount: bigint,
+  additionalScopes?: AztecAddress[],
 ) {
-  await token.methods.mint_to_private(recipient, amount).send({ from: minter });
+  await token.methods.mint_to_private(recipient, amount).send({ from: minter, additionalScopes });
 }
 
 export async function expectTokenBalance(

package/src/shared/jest_setup.ts

@@ -1,8 +1,19 @@
 import { createLogger } from '@aztec/aztec.js/log';
 
-import { beforeEach, expect } from '@jest/globals';
+import { afterAll, afterEach, beforeEach, expect } from '@jest/globals';
+import { readlinkSync } from 'fs';
 import { basename } from 'path';
 
+import { EluMonitor } from '../fixtures/elu_monitor.js';
+
+const eluMonitor = process.env.ELU_MONITOR_FILE
+  ? new EluMonitor(process.env.ELU_MONITOR_FILE, Number(process.env.ELU_MONITOR_INTERVAL_MS) || undefined)
+  : undefined;
+
+if (eluMonitor) {
+  process.on('exit', () => eluMonitor.stop());
+}
+
 beforeEach(() => {
   const { testPath, currentTestName } = expect.getState();
   if (!testPath || !currentTestName) {
@@ -10,4 +21,43 @@ beforeEach(() => {
   }
   const logger = createLogger(`e2e:${basename(testPath).replace('.test.ts', '')}`);
   logger.info(`Running test: ${currentTestName}`);
+  eluMonitor?.startTest(currentTestName);
+});
+
+afterEach(() => {
+  eluMonitor?.stopTest();
+});
+
+// Log leaked handles after all tests complete. This runs after test-level afterAll hooks,
+// so any handles still alive at this point were not properly cleaned up during teardown.
+// This diagnostic helps identify the source of exit hangs without masking them.
+afterAll(() => {
+  const handles = (process as any)._getActiveHandles();
+  if (handles.length > 0) {
+    const details = handles.map((h: any) => {
+      const type = h?.constructor?.name ?? typeof h;
+      const fd = h?.fd ?? h?._handle?.fd ?? '?';
+      const destroyed = h?.destroyed ?? '?';
+      const hasRef = typeof h?.hasRef === 'function' ? h.hasRef() : '?';
+      const localAddr = h?.localAddress ?? '';
+      const remoteAddr = h?.remoteAddress ?? '';
+      const localPort = h?.localPort ?? '';
+      const remotePort = h?.remotePort ?? '';
+      const proto = Object.getPrototypeOf(h)?.constructor?.name ?? '?';
+      const keys = Object.keys(h).slice(0, 10).join(',');
+      let fdTarget = '';
+      if (typeof fd === 'number') {
+        try {
+          fdTarget = ` -> ${readlinkSync(`/proc/self/fd/${fd}`)}`;
+        } catch {
+          // ignore
+        }
+      }
+      return `  ${type}(fd=${fd}, destroyed=${destroyed}, hasRef=${hasRef}${fdTarget}) proto=${proto} addr=${localAddr}:${localPort}->${remoteAddr}:${remotePort} keys=[${keys}]`;
+    });
+    process.stderr.write(
+      `\n[jest_setup] WARNING: ${handles.length} handle(s) still active after teardown:\n${details.join('\n')}\n` +
+        `These may prevent Jest from exiting. Investigate and fix the leak.\n\n`,
+    );
+  }
 });

package/src/simulators/lending_simulator.ts

@@ -94,7 +94,9 @@ export class LendingSimulator {
 
   async prepare() {
     this.accumulator = BASE;
-    const slot = await this.rollup.getSlotAt(BigInt(await this.cc.eth.timestamp()) + BigInt(this.ethereumSlotDuration));
+    const slot = await this.rollup.getSlotAt(
+      BigInt(await this.cc.eth.lastBlockTimestamp()) + BigInt(this.ethereumSlotDuration),
+    );
     this.time = Number(await this.rollup.getTimestampForSlot(slot));
   }
 
@@ -103,7 +105,7 @@ export class LendingSimulator {
       return;
     }
 
-    const slot = await this.rollup.getSlotAt(BigInt(await this.cc.eth.timestamp()));
+    const slot = await this.rollup.getSlotAt(BigInt(await this.cc.eth.lastBlockTimestamp()));
     const targetSlot = SlotNumber(slot + diff);
     const ts = Number(await this.rollup.getTimestampForSlot(targetSlot));
     const timeDiff = ts - this.time;

package/src/simulators/token_simulator.ts

@@ -110,7 +110,7 @@ export class TokenSimulator {
         chunk(calls, 5).map(batch => new BatchCall(this.defaultWallet, batch).simulate({ from: this.defaultAddress })),
       )
     )
-      .flat()
+      .flatMap(r => r.result)
       .map(r => r.result);
     expect(results[0]).toEqual(this.totalSupply);
 

package/src/spartan/setup_test_wallets.ts

@@ -1,4 +1,5 @@
 import { generateSchnorrAccounts } from '@aztec/accounts/testing';
+import { NO_FROM } from '@aztec/aztec.js/account';
 import { AztecAddress } from '@aztec/aztec.js/addresses';
 import { NO_WAIT } from '@aztec/aztec.js/contracts';
 import { L1FeeJuicePortalManager } from '@aztec/aztec.js/ethereum';
@@ -88,11 +89,19 @@ export async function deploySponsoredTestAccountsWithTokens(
 
   const paymentMethod = new SponsoredFeePaymentMethod(await getSponsoredFPCAddress());
   const recipientDeployMethod = await recipientAccount.getDeployMethod();
-  await recipientDeployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod }, wait: { timeout: 2400 } });
+  await recipientDeployMethod.send({
+    from: NO_FROM,
+    fee: { paymentMethod },
+    wait: { timeout: 2400 },
+  });
   await Promise.all(
     fundedAccounts.map(async a => {
       const deployMethod = await a.getDeployMethod();
-      await deployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod }, wait: { timeout: 2400 } }); // increase timeout on purpose in order to account for two empty epochs
+      await deployMethod.send({
+        from: NO_FROM,
+        fee: { paymentMethod },
+        wait: { timeout: 2400 },
+      }); // increase timeout on purpose in order to account for two empty epochs
       logger.info(`Account deployed at ${a.address}`);
     }),
   );
@@ -133,12 +142,12 @@ async function deployAccountWithDiagnostics(
   try {
     let gasSettings;
     if (estimateGas) {
-      const sim = await deployMethod.simulate({ from: AztecAddress.ZERO, fee: { paymentMethod } });
+      const sim = await deployMethod.simulate({ from: NO_FROM, fee: { paymentMethod } });
       gasSettings = sim.estimatedGas;
       logger.info(`${accountLabel} estimated gas: DA=${gasSettings.gasLimits.daGas} L2=${gasSettings.gasLimits.l2Gas}`);
     }
     const deployResult = await deployMethod.send({
-      from: AztecAddress.ZERO,
+      from: NO_FROM,
       fee: { paymentMethod, gasSettings },
       wait: NO_WAIT,
     });
@@ -261,7 +270,7 @@ export async function deployTestAccountsWithTokens(
     fundedAccounts.map(async (a, i) => {
       const paymentMethod = new FeeJuicePaymentMethodWithClaim(a.address, claims[i]);
       const deployMethod = await a.getDeployMethod();
-      await deployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod } });
+      await deployMethod.send({ from: NO_FROM, fee: { paymentMethod } });
       logger.info(`Account deployed at ${a.address}`);
     }),
   );

package/src/spartan/tx_metrics.ts

@@ -296,7 +296,7 @@ export class TxInclusionMetrics {
           value: stats.mean,
         },
         {
-          name: `${group}/median_inclusion`,
+          name: `${group}/p50_inclusion`,
           unit: 's',
           value: stats.median,
         },

package/src/spartan/utils/config.ts

@@ -8,6 +8,7 @@ const logger = createLogger('e2e:k8s-utils');
 const testConfigSchema = z.object({
   NAMESPACE: z.string().default('scenario'),
   REAL_VERIFIER: schemas.Boolean.optional().default(true),
+  DEBUG_FORCE_TX_PROOF_VERIFICATION: schemas.Boolean.optional().default(true),
   CREATE_ETH_DEVNET: schemas.Boolean.optional().default(false),
   L1_RPC_URLS_JSON: z.string().optional(),
   L1_ACCOUNT_MNEMONIC: z.string().optional(),
@@ -16,6 +17,7 @@ const testConfigSchema = z.object({
   AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5),
   AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET: z.coerce.number().optional().default(2),
   FUNDING_PRIVATE_KEY: z.string().optional(),
+  AZTEC_ADMIN_API_KEY: z.string().optional(),
 });
 
 export type TestConfig = z.infer<typeof testConfigSchema>;

package/src/spartan/utils/index.ts

@@ -66,3 +66,6 @@ export { getPublicViemClient, getL1DeploymentAddresses, getNodeClient } from './
 
 // Health checks
 export { ChainHealth, type ChainHealthSnapshot } from './health.js';
+
+// Pod log extraction
+export { type BlockBuiltLogEntry, fetchBlockBuiltLogs } from './pod_logs.js';

package/src/spartan/utils/nodes.ts

@@ -173,7 +173,7 @@ export async function withSequencersAdmin<T>(env: TestConfig, fn: (node: AztecNo
           if (statusRes.status !== 200) {
             throw new Error(`Admin endpoint returned status ${statusRes.status}`);
           }
-          const client = createAztecNodeAdminClient(url);
+          const client = createAztecNodeAdminClient(url, {}, undefined, env.AZTEC_ADMIN_API_KEY);
           return { result: await fn(client), process };
         } catch (err) {
           // Kill the port-forward before retrying
@@ -255,21 +255,18 @@ export async function initHADb(namespace: string): Promise<void> {
 }
 
 /**
- * Enables or disables probabilistic transaction dropping on validators and waits for rollout.
- * Wired to env vars P2P_DROP_TX and P2P_DROP_TX_CHANCE via Helm values.
+ * Sets probabilistic transaction dropping on validators and waits for rollout.
+ * Use probability=0 to disable. Wired to env var P2P_DROP_TX_CHANCE via Helm values.
  */
 export async function setValidatorTxDrop({
   namespace,
-  enabled,
   probability,
   logger: log,
 }: {
   namespace: string;
-  enabled: boolean;
   probability: number;
   logger: Logger;
 }) {
-  const drop = enabled ? 'true' : 'false';
   const prob = String(probability);
 
   const selectors = ['app.kubernetes.io/name=validator', 'app.kubernetes.io/component=validator', 'app=validator'];
@@ -284,7 +281,7 @@ export async function setValidatorTxDrop({
       if (names.length === 0) {
         continue;
       }
-      const cmd = `kubectl set env statefulset -l ${selector} -n ${namespace} P2P_DROP_TX=${drop} P2P_DROP_TX_CHANCE=${prob}`;
+      const cmd = `kubectl set env statefulset -l ${selector} -n ${namespace} P2P_DROP_TX_CHANCE=${prob}`;
       log.info(`command: ${cmd}`);
       await execAsync(cmd);
       updated = true;
@@ -366,16 +363,24 @@ export async function enableValidatorDynamicBootNode(
  */
 export async function rollAztecPods(namespace: string, clearState: boolean = false) {
   // Pod components use 'validator', but StatefulSets and PVCs use 'sequencer-node' for validators
+  // RPC nodes have nodeType='rpc-node' in Helm values, so their component label is 'rpc-node' (not 'rpc')
   const podComponents = [
     'p2p-bootstrap',
     'prover-node',
     'prover-broker',
     'prover-agent',
     'sequencer-node',
-    'rpc',
+    'rpc-node',
+    'validator-ha-db',
+  ];
+  const pvcComponents = [
+    'p2p-bootstrap',
+    'prover-node',
+    'prover-broker',
+    'sequencer-node',
+    'rpc-node',
     'validator-ha-db',
   ];
-  const pvcComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'sequencer-node', 'rpc', 'validator-ha-db'];
   // StatefulSet components that need to be scaled down before PVC deletion
   // Note: validators use 'sequencer-node' as component label, not 'validator'
   const statefulSetComponents = [
@@ -383,7 +388,7 @@ export async function rollAztecPods(namespace: string, clearState: boolean = fal
     'prover-node',
     'prover-broker',
     'sequencer-node',
-    'rpc',
+    'rpc-node',
     'validator-ha-db',
   ];
 

package/src/spartan/utils/pod_logs.ts

@@ -0,0 +1,99 @@
+import type { Logger } from '@aztec/foundation/log';
+
+import { exec } from 'child_process';
+import { promisify } from 'util';
+
+import { getSequencers } from './nodes.js';
+
+const execAsync = promisify(exec);
+
+/** Parsed l2-block-built stats from a sequencer pod log line. */
+export type BlockBuiltLogEntry = {
+  blockNumber: number;
+  txCount: number;
+  duration: number;
+  publicProcessDuration: number;
+  manaPerSec: number;
+  privateLogCount: number;
+  publicLogCount: number;
+  contractClassLogCount: number;
+  contractClassLogSize: number;
+};
+
+const FIELDS: (keyof BlockBuiltLogEntry)[] = [
+  'blockNumber',
+  'txCount',
+  'duration',
+  'publicProcessDuration',
+  'manaPerSec',
+  'privateLogCount',
+  'publicLogCount',
+  'contractClassLogCount',
+  'contractClassLogSize',
+];
+
+/**
+ * Fetches l2-block-built log entries from sequencer pods for given block numbers.
+ * Queries all validator pods (only the proposer will have the log for a given block).
+ *
+ * @param namespace - Kubernetes namespace
+ * @param sinceTime - ISO 8601 timestamp to limit log search (e.g., from before block building was re-enabled)
+ * @param blockNumbers - Set of block numbers to filter for
+ * @param logger - Logger instance
+ * @returns Array of parsed BlockBuiltLogEntry, de-duplicated by blockNumber, sorted ascending
+ */
+export async function fetchBlockBuiltLogs(
+  namespace: string,
+  sinceTime: string,
+  blockNumbers: Set<number>,
+  logger: Logger,
+): Promise<BlockBuiltLogEntry[]> {
+  const pods = await getSequencers(namespace);
+  const entriesByBlock = new Map<number, BlockBuiltLogEntry>();
+
+  // Subtract 60s from sinceTime to account for clock skew between test runner and k8s pods.
+  // Block number filtering ensures we only match the right blocks, so extra lines are harmless.
+  const sinceDate = new Date(new Date(sinceTime).getTime() - 60_000);
+  const sinceFlag = sinceDate.toISOString();
+
+  for (const pod of pods) {
+    try {
+      const cmd = `kubectl logs ${pod} -n ${namespace} -c aztec --since-time=${sinceFlag}`;
+      logger.info(`Fetching logs: ${cmd}`);
+      const { stdout } = await execAsync(cmd, { maxBuffer: 10 * 1024 * 1024 });
+
+      const lines = stdout.split('\n');
+      const matchingLines = lines.filter(l => l.includes('l2-block-built'));
+      logger.info(`Pod ${pod}: ${lines.length} log lines, ${matchingLines.length} contain l2-block-built`);
+
+      for (const line of matchingLines) {
+        try {
+          const parsed = JSON.parse(line);
+          if (parsed.eventName !== 'l2-block-built' || !blockNumbers.has(parsed.blockNumber)) {
+            continue;
+          }
+          if (entriesByBlock.has(parsed.blockNumber)) {
+            continue;
+          }
+          const entry: BlockBuiltLogEntry = {} as BlockBuiltLogEntry;
+          for (const field of FIELDS) {
+            entry[field] = parsed[field] ?? 0;
+          }
+          entriesByBlock.set(entry.blockNumber, entry);
+          logger.verbose(`Parsed l2-block-built log for block ${entry.blockNumber}`, entry);
+        } catch {
+          // Not valid JSON, skip
+        }
+      }
+    } catch (err) {
+      logger.warn(`Failed to fetch logs from pod ${pod}: ${err}`);
+    }
+  }
+
+  if (entriesByBlock.size < blockNumbers.size) {
+    const missing = [...blockNumbers].filter(bn => !entriesByBlock.has(bn));
+    logger.warn(`Missing l2-block-built logs for block(s): ${missing.join(', ')}`);
+  }
+
+  return [...entriesByBlock.values()].sort((a, b) => a.blockNumber - b.blockNumber);
+}