@aztec/end-to-end 0.0.1-commit.3fd054f6 → 0.0.1-commit.431c48d

package/src/spartan/setup_test_wallets.ts

@@ -1,18 +1,23 @@
 import { generateSchnorrAccounts } from '@aztec/accounts/testing';
+import { NO_FROM } from '@aztec/aztec.js/account';
 import { AztecAddress } from '@aztec/aztec.js/addresses';
-import { NO_WAIT } from '@aztec/aztec.js/contracts';
+import { DefaultWaitOpts, NO_WAIT } from '@aztec/aztec.js/contracts';
 import { L1FeeJuicePortalManager } from '@aztec/aztec.js/ethereum';
 import { FeeJuicePaymentMethodWithClaim } from '@aztec/aztec.js/fee';
 import { type FeePaymentMethod, SponsoredFeePaymentMethod } from '@aztec/aztec.js/fee';
 import { Fr } from '@aztec/aztec.js/fields';
+import { waitForL1ToL2MessageReady } from '@aztec/aztec.js/messaging';
 import { type AztecNode, createAztecNodeClient, waitForTx } from '@aztec/aztec.js/node';
 import type { Wallet } from '@aztec/aztec.js/wallet';
 import { createEthereumChain } from '@aztec/ethereum/chain';
 import { createExtendedL1Client } from '@aztec/ethereum/client';
 import type { Logger } from '@aztec/foundation/log';
-import { retryUntil } from '@aztec/foundation/retry';
+import { makeBackoff, retry, retryUntil } from '@aztec/foundation/retry';
 import { TokenContract } from '@aztec/noir-contracts.js/Token';
+import { Gas } from '@aztec/stdlib/gas';
 import type { AztecNodeAdmin } from '@aztec/stdlib/interfaces/client';
+import { TxStatus } from '@aztec/stdlib/tx';
+import { getGasLimits } from '@aztec/wallet-sdk/base-wallet';
 import { registerInitialLocalNetworkAccountsInWallet } from '@aztec/wallets/testing';
 
 import { getACVMConfig } from '../fixtures/get_acvm_config.js';
@@ -51,6 +56,8 @@ export async function setupTestAccountsWithTokens(
 
   const aztecNode = createAztecNodeClient(nodeUrl);
   const wallet = await TestWallet.create(aztecNode);
+  // Remote JSON-RPC node: keep the 1s poll cadence rather than the in-process TestWallet fast default.
+  wallet.setDefaultWaitInterval(DefaultWaitOpts.interval);
 
   const [recipientAccount, ...accounts] = (await registerInitialLocalNetworkAccountsInWallet(wallet)).slice(
     0,
@@ -81,15 +88,17 @@ export async function deploySponsoredTestAccountsWithTokens(
   numberOfFundedWallets = 1,
 ): Promise<TestAccounts> {
   const [recipient, ...funded] = await generateSchnorrAccounts(numberOfFundedWallets + 1);
-  const recipientAccount = await wallet.createSchnorrAccount(recipient.secret, recipient.salt);
-  const fundedAccounts = await Promise.all(funded.map(a => wallet.createSchnorrAccount(a.secret, a.salt)));
+  const recipientAccount = await wallet.createSchnorrAccount(recipient.secret, recipient.salt, recipient.signingKey);
+  const fundedAccounts = await Promise.all(
+    funded.map(a => wallet.createSchnorrAccount(a.secret, a.salt, a.signingKey)),
+  );
 
   await registerSponsoredFPC(wallet);
 
   const paymentMethod = new SponsoredFeePaymentMethod(await getSponsoredFPCAddress());
   const recipientDeployMethod = await recipientAccount.getDeployMethod();
   await recipientDeployMethod.send({
-    from: AztecAddress.ZERO,
+    from: NO_FROM,
     fee: { paymentMethod },
     wait: { timeout: 2400 },
   });
@@ -97,7 +106,7 @@ export async function deploySponsoredTestAccountsWithTokens(
     fundedAccounts.map(async a => {
       const deployMethod = await a.getDeployMethod();
       await deployMethod.send({
-        from: AztecAddress.ZERO,
+        from: NO_FROM,
         fee: { paymentMethod },
         wait: { timeout: 2400 },
       }); // increase timeout on purpose in order to account for two empty epochs
@@ -137,38 +146,58 @@ async function deployAccountWithDiagnostics(
   estimateGas?: boolean,
 ): Promise<void> {
   const deployMethod = await account.getDeployMethod();
-  let txHash;
-  try {
-    let gasSettings;
-    if (estimateGas) {
-      const sim = await deployMethod.simulate({ from: AztecAddress.ZERO, fee: { paymentMethod } });
-      gasSettings = sim.estimatedGas;
-      logger.info(`${accountLabel} estimated gas: DA=${gasSettings.gasLimits.daGas} L2=${gasSettings.gasLimits.l2Gas}`);
-    }
-    const deployResult = await deployMethod.send({
-      from: AztecAddress.ZERO,
-      fee: { paymentMethod, gasSettings },
-      wait: NO_WAIT,
-    });
-    txHash = deployResult.txHash;
-    await waitForTx(aztecNode, txHash, { timeout: 2400 });
-    logger.info(`${accountLabel} deployed at ${account.address}`);
-  } catch (error) {
-    const blockNumber = await aztecNode.getBlockNumber();
-    let receipt;
-    try {
-      receipt = await aztecNode.getTxReceipt(txHash);
-    } catch {
-      receipt = 'unavailable';
-    }
-    logger.error(`${accountLabel} deployment failed`, {
-      txHash: txHash.toString(),
-      receipt: JSON.stringify(receipt),
-      currentBlockNumber: blockNumber,
-      error: String(error),
-    });
-    throw error;
+  let gasSettings: any;
+  if (estimateGas) {
+    const sim = await deployMethod.simulate({ from: NO_FROM, fee: { paymentMethod }, includeMetadata: true });
+    const { txsLimits } = await aztecNode.getNodeInfo();
+    gasSettings = getGasLimits(sim.gasUsed!, Gas.from(txsLimits.gas));
+    logger.info(`${accountLabel} estimated gas: DA=${gasSettings.gasLimits.daGas} L2=${gasSettings.gasLimits.l2Gas}`);
   }
+
+  // Track the tx hash across retries so we don't re-send when the previous tx is still pending.
+  let sentTxHash: { txHash: any } | undefined;
+
+  await retry(
+    async () => {
+      // Check if already deployed (handles case where previous attempt succeeded but waitForTx timed out)
+      const existing = await aztecNode.getContract(account.address);
+      if (existing) {
+        logger.info(`${accountLabel} already deployed at ${account.address}, skipping`);
+        return;
+      }
+
+      // If we already sent a tx, check if it was dropped before deciding to re-send.
+      if (sentTxHash) {
+        const prevReceipt = await aztecNode.getTxReceipt(sentTxHash.txHash);
+        if (prevReceipt.isDropped()) {
+          logger.info(`${accountLabel} previous tx ${sentTxHash.txHash} was dropped, re-sending`);
+          sentTxHash = undefined;
+        } else {
+          logger.info(`${accountLabel} previous tx ${sentTxHash.txHash} still pending, waiting again...`);
+        }
+      }
+
+      if (!sentTxHash) {
+        const deployResult = await deployMethod.send({
+          from: NO_FROM,
+          fee: { paymentMethod, gasSettings },
+          wait: NO_WAIT,
+        });
+        sentTxHash = { txHash: deployResult.txHash };
+        logger.info(`${accountLabel} tx sent`, { txHash: sentTxHash.txHash.toString() });
+      }
+
+      const receipt = await waitForTx(aztecNode, sentTxHash.txHash, { timeout: 600 });
+      if (receipt.isDropped()) {
+        sentTxHash = undefined;
+        throw new Error(`${accountLabel} tx was dropped, retrying...`);
+      }
+      logger.info(`${accountLabel} deployed at ${account.address}`);
+    },
+    `deploy ${accountLabel}`,
+    makeBackoff([1, 2, 4, 8, 16]),
+    logger,
+  );
 }
 
 async function deployAccountsInBatches(
@@ -208,8 +237,10 @@ export async function deploySponsoredTestAccounts(
   opts?: { estimateGas?: boolean },
 ): Promise<TestAccountsWithoutTokens> {
   const [recipient, ...funded] = await generateSchnorrAccounts(numberOfFundedWallets + 1);
-  const recipientAccount = await wallet.createSchnorrAccount(recipient.secret, recipient.salt);
-  const fundedAccounts = await Promise.all(funded.map(a => wallet.createSchnorrAccount(a.secret, a.salt)));
+  const recipientAccount = await wallet.createSchnorrAccount(recipient.secret, recipient.salt, recipient.signingKey);
+  const fundedAccounts = await Promise.all(
+    funded.map(a => wallet.createSchnorrAccount(a.secret, a.salt, a.signingKey)),
+  );
 
   await registerSponsoredFPC(wallet);
 
@@ -251,10 +282,14 @@ export async function deployTestAccountsWithTokens(
 ): Promise<TestAccounts> {
   const aztecNode = createAztecNodeClient(nodeUrl);
   const wallet = await TestWallet.create(aztecNode);
+  // Remote JSON-RPC node: keep the 1s poll cadence rather than the in-process TestWallet fast default.
+  wallet.setDefaultWaitInterval(DefaultWaitOpts.interval);
 
   const [recipient, ...funded] = await generateSchnorrAccounts(numberOfFundedWallets + 1);
-  const recipientAccount = await wallet.createSchnorrAccount(recipient.secret, recipient.salt);
-  const fundedAccounts = await Promise.all(funded.map(a => wallet.createSchnorrAccount(a.secret, a.salt)));
+  const recipientAccount = await wallet.createSchnorrAccount(recipient.secret, recipient.salt, recipient.signingKey);
+  const fundedAccounts = await Promise.all(
+    funded.map(a => wallet.createSchnorrAccount(a.secret, a.salt, a.signingKey)),
+  );
 
   const claims = await Promise.all(
     fundedAccounts.map(a => bridgeL1FeeJuice(l1RpcUrls, mnemonicOrPrivateKey, aztecNode, a.address, undefined, logger)),
@@ -269,7 +304,7 @@ export async function deployTestAccountsWithTokens(
     fundedAccounts.map(async (a, i) => {
       const paymentMethod = new FeeJuicePaymentMethodWithClaim(a.address, claims[i]);
       const deployMethod = await a.getDeployMethod();
-      await deployMethod.send({ from: AztecAddress.ZERO, fee: { paymentMethod } });
+      await deployMethod.send({ from: NO_FROM, fee: { paymentMethod } });
       logger.info(`Account deployed at ${a.address}`);
     }),
   );
@@ -312,9 +347,7 @@ async function bridgeL1FeeJuice(
   const portal = await L1FeeJuicePortalManager.new(aztecNode, l1Client, log);
   const claim = await portal.bridgeTokensPublic(recipient, amount, true /* mint */);
 
-  const isSynced = async () =>
-    (await aztecNode.getL1ToL2MessageCheckpoint(Fr.fromHexString(claim.messageHash))) !== undefined;
-  await retryUntil(isSynced, `message ${claim.messageHash} sync`, 24, 0.5);
+  await waitForL1ToL2MessageReady(aztecNode, Fr.fromHexString(claim.messageHash), { timeoutSeconds: 24 });
 
   log.info(`Created a claim for ${amount} L1 fee juice to ${recipient}.`, claim);
   return claim;
@@ -345,14 +378,18 @@ async function deployTokenAndMint(
   logger: Logger,
 ) {
   logger.verbose(`Deploying TokenContract...`);
-  const {
-    receipt: { contract: tokenContract },
-  } = await TokenContract.deploy(wallet, admin, TOKEN_NAME, TOKEN_SYMBOL, TOKEN_DECIMALS).send({
+  const { contract: tokenContract } = await TokenContract.deploy(
+    wallet,
+    admin,
+    TOKEN_NAME,
+    TOKEN_SYMBOL,
+    TOKEN_DECIMALS,
+  ).send({
     from: admin,
     fee: {
       paymentMethod,
     },
-    wait: { timeout: 600, returnReceipt: true },
+    wait: { timeout: 600 },
   });
 
   const tokenAddress = tokenContract.address;
@@ -403,7 +440,9 @@ export async function performTransfers({
 
     const provenTxs = await Promise.all(txs);
 
-    await Promise.all(provenTxs.map(t => t.send({ wait: { timeout: 600 } })));
+    // Wait only for the txs to be proposed, not checkpointed. This is enough to keep the chain
+    // loaded for the reorg scenario, and avoids each round blocking on the (slower) checkpoint lag.
+    await Promise.all(provenTxs.map(t => t.send({ wait: { timeout: 600, waitForStatus: TxStatus.PROPOSED } })));
 
     logger.info(`Completed round ${i + 1} / ${rounds}`);
   }
@@ -430,6 +469,8 @@ export async function createWalletAndAztecNodeClient(
     proverEnabled,
   };
   const wallet = await TestWallet.create(aztecNode, pxeConfig);
+  // Remote JSON-RPC node: keep the 1s poll cadence rather than the in-process TestWallet fast default.
+  wallet.setDefaultWaitInterval(DefaultWaitOpts.interval);
 
   return {
     wallet,

package/src/spartan/tx_metrics.ts

@@ -1,6 +1,6 @@
 import type { AztecNode } from '@aztec/aztec.js/node';
 import type { Logger } from '@aztec/foundation/log';
-import type { L2Block } from '@aztec/stdlib/block';
+import type { BlockResponse } from '@aztec/stdlib/interfaces/client';
 import type { TopicType } from '@aztec/stdlib/p2p';
 import { Tx, type TxReceipt } from '@aztec/stdlib/tx';
 
@@ -134,9 +134,12 @@ export class ProvingMetrics {
 
 export type TxInclusionData = {
   txHash: string;
-  sentAt: number;
-  minedAt: number;
-  attestedAt: number;
+  /** Wall-clock at client when the tx was submitted, in ms (Date.now()). */
+  sentAtMs: number;
+  /** Wall-clock at client when the block containing the tx first became visible, in ms (Date.now()). -1 if never observed. */
+  minedAtMs: number;
+  /** Reserved for future attestation-observed-at signal; -1 today. */
+  attestedAtMs: number;
   blocknumber: number;
   priorityFee: number;
   totalFee: number;
@@ -147,7 +150,7 @@ export type TxInclusionData = {
 export class TxInclusionMetrics {
   private data = new Map<string, TxInclusionData>();
   private groups = new Set<string>();
-  private blocks = new Map<number, Promise<L2Block | undefined>>();
+  private blocks = new Map<number, Promise<BlockResponse<{ includeTransactions: true }> | undefined>>();
 
   private p2pGossipLatencyByTopic: Partial<Record<TopicType, { p50: number; p95: number }>> = {};
 
@@ -158,6 +161,7 @@ export class TxInclusionMetrics {
   private mempoolMinedDelay:
     | { txP50: number; txP95: number; attestationP50: number; attestationP95: number }
     | undefined;
+  private inclusionOutcome: { mined: number; failed: number } | undefined;
 
   constructor(
     private aztecNode: AztecNode,
@@ -174,9 +178,9 @@ export class TxInclusionMetrics {
 
     this.data.set(txHash, {
       txHash,
-      sentAt: Math.trunc(Date.now() / 1000),
-      minedAt: -1,
-      attestedAt: -1,
+      sentAtMs: Date.now(),
+      minedAtMs: -1,
+      attestedAtMs: -1,
       blocknumber: -1,
       priorityFee: Number(priorityFees.feePerDaGas + priorityFees.feePerL2Gas),
       totalFee: -1,
@@ -186,6 +190,28 @@ export class TxInclusionMetrics {
     this.groups.add(group);
   }
 
+  /**
+   * Stamp mined-at metadata for any tracked tx contained in this block, using
+   * `observedAtMs` (caller-supplied wall-clock at the moment they first saw the
+   * block). Idempotent: existing minedAtMs is preserved so the first observer
+   * wins (typically the block-watcher; recordMinedTx is a fallback).
+   */
+  observeBlockForMinedTxs(
+    blockNumber: number,
+    txHashes: ReadonlyArray<{ toString(): string }>,
+    observedAtMs: number,
+  ): void {
+    txHashes.forEach((txHash, position) => {
+      const data = this.data.get(txHash.toString());
+      if (!data || data.minedAtMs !== -1) {
+        return;
+      }
+      data.blocknumber = blockNumber;
+      data.minedAtMs = observedAtMs;
+      data.positionInBlock = position;
+    });
+  }
+
   async recordMinedTx(txReceipt: TxReceipt): Promise<void> {
     const { txHash, blockNumber } = txReceipt;
     if (!txReceipt.isMined() || !txReceipt.hasExecutionSucceeded() || !blockNumber) {
@@ -197,26 +223,53 @@ export class TxInclusionMetrics {
       return;
     }
 
-    if (!this.blocks.has(blockNumber)) {
-      this.blocks.set(blockNumber, this.aztecNode.getBlock(blockNumber));
-    }
-
-    const block = await this.blocks.get(blockNumber)!;
-    if (!block) {
-      this.logger?.warn('Failed to load block for mined tx receipt', { txHash: txHash.toString(), blockNumber });
-      return;
-    }
     const data = this.data.get(txHash.toString());
     if (!data) {
       const message = `Missing sent tx record for mined tx ${txHash.toString()}`;
       this.logger?.warn(message, { txHash: txHash.toString(), blockNumber });
       throw new Error(message);
     }
-    data.blocknumber = blockNumber;
-    data.minedAt = Number(block.header.globalVariables.timestamp);
-    data.attestedAt = -1;
     data.totalFee = Number(txReceipt.transactionFee ?? 0n);
-    data.positionInBlock = block.body.txEffects.findIndex(txEffect => txEffect.txHash.equals(txHash));
+
+    // Fallback path for txs the block-watcher missed (e.g. observed only after
+    // the watcher stopped). Stamp with the block's L2 slot timestamp; this is
+    // earlier than the true client-observed time by attestation+propagation
+    // lag, but it's the only deterministic timestamp available post-hoc.
+    if (data.minedAtMs === -1) {
+      if (!this.blocks.has(blockNumber)) {
+        this.blocks.set(blockNumber, this.aztecNode.getBlock(blockNumber, { includeTransactions: true }));
+      }
+      const block = await this.blocks.get(blockNumber)!;
+      if (!block) {
+        this.logger?.warn('Failed to load block for mined tx receipt', { txHash: txHash.toString(), blockNumber });
+        return;
+      }
+      data.blocknumber = blockNumber;
+      data.minedAtMs = Number(block.header.globalVariables.timestamp) * 1000;
+      data.positionInBlock = block.body.txEffects.findIndex(txEffect => txEffect.txHash.equals(txHash));
+    }
+  }
+
+  /**
+   * Whether this tx was ever observed in a block (by the block-watcher or a mined receipt).
+   * Idempotent first-sighting semantics: a later reorg / pool eviction never clears it, so callers
+   * can treat "ever mined" as included regardless of what happens to the tx afterwards.
+   */
+  public wasMined(txHash: string): boolean {
+    const d = this.data.get(txHash);
+    return !!d && d.minedAtMs !== -1;
+  }
+
+  /** Per-tx inclusion records for a group. Used to serialise out for downstream tooling. */
+  getInclusionRecords(group?: string): TxInclusionData[] {
+    const out: TxInclusionData[] = [];
+    for (const tx of this.data.values()) {
+      if (group !== undefined && tx.group !== group) {
+        continue;
+      }
+      out.push({ ...tx });
+    }
+    return out;
   }
 
   public inclusionTimeInSeconds(group: string): {
@@ -229,12 +282,28 @@ export class TxInclusionMetrics {
     p99: number;
   } {
     const histogram = createHistogram({});
+    let nonPositive = 0;
     for (const tx of this.data.values()) {
-      if (!tx.blocknumber || tx.group !== group || tx.minedAt === -1) {
+      if (!tx.blocknumber || tx.group !== group || tx.minedAtMs === -1) {
         continue;
       }
 
-      histogram.record(tx.minedAt - tx.sentAt);
+      // Both timestamps are client wall-clock (ms). A negative delta should be
+      // impossible since the watcher stamps minedAtMs strictly after sentAtMs,
+      // but the fallback path (recordMinedTx via L2 slot timestamp) can stamp
+      // earlier than sentAtMs. perf_hooks.createHistogram rejects <=0; skip
+      // those instead of crashing.
+      const deltaMs = tx.minedAtMs - tx.sentAtMs;
+      if (deltaMs <= 0) {
+        nonPositive++;
+        continue;
+      }
+      // Histogram is recorded in seconds (rounded) to match the existing
+      // toGithubActionBenchmarkJSON output unit; per-tx records carry the raw ms.
+      histogram.record(Math.max(1, Math.round(deltaMs / 1000)));
+    }
+    if (nonPositive > 0) {
+      this.logger?.debug(`Dropped ${nonPositive} tx inclusion samples with non-positive delta`, { group });
     }
 
     if (histogram.count === 0) {
@@ -284,6 +353,11 @@ export class TxInclusionMetrics {
     this.mempoolMinedDelay = { txP50, txP95, attestationP50, attestationP95 };
   }
 
+  /** Mined vs failed counts for the high-value lane — recorded instead of asserting strict 1:1 inclusion. */
+  public recordInclusionOutcome(mined: number, failed: number): void {
+    this.inclusionOutcome = { mined, failed };
+  }
+
   toGithubActionBenchmarkJSON(): Array<{ name: string; unit: string; value: number; range?: number; extra?: string }> {
     const data: Array<{ name: string; unit: string; value: number; range?: number; extra?: string }> = [];
     for (const group of this.groups) {
@@ -365,6 +439,16 @@ export class TxInclusionMetrics {
       );
     }
 
+    if (this.inclusionOutcome) {
+      const { mined, failed } = this.inclusionOutcome;
+      const total = mined + failed;
+      data.push(
+        { name: 'inclusion/mined_count', unit: 'count', value: mined },
+        { name: 'inclusion/failed_count', unit: 'count', value: failed },
+        { name: 'inclusion/success_ratio', unit: 'ratio', value: total > 0 ? mined / total : 0 },
+      );
+    }
+
     const scenario = process.env.BENCH_SCENARIO?.trim();
     if (!scenario) {
       return data;

package/src/spartan/utils/gossip_metrics.ts

@@ -0,0 +1,108 @@
+import type { Logger } from '@aztec/foundation/log';
+
+import type { PrometheusClient } from '../../quality_of_service/prometheus_client.js';
+
+/** Runs a PromQL query and returns each series as a label-record plus value. Returns [] on error. */
+async function queryVector(
+  prometheus: PrometheusClient,
+  query: string,
+  logger: Logger,
+): Promise<{ labels: Record<string, string>; value: number }[]> {
+  try {
+    const resp = await prometheus.queryRaw(query);
+    if (resp.status !== 'success' || resp.data.resultType !== 'vector') {
+      logger.warn(`Unexpected Prometheus response for query`, { query, resp });
+      return [];
+    }
+    return resp.data.result.map(({ metric, value }) => ({
+      labels: (metric ?? {}) as Record<string, string>,
+      value: parseFloat(value[1]),
+    }));
+  } catch (err) {
+    logger.warn(`Failed to run Prometheus query: ${err}`, { query });
+    return [];
+  }
+}
+
+/** Formats a grouped vector result as a { labelValue: value } record for structured logging. */
+function toRecord(series: { labels: Record<string, string>; value: number }[], label: string): Record<string, number> {
+  const out: Record<string, number> = {};
+  for (const { labels, value } of series) {
+    out[labels[label] ?? 'unknown'] = Math.round(value * 100) / 100;
+  }
+  return out;
+}
+
+/**
+ * Scrapes and logs gossip tx validation timing breakdowns from Prometheus: per-stage validation
+ * durations, tx pool serial queue wait/execution times, and chonk (IVC) proof verifier timings.
+ * Used by the spartan TPS benchmarks to attribute slow gossip validations to a specific stage.
+ */
+export async function logGossipTxValidationMetrics(
+  prometheus: PrometheusClient,
+  namespace: string,
+  windowSeconds: number,
+  logger: Logger,
+): Promise<void> {
+  const ns = `k8s_namespace_name="${namespace}"`;
+  const window = `[${Math.max(60, Math.ceil(windowSeconds))}s]`;
+
+  const stageQuantile = (perc: string) =>
+    `histogram_quantile(${perc}, sum(rate(aztec_p2p_gossip_tx_validation_stage_duration_milliseconds_bucket{${ns}}${window})) by (le, aztec_p2p_tx_validation_stage))`;
+  const stageAvg = () =>
+    `sum(rate(aztec_p2p_gossip_tx_validation_stage_duration_milliseconds_sum{${ns}}${window})) by (aztec_p2p_tx_validation_stage) / ` +
+    `sum(rate(aztec_p2p_gossip_tx_validation_stage_duration_milliseconds_count{${ns}}${window})) by (aztec_p2p_tx_validation_stage)`;
+  const validationQuantile = (perc: string) =>
+    `histogram_quantile(${perc}, sum(rate(aztec_p2p_gossip_message_validation_duration_milliseconds_bucket{${ns}}${window})) by (le, aztec_gossip_topic_name))`;
+  const queueQuantile = (metric: string, perc: string) =>
+    `topk(10, histogram_quantile(${perc}, sum(rate(${metric}{${ns}}${window})) by (le, aztec_mempool_operation)))`;
+  const ivcQuantile = (metric: string, perc: string) =>
+    `histogram_quantile(${perc}, sum(rate(${metric}{${ns}}${window})) by (le))`;
+
+  const stageLabel = 'aztec_p2p_tx_validation_stage';
+  const topicLabel = 'aztec_gossip_topic_name';
+  const operationLabel = 'aztec_mempool_operation';
+
+  const [stageP50, stageP95, stageAvgs, validationP95, slowCount] = await Promise.all([
+    queryVector(prometheus, stageQuantile('0.50'), logger),
+    queryVector(prometheus, stageQuantile('0.95'), logger),
+    queryVector(prometheus, stageAvg(), logger),
+    queryVector(prometheus, validationQuantile('0.95'), logger),
+    queryVector(prometheus, `sum(aztec_p2p_gossip_slow_validation_count{${ns}}) by (${topicLabel})`, logger),
+  ]);
+
+  logger.info('Gossip tx validation stage timings (ms)', {
+    stageP50: toRecord(stageP50, stageLabel),
+    stageP95: toRecord(stageP95, stageLabel),
+    stageAvg: toRecord(stageAvgs, stageLabel),
+    validationP95ByTopic: toRecord(validationP95, topicLabel),
+    slowValidationCountByTopic: toRecord(slowCount, topicLabel),
+  });
+
+  const [queueWaitP95, queueExecutionP95, queueLengthMax] = await Promise.all([
+    queryVector(prometheus, queueQuantile('aztec_mempool_tx_pool_v2_queue_wait_milliseconds_bucket', '0.95'), logger),
+    queryVector(
+      prometheus,
+      queueQuantile('aztec_mempool_tx_pool_v2_queue_execution_milliseconds_bucket', '0.95'),
+      logger,
+    ),
+    queryVector(prometheus, `max(max_over_time(aztec_mempool_tx_pool_v2_queue_length{${ns}}${window}))`, logger),
+  ]);
+
+  logger.info('Tx pool serial queue stats (ms)', {
+    queueWaitP95: toRecord(queueWaitP95, operationLabel),
+    queueExecutionP95: toRecord(queueExecutionP95, operationLabel),
+    queueLengthMax: queueLengthMax[0]?.value,
+  });
+
+  const [ivcVerifyP95, ivcTotalP95] = await Promise.all([
+    queryVector(prometheus, ivcQuantile('aztec_ivc_verifier_time_milliseconds_bucket', '0.95'), logger),
+    queryVector(prometheus, ivcQuantile('aztec_ivc_verifier_total_time_milliseconds_bucket', '0.95'), logger),
+  ]);
+
+  // A large gap between total (queue + verify) and verify indicates a pile-up in the verifier queue.
+  logger.info('Chonk (IVC) proof verifier timings (ms)', {
+    verifyP95: ivcVerifyP95[0]?.value,
+    totalP95: ivcTotalP95[0]?.value,
+  });
+}

package/src/spartan/utils/index.ts

@@ -69,3 +69,6 @@ export { ChainHealth, type ChainHealthSnapshot } from './health.js';
 
 // Pod log extraction
 export { type BlockBuiltLogEntry, fetchBlockBuiltLogs } from './pod_logs.js';
+
+// Gossip validation metrics scraping
+export { logGossipTxValidationMetrics } from './gossip_metrics.js';

package/src/spartan/utils/nodes.ts

@@ -71,7 +71,7 @@ export async function waitForProvenToAdvance(
   // Get current proven block number
   let initialProvenBlock: number;
   try {
-    const tips = await node.getL2Tips();
+    const tips = await node.getChainTips();
     initialProvenBlock = Number(tips.proven.block.number);
     log.info(`Current proven block: ${initialProvenBlock}. Waiting for it to increase...`);
   } catch (err) {
@@ -82,7 +82,7 @@ export async function waitForProvenToAdvance(
   await retryUntil(
     async () => {
       try {
-        const tips = await node.getL2Tips();
+        const tips = await node.getChainTips();
         const currentProvenBlock = Number(tips.proven.block.number);
         const proposedBlock = Number(tips.proposed.number);