@aztec/end-to-end 0.0.1-commit.7b97ef96e → 0.0.1-commit.7cbc774

package/src/spartan/tx_metrics.ts

@@ -1,6 +1,6 @@
 import type { AztecNode } from '@aztec/aztec.js/node';
 import type { Logger } from '@aztec/foundation/log';
-import type { L2Block } from '@aztec/stdlib/block';
+import type { BlockResponse } from '@aztec/stdlib/interfaces/client';
 import type { TopicType } from '@aztec/stdlib/p2p';
 import { Tx, type TxReceipt } from '@aztec/stdlib/tx';
 
@@ -134,9 +134,12 @@ export class ProvingMetrics {
 
 export type TxInclusionData = {
   txHash: string;
-  sentAt: number;
-  minedAt: number;
-  attestedAt: number;
+  /** Wall-clock at client when the tx was submitted, in ms (Date.now()). */
+  sentAtMs: number;
+  /** Wall-clock at client when the block containing the tx first became visible, in ms (Date.now()). -1 if never observed. */
+  minedAtMs: number;
+  /** Reserved for future attestation-observed-at signal; -1 today. */
+  attestedAtMs: number;
   blocknumber: number;
   priorityFee: number;
   totalFee: number;
@@ -147,7 +150,7 @@ export type TxInclusionData = {
 export class TxInclusionMetrics {
   private data = new Map<string, TxInclusionData>();
   private groups = new Set<string>();
-  private blocks = new Map<number, Promise<L2Block | undefined>>();
+  private blocks = new Map<number, Promise<BlockResponse<{ includeTransactions: true }> | undefined>>();
 
   private p2pGossipLatencyByTopic: Partial<Record<TopicType, { p50: number; p95: number }>> = {};
 
@@ -174,9 +177,9 @@ export class TxInclusionMetrics {
 
     this.data.set(txHash, {
       txHash,
-      sentAt: Math.trunc(Date.now() / 1000),
-      minedAt: -1,
-      attestedAt: -1,
+      sentAtMs: Date.now(),
+      minedAtMs: -1,
+      attestedAtMs: -1,
       blocknumber: -1,
       priorityFee: Number(priorityFees.feePerDaGas + priorityFees.feePerL2Gas),
       totalFee: -1,
@@ -186,6 +189,28 @@ export class TxInclusionMetrics {
     this.groups.add(group);
   }
 
+  /**
+   * Stamp mined-at metadata for any tracked tx contained in this block, using
+   * `observedAtMs` (caller-supplied wall-clock at the moment they first saw the
+   * block). Idempotent: existing minedAtMs is preserved so the first observer
+   * wins (typically the block-watcher; recordMinedTx is a fallback).
+   */
+  observeBlockForMinedTxs(
+    blockNumber: number,
+    txHashes: ReadonlyArray<{ toString(): string }>,
+    observedAtMs: number,
+  ): void {
+    txHashes.forEach((txHash, position) => {
+      const data = this.data.get(txHash.toString());
+      if (!data || data.minedAtMs !== -1) {
+        return;
+      }
+      data.blocknumber = blockNumber;
+      data.minedAtMs = observedAtMs;
+      data.positionInBlock = position;
+    });
+  }
+
   async recordMinedTx(txReceipt: TxReceipt): Promise<void> {
     const { txHash, blockNumber } = txReceipt;
     if (!txReceipt.isMined() || !txReceipt.hasExecutionSucceeded() || !blockNumber) {
@@ -197,26 +222,43 @@ export class TxInclusionMetrics {
       return;
     }
 
-    if (!this.blocks.has(blockNumber)) {
-      this.blocks.set(blockNumber, this.aztecNode.getBlock(blockNumber));
-    }
-
-    const block = await this.blocks.get(blockNumber)!;
-    if (!block) {
-      this.logger?.warn('Failed to load block for mined tx receipt', { txHash: txHash.toString(), blockNumber });
-      return;
-    }
     const data = this.data.get(txHash.toString());
     if (!data) {
       const message = `Missing sent tx record for mined tx ${txHash.toString()}`;
       this.logger?.warn(message, { txHash: txHash.toString(), blockNumber });
       throw new Error(message);
     }
-    data.blocknumber = blockNumber;
-    data.minedAt = Number(block.header.globalVariables.timestamp);
-    data.attestedAt = -1;
     data.totalFee = Number(txReceipt.transactionFee ?? 0n);
-    data.positionInBlock = block.body.txEffects.findIndex(txEffect => txEffect.txHash.equals(txHash));
+
+    // Fallback path for txs the block-watcher missed (e.g. observed only after
+    // the watcher stopped). Stamp with the block's L2 slot timestamp; this is
+    // earlier than the true client-observed time by attestation+propagation
+    // lag, but it's the only deterministic timestamp available post-hoc.
+    if (data.minedAtMs === -1) {
+      if (!this.blocks.has(blockNumber)) {
+        this.blocks.set(blockNumber, this.aztecNode.getBlock(blockNumber, { includeTransactions: true }));
+      }
+      const block = await this.blocks.get(blockNumber)!;
+      if (!block) {
+        this.logger?.warn('Failed to load block for mined tx receipt', { txHash: txHash.toString(), blockNumber });
+        return;
+      }
+      data.blocknumber = blockNumber;
+      data.minedAtMs = Number(block.header.globalVariables.timestamp) * 1000;
+      data.positionInBlock = block.body.txEffects.findIndex(txEffect => txEffect.txHash.equals(txHash));
+    }
+  }
+
+  /** Per-tx inclusion records for a group. Used to serialise out for downstream tooling. */
+  getInclusionRecords(group?: string): TxInclusionData[] {
+    const out: TxInclusionData[] = [];
+    for (const tx of this.data.values()) {
+      if (group !== undefined && tx.group !== group) {
+        continue;
+      }
+      out.push({ ...tx });
+    }
+    return out;
   }
 
   public inclusionTimeInSeconds(group: string): {
@@ -229,12 +271,28 @@ export class TxInclusionMetrics {
     p99: number;
   } {
     const histogram = createHistogram({});
+    let nonPositive = 0;
     for (const tx of this.data.values()) {
-      if (!tx.blocknumber || tx.group !== group || tx.minedAt === -1) {
+      if (!tx.blocknumber || tx.group !== group || tx.minedAtMs === -1) {
         continue;
       }
 
-      histogram.record(tx.minedAt - tx.sentAt);
+      // Both timestamps are client wall-clock (ms). A negative delta should be
+      // impossible since the watcher stamps minedAtMs strictly after sentAtMs,
+      // but the fallback path (recordMinedTx via L2 slot timestamp) can stamp
+      // earlier than sentAtMs. perf_hooks.createHistogram rejects <=0; skip
+      // those instead of crashing.
+      const deltaMs = tx.minedAtMs - tx.sentAtMs;
+      if (deltaMs <= 0) {
+        nonPositive++;
+        continue;
+      }
+      // Histogram is recorded in seconds (rounded) to match the existing
+      // toGithubActionBenchmarkJSON output unit; per-tx records carry the raw ms.
+      histogram.record(Math.max(1, Math.round(deltaMs / 1000)));
+    }
+    if (nonPositive > 0) {
+      this.logger?.debug(`Dropped ${nonPositive} tx inclusion samples with non-positive delta`, { group });
     }
 
     if (histogram.count === 0) {
@@ -296,7 +354,7 @@ export class TxInclusionMetrics {
           value: stats.mean,
         },
         {
-          name: `${group}/median_inclusion`,
+          name: `${group}/p50_inclusion`,
           unit: 's',
           value: stats.median,
         },

package/src/spartan/utils/config.ts

@@ -8,6 +8,7 @@ const logger = createLogger('e2e:k8s-utils');
 const testConfigSchema = z.object({
   NAMESPACE: z.string().default('scenario'),
   REAL_VERIFIER: schemas.Boolean.optional().default(true),
+  DEBUG_FORCE_TX_PROOF_VERIFICATION: schemas.Boolean.optional().default(true),
   CREATE_ETH_DEVNET: schemas.Boolean.optional().default(false),
   L1_RPC_URLS_JSON: z.string().optional(),
   L1_ACCOUNT_MNEMONIC: z.string().optional(),
@@ -16,6 +17,7 @@ const testConfigSchema = z.object({
   AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5),
   AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET: z.coerce.number().optional().default(2),
   FUNDING_PRIVATE_KEY: z.string().optional(),
+  AZTEC_ADMIN_API_KEY: z.string().optional(),
 });
 
 export type TestConfig = z.infer<typeof testConfigSchema>;

package/src/spartan/utils/index.ts

@@ -41,6 +41,9 @@ export {
   applyNetworkShaping,
 } from './chaos.js';
 
+// Helm
+export { hasDeployedHelmRelease } from './helm.js';
+
 // Bot management
 export { restartBot, installTransferBot, uninstallTransferBot } from './bot.js';
 
@@ -63,3 +66,6 @@ export { getPublicViemClient, getL1DeploymentAddresses, getNodeClient } from './
 
 // Health checks
 export { ChainHealth, type ChainHealthSnapshot } from './health.js';
+
+// Pod log extraction
+export { type BlockBuiltLogEntry, fetchBlockBuiltLogs } from './pod_logs.js';

package/src/spartan/utils/nodes.ts

@@ -71,7 +71,7 @@ export async function waitForProvenToAdvance(
   // Get current proven block number
   let initialProvenBlock: number;
   try {
-    const tips = await node.getL2Tips();
+    const tips = await node.getChainTips();
     initialProvenBlock = Number(tips.proven.block.number);
     log.info(`Current proven block: ${initialProvenBlock}. Waiting for it to increase...`);
   } catch (err) {
@@ -82,7 +82,7 @@ export async function waitForProvenToAdvance(
   await retryUntil(
     async () => {
       try {
-        const tips = await node.getL2Tips();
+        const tips = await node.getChainTips();
         const currentProvenBlock = Number(tips.proven.block.number);
         const proposedBlock = Number(tips.proposed.number);
 
@@ -173,7 +173,7 @@ export async function withSequencersAdmin<T>(env: TestConfig, fn: (node: AztecNo
           if (statusRes.status !== 200) {
             throw new Error(`Admin endpoint returned status ${statusRes.status}`);
           }
-          const client = createAztecNodeAdminClient(url);
+          const client = createAztecNodeAdminClient(url, {}, undefined, env.AZTEC_ADMIN_API_KEY);
           return { result: await fn(client), process };
         } catch (err) {
           // Kill the port-forward before retrying
@@ -255,21 +255,18 @@ export async function initHADb(namespace: string): Promise<void> {
 }
 
 /**
- * Enables or disables probabilistic transaction dropping on validators and waits for rollout.
- * Wired to env vars P2P_DROP_TX and P2P_DROP_TX_CHANCE via Helm values.
+ * Sets probabilistic transaction dropping on validators and waits for rollout.
+ * Use probability=0 to disable. Wired to env var P2P_DROP_TX_CHANCE via Helm values.
  */
 export async function setValidatorTxDrop({
   namespace,
-  enabled,
   probability,
   logger: log,
 }: {
   namespace: string;
-  enabled: boolean;
   probability: number;
   logger: Logger;
 }) {
-  const drop = enabled ? 'true' : 'false';
   const prob = String(probability);
 
   const selectors = ['app.kubernetes.io/name=validator', 'app.kubernetes.io/component=validator', 'app=validator'];
@@ -284,7 +281,7 @@ export async function setValidatorTxDrop({
       if (names.length === 0) {
         continue;
       }
-      const cmd = `kubectl set env statefulset -l ${selector} -n ${namespace} P2P_DROP_TX=${drop} P2P_DROP_TX_CHANCE=${prob}`;
+      const cmd = `kubectl set env statefulset -l ${selector} -n ${namespace} P2P_DROP_TX_CHANCE=${prob}`;
       log.info(`command: ${cmd}`);
       await execAsync(cmd);
       updated = true;
@@ -366,16 +363,24 @@ export async function enableValidatorDynamicBootNode(
  */
 export async function rollAztecPods(namespace: string, clearState: boolean = false) {
   // Pod components use 'validator', but StatefulSets and PVCs use 'sequencer-node' for validators
+  // RPC nodes have nodeType='rpc-node' in Helm values, so their component label is 'rpc-node' (not 'rpc')
   const podComponents = [
     'p2p-bootstrap',
     'prover-node',
     'prover-broker',
     'prover-agent',
     'sequencer-node',
-    'rpc',
+    'rpc-node',
+    'validator-ha-db',
+  ];
+  const pvcComponents = [
+    'p2p-bootstrap',
+    'prover-node',
+    'prover-broker',
+    'sequencer-node',
+    'rpc-node',
     'validator-ha-db',
   ];
-  const pvcComponents = ['p2p-bootstrap', 'prover-node', 'prover-broker', 'sequencer-node', 'rpc', 'validator-ha-db'];
   // StatefulSet components that need to be scaled down before PVC deletion
   // Note: validators use 'sequencer-node' as component label, not 'validator'
   const statefulSetComponents = [
@@ -383,7 +388,7 @@ export async function rollAztecPods(namespace: string, clearState: boolean = fal
     'prover-node',
     'prover-broker',
     'sequencer-node',
-    'rpc',
+    'rpc-node',
     'validator-ha-db',
   ];
 

package/src/spartan/utils/pod_logs.ts

@@ -0,0 +1,99 @@
+import type { Logger } from '@aztec/foundation/log';
+
+import { exec } from 'child_process';
+import { promisify } from 'util';
+
+import { getSequencers } from './nodes.js';
+
+const execAsync = promisify(exec);
+
+/** Parsed l2-block-built stats from a sequencer pod log line. */
+export type BlockBuiltLogEntry = {
+  blockNumber: number;
+  txCount: number;
+  duration: number;
+  publicProcessDuration: number;
+  manaPerSec: number;
+  privateLogCount: number;
+  publicLogCount: number;
+  contractClassLogCount: number;
+  contractClassLogSize: number;
+};
+
+const FIELDS: (keyof BlockBuiltLogEntry)[] = [
+  'blockNumber',
+  'txCount',
+  'duration',
+  'publicProcessDuration',
+  'manaPerSec',
+  'privateLogCount',
+  'publicLogCount',
+  'contractClassLogCount',
+  'contractClassLogSize',
+];
+
+/**
+ * Fetches l2-block-built log entries from sequencer pods for given block numbers.
+ * Queries all validator pods (only the proposer will have the log for a given block).
+ *
+ * @param namespace - Kubernetes namespace
+ * @param sinceTime - ISO 8601 timestamp to limit log search (e.g., from before block building was re-enabled)
+ * @param blockNumbers - Set of block numbers to filter for
+ * @param logger - Logger instance
+ * @returns Array of parsed BlockBuiltLogEntry, de-duplicated by blockNumber, sorted ascending
+ */
+export async function fetchBlockBuiltLogs(
+  namespace: string,
+  sinceTime: string,
+  blockNumbers: Set<number>,
+  logger: Logger,
+): Promise<BlockBuiltLogEntry[]> {
+  const pods = await getSequencers(namespace);
+  const entriesByBlock = new Map<number, BlockBuiltLogEntry>();
+
+  // Subtract 60s from sinceTime to account for clock skew between test runner and k8s pods.
+  // Block number filtering ensures we only match the right blocks, so extra lines are harmless.
+  const sinceDate = new Date(new Date(sinceTime).getTime() - 60_000);
+  const sinceFlag = sinceDate.toISOString();
+
+  for (const pod of pods) {
+    try {
+      const cmd = `kubectl logs ${pod} -n ${namespace} -c aztec --since-time=${sinceFlag}`;
+      logger.info(`Fetching logs: ${cmd}`);
+      const { stdout } = await execAsync(cmd, { maxBuffer: 10 * 1024 * 1024 });
+
+      const lines = stdout.split('\n');
+      const matchingLines = lines.filter(l => l.includes('l2-block-built'));
+      logger.info(`Pod ${pod}: ${lines.length} log lines, ${matchingLines.length} contain l2-block-built`);
+
+      for (const line of matchingLines) {
+        try {
+          const parsed = JSON.parse(line);
+          if (parsed.eventName !== 'l2-block-built' || !blockNumbers.has(parsed.blockNumber)) {
+            continue;
+          }
+          if (entriesByBlock.has(parsed.blockNumber)) {
+            continue;
+          }
+          const entry: BlockBuiltLogEntry = {} as BlockBuiltLogEntry;
+          for (const field of FIELDS) {
+            entry[field] = parsed[field] ?? 0;
+          }
+          entriesByBlock.set(entry.blockNumber, entry);
+          logger.verbose(`Parsed l2-block-built log for block ${entry.blockNumber}`, entry);
+        } catch {
+          // Not valid JSON, skip
+        }
+      }
+    } catch (err) {
+      logger.warn(`Failed to fetch logs from pod ${pod}: ${err}`);
+    }
+  }
+
+  if (entriesByBlock.size < blockNumbers.size) {
+    const missing = [...blockNumbers].filter(bn => !entriesByBlock.has(bn));
+    logger.warn(`Missing l2-block-built logs for block(s): ${missing.join(', ')}`);
+  }
+
+  return [...entriesByBlock.values()].sort((a, b) => a.blockNumber - b.blockNumber);
+}