@aztec/end-to-end 0.0.1-commit.d431d1c → 0.0.1-commit.d58ff9d0

package/src/spartan/tx_metrics.ts

@@ -1,15 +1,145 @@
 import type { AztecNode } from '@aztec/aztec.js/node';
-import type { L2BlockNew } from '@aztec/stdlib/block';
+import type { Logger } from '@aztec/foundation/log';
+import type { BlockResponse } from '@aztec/stdlib/interfaces/client';
 import type { TopicType } from '@aztec/stdlib/p2p';
-import { Tx, type TxReceipt, TxStatus } from '@aztec/stdlib/tx';
+import { Tx, type TxReceipt } from '@aztec/stdlib/tx';
 
 import { createHistogram } from 'perf_hooks';
 
+/** Metrics class for proving-related benchmarks. */
+export class ProvingMetrics {
+  private successfulTxs: number | undefined;
+  private proofDuration: number | undefined;
+  private activeAgents: number | undefined;
+  private avgQueueTime: number | undefined;
+  private jobRetries: number | undefined;
+  private jobDuration: number | undefined;
+  private timedOutJobs: number | undefined;
+  private resolvedJobs: number | undefined;
+  private rejectedJobs: number | undefined;
+  private epochProvingDuration: number | undefined;
+  private provenTransactions: number | undefined;
+  private provenBlocks: number | undefined;
+
+  constructor(private prefix: string) {}
+
+  recordSuccessfulTxs(count: number): void {
+    this.successfulTxs = count;
+  }
+
+  recordProofDuration(seconds: number): void {
+    this.proofDuration = seconds;
+  }
+
+  recordActiveAgents(count: number): void {
+    this.activeAgents = count;
+  }
+
+  recordAvgQueueTime(ms: number): void {
+    this.avgQueueTime = ms;
+  }
+
+  recordJobRetries(count: number): void {
+    this.jobRetries = count;
+  }
+
+  recordJobDuration(ms: number): void {
+    this.jobDuration = ms;
+  }
+
+  recordTimedOutJobs(count: number): void {
+    this.timedOutJobs = count;
+  }
+
+  recordResolvedJobs(count: number): void {
+    this.resolvedJobs = count;
+  }
+
+  recordRejectedJobs(count: number): void {
+    this.rejectedJobs = count;
+  }
+
+  recordEpochProvingDuration(seconds: number): void {
+    this.epochProvingDuration = seconds;
+  }
+
+  recordProvenTransactions(count: number): void {
+    this.provenTransactions = count;
+  }
+
+  recordProvenBlocks(count: number): void {
+    this.provenBlocks = count;
+  }
+
+  toGithubActionBenchmarkJSON(): Array<{ name: string; unit: string; value: number }> {
+    const data: Array<{ name: string; unit: string; value: number }> = [];
+
+    if (this.successfulTxs !== undefined) {
+      data.push({ name: `${this.prefix}/successful_txs`, unit: 'count', value: this.successfulTxs });
+    }
+
+    if (this.proofDuration !== undefined) {
+      data.push({ name: `${this.prefix}/proof_duration`, unit: 's', value: this.proofDuration });
+    }
+
+    if (this.activeAgents !== undefined) {
+      data.push({ name: `${this.prefix}/active_agents`, unit: 'count', value: this.activeAgents });
+    }
+
+    if (this.avgQueueTime !== undefined) {
+      data.push({ name: `${this.prefix}/avg_queue_time`, unit: 'ms', value: this.avgQueueTime });
+    }
+
+    if (this.jobRetries !== undefined) {
+      data.push({ name: `${this.prefix}/job_retries`, unit: 'count', value: this.jobRetries });
+    }
+
+    if (this.jobDuration !== undefined) {
+      data.push({ name: `${this.prefix}/job_duration`, unit: 'ms', value: this.jobDuration });
+    }
+
+    if (this.timedOutJobs !== undefined) {
+      data.push({ name: `${this.prefix}/timed_out_jobs`, unit: 'count', value: this.timedOutJobs });
+    }
+
+    if (this.resolvedJobs !== undefined) {
+      data.push({ name: `${this.prefix}/resolved_jobs`, unit: 'count', value: this.resolvedJobs });
+    }
+
+    if (this.rejectedJobs !== undefined) {
+      data.push({ name: `${this.prefix}/rejected_jobs`, unit: 'count', value: this.rejectedJobs });
+    }
+
+    if (this.epochProvingDuration !== undefined) {
+      data.push({ name: `${this.prefix}/epoch_proving_duration`, unit: 's', value: this.epochProvingDuration });
+    }
+
+    if (this.provenTransactions !== undefined) {
+      data.push({ name: `${this.prefix}/proven_transactions`, unit: 'count', value: this.provenTransactions });
+    }
+
+    if (this.provenBlocks !== undefined) {
+      data.push({ name: `${this.prefix}/proven_blocks`, unit: 'count', value: this.provenBlocks });
+    }
+
+    const scenario = process.env.BENCH_SCENARIO?.trim();
+    if (!scenario) {
+      return data;
+    }
+
+    const scenarioPrefix = `scenario/${scenario}/`;
+    return data.map(entry => ({ ...entry, name: `${scenarioPrefix}${entry.name}` }));
+  }
+}
+
 export type TxInclusionData = {
   txHash: string;
-  sentAt: number;
-  minedAt: number;
-  attestedAt: number;
+  /** Wall-clock at client when the tx was submitted, in ms (Date.now()). */
+  sentAtMs: number;
+  /** Wall-clock at client when the block containing the tx first became visible, in ms (Date.now()). -1 if never observed. */
+  minedAtMs: number;
+  /** Reserved for future attestation-observed-at signal; -1 today. */
+  attestedAtMs: number;
   blocknumber: number;
   priorityFee: number;
   totalFee: number;
@@ -20,7 +150,7 @@ export type TxInclusionData = {
 export class TxInclusionMetrics {
   private data = new Map<string, TxInclusionData>();
   private groups = new Set<string>();
-  private blocks = new Map<number, Promise<L2BlockNew | undefined>>();
+  private blocks = new Map<number, Promise<BlockResponse<{ includeTransactions: true }> | undefined>>();
 
   private p2pGossipLatencyByTopic: Partial<Record<TopicType, { p50: number; p95: number }>> = {};
 
@@ -31,18 +161,26 @@ export class TxInclusionMetrics {
   private mempoolMinedDelay:
     | { txP50: number; txP95: number; attestationP50: number; attestationP95: number }
     | undefined;
+  private inclusionOutcome: { mined: number; failed: number } | undefined;
 
-  constructor(private aztecNode: AztecNode) {}
+  constructor(
+    private aztecNode: AztecNode,
+    private logger?: Logger,
+  ) {}
 
   recordSentTx(tx: Tx, group: string): void {
     const txHash = tx.getTxHash().toString();
     const priorityFees = tx.getGasSettings().maxPriorityFeesPerGas;
 
+    if (this.data.has(txHash)) {
+      this.logger?.debug(`Overwriting tx inclusion data for ${txHash}`, { txHash, group });
+    }
+
     this.data.set(txHash, {
       txHash,
-      sentAt: Math.trunc(Date.now() / 1000),
-      minedAt: -1,
-      attestedAt: -1,
+      sentAtMs: Date.now(),
+      minedAtMs: -1,
+      attestedAtMs: -1,
       blocknumber: -1,
       priorityFee: Number(priorityFees.feePerDaGas + priorityFees.feePerL2Gas),
       totalFee: -1,
@@ -52,26 +190,86 @@ export class TxInclusionMetrics {
     this.groups.add(group);
   }
 
+  /**
+   * Stamp mined-at metadata for any tracked tx contained in this block, using
+   * `observedAtMs` (caller-supplied wall-clock at the moment they first saw the
+   * block). Idempotent: existing minedAtMs is preserved so the first observer
+   * wins (typically the block-watcher; recordMinedTx is a fallback).
+   */
+  observeBlockForMinedTxs(
+    blockNumber: number,
+    txHashes: ReadonlyArray<{ toString(): string }>,
+    observedAtMs: number,
+  ): void {
+    txHashes.forEach((txHash, position) => {
+      const data = this.data.get(txHash.toString());
+      if (!data || data.minedAtMs !== -1) {
+        return;
+      }
+      data.blocknumber = blockNumber;
+      data.minedAtMs = observedAtMs;
+      data.positionInBlock = position;
+    });
+  }
+
   async recordMinedTx(txReceipt: TxReceipt): Promise<void> {
-    const { status, txHash, blockNumber } = txReceipt;
-    if (status !== TxStatus.SUCCESS || !blockNumber) {
+    const { txHash, blockNumber } = txReceipt;
+    if (!txReceipt.isMined() || !txReceipt.hasExecutionSucceeded() || !blockNumber) {
+      this.logger?.debug('Skipping mined tx record due to receipt status', {
+        txHash: txHash.toString(),
+        status: txReceipt.status,
+        blockNumber,
+      });
       return;
     }
 
-    if (!this.blocks.has(blockNumber)) {
-      this.blocks.set(blockNumber, this.aztecNode.getBlock(blockNumber));
+    const data = this.data.get(txHash.toString());
+    if (!data) {
+      const message = `Missing sent tx record for mined tx ${txHash.toString()}`;
+      this.logger?.warn(message, { txHash: txHash.toString(), blockNumber });
+      throw new Error(message);
     }
+    data.totalFee = Number(txReceipt.transactionFee ?? 0n);
 
-    const block = await this.blocks.get(blockNumber)!;
-    if (!block) {
-      return;
+    // Fallback path for txs the block-watcher missed (e.g. observed only after
+    // the watcher stopped). Stamp with the block's L2 slot timestamp; this is
+    // earlier than the true client-observed time by attestation+propagation
+    // lag, but it's the only deterministic timestamp available post-hoc.
+    if (data.minedAtMs === -1) {
+      if (!this.blocks.has(blockNumber)) {
+        this.blocks.set(blockNumber, this.aztecNode.getBlock(blockNumber, { includeTransactions: true }));
+      }
+      const block = await this.blocks.get(blockNumber)!;
+      if (!block) {
+        this.logger?.warn('Failed to load block for mined tx receipt', { txHash: txHash.toString(), blockNumber });
+        return;
+      }
+      data.blocknumber = blockNumber;
+      data.minedAtMs = Number(block.header.globalVariables.timestamp) * 1000;
+      data.positionInBlock = block.body.txEffects.findIndex(txEffect => txEffect.txHash.equals(txHash));
     }
-    const data = this.data.get(txHash.toString())!;
-    data.blocknumber = blockNumber;
-    data.minedAt = Number(block.header.globalVariables.timestamp);
-    data.attestedAt = -1;
-    data.totalFee = Number(txReceipt.transactionFee ?? 0n);
-    data.positionInBlock = block.body.txEffects.findIndex(txEffect => txEffect.txHash.equals(txHash));
+  }
+
+  /**
+   * Whether this tx was ever observed in a block (by the block-watcher or a mined receipt).
+   * Idempotent first-sighting semantics: a later reorg / pool eviction never clears it, so callers
+   * can treat "ever mined" as included regardless of what happens to the tx afterwards.
+   */
+  public wasMined(txHash: string): boolean {
+    const d = this.data.get(txHash);
+    return !!d && d.minedAtMs !== -1;
+  }
+
+  /** Per-tx inclusion records for a group. Used to serialise out for downstream tooling. */
+  getInclusionRecords(group?: string): TxInclusionData[] {
+    const out: TxInclusionData[] = [];
+    for (const tx of this.data.values()) {
+      if (group !== undefined && tx.group !== group) {
+        continue;
+      }
+      out.push({ ...tx });
+    }
+    return out;
   }
 
   public inclusionTimeInSeconds(group: string): {
@@ -84,12 +282,28 @@ export class TxInclusionMetrics {
     p99: number;
   } {
     const histogram = createHistogram({});
+    let nonPositive = 0;
     for (const tx of this.data.values()) {
-      if (!tx.blocknumber || tx.group !== group || tx.minedAt === -1) {
+      if (!tx.blocknumber || tx.group !== group || tx.minedAtMs === -1) {
         continue;
       }
 
-      histogram.record(tx.minedAt - tx.sentAt);
+      // Both timestamps are client wall-clock (ms). A negative delta should be
+      // impossible since the watcher stamps minedAtMs strictly after sentAtMs,
+      // but the fallback path (recordMinedTx via L2 slot timestamp) can stamp
+      // earlier than sentAtMs. perf_hooks.createHistogram rejects <=0; skip
+      // those instead of crashing.
+      const deltaMs = tx.minedAtMs - tx.sentAtMs;
+      if (deltaMs <= 0) {
+        nonPositive++;
+        continue;
+      }
+      // Histogram is recorded in seconds (rounded) to match the existing
+      // toGithubActionBenchmarkJSON output unit; per-tx records carry the raw ms.
+      histogram.record(Math.max(1, Math.round(deltaMs / 1000)));
+    }
+    if (nonPositive > 0) {
+      this.logger?.debug(`Dropped ${nonPositive} tx inclusion samples with non-positive delta`, { group });
     }
 
     if (histogram.count === 0) {
@@ -139,6 +353,11 @@ export class TxInclusionMetrics {
     this.mempoolMinedDelay = { txP50, txP95, attestationP50, attestationP95 };
   }
 
+  /** Mined vs failed counts for the high-value lane — recorded instead of asserting strict 1:1 inclusion. */
+  public recordInclusionOutcome(mined: number, failed: number): void {
+    this.inclusionOutcome = { mined, failed };
+  }
+
   toGithubActionBenchmarkJSON(): Array<{ name: string; unit: string; value: number; range?: number; extra?: string }> {
     const data: Array<{ name: string; unit: string; value: number; range?: number; extra?: string }> = [];
     for (const group of this.groups) {
@@ -151,7 +370,7 @@ export class TxInclusionMetrics {
           value: stats.mean,
         },
         {
-          name: `${group}/median_inclusion`,
+          name: `${group}/p50_inclusion`,
           unit: 's',
           value: stats.median,
         },
@@ -220,6 +439,16 @@ export class TxInclusionMetrics {
       );
     }
 
+    if (this.inclusionOutcome) {
+      const { mined, failed } = this.inclusionOutcome;
+      const total = mined + failed;
+      data.push(
+        { name: 'inclusion/mined_count', unit: 'count', value: mined },
+        { name: 'inclusion/failed_count', unit: 'count', value: failed },
+        { name: 'inclusion/success_ratio', unit: 'ratio', value: total > 0 ? mined / total : 0 },
+      );
+    }
+
     const scenario = process.env.BENCH_SCENARIO?.trim();
     if (!scenario) {
       return data;

package/src/spartan/utils/bot.ts

@@ -0,0 +1,188 @@
+import type { Logger } from '@aztec/foundation/log';
+import { sleep } from '@aztec/foundation/sleep';
+
+import { exec } from 'child_process';
+import { promisify } from 'util';
+
+import { execHelmCommand, forceDeleteHelmReleaseRecord, getHelmReleaseStatus, hasDeployedHelmRelease } from './helm.js';
+import { deleteResourceByLabel, getChartDir, waitForResourceByLabel } from './k8s.js';
+
+const execAsync = promisify(exec);
+
+export async function restartBot(namespace: string, log: Logger) {
+  log.info(`Restarting bot`);
+  await deleteResourceByLabel({ resource: 'pods', namespace, label: 'app.kubernetes.io/name=bot' });
+  await sleep(10 * 1000);
+  // Some bot images may take time to report Ready due to heavy boot-time proving.
+  // Waiting for PodReadyToStartContainers ensures the pod is scheduled and starting without blocking on full readiness.
+  await waitForResourceByLabel({
+    resource: 'pods',
+    namespace,
+    label: 'app.kubernetes.io/name=bot',
+    condition: 'PodReadyToStartContainers',
+  });
+  log.info(`Bot restarted`);
+}
+
+/**
+ * Installs or upgrades the transfer bot Helm release for the given namespace.
+ * Intended for test setup to enable L2 traffic generation only when needed.
+ */
+export async function installTransferBot({
+  namespace,
+  spartanDir,
+  logger: log,
+  replicas = 1,
+  txIntervalSeconds = 10,
+  followChain = 'CHECKPOINTED',
+  pxeSyncChainTip = 'proposed',
+  mnemonic = process.env.LABS_INFRA_MNEMONIC ?? 'test test test test test test test test test test test junk',
+  mnemonicStartIndex,
+  botPrivateKey = process.env.BOT_TRANSFERS_L2_PRIVATE_KEY ?? '0xcafe01',
+  nodeUrl,
+  timeout = '15m',
+  reuseValues = true,
+  aztecSlotDuration = Number(process.env.AZTEC_SLOT_DURATION ?? 12),
+}: {
+  namespace: string;
+  spartanDir: string;
+  logger: Logger;
+  replicas?: number;
+  txIntervalSeconds?: number;
+  followChain?: string;
+  pxeSyncChainTip?: string;
+  mnemonic?: string;
+  mnemonicStartIndex?: number | string;
+  botPrivateKey?: string;
+  nodeUrl?: string;
+  timeout?: string;
+  reuseValues?: boolean;
+  aztecSlotDuration?: number;
+}) {
+  const instanceName = `${namespace}-bot-transfers`;
+  const helmChartDir = getChartDir(spartanDir, 'aztec-bot');
+  const resolvedNodeUrl = nodeUrl ?? `http://${namespace}-rpc-aztec-node.${namespace}.svc.cluster.local:8080`;
+
+  log.info(`Installing/upgrading transfer bot: replicas=${replicas}, followChain=${followChain}`);
+
+  const values: Record<string, string | number | boolean> = {
+    'bot.replicaCount': replicas,
+    'bot.txIntervalSeconds': txIntervalSeconds,
+    'bot.followChain': followChain,
+    'bot.pxeSyncChainTip': pxeSyncChainTip,
+    'bot.botPrivateKey': botPrivateKey,
+    'bot.nodeUrl': resolvedNodeUrl,
+    'bot.mnemonic': mnemonic,
+    'bot.feePaymentMethod': 'fee_juice',
+    'aztec.slotDuration': aztecSlotDuration,
+    // Ensure bot can reach its own PXE started in-process (default rpc.port is 8080)
+    // Note: since aztec-bot depends on aztec-node with alias `bot`, env vars go under `bot.node.env`.
+    'bot.node.env.BOT_PXE_URL': 'http://127.0.0.1:8080',
+    // Provide L1 execution RPC for bridging fee juice
+    'bot.node.env.ETHEREUM_HOSTS': `http://${namespace}-eth-execution.${namespace}.svc.cluster.local:8545`,
+    // Provide L1 mnemonic for bridging (falls back to labs mnemonic)
+    'bot.node.env.BOT_L1_MNEMONIC': mnemonic,
+
+    // The bot does not need Kubernetes API access. Disable RBAC + ServiceAccount creation so the chart
+    // can be installed by users without cluster-scoped RBAC permissions.
+    'bot.rbac.create': false,
+    'bot.serviceAccount.create': false,
+    'bot.serviceAccount.name': 'default',
+  };
+  // Ensure we derive a funded L1 key (index 0 is funded on anvil default mnemonic)
+  if (mnemonicStartIndex === undefined) {
+    values['bot.mnemonicStartIndex'] = 0;
+  }
+  // Also pass a funded private key directly if available
+  if (process.env.FUNDING_PRIVATE_KEY) {
+    values['bot.node.env.BOT_L1_PRIVATE_KEY'] = process.env.FUNDING_PRIVATE_KEY;
+  }
+  // Align bot image with the running network image: prefer env var, else detect from a validator pod
+  let repositoryFromEnv: string | undefined;
+  let tagFromEnv: string | undefined;
+  const aztecDockerImage = process.env.AZTEC_DOCKER_IMAGE;
+  if (aztecDockerImage && aztecDockerImage.includes(':')) {
+    const lastColon = aztecDockerImage.lastIndexOf(':');
+    repositoryFromEnv = aztecDockerImage.slice(0, lastColon);
+    tagFromEnv = aztecDockerImage.slice(lastColon + 1);
+  }
+
+  let repository = repositoryFromEnv;
+  let tag = tagFromEnv;
+  if (!repository || !tag) {
+    try {
+      const { stdout } = await execAsync(
+        `kubectl get pods -l app.kubernetes.io/name=validator -n ${namespace} -o jsonpath='{.items[0].spec.containers[?(@.name=="aztec")].image}' | cat`,
+      );
+      const image = stdout.trim().replace(/^'|'$/g, '');
+      if (image && image.includes(':')) {
+        const lastColon = image.lastIndexOf(':');
+        repository = image.slice(0, lastColon);
+        tag = image.slice(lastColon + 1);
+      }
+    } catch (err) {
+      log.warn(`Could not detect aztec image from validator pod: ${String(err)}`);
+    }
+  }
+  if (repository && tag) {
+    values['global.aztecImage.repository'] = repository;
+    values['global.aztecImage.tag'] = tag;
+  }
+  if (mnemonicStartIndex !== undefined) {
+    values['bot.mnemonicStartIndex'] =
+      typeof mnemonicStartIndex === 'string' ? mnemonicStartIndex : Number(mnemonicStartIndex);
+  }
+
+  // If a previous install attempt left the release in a non-deployed state (e.g. FAILED),
+  // `helm upgrade --install` can error with "has no deployed releases".
+  // In that case, clear the release record and do a clean install.
+  const existingStatus = await getHelmReleaseStatus(instanceName, namespace);
+  if (existingStatus && existingStatus.toLowerCase() !== 'deployed') {
+    log.warn(`Transfer bot release ${instanceName} is in status '${existingStatus}'. Reinstalling cleanly.`);
+    await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`).catch(
+      () => undefined,
+    );
+    // If helm left the release in `uninstalling`, force-delete the record so we can reinstall.
+    const afterUninstallStatus = await getHelmReleaseStatus(instanceName, namespace);
+    if (afterUninstallStatus?.toLowerCase() === 'uninstalling') {
+      await forceDeleteHelmReleaseRecord(instanceName, namespace, log);
+    }
+  }
+
+  // `--reuse-values` fails if the release has never successfully deployed (e.g. first install, or a previous failed install).
+  // Only reuse values when we have a deployed release to reuse from.
+  const effectiveReuseValues = reuseValues && (await hasDeployedHelmRelease(instanceName, namespace));
+
+  await execHelmCommand({
+    instanceName,
+    helmChartDir,
+    namespace,
+    valuesFile: undefined,
+    timeout,
+    values: values as unknown as Record<string, string | number | boolean>,
+    reuseValues: effectiveReuseValues,
+  });
+
+  if (replicas > 0) {
+    await waitForResourceByLabel({
+      resource: 'pods',
+      namespace,
+      label: 'app.kubernetes.io/name=bot',
+      condition: 'PodReadyToStartContainers',
+    });
+  }
+}
+
+/**
+ * Uninstalls the transfer bot Helm release from the given namespace.
+ * Intended for test teardown to clean up bot resources.
+ */
+export async function uninstallTransferBot(namespace: string, log: Logger) {
+  const instanceName = `${namespace}-bot-transfers`;
+  log.info(`Uninstalling transfer bot release ${instanceName}`);
+  await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`);
+  // Ensure any leftover pods are removed
+  await deleteResourceByLabel({ resource: 'pods', namespace, label: 'app.kubernetes.io/name=bot' }).catch(
+    () => undefined,
+  );
+}