@aztec/end-to-end 0.0.1-commit.d431d1c → 0.0.1-commit.d58ff9d0

package/dest/spartan/tx_metrics.js

@@ -1,7 +1,157 @@
-import { TxStatus } from '@aztec/stdlib/tx';
 import { createHistogram } from 'perf_hooks';
+/** Metrics class for proving-related benchmarks. */ export class ProvingMetrics {
+    prefix;
+    successfulTxs;
+    proofDuration;
+    activeAgents;
+    avgQueueTime;
+    jobRetries;
+    jobDuration;
+    timedOutJobs;
+    resolvedJobs;
+    rejectedJobs;
+    epochProvingDuration;
+    provenTransactions;
+    provenBlocks;
+    constructor(prefix){
+        this.prefix = prefix;
+    }
+    recordSuccessfulTxs(count) {
+        this.successfulTxs = count;
+    }
+    recordProofDuration(seconds) {
+        this.proofDuration = seconds;
+    }
+    recordActiveAgents(count) {
+        this.activeAgents = count;
+    }
+    recordAvgQueueTime(ms) {
+        this.avgQueueTime = ms;
+    }
+    recordJobRetries(count) {
+        this.jobRetries = count;
+    }
+    recordJobDuration(ms) {
+        this.jobDuration = ms;
+    }
+    recordTimedOutJobs(count) {
+        this.timedOutJobs = count;
+    }
+    recordResolvedJobs(count) {
+        this.resolvedJobs = count;
+    }
+    recordRejectedJobs(count) {
+        this.rejectedJobs = count;
+    }
+    recordEpochProvingDuration(seconds) {
+        this.epochProvingDuration = seconds;
+    }
+    recordProvenTransactions(count) {
+        this.provenTransactions = count;
+    }
+    recordProvenBlocks(count) {
+        this.provenBlocks = count;
+    }
+    toGithubActionBenchmarkJSON() {
+        const data = [];
+        if (this.successfulTxs !== undefined) {
+            data.push({
+                name: `${this.prefix}/successful_txs`,
+                unit: 'count',
+                value: this.successfulTxs
+            });
+        }
+        if (this.proofDuration !== undefined) {
+            data.push({
+                name: `${this.prefix}/proof_duration`,
+                unit: 's',
+                value: this.proofDuration
+            });
+        }
+        if (this.activeAgents !== undefined) {
+            data.push({
+                name: `${this.prefix}/active_agents`,
+                unit: 'count',
+                value: this.activeAgents
+            });
+        }
+        if (this.avgQueueTime !== undefined) {
+            data.push({
+                name: `${this.prefix}/avg_queue_time`,
+                unit: 'ms',
+                value: this.avgQueueTime
+            });
+        }
+        if (this.jobRetries !== undefined) {
+            data.push({
+                name: `${this.prefix}/job_retries`,
+                unit: 'count',
+                value: this.jobRetries
+            });
+        }
+        if (this.jobDuration !== undefined) {
+            data.push({
+                name: `${this.prefix}/job_duration`,
+                unit: 'ms',
+                value: this.jobDuration
+            });
+        }
+        if (this.timedOutJobs !== undefined) {
+            data.push({
+                name: `${this.prefix}/timed_out_jobs`,
+                unit: 'count',
+                value: this.timedOutJobs
+            });
+        }
+        if (this.resolvedJobs !== undefined) {
+            data.push({
+                name: `${this.prefix}/resolved_jobs`,
+                unit: 'count',
+                value: this.resolvedJobs
+            });
+        }
+        if (this.rejectedJobs !== undefined) {
+            data.push({
+                name: `${this.prefix}/rejected_jobs`,
+                unit: 'count',
+                value: this.rejectedJobs
+            });
+        }
+        if (this.epochProvingDuration !== undefined) {
+            data.push({
+                name: `${this.prefix}/epoch_proving_duration`,
+                unit: 's',
+                value: this.epochProvingDuration
+            });
+        }
+        if (this.provenTransactions !== undefined) {
+            data.push({
+                name: `${this.prefix}/proven_transactions`,
+                unit: 'count',
+                value: this.provenTransactions
+            });
+        }
+        if (this.provenBlocks !== undefined) {
+            data.push({
+                name: `${this.prefix}/proven_blocks`,
+                unit: 'count',
+                value: this.provenBlocks
+            });
+        }
+        const scenario = process.env.BENCH_SCENARIO?.trim();
+        if (!scenario) {
+            return data;
+        }
+        const scenarioPrefix = `scenario/${scenario}/`;
+        return data.map((entry)=>({
+                ...entry,
+                name: `${scenarioPrefix}${entry.name}`
+            }));
+    }
+}
 export class TxInclusionMetrics {
     aztecNode;
+    logger;
     data;
     groups;
     blocks;
@@ -11,8 +161,10 @@ export class TxInclusionMetrics {
     reqRespStats;
     peerStats;
     mempoolMinedDelay;
-    constructor(aztecNode){
+    inclusionOutcome;
+    constructor(aztecNode, logger){
         this.aztecNode = aztecNode;
+        this.logger = logger;
         this.data = new Map();
         this.groups = new Set();
         this.blocks = new Map();
@@ -21,11 +173,17 @@ export class TxInclusionMetrics {
     recordSentTx(tx, group) {
         const txHash = tx.getTxHash().toString();
         const priorityFees = tx.getGasSettings().maxPriorityFeesPerGas;
+        if (this.data.has(txHash)) {
+            this.logger?.debug(`Overwriting tx inclusion data for ${txHash}`, {
+                txHash,
+                group
+            });
+        }
         this.data.set(txHash, {
             txHash,
-            sentAt: Math.trunc(Date.now() / 1000),
-            minedAt: -1,
-            attestedAt: -1,
+            sentAtMs: Date.now(),
+            minedAtMs: -1,
+            attestedAtMs: -1,
             blocknumber: -1,
             priorityFee: Number(priorityFees.feePerDaGas + priorityFees.feePerL2Gas),
             totalFee: -1,
@@ -34,32 +192,110 @@ export class TxInclusionMetrics {
         });
         this.groups.add(group);
     }
+    /**
+   * Stamp mined-at metadata for any tracked tx contained in this block, using
+   * `observedAtMs` (caller-supplied wall-clock at the moment they first saw the
+   * block). Idempotent: existing minedAtMs is preserved so the first observer
+   * wins (typically the block-watcher; recordMinedTx is a fallback).
+   */ observeBlockForMinedTxs(blockNumber, txHashes, observedAtMs) {
+        txHashes.forEach((txHash, position)=>{
+            const data = this.data.get(txHash.toString());
+            if (!data || data.minedAtMs !== -1) {
+                return;
+            }
+            data.blocknumber = blockNumber;
+            data.minedAtMs = observedAtMs;
+            data.positionInBlock = position;
+        });
+    }
     async recordMinedTx(txReceipt) {
-        const { status, txHash, blockNumber } = txReceipt;
-        if (status !== TxStatus.SUCCESS || !blockNumber) {
-            return;
-        }
-        if (!this.blocks.has(blockNumber)) {
-            this.blocks.set(blockNumber, this.aztecNode.getBlock(blockNumber));
-        }
-        const block = await this.blocks.get(blockNumber);
-        if (!block) {
+        const { txHash, blockNumber } = txReceipt;
+        if (!txReceipt.isMined() || !txReceipt.hasExecutionSucceeded() || !blockNumber) {
+            this.logger?.debug('Skipping mined tx record due to receipt status', {
+                txHash: txHash.toString(),
+                status: txReceipt.status,
+                blockNumber
+            });
             return;
         }
         const data = this.data.get(txHash.toString());
-        data.blocknumber = blockNumber;
-        data.minedAt = Number(block.header.globalVariables.timestamp);
-        data.attestedAt = -1;
+        if (!data) {
+            const message = `Missing sent tx record for mined tx ${txHash.toString()}`;
+            this.logger?.warn(message, {
+                txHash: txHash.toString(),
+                blockNumber
+            });
+            throw new Error(message);
+        }
         data.totalFee = Number(txReceipt.transactionFee ?? 0n);
-        data.positionInBlock = block.body.txEffects.findIndex((txEffect)=>txEffect.txHash.equals(txHash));
+        // Fallback path for txs the block-watcher missed (e.g. observed only after
+        // the watcher stopped). Stamp with the block's L2 slot timestamp; this is
+        // earlier than the true client-observed time by attestation+propagation
+        // lag, but it's the only deterministic timestamp available post-hoc.
+        if (data.minedAtMs === -1) {
+            if (!this.blocks.has(blockNumber)) {
+                this.blocks.set(blockNumber, this.aztecNode.getBlock(blockNumber, {
+                    includeTransactions: true
+                }));
+            }
+            const block = await this.blocks.get(blockNumber);
+            if (!block) {
+                this.logger?.warn('Failed to load block for mined tx receipt', {
+                    txHash: txHash.toString(),
+                    blockNumber
+                });
+                return;
+            }
+            data.blocknumber = blockNumber;
+            data.minedAtMs = Number(block.header.globalVariables.timestamp) * 1000;
+            data.positionInBlock = block.body.txEffects.findIndex((txEffect)=>txEffect.txHash.equals(txHash));
+        }
+    }
+    /**
+   * Whether this tx was ever observed in a block (by the block-watcher or a mined receipt).
+   * Idempotent first-sighting semantics: a later reorg / pool eviction never clears it, so callers
+   * can treat "ever mined" as included regardless of what happens to the tx afterwards.
+   */ wasMined(txHash) {
+        const d = this.data.get(txHash);
+        return !!d && d.minedAtMs !== -1;
+    }
+    /** Per-tx inclusion records for a group. Used to serialise out for downstream tooling. */ getInclusionRecords(group) {
+        const out = [];
+        for (const tx of this.data.values()){
+            if (group !== undefined && tx.group !== group) {
+                continue;
+            }
+            out.push({
+                ...tx
+            });
+        }
+        return out;
     }
     inclusionTimeInSeconds(group) {
         const histogram = createHistogram({});
+        let nonPositive = 0;
         for (const tx of this.data.values()){
-            if (!tx.blocknumber || tx.group !== group || tx.minedAt === -1) {
+            if (!tx.blocknumber || tx.group !== group || tx.minedAtMs === -1) {
                 continue;
             }
-            histogram.record(tx.minedAt - tx.sentAt);
+            // Both timestamps are client wall-clock (ms). A negative delta should be
+            // impossible since the watcher stamps minedAtMs strictly after sentAtMs,
+            // but the fallback path (recordMinedTx via L2 slot timestamp) can stamp
+            // earlier than sentAtMs. perf_hooks.createHistogram rejects <=0; skip
+            // those instead of crashing.
+            const deltaMs = tx.minedAtMs - tx.sentAtMs;
+            if (deltaMs <= 0) {
+                nonPositive++;
+                continue;
+            }
+            // Histogram is recorded in seconds (rounded) to match the existing
+            // toGithubActionBenchmarkJSON output unit; per-tx records carry the raw ms.
+            histogram.record(Math.max(1, Math.round(deltaMs / 1000)));
+        }
+        if (nonPositive > 0) {
+            this.logger?.debug(`Dropped ${nonPositive} tx inclusion samples with non-positive delta`, {
+                group
+            });
         }
         if (histogram.count === 0) {
             return {
@@ -123,6 +359,12 @@ export class TxInclusionMetrics {
             attestationP95
         };
     }
+    /** Mined vs failed counts for the high-value lane — recorded instead of asserting strict 1:1 inclusion. */ recordInclusionOutcome(mined, failed) {
+        this.inclusionOutcome = {
+            mined,
+            failed
+        };
+    }
     toGithubActionBenchmarkJSON() {
         const data = [];
         for (const group of this.groups){
@@ -132,7 +374,7 @@ export class TxInclusionMetrics {
                 unit: 's',
                 value: stats.mean
             }, {
-                name: `${group}/median_inclusion`,
+                name: `${group}/p50_inclusion`,
                 unit: 's',
                 value: stats.median
             }, {
@@ -235,6 +477,23 @@ export class TxInclusionMetrics {
                 value: this.mempoolMinedDelay.attestationP95
             });
         }
+        if (this.inclusionOutcome) {
+            const { mined, failed } = this.inclusionOutcome;
+            const total = mined + failed;
+            data.push({
+                name: 'inclusion/mined_count',
+                unit: 'count',
+                value: mined
+            }, {
+                name: 'inclusion/failed_count',
+                unit: 'count',
+                value: failed
+            }, {
+                name: 'inclusion/success_ratio',
+                unit: 'ratio',
+                value: total > 0 ? mined / total : 0
+            });
+        }
         const scenario = process.env.BENCH_SCENARIO?.trim();
         if (!scenario) {
             return data;

package/dest/spartan/utils/bot.d.ts

@@ -0,0 +1,28 @@
+import type { Logger } from '@aztec/foundation/log';
+export declare function restartBot(namespace: string, log: Logger): Promise<void>;
+/**
+ * Installs or upgrades the transfer bot Helm release for the given namespace.
+ * Intended for test setup to enable L2 traffic generation only when needed.
+ */
+export declare function installTransferBot({ namespace, spartanDir, logger: log, replicas, txIntervalSeconds, followChain, pxeSyncChainTip, mnemonic, mnemonicStartIndex, botPrivateKey, nodeUrl, timeout, reuseValues, aztecSlotDuration }: {
+    namespace: string;
+    spartanDir: string;
+    logger: Logger;
+    replicas?: number;
+    txIntervalSeconds?: number;
+    followChain?: string;
+    pxeSyncChainTip?: string;
+    mnemonic?: string;
+    mnemonicStartIndex?: number | string;
+    botPrivateKey?: string;
+    nodeUrl?: string;
+    timeout?: string;
+    reuseValues?: boolean;
+    aztecSlotDuration?: number;
+}): Promise<void>;
+/**
+ * Uninstalls the transfer bot Helm release from the given namespace.
+ * Intended for test teardown to clean up bot resources.
+ */
+export declare function uninstallTransferBot(namespace: string, log: Logger): Promise<void>;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYm90LmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc3BhcnRhbi91dGlscy9ib3QudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUsTUFBTSxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFXcEQsd0JBQXNCLFVBQVUsQ0FBQyxTQUFTLEVBQUUsTUFBTSxFQUFFLEdBQUcsRUFBRSxNQUFNLGlCQWE5RDtBQUVEOzs7R0FHRztBQUNILHdCQUFzQixrQkFBa0IsQ0FBQyxFQUN2QyxTQUFTLEVBQ1QsVUFBVSxFQUNWLE1BQU0sRUFBRSxHQUFHLEVBQ1gsUUFBWSxFQUNaLGlCQUFzQixFQUN0QixXQUE0QixFQUM1QixlQUE0QixFQUM1QixRQUEyRyxFQUMzRyxrQkFBa0IsRUFDbEIsYUFBc0UsRUFDdEUsT0FBTyxFQUNQLE9BQWUsRUFDZixXQUFrQixFQUNsQixpQkFBaUUsRUFDbEUsRUFBRTtJQUNELFNBQVMsRUFBRSxNQUFNLENBQUM7SUFDbEIsVUFBVSxFQUFFLE1BQU0sQ0FBQztJQUNuQixNQUFNLEVBQUUsTUFBTSxDQUFDO0lBQ2YsUUFBUSxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLGlCQUFpQixDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQzNCLFdBQVcsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUNyQixlQUFlLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDekIsUUFBUSxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLGtCQUFrQixDQUFDLEVBQUUsTUFBTSxHQUFHLE1BQU0sQ0FBQztJQUNyQyxhQUFhLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDdkIsT0FBTyxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ2pCLE9BQU8sQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUNqQixXQUFXLENBQUMsRUFBRSxPQUFPLENBQUM7SUFDdEIsaUJBQWlCLENBQUMsRUFBRSxNQUFNLENBQUM7Q0FDNUIsaUJBaUhBO0FBRUQ7OztHQUdHO0FBQ0gsd0JBQXNCLG9CQUFvQixDQUFDLFNBQVMsRUFBRSxNQUFNLEVBQUUsR0FBRyxFQUFFLE1BQU0saUJBUXhFIn0=

package/dest/spartan/utils/bot.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bot.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/bot.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAWpD,wBAAsB,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,iBAa9D;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,EACvC,SAAS,EACT,UAAU,EACV,MAAM,EAAE,GAAG,EACX,QAAY,EACZ,iBAAsB,EACtB,WAA4B,EAC5B,eAA4B,EAC5B,QAA2G,EAC3G,kBAAkB,EAClB,aAAsE,EACtE,OAAO,EACP,OAAe,EACf,WAAkB,EAClB,iBAAiE,EAClE,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACrC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,iBAiHA;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,iBAQxE"}

package/dest/spartan/utils/bot.js

@@ -0,0 +1,142 @@
+import { sleep } from '@aztec/foundation/sleep';
+import { exec } from 'child_process';
+import { promisify } from 'util';
+import { execHelmCommand, forceDeleteHelmReleaseRecord, getHelmReleaseStatus, hasDeployedHelmRelease } from './helm.js';
+import { deleteResourceByLabel, getChartDir, waitForResourceByLabel } from './k8s.js';
+const execAsync = promisify(exec);
+export async function restartBot(namespace, log) {
+    log.info(`Restarting bot`);
+    await deleteResourceByLabel({
+        resource: 'pods',
+        namespace,
+        label: 'app.kubernetes.io/name=bot'
+    });
+    await sleep(10 * 1000);
+    // Some bot images may take time to report Ready due to heavy boot-time proving.
+    // Waiting for PodReadyToStartContainers ensures the pod is scheduled and starting without blocking on full readiness.
+    await waitForResourceByLabel({
+        resource: 'pods',
+        namespace,
+        label: 'app.kubernetes.io/name=bot',
+        condition: 'PodReadyToStartContainers'
+    });
+    log.info(`Bot restarted`);
+}
+/**
+ * Installs or upgrades the transfer bot Helm release for the given namespace.
+ * Intended for test setup to enable L2 traffic generation only when needed.
+ */ export async function installTransferBot({ namespace, spartanDir, logger: log, replicas = 1, txIntervalSeconds = 10, followChain = 'CHECKPOINTED', pxeSyncChainTip = 'proposed', mnemonic = process.env.LABS_INFRA_MNEMONIC ?? 'test test test test test test test test test test test junk', mnemonicStartIndex, botPrivateKey = process.env.BOT_TRANSFERS_L2_PRIVATE_KEY ?? '0xcafe01', nodeUrl, timeout = '15m', reuseValues = true, aztecSlotDuration = Number(process.env.AZTEC_SLOT_DURATION ?? 12) }) {
+    const instanceName = `${namespace}-bot-transfers`;
+    const helmChartDir = getChartDir(spartanDir, 'aztec-bot');
+    const resolvedNodeUrl = nodeUrl ?? `http://${namespace}-rpc-aztec-node.${namespace}.svc.cluster.local:8080`;
+    log.info(`Installing/upgrading transfer bot: replicas=${replicas}, followChain=${followChain}`);
+    const values = {
+        'bot.replicaCount': replicas,
+        'bot.txIntervalSeconds': txIntervalSeconds,
+        'bot.followChain': followChain,
+        'bot.pxeSyncChainTip': pxeSyncChainTip,
+        'bot.botPrivateKey': botPrivateKey,
+        'bot.nodeUrl': resolvedNodeUrl,
+        'bot.mnemonic': mnemonic,
+        'bot.feePaymentMethod': 'fee_juice',
+        'aztec.slotDuration': aztecSlotDuration,
+        // Ensure bot can reach its own PXE started in-process (default rpc.port is 8080)
+        // Note: since aztec-bot depends on aztec-node with alias `bot`, env vars go under `bot.node.env`.
+        'bot.node.env.BOT_PXE_URL': 'http://127.0.0.1:8080',
+        // Provide L1 execution RPC for bridging fee juice
+        'bot.node.env.ETHEREUM_HOSTS': `http://${namespace}-eth-execution.${namespace}.svc.cluster.local:8545`,
+        // Provide L1 mnemonic for bridging (falls back to labs mnemonic)
+        'bot.node.env.BOT_L1_MNEMONIC': mnemonic,
+        // The bot does not need Kubernetes API access. Disable RBAC + ServiceAccount creation so the chart
+        // can be installed by users without cluster-scoped RBAC permissions.
+        'bot.rbac.create': false,
+        'bot.serviceAccount.create': false,
+        'bot.serviceAccount.name': 'default'
+    };
+    // Ensure we derive a funded L1 key (index 0 is funded on anvil default mnemonic)
+    if (mnemonicStartIndex === undefined) {
+        values['bot.mnemonicStartIndex'] = 0;
+    }
+    // Also pass a funded private key directly if available
+    if (process.env.FUNDING_PRIVATE_KEY) {
+        values['bot.node.env.BOT_L1_PRIVATE_KEY'] = process.env.FUNDING_PRIVATE_KEY;
+    }
+    // Align bot image with the running network image: prefer env var, else detect from a validator pod
+    let repositoryFromEnv;
+    let tagFromEnv;
+    const aztecDockerImage = process.env.AZTEC_DOCKER_IMAGE;
+    if (aztecDockerImage && aztecDockerImage.includes(':')) {
+        const lastColon = aztecDockerImage.lastIndexOf(':');
+        repositoryFromEnv = aztecDockerImage.slice(0, lastColon);
+        tagFromEnv = aztecDockerImage.slice(lastColon + 1);
+    }
+    let repository = repositoryFromEnv;
+    let tag = tagFromEnv;
+    if (!repository || !tag) {
+        try {
+            const { stdout } = await execAsync(`kubectl get pods -l app.kubernetes.io/name=validator -n ${namespace} -o jsonpath='{.items[0].spec.containers[?(@.name=="aztec")].image}' | cat`);
+            const image = stdout.trim().replace(/^'|'$/g, '');
+            if (image && image.includes(':')) {
+                const lastColon = image.lastIndexOf(':');
+                repository = image.slice(0, lastColon);
+                tag = image.slice(lastColon + 1);
+            }
+        } catch (err) {
+            log.warn(`Could not detect aztec image from validator pod: ${String(err)}`);
+        }
+    }
+    if (repository && tag) {
+        values['global.aztecImage.repository'] = repository;
+        values['global.aztecImage.tag'] = tag;
+    }
+    if (mnemonicStartIndex !== undefined) {
+        values['bot.mnemonicStartIndex'] = typeof mnemonicStartIndex === 'string' ? mnemonicStartIndex : Number(mnemonicStartIndex);
+    }
+    // If a previous install attempt left the release in a non-deployed state (e.g. FAILED),
+    // `helm upgrade --install` can error with "has no deployed releases".
+    // In that case, clear the release record and do a clean install.
+    const existingStatus = await getHelmReleaseStatus(instanceName, namespace);
+    if (existingStatus && existingStatus.toLowerCase() !== 'deployed') {
+        log.warn(`Transfer bot release ${instanceName} is in status '${existingStatus}'. Reinstalling cleanly.`);
+        await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`).catch(()=>undefined);
+        // If helm left the release in `uninstalling`, force-delete the record so we can reinstall.
+        const afterUninstallStatus = await getHelmReleaseStatus(instanceName, namespace);
+        if (afterUninstallStatus?.toLowerCase() === 'uninstalling') {
+            await forceDeleteHelmReleaseRecord(instanceName, namespace, log);
+        }
+    }
+    // `--reuse-values` fails if the release has never successfully deployed (e.g. first install, or a previous failed install).
+    // Only reuse values when we have a deployed release to reuse from.
+    const effectiveReuseValues = reuseValues && await hasDeployedHelmRelease(instanceName, namespace);
+    await execHelmCommand({
+        instanceName,
+        helmChartDir,
+        namespace,
+        valuesFile: undefined,
+        timeout,
+        values: values,
+        reuseValues: effectiveReuseValues
+    });
+    if (replicas > 0) {
+        await waitForResourceByLabel({
+            resource: 'pods',
+            namespace,
+            label: 'app.kubernetes.io/name=bot',
+            condition: 'PodReadyToStartContainers'
+        });
+    }
+}
+/**
+ * Uninstalls the transfer bot Helm release from the given namespace.
+ * Intended for test teardown to clean up bot resources.
+ */ export async function uninstallTransferBot(namespace, log) {
+    const instanceName = `${namespace}-bot-transfers`;
+    log.info(`Uninstalling transfer bot release ${instanceName}`);
+    await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`);
+    // Ensure any leftover pods are removed
+    await deleteResourceByLabel({
+        resource: 'pods',
+        namespace,
+        label: 'app.kubernetes.io/name=bot'
+    }).catch(()=>undefined);
+}

package/dest/spartan/utils/chaos.d.ts

@@ -0,0 +1,79 @@
+import type { Logger } from '@aztec/foundation/log';
+export declare function uninstallChaosMesh(instanceName: string, namespace: string, log: Logger): Promise<void>;
+/**
+ * Installs a Helm chart with the given parameters.
+ * @param instanceName - The name of the Helm chart instance.
+ * @param targetNamespace - The namespace with the resources to be affected by the Helm chart.
+ * @param valuesFile - The values file to use for the Helm chart.
+ * @param chaosMeshNamespace - The namespace to install the Helm chart in.
+ * @param timeout - The timeout for the Helm command.
+ * @param clean - Whether to clean up the Helm chart before installing it.
+ * @returns The stdout of the Helm command.
+ * @throws If the Helm command fails.
+ *
+ * Example usage:
+ * ```typescript
+ * const stdout = await installChaosMeshChart({ instanceName: 'force-reorg', targetNamespace: 'smoke', valuesFile: 'prover-failure.yaml'});
+ * console.log(stdout);
+ * ```
+ */
+export declare function installChaosMeshChart({ instanceName, targetNamespace, valuesFile, helmChartDir, timeout, clean, values, logger: log }: {
+    instanceName: string;
+    targetNamespace: string;
+    valuesFile: string;
+    helmChartDir: string;
+    chaosMeshNamespace?: string;
+    timeout?: string;
+    clean?: boolean;
+    values?: Record<string, string | number>;
+    logger: Logger;
+}): Promise<string>;
+export declare function applyProverFailure({ namespace, spartanDir, durationSeconds, logger: log }: {
+    namespace: string;
+    spartanDir: string;
+    durationSeconds: number;
+    logger: Logger;
+}): Promise<string>;
+export declare function applyValidatorFailure({ namespace, spartanDir, logger: log, values, instanceName }: {
+    namespace: string;
+    spartanDir: string;
+    logger: Logger;
+    values?: Record<string, string | number>;
+    instanceName?: string;
+}): Promise<string>;
+export declare function applyProverKill({ namespace, spartanDir, logger: log, values }: {
+    namespace: string;
+    spartanDir: string;
+    logger: Logger;
+    values?: Record<string, string | number>;
+}): Promise<string>;
+export declare function applyProverBrokerKill({ namespace, spartanDir, logger: log, values }: {
+    namespace: string;
+    spartanDir: string;
+    logger: Logger;
+    values?: Record<string, string | number>;
+}): Promise<string>;
+export declare function applyBootNodeFailure({ instanceName, namespace, spartanDir, durationSeconds, logger: log, values }: {
+    instanceName?: string;
+    namespace: string;
+    spartanDir: string;
+    durationSeconds: number;
+    logger: Logger;
+    values?: Record<string, string | number>;
+}): Promise<string>;
+export declare function applyValidatorKill({ instanceName, namespace, spartanDir, logger: log, values, clean }: {
+    instanceName?: string;
+    namespace: string;
+    spartanDir: string;
+    logger: Logger;
+    values?: Record<string, string | number>;
+    clean?: boolean;
+}): Promise<string>;
+export declare function applyNetworkShaping({ instanceName, valuesFile, namespace, spartanDir, logger: log }: {
+    instanceName?: string;
+    valuesFile: string;
+    namespace: string;
+    spartanDir: string;
+    logger: Logger;
+}): Promise<string>;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY2hhb3MuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zcGFydGFuL3V0aWxzL2NoYW9zLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLHVCQUF1QixDQUFDO0FBVXBELHdCQUFzQixrQkFBa0IsQ0FBQyxZQUFZLEVBQUUsTUFBTSxFQUFFLFNBQVMsRUFBRSxNQUFNLEVBQUUsR0FBRyxFQUFFLE1BQU0saUJBd0I1RjtBQUVEOzs7Ozs7Ozs7Ozs7Ozs7O0dBZ0JHO0FBQ0gsd0JBQXNCLHFCQUFxQixDQUFDLEVBQzFDLFlBQVksRUFDWixlQUFlLEVBQ2YsVUFBVSxFQUNWLFlBQVksRUFDWixPQUFlLEVBQ2YsS0FBWSxFQUNaLE1BQVcsRUFDWCxNQUFNLEVBQUUsR0FBRyxFQUNaLEVBQUU7SUFDRCxZQUFZLEVBQUUsTUFBTSxDQUFDO0lBQ3JCLGVBQWUsRUFBRSxNQUFNLENBQUM7SUFDeEIsVUFBVSxFQUFFLE1BQU0sQ0FBQztJQUNuQixZQUFZLEVBQUUsTUFBTSxDQUFDO0lBQ3JCLGtCQUFrQixDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQzVCLE9BQU8sQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUNqQixLQUFLLENBQUMsRUFBRSxPQUFPLENBQUM7SUFDaEIsTUFBTSxDQUFDLEVBQUUsTUFBTSxDQUFDLE1BQU0sRUFBRSxNQUFNLEdBQUcsTUFBTSxDQUFDLENBQUM7SUFDekMsTUFBTSxFQUFFLE1BQU0sQ0FBQztDQUNoQixtQkFhQTtBQUVELHdCQUFnQixrQkFBa0IsQ0FBQyxFQUNqQyxTQUFTLEVBQ1QsVUFBVSxFQUNWLGVBQWUsRUFDZixNQUFNLEVBQUUsR0FBRyxFQUNaLEVBQUU7SUFDRCxTQUFTLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLFVBQVUsRUFBRSxNQUFNLENBQUM7SUFDbkIsZUFBZSxFQUFFLE1BQU0sQ0FBQztJQUN4QixNQUFNLEVBQUUsTUFBTSxDQUFDO0NBQ2hCLG1CQVdBO0FBRUQsd0JBQWdCLHFCQUFxQixDQUFDLEVBQ3BDLFNBQVMsRUFDVCxVQUFVLEVBQ1YsTUFBTSxFQUFFLEdBQUcsRUFDWCxNQUFNLEVBQ04sWUFBWSxFQUNiLEVBQUU7SUFDRCxTQUFTLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLFVBQVUsRUFBRSxNQUFNLENBQUM7SUFDbkIsTUFBTSxFQUFFLE1BQU0sQ0FBQztJQUNmLE1BQU0sQ0FBQyxFQUFFLE1BQU0sQ0FBQyxNQUFNLEVBQUUsTUFBTSxHQUFHLE1BQU0sQ0FBQyxDQUFDO0lBQ3pDLFlBQVksQ0FBQyxFQUFFLE1BQU0sQ0FBQztDQUN2QixtQkFTQTtBQUVELHdCQUFnQixlQUFlLENBQUMsRUFDOUIsU0FBUyxFQUNULFVBQVUsRUFDVixNQUFNLEVBQUUsR0FBRyxFQUNYLE1BQU0sRUFDUCxFQUFFO0lBQ0QsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixVQUFVLEVBQUUsTUFBTSxDQUFDO0lBQ25CLE1BQU0sRUFBRSxNQUFNLENBQUM7SUFDZixNQUFNLENBQUMsRUFBRSxNQUFNLENBQUMsTUFBTSxFQUFFLE1BQU0sR0FBRyxNQUFNLENBQUMsQ0FBQztDQUMxQyxtQkFXQTtBQUVELHdCQUFnQixxQkFBcUIsQ0FBQyxFQUNwQyxTQUFTLEVBQ1QsVUFBVSxFQUNWLE1BQU0sRUFBRSxHQUFHLEVBQ1gsTUFBTSxFQUNQLEVBQUU7SUFDRCxTQUFTLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLFVBQVUsRUFBRSxNQUFNLENBQUM7SUFDbkIsTUFBTSxFQUFFLE1BQU0sQ0FBQztJQUNmLE1BQU0sQ0FBQyxFQUFFLE1BQU0sQ0FBQyxNQUFNLEVBQUUsTUFBTSxHQUFHLE1BQU0sQ0FBQyxDQUFDO0NBQzFDLG1CQVVBO0FBRUQsd0JBQWdCLG9CQUFvQixDQUFDLEVBQ25DLFlBQWtDLEVBQ2xDLFNBQVMsRUFDVCxVQUFVLEVBQ1YsZUFBZSxFQUNmLE1BQU0sRUFBRSxHQUFHLEVBQ1gsTUFBTSxFQUNQLEVBQUU7SUFDRCxZQUFZLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDdEIsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixVQUFVLEVBQUUsTUFBTSxDQUFDO0lBQ25CLGVBQWUsRUFBRSxNQUFNLENBQUM7SUFDeEIsTUFBTSxFQUFFLE1BQU0sQ0FBQztJQUNmLE1BQU0sQ0FBQyxFQUFFLE1BQU0sQ0FBQyxNQUFNLEVBQUUsTUFBTSxHQUFHLE1BQU0sQ0FBQyxDQUFDO0NBQzFDLG1CQVlBO0FBRUQsd0JBQWdCLGtCQUFrQixDQUFDLEVBQ2pDLFlBQStCLEVBQy9CLFNBQVMsRUFDVCxVQUFVLEVBQ1YsTUFBTSxFQUFFLEdBQUcsRUFDWCxNQUFNLEVBQ04sS0FBWSxFQUNiLEVBQUU7SUFDRCxZQUFZLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDdEIsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixVQUFVLEVBQUUsTUFBTSxDQUFDO0lBQ25CLE1BQU0sRUFBRSxNQUFNLENBQUM7SUFDZixNQUFNLENBQUMsRUFBRSxNQUFNLENBQUMsTUFBTSxFQUFFLE1BQU0sR0FBRyxNQUFNLENBQUMsQ0FBQztJQUN6QyxLQUFLLENBQUMsRUFBRSxPQUFPLENBQUM7Q0FDakIsbUJBVUE7QUFFRCx3QkFBZ0IsbUJBQW1CLENBQUMsRUFDbEMsWUFBZ0MsRUFDaEMsVUFBVSxFQUNWLFNBQVMsRUFDVCxVQUFVLEVBQ1YsTUFBTSxFQUFFLEdBQUcsRUFDWixFQUFFO0lBQ0QsWUFBWSxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ3RCLFVBQVUsRUFBRSxNQUFNLENBQUM7SUFDbkIsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixVQUFVLEVBQUUsTUFBTSxDQUFDO0lBQ25CLE1BQU0sRUFBRSxNQUFNLENBQUM7Q0FDaEIsbUJBUUEifQ==

package/dest/spartan/utils/chaos.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chaos.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/chaos.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAUpD,wBAAsB,kBAAkB,CAAC,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,iBAwB5F;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,qBAAqB,CAAC,EAC1C,YAAY,EACZ,eAAe,EACf,UAAU,EACV,YAAY,EACZ,OAAe,EACf,KAAY,EACZ,MAAW,EACX,MAAM,EAAE,GAAG,EACZ,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACzC,MAAM,EAAE,MAAM,CAAC;CAChB,mBAaA;AAED,wBAAgB,kBAAkB,CAAC,EACjC,SAAS,EACT,UAAU,EACV,eAAe,EACf,MAAM,EAAE,GAAG,EACZ,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;CAChB,mBAWA;AAED,wBAAgB,qBAAqB,CAAC,EACpC,SAAS,EACT,UAAU,EACV,MAAM,EAAE,GAAG,EACX,MAAM,EACN,YAAY,EACb,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACzC,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,mBASA;AAED,wBAAgB,eAAe,CAAC,EAC9B,SAAS,EACT,UAAU,EACV,MAAM,EAAE,GAAG,EACX,MAAM,EACP,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;CAC1C,mBAWA;AAED,wBAAgB,qBAAqB,CAAC,EACpC,SAAS,EACT,UAAU,EACV,MAAM,EAAE,GAAG,EACX,MAAM,EACP,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;CAC1C,mBAUA;AAED,wBAAgB,oBAAoB,CAAC,EACnC,YAAkC,EAClC,SAAS,EACT,UAAU,EACV,eAAe,EACf,MAAM,EAAE,GAAG,EACX,MAAM,EACP,EAAE;IACD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;CAC1C,mBAYA;AAED,wBAAgB,kBAAkB,CAAC,EACjC,YAA+B,EAC/B,SAAS,EACT,UAAU,EACV,MAAM,EAAE,GAAG,EACX,MAAM,EACN,KAAY,EACb,EAAE;IACD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACzC,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,mBAUA;AAED,wBAAgB,mBAAmB,CAAC,EAClC,YAAgC,EAChC,UAAU,EACV,SAAS,EACT,UAAU,EACV,MAAM,EAAE,GAAG,EACZ,EAAE;IACD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;CAChB,mBAQA"}