@aztec/end-to-end 0.0.1-commit.21caa21 → 0.0.1-commit.29c6b1a3

package/dest/spartan/tx_metrics.js

@@ -0,0 +1,422 @@
+import { createHistogram } from 'perf_hooks';
+/** Metrics class for proving-related benchmarks. */ export class ProvingMetrics {
+    prefix;
+    successfulTxs;
+    proofDuration;
+    activeAgents;
+    avgQueueTime;
+    jobRetries;
+    jobDuration;
+    timedOutJobs;
+    resolvedJobs;
+    rejectedJobs;
+    epochProvingDuration;
+    provenTransactions;
+    provenBlocks;
+    constructor(prefix){
+        this.prefix = prefix;
+    }
+    recordSuccessfulTxs(count) {
+        this.successfulTxs = count;
+    }
+    recordProofDuration(seconds) {
+        this.proofDuration = seconds;
+    }
+    recordActiveAgents(count) {
+        this.activeAgents = count;
+    }
+    recordAvgQueueTime(ms) {
+        this.avgQueueTime = ms;
+    }
+    recordJobRetries(count) {
+        this.jobRetries = count;
+    }
+    recordJobDuration(ms) {
+        this.jobDuration = ms;
+    }
+    recordTimedOutJobs(count) {
+        this.timedOutJobs = count;
+    }
+    recordResolvedJobs(count) {
+        this.resolvedJobs = count;
+    }
+    recordRejectedJobs(count) {
+        this.rejectedJobs = count;
+    }
+    recordEpochProvingDuration(seconds) {
+        this.epochProvingDuration = seconds;
+    }
+    recordProvenTransactions(count) {
+        this.provenTransactions = count;
+    }
+    recordProvenBlocks(count) {
+        this.provenBlocks = count;
+    }
+    toGithubActionBenchmarkJSON() {
+        const data = [];
+        if (this.successfulTxs !== undefined) {
+            data.push({
+                name: `${this.prefix}/successful_txs`,
+                unit: 'count',
+                value: this.successfulTxs
+            });
+        }
+        if (this.proofDuration !== undefined) {
+            data.push({
+                name: `${this.prefix}/proof_duration`,
+                unit: 's',
+                value: this.proofDuration
+            });
+        }
+        if (this.activeAgents !== undefined) {
+            data.push({
+                name: `${this.prefix}/active_agents`,
+                unit: 'count',
+                value: this.activeAgents
+            });
+        }
+        if (this.avgQueueTime !== undefined) {
+            data.push({
+                name: `${this.prefix}/avg_queue_time`,
+                unit: 'ms',
+                value: this.avgQueueTime
+            });
+        }
+        if (this.jobRetries !== undefined) {
+            data.push({
+                name: `${this.prefix}/job_retries`,
+                unit: 'count',
+                value: this.jobRetries
+            });
+        }
+        if (this.jobDuration !== undefined) {
+            data.push({
+                name: `${this.prefix}/job_duration`,
+                unit: 'ms',
+                value: this.jobDuration
+            });
+        }
+        if (this.timedOutJobs !== undefined) {
+            data.push({
+                name: `${this.prefix}/timed_out_jobs`,
+                unit: 'count',
+                value: this.timedOutJobs
+            });
+        }
+        if (this.resolvedJobs !== undefined) {
+            data.push({
+                name: `${this.prefix}/resolved_jobs`,
+                unit: 'count',
+                value: this.resolvedJobs
+            });
+        }
+        if (this.rejectedJobs !== undefined) {
+            data.push({
+                name: `${this.prefix}/rejected_jobs`,
+                unit: 'count',
+                value: this.rejectedJobs
+            });
+        }
+        if (this.epochProvingDuration !== undefined) {
+            data.push({
+                name: `${this.prefix}/epoch_proving_duration`,
+                unit: 's',
+                value: this.epochProvingDuration
+            });
+        }
+        if (this.provenTransactions !== undefined) {
+            data.push({
+                name: `${this.prefix}/proven_transactions`,
+                unit: 'count',
+                value: this.provenTransactions
+            });
+        }
+        if (this.provenBlocks !== undefined) {
+            data.push({
+                name: `${this.prefix}/proven_blocks`,
+                unit: 'count',
+                value: this.provenBlocks
+            });
+        }
+        const scenario = process.env.BENCH_SCENARIO?.trim();
+        if (!scenario) {
+            return data;
+        }
+        const scenarioPrefix = `scenario/${scenario}/`;
+        return data.map((entry)=>({
+                ...entry,
+                name: `${scenarioPrefix}${entry.name}`
+            }));
+    }
+}
+export class TxInclusionMetrics {
+    aztecNode;
+    logger;
+    data;
+    groups;
+    blocks;
+    p2pGossipLatencyByTopic;
+    attestationLatency;
+    attestationCounts;
+    reqRespStats;
+    peerStats;
+    mempoolMinedDelay;
+    constructor(aztecNode, logger){
+        this.aztecNode = aztecNode;
+        this.logger = logger;
+        this.data = new Map();
+        this.groups = new Set();
+        this.blocks = new Map();
+        this.p2pGossipLatencyByTopic = {};
+    }
+    recordSentTx(tx, group) {
+        const txHash = tx.getTxHash().toString();
+        const priorityFees = tx.getGasSettings().maxPriorityFeesPerGas;
+        if (this.data.has(txHash)) {
+            this.logger?.debug(`Overwriting tx inclusion data for ${txHash}`, {
+                txHash,
+                group
+            });
+        }
+        this.data.set(txHash, {
+            txHash,
+            sentAt: Math.trunc(Date.now() / 1000),
+            minedAt: -1,
+            attestedAt: -1,
+            blocknumber: -1,
+            priorityFee: Number(priorityFees.feePerDaGas + priorityFees.feePerL2Gas),
+            totalFee: -1,
+            positionInBlock: -1,
+            group
+        });
+        this.groups.add(group);
+    }
+    async recordMinedTx(txReceipt) {
+        const { txHash, blockNumber } = txReceipt;
+        if (!txReceipt.isMined() || !txReceipt.hasExecutionSucceeded() || !blockNumber) {
+            this.logger?.debug('Skipping mined tx record due to receipt status', {
+                txHash: txHash.toString(),
+                status: txReceipt.status,
+                blockNumber
+            });
+            return;
+        }
+        if (!this.blocks.has(blockNumber)) {
+            this.blocks.set(blockNumber, this.aztecNode.getBlock(blockNumber));
+        }
+        const block = await this.blocks.get(blockNumber);
+        if (!block) {
+            this.logger?.warn('Failed to load block for mined tx receipt', {
+                txHash: txHash.toString(),
+                blockNumber
+            });
+            return;
+        }
+        const data = this.data.get(txHash.toString());
+        if (!data) {
+            const message = `Missing sent tx record for mined tx ${txHash.toString()}`;
+            this.logger?.warn(message, {
+                txHash: txHash.toString(),
+                blockNumber
+            });
+            throw new Error(message);
+        }
+        data.blocknumber = blockNumber;
+        data.minedAt = Number(block.header.globalVariables.timestamp);
+        data.attestedAt = -1;
+        data.totalFee = Number(txReceipt.transactionFee ?? 0n);
+        data.positionInBlock = block.body.txEffects.findIndex((txEffect)=>txEffect.txHash.equals(txHash));
+    }
+    inclusionTimeInSeconds(group) {
+        const histogram = createHistogram({});
+        for (const tx of this.data.values()){
+            if (!tx.blocknumber || tx.group !== group || tx.minedAt === -1) {
+                continue;
+            }
+            histogram.record(tx.minedAt - tx.sentAt);
+        }
+        if (histogram.count === 0) {
+            return {
+                group,
+                count: 0,
+                mean: 0,
+                max: 0,
+                median: 0,
+                min: 0,
+                p99: 0
+            };
+        }
+        return {
+            group,
+            count: histogram.count,
+            mean: histogram.mean,
+            max: histogram.max,
+            median: histogram.percentile(50),
+            min: histogram.min,
+            p99: histogram.percentile(99)
+        };
+    }
+    recordP2PGossipLatency(topicName, p50, p95) {
+        this.p2pGossipLatencyByTopic[topicName] = {
+            p50,
+            p95
+        };
+    }
+    recordAttestationLatency(p50, p95) {
+        this.attestationLatency = {
+            p50,
+            p95
+        };
+    }
+    recordAttestationCounts(success, failedBad, failedNode) {
+        this.attestationCounts = {
+            success,
+            failedBad,
+            failedNode
+        };
+    }
+    recordReqRespStats(fraction, delayP50, delayP95) {
+        this.reqRespStats = {
+            fraction,
+            delayP50,
+            delayP95
+        };
+    }
+    recordPeerStats(avgCount, connectionDurationP50, connectionDurationP95) {
+        this.peerStats = {
+            avgCount,
+            connectionDurationP50,
+            connectionDurationP95
+        };
+    }
+    recordMempoolMinedDelay(txP50, txP95, attestationP50, attestationP95) {
+        this.mempoolMinedDelay = {
+            txP50,
+            txP95,
+            attestationP50,
+            attestationP95
+        };
+    }
+    toGithubActionBenchmarkJSON() {
+        const data = [];
+        for (const group of this.groups){
+            const stats = this.inclusionTimeInSeconds(group);
+            data.push({
+                name: `${group}/avg_inclusion`,
+                unit: 's',
+                value: stats.mean
+            }, {
+                name: `${group}/median_inclusion`,
+                unit: 's',
+                value: stats.median
+            }, {
+                name: `${group}/p99_inclusion`,
+                unit: 's',
+                value: stats.p99
+            });
+        }
+        for (const [topic, { p50, p95 }] of Object.entries(this.p2pGossipLatencyByTopic)){
+            data.push({
+                name: `p2p_gossip_latency/${topic}/p50`,
+                unit: 'ms',
+                value: p50
+            });
+            data.push({
+                name: `p2p_gossip_latency/${topic}/p95`,
+                unit: 'ms',
+                value: p95
+            });
+        }
+        if (this.attestationLatency) {
+            data.push({
+                name: 'attestation_latency/p50',
+                unit: 'ms',
+                value: this.attestationLatency.p50
+            }, {
+                name: 'attestation_latency/p95',
+                unit: 'ms',
+                value: this.attestationLatency.p95
+            });
+        }
+        if (this.attestationCounts) {
+            const { success, failedBad, failedNode } = this.attestationCounts;
+            const total = success + failedBad + failedNode;
+            const ratio = total > 0 ? success / total : 0;
+            data.push({
+                name: 'attestation/success_count',
+                unit: 'count',
+                value: success
+            }, {
+                name: 'attestation/failed_bad_proposal_count',
+                unit: 'count',
+                value: failedBad
+            }, {
+                name: 'attestation/failed_node_issue_count',
+                unit: 'count',
+                value: failedNode
+            }, {
+                name: 'attestation/success_ratio',
+                unit: 'ratio',
+                value: ratio
+            });
+        }
+        if (this.reqRespStats) {
+            data.push({
+                name: 'req_resp/txs_requested_fraction',
+                unit: 'ratio',
+                value: this.reqRespStats.fraction
+            }, {
+                name: 'req_resp/delay_p50',
+                unit: 'ms',
+                value: this.reqRespStats.delayP50
+            }, {
+                name: 'req_resp/delay_p95',
+                unit: 'ms',
+                value: this.reqRespStats.delayP95
+            });
+        }
+        if (this.peerStats) {
+            data.push({
+                name: 'peers/avg_count',
+                unit: 'peers',
+                value: this.peerStats.avgCount
+            }, {
+                name: 'peers/connection_duration_p50',
+                unit: 'ms',
+                value: this.peerStats.connectionDurationP50
+            }, {
+                name: 'peers/connection_duration_p95',
+                unit: 'ms',
+                value: this.peerStats.connectionDurationP95
+            });
+        }
+        if (this.mempoolMinedDelay) {
+            data.push({
+                name: 'mempool/tx_mined_delay_p50',
+                unit: 'ms',
+                value: this.mempoolMinedDelay.txP50
+            }, {
+                name: 'mempool/tx_mined_delay_p95',
+                unit: 'ms',
+                value: this.mempoolMinedDelay.txP95
+            }, {
+                name: 'mempool/attestation_mined_delay_p50',
+                unit: 'ms',
+                value: this.mempoolMinedDelay.attestationP50
+            }, {
+                name: 'mempool/attestation_mined_delay_p95',
+                unit: 'ms',
+                value: this.mempoolMinedDelay.attestationP95
+            });
+        }
+        const scenario = process.env.BENCH_SCENARIO?.trim();
+        if (!scenario) {
+            return data;
+        }
+        const scenarioPrefix = `scenario/${scenario}/`;
+        return data.map((entry)=>({
+                ...entry,
+                name: `${scenarioPrefix}${entry.name}`
+            }));
+    }
+}

package/dest/spartan/utils/bot.d.ts

@@ -0,0 +1,27 @@
+import type { Logger } from '@aztec/foundation/log';
+export declare function restartBot(namespace: string, log: Logger): Promise<void>;
+/**
+ * Installs or upgrades the transfer bot Helm release for the given namespace.
+ * Intended for test setup to enable L2 traffic generation only when needed.
+ */
+export declare function installTransferBot({ namespace, spartanDir, logger: log, replicas, txIntervalSeconds, followChain, mnemonic, mnemonicStartIndex, botPrivateKey, nodeUrl, timeout, reuseValues, aztecSlotDuration }: {
+    namespace: string;
+    spartanDir: string;
+    logger: Logger;
+    replicas?: number;
+    txIntervalSeconds?: number;
+    followChain?: string;
+    mnemonic?: string;
+    mnemonicStartIndex?: number | string;
+    botPrivateKey?: string;
+    nodeUrl?: string;
+    timeout?: string;
+    reuseValues?: boolean;
+    aztecSlotDuration?: number;
+}): Promise<void>;
+/**
+ * Uninstalls the transfer bot Helm release from the given namespace.
+ * Intended for test teardown to clean up bot resources.
+ */
+export declare function uninstallTransferBot(namespace: string, log: Logger): Promise<void>;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYm90LmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc3BhcnRhbi91dGlscy9ib3QudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUsTUFBTSxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFXcEQsd0JBQXNCLFVBQVUsQ0FBQyxTQUFTLEVBQUUsTUFBTSxFQUFFLEdBQUcsRUFBRSxNQUFNLGlCQWE5RDtBQUVEOzs7R0FHRztBQUNILHdCQUFzQixrQkFBa0IsQ0FBQyxFQUN2QyxTQUFTLEVBQ1QsVUFBVSxFQUNWLE1BQU0sRUFBRSxHQUFHLEVBQ1gsUUFBWSxFQUNaLGlCQUFzQixFQUN0QixXQUF1QixFQUN2QixRQUEyRyxFQUMzRyxrQkFBa0IsRUFDbEIsYUFBc0UsRUFDdEUsT0FBTyxFQUNQLE9BQWUsRUFDZixXQUFrQixFQUNsQixpQkFBaUUsRUFDbEUsRUFBRTtJQUNELFNBQVMsRUFBRSxNQUFNLENBQUM7SUFDbEIsVUFBVSxFQUFFLE1BQU0sQ0FBQztJQUNuQixNQUFNLEVBQUUsTUFBTSxDQUFDO0lBQ2YsUUFBUSxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLGlCQUFpQixDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQzNCLFdBQVcsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUNyQixRQUFRLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDbEIsa0JBQWtCLENBQUMsRUFBRSxNQUFNLEdBQUcsTUFBTSxDQUFDO0lBQ3JDLGFBQWEsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUN2QixPQUFPLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDakIsT0FBTyxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ2pCLFdBQVcsQ0FBQyxFQUFFLE9BQU8sQ0FBQztJQUN0QixpQkFBaUIsQ0FBQyxFQUFFLE1BQU0sQ0FBQztDQUM1QixpQkFnSEE7QUFFRDs7O0dBR0c7QUFDSCx3QkFBc0Isb0JBQW9CLENBQUMsU0FBUyxFQUFFLE1BQU0sRUFBRSxHQUFHLEVBQUUsTUFBTSxpQkFReEUifQ==

package/dest/spartan/utils/bot.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bot.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/bot.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAWpD,wBAAsB,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,iBAa9D;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,EACvC,SAAS,EACT,UAAU,EACV,MAAM,EAAE,GAAG,EACX,QAAY,EACZ,iBAAsB,EACtB,WAAuB,EACvB,QAA2G,EAC3G,kBAAkB,EAClB,aAAsE,EACtE,OAAO,EACP,OAAe,EACf,WAAkB,EAClB,iBAAiE,EAClE,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACrC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,iBAgHA;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,iBAQxE"}

package/dest/spartan/utils/bot.js

@@ -0,0 +1,141 @@
+import { sleep } from '@aztec/foundation/sleep';
+import { exec } from 'child_process';
+import { promisify } from 'util';
+import { execHelmCommand, forceDeleteHelmReleaseRecord, getHelmReleaseStatus, hasDeployedHelmRelease } from './helm.js';
+import { deleteResourceByLabel, getChartDir, waitForResourceByLabel } from './k8s.js';
+const execAsync = promisify(exec);
+export async function restartBot(namespace, log) {
+    log.info(`Restarting bot`);
+    await deleteResourceByLabel({
+        resource: 'pods',
+        namespace,
+        label: 'app.kubernetes.io/name=bot'
+    });
+    await sleep(10 * 1000);
+    // Some bot images may take time to report Ready due to heavy boot-time proving.
+    // Waiting for PodReadyToStartContainers ensures the pod is scheduled and starting without blocking on full readiness.
+    await waitForResourceByLabel({
+        resource: 'pods',
+        namespace,
+        label: 'app.kubernetes.io/name=bot',
+        condition: 'PodReadyToStartContainers'
+    });
+    log.info(`Bot restarted`);
+}
+/**
+ * Installs or upgrades the transfer bot Helm release for the given namespace.
+ * Intended for test setup to enable L2 traffic generation only when needed.
+ */ export async function installTransferBot({ namespace, spartanDir, logger: log, replicas = 1, txIntervalSeconds = 10, followChain = 'PENDING', mnemonic = process.env.LABS_INFRA_MNEMONIC ?? 'test test test test test test test test test test test junk', mnemonicStartIndex, botPrivateKey = process.env.BOT_TRANSFERS_L2_PRIVATE_KEY ?? '0xcafe01', nodeUrl, timeout = '15m', reuseValues = true, aztecSlotDuration = Number(process.env.AZTEC_SLOT_DURATION ?? 12) }) {
+    const instanceName = `${namespace}-bot-transfers`;
+    const helmChartDir = getChartDir(spartanDir, 'aztec-bot');
+    const resolvedNodeUrl = nodeUrl ?? `http://${namespace}-rpc-aztec-node.${namespace}.svc.cluster.local:8080`;
+    log.info(`Installing/upgrading transfer bot: replicas=${replicas}, followChain=${followChain}`);
+    const values = {
+        'bot.replicaCount': replicas,
+        'bot.txIntervalSeconds': txIntervalSeconds,
+        'bot.followChain': followChain,
+        'bot.botPrivateKey': botPrivateKey,
+        'bot.nodeUrl': resolvedNodeUrl,
+        'bot.mnemonic': mnemonic,
+        'bot.feePaymentMethod': 'fee_juice',
+        'aztec.slotDuration': aztecSlotDuration,
+        // Ensure bot can reach its own PXE started in-process (default rpc.port is 8080)
+        // Note: since aztec-bot depends on aztec-node with alias `bot`, env vars go under `bot.node.env`.
+        'bot.node.env.BOT_PXE_URL': 'http://127.0.0.1:8080',
+        // Provide L1 execution RPC for bridging fee juice
+        'bot.node.env.ETHEREUM_HOSTS': `http://${namespace}-eth-execution.${namespace}.svc.cluster.local:8545`,
+        // Provide L1 mnemonic for bridging (falls back to labs mnemonic)
+        'bot.node.env.BOT_L1_MNEMONIC': mnemonic,
+        // The bot does not need Kubernetes API access. Disable RBAC + ServiceAccount creation so the chart
+        // can be installed by users without cluster-scoped RBAC permissions.
+        'bot.rbac.create': false,
+        'bot.serviceAccount.create': false,
+        'bot.serviceAccount.name': 'default'
+    };
+    // Ensure we derive a funded L1 key (index 0 is funded on anvil default mnemonic)
+    if (mnemonicStartIndex === undefined) {
+        values['bot.mnemonicStartIndex'] = 0;
+    }
+    // Also pass a funded private key directly if available
+    if (process.env.FUNDING_PRIVATE_KEY) {
+        values['bot.node.env.BOT_L1_PRIVATE_KEY'] = process.env.FUNDING_PRIVATE_KEY;
+    }
+    // Align bot image with the running network image: prefer env var, else detect from a validator pod
+    let repositoryFromEnv;
+    let tagFromEnv;
+    const aztecDockerImage = process.env.AZTEC_DOCKER_IMAGE;
+    if (aztecDockerImage && aztecDockerImage.includes(':')) {
+        const lastColon = aztecDockerImage.lastIndexOf(':');
+        repositoryFromEnv = aztecDockerImage.slice(0, lastColon);
+        tagFromEnv = aztecDockerImage.slice(lastColon + 1);
+    }
+    let repository = repositoryFromEnv;
+    let tag = tagFromEnv;
+    if (!repository || !tag) {
+        try {
+            const { stdout } = await execAsync(`kubectl get pods -l app.kubernetes.io/name=validator -n ${namespace} -o jsonpath='{.items[0].spec.containers[?(@.name=="aztec")].image}' | cat`);
+            const image = stdout.trim().replace(/^'|'$/g, '');
+            if (image && image.includes(':')) {
+                const lastColon = image.lastIndexOf(':');
+                repository = image.slice(0, lastColon);
+                tag = image.slice(lastColon + 1);
+            }
+        } catch (err) {
+            log.warn(`Could not detect aztec image from validator pod: ${String(err)}`);
+        }
+    }
+    if (repository && tag) {
+        values['global.aztecImage.repository'] = repository;
+        values['global.aztecImage.tag'] = tag;
+    }
+    if (mnemonicStartIndex !== undefined) {
+        values['bot.mnemonicStartIndex'] = typeof mnemonicStartIndex === 'string' ? mnemonicStartIndex : Number(mnemonicStartIndex);
+    }
+    // If a previous install attempt left the release in a non-deployed state (e.g. FAILED),
+    // `helm upgrade --install` can error with "has no deployed releases".
+    // In that case, clear the release record and do a clean install.
+    const existingStatus = await getHelmReleaseStatus(instanceName, namespace);
+    if (existingStatus && existingStatus.toLowerCase() !== 'deployed') {
+        log.warn(`Transfer bot release ${instanceName} is in status '${existingStatus}'. Reinstalling cleanly.`);
+        await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`).catch(()=>undefined);
+        // If helm left the release in `uninstalling`, force-delete the record so we can reinstall.
+        const afterUninstallStatus = await getHelmReleaseStatus(instanceName, namespace);
+        if (afterUninstallStatus?.toLowerCase() === 'uninstalling') {
+            await forceDeleteHelmReleaseRecord(instanceName, namespace, log);
+        }
+    }
+    // `--reuse-values` fails if the release has never successfully deployed (e.g. first install, or a previous failed install).
+    // Only reuse values when we have a deployed release to reuse from.
+    const effectiveReuseValues = reuseValues && await hasDeployedHelmRelease(instanceName, namespace);
+    await execHelmCommand({
+        instanceName,
+        helmChartDir,
+        namespace,
+        valuesFile: undefined,
+        timeout,
+        values: values,
+        reuseValues: effectiveReuseValues
+    });
+    if (replicas > 0) {
+        await waitForResourceByLabel({
+            resource: 'pods',
+            namespace,
+            label: 'app.kubernetes.io/name=bot',
+            condition: 'PodReadyToStartContainers'
+        });
+    }
+}
+/**
+ * Uninstalls the transfer bot Helm release from the given namespace.
+ * Intended for test teardown to clean up bot resources.
+ */ export async function uninstallTransferBot(namespace, log) {
+    const instanceName = `${namespace}-bot-transfers`;
+    log.info(`Uninstalling transfer bot release ${instanceName}`);
+    await execAsync(`helm uninstall ${instanceName} --namespace ${namespace} --wait --ignore-not-found`);
+    // Ensure any leftover pods are removed
+    await deleteResourceByLabel({
+        resource: 'pods',
+        namespace,
+        label: 'app.kubernetes.io/name=bot'
+    }).catch(()=>undefined);
+}

package/dest/spartan/utils/chaos.d.ts

@@ -0,0 +1,79 @@
+import type { Logger } from '@aztec/foundation/log';
+export declare function uninstallChaosMesh(instanceName: string, namespace: string, log: Logger): Promise<void>;
+/**
+ * Installs a Helm chart with the given parameters.
+ * @param instanceName - The name of the Helm chart instance.
+ * @param targetNamespace - The namespace with the resources to be affected by the Helm chart.
+ * @param valuesFile - The values file to use for the Helm chart.
+ * @param chaosMeshNamespace - The namespace to install the Helm chart in.
+ * @param timeout - The timeout for the Helm command.
+ * @param clean - Whether to clean up the Helm chart before installing it.
+ * @returns The stdout of the Helm command.
+ * @throws If the Helm command fails.
+ *
+ * Example usage:
+ * ```typescript
+ * const stdout = await installChaosMeshChart({ instanceName: 'force-reorg', targetNamespace: 'smoke', valuesFile: 'prover-failure.yaml'});
+ * console.log(stdout);
+ * ```
+ */
+export declare function installChaosMeshChart({ instanceName, targetNamespace, valuesFile, helmChartDir, timeout, clean, values, logger: log }: {
+    instanceName: string;
+    targetNamespace: string;
+    valuesFile: string;
+    helmChartDir: string;
+    chaosMeshNamespace?: string;
+    timeout?: string;
+    clean?: boolean;
+    values?: Record<string, string | number>;
+    logger: Logger;
+}): Promise<string>;
+export declare function applyProverFailure({ namespace, spartanDir, durationSeconds, logger: log }: {
+    namespace: string;
+    spartanDir: string;
+    durationSeconds: number;
+    logger: Logger;
+}): Promise<string>;
+export declare function applyValidatorFailure({ namespace, spartanDir, logger: log, values, instanceName }: {
+    namespace: string;
+    spartanDir: string;
+    logger: Logger;
+    values?: Record<string, string | number>;
+    instanceName?: string;
+}): Promise<string>;
+export declare function applyProverKill({ namespace, spartanDir, logger: log, values }: {
+    namespace: string;
+    spartanDir: string;
+    logger: Logger;
+    values?: Record<string, string | number>;
+}): Promise<string>;
+export declare function applyProverBrokerKill({ namespace, spartanDir, logger: log, values }: {
+    namespace: string;
+    spartanDir: string;
+    logger: Logger;
+    values?: Record<string, string | number>;
+}): Promise<string>;
+export declare function applyBootNodeFailure({ instanceName, namespace, spartanDir, durationSeconds, logger: log, values }: {
+    instanceName?: string;
+    namespace: string;
+    spartanDir: string;
+    durationSeconds: number;
+    logger: Logger;
+    values?: Record<string, string | number>;
+}): Promise<string>;
+export declare function applyValidatorKill({ instanceName, namespace, spartanDir, logger: log, values, clean }: {
+    instanceName?: string;
+    namespace: string;
+    spartanDir: string;
+    logger: Logger;
+    values?: Record<string, string | number>;
+    clean?: boolean;
+}): Promise<string>;
+export declare function applyNetworkShaping({ instanceName, valuesFile, namespace, spartanDir, logger: log }: {
+    instanceName?: string;
+    valuesFile: string;
+    namespace: string;
+    spartanDir: string;
+    logger: Logger;
+}): Promise<string>;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY2hhb3MuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zcGFydGFuL3V0aWxzL2NoYW9zLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLHVCQUF1QixDQUFDO0FBVXBELHdCQUFzQixrQkFBa0IsQ0FBQyxZQUFZLEVBQUUsTUFBTSxFQUFFLFNBQVMsRUFBRSxNQUFNLEVBQUUsR0FBRyxFQUFFLE1BQU0saUJBd0I1RjtBQUVEOzs7Ozs7Ozs7Ozs7Ozs7O0dBZ0JHO0FBQ0gsd0JBQXNCLHFCQUFxQixDQUFDLEVBQzFDLFlBQVksRUFDWixlQUFlLEVBQ2YsVUFBVSxFQUNWLFlBQVksRUFDWixPQUFlLEVBQ2YsS0FBWSxFQUNaLE1BQVcsRUFDWCxNQUFNLEVBQUUsR0FBRyxFQUNaLEVBQUU7SUFDRCxZQUFZLEVBQUUsTUFBTSxDQUFDO0lBQ3JCLGVBQWUsRUFBRSxNQUFNLENBQUM7SUFDeEIsVUFBVSxFQUFFLE1BQU0sQ0FBQztJQUNuQixZQUFZLEVBQUUsTUFBTSxDQUFDO0lBQ3JCLGtCQUFrQixDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQzVCLE9BQU8sQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUNqQixLQUFLLENBQUMsRUFBRSxPQUFPLENBQUM7SUFDaEIsTUFBTSxDQUFDLEVBQUUsTUFBTSxDQUFDLE1BQU0sRUFBRSxNQUFNLEdBQUcsTUFBTSxDQUFDLENBQUM7SUFDekMsTUFBTSxFQUFFLE1BQU0sQ0FBQztDQUNoQixtQkFhQTtBQUVELHdCQUFnQixrQkFBa0IsQ0FBQyxFQUNqQyxTQUFTLEVBQ1QsVUFBVSxFQUNWLGVBQWUsRUFDZixNQUFNLEVBQUUsR0FBRyxFQUNaLEVBQUU7SUFDRCxTQUFTLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLFVBQVUsRUFBRSxNQUFNLENBQUM7SUFDbkIsZUFBZSxFQUFFLE1BQU0sQ0FBQztJQUN4QixNQUFNLEVBQUUsTUFBTSxDQUFDO0NBQ2hCLG1CQVdBO0FBRUQsd0JBQWdCLHFCQUFxQixDQUFDLEVBQ3BDLFNBQVMsRUFDVCxVQUFVLEVBQ1YsTUFBTSxFQUFFLEdBQUcsRUFDWCxNQUFNLEVBQ04sWUFBWSxFQUNiLEVBQUU7SUFDRCxTQUFTLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLFVBQVUsRUFBRSxNQUFNLENBQUM7SUFDbkIsTUFBTSxFQUFFLE1BQU0sQ0FBQztJQUNmLE1BQU0sQ0FBQyxFQUFFLE1BQU0sQ0FBQyxNQUFNLEVBQUUsTUFBTSxHQUFHLE1BQU0sQ0FBQyxDQUFDO0lBQ3pDLFlBQVksQ0FBQyxFQUFFLE1BQU0sQ0FBQztDQUN2QixtQkFTQTtBQUVELHdCQUFnQixlQUFlLENBQUMsRUFDOUIsU0FBUyxFQUNULFVBQVUsRUFDVixNQUFNLEVBQUUsR0FBRyxFQUNYLE1BQU0sRUFDUCxFQUFFO0lBQ0QsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixVQUFVLEVBQUUsTUFBTSxDQUFDO0lBQ25CLE1BQU0sRUFBRSxNQUFNLENBQUM7SUFDZixNQUFNLENBQUMsRUFBRSxNQUFNLENBQUMsTUFBTSxFQUFFLE1BQU0sR0FBRyxNQUFNLENBQUMsQ0FBQztDQUMxQyxtQkFXQTtBQUVELHdCQUFnQixxQkFBcUIsQ0FBQyxFQUNwQyxTQUFTLEVBQ1QsVUFBVSxFQUNWLE1BQU0sRUFBRSxHQUFHLEVBQ1gsTUFBTSxFQUNQLEVBQUU7SUFDRCxTQUFTLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLFVBQVUsRUFBRSxNQUFNLENBQUM7SUFDbkIsTUFBTSxFQUFFLE1BQU0sQ0FBQztJQUNmLE1BQU0sQ0FBQyxFQUFFLE1BQU0sQ0FBQyxNQUFNLEVBQUUsTUFBTSxHQUFHLE1BQU0sQ0FBQyxDQUFDO0NBQzFDLG1CQVVBO0FBRUQsd0JBQWdCLG9CQUFvQixDQUFDLEVBQ25DLFlBQWtDLEVBQ2xDLFNBQVMsRUFDVCxVQUFVLEVBQ1YsZUFBZSxFQUNmLE1BQU0sRUFBRSxHQUFHLEVBQ1gsTUFBTSxFQUNQLEVBQUU7SUFDRCxZQUFZLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDdEIsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixVQUFVLEVBQUUsTUFBTSxDQUFDO0lBQ25CLGVBQWUsRUFBRSxNQUFNLENBQUM7SUFDeEIsTUFBTSxFQUFFLE1BQU0sQ0FBQztJQUNmLE1BQU0sQ0FBQyxFQUFFLE1BQU0sQ0FBQyxNQUFNLEVBQUUsTUFBTSxHQUFHLE1BQU0sQ0FBQyxDQUFDO0NBQzFDLG1CQVlBO0FBRUQsd0JBQWdCLGtCQUFrQixDQUFDLEVBQ2pDLFlBQStCLEVBQy9CLFNBQVMsRUFDVCxVQUFVLEVBQ1YsTUFBTSxFQUFFLEdBQUcsRUFDWCxNQUFNLEVBQ04sS0FBWSxFQUNiLEVBQUU7SUFDRCxZQUFZLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDdEIsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixVQUFVLEVBQUUsTUFBTSxDQUFDO0lBQ25CLE1BQU0sRUFBRSxNQUFNLENBQUM7SUFDZixNQUFNLENBQUMsRUFBRSxNQUFNLENBQUMsTUFBTSxFQUFFLE1BQU0sR0FBRyxNQUFNLENBQUMsQ0FBQztJQUN6QyxLQUFLLENBQUMsRUFBRSxPQUFPLENBQUM7Q0FDakIsbUJBVUE7QUFFRCx3QkFBZ0IsbUJBQW1CLENBQUMsRUFDbEMsWUFBZ0MsRUFDaEMsVUFBVSxFQUNWLFNBQVMsRUFDVCxVQUFVLEVBQ1YsTUFBTSxFQUFFLEdBQUcsRUFDWixFQUFFO0lBQ0QsWUFBWSxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ3RCLFVBQVUsRUFBRSxNQUFNLENBQUM7SUFDbkIsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixVQUFVLEVBQUUsTUFBTSxDQUFDO0lBQ25CLE1BQU0sRUFBRSxNQUFNLENBQUM7Q0FDaEIsbUJBUUEifQ==

package/dest/spartan/utils/chaos.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chaos.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/chaos.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAUpD,wBAAsB,kBAAkB,CAAC,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,iBAwB5F;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,qBAAqB,CAAC,EAC1C,YAAY,EACZ,eAAe,EACf,UAAU,EACV,YAAY,EACZ,OAAe,EACf,KAAY,EACZ,MAAW,EACX,MAAM,EAAE,GAAG,EACZ,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACzC,MAAM,EAAE,MAAM,CAAC;CAChB,mBAaA;AAED,wBAAgB,kBAAkB,CAAC,EACjC,SAAS,EACT,UAAU,EACV,eAAe,EACf,MAAM,EAAE,GAAG,EACZ,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;CAChB,mBAWA;AAED,wBAAgB,qBAAqB,CAAC,EACpC,SAAS,EACT,UAAU,EACV,MAAM,EAAE,GAAG,EACX,MAAM,EACN,YAAY,EACb,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACzC,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,mBASA;AAED,wBAAgB,eAAe,CAAC,EAC9B,SAAS,EACT,UAAU,EACV,MAAM,EAAE,GAAG,EACX,MAAM,EACP,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;CAC1C,mBAWA;AAED,wBAAgB,qBAAqB,CAAC,EACpC,SAAS,EACT,UAAU,EACV,MAAM,EAAE,GAAG,EACX,MAAM,EACP,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;CAC1C,mBAUA;AAED,wBAAgB,oBAAoB,CAAC,EACnC,YAAkC,EAClC,SAAS,EACT,UAAU,EACV,eAAe,EACf,MAAM,EAAE,GAAG,EACX,MAAM,EACP,EAAE;IACD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;CAC1C,mBAYA;AAED,wBAAgB,kBAAkB,CAAC,EACjC,YAA+B,EAC/B,SAAS,EACT,UAAU,EACV,MAAM,EAAE,GAAG,EACX,MAAM,EACN,KAAY,EACb,EAAE;IACD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACzC,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,mBAUA;AAED,wBAAgB,mBAAmB,CAAC,EAClC,YAAgC,EAChC,UAAU,EACV,SAAS,EACT,UAAU,EACV,MAAM,EAAE,GAAG,EACZ,EAAE;IACD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;CAChB,mBAQA"}