@aztec/end-to-end 0.0.1-commit.1bea0213 → 0.0.1-commit.21ecf947b

package/dest/spartan/tx_metrics.js

@@ -1,4 +1,154 @@
 import { createHistogram } from 'perf_hooks';
+/** Metrics class for proving-related benchmarks. */ export class ProvingMetrics {
+    prefix;
+    successfulTxs;
+    proofDuration;
+    activeAgents;
+    avgQueueTime;
+    jobRetries;
+    jobDuration;
+    timedOutJobs;
+    resolvedJobs;
+    rejectedJobs;
+    epochProvingDuration;
+    provenTransactions;
+    provenBlocks;
+    constructor(prefix){
+        this.prefix = prefix;
+    }
+    recordSuccessfulTxs(count) {
+        this.successfulTxs = count;
+    }
+    recordProofDuration(seconds) {
+        this.proofDuration = seconds;
+    }
+    recordActiveAgents(count) {
+        this.activeAgents = count;
+    }
+    recordAvgQueueTime(ms) {
+        this.avgQueueTime = ms;
+    }
+    recordJobRetries(count) {
+        this.jobRetries = count;
+    }
+    recordJobDuration(ms) {
+        this.jobDuration = ms;
+    }
+    recordTimedOutJobs(count) {
+        this.timedOutJobs = count;
+    }
+    recordResolvedJobs(count) {
+        this.resolvedJobs = count;
+    }
+    recordRejectedJobs(count) {
+        this.rejectedJobs = count;
+    }
+    recordEpochProvingDuration(seconds) {
+        this.epochProvingDuration = seconds;
+    }
+    recordProvenTransactions(count) {
+        this.provenTransactions = count;
+    }
+    recordProvenBlocks(count) {
+        this.provenBlocks = count;
+    }
+    toGithubActionBenchmarkJSON() {
+        const data = [];
+        if (this.successfulTxs !== undefined) {
+            data.push({
+                name: `${this.prefix}/successful_txs`,
+                unit: 'count',
+                value: this.successfulTxs
+            });
+        }
+        if (this.proofDuration !== undefined) {
+            data.push({
+                name: `${this.prefix}/proof_duration`,
+                unit: 's',
+                value: this.proofDuration
+            });
+        }
+        if (this.activeAgents !== undefined) {
+            data.push({
+                name: `${this.prefix}/active_agents`,
+                unit: 'count',
+                value: this.activeAgents
+            });
+        }
+        if (this.avgQueueTime !== undefined) {
+            data.push({
+                name: `${this.prefix}/avg_queue_time`,
+                unit: 'ms',
+                value: this.avgQueueTime
+            });
+        }
+        if (this.jobRetries !== undefined) {
+            data.push({
+                name: `${this.prefix}/job_retries`,
+                unit: 'count',
+                value: this.jobRetries
+            });
+        }
+        if (this.jobDuration !== undefined) {
+            data.push({
+                name: `${this.prefix}/job_duration`,
+                unit: 'ms',
+                value: this.jobDuration
+            });
+        }
+        if (this.timedOutJobs !== undefined) {
+            data.push({
+                name: `${this.prefix}/timed_out_jobs`,
+                unit: 'count',
+                value: this.timedOutJobs
+            });
+        }
+        if (this.resolvedJobs !== undefined) {
+            data.push({
+                name: `${this.prefix}/resolved_jobs`,
+                unit: 'count',
+                value: this.resolvedJobs
+            });
+        }
+        if (this.rejectedJobs !== undefined) {
+            data.push({
+                name: `${this.prefix}/rejected_jobs`,
+                unit: 'count',
+                value: this.rejectedJobs
+            });
+        }
+        if (this.epochProvingDuration !== undefined) {
+            data.push({
+                name: `${this.prefix}/epoch_proving_duration`,
+                unit: 's',
+                value: this.epochProvingDuration
+            });
+        }
+        if (this.provenTransactions !== undefined) {
+            data.push({
+                name: `${this.prefix}/proven_transactions`,
+                unit: 'count',
+                value: this.provenTransactions
+            });
+        }
+        if (this.provenBlocks !== undefined) {
+            data.push({
+                name: `${this.prefix}/proven_blocks`,
+                unit: 'count',
+                value: this.provenBlocks
+            });
+        }
+        const scenario = process.env.BENCH_SCENARIO?.trim();
+        if (!scenario) {
+            return data;
+        }
+        const scenarioPrefix = `scenario/${scenario}/`;
+        return data.map((entry)=>({
+                ...entry,
+                name: `${scenarioPrefix}${entry.name}`
+            }));
+    }
+}
 export class TxInclusionMetrics {
     aztecNode;
     logger;

package/dest/spartan/utils/bot.d.ts

@@ -24,4 +24,4 @@ export declare function installTransferBot({ namespace, spartanDir, logger: log,
  * Intended for test teardown to clean up bot resources.
  */
 export declare function uninstallTransferBot(namespace: string, log: Logger): Promise<void>;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYm90LmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc3BhcnRhbi91dGlscy9ib3QudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUsTUFBTSxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFXcEQsd0JBQXNCLFVBQVUsQ0FBQyxTQUFTLEVBQUUsTUFBTSxFQUFFLEdBQUcsRUFBRSxNQUFNLGlCQWE5RDtBQUVEOzs7R0FHRztBQUNILHdCQUFzQixrQkFBa0IsQ0FBQyxFQUN2QyxTQUFTLEVBQ1QsVUFBVSxFQUNWLE1BQU0sRUFBRSxHQUFHLEVBQ1gsUUFBWSxFQUNaLGlCQUFzQixFQUN0QixXQUF1QixFQUN2QixRQUEyRyxFQUMzRyxrQkFBa0IsRUFDbEIsYUFBc0UsRUFDdEUsT0FBTyxFQUNQLE9BQWUsRUFDZixXQUFrQixFQUNsQixpQkFBaUUsRUFDbEUsRUFBRTtJQUNELFNBQVMsRUFBRSxNQUFNLENBQUM7SUFDbEIsVUFBVSxFQUFFLE1BQU0sQ0FBQztJQUNuQixNQUFNLEVBQUUsTUFBTSxDQUFDO0lBQ2YsUUFBUSxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLGlCQUFpQixDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQzNCLFdBQVcsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUNyQixRQUFRLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDbEIsa0JBQWtCLENBQUMsRUFBRSxNQUFNLEdBQUcsTUFBTSxDQUFDO0lBQ3JDLGFBQWEsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUN2QixPQUFPLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDakIsT0FBTyxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ2pCLFdBQVcsQ0FBQyxFQUFFLE9BQU8sQ0FBQztJQUN0QixpQkFBaUIsQ0FBQyxFQUFFLE1BQU0sQ0FBQztDQUM1QixpQkFnSEE7QUFFRDs7O0dBR0c7QUFDSCx3QkFBc0Isb0JBQW9CLENBQUMsU0FBUyxFQUFFLE1BQU0sRUFBRSxHQUFHLEVBQUUsTUFBTSxpQkFReEUifQ==
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYm90LmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc3BhcnRhbi91dGlscy9ib3QudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUsTUFBTSxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFXcEQsd0JBQXNCLFVBQVUsQ0FBQyxTQUFTLEVBQUUsTUFBTSxFQUFFLEdBQUcsRUFBRSxNQUFNLGlCQWE5RDtBQUVEOzs7R0FHRztBQUNILHdCQUFzQixrQkFBa0IsQ0FBQyxFQUN2QyxTQUFTLEVBQ1QsVUFBVSxFQUNWLE1BQU0sRUFBRSxHQUFHLEVBQ1gsUUFBWSxFQUNaLGlCQUFzQixFQUN0QixXQUE0QixFQUM1QixRQUEyRyxFQUMzRyxrQkFBa0IsRUFDbEIsYUFBc0UsRUFDdEUsT0FBTyxFQUNQLE9BQWUsRUFDZixXQUFrQixFQUNsQixpQkFBaUUsRUFDbEUsRUFBRTtJQUNELFNBQVMsRUFBRSxNQUFNLENBQUM7SUFDbEIsVUFBVSxFQUFFLE1BQU0sQ0FBQztJQUNuQixNQUFNLEVBQUUsTUFBTSxDQUFDO0lBQ2YsUUFBUSxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLGlCQUFpQixDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQzNCLFdBQVcsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUNyQixRQUFRLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDbEIsa0JBQWtCLENBQUMsRUFBRSxNQUFNLEdBQUcsTUFBTSxDQUFDO0lBQ3JDLGFBQWEsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUN2QixPQUFPLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDakIsT0FBTyxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ2pCLFdBQVcsQ0FBQyxFQUFFLE9BQU8sQ0FBQztJQUN0QixpQkFBaUIsQ0FBQyxFQUFFLE1BQU0sQ0FBQztDQUM1QixpQkFnSEE7QUFFRDs7O0dBR0c7QUFDSCx3QkFBc0Isb0JBQW9CLENBQUMsU0FBUyxFQUFFLE1BQU0sRUFBRSxHQUFHLEVBQUUsTUFBTSxpQkFReEUifQ==

package/dest/spartan/utils/bot.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bot.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/bot.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAWpD,wBAAsB,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,iBAa9D;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,EACvC,SAAS,EACT,UAAU,EACV,MAAM,EAAE,GAAG,EACX,QAAY,EACZ,iBAAsB,EACtB,WAAuB,EACvB,QAA2G,EAC3G,kBAAkB,EAClB,aAAsE,EACtE,OAAO,EACP,OAAe,EACf,WAAkB,EAClB,iBAAiE,EAClE,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACrC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,iBAgHA;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,iBAQxE"}
+{"version":3,"file":"bot.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/bot.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAWpD,wBAAsB,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,iBAa9D;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,EACvC,SAAS,EACT,UAAU,EACV,MAAM,EAAE,GAAG,EACX,QAAY,EACZ,iBAAsB,EACtB,WAA4B,EAC5B,QAA2G,EAC3G,kBAAkB,EAClB,aAAsE,EACtE,OAAO,EACP,OAAe,EACf,WAAkB,EAClB,iBAAiE,EAClE,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACrC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,iBAgHA;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,iBAQxE"}

package/dest/spartan/utils/bot.js

@@ -25,7 +25,7 @@ export async function restartBot(namespace, log) {
 /**
  * Installs or upgrades the transfer bot Helm release for the given namespace.
  * Intended for test setup to enable L2 traffic generation only when needed.
- */ export async function installTransferBot({ namespace, spartanDir, logger: log, replicas = 1, txIntervalSeconds = 10, followChain = 'PENDING', mnemonic = process.env.LABS_INFRA_MNEMONIC ?? 'test test test test test test test test test test test junk', mnemonicStartIndex, botPrivateKey = process.env.BOT_TRANSFERS_L2_PRIVATE_KEY ?? '0xcafe01', nodeUrl, timeout = '15m', reuseValues = true, aztecSlotDuration = Number(process.env.AZTEC_SLOT_DURATION ?? 12) }) {
+ */ export async function installTransferBot({ namespace, spartanDir, logger: log, replicas = 1, txIntervalSeconds = 10, followChain = 'CHECKPOINTED', mnemonic = process.env.LABS_INFRA_MNEMONIC ?? 'test test test test test test test test test test test junk', mnemonicStartIndex, botPrivateKey = process.env.BOT_TRANSFERS_L2_PRIVATE_KEY ?? '0xcafe01', nodeUrl, timeout = '15m', reuseValues = true, aztecSlotDuration = Number(process.env.AZTEC_SLOT_DURATION ?? 12) }) {
     const instanceName = `${namespace}-bot-transfers`;
     const helmChartDir = getChartDir(spartanDir, 'aztec-bot');
     const resolvedNodeUrl = nodeUrl ?? `http://${namespace}-rpc-aztec-node.${namespace}.svc.cluster.local:8080`;

package/dest/spartan/utils/config.d.ts

@@ -9,6 +9,7 @@ declare const testConfigSchema: z.ZodObject<{
     AZTEC_EPOCH_DURATION: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
     AZTEC_PROOF_SUBMISSION_WINDOW: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
     AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+    FUNDING_PRIVATE_KEY: z.ZodOptional<z.ZodString>;
 }, "strip", z.ZodTypeAny, {
     NAMESPACE: string;
     REAL_VERIFIER: boolean;
@@ -19,6 +20,7 @@ declare const testConfigSchema: z.ZodObject<{
     AZTEC_EPOCH_DURATION: number;
     AZTEC_PROOF_SUBMISSION_WINDOW: number;
     AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET: number;
+    FUNDING_PRIVATE_KEY?: string | undefined;
 }, {
     NAMESPACE?: string | undefined;
     REAL_VERIFIER?: string | number | boolean | undefined;
@@ -29,8 +31,9 @@ declare const testConfigSchema: z.ZodObject<{
     AZTEC_EPOCH_DURATION?: number | undefined;
     AZTEC_PROOF_SUBMISSION_WINDOW?: number | undefined;
     AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET?: number | undefined;
+    FUNDING_PRIVATE_KEY?: string | undefined;
 }>;
 export type TestConfig = z.infer<typeof testConfigSchema>;
 export declare function setupEnvironment(env: unknown): TestConfig;
 export {};
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29uZmlnLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc3BhcnRhbi91dGlscy9jb25maWcudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBR0EsT0FBTyxFQUFFLENBQUMsRUFBRSxNQUFNLEtBQUssQ0FBQztBQUl4QixRQUFBLE1BQU0sZ0JBQWdCOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7RUFVcEIsQ0FBQztBQUVILE1BQU0sTUFBTSxVQUFVLEdBQUcsQ0FBQyxDQUFDLEtBQUssQ0FBQyxPQUFPLGdCQUFnQixDQUFDLENBQUM7QUFFMUQsd0JBQWdCLGdCQUFnQixDQUFDLEdBQUcsRUFBRSxPQUFPLEdBQUcsVUFBVSxDQUl6RCJ9
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29uZmlnLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc3BhcnRhbi91dGlscy9jb25maWcudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBR0EsT0FBTyxFQUFFLENBQUMsRUFBRSxNQUFNLEtBQUssQ0FBQztBQUl4QixRQUFBLE1BQU0sZ0JBQWdCOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7RUFXcEIsQ0FBQztBQUVILE1BQU0sTUFBTSxVQUFVLEdBQUcsQ0FBQyxDQUFDLEtBQUssQ0FBQyxPQUFPLGdCQUFnQixDQUFDLENBQUM7QUFFMUQsd0JBQWdCLGdCQUFnQixDQUFDLEdBQUcsRUFBRSxPQUFPLEdBQUcsVUFBVSxDQUl6RCJ9

package/dest/spartan/utils/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/config.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,QAAA,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAUpB,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,UAAU,CAIzD"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/config.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,QAAA,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAWpB,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,UAAU,CAIzD"}

package/dest/spartan/utils/config.js

@@ -11,7 +11,8 @@ const testConfigSchema = z.object({
     AZTEC_SLOT_DURATION: z.coerce.number().optional().default(24),
     AZTEC_EPOCH_DURATION: z.coerce.number().optional().default(32),
     AZTEC_PROOF_SUBMISSION_WINDOW: z.coerce.number().optional().default(5),
-    AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET: z.coerce.number().optional().default(2)
+    AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET: z.coerce.number().optional().default(2),
+    FUNDING_PRIVATE_KEY: z.string().optional()
 });
 export function setupEnvironment(env) {
     const config = testConfigSchema.parse(env);

package/dest/spartan/utils/index.d.ts

@@ -1,9 +1,9 @@
 export { type TestConfig, setupEnvironment } from './config.js';
-export { getGitProjectRoot, getAztecBin, runAztecBin, runProjectScript } from './scripts.js';
-export { startPortForward, getExternalIP, startPortForwardForPrometeheus, startPortForwardForRPC, startPortForwardForEthereum, deleteResourceByName, deleteResourceByLabel, waitForResourceByLabel, waitForResourceByName, waitForResourcesByName, getChartDir, type ServiceEndpoint, getServiceEndpoint, getRPCEndpoint, getEthereumEndpoint, } from './k8s.js';
+export { type ScriptResult, getGitProjectRoot, getAztecBin, runAztecBin, runProjectScript } from './scripts.js';
+export { startPortForward, getExternalIP, startPortForwardForPrometeheus, startPortForwardForRPC, startPortForwardForEthereum, deleteResourceByName, deleteResourceByLabel, waitForResourceByLabel, waitForResourceByName, waitForResourcesByName, getChartDir, type ServiceEndpoint, getServiceEndpoint, getRPCEndpoint, getEthereumEndpoint, createResilientPrometheusConnection, } from './k8s.js';
 export { uninstallChaosMesh, installChaosMeshChart, applyProverFailure, applyValidatorFailure, applyProverKill, applyProverBrokerKill, applyBootNodeFailure, applyValidatorKill, applyNetworkShaping, } from './chaos.js';
 export { restartBot, installTransferBot, uninstallTransferBot } from './bot.js';
-export { awaitCheckpointNumber, getSequencers, updateSequencersConfig, getSequencersConfig, withSequencersAdmin, setValidatorTxDrop, restartValidators, enableValidatorDynamicBootNode, rollAztecPods, } from './nodes.js';
+export { awaitCheckpointNumber, waitForProvenToAdvance, getSequencers, updateSequencersConfig, getSequencersConfig, withSequencersAdmin, setValidatorTxDrop, restartValidators, enableValidatorDynamicBootNode, rollAztecPods, } from './nodes.js';
 export { getPublicViemClient, getL1DeploymentAddresses, getNodeClient } from './clients.js';
 export { ChainHealth, type ChainHealthSnapshot } from './health.js';
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zcGFydGFuL3V0aWxzL2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUlBLE9BQU8sRUFBRSxLQUFLLFVBQVUsRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLGFBQWEsQ0FBQztBQUdoRSxPQUFPLEVBQUUsaUJBQWlCLEVBQUUsV0FBVyxFQUFFLFdBQVcsRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLGNBQWMsQ0FBQztBQUc3RixPQUFPLEVBQ0wsZ0JBQWdCLEVBQ2hCLGFBQWEsRUFDYiw4QkFBOEIsRUFDOUIsc0JBQXNCLEVBQ3RCLDJCQUEyQixFQUMzQixvQkFBb0IsRUFDcEIscUJBQXFCLEVBQ3JCLHNCQUFzQixFQUN0QixxQkFBcUIsRUFDckIsc0JBQXNCLEVBQ3RCLFdBQVcsRUFDWCxLQUFLLGVBQWUsRUFDcEIsa0JBQWtCLEVBQ2xCLGNBQWMsRUFDZCxtQkFBbUIsR0FDcEIsTUFBTSxVQUFVLENBQUM7QUFHbEIsT0FBTyxFQUNMLGtCQUFrQixFQUNsQixxQkFBcUIsRUFDckIsa0JBQWtCLEVBQ2xCLHFCQUFxQixFQUNyQixlQUFlLEVBQ2YscUJBQXFCLEVBQ3JCLG9CQUFvQixFQUNwQixrQkFBa0IsRUFDbEIsbUJBQW1CLEdBQ3BCLE1BQU0sWUFBWSxDQUFDO0FBR3BCLE9BQU8sRUFBRSxVQUFVLEVBQUUsa0JBQWtCLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSxVQUFVLENBQUM7QUFHaEYsT0FBTyxFQUNMLHFCQUFxQixFQUNyQixhQUFhLEVBQ2Isc0JBQXNCLEVBQ3RCLG1CQUFtQixFQUNuQixtQkFBbUIsRUFDbkIsa0JBQWtCLEVBQ2xCLGlCQUFpQixFQUNqQiw4QkFBOEIsRUFDOUIsYUFBYSxHQUNkLE1BQU0sWUFBWSxDQUFDO0FBR3BCLE9BQU8sRUFBRSxtQkFBbUIsRUFBRSx3QkFBd0IsRUFBRSxhQUFhLEVBQUUsTUFBTSxjQUFjLENBQUM7QUFHNUYsT0FBTyxFQUFFLFdBQVcsRUFBRSxLQUFLLG1CQUFtQixFQUFFLE1BQU0sYUFBYSxDQUFDIn0=
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zcGFydGFuL3V0aWxzL2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUlBLE9BQU8sRUFBRSxLQUFLLFVBQVUsRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLGFBQWEsQ0FBQztBQUdoRSxPQUFPLEVBQUUsS0FBSyxZQUFZLEVBQUUsaUJBQWlCLEVBQUUsV0FBVyxFQUFFLFdBQVcsRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLGNBQWMsQ0FBQztBQUdoSCxPQUFPLEVBQ0wsZ0JBQWdCLEVBQ2hCLGFBQWEsRUFDYiw4QkFBOEIsRUFDOUIsc0JBQXNCLEVBQ3RCLDJCQUEyQixFQUMzQixvQkFBb0IsRUFDcEIscUJBQXFCLEVBQ3JCLHNCQUFzQixFQUN0QixxQkFBcUIsRUFDckIsc0JBQXNCLEVBQ3RCLFdBQVcsRUFDWCxLQUFLLGVBQWUsRUFDcEIsa0JBQWtCLEVBQ2xCLGNBQWMsRUFDZCxtQkFBbUIsRUFDbkIsbUNBQW1DLEdBQ3BDLE1BQU0sVUFBVSxDQUFDO0FBR2xCLE9BQU8sRUFDTCxrQkFBa0IsRUFDbEIscUJBQXFCLEVBQ3JCLGtCQUFrQixFQUNsQixxQkFBcUIsRUFDckIsZUFBZSxFQUNmLHFCQUFxQixFQUNyQixvQkFBb0IsRUFDcEIsa0JBQWtCLEVBQ2xCLG1CQUFtQixHQUNwQixNQUFNLFlBQVksQ0FBQztBQUdwQixPQUFPLEVBQUUsVUFBVSxFQUFFLGtCQUFrQixFQUFFLG9CQUFvQixFQUFFLE1BQU0sVUFBVSxDQUFDO0FBR2hGLE9BQU8sRUFDTCxxQkFBcUIsRUFDckIsc0JBQXNCLEVBQ3RCLGFBQWEsRUFDYixzQkFBc0IsRUFDdEIsbUJBQW1CLEVBQ25CLG1CQUFtQixFQUNuQixrQkFBa0IsRUFDbEIsaUJBQWlCLEVBQ2pCLDhCQUE4QixFQUM5QixhQUFhLEdBQ2QsTUFBTSxZQUFZLENBQUM7QUFHcEIsT0FBTyxFQUFFLG1CQUFtQixFQUFFLHdCQUF3QixFQUFFLGFBQWEsRUFBRSxNQUFNLGNBQWMsQ0FBQztBQUc1RixPQUFPLEVBQUUsV0FBVyxFQUFFLEtBQUssbUJBQW1CLEVBQUUsTUFBTSxhQUFhLENBQUMifQ==

package/dest/spartan/utils/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,UAAU,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAGhE,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAG7F,OAAO,EACL,gBAAgB,EAChB,aAAa,EACb,8BAA8B,EAC9B,sBAAsB,EACtB,2BAA2B,EAC3B,oBAAoB,EACpB,qBAAqB,EACrB,sBAAsB,EACtB,qBAAqB,EACrB,sBAAsB,EACtB,WAAW,EACX,KAAK,eAAe,EACpB,kBAAkB,EAClB,cAAc,EACd,mBAAmB,GACpB,MAAM,UAAU,CAAC;AAGlB,OAAO,EACL,kBAAkB,EAClB,qBAAqB,EACrB,kBAAkB,EAClB,qBAAqB,EACrB,eAAe,EACf,qBAAqB,EACrB,oBAAoB,EACpB,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAGhF,OAAO,EACL,qBAAqB,EACrB,aAAa,EACb,sBAAsB,EACtB,mBAAmB,EACnB,mBAAmB,EACnB,kBAAkB,EAClB,iBAAiB,EACjB,8BAA8B,EAC9B,aAAa,GACd,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAG5F,OAAO,EAAE,WAAW,EAAE,KAAK,mBAAmB,EAAE,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,UAAU,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAGhE,OAAO,EAAE,KAAK,YAAY,EAAE,iBAAiB,EAAE,WAAW,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAGhH,OAAO,EACL,gBAAgB,EAChB,aAAa,EACb,8BAA8B,EAC9B,sBAAsB,EACtB,2BAA2B,EAC3B,oBAAoB,EACpB,qBAAqB,EACrB,sBAAsB,EACtB,qBAAqB,EACrB,sBAAsB,EACtB,WAAW,EACX,KAAK,eAAe,EACpB,kBAAkB,EAClB,cAAc,EACd,mBAAmB,EACnB,mCAAmC,GACpC,MAAM,UAAU,CAAC;AAGlB,OAAO,EACL,kBAAkB,EAClB,qBAAqB,EACrB,kBAAkB,EAClB,qBAAqB,EACrB,eAAe,EACf,qBAAqB,EACrB,oBAAoB,EACpB,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAGhF,OAAO,EACL,qBAAqB,EACrB,sBAAsB,EACtB,aAAa,EACb,sBAAsB,EACtB,mBAAmB,EACnB,mBAAmB,EACnB,kBAAkB,EAClB,iBAAiB,EACjB,8BAA8B,EAC9B,aAAa,GACd,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAG5F,OAAO,EAAE,WAAW,EAAE,KAAK,mBAAmB,EAAE,MAAM,aAAa,CAAC"}

package/dest/spartan/utils/index.js

@@ -5,13 +5,13 @@ export { setupEnvironment } from './config.js';
 // Scripts
 export { getGitProjectRoot, getAztecBin, runAztecBin, runProjectScript } from './scripts.js';
 // K8s operations
-export { startPortForward, getExternalIP, startPortForwardForPrometeheus, startPortForwardForRPC, startPortForwardForEthereum, deleteResourceByName, deleteResourceByLabel, waitForResourceByLabel, waitForResourceByName, waitForResourcesByName, getChartDir, getServiceEndpoint, getRPCEndpoint, getEthereumEndpoint } from './k8s.js';
+export { startPortForward, getExternalIP, startPortForwardForPrometeheus, startPortForwardForRPC, startPortForwardForEthereum, deleteResourceByName, deleteResourceByLabel, waitForResourceByLabel, waitForResourceByName, waitForResourcesByName, getChartDir, getServiceEndpoint, getRPCEndpoint, getEthereumEndpoint, createResilientPrometheusConnection } from './k8s.js';
 // Chaos Mesh
 export { uninstallChaosMesh, installChaosMeshChart, applyProverFailure, applyValidatorFailure, applyProverKill, applyProverBrokerKill, applyBootNodeFailure, applyValidatorKill, applyNetworkShaping } from './chaos.js';
 // Bot management
 export { restartBot, installTransferBot, uninstallTransferBot } from './bot.js';
 // Node operations (sequencers, validators, pods)
-export { awaitCheckpointNumber, getSequencers, updateSequencersConfig, getSequencersConfig, withSequencersAdmin, setValidatorTxDrop, restartValidators, enableValidatorDynamicBootNode, rollAztecPods } from './nodes.js';
+export { awaitCheckpointNumber, waitForProvenToAdvance, getSequencers, updateSequencersConfig, getSequencersConfig, withSequencersAdmin, setValidatorTxDrop, restartValidators, enableValidatorDynamicBootNode, rollAztecPods } from './nodes.js';
 // Client utilities
 export { getPublicViemClient, getL1DeploymentAddresses, getNodeClient } from './clients.js';
 // Health checks

package/dest/spartan/utils/k8s.d.ts

@@ -1,4 +1,6 @@
+import type { Logger } from '@aztec/foundation/log';
 import { type ChildProcess } from 'child_process';
+import { GrafanaClient } from '../../quality_of_service/grafana_client.js';
 /**
  * Represents an endpoint to reach a K8s service.
  * May be a LoadBalancer external IP or a port-forward.
@@ -94,5 +96,31 @@ export declare function waitForResourcesByName({ resource, names, namespace, con
     condition?: string;
     timeout?: string;
 }): Promise<void>;
+/**
+ * Waits for all StatefulSets matching a label to have all their replicas ready.
+ *
+ * @param namespace - Kubernetes namespace
+ * @param label - Label selector for StatefulSets (e.g., "app.kubernetes.io/component=sequencer-node")
+ * @param timeoutSeconds - Maximum time to wait in seconds
+ * @param pollIntervalSeconds - How often to check status
+ */
+export declare function waitForStatefulSetsReady({ namespace, label, timeoutSeconds, pollIntervalSeconds }: {
+    namespace: string;
+    label: string;
+    timeoutSeconds?: number;
+    pollIntervalSeconds?: number;
+}): Promise<void>;
+/**
+ * Creates a Prometheus connection that can re-establish port-forward on failure.
+ * Returns functions to connect and run alert checks that automatically reconnect if needed.
+ *
+ * @param namespace - K8s namespace to fall back to if metrics namespace doesn't have Prometheus
+ * @param endpoints - Array to track created endpoints for cleanup
+ * @param log - Logger instance
+ */
+export declare function createResilientPrometheusConnection(namespace: string, endpoints: ServiceEndpoint[], log: Logger): {
+    connect: () => Promise<GrafanaClient>;
+    runAlertCheck: (alerts: Parameters<GrafanaClient['runAlertCheck']>[0]) => Promise<void>;
+};
 export declare function getChartDir(spartanDir: string, chartName: string): string;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiazhzLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc3BhcnRhbi91dGlscy9rOHMudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBSUEsT0FBTyxFQUFFLEtBQUssWUFBWSxFQUFlLE1BQU0sZUFBZSxDQUFDO0FBUS9EOzs7R0FHRztBQUNILE1BQU0sV0FBVyxlQUFlO0lBQzlCLEdBQUcsRUFBRSxNQUFNLENBQUM7SUFDWixPQUFPLENBQUMsRUFBRSxZQUFZLENBQUM7Q0FDeEI7QUFFRCx3QkFBc0IsZ0JBQWdCLENBQUMsRUFDckMsUUFBUSxFQUNSLFNBQVMsRUFDVCxhQUFhLEVBQ2IsUUFBUSxFQUNULEVBQUU7SUFDRCxRQUFRLEVBQUUsTUFBTSxDQUFDO0lBQ2pCLFNBQVMsRUFBRSxNQUFNLENBQUM7SUFDbEIsYUFBYSxFQUFFLE1BQU0sQ0FBQztJQUV0QixRQUFRLENBQUMsRUFBRSxNQUFNLENBQUM7Q0FDbkIsR0FBRyxPQUFPLENBQUM7SUFDVixPQUFPLEVBQUUsWUFBWSxDQUFDO0lBQ3RCLElBQUksRUFBRSxNQUFNLENBQUM7Q0FDZCxDQUFDLENBdUVEO0FBRUQsd0JBQWdCLGFBQWEsQ0FBQyxTQUFTLEVBQUUsTUFBTSxFQUFFLFdBQVcsRUFBRSxNQUFNLEdBQUcsT0FBTyxDQUFDLE1BQU0sQ0FBQyxDQStCckY7QUFFRDs7Ozs7Ozs7R0FRRztBQUNILHdCQUFzQixrQkFBa0IsQ0FBQyxJQUFJLEVBQUU7SUFDN0MsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixXQUFXLEVBQUUsTUFBTSxDQUFDO0lBQ3BCLGFBQWEsRUFBRSxNQUFNLENBQUM7SUFDdEIsZ0JBQWdCLENBQUMsRUFBRSxPQUFPLENBQUM7Q0FDNUIsR0FBRyxPQUFPLENBQUMsZUFBZSxDQUFDLENBc0MzQjtBQUVEOzs7Ozs7R0FNRztBQUNILHdCQUFzQixjQUFjLENBQUMsU0FBUyxFQUFFLE1BQU0sRUFBRSxnQkFBZ0IsQ0FBQyxFQUFFLE9BQU8sR0FBRyxPQUFPLENBQUMsZUFBZSxDQUFDLENBTzVHO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsd0JBQXNCLG1CQUFtQixDQUFDLFNBQVMsRUFBRSxNQUFNLEVBQUUsZ0JBQWdCLENBQUMsRUFBRSxPQUFPLEdBQUcsT0FBTyxDQUFDLGVBQWUsQ0FBQyxDQU9qSDtBQUVELHdCQUFnQiw4QkFBOEIsQ0FBQyxTQUFTLEVBQUUsTUFBTTs7O0dBTS9EO0FBRUQsd0JBQWdCLHNCQUFzQixDQUFDLFNBQVMsRUFBRSxNQUFNLEVBQUUsS0FBSyxTQUFJOzs7R0FNbEU7QUFFRCx3QkFBZ0IsMkJBQTJCLENBQUMsU0FBUyxFQUFFLE1BQU07OztHQU01RDtBQUVELHdCQUFzQixvQkFBb0IsQ0FBQyxFQUN6QyxRQUFRLEVBQ1IsU0FBUyxFQUNULElBQUksRUFDSixLQUFhLEVBQ2QsRUFBRTtJQUNELFFBQVEsRUFBRSxNQUFNLENBQUM7SUFDakIsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixJQUFJLEVBQUUsTUFBTSxDQUFDO0lBQ2IsS0FBSyxDQUFDLEVBQUUsT0FBTyxDQUFDO0NBQ2pCLG1CQU9BO0FBRUQsd0JBQXNCLHFCQUFxQixDQUFDLEVBQzFDLFFBQVEsRUFDUixTQUFTLEVBQ1QsS0FBSyxFQUNMLE9BQWMsRUFDZCxLQUFhLEVBQ2QsRUFBRTtJQUNELFFBQVEsRUFBRSxNQUFNLENBQUM7SUFDakIsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixLQUFLLEVBQUUsTUFBTSxDQUFDO0lBQ2QsT0FBTyxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ2pCLEtBQUssQ0FBQyxFQUFFLE9BQU8sQ0FBQztDQUNqQixtQkFpQkE7QUFFRCx3QkFBc0Isc0JBQXNCLENBQUMsRUFDM0MsUUFBUSxFQUNSLEtBQUssRUFDTCxTQUFTLEVBQ1QsU0FBbUIsRUFDbkIsT0FBZSxFQUNoQixFQUFFO0lBQ0QsUUFBUSxFQUFFLE1BQU0sQ0FBQztJQUNqQixLQUFLLEVBQUUsTUFBTSxDQUFDO0lBQ2QsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixTQUFTLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDbkIsT0FBTyxDQUFDLEVBQUUsTUFBTSxDQUFDO0NBQ2xCLG1CQUtBO0FBRUQsd0JBQXNCLHFCQUFxQixDQUFDLEVBQzFDLFFBQVEsRUFDUixJQUFJLEVBQ0osU0FBUyxFQUNULFNBQW1CLEVBQ25CLE9BQWUsRUFDaEIsRUFBRTtJQUNELFFBQVEsRUFBRSxNQUFNLENBQUM7SUFDakIsSUFBSSxFQUFFLE1BQU0sQ0FBQztJQUNiLFNBQVMsRUFBRSxNQUFNLENBQUM7SUFDbEIsU0FBUyxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ25CLE9BQU8sQ0FBQyxFQUFFLE1BQU0sQ0FBQztDQUNsQixtQkFLQTtBQUVELHdCQUFzQixzQkFBc0IsQ0FBQyxFQUMzQyxRQUFRLEVBQ1IsS0FBSyxFQUNMLFNBQVMsRUFDVCxTQUFtQixFQUNuQixPQUFlLEVBQ2hCLEVBQUU7SUFDRCxRQUFRLEVBQUUsTUFBTSxDQUFDO0lBQ2pCLEtBQUssRUFBRSxNQUFNLEVBQUUsQ0FBQztJQUNoQixTQUFTLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLFNBQVMsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUNuQixPQUFPLENBQUMsRUFBRSxNQUFNLENBQUM7Q0FDbEIsaUJBbUJBO0FBRUQsd0JBQWdCLFdBQVcsQ0FBQyxVQUFVLEVBQUUsTUFBTSxFQUFFLFNBQVMsRUFBRSxNQUFNLFVBRWhFIn0=
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiazhzLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc3BhcnRhbi91dGlscy9rOHMudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQ0EsT0FBTyxLQUFLLEVBQUUsTUFBTSxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFJcEQsT0FBTyxFQUFFLEtBQUssWUFBWSxFQUFlLE1BQU0sZUFBZSxDQUFDO0FBSS9ELE9BQU8sRUFBdUIsYUFBYSxFQUFFLE1BQU0sNENBQTRDLENBQUM7QUFNaEc7OztHQUdHO0FBQ0gsTUFBTSxXQUFXLGVBQWU7SUFDOUIsR0FBRyxFQUFFLE1BQU0sQ0FBQztJQUNaLE9BQU8sQ0FBQyxFQUFFLFlBQVksQ0FBQztDQUN4QjtBQUVELHdCQUFzQixnQkFBZ0IsQ0FBQyxFQUNyQyxRQUFRLEVBQ1IsU0FBUyxFQUNULGFBQWEsRUFDYixRQUFRLEVBQ1QsRUFBRTtJQUNELFFBQVEsRUFBRSxNQUFNLENBQUM7SUFDakIsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixhQUFhLEVBQUUsTUFBTSxDQUFDO0lBRXRCLFFBQVEsQ0FBQyxFQUFFLE1BQU0sQ0FBQztDQUNuQixHQUFHLE9BQU8sQ0FBQztJQUNWLE9BQU8sRUFBRSxZQUFZLENBQUM7SUFDdEIsSUFBSSxFQUFFLE1BQU0sQ0FBQztDQUNkLENBQUMsQ0F1RUQ7QUFFRCx3QkFBZ0IsYUFBYSxDQUFDLFNBQVMsRUFBRSxNQUFNLEVBQUUsV0FBVyxFQUFFLE1BQU0sR0FBRyxPQUFPLENBQUMsTUFBTSxDQUFDLENBK0JyRjtBQUVEOzs7Ozs7OztHQVFHO0FBQ0gsd0JBQXNCLGtCQUFrQixDQUFDLElBQUksRUFBRTtJQUM3QyxTQUFTLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLFdBQVcsRUFBRSxNQUFNLENBQUM7SUFDcEIsYUFBYSxFQUFFLE1BQU0sQ0FBQztJQUN0QixnQkFBZ0IsQ0FBQyxFQUFFLE9BQU8sQ0FBQztDQUM1QixHQUFHLE9BQU8sQ0FBQyxlQUFlLENBQUMsQ0FzQzNCO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsd0JBQXNCLGNBQWMsQ0FBQyxTQUFTLEVBQUUsTUFBTSxFQUFFLGdCQUFnQixDQUFDLEVBQUUsT0FBTyxHQUFHLE9BQU8sQ0FBQyxlQUFlLENBQUMsQ0FPNUc7QUFFRDs7Ozs7O0dBTUc7QUFDSCx3QkFBc0IsbUJBQW1CLENBQUMsU0FBUyxFQUFFLE1BQU0sRUFBRSxnQkFBZ0IsQ0FBQyxFQUFFLE9BQU8sR0FBRyxPQUFPLENBQUMsZUFBZSxDQUFDLENBT2pIO0FBRUQsd0JBQWdCLDhCQUE4QixDQUFDLFNBQVMsRUFBRSxNQUFNOzs7R0FNL0Q7QUFFRCx3QkFBZ0Isc0JBQXNCLENBQUMsU0FBUyxFQUFFLE1BQU0sRUFBRSxLQUFLLFNBQUk7OztHQU1sRTtBQUVELHdCQUFnQiwyQkFBMkIsQ0FBQyxTQUFTLEVBQUUsTUFBTTs7O0dBTTVEO0FBRUQsd0JBQXNCLG9CQUFvQixDQUFDLEVBQ3pDLFFBQVEsRUFDUixTQUFTLEVBQ1QsSUFBSSxFQUNKLEtBQWEsRUFDZCxFQUFFO0lBQ0QsUUFBUSxFQUFFLE1BQU0sQ0FBQztJQUNqQixTQUFTLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLElBQUksRUFBRSxNQUFNLENBQUM7SUFDYixLQUFLLENBQUMsRUFBRSxPQUFPLENBQUM7Q0FDakIsbUJBT0E7QUFFRCx3QkFBc0IscUJBQXFCLENBQUMsRUFDMUMsUUFBUSxFQUNSLFNBQVMsRUFDVCxLQUFLLEVBQ0wsT0FBYyxFQUNkLEtBQWEsRUFDZCxFQUFFO0lBQ0QsUUFBUSxFQUFFLE1BQU0sQ0FBQztJQUNqQixTQUFTLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLEtBQUssRUFBRSxNQUFNLENBQUM7SUFDZCxPQUFPLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDakIsS0FBSyxDQUFDLEVBQUUsT0FBTyxDQUFDO0NBQ2pCLG1CQWlCQTtBQUVELHdCQUFzQixzQkFBc0IsQ0FBQyxFQUMzQyxRQUFRLEVBQ1IsS0FBSyxFQUNMLFNBQVMsRUFDVCxTQUFtQixFQUNuQixPQUFlLEVBQ2hCLEVBQUU7SUFDRCxRQUFRLEVBQUUsTUFBTSxDQUFDO0lBQ2pCLEtBQUssRUFBRSxNQUFNLENBQUM7SUFDZCxTQUFTLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLFNBQVMsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUNuQixPQUFPLENBQUMsRUFBRSxNQUFNLENBQUM7Q0FDbEIsbUJBS0E7QUFFRCx3QkFBc0IscUJBQXFCLENBQUMsRUFDMUMsUUFBUSxFQUNSLElBQUksRUFDSixTQUFTLEVBQ1QsU0FBbUIsRUFDbkIsT0FBZSxFQUNoQixFQUFFO0lBQ0QsUUFBUSxFQUFFLE1BQU0sQ0FBQztJQUNqQixJQUFJLEVBQUUsTUFBTSxDQUFDO0lBQ2IsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixTQUFTLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDbkIsT0FBTyxDQUFDLEVBQUUsTUFBTSxDQUFDO0NBQ2xCLG1CQUtBO0FBRUQsd0JBQXNCLHNCQUFzQixDQUFDLEVBQzNDLFFBQVEsRUFDUixLQUFLLEVBQ0wsU0FBUyxFQUNULFNBQW1CLEVBQ25CLE9BQWUsRUFDaEIsRUFBRTtJQUNELFFBQVEsRUFBRSxNQUFNLENBQUM7SUFDakIsS0FBSyxFQUFFLE1BQU0sRUFBRSxDQUFDO0lBQ2hCLFNBQVMsRUFBRSxNQUFNLENBQUM7SUFDbEIsU0FBUyxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ25CLE9BQU8sQ0FBQyxFQUFFLE1BQU0sQ0FBQztDQUNsQixpQkFtQkE7QUFFRDs7Ozs7OztHQU9HO0FBQ0gsd0JBQXNCLHdCQUF3QixDQUFDLEVBQzdDLFNBQVMsRUFDVCxLQUFLLEVBQ0wsY0FBb0IsRUFDcEIsbUJBQXVCLEVBQ3hCLEVBQUU7SUFDRCxTQUFTLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLEtBQUssRUFBRSxNQUFNLENBQUM7SUFDZCxjQUFjLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDeEIsbUJBQW1CLENBQUMsRUFBRSxNQUFNLENBQUM7Q0FDOUIsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBbUNoQjtBQUVEOzs7Ozs7O0dBT0c7QUFDSCx3QkFBZ0IsbUNBQW1DLENBQ2pELFNBQVMsRUFBRSxNQUFNLEVBQ2pCLFNBQVMsRUFBRSxlQUFlLEVBQUUsRUFDNUIsR0FBRyxFQUFFLE1BQU0sR0FDVjtJQUNELE9BQU8sRUFBRSxNQUFNLE9BQU8sQ0FBQyxhQUFhLENBQUMsQ0FBQztJQUN0QyxhQUFhLEVBQUUsQ0FBQyxNQUFNLEVBQUUsVUFBVSxDQUFDLGFBQWEsQ0FBQyxlQUFlLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxLQUFLLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQztDQUN6RixDQTZFQTtBQUVELHdCQUFnQixXQUFXLENBQUMsVUFBVSxFQUFFLE1BQU0sRUFBRSxTQUFTLEVBQUUsTUFBTSxVQUVoRSJ9

package/dest/spartan/utils/k8s.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"k8s.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/k8s.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,YAAY,EAAe,MAAM,eAAe,CAAC;AAQ/D;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,YAAY,CAAC;CACxB;AAED,wBAAsB,gBAAgB,CAAC,EACrC,QAAQ,EACR,SAAS,EACT,aAAa,EACb,QAAQ,EACT,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IAEtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,OAAO,CAAC;IACV,OAAO,EAAE,YAAY,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC,CAuED;AAED,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA+BrF;AAED;;;;;;;;GAQG;AACH,wBAAsB,kBAAkB,CAAC,IAAI,EAAE;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,GAAG,OAAO,CAAC,eAAe,CAAC,CAsC3B;AAED;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,gBAAgB,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC,CAO5G;AAED;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CAAC,SAAS,EAAE,MAAM,EAAE,gBAAgB,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC,CAOjH;AAED,wBAAgB,8BAA8B,CAAC,SAAS,EAAE,MAAM;;;GAM/D;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,SAAI;;;GAMlE;AAED,wBAAgB,2BAA2B,CAAC,SAAS,EAAE,MAAM;;;GAM5D;AAED,wBAAsB,oBAAoB,CAAC,EACzC,QAAQ,EACR,SAAS,EACT,IAAI,EACJ,KAAa,EACd,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,mBAOA;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,SAAS,EACT,KAAK,EACL,OAAc,EACd,KAAa,EACd,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,mBAiBA;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,QAAQ,EACR,KAAK,EACL,SAAS,EACT,SAAmB,EACnB,OAAe,EAChB,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,mBAKA;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,IAAI,EACJ,SAAS,EACT,SAAmB,EACnB,OAAe,EAChB,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,mBAKA;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,QAAQ,EACR,KAAK,EACL,SAAS,EACT,SAAmB,EACnB,OAAe,EAChB,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,iBAmBA;AAED,wBAAgB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,UAEhE"}
+{"version":3,"file":"k8s.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/k8s.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIpD,OAAO,EAAE,KAAK,YAAY,EAAe,MAAM,eAAe,CAAC;AAI/D,OAAO,EAAuB,aAAa,EAAE,MAAM,4CAA4C,CAAC;AAMhG;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,YAAY,CAAC;CACxB;AAED,wBAAsB,gBAAgB,CAAC,EACrC,QAAQ,EACR,SAAS,EACT,aAAa,EACb,QAAQ,EACT,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IAEtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,OAAO,CAAC;IACV,OAAO,EAAE,YAAY,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC,CAuED;AAED,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA+BrF;AAED;;;;;;;;GAQG;AACH,wBAAsB,kBAAkB,CAAC,IAAI,EAAE;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,GAAG,OAAO,CAAC,eAAe,CAAC,CAsC3B;AAED;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,gBAAgB,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC,CAO5G;AAED;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CAAC,SAAS,EAAE,MAAM,EAAE,gBAAgB,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC,CAOjH;AAED,wBAAgB,8BAA8B,CAAC,SAAS,EAAE,MAAM;;;GAM/D;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,SAAI;;;GAMlE;AAED,wBAAgB,2BAA2B,CAAC,SAAS,EAAE,MAAM;;;GAM5D;AAED,wBAAsB,oBAAoB,CAAC,EACzC,QAAQ,EACR,SAAS,EACT,IAAI,EACJ,KAAa,EACd,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,mBAOA;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,SAAS,EACT,KAAK,EACL,OAAc,EACd,KAAa,EACd,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,mBAiBA;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,QAAQ,EACR,KAAK,EACL,SAAS,EACT,SAAmB,EACnB,OAAe,EAChB,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,mBAKA;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,IAAI,EACJ,SAAS,EACT,SAAmB,EACnB,OAAe,EAChB,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,mBAKA;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,QAAQ,EACR,KAAK,EACL,SAAS,EACT,SAAmB,EACnB,OAAe,EAChB,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,iBAmBA;AAED;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAAC,EAC7C,SAAS,EACT,KAAK,EACL,cAAoB,EACpB,mBAAuB,EACxB,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B,GAAG,OAAO,CAAC,IAAI,CAAC,CAmChB;AAED;;;;;;;GAOG;AACH,wBAAgB,mCAAmC,CACjD,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,eAAe,EAAE,EAC5B,GAAG,EAAE,MAAM,GACV;IACD,OAAO,EAAE,MAAM,OAAO,CAAC,aAAa,CAAC,CAAC;IACtC,aAAa,EAAE,CAAC,MAAM,EAAE,UAAU,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACzF,CA6EA;AAED,wBAAgB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,UAEhE"}

package/dest/spartan/utils/k8s.js

@@ -4,6 +4,7 @@ import { retryUntil } from '@aztec/foundation/retry';
 import { exec, spawn } from 'child_process';
 import path from 'path';
 import { promisify } from 'util';
+import { AlertTriggeredError, GrafanaClient } from '../../quality_of_service/grafana_client.js';
 const execAsync = promisify(exec);
 const logger = createLogger('e2e:k8s-utils');
 export async function startPortForward({ resource, namespace, containerPort, hostPort }) {
@@ -252,6 +253,123 @@ export async function waitForResourcesByName({ resource, names, namespace, condi
         }
     }));
 }
+/**
+ * Waits for all StatefulSets matching a label to have all their replicas ready.
+ *
+ * @param namespace - Kubernetes namespace
+ * @param label - Label selector for StatefulSets (e.g., "app.kubernetes.io/component=sequencer-node")
+ * @param timeoutSeconds - Maximum time to wait in seconds
+ * @param pollIntervalSeconds - How often to check status
+ */ export async function waitForStatefulSetsReady({ namespace, label, timeoutSeconds = 600, pollIntervalSeconds = 5 }) {
+    logger.info(`Waiting for StatefulSets with label ${label} to have all replicas ready (timeout: ${timeoutSeconds}s)`);
+    await retryUntil(async ()=>{
+        // Get all StatefulSets matching the label
+        const getCmd = `kubectl get statefulset -l ${label} -n ${namespace} -o json`;
+        const { stdout } = await execAsync(getCmd);
+        const result = JSON.parse(stdout);
+        if (!result.items || result.items.length === 0) {
+            logger.verbose(`No StatefulSets found with label ${label}`);
+            return false;
+        }
+        // Check each StatefulSet
+        for (const sts of result.items){
+            const name = sts.metadata.name;
+            const desired = sts.spec.replicas ?? 0;
+            const ready = sts.status.readyReplicas ?? 0;
+            const updated = sts.status.updatedReplicas ?? 0;
+            if (ready < desired || updated < desired) {
+                logger.verbose(`StatefulSet ${name}: ${ready}/${desired} ready, ${updated}/${desired} updated`);
+                return false;
+            }
+        }
+        logger.info(`All StatefulSets with label ${label} are ready`);
+        return true;
+    }, `StatefulSets with label ${label} to be ready`, timeoutSeconds, pollIntervalSeconds);
+}
+/**
+ * Creates a Prometheus connection that can re-establish port-forward on failure.
+ * Returns functions to connect and run alert checks that automatically reconnect if needed.
+ *
+ * @param namespace - K8s namespace to fall back to if metrics namespace doesn't have Prometheus
+ * @param endpoints - Array to track created endpoints for cleanup
+ * @param log - Logger instance
+ */ export function createResilientPrometheusConnection(namespace, endpoints, log) {
+    let alertChecker;
+    let currentEndpoint;
+    const connect = async ()=>{
+        // Kill existing connection if any
+        if (currentEndpoint?.process) {
+            currentEndpoint.process.kill();
+        }
+        // Try metrics namespace first, then network namespace
+        let promPort = 0;
+        let promUrl = '';
+        let promProc;
+        try {
+            const metricsResult = await startPortForward({
+                resource: `svc/metrics-prometheus-server`,
+                namespace: 'metrics',
+                containerPort: 80
+            });
+            promProc = metricsResult.process;
+            promPort = metricsResult.port;
+            promUrl = `http://127.0.0.1:${promPort}/api/v1`;
+        } catch  {
+            // Metrics namespace might not have Prometheus, try network namespace
+            log.verbose('Metrics namespace Prometheus not available, trying network namespace');
+        }
+        if (promPort === 0) {
+            const nsResult = await startPortForward({
+                resource: `svc/prometheus-server`,
+                namespace,
+                containerPort: 80
+            });
+            promProc = nsResult.process;
+            promPort = nsResult.port;
+            promUrl = `http://127.0.0.1:${promPort}/api/v1`;
+        }
+        if (!promProc || promPort === 0) {
+            throw new Error('Unable to port-forward to Prometheus');
+        }
+        currentEndpoint = {
+            url: promUrl,
+            process: promProc
+        };
+        endpoints.push(currentEndpoint);
+        alertChecker = new GrafanaClient(log, {
+            grafanaEndpoint: promUrl,
+            grafanaCredentials: ''
+        });
+        log.info(`Established Prometheus connection at ${promUrl}`);
+        return alertChecker;
+    };
+    const runAlertCheck = async (alerts)=>{
+        if (!alertChecker) {
+            alertChecker = await connect();
+        }
+        try {
+            await alertChecker.runAlertCheck(alerts);
+        } catch (err) {
+            // If it's an AlertTriggeredError (expected behavior)
+            if (err instanceof AlertTriggeredError) {
+                throw err;
+            }
+            // Check if it's a connection error (port-forward died)
+            const errorStr = String(err);
+            if (errorStr.includes('fetch failed') || errorStr.includes('ECONNREFUSED') || errorStr.includes('ECONNRESET')) {
+                log.warn(`Prometheus connection lost, re-establishing port-forward...`);
+                alertChecker = await connect();
+                await alertChecker.runAlertCheck(alerts);
+            } else {
+                throw err;
+            }
+        }
+    };
+    return {
+        connect,
+        runAlertCheck
+    };
+}
 export function getChartDir(spartanDir, chartName) {
     return path.join(spartanDir.trim(), chartName);
 }

package/dest/spartan/utils/nodes.d.ts

@@ -4,10 +4,20 @@ import type { Logger } from '@aztec/foundation/log';
 import { type AztecNodeAdmin, type AztecNodeAdminConfig } from '@aztec/stdlib/interfaces/client';
 import type { TestConfig } from './config.js';
 export declare function awaitCheckpointNumber(rollupCheatCodes: RollupCheatCodes, checkpointNumber: CheckpointNumber, timeoutSeconds: number, log: Logger): Promise<void>;
+/**
+ * Waits until the proven block number increases.
+ *
+ * @param rpcUrl - URL of an Aztec RPC node to query
+ * @param log - Logger instance
+ * @param timeoutSeconds - Maximum time to wait
+ * @param pollIntervalSeconds - How often to check
+ */
+export declare function waitForProvenToAdvance(rpcUrl: string, log: Logger, timeoutSeconds?: number, pollIntervalSeconds?: number): Promise<void>;
 export declare function getSequencers(namespace: string): Promise<string[]>;
 export declare function updateSequencersConfig(env: TestConfig, config: Partial<AztecNodeAdminConfig>): Promise<AztecNodeAdminConfig[]>;
 export declare function getSequencersConfig(env: TestConfig): Promise<AztecNodeAdminConfig[]>;
 export declare function withSequencersAdmin<T>(env: TestConfig, fn: (node: AztecNodeAdmin) => Promise<T>): Promise<T[]>;
+export declare function initHADb(namespace: string): Promise<void>;
 /**
  * Enables or disables probabilistic transaction dropping on validators and waits for rollout.
  * Wired to env vars P2P_DROP_TX and P2P_DROP_TX_CHANCE via Helm values.
@@ -28,4 +38,4 @@ export declare function enableValidatorDynamicBootNode(instanceName: string, nam
  *        Defaults to false, which preserves the existing storage.
  */
 export declare function rollAztecPods(namespace: string, clearState?: boolean): Promise<void>;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibm9kZXMuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zcGFydGFuL3V0aWxzL25vZGVzLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFDN0QsT0FBTyxLQUFLLEVBQUUsZ0JBQWdCLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUN4RSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUdwRCxPQUFPLEVBQ0wsS0FBSyxjQUFjLEVBQ25CLEtBQUssb0JBQW9CLEVBRTFCLE1BQU0saUNBQWlDLENBQUM7QUFLekMsT0FBTyxLQUFLLEVBQUUsVUFBVSxFQUFFLE1BQU0sYUFBYSxDQUFDO0FBUTlDLHdCQUFzQixxQkFBcUIsQ0FDekMsZ0JBQWdCLEVBQUUsZ0JBQWdCLEVBQ2xDLGdCQUFnQixFQUFFLGdCQUFnQixFQUNsQyxjQUFjLEVBQUUsTUFBTSxFQUN0QixHQUFHLEVBQUUsTUFBTSxpQkFlWjtBQUVELHdCQUFzQixhQUFhLENBQUMsU0FBUyxFQUFFLE1BQU0scUJBNEJwRDtBQUVELHdCQUFnQixzQkFBc0IsQ0FBQyxHQUFHLEVBQUUsVUFBVSxFQUFFLE1BQU0sRUFBRSxPQUFPLENBQUMsb0JBQW9CLENBQUMsbUNBSzVGO0FBRUQsd0JBQWdCLG1CQUFtQixDQUFDLEdBQUcsRUFBRSxVQUFVLG1DQUVsRDtBQUVELHdCQUFzQixtQkFBbUIsQ0FBQyxDQUFDLEVBQUUsR0FBRyxFQUFFLFVBQVUsRUFBRSxFQUFFLEVBQUUsQ0FBQyxJQUFJLEVBQUUsY0FBYyxLQUFLLE9BQU8sQ0FBQyxDQUFDLENBQUMsR0FBRyxPQUFPLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0EwQ3BIO0FBRUQ7OztHQUdHO0FBQ0gsd0JBQXNCLGtCQUFrQixDQUFDLEVBQ3ZDLFNBQVMsRUFDVCxPQUFPLEVBQ1AsV0FBVyxFQUNYLE1BQU0sRUFBRSxHQUFHLEVBQ1osRUFBRTtJQUNELFNBQVMsRUFBRSxNQUFNLENBQUM7SUFDbEIsT0FBTyxFQUFFLE9BQU8sQ0FBQztJQUNqQixXQUFXLEVBQUUsTUFBTSxDQUFDO0lBQ3BCLE1BQU0sRUFBRSxNQUFNLENBQUM7Q0FDaEIsaUJBZ0NBO0FBRUQsd0JBQXNCLGlCQUFpQixDQUFDLFNBQVMsRUFBRSxNQUFNLEVBQUUsR0FBRyxFQUFFLE1BQU0saUJBK0JyRTtBQUVELHdCQUFzQiw4QkFBOEIsQ0FDbEQsWUFBWSxFQUFFLE1BQU0sRUFDcEIsU0FBUyxFQUFFLE1BQU0sRUFDakIsVUFBVSxFQUFFLE1BQU0sRUFDbEIsR0FBRyxFQUFFLE1BQU0saUJBZ0JaO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsd0JBQXNCLGFBQWEsQ0FBQyxTQUFTLEVBQUUsTUFBTSxFQUFFLFVBQVUsR0FBRSxPQUFlLGlCQW1GakYifQ==
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibm9kZXMuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zcGFydGFuL3V0aWxzL25vZGVzLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUVBLE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFDN0QsT0FBTyxLQUFLLEVBQUUsZ0JBQWdCLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUN4RSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUdwRCxPQUFPLEVBQ0wsS0FBSyxjQUFjLEVBQ25CLEtBQUssb0JBQW9CLEVBRTFCLE1BQU0saUNBQWlDLENBQUM7QUFLekMsT0FBTyxLQUFLLEVBQUUsVUFBVSxFQUFFLE1BQU0sYUFBYSxDQUFDO0FBZTlDLHdCQUFzQixxQkFBcUIsQ0FDekMsZ0JBQWdCLEVBQUUsZ0JBQWdCLEVBQ2xDLGdCQUFnQixFQUFFLGdCQUFnQixFQUNsQyxjQUFjLEVBQUUsTUFBTSxFQUN0QixHQUFHLEVBQUUsTUFBTSxpQkFlWjtBQUVEOzs7Ozs7O0dBT0c7QUFDSCx3QkFBc0Isc0JBQXNCLENBQzFDLE1BQU0sRUFBRSxNQUFNLEVBQ2QsR0FBRyxFQUFFLE1BQU0sRUFDWCxjQUFjLEdBQUUsTUFBWSxFQUM1QixtQkFBbUIsR0FBRSxNQUFXLEdBQy9CLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0EwQ2Y7QUFFRCx3QkFBc0IsYUFBYSxDQUFDLFNBQVMsRUFBRSxNQUFNLHFCQTRCcEQ7QUFFRCx3QkFBZ0Isc0JBQXNCLENBQUMsR0FBRyxFQUFFLFVBQVUsRUFBRSxNQUFNLEVBQUUsT0FBTyxDQUFDLG9CQUFvQixDQUFDLG1DQUs1RjtBQUVELHdCQUFnQixtQkFBbUIsQ0FBQyxHQUFHLEVBQUUsVUFBVSxtQ0FFbEQ7QUFFRCx3QkFBc0IsbUJBQW1CLENBQUMsQ0FBQyxFQUFFLEdBQUcsRUFBRSxVQUFVLEVBQUUsRUFBRSxFQUFFLENBQUMsSUFBSSxFQUFFLGNBQWMsS0FBSyxPQUFPLENBQUMsQ0FBQyxDQUFDLEdBQUcsT0FBTyxDQUFDLENBQUMsRUFBRSxDQUFDLENBNENwSDtBQWtDRCx3QkFBc0IsUUFBUSxDQUFDLFNBQVMsRUFBRSxNQUFNLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQTBCL0Q7QUFFRDs7O0dBR0c7QUFDSCx3QkFBc0Isa0JBQWtCLENBQUMsRUFDdkMsU0FBUyxFQUNULE9BQU8sRUFDUCxXQUFXLEVBQ1gsTUFBTSxFQUFFLEdBQUcsRUFDWixFQUFFO0lBQ0QsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixPQUFPLEVBQUUsT0FBTyxDQUFDO0lBQ2pCLFdBQVcsRUFBRSxNQUFNLENBQUM7SUFDcEIsTUFBTSxFQUFFLE1BQU0sQ0FBQztDQUNoQixpQkFnQ0E7QUFFRCx3QkFBc0IsaUJBQWlCLENBQUMsU0FBUyxFQUFFLE1BQU0sRUFBRSxHQUFHLEVBQUUsTUFBTSxpQkErQnJFO0FBRUQsd0JBQXNCLDhCQUE4QixDQUNsRCxZQUFZLEVBQUUsTUFBTSxFQUNwQixTQUFTLEVBQUUsTUFBTSxFQUNqQixVQUFVLEVBQUUsTUFBTSxFQUNsQixHQUFHLEVBQUUsTUFBTSxpQkFnQlo7QUFFRDs7Ozs7O0dBTUc7QUFDSCx3QkFBc0IsYUFBYSxDQUFDLFNBQVMsRUFBRSxNQUFNLEVBQUUsVUFBVSxHQUFFLE9BQWUsaUJBMktqRiJ9

package/dest/spartan/utils/nodes.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"nodes.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/nodes.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACxE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAGpD,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,oBAAoB,EAE1B,MAAM,iCAAiC,CAAC;AAKzC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAQ9C,wBAAsB,qBAAqB,CACzC,gBAAgB,EAAE,gBAAgB,EAClC,gBAAgB,EAAE,gBAAgB,EAClC,cAAc,EAAE,MAAM,EACtB,GAAG,EAAE,MAAM,iBAeZ;AAED,wBAAsB,aAAa,CAAC,SAAS,EAAE,MAAM,qBA4BpD;AAED,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,oBAAoB,CAAC,mCAK5F;AAED,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,UAAU,mCAElD;AAED,wBAAsB,mBAAmB,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CA0CpH;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,EACvC,SAAS,EACT,OAAO,EACP,WAAW,EACX,MAAM,EAAE,GAAG,EACZ,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB,iBAgCA;AAED,wBAAsB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,iBA+BrE;AAED,wBAAsB,8BAA8B,CAClD,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,MAAM,iBAgBZ;AAED;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,GAAE,OAAe,iBAmFjF"}
+{"version":3,"file":"nodes.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/nodes.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACxE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAGpD,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,oBAAoB,EAE1B,MAAM,iCAAiC,CAAC;AAKzC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAe9C,wBAAsB,qBAAqB,CACzC,gBAAgB,EAAE,gBAAgB,EAClC,gBAAgB,EAAE,gBAAgB,EAClC,cAAc,EAAE,MAAM,EACtB,GAAG,EAAE,MAAM,iBAeZ;AAED;;;;;;;GAOG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,EACX,cAAc,GAAE,MAAY,EAC5B,mBAAmB,GAAE,MAAW,GAC/B,OAAO,CAAC,IAAI,CAAC,CA0Cf;AAED,wBAAsB,aAAa,CAAC,SAAS,EAAE,MAAM,qBA4BpD;AAED,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,oBAAoB,CAAC,mCAK5F;AAED,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,UAAU,mCAElD;AAED,wBAAsB,mBAAmB,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CA4CpH;AAkCD,wBAAsB,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA0B/D;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,EACvC,SAAS,EACT,OAAO,EACP,WAAW,EACX,MAAM,EAAE,GAAG,EACZ,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB,iBAgCA;AAED,wBAAsB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,iBA+BrE;AAED,wBAAsB,8BAA8B,CAClD,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,MAAM,iBAgBZ;AAED;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,GAAE,OAAe,iBA2KjF"}