@aztec/end-to-end 0.0.1-commit.2ed92850 → 0.0.1-commit.3d8f95d

package/dest/spartan/utils/nodes.js

@@ -1,11 +1,12 @@
 import { createLogger } from '@aztec/aztec.js/log';
-import { makeBackoff, retry } from '@aztec/foundation/retry';
+import { createAztecNodeClient } from '@aztec/aztec.js/node';
+import { makeBackoff, retry, retryUntil } from '@aztec/foundation/retry';
 import { sleep } from '@aztec/foundation/sleep';
 import { createAztecNodeAdminClient } from '@aztec/stdlib/interfaces/client';
 import { exec } from 'child_process';
 import { promisify } from 'util';
 import { execHelmCommand } from './helm.js';
-import { deleteResourceByLabel, getChartDir, startPortForward, waitForResourceByLabel } from './k8s.js';
+import { deleteResourceByLabel, getChartDir, startPortForward, waitForResourceByLabel, waitForResourceByName, waitForStatefulSetsReady } from './k8s.js';
 const execAsync = promisify(exec);
 const logger = createLogger('e2e:k8s-utils');
 export async function awaitCheckpointNumber(rollupCheatCodes, checkpointNumber, timeoutSeconds, log) {
@@ -23,6 +24,43 @@ export async function awaitCheckpointNumber(rollupCheatCodes, checkpointNumber,
         log.info(`Reached checkpoint ${tips.pending}`);
     }
 }
+/**
+ * Waits until the proven block number increases.
+ *
+ * @param rpcUrl - URL of an Aztec RPC node to query
+ * @param log - Logger instance
+ * @param timeoutSeconds - Maximum time to wait
+ * @param pollIntervalSeconds - How often to check
+ */ export async function waitForProvenToAdvance(rpcUrl, log, timeoutSeconds = 300, pollIntervalSeconds = 12) {
+    const node = createAztecNodeClient(rpcUrl);
+    log.info('Waiting for proven block to advance (indicating epoch proof just submitted)...');
+    // Get current proven block number
+    let initialProvenBlock;
+    try {
+        const tips = await node.getL2Tips();
+        initialProvenBlock = Number(tips.proven.block.number);
+        log.info(`Current proven block: ${initialProvenBlock}. Waiting for it to increase...`);
+    } catch (err) {
+        log.warn(`Error getting initial tips: ${err}. Will poll until successful.`);
+        initialProvenBlock = 0;
+    }
+    await retryUntil(async ()=>{
+        try {
+            const tips = await node.getL2Tips();
+            const currentProvenBlock = Number(tips.proven.block.number);
+            const proposedBlock = Number(tips.proposed.number);
+            log.verbose(`Chain state: proposed=${proposedBlock}, proven=${currentProvenBlock} (waiting for > ${initialProvenBlock})`);
+            if (currentProvenBlock > initialProvenBlock) {
+                log.info(`Proven block advanced from ${initialProvenBlock} to ${currentProvenBlock}.`);
+                return true;
+            }
+            return false;
+        } catch (err) {
+            log.verbose(`Error checking tips: ${err}`);
+            return false;
+        }
+    }, 'proven block to advance', timeoutSeconds, pollIntervalSeconds);
+}
 export async function getSequencers(namespace) {
     const selectors = [
         'app.kubernetes.io/name=validator',
@@ -61,9 +99,15 @@ export async function withSequencersAdmin(env, fn) {
     const sequencers = await getSequencers(namespace);
     const results = [];
     for (const sequencer of sequencers){
+        // Ensure pod is Ready before attempting port-forward.
+        await waitForResourceByName({
+            resource: 'pods',
+            name: sequencer,
+            namespace
+        });
         // Wrap port-forward + fetch in a retry to handle flaky port-forwards
         const result = await retry(async ()=>{
-            const { process, port } = await startPortForward({
+            const { process: process1, port } = await startPortForward({
                 resource: `pod/${sequencer}`,
                 namespace,
                 containerPort: adminContainerPort
@@ -78,11 +122,11 @@ export async function withSequencersAdmin(env, fn) {
                 const client = createAztecNodeAdminClient(url);
                 return {
                     result: await fn(client),
-                    process
+                    process: process1
                 };
             } catch (err) {
                 // Kill the port-forward before retrying
-                process.kill();
+                process1.kill();
                 throw err;
             }
         }, 'connect to node admin', makeBackoff([
@@ -96,6 +140,58 @@ export async function withSequencersAdmin(env, fn) {
     }
     return results;
 }
+async function getAztecImageForMigrations(namespace) {
+    const aztecDockerImage = process.env.AZTEC_DOCKER_IMAGE;
+    if (aztecDockerImage) {
+        return aztecDockerImage;
+    }
+    const { stdout } = await execAsync(`kubectl get pods -l app.kubernetes.io/name=validator -n ${namespace} -o jsonpath='{.items[0].spec.containers[?(@.name=="aztec")].image}' | cat`);
+    const image = stdout.trim().replace(/^'|'$/g, '');
+    if (!image) {
+        throw new Error(`Could not detect aztec image from validator pod in namespace ${namespace}`);
+    }
+    return image;
+}
+async function getHaDbConnectionUrl(namespace) {
+    const secretName = `${namespace}-validator-ha-db-postgres`;
+    const { stdout } = await execAsync(`kubectl get secret ${secretName} -n ${namespace} -o json`);
+    const secret = JSON.parse(stdout);
+    const data = secret?.data ?? {};
+    const decode = (value)=>value ? Buffer.from(value, 'base64').toString('utf8') : '';
+    const user = decode(data.POSTGRES_USER);
+    const password = decode(data.POSTGRES_PASSWORD);
+    const database = decode(data.POSTGRES_DB);
+    if (!user || !password || !database) {
+        throw new Error(`Missing HA DB credentials in secret ${secretName}`);
+    }
+    const host = `${namespace}-validator-ha-db-postgres.${namespace}.svc.cluster.local`;
+    return `postgresql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${host}:5432/${database}`;
+}
+export async function initHADb(namespace) {
+    const databaseUrl = await getHaDbConnectionUrl(namespace);
+    const image = await getAztecImageForMigrations(namespace);
+    const jobName = `${namespace}-validator-ha-db-migrate`;
+    await execAsync(`kubectl delete pod ${jobName} -n ${namespace} --ignore-not-found=true`).catch(()=>undefined);
+    const migrateCmd = [
+        `kubectl run ${jobName} -n ${namespace}`,
+        '--rm -i',
+        '--restart=Never',
+        `--image=${image}`,
+        `--env=DATABASE_URL=${databaseUrl}`,
+        '--command -- node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js migrate-ha-db up'
+    ].join(' ');
+    const migrateCmdForLog = migrateCmd.replace(/--env=DATABASE_URL=\S+/, '--env=DATABASE_URL=<redacted>');
+    await retry(async ()=>{
+        logger.info(`command: ${migrateCmdForLog}`);
+        await execAsync(migrateCmd);
+    }, 'run HA DB migrations', makeBackoff([
+        1,
+        2,
+        4,
+        8,
+        16
+    ]), logger, true);
+}
 /**
  * Enables or disables probabilistic transaction dropping on validators and waits for rollout.
  * Wired to env vars P2P_DROP_TX and P2P_DROP_TX_CHANCE via Helm values.
@@ -201,14 +297,16 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
         'prover-broker',
         'prover-agent',
         'sequencer-node',
-        'rpc'
+        'rpc',
+        'validator-ha-db'
     ];
     const pvcComponents = [
         'p2p-bootstrap',
         'prover-node',
         'prover-broker',
         'sequencer-node',
-        'rpc'
+        'rpc',
+        'validator-ha-db'
     ];
     // StatefulSet components that need to be scaled down before PVC deletion
     // Note: validators use 'sequencer-node' as component label, not 'validator'
@@ -217,20 +315,27 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
         'prover-node',
         'prover-broker',
         'sequencer-node',
-        'rpc'
+        'rpc',
+        'validator-ha-db'
     ];
     if (clearState) {
         // To delete PVCs, we must first scale down StatefulSets so pods release the volumes
         // Otherwise PVC deletion will hang waiting for pods to terminate
-        // First, save original replica counts
+        // Save original replica counts for all StatefulSets
         const originalReplicas = new Map();
         for (const component of statefulSetComponents){
             try {
-                const getCmd = `kubectl get statefulset -l app.kubernetes.io/component=${component} -n ${namespace} -o jsonpath='{.items[0].spec.replicas}'`;
+                // Get all StatefulSets that match the component label
+                const getCmd = `kubectl get statefulset -l app.kubernetes.io/component=${component} -n ${namespace} -o json`;
                 const { stdout } = await execAsync(getCmd);
-                const replicas = parseInt(stdout.replace(/'/g, '').trim(), 10);
-                if (!isNaN(replicas) && replicas > 0) {
-                    originalReplicas.set(component, replicas);
+                const result = JSON.parse(stdout);
+                for (const sts of result.items || []){
+                    const name = sts.metadata.name;
+                    const replicas = sts.spec.replicas ?? 1;
+                    if (replicas > 0) {
+                        originalReplicas.set(name, replicas);
+                        logger.debug(`Saved replica count for StatefulSet ${name}: ${replicas}`);
+                    }
                 }
             } catch  {
             // Component might not exist, continue
@@ -247,25 +352,79 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
                 logger.verbose(`Scale down ${component} skipped: ${e}`);
             }
         }
-        // Wait for pods to terminate
-        await sleep(15 * 1000);
+        // Wait for all pods to fully terminate before deleting PVCs.
+        // terminationGracePeriodSeconds default is 30s.
+        logger.info('Waiting for pods to fully terminate before deleting PVCs...');
+        for (const component of statefulSetComponents){
+            try {
+                // Wait for all pods with this component label to be deleted
+                const waitCmd = `kubectl wait pods -l app.kubernetes.io/component=${component} --for=delete -n ${namespace} --timeout=2m`;
+                logger.info(`command: ${waitCmd}`);
+                await execAsync(waitCmd);
+            } catch (e) {
+                logger.verbose(`Wait for pod deletion ${component} skipped: ${e}`);
+            }
+        }
+        // Extra buffer to ensure PVC protection finalizers are cleared
+        await sleep(5 * 1000);
         // Now delete PVCs (they should no longer be in use)
         for (const component of pvcComponents){
-            await deleteResourceByLabel({
-                resource: 'persistentvolumeclaims',
-                namespace: namespace,
-                label: `app.kubernetes.io/component=${component}`
-            });
+            try {
+                await deleteResourceByLabel({
+                    resource: 'persistentvolumeclaims',
+                    namespace: namespace,
+                    label: `app.kubernetes.io/component=${component}`
+                });
+            } catch (e) {
+                logger.warn(`Failed to delete PVCs for ${component}: ${e}`);
+            }
         }
-        // Scale StatefulSets back up to original replica counts
-        for (const component of statefulSetComponents){
-            const replicas = originalReplicas.get(component) ?? 1;
+        // Verify PVCs are deleted
+        for (const component of pvcComponents){
+            try {
+                const waitCmd = `kubectl wait pvc -l app.kubernetes.io/component=${component} --for=delete -n ${namespace} --timeout=2m`;
+                logger.info(`command: ${waitCmd}`);
+                await execAsync(waitCmd);
+            } catch (e) {
+                logger.verbose(`Wait for PVC deletion ${component} skipped: ${e}`);
+            }
+        }
+        const haDbStatefulSets = [
+            ...originalReplicas.entries()
+        ].filter(([name])=>name.includes('validator-ha-db'));
+        const otherStatefulSets = [
+            ...originalReplicas.entries()
+        ].filter(([name])=>!name.includes('validator-ha-db'));
+        // Bring up HA DB first so we can run migrations before validators start
+        for (const [stsName, replicas] of haDbStatefulSets){
+            try {
+                const scaleCmd = `kubectl scale statefulset ${stsName} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+                logger.info(`command: ${scaleCmd}`);
+                await execAsync(scaleCmd);
+            } catch (e) {
+                logger.verbose(`Scale up ${stsName} skipped: ${e}`);
+            }
+        }
+        if (haDbStatefulSets.length > 0) {
+            try {
+                await waitForStatefulSetsReady({
+                    namespace,
+                    label: 'app.kubernetes.io/component=validator-ha-db',
+                    timeoutSeconds: 600
+                });
+                await initHADb(namespace);
+            } catch (e) {
+                logger.warn(`HA DB migration step skipped or failed: ${e}`);
+            }
+        }
+        // Scale remaining StatefulSets back up to original replica counts (by name, not label)
+        for (const [stsName, replicas] of otherStatefulSets){
             try {
-                const scaleCmd = `kubectl scale statefulset -l app.kubernetes.io/component=${component} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+                const scaleCmd = `kubectl scale statefulset ${stsName} -n ${namespace} --replicas=${replicas} --timeout=2m`;
                 logger.info(`command: ${scaleCmd}`);
                 await execAsync(scaleCmd);
             } catch (e) {
-                logger.verbose(`Scale up ${component} skipped: ${e}`);
+                logger.verbose(`Scale up ${stsName} skipped: ${e}`);
             }
         }
     } else {
@@ -279,8 +438,20 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
         }
     }
     await sleep(10 * 1000);
-    // Wait for pods to come back
-    for (const component of podComponents){
+    // Wait for StatefulSets to have all replicas ready.
+    for (const component of statefulSetComponents){
+        try {
+            await waitForStatefulSetsReady({
+                namespace,
+                label: `app.kubernetes.io/component=${component}`,
+                timeoutSeconds: 600
+            });
+        } catch (e) {
+            logger.warn(`StatefulSet component ${component} may not be fully ready: ${e}`);
+        }
+    }
+    const nonStatefulSetComponents = podComponents.filter((c)=>!statefulSetComponents.includes(c));
+    for (const component of nonStatefulSetComponents){
         await waitForResourceByLabel({
             resource: 'pods',
             namespace: namespace,

package/dest/spartan/utils/scripts.d.ts

@@ -1,4 +1,10 @@
 import type { Logger } from '@aztec/foundation/log';
+/** Result from running a script */
+export type ScriptResult = {
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+};
 /**
  * Returns the absolute path to the git repository root
  */
@@ -9,8 +15,16 @@ export declare function getAztecBin(): string;
  * @param args - The arguments to pass to the Aztec binary
  * @param logger - The logger to use
  * @param env - Optional environment variables to set for the process
- * @returns The exit code of the Aztec binary
+ * @returns The exit code, stdout, and stderr of the Aztec binary
  */
-export declare function runAztecBin(args: string[], logger: Logger, env?: Record<string, string>): Promise<number>;
-export declare function runProjectScript(script: string, args: string[], logger: Logger, env?: Record<string, string>): Promise<number>;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2NyaXB0cy5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL3NwYXJ0YW4vdXRpbHMvc2NyaXB0cy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQTRCcEQ7O0dBRUc7QUFDSCx3QkFBZ0IsaUJBQWlCLElBQUksTUFBTSxDQVcxQztBQUVELHdCQUFnQixXQUFXLFdBRTFCO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsd0JBQWdCLFdBQVcsQ0FBQyxJQUFJLEVBQUUsTUFBTSxFQUFFLEVBQUUsTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLENBQUMsRUFBRSxNQUFNLENBQUMsTUFBTSxFQUFFLE1BQU0sQ0FBQyxtQkFFdkY7QUFFRCx3QkFBZ0IsZ0JBQWdCLENBQUMsTUFBTSxFQUFFLE1BQU0sRUFBRSxJQUFJLEVBQUUsTUFBTSxFQUFFLEVBQUUsTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLENBQUMsRUFBRSxNQUFNLENBQUMsTUFBTSxFQUFFLE1BQU0sQ0FBQyxtQkFHNUcifQ==
+export declare function runAztecBin(args: string[], logger: Logger, env?: Record<string, string>): Promise<ScriptResult>;
+/**
+ * Runs a script from the project root
+ * @param script - The path to the script, relative to the project root
+ * @param args - The arguments to pass to the script
+ * @param logger - The logger to use
+ * @param env - Optional environment variables to set for the process
+ * @returns The exit code, stdout, and stderr of the script
+ */
+export declare function runProjectScript(script: string, args: string[], logger: Logger, env?: Record<string, string>): Promise<ScriptResult>;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2NyaXB0cy5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL3NwYXJ0YW4vdXRpbHMvc2NyaXB0cy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUtwRCxtQ0FBbUM7QUFDbkMsTUFBTSxNQUFNLFlBQVksR0FBRztJQUN6QixRQUFRLEVBQUUsTUFBTSxDQUFDO0lBQ2pCLE1BQU0sRUFBRSxNQUFNLENBQUM7SUFDZixNQUFNLEVBQUUsTUFBTSxDQUFDO0NBQ2hCLENBQUM7QUF5Q0Y7O0dBRUc7QUFDSCx3QkFBZ0IsaUJBQWlCLElBQUksTUFBTSxDQVcxQztBQUVELHdCQUFnQixXQUFXLFdBRTFCO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsd0JBQWdCLFdBQVcsQ0FBQyxJQUFJLEVBQUUsTUFBTSxFQUFFLEVBQUUsTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLENBQUMsRUFBRSxNQUFNLENBQUMsTUFBTSxFQUFFLE1BQU0sQ0FBQyxHQUFHLE9BQU8sQ0FBQyxZQUFZLENBQUMsQ0FFL0c7QUFFRDs7Ozs7OztHQU9HO0FBQ0gsd0JBQWdCLGdCQUFnQixDQUM5QixNQUFNLEVBQUUsTUFBTSxFQUNkLElBQUksRUFBRSxNQUFNLEVBQUUsRUFDZCxNQUFNLEVBQUUsTUFBTSxFQUNkLEdBQUcsQ0FBQyxFQUFFLE1BQU0sQ0FBQyxNQUFNLEVBQUUsTUFBTSxDQUFDLEdBQzNCLE9BQU8sQ0FBQyxZQUFZLENBQUMsQ0FHdkIifQ==

package/dest/spartan/utils/scripts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scripts.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/scripts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AA4BpD;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,MAAM,CAW1C;AAED,wBAAgB,WAAW,WAE1B;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,mBAEvF;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,mBAG5G"}
+{"version":3,"file":"scripts.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/scripts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAKpD,mCAAmC;AACnC,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAyCF;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,MAAM,CAW1C;AAED,wBAAgB,WAAW,WAE1B;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC,CAE/G;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,MAAM,EACd,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC3B,OAAO,CAAC,YAAY,CAAC,CAGvB"}

package/dest/spartan/utils/scripts.js

@@ -4,7 +4,7 @@ import path from 'path';
  * @param scriptPath - The path to the script, relative to the project root
  * @param args - The arguments to pass to the script
  * @param logger - The logger to use
- * @returns The exit code of the script
+ * @returns The exit code, stdout, and stderr of the script
  */ function runScript(scriptPath, args, logger, env) {
     const childProcess = spawn(scriptPath, args, {
         stdio: [
@@ -17,13 +17,21 @@ import path from 'path';
             ...env
         } : process.env
     });
+    const stdoutChunks = [];
+    const stderrChunks = [];
     return new Promise((resolve, reject)=>{
-        childProcess.on('close', (code)=>resolve(code ?? 0));
+        childProcess.on('close', (code)=>resolve({
+                exitCode: code ?? 0,
+                stdout: Buffer.concat(stdoutChunks).toString(),
+                stderr: Buffer.concat(stderrChunks).toString()
+            }));
         childProcess.on('error', reject);
         childProcess.stdout?.on('data', (data)=>{
+            stdoutChunks.push(data);
             logger.info(data.toString());
         });
         childProcess.stderr?.on('data', (data)=>{
+            stderrChunks.push(data);
             logger.error(data.toString());
         });
     });
@@ -53,14 +61,21 @@ export function getAztecBin() {
  * @param args - The arguments to pass to the Aztec binary
  * @param logger - The logger to use
  * @param env - Optional environment variables to set for the process
- * @returns The exit code of the Aztec binary
+ * @returns The exit code, stdout, and stderr of the Aztec binary
  */ export function runAztecBin(args, logger, env) {
     return runScript('node', [
         getAztecBin(),
         ...args
     ], logger, env);
 }
-export function runProjectScript(script, args, logger, env) {
+/**
+ * Runs a script from the project root
+ * @param script - The path to the script, relative to the project root
+ * @param args - The arguments to pass to the script
+ * @param logger - The logger to use
+ * @param env - Optional environment variables to set for the process
+ * @returns The exit code, stdout, and stderr of the script
+ */ export function runProjectScript(script, args, logger, env) {
     const scriptPath = script.startsWith('/') ? script : path.join(getGitProjectRoot(), script);
     return runScript(scriptPath, args, logger, env);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aztec/end-to-end",
-  "version": "0.0.1-commit.2ed92850",
+  "version": "0.0.1-commit.3d8f95d",
   "type": "module",
   "exports": "./dest/index.js",
   "inherits": [
@@ -22,47 +22,48 @@
     "test:integration:run": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --no-cache --runInBand --config jest.integration.config.json",
     "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests src/fixtures",
     "test:compose": "./scripts/run_test.sh compose",
+    "test:ha": "./scripts/run_test.sh ha",
     "formatting": "run -T prettier --check ./src && run -T eslint ./src"
   },
   "dependencies": {
-    "@aztec/accounts": "0.0.1-commit.2ed92850",
-    "@aztec/archiver": "0.0.1-commit.2ed92850",
-    "@aztec/aztec": "0.0.1-commit.2ed92850",
-    "@aztec/aztec-node": "0.0.1-commit.2ed92850",
-    "@aztec/aztec.js": "0.0.1-commit.2ed92850",
-    "@aztec/bb-prover": "0.0.1-commit.2ed92850",
-    "@aztec/bb.js": "0.0.1-commit.2ed92850",
-    "@aztec/blob-client": "0.0.1-commit.2ed92850",
-    "@aztec/blob-lib": "0.0.1-commit.2ed92850",
-    "@aztec/bot": "0.0.1-commit.2ed92850",
-    "@aztec/cli": "0.0.1-commit.2ed92850",
-    "@aztec/constants": "0.0.1-commit.2ed92850",
-    "@aztec/entrypoints": "0.0.1-commit.2ed92850",
-    "@aztec/epoch-cache": "0.0.1-commit.2ed92850",
-    "@aztec/ethereum": "0.0.1-commit.2ed92850",
-    "@aztec/foundation": "0.0.1-commit.2ed92850",
-    "@aztec/kv-store": "0.0.1-commit.2ed92850",
-    "@aztec/l1-artifacts": "0.0.1-commit.2ed92850",
-    "@aztec/merkle-tree": "0.0.1-commit.2ed92850",
-    "@aztec/node-keystore": "0.0.1-commit.2ed92850",
-    "@aztec/noir-contracts.js": "0.0.1-commit.2ed92850",
-    "@aztec/noir-noirc_abi": "0.0.1-commit.2ed92850",
-    "@aztec/noir-protocol-circuits-types": "0.0.1-commit.2ed92850",
-    "@aztec/noir-test-contracts.js": "0.0.1-commit.2ed92850",
-    "@aztec/p2p": "0.0.1-commit.2ed92850",
-    "@aztec/protocol-contracts": "0.0.1-commit.2ed92850",
-    "@aztec/prover-client": "0.0.1-commit.2ed92850",
-    "@aztec/prover-node": "0.0.1-commit.2ed92850",
-    "@aztec/pxe": "0.0.1-commit.2ed92850",
-    "@aztec/sequencer-client": "0.0.1-commit.2ed92850",
-    "@aztec/simulator": "0.0.1-commit.2ed92850",
-    "@aztec/slasher": "0.0.1-commit.2ed92850",
-    "@aztec/stdlib": "0.0.1-commit.2ed92850",
-    "@aztec/telemetry-client": "0.0.1-commit.2ed92850",
-    "@aztec/test-wallet": "0.0.1-commit.2ed92850",
-    "@aztec/validator-client": "0.0.1-commit.2ed92850",
-    "@aztec/validator-ha-signer": "0.0.1-commit.2ed92850",
-    "@aztec/world-state": "0.0.1-commit.2ed92850",
+    "@aztec/accounts": "0.0.1-commit.3d8f95d",
+    "@aztec/archiver": "0.0.1-commit.3d8f95d",
+    "@aztec/aztec": "0.0.1-commit.3d8f95d",
+    "@aztec/aztec-node": "0.0.1-commit.3d8f95d",
+    "@aztec/aztec.js": "0.0.1-commit.3d8f95d",
+    "@aztec/bb-prover": "0.0.1-commit.3d8f95d",
+    "@aztec/bb.js": "0.0.1-commit.3d8f95d",
+    "@aztec/blob-client": "0.0.1-commit.3d8f95d",
+    "@aztec/blob-lib": "0.0.1-commit.3d8f95d",
+    "@aztec/bot": "0.0.1-commit.3d8f95d",
+    "@aztec/cli": "0.0.1-commit.3d8f95d",
+    "@aztec/constants": "0.0.1-commit.3d8f95d",
+    "@aztec/entrypoints": "0.0.1-commit.3d8f95d",
+    "@aztec/epoch-cache": "0.0.1-commit.3d8f95d",
+    "@aztec/ethereum": "0.0.1-commit.3d8f95d",
+    "@aztec/foundation": "0.0.1-commit.3d8f95d",
+    "@aztec/kv-store": "0.0.1-commit.3d8f95d",
+    "@aztec/l1-artifacts": "0.0.1-commit.3d8f95d",
+    "@aztec/merkle-tree": "0.0.1-commit.3d8f95d",
+    "@aztec/node-keystore": "0.0.1-commit.3d8f95d",
+    "@aztec/noir-contracts.js": "0.0.1-commit.3d8f95d",
+    "@aztec/noir-noirc_abi": "0.0.1-commit.3d8f95d",
+    "@aztec/noir-protocol-circuits-types": "0.0.1-commit.3d8f95d",
+    "@aztec/noir-test-contracts.js": "0.0.1-commit.3d8f95d",
+    "@aztec/p2p": "0.0.1-commit.3d8f95d",
+    "@aztec/protocol-contracts": "0.0.1-commit.3d8f95d",
+    "@aztec/prover-client": "0.0.1-commit.3d8f95d",
+    "@aztec/prover-node": "0.0.1-commit.3d8f95d",
+    "@aztec/pxe": "0.0.1-commit.3d8f95d",
+    "@aztec/sequencer-client": "0.0.1-commit.3d8f95d",
+    "@aztec/simulator": "0.0.1-commit.3d8f95d",
+    "@aztec/slasher": "0.0.1-commit.3d8f95d",
+    "@aztec/stdlib": "0.0.1-commit.3d8f95d",
+    "@aztec/telemetry-client": "0.0.1-commit.3d8f95d",
+    "@aztec/test-wallet": "0.0.1-commit.3d8f95d",
+    "@aztec/validator-client": "0.0.1-commit.3d8f95d",
+    "@aztec/validator-ha-signer": "0.0.1-commit.3d8f95d",
+    "@aztec/world-state": "0.0.1-commit.3d8f95d",
     "@iarna/toml": "^2.2.5",
     "@jest/globals": "^30.0.0",
     "@noble/curves": "=1.0.0",
@@ -90,6 +91,7 @@
     "lodash.every": "^4.6.0",
     "lodash.omit": "^4.5.0",
     "msgpackr": "^1.11.2",
+    "pg": "^8.17.1",
     "process": "^0.11.10",
     "snappy": "^7.2.2",
     "stream-browserify": "^3.0.0",
@@ -108,6 +110,7 @@
     "@types/jest": "^30.0.0",
     "@types/js-yaml": "^4.0.9",
     "@types/lodash.chunk": "^4.2.9",
+    "@types/pg": "^8",
     "@typescript/native-preview": "7.0.0-dev.20260113.1",
     "concurrently": "^7.6.0",
     "jest": "^30.0.0",

package/src/bench/client_flows/client_flows_benchmark.ts

@@ -138,10 +138,10 @@ export class ClientFlowsBenchmark {
     });
     await this.applyBaseSetup();
 
-    await this.context.aztecNodeService!.setConfig({ feeRecipient: this.sequencerAddress, coinbase: this.coinbase });
+    await this.context.aztecNodeService.setConfig({ feeRecipient: this.sequencerAddress, coinbase: this.coinbase });
 
     const rollupContract = RollupContract.getFromConfig(this.context.config);
-    this.chainMonitor = new ChainMonitor(rollupContract, this.context.dateProvider!, this.logger, 200).start();
+    this.chainMonitor = new ChainMonitor(rollupContract, this.context.dateProvider, this.logger, 200).start();
 
     return this;
   }
@@ -207,7 +207,7 @@ export class ClientFlowsBenchmark {
     const [{ address: adminAddress }, { address: sequencerAddress }] = deployedAccounts;
 
     this.adminWallet = this.context.wallet;
-    this.aztecNode = this.context.aztecNodeService!;
+    this.aztecNode = this.context.aztecNodeService;
     this.cheatCodes = this.context.cheatCodes;
 
     this.adminAddress = adminAddress;
@@ -235,8 +235,8 @@ export class ClientFlowsBenchmark {
     this.feeJuiceContract = FeeJuiceContract.at(ProtocolContractAddress.FeeJuice, this.adminWallet);
 
     this.feeJuiceBridgeTestHarness = await FeeJuicePortalTestingHarnessFactory.create({
-      aztecNode: this.context.aztecNodeService!,
-      aztecNodeAdmin: this.context.aztecNodeService!,
+      aztecNode: this.context.aztecNodeService,
+      aztecNodeAdmin: this.context.aztecNodeService,
       l1Client: this.context.deployL1ContractsValues.l1Client,
       wallet: this.adminWallet,
       logger: this.logger,

package/src/bench/client_flows/config.ts

@@ -6,7 +6,7 @@ export type ClientFlowConfig = {
   recursions?: number[];
 };
 
-type ClientFlows = 'accountDeployments' | 'deployments' | 'transfers' | 'bridging' | 'amm';
+type ClientFlows = 'accountDeployments' | 'deployments' | 'transfers' | 'bridging' | 'amm' | 'storageProof';
 
 export type ClientFlowsConfig = {
   [key in ClientFlows]: ClientFlowConfig;
@@ -34,6 +34,10 @@ export const KEY_FLOWS_CONFIG: ClientFlowsConfig = {
     feePaymentMethods: ['sponsored_fpc', 'private_fpc'],
     recursions: [0, 1],
   },
+  storageProof: {
+    accounts: ['ecdsar1'],
+    feePaymentMethods: ['sponsored_fpc'],
+  },
 };
 
 export const FULL_FLOWS_CONFIG: ClientFlowsConfig = {
@@ -58,4 +62,8 @@ export const FULL_FLOWS_CONFIG: ClientFlowsConfig = {
     feePaymentMethods: ['sponsored_fpc', 'private_fpc'],
     recursions: [0, 1, 2],
   },
+  storageProof: {
+    accounts: ['ecdsar1', 'schnorr'],
+    feePaymentMethods: ['sponsored_fpc', 'private_fpc'],
+  },
 };

package/src/bench/utils.ts

@@ -28,7 +28,7 @@ export async function benchmarkSetup(
   const contract = await BenchmarkingContract.deploy(context.wallet).send({ from: defaultAccountAddress });
   context.logger.info(`Deployed benchmarking contract at ${contract.address}`);
   const sequencer = (context.aztecNode as AztecNodeService).getSequencer()!;
-  const telemetry = context.telemetryClient! as BenchmarkTelemetryClient;
+  const telemetry = context.telemetryClient as BenchmarkTelemetryClient;
   context.logger.warn(`Cleared benchmark data points from setup`);
   telemetry.clear();
   const origTeardown = context.teardown.bind(context);

package/src/e2e_blacklist_token_contract/blacklist_token_contract_test.ts

@@ -92,7 +92,7 @@ export class BlacklistTokenContractTest {
     });
 
     this.cheatCodes = this.context.cheatCodes;
-    this.aztecNode = this.context.aztecNodeService!;
+    this.aztecNode = this.context.aztecNodeService;
     this.sequencer = this.context.sequencer!;
     this.wallet = this.context.wallet;
     this.adminAddress = deployedAccounts[0].address;

package/src/e2e_cross_chain_messaging/cross_chain_messaging_test.ts

@@ -13,7 +13,7 @@ import type {
 } from '@aztec/ethereum/deploy-aztec-l1-contracts';
 import { deployL1Contract } from '@aztec/ethereum/deploy-l1-contract';
 import type { ExtendedViemWalletClient } from '@aztec/ethereum/types';
-import { CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
+import { EpochNumber } from '@aztec/foundation/branded-types';
 import { sleep } from '@aztec/foundation/sleep';
 import { TestERC20Abi, TestERC20Bytecode } from '@aztec/l1-artifacts';
 import { TokenContract } from '@aztec/noir-contracts.js/Token';
@@ -86,9 +86,8 @@ export class CrossChainMessagingTest {
   }
 
   async advanceToEpochProven(l2TxReceipt: TxReceipt): Promise<EpochNumber> {
-    const epoch = await this.rollup.getEpochNumberForCheckpoint(
-      CheckpointNumber.fromBlockNumber(l2TxReceipt.blockNumber!),
-    );
+    const block = await this.aztecNode.getBlock(l2TxReceipt.blockNumber!);
+    const epoch = await this.rollup.getEpochNumberForCheckpoint(block!.checkpointNumber);
     // Warp to the next epoch.
     await this.cheatCodes.rollup.advanceToEpoch(EpochNumber(epoch + 1));
     // Wait for the tx to be proven.
@@ -110,16 +109,16 @@ export class CrossChainMessagingTest {
 
   async applyBaseSetup() {
     // Set up base context fields
-    this.aztecNode = this.context.aztecNodeService!;
+    this.aztecNode = this.context.aztecNodeService;
     this.wallet = this.context.wallet;
     this.aztecNodeConfig = this.context.config;
     this.cheatCodes = this.context.cheatCodes;
     this.deployL1ContractsValues = this.context.deployL1ContractsValues;
-    this.aztecNodeAdmin = this.context.aztecNodeService!;
+    this.aztecNodeAdmin = this.context.aztecNodeService;
 
     if (this.requireEpochProven) {
       // Turn off the watcher to prevent it from keep marking blocks as proven.
-      this.context.watcher!.setIsMarkingAsProven(false);
+      this.context.watcher.setIsMarkingAsProven(false);
     }
 
     // Deploy 3 accounts