@aztec/end-to-end 0.0.1-commit.e61ad554 → 0.0.1-commit.ee80a48

package/dest/spartan/utils/nodes.js

@@ -1,11 +1,12 @@
 import { createLogger } from '@aztec/aztec.js/log';
-import { makeBackoff, retry } from '@aztec/foundation/retry';
+import { createAztecNodeClient } from '@aztec/aztec.js/node';
+import { makeBackoff, retry, retryUntil } from '@aztec/foundation/retry';
 import { sleep } from '@aztec/foundation/sleep';
 import { createAztecNodeAdminClient } from '@aztec/stdlib/interfaces/client';
 import { exec } from 'child_process';
 import { promisify } from 'util';
 import { execHelmCommand } from './helm.js';
-import { deleteResourceByLabel, getChartDir, startPortForward, waitForResourceByLabel } from './k8s.js';
+import { deleteResourceByLabel, getChartDir, startPortForward, waitForResourceByLabel, waitForResourceByName, waitForStatefulSetsReady } from './k8s.js';
 const execAsync = promisify(exec);
 const logger = createLogger('e2e:k8s-utils');
 export async function awaitCheckpointNumber(rollupCheatCodes, checkpointNumber, timeoutSeconds, log) {
@@ -23,6 +24,43 @@ export async function awaitCheckpointNumber(rollupCheatCodes, checkpointNumber,
         log.info(`Reached checkpoint ${tips.pending}`);
     }
 }
+/**
+ * Waits until the proven block number increases.
+ *
+ * @param rpcUrl - URL of an Aztec RPC node to query
+ * @param log - Logger instance
+ * @param timeoutSeconds - Maximum time to wait
+ * @param pollIntervalSeconds - How often to check
+ */ export async function waitForProvenToAdvance(rpcUrl, log, timeoutSeconds = 300, pollIntervalSeconds = 12) {
+    const node = createAztecNodeClient(rpcUrl);
+    log.info('Waiting for proven block to advance (indicating epoch proof just submitted)...');
+    // Get current proven block number
+    let initialProvenBlock;
+    try {
+        const tips = await node.getL2Tips();
+        initialProvenBlock = Number(tips.proven.block.number);
+        log.info(`Current proven block: ${initialProvenBlock}. Waiting for it to increase...`);
+    } catch (err) {
+        log.warn(`Error getting initial tips: ${err}. Will poll until successful.`);
+        initialProvenBlock = 0;
+    }
+    await retryUntil(async ()=>{
+        try {
+            const tips = await node.getL2Tips();
+            const currentProvenBlock = Number(tips.proven.block.number);
+            const proposedBlock = Number(tips.proposed.number);
+            log.verbose(`Chain state: proposed=${proposedBlock}, proven=${currentProvenBlock} (waiting for > ${initialProvenBlock})`);
+            if (currentProvenBlock > initialProvenBlock) {
+                log.info(`Proven block advanced from ${initialProvenBlock} to ${currentProvenBlock}.`);
+                return true;
+            }
+            return false;
+        } catch (err) {
+            log.verbose(`Error checking tips: ${err}`);
+            return false;
+        }
+    }, 'proven block to advance', timeoutSeconds, pollIntervalSeconds);
+}
 export async function getSequencers(namespace) {
     const selectors = [
         'app.kubernetes.io/name=validator',
@@ -61,9 +99,15 @@ export async function withSequencersAdmin(env, fn) {
     const sequencers = await getSequencers(namespace);
     const results = [];
     for (const sequencer of sequencers){
+        // Ensure pod is Ready before attempting port-forward.
+        await waitForResourceByName({
+            resource: 'pods',
+            name: sequencer,
+            namespace
+        });
         // Wrap port-forward + fetch in a retry to handle flaky port-forwards
         const result = await retry(async ()=>{
-            const { process, port } = await startPortForward({
+            const { process: process1, port } = await startPortForward({
                 resource: `pod/${sequencer}`,
                 namespace,
                 containerPort: adminContainerPort
@@ -78,11 +122,11 @@ export async function withSequencersAdmin(env, fn) {
                 const client = createAztecNodeAdminClient(url);
                 return {
                     result: await fn(client),
-                    process
+                    process: process1
                 };
             } catch (err) {
                 // Kill the port-forward before retrying
-                process.kill();
+                process1.kill();
                 throw err;
             }
         }, 'connect to node admin', makeBackoff([
@@ -96,6 +140,58 @@ export async function withSequencersAdmin(env, fn) {
     }
     return results;
 }
+async function getAztecImageForMigrations(namespace) {
+    const aztecDockerImage = process.env.AZTEC_DOCKER_IMAGE;
+    if (aztecDockerImage) {
+        return aztecDockerImage;
+    }
+    const { stdout } = await execAsync(`kubectl get pods -l app.kubernetes.io/name=validator -n ${namespace} -o jsonpath='{.items[0].spec.containers[?(@.name=="aztec")].image}' | cat`);
+    const image = stdout.trim().replace(/^'|'$/g, '');
+    if (!image) {
+        throw new Error(`Could not detect aztec image from validator pod in namespace ${namespace}`);
+    }
+    return image;
+}
+async function getHaDbConnectionUrl(namespace) {
+    const secretName = `${namespace}-validator-ha-db-postgres`;
+    const { stdout } = await execAsync(`kubectl get secret ${secretName} -n ${namespace} -o json`);
+    const secret = JSON.parse(stdout);
+    const data = secret?.data ?? {};
+    const decode = (value)=>value ? Buffer.from(value, 'base64').toString('utf8') : '';
+    const user = decode(data.POSTGRES_USER);
+    const password = decode(data.POSTGRES_PASSWORD);
+    const database = decode(data.POSTGRES_DB);
+    if (!user || !password || !database) {
+        throw new Error(`Missing HA DB credentials in secret ${secretName}`);
+    }
+    const host = `${namespace}-validator-ha-db-postgres.${namespace}.svc.cluster.local`;
+    return `postgresql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${host}:5432/${database}`;
+}
+export async function initHADb(namespace) {
+    const databaseUrl = await getHaDbConnectionUrl(namespace);
+    const image = await getAztecImageForMigrations(namespace);
+    const jobName = `${namespace}-validator-ha-db-migrate`;
+    await execAsync(`kubectl delete pod ${jobName} -n ${namespace} --ignore-not-found=true`).catch(()=>undefined);
+    const migrateCmd = [
+        `kubectl run ${jobName} -n ${namespace}`,
+        '--rm -i',
+        '--restart=Never',
+        `--image=${image}`,
+        `--env=DATABASE_URL=${databaseUrl}`,
+        '--command -- node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js migrate-ha-db up'
+    ].join(' ');
+    const migrateCmdForLog = migrateCmd.replace(/--env=DATABASE_URL=\S+/, '--env=DATABASE_URL=<redacted>');
+    await retry(async ()=>{
+        logger.info(`command: ${migrateCmdForLog}`);
+        await execAsync(migrateCmd);
+    }, 'run HA DB migrations', makeBackoff([
+        1,
+        2,
+        4,
+        8,
+        16
+    ]), logger, true);
+}
 /**
  * Enables or disables probabilistic transaction dropping on validators and waits for rollout.
  * Wired to env vars P2P_DROP_TX and P2P_DROP_TX_CHANCE via Helm values.
@@ -201,14 +297,16 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
         'prover-broker',
         'prover-agent',
         'sequencer-node',
-        'rpc'
+        'rpc',
+        'validator-ha-db'
     ];
     const pvcComponents = [
         'p2p-bootstrap',
         'prover-node',
         'prover-broker',
         'sequencer-node',
-        'rpc'
+        'rpc',
+        'validator-ha-db'
     ];
     // StatefulSet components that need to be scaled down before PVC deletion
     // Note: validators use 'sequencer-node' as component label, not 'validator'
@@ -217,20 +315,27 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
         'prover-node',
         'prover-broker',
         'sequencer-node',
-        'rpc'
+        'rpc',
+        'validator-ha-db'
     ];
     if (clearState) {
         // To delete PVCs, we must first scale down StatefulSets so pods release the volumes
         // Otherwise PVC deletion will hang waiting for pods to terminate
-        // First, save original replica counts
+        // Save original replica counts for all StatefulSets
         const originalReplicas = new Map();
         for (const component of statefulSetComponents){
             try {
-                const getCmd = `kubectl get statefulset -l app.kubernetes.io/component=${component} -n ${namespace} -o jsonpath='{.items[0].spec.replicas}'`;
+                // Get all StatefulSets that match the component label
+                const getCmd = `kubectl get statefulset -l app.kubernetes.io/component=${component} -n ${namespace} -o json`;
                 const { stdout } = await execAsync(getCmd);
-                const replicas = parseInt(stdout.replace(/'/g, '').trim(), 10);
-                if (!isNaN(replicas) && replicas > 0) {
-                    originalReplicas.set(component, replicas);
+                const result = JSON.parse(stdout);
+                for (const sts of result.items || []){
+                    const name = sts.metadata.name;
+                    const replicas = sts.spec.replicas ?? 1;
+                    if (replicas > 0) {
+                        originalReplicas.set(name, replicas);
+                        logger.debug(`Saved replica count for StatefulSet ${name}: ${replicas}`);
+                    }
                 }
             } catch  {
             // Component might not exist, continue
@@ -247,25 +352,79 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
                 logger.verbose(`Scale down ${component} skipped: ${e}`);
             }
         }
-        // Wait for pods to terminate
-        await sleep(15 * 1000);
+        // Wait for all pods to fully terminate before deleting PVCs.
+        // terminationGracePeriodSeconds default is 30s.
+        logger.info('Waiting for pods to fully terminate before deleting PVCs...');
+        for (const component of statefulSetComponents){
+            try {
+                // Wait for all pods with this component label to be deleted
+                const waitCmd = `kubectl wait pods -l app.kubernetes.io/component=${component} --for=delete -n ${namespace} --timeout=2m`;
+                logger.info(`command: ${waitCmd}`);
+                await execAsync(waitCmd);
+            } catch (e) {
+                logger.verbose(`Wait for pod deletion ${component} skipped: ${e}`);
+            }
+        }
+        // Extra buffer to ensure PVC protection finalizers are cleared
+        await sleep(5 * 1000);
         // Now delete PVCs (they should no longer be in use)
         for (const component of pvcComponents){
-            await deleteResourceByLabel({
-                resource: 'persistentvolumeclaims',
-                namespace: namespace,
-                label: `app.kubernetes.io/component=${component}`
-            });
+            try {
+                await deleteResourceByLabel({
+                    resource: 'persistentvolumeclaims',
+                    namespace: namespace,
+                    label: `app.kubernetes.io/component=${component}`
+                });
+            } catch (e) {
+                logger.warn(`Failed to delete PVCs for ${component}: ${e}`);
+            }
         }
-        // Scale StatefulSets back up to original replica counts
-        for (const component of statefulSetComponents){
-            const replicas = originalReplicas.get(component) ?? 1;
+        // Verify PVCs are deleted
+        for (const component of pvcComponents){
+            try {
+                const waitCmd = `kubectl wait pvc -l app.kubernetes.io/component=${component} --for=delete -n ${namespace} --timeout=2m`;
+                logger.info(`command: ${waitCmd}`);
+                await execAsync(waitCmd);
+            } catch (e) {
+                logger.verbose(`Wait for PVC deletion ${component} skipped: ${e}`);
+            }
+        }
+        const haDbStatefulSets = [
+            ...originalReplicas.entries()
+        ].filter(([name])=>name.includes('validator-ha-db'));
+        const otherStatefulSets = [
+            ...originalReplicas.entries()
+        ].filter(([name])=>!name.includes('validator-ha-db'));
+        // Bring up HA DB first so we can run migrations before validators start
+        for (const [stsName, replicas] of haDbStatefulSets){
+            try {
+                const scaleCmd = `kubectl scale statefulset ${stsName} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+                logger.info(`command: ${scaleCmd}`);
+                await execAsync(scaleCmd);
+            } catch (e) {
+                logger.verbose(`Scale up ${stsName} skipped: ${e}`);
+            }
+        }
+        if (haDbStatefulSets.length > 0) {
+            try {
+                await waitForStatefulSetsReady({
+                    namespace,
+                    label: 'app.kubernetes.io/component=validator-ha-db',
+                    timeoutSeconds: 600
+                });
+                await initHADb(namespace);
+            } catch (e) {
+                logger.warn(`HA DB migration step skipped or failed: ${e}`);
+            }
+        }
+        // Scale remaining StatefulSets back up to original replica counts (by name, not label)
+        for (const [stsName, replicas] of otherStatefulSets){
             try {
-                const scaleCmd = `kubectl scale statefulset -l app.kubernetes.io/component=${component} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+                const scaleCmd = `kubectl scale statefulset ${stsName} -n ${namespace} --replicas=${replicas} --timeout=2m`;
                 logger.info(`command: ${scaleCmd}`);
                 await execAsync(scaleCmd);
             } catch (e) {
-                logger.verbose(`Scale up ${component} skipped: ${e}`);
+                logger.verbose(`Scale up ${stsName} skipped: ${e}`);
             }
         }
     } else {
@@ -279,8 +438,20 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
         }
     }
     await sleep(10 * 1000);
-    // Wait for pods to come back
-    for (const component of podComponents){
+    // Wait for StatefulSets to have all replicas ready.
+    for (const component of statefulSetComponents){
+        try {
+            await waitForStatefulSetsReady({
+                namespace,
+                label: `app.kubernetes.io/component=${component}`,
+                timeoutSeconds: 600
+            });
+        } catch (e) {
+            logger.warn(`StatefulSet component ${component} may not be fully ready: ${e}`);
+        }
+    }
+    const nonStatefulSetComponents = podComponents.filter((c)=>!statefulSetComponents.includes(c));
+    for (const component of nonStatefulSetComponents){
         await waitForResourceByLabel({
             resource: 'pods',
             namespace: namespace,

package/dest/spartan/utils/scripts.d.ts

@@ -1,4 +1,10 @@
 import type { Logger } from '@aztec/foundation/log';
+/** Result from running a script */
+export type ScriptResult = {
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+};
 /**
  * Returns the absolute path to the git repository root
  */
@@ -9,8 +15,16 @@ export declare function getAztecBin(): string;
  * @param args - The arguments to pass to the Aztec binary
  * @param logger - The logger to use
  * @param env - Optional environment variables to set for the process
- * @returns The exit code of the Aztec binary
+ * @returns The exit code, stdout, and stderr of the Aztec binary
  */
-export declare function runAztecBin(args: string[], logger: Logger, env?: Record<string, string>): Promise<number>;
-export declare function runProjectScript(script: string, args: string[], logger: Logger, env?: Record<string, string>): Promise<number>;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2NyaXB0cy5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL3NwYXJ0YW4vdXRpbHMvc2NyaXB0cy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQTRCcEQ7O0dBRUc7QUFDSCx3QkFBZ0IsaUJBQWlCLElBQUksTUFBTSxDQVcxQztBQUVELHdCQUFnQixXQUFXLFdBRTFCO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsd0JBQWdCLFdBQVcsQ0FBQyxJQUFJLEVBQUUsTUFBTSxFQUFFLEVBQUUsTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLENBQUMsRUFBRSxNQUFNLENBQUMsTUFBTSxFQUFFLE1BQU0sQ0FBQyxtQkFFdkY7QUFFRCx3QkFBZ0IsZ0JBQWdCLENBQUMsTUFBTSxFQUFFLE1BQU0sRUFBRSxJQUFJLEVBQUUsTUFBTSxFQUFFLEVBQUUsTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLENBQUMsRUFBRSxNQUFNLENBQUMsTUFBTSxFQUFFLE1BQU0sQ0FBQyxtQkFHNUcifQ==
+export declare function runAztecBin(args: string[], logger: Logger, env?: Record<string, string>): Promise<ScriptResult>;
+/**
+ * Runs a script from the project root
+ * @param script - The path to the script, relative to the project root
+ * @param args - The arguments to pass to the script
+ * @param logger - The logger to use
+ * @param env - Optional environment variables to set for the process
+ * @returns The exit code, stdout, and stderr of the script
+ */
+export declare function runProjectScript(script: string, args: string[], logger: Logger, env?: Record<string, string>): Promise<ScriptResult>;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2NyaXB0cy5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL3NwYXJ0YW4vdXRpbHMvc2NyaXB0cy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUtwRCxtQ0FBbUM7QUFDbkMsTUFBTSxNQUFNLFlBQVksR0FBRztJQUN6QixRQUFRLEVBQUUsTUFBTSxDQUFDO0lBQ2pCLE1BQU0sRUFBRSxNQUFNLENBQUM7SUFDZixNQUFNLEVBQUUsTUFBTSxDQUFDO0NBQ2hCLENBQUM7QUF5Q0Y7O0dBRUc7QUFDSCx3QkFBZ0IsaUJBQWlCLElBQUksTUFBTSxDQVcxQztBQUVELHdCQUFnQixXQUFXLFdBRTFCO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsd0JBQWdCLFdBQVcsQ0FBQyxJQUFJLEVBQUUsTUFBTSxFQUFFLEVBQUUsTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLENBQUMsRUFBRSxNQUFNLENBQUMsTUFBTSxFQUFFLE1BQU0sQ0FBQyxHQUFHLE9BQU8sQ0FBQyxZQUFZLENBQUMsQ0FFL0c7QUFFRDs7Ozs7OztHQU9HO0FBQ0gsd0JBQWdCLGdCQUFnQixDQUM5QixNQUFNLEVBQUUsTUFBTSxFQUNkLElBQUksRUFBRSxNQUFNLEVBQUUsRUFDZCxNQUFNLEVBQUUsTUFBTSxFQUNkLEdBQUcsQ0FBQyxFQUFFLE1BQU0sQ0FBQyxNQUFNLEVBQUUsTUFBTSxDQUFDLEdBQzNCLE9BQU8sQ0FBQyxZQUFZLENBQUMsQ0FHdkIifQ==

package/dest/spartan/utils/scripts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scripts.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/scripts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AA4BpD;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,MAAM,CAW1C;AAED,wBAAgB,WAAW,WAE1B;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,mBAEvF;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,mBAG5G"}
+{"version":3,"file":"scripts.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/scripts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAKpD,mCAAmC;AACnC,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAyCF;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,MAAM,CAW1C;AAED,wBAAgB,WAAW,WAE1B;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC,CAE/G;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,MAAM,EACd,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC3B,OAAO,CAAC,YAAY,CAAC,CAGvB"}

package/dest/spartan/utils/scripts.js

@@ -4,7 +4,7 @@ import path from 'path';
  * @param scriptPath - The path to the script, relative to the project root
  * @param args - The arguments to pass to the script
  * @param logger - The logger to use
- * @returns The exit code of the script
+ * @returns The exit code, stdout, and stderr of the script
  */ function runScript(scriptPath, args, logger, env) {
     const childProcess = spawn(scriptPath, args, {
         stdio: [
@@ -17,13 +17,21 @@ import path from 'path';
             ...env
         } : process.env
     });
+    const stdoutChunks = [];
+    const stderrChunks = [];
     return new Promise((resolve, reject)=>{
-        childProcess.on('close', (code)=>resolve(code ?? 0));
+        childProcess.on('close', (code)=>resolve({
+                exitCode: code ?? 0,
+                stdout: Buffer.concat(stdoutChunks).toString(),
+                stderr: Buffer.concat(stderrChunks).toString()
+            }));
         childProcess.on('error', reject);
         childProcess.stdout?.on('data', (data)=>{
+            stdoutChunks.push(data);
             logger.info(data.toString());
         });
         childProcess.stderr?.on('data', (data)=>{
+            stderrChunks.push(data);
             logger.error(data.toString());
         });
     });
@@ -53,14 +61,21 @@ export function getAztecBin() {
  * @param args - The arguments to pass to the Aztec binary
  * @param logger - The logger to use
  * @param env - Optional environment variables to set for the process
- * @returns The exit code of the Aztec binary
+ * @returns The exit code, stdout, and stderr of the Aztec binary
  */ export function runAztecBin(args, logger, env) {
     return runScript('node', [
         getAztecBin(),
         ...args
     ], logger, env);
 }
-export function runProjectScript(script, args, logger, env) {
+/**
+ * Runs a script from the project root
+ * @param script - The path to the script, relative to the project root
+ * @param args - The arguments to pass to the script
+ * @param logger - The logger to use
+ * @param env - Optional environment variables to set for the process
+ * @returns The exit code, stdout, and stderr of the script
+ */ export function runProjectScript(script, args, logger, env) {
     const scriptPath = script.startsWith('/') ? script : path.join(getGitProjectRoot(), script);
     return runScript(scriptPath, args, logger, env);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aztec/end-to-end",
-  "version": "0.0.1-commit.e61ad554",
+  "version": "0.0.1-commit.ee80a48",
   "type": "module",
   "exports": "./dest/index.js",
   "inherits": [
@@ -22,47 +22,48 @@
     "test:integration:run": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --no-cache --runInBand --config jest.integration.config.json",
     "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests src/fixtures",
     "test:compose": "./scripts/run_test.sh compose",
+    "test:ha": "./scripts/run_test.sh ha",
     "formatting": "run -T prettier --check ./src && run -T eslint ./src"
   },
   "dependencies": {
-    "@aztec/accounts": "0.0.1-commit.e61ad554",
-    "@aztec/archiver": "0.0.1-commit.e61ad554",
-    "@aztec/aztec": "0.0.1-commit.e61ad554",
-    "@aztec/aztec-node": "0.0.1-commit.e61ad554",
-    "@aztec/aztec.js": "0.0.1-commit.e61ad554",
-    "@aztec/bb-prover": "0.0.1-commit.e61ad554",
-    "@aztec/bb.js": "0.0.1-commit.e61ad554",
-    "@aztec/blob-client": "0.0.1-commit.e61ad554",
-    "@aztec/blob-lib": "0.0.1-commit.e61ad554",
-    "@aztec/bot": "0.0.1-commit.e61ad554",
-    "@aztec/cli": "0.0.1-commit.e61ad554",
-    "@aztec/constants": "0.0.1-commit.e61ad554",
-    "@aztec/entrypoints": "0.0.1-commit.e61ad554",
-    "@aztec/epoch-cache": "0.0.1-commit.e61ad554",
-    "@aztec/ethereum": "0.0.1-commit.e61ad554",
-    "@aztec/foundation": "0.0.1-commit.e61ad554",
-    "@aztec/kv-store": "0.0.1-commit.e61ad554",
-    "@aztec/l1-artifacts": "0.0.1-commit.e61ad554",
-    "@aztec/merkle-tree": "0.0.1-commit.e61ad554",
-    "@aztec/node-keystore": "0.0.1-commit.e61ad554",
-    "@aztec/noir-contracts.js": "0.0.1-commit.e61ad554",
-    "@aztec/noir-noirc_abi": "0.0.1-commit.e61ad554",
-    "@aztec/noir-protocol-circuits-types": "0.0.1-commit.e61ad554",
-    "@aztec/noir-test-contracts.js": "0.0.1-commit.e61ad554",
-    "@aztec/p2p": "0.0.1-commit.e61ad554",
-    "@aztec/protocol-contracts": "0.0.1-commit.e61ad554",
-    "@aztec/prover-client": "0.0.1-commit.e61ad554",
-    "@aztec/prover-node": "0.0.1-commit.e61ad554",
-    "@aztec/pxe": "0.0.1-commit.e61ad554",
-    "@aztec/sequencer-client": "0.0.1-commit.e61ad554",
-    "@aztec/simulator": "0.0.1-commit.e61ad554",
-    "@aztec/slasher": "0.0.1-commit.e61ad554",
-    "@aztec/stdlib": "0.0.1-commit.e61ad554",
-    "@aztec/telemetry-client": "0.0.1-commit.e61ad554",
-    "@aztec/test-wallet": "0.0.1-commit.e61ad554",
-    "@aztec/validator-client": "0.0.1-commit.e61ad554",
-    "@aztec/validator-ha-signer": "0.0.1-commit.e61ad554",
-    "@aztec/world-state": "0.0.1-commit.e61ad554",
+    "@aztec/accounts": "0.0.1-commit.ee80a48",
+    "@aztec/archiver": "0.0.1-commit.ee80a48",
+    "@aztec/aztec": "0.0.1-commit.ee80a48",
+    "@aztec/aztec-node": "0.0.1-commit.ee80a48",
+    "@aztec/aztec.js": "0.0.1-commit.ee80a48",
+    "@aztec/bb-prover": "0.0.1-commit.ee80a48",
+    "@aztec/bb.js": "0.0.1-commit.ee80a48",
+    "@aztec/blob-client": "0.0.1-commit.ee80a48",
+    "@aztec/blob-lib": "0.0.1-commit.ee80a48",
+    "@aztec/bot": "0.0.1-commit.ee80a48",
+    "@aztec/cli": "0.0.1-commit.ee80a48",
+    "@aztec/constants": "0.0.1-commit.ee80a48",
+    "@aztec/entrypoints": "0.0.1-commit.ee80a48",
+    "@aztec/epoch-cache": "0.0.1-commit.ee80a48",
+    "@aztec/ethereum": "0.0.1-commit.ee80a48",
+    "@aztec/foundation": "0.0.1-commit.ee80a48",
+    "@aztec/kv-store": "0.0.1-commit.ee80a48",
+    "@aztec/l1-artifacts": "0.0.1-commit.ee80a48",
+    "@aztec/merkle-tree": "0.0.1-commit.ee80a48",
+    "@aztec/node-keystore": "0.0.1-commit.ee80a48",
+    "@aztec/noir-contracts.js": "0.0.1-commit.ee80a48",
+    "@aztec/noir-noirc_abi": "0.0.1-commit.ee80a48",
+    "@aztec/noir-protocol-circuits-types": "0.0.1-commit.ee80a48",
+    "@aztec/noir-test-contracts.js": "0.0.1-commit.ee80a48",
+    "@aztec/p2p": "0.0.1-commit.ee80a48",
+    "@aztec/protocol-contracts": "0.0.1-commit.ee80a48",
+    "@aztec/prover-client": "0.0.1-commit.ee80a48",
+    "@aztec/prover-node": "0.0.1-commit.ee80a48",
+    "@aztec/pxe": "0.0.1-commit.ee80a48",
+    "@aztec/sequencer-client": "0.0.1-commit.ee80a48",
+    "@aztec/simulator": "0.0.1-commit.ee80a48",
+    "@aztec/slasher": "0.0.1-commit.ee80a48",
+    "@aztec/stdlib": "0.0.1-commit.ee80a48",
+    "@aztec/telemetry-client": "0.0.1-commit.ee80a48",
+    "@aztec/test-wallet": "0.0.1-commit.ee80a48",
+    "@aztec/validator-client": "0.0.1-commit.ee80a48",
+    "@aztec/validator-ha-signer": "0.0.1-commit.ee80a48",
+    "@aztec/world-state": "0.0.1-commit.ee80a48",
     "@iarna/toml": "^2.2.5",
     "@jest/globals": "^30.0.0",
     "@noble/curves": "=1.0.0",
@@ -90,6 +91,7 @@
     "lodash.every": "^4.6.0",
     "lodash.omit": "^4.5.0",
     "msgpackr": "^1.11.2",
+    "pg": "^8.17.1",
     "process": "^0.11.10",
     "snappy": "^7.2.2",
     "stream-browserify": "^3.0.0",
@@ -108,6 +110,7 @@
     "@types/jest": "^30.0.0",
     "@types/js-yaml": "^4.0.9",
     "@types/lodash.chunk": "^4.2.9",
+    "@types/pg": "^8",
     "@typescript/native-preview": "7.0.0-dev.20260113.1",
     "concurrently": "^7.6.0",
     "jest": "^30.0.0",

package/src/e2e_epochs/epochs_test.ts

@@ -14,13 +14,15 @@ import type { ExtendedViemWalletClient } from '@aztec/ethereum/types';
 import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
 import { SecretValue } from '@aztec/foundation/config';
 import { randomBytes } from '@aztec/foundation/crypto/random';
-import { withLogNameSuffix } from '@aztec/foundation/log';
+import { withLoggerBindings } from '@aztec/foundation/log/server';
 import { retryUntil } from '@aztec/foundation/retry';
 import { sleep } from '@aztec/foundation/sleep';
 import { SpamContract } from '@aztec/noir-test-contracts.js/Spam';
+import { TestContract } from '@aztec/noir-test-contracts.js/Test';
 import { getMockPubSubP2PServiceFactory } from '@aztec/p2p/test-helpers';
 import { ProverNode, type ProverNodeConfig, ProverNodePublisher } from '@aztec/prover-node';
 import type { TestProverNode } from '@aztec/prover-node/test';
+import type { PXEConfig } from '@aztec/pxe/config';
 import {
   type SequencerClient,
   type SequencerEvents,
@@ -49,7 +51,11 @@ export const WORLD_STATE_BLOCK_CHECK_INTERVAL = 50;
 export const ARCHIVER_POLL_INTERVAL = 50;
 export const DEFAULT_L1_BLOCK_TIME = process.env.CI ? 12 : 8;
 
-export type EpochsTestOpts = Partial<SetupOptions> & { numberOfAccounts?: number };
+export type EpochsTestOpts = Partial<SetupOptions> & {
+  numberOfAccounts?: number;
+  pxeOpts?: Partial<PXEConfig>;
+  aztecSlotDurationInL1Slots?: number;
+};
 
 export type TrackedSequencerEvent = {
   [K in keyof SequencerEvents]: Parameters<SequencerEvents[K]>[0] & {
@@ -94,7 +100,7 @@ export class EpochsTestContext {
       ? parseInt(process.env.L1_BLOCK_TIME)
       : DEFAULT_L1_BLOCK_TIME;
     const ethereumSlotDuration = opts.ethereumSlotDuration ?? envEthereumSlotDuration;
-    const aztecSlotDuration = opts.aztecSlotDuration ?? ethereumSlotDuration * 2;
+    const aztecSlotDuration = opts.aztecSlotDuration ?? (opts.aztecSlotDurationInL1Slots ?? 2) * ethereumSlotDuration;
     const aztecEpochDuration = opts.aztecEpochDuration ?? 6;
     const aztecProofSubmissionEpochs = opts.aztecProofSubmissionEpochs ?? 1;
     const l1PublishingTime = opts.l1PublishingTime ?? 1;
@@ -147,8 +153,9 @@ export class EpochsTestContext {
         l1PublishingTime,
         ...opts,
       },
-      // Use checkpointed chain tip for PXE to avoid issues with blocks being dropped due to pruned anchor blocks.
-      { syncChainTip: 'checkpointed' },
+      // Use checkpointed chain tip for PXE by default to avoid issues with blocks being dropped due to pruned anchor blocks.
+      // Can be overridden via opts.pxeOpts.
+      { syncChainTip: 'checkpointed', ...opts.pxeOpts },
     );
 
     this.context = context;
@@ -204,14 +211,14 @@ export class EpochsTestContext {
   public async createProverNode(opts: { dontStart?: boolean } & Partial<ProverNodeConfig> = {}) {
     this.logger.warn('Creating and syncing a simulated prover node...');
     const proverNodePrivateKey = this.getNextPrivateKey();
-    const suffix = (this.proverNodes.length + 1).toString();
-    const proverNode = await withLogNameSuffix(suffix, () =>
+    const proverIndex = this.proverNodes.length + 1;
+    const proverNode = await withLoggerBindings({ actor: `prover-${proverIndex}` }, () =>
       createAndSyncProverNode(
         proverNodePrivateKey,
         { ...this.context.config },
         {
           dataDirectory: join(this.context.config.dataDirectory!, randomBytes(8).toString('hex')),
-          proverId: EthAddress.fromNumber(parseInt(suffix, 10)),
+          proverId: EthAddress.fromNumber(proverIndex),
           dontStart: opts.dontStart,
           ...opts,
         },
@@ -240,12 +247,13 @@ export class EpochsTestContext {
   private async createNode(
     opts: Partial<AztecNodeConfig> & { txDelayerMaxInclusionTimeIntoSlot?: number; dontStartSequencer?: boolean } = {},
   ) {
-    const suffix = (this.nodes.length + 1).toString();
+    const nodeIndex = this.nodes.length + 1;
+    const actorPrefix = opts.disableValidator ? 'node' : 'validator';
     const { mockGossipSubNetwork } = this.context;
     const resolvedConfig = { ...this.context.config, ...opts };
     const p2pEnabled = resolvedConfig.p2pEnabled || mockGossipSubNetwork !== undefined;
     const p2pIp = resolvedConfig.p2pIp ?? (p2pEnabled ? '127.0.0.1' : undefined);
-    const node = await withLogNameSuffix(suffix, () =>
+    const node = await withLoggerBindings({ actor: `${actorPrefix}-${nodeIndex}` }, () =>
       AztecNodeService.createAndSync(
         {
           ...resolvedConfig,
@@ -375,6 +383,19 @@ export class EpochsTestContext {
     return SpamContract.at(instance.address, wallet);
   }
 
+  /** Registers the TestContract on the given wallet. */
+  public async registerTestContract(wallet: Wallet, salt = Fr.ZERO) {
+    const instance = await getContractInstanceFromInstantiationParams(TestContract.artifact, {
+      constructorArgs: [],
+      constructorArtifact: undefined,
+      salt,
+      publicKeys: undefined,
+      deployer: undefined,
+    });
+    await wallet.registerContract(instance, TestContract.artifact);
+    return TestContract.at(instance.address, wallet);
+  }
+
   /** Creates an L1 client using a fresh account with funds from anvil, with a tx delayer already set up. */
   public async createL1Client() {
     const { client, delayer } = withDelayer(

package/src/e2e_l1_publisher/write_json.ts

@@ -15,6 +15,7 @@ const AZTEC_GENERATE_TEST_DATA = !!process.env.AZTEC_GENERATE_TEST_DATA;
  */
 export async function writeJson(
   fileName: string,
+  checkpointHeader: CheckpointHeader,
   block: L2Block,
   l1ToL2Content: Fr[],
   blobs: Blob[],
@@ -33,12 +34,6 @@ export async function writeJson(
     return `0x${buffer.toString('hex').padStart(size, '0')}`;
   };
 
-  // Create a checkpoint header for this block
-  const checkpointHeader = CheckpointHeader.random({
-    slotNumber: block.slot,
-    timestamp: block.timestamp,
-  });
-
   const jsonObject = {
     populate: {
       l1ToL2Content: l1ToL2Content.map(value => asHex(value)),