@aztec/end-to-end 0.0.1-commit.c80b6263 → 0.0.1-commit.c949de6bc

package/dest/spartan/utils/nodes.js

@@ -1,11 +1,12 @@
 import { createLogger } from '@aztec/aztec.js/log';
-import { makeBackoff, retry } from '@aztec/foundation/retry';
+import { createAztecNodeClient } from '@aztec/aztec.js/node';
+import { makeBackoff, retry, retryUntil } from '@aztec/foundation/retry';
 import { sleep } from '@aztec/foundation/sleep';
 import { createAztecNodeAdminClient } from '@aztec/stdlib/interfaces/client';
 import { exec } from 'child_process';
 import { promisify } from 'util';
 import { execHelmCommand } from './helm.js';
-import { deleteResourceByLabel, getChartDir, startPortForward, waitForResourceByLabel } from './k8s.js';
+import { deleteResourceByLabel, getChartDir, startPortForward, waitForResourceByLabel, waitForResourceByName, waitForStatefulSetsReady } from './k8s.js';
 const execAsync = promisify(exec);
 const logger = createLogger('e2e:k8s-utils');
 export async function awaitCheckpointNumber(rollupCheatCodes, checkpointNumber, timeoutSeconds, log) {
@@ -23,6 +24,43 @@ export async function awaitCheckpointNumber(rollupCheatCodes, checkpointNumber,
         log.info(`Reached checkpoint ${tips.pending}`);
     }
 }
+/**
+ * Waits until the proven block number increases.
+ *
+ * @param rpcUrl - URL of an Aztec RPC node to query
+ * @param log - Logger instance
+ * @param timeoutSeconds - Maximum time to wait
+ * @param pollIntervalSeconds - How often to check
+ */ export async function waitForProvenToAdvance(rpcUrl, log, timeoutSeconds = 300, pollIntervalSeconds = 12) {
+    const node = createAztecNodeClient(rpcUrl);
+    log.info('Waiting for proven block to advance (indicating epoch proof just submitted)...');
+    // Get current proven block number
+    let initialProvenBlock;
+    try {
+        const tips = await node.getL2Tips();
+        initialProvenBlock = Number(tips.proven.block.number);
+        log.info(`Current proven block: ${initialProvenBlock}. Waiting for it to increase...`);
+    } catch (err) {
+        log.warn(`Error getting initial tips: ${err}. Will poll until successful.`);
+        initialProvenBlock = 0;
+    }
+    await retryUntil(async ()=>{
+        try {
+            const tips = await node.getL2Tips();
+            const currentProvenBlock = Number(tips.proven.block.number);
+            const proposedBlock = Number(tips.proposed.number);
+            log.verbose(`Chain state: proposed=${proposedBlock}, proven=${currentProvenBlock} (waiting for > ${initialProvenBlock})`);
+            if (currentProvenBlock > initialProvenBlock) {
+                log.info(`Proven block advanced from ${initialProvenBlock} to ${currentProvenBlock}.`);
+                return true;
+            }
+            return false;
+        } catch (err) {
+            log.verbose(`Error checking tips: ${err}`);
+            return false;
+        }
+    }, 'proven block to advance', timeoutSeconds, pollIntervalSeconds);
+}
 export async function getSequencers(namespace) {
     const selectors = [
         'app.kubernetes.io/name=validator',
@@ -61,9 +99,15 @@ export async function withSequencersAdmin(env, fn) {
     const sequencers = await getSequencers(namespace);
     const results = [];
     for (const sequencer of sequencers){
+        // Ensure pod is Ready before attempting port-forward.
+        await waitForResourceByName({
+            resource: 'pods',
+            name: sequencer,
+            namespace
+        });
         // Wrap port-forward + fetch in a retry to handle flaky port-forwards
         const result = await retry(async ()=>{
-            const { process, port } = await startPortForward({
+            const { process: process1, port } = await startPortForward({
                 resource: `pod/${sequencer}`,
                 namespace,
                 containerPort: adminContainerPort
@@ -75,14 +119,14 @@ export async function withSequencersAdmin(env, fn) {
                 if (statusRes.status !== 200) {
                     throw new Error(`Admin endpoint returned status ${statusRes.status}`);
                 }
-                const client = createAztecNodeAdminClient(url);
+                const client = createAztecNodeAdminClient(url, {}, undefined, env.AZTEC_ADMIN_API_KEY);
                 return {
                     result: await fn(client),
-                    process
+                    process: process1
                 };
             } catch (err) {
                 // Kill the port-forward before retrying
-                process.kill();
+                process1.kill();
                 throw err;
             }
         }, 'connect to node admin', makeBackoff([
@@ -96,11 +140,62 @@ export async function withSequencersAdmin(env, fn) {
     }
     return results;
 }
+async function getAztecImageForMigrations(namespace) {
+    const aztecDockerImage = process.env.AZTEC_DOCKER_IMAGE;
+    if (aztecDockerImage) {
+        return aztecDockerImage;
+    }
+    const { stdout } = await execAsync(`kubectl get pods -l app.kubernetes.io/name=validator -n ${namespace} -o jsonpath='{.items[0].spec.containers[?(@.name=="aztec")].image}' | cat`);
+    const image = stdout.trim().replace(/^'|'$/g, '');
+    if (!image) {
+        throw new Error(`Could not detect aztec image from validator pod in namespace ${namespace}`);
+    }
+    return image;
+}
+async function getHaDbConnectionUrl(namespace) {
+    const secretName = `${namespace}-validator-ha-db-postgres`;
+    const { stdout } = await execAsync(`kubectl get secret ${secretName} -n ${namespace} -o json`);
+    const secret = JSON.parse(stdout);
+    const data = secret?.data ?? {};
+    const decode = (value)=>value ? Buffer.from(value, 'base64').toString('utf8') : '';
+    const user = decode(data.POSTGRES_USER);
+    const password = decode(data.POSTGRES_PASSWORD);
+    const database = decode(data.POSTGRES_DB);
+    if (!user || !password || !database) {
+        throw new Error(`Missing HA DB credentials in secret ${secretName}`);
+    }
+    const host = `${namespace}-validator-ha-db-postgres.${namespace}.svc.cluster.local`;
+    return `postgresql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${host}:5432/${database}`;
+}
+export async function initHADb(namespace) {
+    const databaseUrl = await getHaDbConnectionUrl(namespace);
+    const image = await getAztecImageForMigrations(namespace);
+    const jobName = `${namespace}-validator-ha-db-migrate`;
+    await execAsync(`kubectl delete pod ${jobName} -n ${namespace} --ignore-not-found=true`).catch(()=>undefined);
+    const migrateCmd = [
+        `kubectl run ${jobName} -n ${namespace}`,
+        '--rm -i',
+        '--restart=Never',
+        `--image=${image}`,
+        `--env=DATABASE_URL=${databaseUrl}`,
+        '--command -- node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js migrate-ha-db up'
+    ].join(' ');
+    const migrateCmdForLog = migrateCmd.replace(/--env=DATABASE_URL=\S+/, '--env=DATABASE_URL=<redacted>');
+    await retry(async ()=>{
+        logger.info(`command: ${migrateCmdForLog}`);
+        await execAsync(migrateCmd);
+    }, 'run HA DB migrations', makeBackoff([
+        1,
+        2,
+        4,
+        8,
+        16
+    ]), logger, true);
+}
 /**
- * Enables or disables probabilistic transaction dropping on validators and waits for rollout.
- * Wired to env vars P2P_DROP_TX and P2P_DROP_TX_CHANCE via Helm values.
- */ export async function setValidatorTxDrop({ namespace, enabled, probability, logger: log }) {
-    const drop = enabled ? 'true' : 'false';
+ * Sets probabilistic transaction dropping on validators and waits for rollout.
+ * Use probability=0 to disable. Wired to env var P2P_DROP_TX_CHANCE via Helm values.
+ */ export async function setValidatorTxDrop({ namespace, probability, logger: log }) {
     const prob = String(probability);
     const selectors = [
         'app.kubernetes.io/name=validator',
@@ -115,7 +210,7 @@ export async function withSequencersAdmin(env, fn) {
             if (names.length === 0) {
                 continue;
             }
-            const cmd = `kubectl set env statefulset -l ${selector} -n ${namespace} P2P_DROP_TX=${drop} P2P_DROP_TX_CHANCE=${prob}`;
+            const cmd = `kubectl set env statefulset -l ${selector} -n ${namespace} P2P_DROP_TX_CHANCE=${prob}`;
             log.info(`command: ${cmd}`);
             await execAsync(cmd);
             updated = true;
@@ -195,20 +290,23 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
  *        Defaults to false, which preserves the existing storage.
  */ export async function rollAztecPods(namespace, clearState = false) {
     // Pod components use 'validator', but StatefulSets and PVCs use 'sequencer-node' for validators
+    // RPC nodes have nodeType='rpc-node' in Helm values, so their component label is 'rpc-node' (not 'rpc')
     const podComponents = [
         'p2p-bootstrap',
         'prover-node',
         'prover-broker',
         'prover-agent',
         'sequencer-node',
-        'rpc'
+        'rpc-node',
+        'validator-ha-db'
     ];
     const pvcComponents = [
         'p2p-bootstrap',
         'prover-node',
         'prover-broker',
         'sequencer-node',
-        'rpc'
+        'rpc-node',
+        'validator-ha-db'
     ];
     // StatefulSet components that need to be scaled down before PVC deletion
     // Note: validators use 'sequencer-node' as component label, not 'validator'
@@ -217,20 +315,27 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
         'prover-node',
         'prover-broker',
         'sequencer-node',
-        'rpc'
+        'rpc-node',
+        'validator-ha-db'
     ];
     if (clearState) {
         // To delete PVCs, we must first scale down StatefulSets so pods release the volumes
         // Otherwise PVC deletion will hang waiting for pods to terminate
-        // First, save original replica counts
+        // Save original replica counts for all StatefulSets
         const originalReplicas = new Map();
         for (const component of statefulSetComponents){
             try {
-                const getCmd = `kubectl get statefulset -l app.kubernetes.io/component=${component} -n ${namespace} -o jsonpath='{.items[0].spec.replicas}'`;
+                // Get all StatefulSets that match the component label
+                const getCmd = `kubectl get statefulset -l app.kubernetes.io/component=${component} -n ${namespace} -o json`;
                 const { stdout } = await execAsync(getCmd);
-                const replicas = parseInt(stdout.replace(/'/g, '').trim(), 10);
-                if (!isNaN(replicas) && replicas > 0) {
-                    originalReplicas.set(component, replicas);
+                const result = JSON.parse(stdout);
+                for (const sts of result.items || []){
+                    const name = sts.metadata.name;
+                    const replicas = sts.spec.replicas ?? 1;
+                    if (replicas > 0) {
+                        originalReplicas.set(name, replicas);
+                        logger.debug(`Saved replica count for StatefulSet ${name}: ${replicas}`);
+                    }
                 }
             } catch  {
             // Component might not exist, continue
@@ -247,25 +352,79 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
                 logger.verbose(`Scale down ${component} skipped: ${e}`);
             }
         }
-        // Wait for pods to terminate
-        await sleep(15 * 1000);
+        // Wait for all pods to fully terminate before deleting PVCs.
+        // terminationGracePeriodSeconds default is 30s.
+        logger.info('Waiting for pods to fully terminate before deleting PVCs...');
+        for (const component of statefulSetComponents){
+            try {
+                // Wait for all pods with this component label to be deleted
+                const waitCmd = `kubectl wait pods -l app.kubernetes.io/component=${component} --for=delete -n ${namespace} --timeout=2m`;
+                logger.info(`command: ${waitCmd}`);
+                await execAsync(waitCmd);
+            } catch (e) {
+                logger.verbose(`Wait for pod deletion ${component} skipped: ${e}`);
+            }
+        }
+        // Extra buffer to ensure PVC protection finalizers are cleared
+        await sleep(5 * 1000);
         // Now delete PVCs (they should no longer be in use)
         for (const component of pvcComponents){
-            await deleteResourceByLabel({
-                resource: 'persistentvolumeclaims',
-                namespace: namespace,
-                label: `app.kubernetes.io/component=${component}`
-            });
+            try {
+                await deleteResourceByLabel({
+                    resource: 'persistentvolumeclaims',
+                    namespace: namespace,
+                    label: `app.kubernetes.io/component=${component}`
+                });
+            } catch (e) {
+                logger.warn(`Failed to delete PVCs for ${component}: ${e}`);
+            }
         }
-        // Scale StatefulSets back up to original replica counts
-        for (const component of statefulSetComponents){
-            const replicas = originalReplicas.get(component) ?? 1;
+        // Verify PVCs are deleted
+        for (const component of pvcComponents){
+            try {
+                const waitCmd = `kubectl wait pvc -l app.kubernetes.io/component=${component} --for=delete -n ${namespace} --timeout=2m`;
+                logger.info(`command: ${waitCmd}`);
+                await execAsync(waitCmd);
+            } catch (e) {
+                logger.verbose(`Wait for PVC deletion ${component} skipped: ${e}`);
+            }
+        }
+        const haDbStatefulSets = [
+            ...originalReplicas.entries()
+        ].filter(([name])=>name.includes('validator-ha-db'));
+        const otherStatefulSets = [
+            ...originalReplicas.entries()
+        ].filter(([name])=>!name.includes('validator-ha-db'));
+        // Bring up HA DB first so we can run migrations before validators start
+        for (const [stsName, replicas] of haDbStatefulSets){
+            try {
+                const scaleCmd = `kubectl scale statefulset ${stsName} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+                logger.info(`command: ${scaleCmd}`);
+                await execAsync(scaleCmd);
+            } catch (e) {
+                logger.verbose(`Scale up ${stsName} skipped: ${e}`);
+            }
+        }
+        if (haDbStatefulSets.length > 0) {
+            try {
+                await waitForStatefulSetsReady({
+                    namespace,
+                    label: 'app.kubernetes.io/component=validator-ha-db',
+                    timeoutSeconds: 600
+                });
+                await initHADb(namespace);
+            } catch (e) {
+                logger.warn(`HA DB migration step skipped or failed: ${e}`);
+            }
+        }
+        // Scale remaining StatefulSets back up to original replica counts (by name, not label)
+        for (const [stsName, replicas] of otherStatefulSets){
             try {
-                const scaleCmd = `kubectl scale statefulset -l app.kubernetes.io/component=${component} -n ${namespace} --replicas=${replicas} --timeout=2m`;
+                const scaleCmd = `kubectl scale statefulset ${stsName} -n ${namespace} --replicas=${replicas} --timeout=2m`;
                 logger.info(`command: ${scaleCmd}`);
                 await execAsync(scaleCmd);
             } catch (e) {
-                logger.verbose(`Scale up ${component} skipped: ${e}`);
+                logger.verbose(`Scale up ${stsName} skipped: ${e}`);
             }
         }
     } else {
@@ -279,8 +438,20 @@ export async function enableValidatorDynamicBootNode(instanceName, namespace, sp
         }
     }
     await sleep(10 * 1000);
-    // Wait for pods to come back
-    for (const component of podComponents){
+    // Wait for StatefulSets to have all replicas ready.
+    for (const component of statefulSetComponents){
+        try {
+            await waitForStatefulSetsReady({
+                namespace,
+                label: `app.kubernetes.io/component=${component}`,
+                timeoutSeconds: 600
+            });
+        } catch (e) {
+            logger.warn(`StatefulSet component ${component} may not be fully ready: ${e}`);
+        }
+    }
+    const nonStatefulSetComponents = podComponents.filter((c)=>!statefulSetComponents.includes(c));
+    for (const component of nonStatefulSetComponents){
         await waitForResourceByLabel({
             resource: 'pods',
             namespace: namespace,

package/dest/spartan/utils/scripts.d.ts

@@ -1,4 +1,10 @@
 import type { Logger } from '@aztec/foundation/log';
+/** Result from running a script */
+export type ScriptResult = {
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+};
 /**
  * Returns the absolute path to the git repository root
  */
@@ -9,8 +15,16 @@ export declare function getAztecBin(): string;
  * @param args - The arguments to pass to the Aztec binary
  * @param logger - The logger to use
  * @param env - Optional environment variables to set for the process
- * @returns The exit code of the Aztec binary
+ * @returns The exit code, stdout, and stderr of the Aztec binary
  */
-export declare function runAztecBin(args: string[], logger: Logger, env?: Record<string, string>): Promise<number>;
-export declare function runProjectScript(script: string, args: string[], logger: Logger, env?: Record<string, string>): Promise<number>;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2NyaXB0cy5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL3NwYXJ0YW4vdXRpbHMvc2NyaXB0cy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQTRCcEQ7O0dBRUc7QUFDSCx3QkFBZ0IsaUJBQWlCLElBQUksTUFBTSxDQVcxQztBQUVELHdCQUFnQixXQUFXLFdBRTFCO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsd0JBQWdCLFdBQVcsQ0FBQyxJQUFJLEVBQUUsTUFBTSxFQUFFLEVBQUUsTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLENBQUMsRUFBRSxNQUFNLENBQUMsTUFBTSxFQUFFLE1BQU0sQ0FBQyxtQkFFdkY7QUFFRCx3QkFBZ0IsZ0JBQWdCLENBQUMsTUFBTSxFQUFFLE1BQU0sRUFBRSxJQUFJLEVBQUUsTUFBTSxFQUFFLEVBQUUsTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLENBQUMsRUFBRSxNQUFNLENBQUMsTUFBTSxFQUFFLE1BQU0sQ0FBQyxtQkFHNUcifQ==
+export declare function runAztecBin(args: string[], logger: Logger, env?: Record<string, string>): Promise<ScriptResult>;
+/**
+ * Runs a script from the project root
+ * @param script - The path to the script, relative to the project root
+ * @param args - The arguments to pass to the script
+ * @param logger - The logger to use
+ * @param env - Optional environment variables to set for the process
+ * @returns The exit code, stdout, and stderr of the script
+ */
+export declare function runProjectScript(script: string, args: string[], logger: Logger, env?: Record<string, string>): Promise<ScriptResult>;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2NyaXB0cy5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL3NwYXJ0YW4vdXRpbHMvc2NyaXB0cy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUtwRCxtQ0FBbUM7QUFDbkMsTUFBTSxNQUFNLFlBQVksR0FBRztJQUN6QixRQUFRLEVBQUUsTUFBTSxDQUFDO0lBQ2pCLE1BQU0sRUFBRSxNQUFNLENBQUM7SUFDZixNQUFNLEVBQUUsTUFBTSxDQUFDO0NBQ2hCLENBQUM7QUF5Q0Y7O0dBRUc7QUFDSCx3QkFBZ0IsaUJBQWlCLElBQUksTUFBTSxDQVcxQztBQUVELHdCQUFnQixXQUFXLFdBRTFCO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsd0JBQWdCLFdBQVcsQ0FBQyxJQUFJLEVBQUUsTUFBTSxFQUFFLEVBQUUsTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLENBQUMsRUFBRSxNQUFNLENBQUMsTUFBTSxFQUFFLE1BQU0sQ0FBQyxHQUFHLE9BQU8sQ0FBQyxZQUFZLENBQUMsQ0FFL0c7QUFFRDs7Ozs7OztHQU9HO0FBQ0gsd0JBQWdCLGdCQUFnQixDQUM5QixNQUFNLEVBQUUsTUFBTSxFQUNkLElBQUksRUFBRSxNQUFNLEVBQUUsRUFDZCxNQUFNLEVBQUUsTUFBTSxFQUNkLEdBQUcsQ0FBQyxFQUFFLE1BQU0sQ0FBQyxNQUFNLEVBQUUsTUFBTSxDQUFDLEdBQzNCLE9BQU8sQ0FBQyxZQUFZLENBQUMsQ0FHdkIifQ==

package/dest/spartan/utils/scripts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scripts.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/scripts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AA4BpD;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,MAAM,CAW1C;AAED,wBAAgB,WAAW,WAE1B;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,mBAEvF;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,mBAG5G"}
+{"version":3,"file":"scripts.d.ts","sourceRoot":"","sources":["../../../src/spartan/utils/scripts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAKpD,mCAAmC;AACnC,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAyCF;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,MAAM,CAW1C;AAED,wBAAgB,WAAW,WAE1B;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC,CAE/G;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,MAAM,EACd,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC3B,OAAO,CAAC,YAAY,CAAC,CAGvB"}

package/dest/spartan/utils/scripts.js

@@ -4,7 +4,7 @@ import path from 'path';
  * @param scriptPath - The path to the script, relative to the project root
  * @param args - The arguments to pass to the script
  * @param logger - The logger to use
- * @returns The exit code of the script
+ * @returns The exit code, stdout, and stderr of the script
  */ function runScript(scriptPath, args, logger, env) {
     const childProcess = spawn(scriptPath, args, {
         stdio: [
@@ -17,13 +17,21 @@ import path from 'path';
             ...env
         } : process.env
     });
+    const stdoutChunks = [];
+    const stderrChunks = [];
     return new Promise((resolve, reject)=>{
-        childProcess.on('close', (code)=>resolve(code ?? 0));
+        childProcess.on('close', (code)=>resolve({
+                exitCode: code ?? 0,
+                stdout: Buffer.concat(stdoutChunks).toString(),
+                stderr: Buffer.concat(stderrChunks).toString()
+            }));
         childProcess.on('error', reject);
         childProcess.stdout?.on('data', (data)=>{
+            stdoutChunks.push(data);
             logger.info(data.toString());
         });
         childProcess.stderr?.on('data', (data)=>{
+            stderrChunks.push(data);
             logger.error(data.toString());
         });
     });
@@ -53,14 +61,21 @@ export function getAztecBin() {
  * @param args - The arguments to pass to the Aztec binary
  * @param logger - The logger to use
  * @param env - Optional environment variables to set for the process
- * @returns The exit code of the Aztec binary
+ * @returns The exit code, stdout, and stderr of the Aztec binary
  */ export function runAztecBin(args, logger, env) {
     return runScript('node', [
         getAztecBin(),
         ...args
     ], logger, env);
 }
-export function runProjectScript(script, args, logger, env) {
+/**
+ * Runs a script from the project root
+ * @param script - The path to the script, relative to the project root
+ * @param args - The arguments to pass to the script
+ * @param logger - The logger to use
+ * @param env - Optional environment variables to set for the process
+ * @returns The exit code, stdout, and stderr of the script
+ */ export function runProjectScript(script, args, logger, env) {
     const scriptPath = script.startsWith('/') ? script : path.join(getGitProjectRoot(), script);
     return runScript(scriptPath, args, logger, env);
 }

package/dest/test-wallet/test_wallet.d.ts

@@ -0,0 +1,83 @@
+import { type Account, type AccountContract } from '@aztec/aztec.js/account';
+import { type CallIntent, type ContractFunctionInteractionCallIntent, type IntentInnerHash, SetPublicAuthwitContractInteraction } from '@aztec/aztec.js/authorization';
+import type { AztecNode } from '@aztec/aztec.js/node';
+import { AccountManager, type SendOptions } from '@aztec/aztec.js/wallet';
+import { Fq, Fr } from '@aztec/foundation/curves/bn254';
+import type { AccessScopes, NotesFilter } from '@aztec/pxe/client/lazy';
+import { type PXEConfig } from '@aztec/pxe/config';
+import { PXE, type PXECreationOptions } from '@aztec/pxe/server';
+import { AuthWitness } from '@aztec/stdlib/auth-witness';
+import { AztecAddress } from '@aztec/stdlib/aztec-address';
+import type { NoteDao } from '@aztec/stdlib/note';
+import type { BlockHeader, TxHash, TxReceipt, TxSimulationResult } from '@aztec/stdlib/tx';
+import { ExecutionPayload } from '@aztec/stdlib/tx';
+import { BaseWallet, type FeeOptions } from '@aztec/wallet-sdk/base-wallet';
+import { AztecNodeProxy, ProvenTx } from './utils.js';
+/**
+ * Data for generating an account.
+ */
+export interface AccountData {
+    secret: Fr;
+    salt: Fr;
+    contract: AccountContract;
+}
+/**
+ * Wallet implementation that stores accounts in memory and provides extra debugging
+ * utilities
+ * It is intended to be used in e2e tests.
+ */
+export declare class TestWallet extends BaseWallet {
+    private readonly nodeRef;
+    constructor(pxe: PXE, nodeRef: AztecNodeProxy);
+    static create(node: AztecNode, overridePXEConfig?: Partial<PXEConfig>, options?: PXECreationOptions): Promise<TestWallet>;
+    /**
+     * Updates the underlying node that this wallet and its PXE communicate with.
+     * @param node - The new AztecNode to forward all calls to.
+     */
+    updateNode(node: AztecNode): void;
+    createSchnorrAccount(secret: Fr, salt: Fr, signingKey?: Fq): Promise<AccountManager>;
+    createECDSARAccount(secret: Fr, salt: Fr, signingKey: Buffer): Promise<AccountManager>;
+    createECDSAKAccount(secret: Fr, salt: Fr, signingKey: Buffer): Promise<AccountManager>;
+    getFakeAccountDataFor(address: AztecAddress): Promise<{
+        account: import("@aztec/aztec.js/account").BaseAccount;
+        instance: import("@aztec/stdlib/contract").ContractInstanceWithAddress;
+        artifact: import("@aztec/stdlib/abi").ContractArtifact;
+    }>;
+    protected accounts: Map<string, Account>;
+    /**
+     * Toggle for running "simulated simulations" when calling simulateTx.
+     *
+     * When this flag is true, simulateViaEntrypoint constructs a request using a fake account
+     * (and accepts contract overrides on the input) and the PXE emulates kernel effects without
+     * generating kernel witnesses. When false, simulateViaEntrypoint defers to the standard
+     * simulation path via the real account entrypoint.
+     */
+    private simulatedSimulations;
+    enableSimulatedSimulations(): void;
+    disableSimulatedSimulations(): void;
+    setMinFeePadding(value?: number): void;
+    protected getAccountFromAddress(address: AztecAddress): Promise<Account>;
+    getAccounts(): Promise<{
+        alias: string;
+        item: AztecAddress;
+    }[]>;
+    createAccount(accountData?: AccountData): Promise<AccountManager>;
+    lookupValidity(onBehalfOf: AztecAddress, intent: IntentInnerHash | CallIntent | ContractFunctionInteractionCallIntent, witness: AuthWitness): Promise<{
+        isValidInPrivate: boolean;
+        isValidInPublic: boolean;
+    }>;
+    setPublicAuthWit(from: AztecAddress, messageHashOrIntent: Fr | IntentInnerHash | CallIntent | ContractFunctionInteractionCallIntent, authorized: boolean): Promise<SetPublicAuthwitContractInteraction>;
+    createAuthWit(from: AztecAddress, intent: IntentInnerHash | CallIntent | ContractFunctionInteractionCallIntent): Promise<AuthWitness>;
+    /**
+     * Override simulateViaEntrypoint to use fake accounts for kernelless simulation
+     * when simulatedSimulations is enabled. Otherwise falls through to the real entrypoint path.
+     */
+    protected simulateViaEntrypoint(executionPayload: ExecutionPayload, from: AztecAddress, feeOptions: FeeOptions, scopes: AccessScopes, skipTxValidation?: boolean, skipFeeEnforcement?: boolean): Promise<TxSimulationResult>;
+    proveTx(exec: ExecutionPayload, opts: Omit<SendOptions, 'wait'>): Promise<ProvenTx>;
+    getTxReceipt(txHash: TxHash): Promise<TxReceipt>;
+    getNotes(filter: NotesFilter): Promise<NoteDao[]>;
+    getSyncedBlockHeader(): Promise<BlockHeader>;
+    sync(): Promise<void>;
+    stop(): Promise<void>;
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidGVzdF93YWxsZXQuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy90ZXN0LXdhbGxldC90ZXN0X3dhbGxldC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFHQSxPQUFPLEVBQUUsS0FBSyxPQUFPLEVBQUUsS0FBSyxlQUFlLEVBQXFCLE1BQU0seUJBQXlCLENBQUM7QUFDaEcsT0FBTyxFQUNMLEtBQUssVUFBVSxFQUNmLEtBQUsscUNBQXFDLEVBQzFDLEtBQUssZUFBZSxFQUNwQixtQ0FBbUMsRUFJcEMsTUFBTSwrQkFBK0IsQ0FBQztBQUN2QyxPQUFPLEtBQUssRUFBRSxTQUFTLEVBQUUsTUFBTSxzQkFBc0IsQ0FBQztBQUN0RCxPQUFPLEVBQUUsY0FBYyxFQUFFLEtBQUssV0FBVyxFQUFFLE1BQU0sd0JBQXdCLENBQUM7QUFFMUUsT0FBTyxFQUFFLEVBQUUsRUFBRSxFQUFFLEVBQUUsTUFBTSxnQ0FBZ0MsQ0FBQztBQUV4RCxPQUFPLEtBQUssRUFBRSxZQUFZLEVBQUUsV0FBVyxFQUFFLE1BQU0sd0JBQXdCLENBQUM7QUFDeEUsT0FBTyxFQUFFLEtBQUssU0FBUyxFQUFnQixNQUFNLG1CQUFtQixDQUFDO0FBQ2pFLE9BQU8sRUFBRSxHQUFHLEVBQUUsS0FBSyxrQkFBa0IsRUFBYSxNQUFNLG1CQUFtQixDQUFDO0FBQzVFLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSw0QkFBNEIsQ0FBQztBQUN6RCxPQUFPLEVBQUUsWUFBWSxFQUFFLE1BQU0sNkJBQTZCLENBQUM7QUFHM0QsT0FBTyxLQUFLLEVBQUUsT0FBTyxFQUFFLE1BQU0sb0JBQW9CLENBQUM7QUFDbEQsT0FBTyxLQUFLLEVBQUUsV0FBVyxFQUFFLE1BQU0sRUFBRSxTQUFTLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQztBQUMzRixPQUFPLEVBQUUsZ0JBQWdCLEVBQTBCLE1BQU0sa0JBQWtCLENBQUM7QUFDNUUsT0FBTyxFQUFFLFVBQVUsRUFBRSxLQUFLLFVBQVUsRUFBRSxNQUFNLCtCQUErQixDQUFDO0FBRTVFLE9BQU8sRUFBRSxjQUFjLEVBQUUsUUFBUSxFQUFFLE1BQU0sWUFBWSxDQUFDO0FBRXREOztHQUVHO0FBQ0gsTUFBTSxXQUFXLFdBQVc7SUFDMUIsTUFBTSxFQUFFLEVBQUUsQ0FBQztJQUNYLElBQUksRUFBRSxFQUFFLENBQUM7SUFDVCxRQUFRLEVBQUUsZUFBZSxDQUFDO0NBQzNCO0FBRUQ7Ozs7R0FJRztBQUNILHFCQUFhLFVBQVcsU0FBUSxVQUFVO0lBR3RDLE9BQU8sQ0FBQyxRQUFRLENBQUMsT0FBTztJQUYxQixZQUNFLEdBQUcsRUFBRSxHQUFHLEVBQ1MsT0FBTyxFQUFFLGNBQWMsRUFHekM7SUFFRCxPQUFhLE1BQU0sQ0FDakIsSUFBSSxFQUFFLFNBQVMsRUFDZixpQkFBaUIsQ0FBQyxFQUFFLE9BQU8sQ0FBQyxTQUFTLENBQUMsRUFDdEMsT0FBTyxHQUFFLGtCQUFvQyxHQUM1QyxPQUFPLENBQUMsVUFBVSxDQUFDLENBUXJCO0lBRUQ7OztPQUdHO0lBQ0gsVUFBVSxDQUFDLElBQUksRUFBRSxTQUFTLEdBQUcsSUFBSSxDQUVoQztJQUVELG9CQUFvQixDQUFDLE1BQU0sRUFBRSxFQUFFLEVBQUUsSUFBSSxFQUFFLEVBQUUsRUFBRSxVQUFVLENBQUMsRUFBRSxFQUFFLEdBQUcsT0FBTyxDQUFDLGNBQWMsQ0FBQyxDQVFuRjtJQUVELG1CQUFtQixDQUFDLE1BQU0sRUFBRSxFQUFFLEVBQUUsSUFBSSxFQUFFLEVBQUUsRUFBRSxVQUFVLEVBQUUsTUFBTSxHQUFHLE9BQU8sQ0FBQyxjQUFjLENBQUMsQ0FPckY7SUFFRCxtQkFBbUIsQ0FBQyxNQUFNLEVBQUUsRUFBRSxFQUFFLElBQUksRUFBRSxFQUFFLEVBQUUsVUFBVSxFQUFFLE1BQU0sR0FBRyxPQUFPLENBQUMsY0FBYyxDQUFDLENBT3JGO0lBRUsscUJBQXFCLENBQUMsT0FBTyxFQUFFLFlBQVk7Ozs7T0FtQmhEO0lBQ0QsU0FBUyxDQUFDLFFBQVEsRUFBRSxHQUFHLENBQUMsTUFBTSxFQUFFLE9BQU8sQ0FBQyxDQUFhO0lBRXJEOzs7Ozs7O09BT0c7SUFDSCxPQUFPLENBQUMsb0JBQW9CLENBQVM7SUFFckMsMEJBQTBCLFNBRXpCO0lBRUQsMkJBQTJCLFNBRTFCO0lBRUQsZ0JBQWdCLENBQUMsS0FBSyxDQUFDLEVBQUUsTUFBTSxRQUU5QjtJQUVELFNBQVMsQ0FBQyxxQkFBcUIsQ0FBQyxPQUFPLEVBQUUsWUFBWSxHQUFHLE9BQU8sQ0FBQyxPQUFPLENBQUMsQ0FhdkU7SUFFRCxXQUFXOzs7U0FFVjtJQUVLLGFBQWEsQ0FBQyxXQUFXLENBQUMsRUFBRSxXQUFXLEdBQUcsT0FBTyxDQUFDLGNBQWMsQ0FBQyxDQWV0RTtJQUVELGNBQWMsQ0FDWixVQUFVLEVBQUUsWUFBWSxFQUN4QixNQUFNLEVBQUUsZUFBZSxHQUFHLFVBQVUsR0FBRyxxQ0FBcUMsRUFDNUUsT0FBTyxFQUFFLFdBQVcsR0FDbkIsT0FBTyxDQUFDO1FBQ1QsZ0JBQWdCLEVBQUUsT0FBTyxDQUFDO1FBQzFCLGVBQWUsRUFBRSxPQUFPLENBQUM7S0FDMUIsQ0FBQyxDQUVEO0lBRU0sZ0JBQWdCLENBQ3JCLElBQUksRUFBRSxZQUFZLEVBQ2xCLG1CQUFtQixFQUFFLEVBQUUsR0FBRyxlQUFlLEdBQUcsVUFBVSxHQUFHLHFDQUFxQyxFQUM5RixVQUFVLEVBQUUsT0FBTyxHQUNsQixPQUFPLENBQUMsbUNBQW1DLENBQUMsQ0FFOUM7SUFFcUIsYUFBYSxDQUNqQyxJQUFJLEVBQUUsWUFBWSxFQUNsQixNQUFNLEVBQUUsZUFBZSxHQUFHLFVBQVUsR0FBRyxxQ0FBcUMsR0FDM0UsT0FBTyxDQUFDLFdBQVcsQ0FBQyxDQWN0QjtJQUVEOzs7T0FHRztJQUNILFVBQXlCLHFCQUFxQixDQUM1QyxnQkFBZ0IsRUFBRSxnQkFBZ0IsRUFDbEMsSUFBSSxFQUFFLFlBQVksRUFDbEIsVUFBVSxFQUFFLFVBQVUsRUFDdEIsTUFBTSxFQUFFLFlBQVksRUFDcEIsZ0JBQWdCLENBQUMsRUFBRSxPQUFPLEVBQzFCLGtCQUFrQixDQUFDLEVBQUUsT0FBTyxHQUMzQixPQUFPLENBQUMsa0JBQWtCLENBQUMsQ0F1QzdCO0lBRUssT0FBTyxDQUFDLElBQUksRUFBRSxnQkFBZ0IsRUFBRSxJQUFJLEVBQUUsSUFBSSxDQUFDLFdBQVcsRUFBRSxNQUFNLENBQUMsR0FBRyxPQUFPLENBQUMsUUFBUSxDQUFDLENBVXhGO0lBRUQsWUFBWSxDQUFDLE1BQU0sRUFBRSxNQUFNLEdBQUcsT0FBTyxDQUFDLFNBQVMsQ0FBQyxDQUUvQztJQUVELFFBQVEsQ0FBQyxNQUFNLEVBQUUsV0FBVyxHQUFHLE9BQU8sQ0FBQyxPQUFPLEVBQUUsQ0FBQyxDQUVoRDtJQUVELG9CQUFvQixJQUFJLE9BQU8sQ0FBQyxXQUFXLENBQUMsQ0FFM0M7SUFFRCxJQUFJLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQUVwQjtJQUVELElBQUksSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBRXBCO0NBQ0YifQ==

package/dest/test-wallet/test_wallet.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test_wallet.d.ts","sourceRoot":"","sources":["../../src/test-wallet/test_wallet.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,OAAO,EAAE,KAAK,eAAe,EAAqB,MAAM,yBAAyB,CAAC;AAChG,OAAO,EACL,KAAK,UAAU,EACf,KAAK,qCAAqC,EAC1C,KAAK,eAAe,EACpB,mCAAmC,EAIpC,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAE1E,OAAO,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,gCAAgC,CAAC;AAExD,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACxE,OAAO,EAAE,KAAK,SAAS,EAAgB,MAAM,mBAAmB,CAAC;AACjE,OAAO,EAAE,GAAG,EAAE,KAAK,kBAAkB,EAAa,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAG3D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAC3F,OAAO,EAAE,gBAAgB,EAA0B,MAAM,kBAAkB,CAAC;AAC5E,OAAO,EAAE,UAAU,EAAE,KAAK,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAE5E,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtD;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,EAAE,CAAC;IACX,IAAI,EAAE,EAAE,CAAC;IACT,QAAQ,EAAE,eAAe,CAAC;CAC3B;AAED;;;;GAIG;AACH,qBAAa,UAAW,SAAQ,UAAU;IAGtC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAF1B,YACE,GAAG,EAAE,GAAG,EACS,OAAO,EAAE,cAAc,EAGzC;IAED,OAAa,MAAM,CACjB,IAAI,EAAE,SAAS,EACf,iBAAiB,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,EACtC,OAAO,GAAE,kBAAoC,GAC5C,OAAO,CAAC,UAAU,CAAC,CAQrB;IAED;;;OAGG;IACH,UAAU,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI,CAEhC;IAED,oBAAoB,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,cAAc,CAAC,CAQnF;IAED,mBAAmB,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAOrF;IAED,mBAAmB,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAOrF;IAEK,qBAAqB,CAAC,OAAO,EAAE,YAAY;;;;OAmBhD;IACD,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAa;IAErD;;;;;;;OAOG;IACH,OAAO,CAAC,oBAAoB,CAAS;IAErC,0BAA0B,SAEzB;IAED,2BAA2B,SAE1B;IAED,gBAAgB,CAAC,KAAK,CAAC,EAAE,MAAM,QAE9B;IAED,SAAS,CAAC,qBAAqB,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,CAavE;IAED,WAAW;;;SAEV;IAEK,aAAa,CAAC,WAAW,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC,CAetE;IAED,cAAc,CACZ,UAAU,EAAE,YAAY,EACxB,MAAM,EAAE,eAAe,GAAG,UAAU,GAAG,qCAAqC,EAC5E,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC;QACT,gBAAgB,EAAE,OAAO,CAAC;QAC1B,eAAe,EAAE,OAAO,CAAC;KAC1B,CAAC,CAED;IAEM,gBAAgB,CACrB,IAAI,EAAE,YAAY,EAClB,mBAAmB,EAAE,EAAE,GAAG,eAAe,GAAG,UAAU,GAAG,qCAAqC,EAC9F,UAAU,EAAE,OAAO,GAClB,OAAO,CAAC,mCAAmC,CAAC,CAE9C;IAEqB,aAAa,CACjC,IAAI,EAAE,YAAY,EAClB,MAAM,EAAE,eAAe,GAAG,UAAU,GAAG,qCAAqC,GAC3E,OAAO,CAAC,WAAW,CAAC,CActB;IAED;;;OAGG;IACH,UAAyB,qBAAqB,CAC5C,gBAAgB,EAAE,gBAAgB,EAClC,IAAI,EAAE,YAAY,EAClB,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,YAAY,EACpB,gBAAgB,CAAC,EAAE,OAAO,EAC1B,kBAAkB,CAAC,EAAE,OAAO,GAC3B,OAAO,CAAC,kBAAkB,CAAC,CAuC7B;IAEK,OAAO,CAAC,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,CAUxF;IAED,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAE/C;IAED,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAEhD;IAED,oBAAoB,IAAI,OAAO,CAAC,WAAW,CAAC,CAE3C;IAED,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAEpB;IAED,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAEpB;CACF"}