@aztec/cli 0.0.1-commit.3469e52 → 0.0.1-commit.350e0a4d

package/src/cmds/validator_keys/shared.ts

@@ -1,4 +1,5 @@
 import { prettyPrintJSON } from '@aztec/cli/utils';
+import { asyncPool } from '@aztec/foundation/async-pool';
 import { deriveBlsPrivateKey } from '@aztec/foundation/crypto/bls';
 import { createBn254Keystore } from '@aztec/foundation/crypto/bls/bn254_keystore';
 import { computeBn254G1PublicKeyCompressed } from '@aztec/foundation/crypto/bn254';
@@ -10,12 +11,60 @@ import type { AztecAddress } from '@aztec/stdlib/aztec-address';
 import { Wallet } from '@ethersproject/wallet';
 import { constants as fsConstants, mkdirSync } from 'fs';
 import { access, writeFile } from 'fs/promises';
-import { homedir } from 'os';
+import { availableParallelism, homedir } from 'os';
 import { dirname, isAbsolute, join } from 'path';
 import { mnemonicToAccount } from 'viem/accounts';
+import { Worker } from 'worker_threads';
 
 import { defaultBlsPath } from './utils.js';
 
+type EthJsonV3WorkerResult = { json: string } | { error: { message: string; name?: string; stack?: string } };
+
+// ethers' default scrypt parameters use substantial memory, so keep worker fan-out bounded.
+const maxEthKeystoreWorkers = Math.max(1, Math.min(4, availableParallelism()));
+
+function deserializeWorkerError(error: { message: string; name?: string; stack?: string }) {
+  const result = new Error(error.message);
+  result.name = error.name ?? result.name;
+  result.stack = error.stack;
+  return result;
+}
+
+function encryptEthJsonV3InWorker(privateKeyHex: string, password: string): Promise<string> {
+  return new Promise<string>((resolve, reject) => {
+    const worker = new Worker(new URL('./eth_json_v3_worker.js', import.meta.url), {
+      workerData: { privateKeyHex, password },
+    });
+    let settled = false;
+
+    worker.once('message', (result: EthJsonV3WorkerResult) => {
+      settled = true;
+
+      if ('json' in result) {
+        resolve(result.json);
+      } else {
+        reject(deserializeWorkerError(result.error));
+      }
+    });
+    worker.once('error', error => {
+      settled = true;
+      reject(error);
+    });
+    worker.once('exit', code => {
+      if (!settled) {
+        reject(new Error(`ETH JSON V3 worker stopped with exit code ${code}`));
+      }
+    });
+  });
+}
+
+async function encryptEthJsonV3(privateKeyHex: string, password: string): Promise<string> {
+  if (process.env.JEST_WORKER_ID !== undefined) {
+    return await new Wallet(privateKeyHex).encrypt(password);
+  }
+  return await encryptEthJsonV3InWorker(privateKeyHex, password);
+}
+
 export type ValidatorSummary = { attesterEth?: string; attesterBls?: string; publisherEth?: string[] };
 
 export type BuildValidatorsInput = {
@@ -228,7 +277,7 @@ export async function writeBn254BlsKeystore(
 ): Promise<string> {
   mkdirSync(outDir, { recursive: true });
 
-  const keystore = createBn254Keystore(password, privateKeyHex, pubkeyHex, derivationPath);
+  const keystore = await createBn254Keystore(password, privateKeyHex, pubkeyHex, derivationPath);
 
   const safeBase = fileNameBase.replace(/[^a-zA-Z0-9_-]/g, '_');
   const outPath = join(outDir, `keystore-${safeBase}.json`);
@@ -241,28 +290,29 @@ export async function writeBlsBn254ToFile(
   validators: ValidatorKeyStore[],
   options: { outDir: string; password: string; blsPath?: string },
 ): Promise<void> {
-  for (let i = 0; i < validators.length; i++) {
-    const v = validators[i];
-    if (!v || typeof v !== 'object' || !('attester' in v)) {
-      continue;
-    }
-    const att = (v as any).attester;
+  await Promise.all(
+    validators.map(async (v, i) => {
+      if (!v || typeof v !== 'object' || !('attester' in v)) {
+        return;
+      }
+      const att = (v as any).attester;
 
-    // Shapes: { bls: <hex> } or { eth: <ethAccount>, bls?: <hex> } or plain EthAccount
-    const blsKey: string | undefined = typeof att === 'object' && 'bls' in att ? (att as any).bls : undefined;
-    if (!blsKey || typeof blsKey !== 'string') {
-      continue;
-    }
+      // Shapes: { bls: <hex> } or { eth: <ethAccount>, bls?: <hex> } or plain EthAccount
+      const blsKey: string | undefined = typeof att === 'object' && 'bls' in att ? (att as any).bls : undefined;
+      if (!blsKey || typeof blsKey !== 'string') {
+        return;
+      }
 
-    const pub = await computeBlsPublicKeyCompressed(blsKey);
-    const path = options.blsPath ?? defaultBlsPath;
-    const fileBase = `${String(i + 1)}_${pub.slice(2, 18)}`;
-    const keystorePath = await writeBn254BlsKeystore(options.outDir, fileBase, options.password, blsKey, pub, path);
+      const pub = await computeBlsPublicKeyCompressed(blsKey);
+      const path = options.blsPath ?? defaultBlsPath;
+      const fileBase = `${String(i + 1)}_${pub.slice(2, 18)}`;
+      const keystorePath = await writeBn254BlsKeystore(options.outDir, fileBase, options.password, blsKey, pub, path);
 
-    if (typeof att === 'object') {
-      (att as any).bls = { path: keystorePath, password: options.password };
-    }
-  }
+      if (typeof att === 'object') {
+        (att as any).bls = { path: keystorePath, password: options.password };
+      }
+    }),
+  );
 }
 
 /** Writes an Ethereum JSON V3 keystore using ethers, returns absolute path */
@@ -274,8 +324,7 @@ export async function writeEthJsonV3Keystore(
 ): Promise<string> {
   const safeBase = fileNameBase.replace(/[^a-zA-Z0-9_-]/g, '_');
   mkdirSync(outDir, { recursive: true });
-  const wallet = new Wallet(privateKeyHex);
-  const json = await wallet.encrypt(password);
+  const json = await encryptEthJsonV3(privateKeyHex, password);
   const outPath = join(outDir, `keystore-eth-${safeBase}.json`);
   await writeFile(outPath, json, { encoding: 'utf-8' });
   return outPath;
@@ -286,13 +335,16 @@ export async function writeEthJsonV3ToFile(
   validators: ValidatorKeyStore[],
   options: { outDir: string; password: string },
 ): Promise<void> {
-  const maybeEncryptEth = async (account: any, label: string) => {
+  const tasks: (() => Promise<void>)[] = [];
+
+  const maybeQueueEncryptEth = (account: any, label: string, setEncryptedAccount: (account: any) => void) => {
     if (typeof account === 'string' && account.startsWith('0x') && account.length === 66) {
-      const fileBase = `${label}_${account.slice(2, 10)}`;
-      const p = await writeEthJsonV3Keystore(options.outDir, fileBase, options.password, account);
-      return { path: p, password: options.password };
+      tasks.push(async () => {
+        const fileBase = `${label}_${account.slice(2, 10)}`;
+        const path = await writeEthJsonV3Keystore(options.outDir, fileBase, options.password, account);
+        setEncryptedAccount({ path, password: options.password });
+      });
     }
-    return account;
   };
 
   for (let i = 0; i < validators.length; i++) {
@@ -304,23 +356,23 @@ export async function writeEthJsonV3ToFile(
     // attester may be string (eth), object with eth, or remote signer
     const att = (v as any).attester;
     if (typeof att === 'string') {
-      (v as any).attester = await maybeEncryptEth(att, `attester_${i + 1}`);
+      maybeQueueEncryptEth(att, `attester_${i + 1}`, account => ((v as any).attester = account));
     } else if (att && typeof att === 'object' && 'eth' in att) {
-      (att as any).eth = await maybeEncryptEth((att as any).eth, `attester_${i + 1}`);
+      maybeQueueEncryptEth((att as any).eth, `attester_${i + 1}`, account => ((att as any).eth = account));
     }
 
     // publisher can be single or array
     if ('publisher' in v) {
       const pub = (v as any).publisher;
       if (Array.isArray(pub)) {
-        const out: any[] = [];
         for (let j = 0; j < pub.length; j++) {
-          out.push(await maybeEncryptEth(pub[j], `publisher_${i + 1}_${j + 1}`));
+          maybeQueueEncryptEth(pub[j], `publisher_${i + 1}_${j + 1}`, account => (pub[j] = account));
         }
-        (v as any).publisher = out;
       } else if (pub !== undefined) {
-        (v as any).publisher = await maybeEncryptEth(pub, `publisher_${i + 1}`);
+        maybeQueueEncryptEth(pub, `publisher_${i + 1}`, account => ((v as any).publisher = account));
       }
     }
   }
+
+  await asyncPool(maxEthKeystoreWorkers, tasks, task => task());
 }

package/src/config/cached_fetch.ts

@@ -1,24 +1,48 @@
 import { createLogger } from '@aztec/aztec.js/log';
 
-import { mkdir, readFile, stat, writeFile } from 'fs/promises';
+import { mkdir, readFile, writeFile } from 'fs/promises';
 import { dirname } from 'path';
 
 export interface CachedFetchOptions {
-  /** Cache duration in milliseconds */
-  cacheDurationMs: number;
-  /** The cache file */
+  /** The cache file path for storing data. If not provided, no caching is performed. */
   cacheFile?: string;
+  /** Fallback max-age in milliseconds when server sends no Cache-Control header. Defaults to 5 minutes. */
+  defaultMaxAgeMs?: number;
+}
+
+/** Cache metadata stored in a sidecar .meta file alongside the data file. */
+interface CacheMeta {
+  etag?: string;
+  expiresAt: number;
+}
+
+const DEFAULT_MAX_AGE_MS = 5 * 60 * 1000; // 5 minutes
+
+/** Extracts max-age value in milliseconds from a Response's Cache-Control header. Returns undefined if not present. */
+export function parseMaxAge(response: { headers: { get(name: string): string | null } }): number | undefined {
+  const cacheControl = response.headers.get('cache-control');
+  if (!cacheControl) {
+    return undefined;
+  }
+  const match = cacheControl.match(/max-age=(\d+)/);
+  if (!match) {
+    return undefined;
+  }
+  return parseInt(match[1], 10) * 1000;
 }
 
 /**
- * Fetches data from a URL with file-based caching support.
- * This utility can be used by both remote config and bootnodes fetching.
+ * Fetches data from a URL with file-based HTTP conditional caching.
+ *
+ * Data is stored as raw JSON in the cache file (same format as the server returns).
+ * Caching metadata (ETag, expiry) is stored in a separate sidecar `.meta` file.
+ * This keeps the data file human-readable and backward-compatible with older code.
  *
  * @param url - The URL to fetch from
- * @param networkName - Network name for cache directory structure
- * @param options - Caching and error handling options
- * @param cacheDir - Optional cache directory (defaults to no caching)
- * @returns The fetched and parsed JSON data, or undefined if fetch fails and throwOnError is false
+ * @param options - Caching options
+ * @param fetch - Fetch implementation (defaults to globalThis.fetch)
+ * @param log - Logger instance
+ * @returns The fetched and parsed JSON data, or undefined if fetch fails
  */
 export async function cachedFetch<T = any>(
   url: string,
@@ -26,42 +50,106 @@ export async function cachedFetch<T = any>(
   fetch = globalThis.fetch,
   log = createLogger('cached_fetch'),
 ): Promise<T | undefined> {
-  const { cacheDurationMs, cacheFile } = options;
+  const { cacheFile, defaultMaxAgeMs = DEFAULT_MAX_AGE_MS } = options;
+
+  // If no cacheFile, just fetch normally without caching
+  if (!cacheFile) {
+    return fetchAndParse<T>(url, fetch, log);
+  }
+
+  const metaFile = cacheFile + '.meta';
 
-  // Try to read from cache first
+  // Try to read metadata
+  let meta: CacheMeta | undefined;
   try {
-    if (cacheFile) {
-      const info = await stat(cacheFile);
-      if (info.mtimeMs + cacheDurationMs > Date.now()) {
-        const cachedData = JSON.parse(await readFile(cacheFile, 'utf-8'));
-        return cachedData;
-      }
-    }
+    meta = JSON.parse(await readFile(metaFile, 'utf-8'));
   } catch {
-    log.trace('Failed to read data from cache');
+    log.trace('No usable cache metadata found');
   }
 
+  // Try to read cached data
+  let cachedData: T | undefined;
   try {
-    const response = await fetch(url);
+    cachedData = JSON.parse(await readFile(cacheFile, 'utf-8'));
+  } catch {
+    log.trace('No usable cached data found');
+  }
+
+  // If metadata and data exist and cache is fresh, return directly
+  if (meta && cachedData !== undefined && meta.expiresAt > Date.now()) {
+    return cachedData;
+  }
+
+  // Cache is stale or missing — make a (possibly conditional) request
+  try {
+    const headers: Record<string, string> = {};
+    if (meta?.etag && cachedData !== undefined) {
+      headers['If-None-Match'] = meta.etag;
+    }
+
+    const response = await fetch(url, { headers });
+
+    if (response.status === 304 && cachedData !== undefined) {
+      // Not modified — recompute expiry from new response headers and return cached data
+      const maxAgeMs = parseMaxAge(response) ?? defaultMaxAgeMs;
+      await writeMetaFile(metaFile, { etag: meta?.etag, expiresAt: Date.now() + maxAgeMs }, log);
+      return cachedData;
+    }
+
     if (!response.ok) {
       log.warn(`Failed to fetch from ${url}: ${response.status} ${response.statusText}`);
-      return undefined;
+      return cachedData;
     }
 
-    const data = await response.json();
+    // 200 — parse new data and cache it
+    const data = (await response.json()) as T;
+    const maxAgeMs = parseMaxAge(response) ?? defaultMaxAgeMs;
+    const etag = response.headers.get('etag') ?? undefined;
 
-    try {
-      if (cacheFile) {
-        await mkdir(dirname(cacheFile), { recursive: true });
-        await writeFile(cacheFile, JSON.stringify(data), 'utf-8');
-      }
-    } catch (err) {
-      log.warn('Failed to cache data on disk: ' + cacheFile, { cacheFile, err });
-    }
+    await ensureDir(cacheFile, log);
+    await Promise.all([
+      writeFile(cacheFile, JSON.stringify(data), 'utf-8'),
+      writeFile(metaFile, JSON.stringify({ etag, expiresAt: Date.now() + maxAgeMs }), 'utf-8'),
+    ]);
 
     return data;
+  } catch (err) {
+    log.warn(`Failed to fetch from ${url}`, { err });
+    return cachedData;
+  }
+}
+
+async function fetchAndParse<T>(
+  url: string,
+  fetch: typeof globalThis.fetch,
+  log: ReturnType<typeof createLogger>,
+): Promise<T | undefined> {
+  try {
+    const response = await fetch(url);
+    if (!response.ok) {
+      log.warn(`Failed to fetch from ${url}: ${response.status} ${response.statusText}`);
+      return undefined;
+    }
+    return (await response.json()) as T;
   } catch (err) {
     log.warn(`Failed to fetch from ${url}`, { err });
     return undefined;
   }
 }
+
+async function ensureDir(filePath: string, log: ReturnType<typeof createLogger>) {
+  try {
+    await mkdir(dirname(filePath), { recursive: true });
+  } catch (err) {
+    log.warn('Failed to create cache directory for: ' + filePath, { err });
+  }
+}
+
+async function writeMetaFile(metaFile: string, meta: CacheMeta, log: ReturnType<typeof createLogger>) {
+  try {
+    await mkdir(dirname(metaFile), { recursive: true });
+    await writeFile(metaFile, JSON.stringify(meta), 'utf-8');
+  } catch (err) {
+    log.warn('Failed to write cache metadata: ' + metaFile, { err });
+  }
+}

package/src/config/chain_l2_config.ts

@@ -1,4 +1,6 @@
 import type { NetworkNames } from '@aztec/foundation/config';
+import { createLogger } from '@aztec/foundation/log';
+import { type ConsensusEnvVar, checkConsensusEnvOverrides } from '@aztec/stdlib/config';
 
 import path from 'path';
 
@@ -12,6 +14,12 @@ const NetworkConfigs: Partial<Record<NetworkNames, NetworkConfigEnv>> = {
   mainnet: mainnetConfig,
 };
 
+/** Every generated network config must define every consensus-critical env var. */
+export type ConsensusComplete = Record<ConsensusEnvVar, string | number | boolean>;
+({ devnetConfig, testnetConfig, mainnetConfig }) satisfies Record<string, ConsensusComplete>;
+
+const log = createLogger('cli:chain_l2_config');
+
 function enrichEnvironmentWithNetworkConfig(config: NetworkConfigEnv): void {
   for (const [key, value] of Object.entries(config)) {
     if (process.env[key] === undefined && value !== undefined) {
@@ -31,7 +39,9 @@ function getDefaultDataDir(networkName: NetworkNames): string {
  * and DefaultSlasherConfig (which match the 'defaults' section of defaults.yml).
  *
  * For deployed networks: applies network configuration from generated defaults.yml,
- * merging base defaults with network-specific overrides.
+ * merging base defaults with network-specific overrides. Before merging, enforces that operators have not
+ * overridden any consensus-critical env var with a value diverging from the network config (throwing unless
+ * ALLOW_OVERRIDING_NETWORK_CONFIG is set), so all nodes of a network agree on consensus-critical values.
  *
  * @param networkName - The network name
  */
@@ -45,8 +55,13 @@ export function enrichEnvironmentWithChainName(networkName: NetworkNames) {
   }
 
   // Apply generated network config from defaults.yml
-  const generatedConfig = NetworkConfigs[networkName];
+  // For devnet iterations (v4-devnet-1, etc.), use the base devnet config
+  const configKey = /^v\d+-devnet-\d+$/.test(networkName) ? 'devnet' : networkName;
+  const generatedConfig = NetworkConfigs[configKey];
   if (generatedConfig) {
+    // The check is pure; this layer owns env mutation, so apply its canonical writes before enriching.
+    const canonical = checkConsensusEnvOverrides(generatedConfig, process.env, msg => log.warn(msg));
+    Object.assign(process.env, canonical);
     enrichEnvironmentWithNetworkConfig(generatedConfig);
   }