@aztec/aztec 0.0.1-commit.b655e406 → 0.0.1-commit.b8a057fa

package/src/cli/cmds/standby.ts

@@ -0,0 +1,184 @@
+import { getInitialTestAccountsData } from '@aztec/accounts/testing';
+import type { Fr } from '@aztec/aztec.js/fields';
+import { getSponsoredFPCAddress } from '@aztec/cli/cli-utils';
+import { getPublicClient } from '@aztec/ethereum/client';
+import type { GenesisStateConfig } from '@aztec/ethereum/config';
+import { RegistryContract, RollupContract } from '@aztec/ethereum/contracts';
+import type { EthAddress } from '@aztec/foundation/eth-address';
+import { startHttpRpcServer } from '@aztec/foundation/json-rpc/server';
+import { type LogFn, createLogger } from '@aztec/foundation/log';
+import { retryUntil } from '@aztec/foundation/retry';
+import { AztecAddress } from '@aztec/stdlib/aztec-address';
+import type { VersionCheck } from '@aztec/stdlib/update-checker';
+import { getGenesisValues } from '@aztec/world-state/testing';
+
+import Koa from 'koa';
+
+import { isShuttingDown, softShutdown } from '../util.js';
+
+const ROLLUP_POLL_INTERVAL_S = 60;
+
+/**
+ * Computes the expected genesis archive root from the genesis state config.
+ * Reads test accounts and sponsored FPC addresses as specified, then computes
+ * the genesis values including the archive root and prefilled public data.
+ */
+export async function computeExpectedGenesisRoot(config: GenesisStateConfig, userLog: LogFn) {
+  const testAccounts = config.testAccounts ? (await getInitialTestAccountsData()).map(a => a.address) : [];
+  const sponsoredFPCAccounts = config.sponsoredFPC ? [await getSponsoredFPCAddress()] : [];
+  const prefundAddresses = (config.prefundAddresses ?? []).map(a => AztecAddress.fromStringUnsafe(a));
+  const initialFundedAccounts = testAccounts.concat(sponsoredFPCAccounts).concat(prefundAddresses);
+
+  userLog(`Initial funded accounts: ${initialFundedAccounts.map(a => a.toString()).join(', ')}`);
+
+  const { genesisArchiveRoot, genesis } = await getGenesisValues(initialFundedAccounts);
+
+  userLog(`Genesis archive root: ${genesisArchiveRoot.toString()}`);
+
+  return { genesisArchiveRoot, genesis };
+}
+
+/**
+ * Compares the rollup's on-chain protocol constants (genesis archive root, VK tree root, protocol
+ * contracts hash) against the node's expected local values. Returns a list of human-readable mismatch
+ * descriptions, empty if the rollup is compatible.
+ */
+export async function checkRollupCompatibility(
+  rollup: RollupContract,
+  expected: { genesisArchiveRoot: Fr; vkTreeRoot: Fr; protocolContractsHash: Fr },
+): Promise<string[]> {
+  const mismatches: string[] = [];
+  const [l1Genesis, l1Vk, l1Protocol] = await Promise.all([
+    rollup.getGenesisArchiveTreeRoot(),
+    rollup.getVkTreeRoot(),
+    rollup.getProtocolContractsHash(),
+  ]);
+  if (!l1Genesis.equals(expected.genesisArchiveRoot)) {
+    mismatches.push(`genesis archive root (expected ${expected.genesisArchiveRoot}, got ${l1Genesis})`);
+  }
+  if (!l1Vk.equals(expected.vkTreeRoot)) {
+    mismatches.push(`VK tree root (expected ${expected.vkTreeRoot}, got ${l1Vk})`);
+  }
+  if (!l1Protocol.equals(expected.protocolContractsHash)) {
+    mismatches.push(`protocol contracts hash (expected ${expected.protocolContractsHash}, got ${l1Protocol})`);
+  }
+  return mismatches;
+}
+
+/**
+ * Waits until the canonical rollup's genesis archive root, VK tree root, and protocol contracts hash
+ * all match the expected local values. If the rollup is not yet compatible (e.g. during L1 contract upgrades),
+ * enters standby mode: starts a lightweight HTTP server for K8s liveness probes and polls every 60s
+ * until a compatible rollup appears.
+ */
+export async function waitForCompatibleRollup(
+  config: {
+    l1RpcUrls: string[];
+    l1ChainId: number;
+    registryAddress: EthAddress;
+    rollupVersion?: number;
+  },
+  expected: { genesisArchiveRoot: Fr; vkTreeRoot: Fr; protocolContractsHash: Fr },
+  port: number | undefined,
+  userLog: LogFn,
+): Promise<void> {
+  const publicClient = getPublicClient(config);
+  const rollupVersion: number | 'canonical' = config.rollupVersion ?? 'canonical';
+
+  const registry = new RegistryContract(publicClient, config.registryAddress);
+  const rollupAddress = await registry.getRollupAddress(rollupVersion);
+  const rollup = new RollupContract(publicClient, rollupAddress.toString());
+
+  let mismatches: string[];
+  try {
+    mismatches = await checkRollupCompatibility(rollup, expected);
+  } catch (err: any) {
+    throw new Error(`Could not retrieve rollup config from canonical rollup at ${rollupAddress}: ${err.message}`);
+  }
+
+  if (mismatches.length === 0) {
+    return;
+  }
+
+  userLog(
+    `Rollup at ${rollupAddress} is incompatible: ${mismatches.join('; ')}. ` +
+      `Entering standby mode. Will poll every ${ROLLUP_POLL_INTERVAL_S}s for a compatible rollup...`,
+  );
+
+  const standbyServer = await startHttpRpcServer({ getApp: () => new Koa(), isHealthy: () => true }, { port });
+  userLog(`Standby status server listening on port ${standbyServer.port}`);
+
+  try {
+    await retryUntil(
+      async () => {
+        const currentRollupAddress = await registry.getRollupAddress(rollupVersion);
+        const currentRollup = new RollupContract(publicClient, currentRollupAddress.toString());
+
+        let currentMismatches: string[];
+        try {
+          currentMismatches = await checkRollupCompatibility(currentRollup, expected);
+        } catch {
+          userLog(`Failed to fetch rollup config from rollup at ${currentRollupAddress}. Retrying...`);
+          return undefined;
+        }
+
+        if (currentMismatches.length === 0) {
+          userLog(`Compatible rollup found at ${currentRollupAddress}. Exiting standby mode.`);
+          return true;
+        }
+
+        userLog(`Still waiting. Rollup at ${currentRollupAddress}: ${currentMismatches.join('; ')}.`);
+        return undefined;
+      },
+      'compatible rollup',
+      0,
+      ROLLUP_POLL_INTERVAL_S,
+    );
+  } finally {
+    await new Promise<void>((resolve, reject) => standbyServer.close(err => (err ? reject(err) : resolve())));
+  }
+}
+
+/**
+ * Polls the canonical rollup's protocol constants every 10 minutes and soft-shuts-down the node once they
+ * diverge from the node's expected local values (i.e. an incompatible rollup has become canonical on L1).
+ * The HTTP health server is left running so K8s probes keep passing on the wound-down pod. This is the
+ * inverse of {@link waitForCompatibleRollup}; it should only be set up for nodes following the canonical
+ * rollup. Reuses the {@link checkRollupCompatibility} diff and the generic VersionChecker poll primitive.
+ */
+export async function setupAutoShutdown(
+  config: { l1RpcUrls: string[]; l1ChainId: number },
+  registryAddress: EthAddress,
+  rollupVersion: number | 'canonical',
+  expected: { genesisArchiveRoot: Fr; vkTreeRoot: Fr; protocolContractsHash: Fr },
+  signalHandlers: Array<() => Promise<void>>,
+): Promise<void> {
+  const { VersionChecker } = await import('@aztec/stdlib/update-checker');
+
+  const logger = createLogger('auto_shutdown');
+  const publicClient = getPublicClient(config);
+  const registry = new RegistryContract(publicClient, registryAddress);
+
+  const check: VersionCheck = {
+    name: 'rollup',
+    currentVersion: 'compatible',
+    getLatestVersion: async () => {
+      const rollupAddress = await registry.getRollupAddress(rollupVersion);
+      const rollup = new RollupContract(publicClient, rollupAddress.toString());
+      const mismatches = await checkRollupCompatibility(rollup, expected);
+      return mismatches.length === 0 ? 'compatible' : `incompatible: ${mismatches.join('; ')}`;
+    },
+  };
+
+  const checker = new VersionChecker([check], 600_000, logger);
+  checker.on('newVersion', ({ latestVersion }) => {
+    if (isShuttingDown()) {
+      return;
+    }
+    logger.warn('Canonical rollup is no longer compatible; auto-shutting down node', { latestVersion });
+    // softShutdown never rejects (it awaits handlers via allSettled); fire-and-forget from the listener.
+    void softShutdown(logger.info.bind(logger), signalHandlers);
+  });
+  checker.start();
+  signalHandlers.push(() => checker.stop());
+}

package/src/cli/cmds/start_bot.ts

@@ -10,9 +10,9 @@ import {
   initTelemetryClient,
   makeTracedFetch,
 } from '@aztec/telemetry-client';
-import { TestWallet } from '@aztec/test-wallet/server';
+import { EmbeddedWallet } from '@aztec/wallets/embedded';
 
-import { extractRelevantOptions } from '../util.js';
+import { extractRelevantOptions, stringifyConfig } from '../util.js';
 import { getVersions } from '../versioning.js';
 
 export async function startBot(
@@ -21,11 +21,9 @@ export async function startBot(
   services: NamespacedApiHandlers,
   userLog: LogFn,
 ) {
-  const { proverNode, archiver, sequencer, p2pBootstrap, txe, prover } = options;
-  if (proverNode || archiver || sequencer || p2pBootstrap || txe || prover) {
-    userLog(
-      `Starting a bot with --prover-node, --prover, --archiver, --sequencer, --p2p-bootstrap, or --txe is not supported.`,
-    );
+  const { proverNode, sequencer, p2pBootstrap, txe, prover } = options;
+  if (proverNode || sequencer || p2pBootstrap || txe || prover) {
+    userLog(`Starting a bot with --prover-node, --prover, --sequencer, --p2p-bootstrap, or --txe is not supported.`);
     process.exit(1);
   }
 
@@ -38,22 +36,30 @@ export async function startBot(
   const aztecNode = createAztecNodeClient(config.nodeUrl, getVersions(), fetch);
 
   const pxeConfig = extractRelevantOptions<PXEConfig & CliPXEOptions>(options, allPxeConfigMappings, 'pxe');
-  const wallet = await TestWallet.create(aztecNode, pxeConfig);
+  userLog(`Creating bot wallet with config ${stringifyConfig(pxeConfig)}`);
+  const wallet = await EmbeddedWallet.create(aztecNode, { pxeConfig });
 
-  const telemetry = initTelemetryClient(getTelemetryClientConfig());
-  await addBot(options, signalHandlers, services, wallet, aztecNode, telemetry, undefined);
+  const telemetry = await initTelemetryClient(getTelemetryClientConfig());
+  await addBot(options, signalHandlers, services, wallet, aztecNode, telemetry, undefined, userLog);
 }
 
 export async function addBot(
   options: any,
   signalHandlers: (() => Promise<void>)[],
   services: NamespacedApiHandlers,
-  wallet: TestWallet,
+  wallet: EmbeddedWallet,
   aztecNode: AztecNode,
   telemetry: TelemetryClient,
   aztecNodeAdmin?: AztecNodeAdmin,
+  userLog?: LogFn,
 ) {
   const config = extractRelevantOptions<BotConfig>(options, botConfigMappings, 'bot');
+  userLog?.(`Starting bot with config ${stringifyConfig(config)}`);
+
+  // The bot wallet's embedded PXE syncs to this tip (see start_bot.ts/start_node.ts which build the wallet from the
+  // same options). L1-to-L2 readiness checks must be evaluated at this tip rather than at 'latest', or the bot can
+  // consider a message ready while the PXE simulation anchors to an older block that cannot prove its membership yet.
+  const { syncChainTip } = extractRelevantOptions<PXEConfig & CliPXEOptions>(options, allPxeConfigMappings, 'pxe');
 
   const db = await (config.dataDirectory
     ? createStore('bot', BotStore.SCHEMA_VERSION, config)
@@ -62,7 +68,7 @@ export async function addBot(
   const store = new BotStore(db);
   await store.cleanupOldClaims();
 
-  const botRunner = new BotRunner(config, wallet, aztecNode, telemetry, aztecNodeAdmin, store);
+  const botRunner = new BotRunner(config, wallet, aztecNode, telemetry, aztecNodeAdmin, store, syncChainTip);
   if (!config.noStart) {
     void botRunner.start(); // Do not block since bot setup takes time
   }

package/src/cli/cmds/start_node.ts

@@ -1,30 +1,38 @@
-import { getInitialTestAccountsData } from '@aztec/accounts/testing';
-import { type AztecNodeConfig, aztecNodeConfigMappings, getConfigEnvVars } from '@aztec/aztec-node';
+import {
+  type AztecNodeConfig,
+  aztecNodeConfigMappings,
+  getConfigEnvVars,
+  registerAztecNodeRpcHandlers,
+} from '@aztec/aztec-node';
 import { Fr } from '@aztec/aztec.js/fields';
-import { getSponsoredFPCAddress } from '@aztec/cli/cli-utils';
 import { getL1Config } from '@aztec/cli/config';
-import { getPublicClient } from '@aztec/ethereum';
-import { SecretValue } from '@aztec/foundation/config';
+import { getGenesisStateConfigEnvVars } from '@aztec/ethereum/config';
+import { type NetworkNames, SecretValue } from '@aztec/foundation/config';
 import type { NamespacedApiHandlers } from '@aztec/foundation/json-rpc/server';
+import { Agent, makeUndiciFetch } from '@aztec/foundation/json-rpc/undici';
 import type { LogFn } from '@aztec/foundation/log';
+import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
+import { protocolContractsHash } from '@aztec/protocol-contracts';
+import {
+  ProvingJobConsumerSchema,
+  createProvingJobBrokerClient,
+  proverBrokerBackoff,
+} from '@aztec/prover-client/broker';
 import { type CliPXEOptions, type PXEConfig, allPxeConfigMappings } from '@aztec/pxe/config';
-import { AztecNodeAdminApiSchema, AztecNodeApiSchema } from '@aztec/stdlib/interfaces/client';
-import { P2PApiSchema } from '@aztec/stdlib/interfaces/server';
+import { ProverNodeApiSchema, type ProvingJobBroker } from '@aztec/stdlib/interfaces/server';
 import {
   type TelemetryClientConfig,
   initTelemetryClient,
+  makeTracedFetch,
   telemetryClientConfigMappings,
 } from '@aztec/telemetry-client';
-import { TestWallet } from '@aztec/test-wallet/server';
-import { getGenesisValues } from '@aztec/world-state/testing';
+import { EmbeddedWallet } from '@aztec/wallets/embedded';
 
-import { createAztecNode } from '../../sandbox/index.js';
-import {
-  extractNamespacedOptions,
-  extractRelevantOptions,
-  preloadCrsDataForVerifying,
-  setupUpdateMonitor,
-} from '../util.js';
+import { createAztecNode } from '../../local-network/index.js';
+import { extractNamespacedOptions, extractRelevantOptions, preloadCrsDataForVerifying } from '../util.js';
+import { getVersions } from '../versioning.js';
+import { computeExpectedGenesisRoot, setupAutoShutdown, waitForCompatibleRollup } from './standby.js';
+import { startProverBroker } from './start_prover_broker.js';
 
 export async function startNode(
   options: any,
@@ -32,6 +40,7 @@ export async function startNode(
   services: NamespacedApiHandlers,
   adminServices: NamespacedApiHandlers,
   userLog: LogFn,
+  networkName: NetworkNames,
 ): Promise<{ config: AztecNodeConfig }> {
   // All options set from environment variables
   const configFromEnvVars = getConfigEnvVars();
@@ -45,31 +54,53 @@ export async function startNode(
     ...relevantOptions,
   };
 
+  // Prover node configuration and broker setup
+  // REFACTOR: Move the broker setup out of here and into the prover-node factory
+  let broker: ProvingJobBroker | undefined = undefined;
   if (options.proverNode) {
-    userLog(`Running a Prover Node within a Node is not yet supported`);
-    process.exit(1);
+    nodeConfig.enableProverNode = true;
+    if (nodeConfig.proverAgentCount === 0) {
+      userLog(
+        `Running prover node without local prover agent. Connect prover agents or pass --proverAgent.proverAgentCount`,
+      );
+    }
+    if (nodeConfig.proverBrokerUrl) {
+      // at 1TPS we'd enqueue ~1k chonk verifier proofs and ~1k AVM proofs immediately
+      // set a lower connection limit such that we don't overload the server
+      // Retry indefinitely until the epoch proving times out and the chain reorgs
+      const fetch = makeTracedFetch(proverBrokerBackoff, false, makeUndiciFetch(new Agent({ connections: 100 })));
+      broker = createProvingJobBrokerClient(nodeConfig.proverBrokerUrl, getVersions(nodeConfig), fetch);
+    } else if (options.proverBroker) {
+      ({ broker } = await startProverBroker(options, signalHandlers, services, userLog));
+    } else {
+      userLog(`--prover-broker-url or --prover-broker is required to start a Prover Node`);
+      process.exit(1);
+    }
   }
 
   await preloadCrsDataForVerifying(nodeConfig, userLog);
 
-  const testAccounts = nodeConfig.testAccounts ? (await getInitialTestAccountsData()).map(a => a.address) : [];
-  const sponsoredFPCAccounts = nodeConfig.sponsoredFPC ? [await getSponsoredFPCAddress()] : [];
-  const initialFundedAccounts = testAccounts.concat(sponsoredFPCAccounts);
-
-  userLog(`Initial funded accounts: ${initialFundedAccounts.map(a => a.toString()).join(', ')}`);
-
-  const { genesisArchiveRoot, prefilledPublicData } = await getGenesisValues(initialFundedAccounts);
-
-  userLog(`Genesis archive root: ${genesisArchiveRoot.toString()}`);
+  const genesisConfig = getGenesisStateConfigEnvVars();
+  const { genesisArchiveRoot, genesis } = await computeExpectedGenesisRoot(genesisConfig, userLog);
 
   const followsCanonicalRollup =
     typeof nodeConfig.rollupVersion !== 'number' || (nodeConfig.rollupVersion as unknown as string) === 'canonical';
 
-  if (!nodeConfig.l1Contracts.registryAddress || nodeConfig.l1Contracts.registryAddress.isZero()) {
+  if (!nodeConfig.registryAddress || nodeConfig.registryAddress.isZero()) {
     throw new Error('L1 registry address is required to start Aztec Node');
   }
+
+  // Wait for a compatible rollup before proceeding with full L1 config fetch.
+  // This prevents crashes when the canonical rollup hasn't been upgraded yet.
+  await waitForCompatibleRollup(
+    nodeConfig,
+    { genesisArchiveRoot, vkTreeRoot: getVKTreeRoot(), protocolContractsHash },
+    options.port,
+    userLog,
+  );
+
   const { addresses, config } = await getL1Config(
-    nodeConfig.l1Contracts.registryAddress,
+    nodeConfig.registryAddress,
     nodeConfig.l1RpcUrls,
     nodeConfig.l1ChainId,
     nodeConfig.rollupVersion,
@@ -83,17 +114,13 @@ export async function startNode(
     );
   }
 
-  // TODO(#12272): will clean this up.
   nodeConfig = {
     ...nodeConfig,
-    l1Contracts: {
-      ...addresses,
-      slashFactoryAddress: nodeConfig.l1Contracts.slashFactoryAddress,
-    },
+    ...addresses,
     ...config,
   };
 
-  if (!options.sequencer) {
+  if (!options.sequencer && !nodeConfig.fishermanMode) {
     nodeConfig.disableValidator = true;
   } else {
     const sequencerConfig = {
@@ -101,12 +128,17 @@ export async function startNode(
       ...extractNamespacedOptions(options, 'sequencer'),
     };
     // If no publisher private keys have been given, use the first validator key
-    if (sequencerConfig.publisherPrivateKeys === undefined || !sequencerConfig.publisherPrivateKeys.length) {
+    if (
+      sequencerConfig.sequencerPublisherPrivateKeys === undefined ||
+      !sequencerConfig.sequencerPublisherPrivateKeys.length
+    ) {
       if (sequencerConfig.validatorPrivateKeys?.getValue().length) {
-        sequencerConfig.publisherPrivateKeys = [new SecretValue(sequencerConfig.validatorPrivateKeys.getValue()[0])];
+        sequencerConfig.sequencerPublisherPrivateKeys = [
+          new SecretValue(sequencerConfig.validatorPrivateKeys.getValue()[0]),
+        ];
       }
     }
-    nodeConfig.publisherPrivateKeys = sequencerConfig.publisherPrivateKeys;
+    nodeConfig.sequencerPublisherPrivateKeys = sequencerConfig.sequencerPublisherPrivateKeys;
   }
 
   if (nodeConfig.p2pEnabled) {
@@ -117,15 +149,21 @@ export async function startNode(
   }
 
   const telemetryConfig = extractRelevantOptions<TelemetryClientConfig>(options, telemetryClientConfigMappings, 'tel');
-  const telemetry = initTelemetryClient(telemetryConfig);
+  const telemetry = await initTelemetryClient(telemetryConfig);
 
   // Create and start Aztec Node
-  const node = await createAztecNode(nodeConfig, { telemetry }, { prefilledPublicData });
+  const node = await createAztecNode(nodeConfig, { telemetry, proverBroker: broker }, { genesis });
+
+  registerAztecNodeRpcHandlers(node, services, adminServices, { debug: options.nodeDebug });
 
-  // Add node and p2p to services list
-  services.node = [node, AztecNodeApiSchema];
-  services.p2p = [node.getP2P(), P2PApiSchema];
-  adminServices.nodeAdmin = [node, AztecNodeAdminApiSchema];
+  // Register prover-node services if the prover node subsystem is running
+  const proverNode = node.getProverNode();
+  if (proverNode) {
+    adminServices.prover = [proverNode, ProverNodeApiSchema];
+    if (!nodeConfig.proverBrokerUrl) {
+      services.provingJobSource = [proverNode.getProver().getProvingJobSource(), ProvingJobConsumerSchema];
+    }
+  }
 
   // Add node stop function to signal handlers
   signalHandlers.push(node.stop.bind(node));
@@ -135,21 +173,23 @@ export async function startNode(
     const { addBot } = await import('./start_bot.js');
 
     const pxeConfig = extractRelevantOptions<PXEConfig & CliPXEOptions>(options, allPxeConfigMappings, 'pxe');
-    const wallet = await TestWallet.create(node, pxeConfig);
+    const wallet = await EmbeddedWallet.create(node, { pxeConfig });
 
     await addBot(options, signalHandlers, services, wallet, node, telemetry, undefined);
   }
 
-  if (nodeConfig.autoUpdate !== 'disabled' && nodeConfig.autoUpdateUrl) {
-    await setupUpdateMonitor(
-      nodeConfig.autoUpdate,
-      new URL(nodeConfig.autoUpdateUrl),
-      followsCanonicalRollup,
-      getPublicClient(nodeConfig!),
-      nodeConfig.l1Contracts.registryAddress,
-      signalHandlers,
-      async config => node.setConfig((await AztecNodeAdminApiSchema.setConfig.parameters().parseAsync([config]))[0]),
-    );
+  if (nodeConfig.enableAutoShutdown && networkName !== 'local' && followsCanonicalRollup) {
+    try {
+      await setupAutoShutdown(
+        nodeConfig,
+        nodeConfig.registryAddress,
+        (nodeConfig.rollupVersion as number | 'canonical') ?? 'canonical',
+        { genesisArchiveRoot, vkTreeRoot: getVKTreeRoot(), protocolContractsHash },
+        signalHandlers,
+      );
+    } catch {
+      /* no-op */
+    }
   }
 
   return { config: nodeConfig };

package/src/cli/cmds/start_p2p_bootstrap.ts

@@ -1,6 +1,6 @@
 import { jsonStringify } from '@aztec/foundation/json-rpc';
 import type { NamespacedApiHandlers } from '@aztec/foundation/json-rpc/server';
-import { type LogFn, createLogger } from '@aztec/foundation/log';
+import type { LogFn } from '@aztec/foundation/log';
 import { createStore } from '@aztec/kv-store/lmdb-v2';
 import { type BootnodeConfig, BootstrapNode, bootnodeConfigMappings } from '@aztec/p2p';
 import { emptyChainConfig } from '@aztec/stdlib/config';
@@ -25,9 +25,9 @@ export async function startP2PBootstrap(
   userLog(`Starting P2P bootstrap node with config: ${jsonStringify(safeConfig)}`);
 
   const telemetryConfig = extractRelevantOptions<TelemetryClientConfig>(options, telemetryClientConfigMappings, 'tel');
-  const telemetryClient = initTelemetryClient(telemetryConfig);
+  const telemetryClient = await initTelemetryClient(telemetryConfig);
 
-  const store = await createStore('p2p-bootstrap', 1, config, createLogger('p2p:bootstrap:store'));
+  const store = await createStore('p2p-bootstrap', 1, config);
   const node = new BootstrapNode(store, telemetryClient);
   await node.start(config);
   signalHandlers.push(() => node.stop());

package/src/cli/cmds/start_prover_agent.ts

@@ -4,11 +4,12 @@ import { Agent, makeUndiciFetch } from '@aztec/foundation/json-rpc/undici';
 import type { LogFn } from '@aztec/foundation/log';
 import { buildServerCircuitProver } from '@aztec/prover-client';
 import {
-  InlineProofStore,
   type ProverAgentConfig,
   ProvingAgent,
+  createProofStore,
   createProvingJobBrokerClient,
   proverAgentConfigMappings,
+  proverBrokerBackoff,
 } from '@aztec/prover-client/broker';
 import { getProverNodeAgentConfigFromEnv } from '@aztec/prover-node';
 import { ProverAgentApiSchema } from '@aztec/stdlib/interfaces/server';
@@ -23,8 +24,8 @@ export async function startProverAgent(
   services: NamespacedApiHandlers,
   userLog: LogFn,
 ) {
-  if (options.node || options.sequencer || options.pxe || options.p2pBootstrap || options.txe) {
-    userLog(`Starting a prover agent with --node, --sequencer, --pxe, --p2p-bootstrap, or --txe is not supported.`);
+  if (options.node || options.sequencer || options.p2pBootstrap || options.txe) {
+    userLog(`Starting a prover agent with --node, --sequencer, --p2p-bootstrap, or --txe is not supported.`);
     process.exit(1);
   }
 
@@ -45,28 +46,16 @@ export async function startProverAgent(
 
   await preloadCrsDataForServerSideProving(config, userLog);
 
-  const fetch = makeTracedFetch(
-    // retry connections every 3s, up to 30s before giving up
-    [1, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3],
-    false,
-    makeUndiciFetch(new Agent({ connections: 10 })),
-  );
+  // Retry indefinitely until the epoch proving times out and the chain reorgs
+  const fetch = makeTracedFetch(proverBrokerBackoff, false, makeUndiciFetch(new Agent({ connections: 10 })));
   const broker = createProvingJobBrokerClient(config.proverBrokerUrl, getVersions(), fetch);
 
-  const telemetry = initTelemetryClient(extractRelevantOptions(options, telemetryClientConfigMappings, 'tel'));
+  const telemetry = await initTelemetryClient(extractRelevantOptions(options, telemetryClientConfigMappings, 'tel'));
   const prover = await buildServerCircuitProver(config, telemetry);
-  const proofStore = new InlineProofStore();
+  const proofStore = await createProofStore(config.proofStore);
   const agents = times(
     config.proverAgentCount,
-    () =>
-      new ProvingAgent(
-        broker,
-        proofStore,
-        prover,
-        config.proverAgentProofTypes,
-        config.proverAgentPollIntervalMs,
-        telemetry,
-      ),
+    () => new ProvingAgent(broker, proofStore, prover, config.proverAgentProofTypes, config.proverAgentPollIntervalMs),
   );
 
   // expose all agents as individual services

package/src/cli/cmds/start_prover_broker.ts

@@ -1,10 +1,13 @@
 import { getL1Config } from '@aztec/cli/config';
-import { getPublicClient } from '@aztec/ethereum';
+import { getGenesisStateConfigEnvVars } from '@aztec/ethereum/config';
 import type { NamespacedApiHandlers } from '@aztec/foundation/json-rpc/server';
 import type { LogFn } from '@aztec/foundation/log';
+import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
+import { protocolContractsHash } from '@aztec/protocol-contracts';
 import {
   type ProverBrokerConfig,
   ProvingJobBrokerSchema,
+  ProvingJobBrokerSchemaWithDebug,
   createAndStartProvingBroker,
   proverBrokerConfigMappings,
 } from '@aztec/prover-client/broker';
@@ -12,7 +15,8 @@ import { getProverNodeBrokerConfigFromEnv } from '@aztec/prover-node';
 import type { ProvingJobBroker } from '@aztec/stdlib/interfaces/server';
 import { getConfigEnvVars as getTelemetryClientConfig, initTelemetryClient } from '@aztec/telemetry-client';
 
-import { extractRelevantOptions, setupUpdateMonitor } from '../util.js';
+import { extractRelevantOptions } from '../util.js';
+import { computeExpectedGenesisRoot, waitForCompatibleRollup } from './standby.js';
 
 export async function startProverBroker(
   options: any,
@@ -20,46 +24,45 @@ export async function startProverBroker(
   services: NamespacedApiHandlers,
   userLog: LogFn,
 ): Promise<{ broker: ProvingJobBroker; config: ProverBrokerConfig }> {
-  if (options.node || options.sequencer || options.pxe || options.p2pBootstrap || options.txe) {
-    userLog(`Starting a prover broker with --node, --sequencer, --pxe, --p2p-bootstrap, or --txe is not supported.`);
+  if (options.node || options.sequencer || options.p2pBootstrap || options.txe) {
+    userLog(`Starting a prover broker with --node, --sequencer, --p2p-bootstrap, or --txe is not supported.`);
     process.exit(1);
   }
 
-  const config: ProverBrokerConfig = {
+  const baseConfig: ProverBrokerConfig = {
     ...getProverNodeBrokerConfigFromEnv(), // get default config from env
     ...extractRelevantOptions<ProverBrokerConfig>(options, proverBrokerConfigMappings, 'proverBroker'), // override with command line options
   };
 
-  if (!config.l1Contracts.registryAddress || config.l1Contracts.registryAddress.isZero()) {
+  if (!baseConfig.registryAddress || baseConfig.registryAddress.isZero()) {
     throw new Error('L1 registry address is required to start Aztec Node without --deploy-aztec-contracts option');
   }
 
-  const followsCanonicalRollup = typeof config.rollupVersion !== 'number';
+  const genesisConfig = getGenesisStateConfigEnvVars();
+  const { genesisArchiveRoot } = await computeExpectedGenesisRoot(genesisConfig, userLog);
+  await waitForCompatibleRollup(
+    baseConfig,
+    { genesisArchiveRoot, vkTreeRoot: getVKTreeRoot(), protocolContractsHash },
+    options.port,
+    userLog,
+  );
+
   const { addresses, config: rollupConfig } = await getL1Config(
-    config.l1Contracts.registryAddress,
-    config.l1RpcUrls,
-    config.l1ChainId,
-    config.rollupVersion,
+    baseConfig.registryAddress,
+    baseConfig.l1RpcUrls,
+    baseConfig.l1ChainId,
+    baseConfig.rollupVersion,
   );
 
-  config.l1Contracts = addresses;
-  config.rollupVersion = rollupConfig.rollupVersion;
+  const config: ProverBrokerConfig = { ...baseConfig, ...addresses, rollupVersion: rollupConfig.rollupVersion };
 
-  const client = initTelemetryClient(getTelemetryClientConfig());
+  const client = await initTelemetryClient(getTelemetryClientConfig());
   const broker = await createAndStartProvingBroker(config, client);
 
-  if (options.autoUpdate !== 'disabled' && options.autoUpdateUrl) {
-    await setupUpdateMonitor(
-      options.autoUpdate,
-      new URL(options.autoUpdateUrl),
-      followsCanonicalRollup,
-      getPublicClient(config),
-      config.l1Contracts.registryAddress,
-      signalHandlers,
-    );
-  }
-
-  services.proverBroker = [broker, ProvingJobBrokerSchema];
+  services.proverBroker = [
+    broker,
+    config.proverBrokerDebugReplayEnabled ? ProvingJobBrokerSchemaWithDebug : ProvingJobBrokerSchema,
+  ];
   signalHandlers.push(() => broker.stop());
 
   return { broker, config };