@aztec/prover-node 0.85.0 → 0.86.0

package/src/config.ts

@@ -44,9 +44,9 @@ type SpecificProverNodeConfig = {
   proverNodeMaxPendingJobs: number;
   proverNodePollingIntervalMs: number;
   proverNodeMaxParallelBlocksPerEpoch: number;
-  txGatheringTimeoutMs: number;
   txGatheringIntervalMs: number;
-  txGatheringMaxParallelRequests: number;
+  txGatheringBatchSize: number;
+  txGatheringMaxParallelRequestsPerNode: number;
 };
 
 const specificProverNodeConfigMappings: ConfigMappingsType<SpecificProverNodeConfig> = {
@@ -65,19 +65,19 @@ const specificProverNodeConfigMappings: ConfigMappingsType<SpecificProverNodeCon
     description: 'The Maximum number of blocks to process in parallel while proving an epoch',
     ...numberConfigHelper(32),
   },
-  txGatheringTimeoutMs: {
-    env: 'PROVER_NODE_TX_GATHERING_TIMEOUT_MS',
-    description: 'The maximum amount of time to wait for tx data to be available',
-    ...numberConfigHelper(60_000),
-  },
   txGatheringIntervalMs: {
     env: 'PROVER_NODE_TX_GATHERING_INTERVAL_MS',
     description: 'How often to check that tx data is available',
     ...numberConfigHelper(1_000),
   },
-  txGatheringMaxParallelRequests: {
-    env: 'PROVER_NODE_TX_GATHERING_MAX_PARALLEL_REQUESTS',
-    description: 'How many txs to load up a time',
+  txGatheringBatchSize: {
+    env: 'PROVER_NODE_TX_GATHERING_BATCH_SIZE',
+    description: 'How many transactions to gather from a node in a single request',
+    ...numberConfigHelper(10),
+  },
+  txGatheringMaxParallelRequestsPerNode: {
+    env: 'PROVER_NODE_TX_GATHERING_MAX_PARALLEL_REQUESTS_PER_NODE',
+    description: 'How many tx requests to make in parallel to each node',
     ...numberConfigHelper(100),
   },
 };

package/src/factory.ts

@@ -1,19 +1,20 @@
 import { type Archiver, createArchiver } from '@aztec/archiver';
 import { type BlobSinkClientInterface, createBlobSinkClient } from '@aztec/blob-sink/client';
 import { EpochCache } from '@aztec/epoch-cache';
-import { L1TxUtils, RollupContract, createEthereumChain, createL1Clients } from '@aztec/ethereum';
+import { L1TxUtils, RollupContract, createEthereumChain, createExtendedL1Client } from '@aztec/ethereum';
 import { type Logger, createLogger } from '@aztec/foundation/log';
 import type { DataStoreConfig } from '@aztec/kv-store/config';
 import { trySnapshotSync } from '@aztec/node-lib/actions';
 import { createProverClient } from '@aztec/prover-client';
 import { createAndStartProvingBroker } from '@aztec/prover-client/broker';
-import type { ProverCoordination, ProvingJobBroker } from '@aztec/stdlib/interfaces/server';
+import type { ProvingJobBroker } from '@aztec/stdlib/interfaces/server';
 import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees';
 import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
 import { createWorldStateSynchronizer } from '@aztec/world-state';
 
 import { type ProverNodeConfig, resolveConfig } from './config.js';
 import { EpochMonitor } from './monitors/epoch-monitor.js';
+import type { TxSource } from './prover-coordination/combined-prover-coordination.js';
 import { createProverCoordination } from './prover-coordination/factory.js';
 import { ProverNodePublisher } from './prover-node-publisher.js';
 import { ProverNode, type ProverNodeOptions } from './prover-node.js';
@@ -24,7 +25,7 @@ export async function createProverNode(
   deps: {
     telemetry?: TelemetryClient;
     log?: Logger;
-    aztecNodeTxProvider?: ProverCoordination;
+    aztecNodeTxProvider?: TxSource;
     archiver?: Archiver;
     publisher?: ProverNodePublisher;
     blobSinkClient?: BlobSinkClientInterface;
@@ -37,7 +38,8 @@ export async function createProverNode(
 ) {
   const config = resolveConfig(userConfig);
   const telemetry = deps.telemetry ?? getTelemetryClient();
-  const blobSinkClient = deps.blobSinkClient ?? createBlobSinkClient(config);
+  const blobSinkClient =
+    deps.blobSinkClient ?? createBlobSinkClient(config, { logger: createLogger('prover-node:blob-sink:client') });
   const log = deps.log ?? createLogger('prover-node');
 
   await trySnapshotSync(config, log);
@@ -59,17 +61,17 @@ export async function createProverNode(
 
   const { l1RpcUrls: rpcUrls, l1ChainId: chainId, publisherPrivateKey } = config;
   const chain = createEthereumChain(rpcUrls, chainId);
-  const { publicClient, walletClient } = createL1Clients(rpcUrls, publisherPrivateKey, chain.chainInfo);
+  const l1Client = createExtendedL1Client(rpcUrls, publisherPrivateKey, chain.chainInfo);
 
-  const rollupContract = new RollupContract(publicClient, config.l1Contracts.rollupAddress.toString());
+  const rollupContract = new RollupContract(l1Client, config.l1Contracts.rollupAddress.toString());
 
-  const l1TxUtils = deps.l1TxUtils ?? new L1TxUtils(publicClient, walletClient, log, config);
+  const l1TxUtils = deps.l1TxUtils ?? new L1TxUtils(l1Client, log, config);
   const publisher = deps.publisher ?? new ProverNodePublisher(config, { telemetry, rollupContract, l1TxUtils });
 
   const epochCache = await EpochCache.create(config.l1Contracts.rollupAddress, config);
 
   // If config.p2pEnabled is true, createProverCoordination will create a p2p client where txs are requested
-  // If config.p2pEnabled is false, createProverCoordination request information from the AztecNode
+  // If config.proverCoordinationNodeUrls is not empty, createProverCoordination will create set of aztec node clients from which txs are requested
   const proverCoordination = await createProverCoordination(config, {
     aztecNodeTxProvider: deps.aztecNodeTxProvider,
     worldStateSynchronizer,
@@ -82,9 +84,7 @@ export async function createProverNode(
     maxPendingJobs: config.proverNodeMaxPendingJobs,
     pollingIntervalMs: config.proverNodePollingIntervalMs,
     maxParallelBlocksPerEpoch: config.proverNodeMaxParallelBlocksPerEpoch,
-    txGatheringMaxParallelRequests: config.txGatheringMaxParallelRequests,
     txGatheringIntervalMs: config.txGatheringIntervalMs,
-    txGatheringTimeoutMs: config.txGatheringTimeoutMs,
   };
 
   const epochMonitor = await EpochMonitor.create(archiver, proverNodeConfig, telemetry);

package/src/job/epoch-proving-job.ts

@@ -16,7 +16,7 @@ import { Attributes, type Traceable, type Tracer, trackSpan } from '@aztec/telem
 
 import * as crypto from 'node:crypto';
 
-import type { ProverNodeMetrics } from '../metrics.js';
+import type { ProverNodeJobMetrics } from '../metrics.js';
 import type { ProverNodePublisher } from '../prover-node-publisher.js';
 
 /**
@@ -45,12 +45,12 @@ export class EpochProvingJob implements Traceable {
     private publisher: ProverNodePublisher,
     private l2BlockSource: L2BlockSource,
     private l1ToL2MessageSource: L1ToL2MessageSource,
-    private metrics: ProverNodeMetrics,
+    private metrics: ProverNodeJobMetrics,
     private deadline: Date | undefined,
     private config: { parallelBlockLimit: number } = { parallelBlockLimit: 32 },
   ) {
     this.uuid = crypto.randomUUID();
-    this.tracer = metrics.client.getTracer('EpochProvingJob');
+    this.tracer = metrics.tracer;
   }
 
   public getId(): string {

package/src/metrics.ts

@@ -1,107 +1,187 @@
+import type { RollupContract } from '@aztec/ethereum';
+import type { EthAddress } from '@aztec/foundation/eth-address';
 import { createLogger } from '@aztec/foundation/log';
 import type { L1PublishProofStats, L1PublishStats } from '@aztec/stdlib/stats';
 import {
   Attributes,
+  type BatchObservableResult,
   type Gauge,
   type Histogram,
+  type Meter,
   Metrics,
+  type ObservableGauge,
   type TelemetryClient,
+  type Tracer,
   type UpDownCounter,
   ValueType,
 } from '@aztec/telemetry-client';
 
-import { formatEther } from 'viem';
+import { formatEther, formatUnits } from 'viem';
 
-export class ProverNodeMetrics {
+export class ProverNodeJobMetrics {
   proverEpochExecutionDuration: Histogram;
   provingJobDuration: Histogram;
   provingJobBlocks: Gauge;
   provingJobTransactions: Gauge;
 
-  gasPrice: Histogram;
-  txCount: UpDownCounter;
-  txDuration: Histogram;
-  txGas: Histogram;
-  txCalldataSize: Histogram;
-  txCalldataGas: Histogram;
-  txBlobDataGasUsed: Histogram;
-  txBlobDataGasCost: Histogram;
-  txTotalFee: Histogram;
-
-  private senderBalance: Gauge;
-
   constructor(
-    public readonly client: TelemetryClient,
-    name = 'ProverNode',
+    private meter: Meter,
+    public readonly tracer: Tracer,
     private logger = createLogger('prover-node:publisher:metrics'),
   ) {
-    const meter = client.getMeter(name);
-    this.proverEpochExecutionDuration = meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, {
+    this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, {
       description: 'Duration of execution of an epoch by the prover',
       unit: 'ms',
       valueType: ValueType.INT,
     });
-    this.provingJobDuration = meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
+    this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
       description: 'Duration of proving job',
       unit: 's',
       valueType: ValueType.DOUBLE,
     });
-    this.provingJobBlocks = meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
+    this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
       description: 'Number of blocks in a proven epoch',
       valueType: ValueType.INT,
     });
-    this.provingJobTransactions = meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
+    this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
       description: 'Number of transactions in a proven epoch',
       valueType: ValueType.INT,
     });
+  }
+
+  public recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number) {
+    this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
+    this.provingJobDuration.record(totalTimeMs / 1000);
+    this.provingJobBlocks.record(Math.floor(numBlocks));
+    this.provingJobTransactions.record(Math.floor(numTxs));
+  }
+}
+
+export class ProverNodeRewardsMetrics {
+  private rewards: ObservableGauge;
+  private accumulatedRewards: UpDownCounter;
+  private prevEpoch = -1n;
+  private proofSubmissionWindow = 0n;
+
+  constructor(
+    private meter: Meter,
+    private coinbase: EthAddress,
+    private rollup: RollupContract,
+    private logger = createLogger('prover-node:publisher:metrics'),
+  ) {
+    this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH, {
+      valueType: ValueType.DOUBLE,
+      description: 'The rewards earned',
+    });
+
+    this.accumulatedRewards = this.meter.createUpDownCounter(Metrics.PROVER_NODE_REWARDS_TOTAL, {
+      valueType: ValueType.DOUBLE,
+      description: 'The rewards earned (total)',
+    });
+  }
+
+  public async start() {
+    this.proofSubmissionWindow = await this.rollup.getProofSubmissionWindow();
+    this.meter.addBatchObservableCallback(this.observe, [this.rewards]);
+  }
+
+  public stop() {
+    this.meter.removeBatchObservableCallback(this.observe, [this.rewards]);
+  }
+
+  private observe = async (observer: BatchObservableResult): Promise<void> => {
+    const slot = await this.rollup.getSlotNumber();
+
+    // look at the prev epoch so that we get an accurate value, after proof submission window has closed
+    if (slot > this.proofSubmissionWindow) {
+      const closedEpoch = await this.rollup.getEpochNumberForSlotNumber(slot - this.proofSubmissionWindow);
+      const rewards = await this.rollup.getSpecificProverRewardsForEpoch(closedEpoch, this.coinbase);
+
+      const fmt = parseFloat(formatUnits(rewards, 18));
+
+      observer.observe(this.rewards, fmt, {
+        [Attributes.COINBASE]: this.coinbase.toString(),
+      });
+
+      // only accumulate once per epoch
+      if (closedEpoch > this.prevEpoch) {
+        this.prevEpoch = closedEpoch;
+        this.accumulatedRewards.add(fmt, {
+          [Attributes.COINBASE]: this.coinbase.toString(),
+        });
+      }
+    }
+  };
+}
 
-    this.gasPrice = meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, {
+export class ProverNodePublisherMetrics {
+  gasPrice: Histogram;
+  txCount: UpDownCounter;
+  txDuration: Histogram;
+  txGas: Histogram;
+  txCalldataSize: Histogram;
+  txCalldataGas: Histogram;
+  txBlobDataGasUsed: Histogram;
+  txBlobDataGasCost: Histogram;
+  txTotalFee: Histogram;
+
+  private senderBalance: Gauge;
+  private meter: Meter;
+
+  constructor(
+    public readonly client: TelemetryClient,
+    name = 'ProverNode',
+    private logger = createLogger('prover-node:publisher:metrics'),
+  ) {
+    this.meter = client.getMeter(name);
+
+    this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, {
       description: 'The gas price used for transactions',
       unit: 'gwei',
       valueType: ValueType.DOUBLE,
     });
 
-    this.txCount = meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, {
+    this.txCount = this.meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, {
       description: 'The number of transactions processed',
     });
 
-    this.txDuration = meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, {
+    this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, {
       description: 'The duration of transaction processing',
       unit: 'ms',
       valueType: ValueType.INT,
     });
 
-    this.txGas = meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, {
+    this.txGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, {
       description: 'The gas consumed by transactions',
       unit: 'gas',
       valueType: ValueType.INT,
     });
 
-    this.txCalldataSize = meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, {
+    this.txCalldataSize = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, {
       description: 'The size of the calldata in transactions',
       unit: 'By',
       valueType: ValueType.INT,
     });
 
-    this.txCalldataGas = meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, {
+    this.txCalldataGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, {
       description: 'The gas consumed by the calldata in transactions',
       unit: 'gas',
       valueType: ValueType.INT,
     });
 
-    this.txBlobDataGasUsed = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, {
+    this.txBlobDataGasUsed = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, {
       description: 'The amount of blob gas used in transactions',
       unit: 'gas',
       valueType: ValueType.INT,
     });
 
-    this.txBlobDataGasCost = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, {
+    this.txBlobDataGasCost = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, {
       description: 'The gas cost of blobs in transactions',
       unit: 'gwei',
       valueType: ValueType.INT,
     });
 
-    this.txTotalFee = meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE, {
+    this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE, {
       description: 'How much L1 tx costs',
       unit: 'gwei',
       valueType: ValueType.DOUBLE,
@@ -112,7 +192,7 @@ export class ProverNodeMetrics {
       },
     });
 
-    this.senderBalance = meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, {
+    this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, {
       unit: 'eth',
       description: 'The balance of the sender address',
       valueType: ValueType.DOUBLE,
@@ -130,13 +210,6 @@ export class ProverNodeMetrics {
     this.recordTx(durationMs, stats);
   }
 
-  public recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number) {
-    this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
-    this.provingJobDuration.record(totalTimeMs / 1000);
-    this.provingJobBlocks.record(Math.floor(numBlocks));
-    this.provingJobTransactions.record(Math.floor(numTxs));
-  }
-
   public recordSenderBalance(wei: bigint, senderAddress: string) {
     const eth = parseFloat(formatEther(wei, 'wei'));
     this.senderBalance.record(eth, {

package/src/monitors/epoch-monitor.ts

@@ -11,7 +11,7 @@ import {
 } from '@aztec/telemetry-client';
 
 export interface EpochMonitorHandler {
-  handleEpochReadyToProve(epochNumber: bigint): Promise<void>;
+  handleEpochReadyToProve(epochNumber: bigint): Promise<boolean>;
 }
 
 /**
@@ -71,6 +71,7 @@ export class EpochMonitor implements Traceable {
   @trackSpan('EpochMonitor.work')
   public async work() {
     const { epochToProve, blockNumber, slotNumber } = await this.getEpochNumberToProve();
+    this.log.debug(`Epoch to prove: ${epochToProve}`, { blockNumber, slotNumber });
     if (epochToProve === undefined) {
       this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { blockNumber });
       return;
@@ -87,8 +88,9 @@ export class EpochMonitor implements Traceable {
     }
 
     this.log.debug(`Epoch ${epochToProve} is ready to be proven`);
-    await this.handler?.handleEpochReadyToProve(epochToProve);
-    this.latestEpochNumber = epochToProve;
+    if (await this.handler?.handleEpochReadyToProve(epochToProve)) {
+      this.latestEpochNumber = epochToProve;
+    }
   }
 
   private async getEpochNumberToProve() {

package/src/prover-coordination/combined-prover-coordination.ts

@@ -0,0 +1,160 @@
+import { asyncPool } from '@aztec/foundation/async-pool';
+import { createLogger } from '@aztec/foundation/log';
+import type { P2P } from '@aztec/p2p';
+import type { P2PClient, ProverCoordination } from '@aztec/stdlib/interfaces/server';
+import { type Tx, TxHash } from '@aztec/stdlib/tx';
+
+export type CombinedCoordinationOptions = {
+  // These options apply to http tx gathering only
+  txGatheringBatchSize: number;
+  txGatheringMaxParallelRequestsPerNode: number;
+};
+
+interface CoordinationPool {
+  getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
+  hasTxsInPool(txHashes: TxHash[]): Promise<boolean[]>;
+  getTxsByHashFromPool(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
+  addTxs(txs: Tx[]): Promise<void>;
+}
+
+export interface TxSource {
+  getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
+}
+
+// Wraps the p2p client into a coordination pool
+class P2PCoordinationPool implements CoordinationPool {
+  constructor(private readonly p2p: P2P) {}
+  getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
+    return this.p2p.getTxsByHash(txHashes);
+  }
+  hasTxsInPool(txHashes: TxHash[]): Promise<boolean[]> {
+    return this.p2p.hasTxsInPool(txHashes);
+  }
+  getTxsByHashFromPool(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
+    return this.p2p.getTxsByHashFromPool(txHashes);
+  }
+  addTxs(txs: Tx[]): Promise<void> {
+    return this.p2p.addTxs(txs);
+  }
+}
+
+// Wraps an in memory tx pool into a coordination pool. Used for testing when no p2p/tx pool is available.
+class InMemoryCoordinationPool implements CoordinationPool {
+  private txs: Map<string, Tx> = new Map();
+  getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
+    return Promise.resolve(txHashes.map(hash => this.txs.get(hash.toString())));
+  }
+  hasTxsInPool(txHashes: TxHash[]): Promise<boolean[]> {
+    return Promise.resolve(txHashes.map(hash => this.txs.has(hash.toString())));
+  }
+  getTxsByHashFromPool(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
+    return this.getTxsByHash(txHashes);
+  }
+  async addTxs(txs: Tx[]): Promise<void> {
+    const hashes = await Promise.all(txs.map(tx => tx.getTxHash()));
+    txs.forEach((tx, index) => this.txs.set(hashes[index].toString(), tx));
+  }
+}
+
+// Class to implement combined transaction retrieval from p2p and any available nodes
+export class CombinedProverCoordination implements ProverCoordination {
+  constructor(
+    public readonly p2p: P2P | undefined,
+    public readonly aztecNodes: TxSource[],
+    private readonly options: CombinedCoordinationOptions = {
+      txGatheringBatchSize: 10,
+      txGatheringMaxParallelRequestsPerNode: 10,
+    },
+    private readonly log = createLogger('prover-node:combined-prover-coordination'),
+  ) {}
+
+  public getP2PClient(): P2PClient | undefined {
+    return this.p2p;
+  }
+
+  public async getTxsByHash(txHashes: TxHash[]): Promise<Tx[]> {
+    const pool = this.p2p ? new P2PCoordinationPool(this.p2p) : new InMemoryCoordinationPool();
+    await this.#gatherTxs(txHashes, pool);
+    const availability = await pool.hasTxsInPool(txHashes);
+    const notFound = txHashes.filter((_, index) => !availability[index]);
+    if (notFound.length > 0) {
+      throw new Error(`Could not find txs: ${notFound.map(tx => tx.toString())}`);
+    }
+    const txs = await pool.getTxsByHashFromPool(txHashes);
+    return txs.filter(tx => tx !== undefined) as Tx[];
+  }
+
+  public async gatherTxs(txHashes: TxHash[]): Promise<void> {
+    const pool = this.p2p ? new P2PCoordinationPool(this.p2p) : new InMemoryCoordinationPool();
+    await this.#gatherTxs(txHashes, pool);
+  }
+
+  async #gatherTxs(txHashes: TxHash[], pool: CoordinationPool): Promise<void> {
+    const availability = await pool.hasTxsInPool(txHashes);
+    const notFound = txHashes.filter((_, index) => !availability[index]);
+    const txsToFind = new Set(notFound.map(tx => tx.toString()));
+    if (txsToFind.size === 0) {
+      this.log.info(`Check for ${txHashes.length} txs found all in the pool}`);
+      return;
+    }
+    this.log.info(`Check for ${txHashes.length} txs found ${txsToFind.size} missing. Will gather from nodes and p2p`);
+    const originalToFind = txsToFind.size;
+    await this.#gatherTxsFromAllNodes(txsToFind, pool);
+    if (txsToFind.size === 0) {
+      this.log.info(`Found all ${originalToFind} txs directly from nodes`);
+      return;
+    }
+    const toFindFromP2P = txsToFind.size;
+    this.log.verbose(`Gathering ${toFindFromP2P} txs from p2p network`);
+    const foundFromP2P = await pool.getTxsByHash([...txsToFind].map(tx => TxHash.fromString(tx)));
+    const numFoundFromNodes = originalToFind - toFindFromP2P;
+    const numNotFound = toFindFromP2P - foundFromP2P.length;
+    if (numNotFound === 0) {
+      this.log.info(
+        `Found all ${originalToFind} txs. ${numFoundFromNodes} from nodes, ${foundFromP2P.length} from p2p`,
+      );
+      return;
+    }
+    this.log.warn(
+      `Failed to find ${numNotFound} txs from any source. Found ${foundFromP2P.length} from p2p and ${numFoundFromNodes} from nodes`,
+    );
+  }
+
+  async #gatherTxsFromAllNodes(txsToFind: Set<string>, pool: CoordinationPool) {
+    if (txsToFind.size === 0 || this.aztecNodes.length === 0) {
+      return;
+    }
+    await Promise.all(this.aztecNodes.map(aztecNode => this.#gatherTxsFromNode(txsToFind, aztecNode, pool)));
+  }
+
+  async #gatherTxsFromNode(txsToFind: Set<string>, aztecNode: TxSource, pool: CoordinationPool) {
+    const totalTxsRequired = txsToFind.size;
+
+    // It's possible that the set is empty as we already found the txs
+    if (totalTxsRequired === 0) {
+      return;
+    }
+    let totalTxsGathered = 0;
+
+    const batches: string[][] = [];
+    const allTxs: string[] = [...txsToFind];
+    while (allTxs.length) {
+      const batch = allTxs.splice(0, this.options.txGatheringBatchSize);
+      batches.push(batch);
+    }
+
+    await asyncPool(this.options.txGatheringMaxParallelRequestsPerNode, batches, async batch => {
+      try {
+        const txs = (await aztecNode.getTxsByHash(batch.map(b => TxHash.fromString(b)))).filter((tx): tx is Tx => !!tx);
+        const hashes = await Promise.all(txs.map(tx => tx.getTxHash()));
+        await pool.addTxs(txs);
+        hashes.forEach(hash => txsToFind.delete(hash.toString()));
+        totalTxsGathered += txs.length;
+      } catch (err) {
+        this.log.error(`Error gathering txs from aztec node: ${err}`);
+        return;
+      }
+    });
+    this.log.verbose(`Gathered ${totalTxsGathered} of ${totalTxsRequired} txs from a node`);
+  }
+}

package/src/prover-coordination/config.ts

@@ -1,14 +1,14 @@
 import { type ConfigMappingsType, getConfigFromMappings } from '@aztec/foundation/config';
 
 export type ProverCoordinationConfig = {
-  proverCoordinationNodeUrl: string | undefined;
+  proverCoordinationNodeUrls: string[];
 };
 
 export const proverCoordinationConfigMappings: ConfigMappingsType<ProverCoordinationConfig> = {
-  proverCoordinationNodeUrl: {
-    env: 'PROVER_COORDINATION_NODE_URL',
-    description: 'The URL of the tx provider node',
-    parseEnv: (val: string) => val,
+  proverCoordinationNodeUrls: {
+    env: 'PROVER_COORDINATION_NODE_URLS',
+    description: 'The URLs of the tx provider nodes',
+    parseEnv: (val: string) => val.split(',').map(url => url.trim().replace(/\/$/, '')),
   },
 };