@aztec/prover-node 0.85.0 → 0.86.0-nightly.20250426

package/src/job/epoch-proving-job.ts

@@ -10,14 +10,14 @@ import {
   EpochProvingJobTerminalState,
   type ForkMerkleTreeOperations,
 } from '@aztec/stdlib/interfaces/server';
-import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
 import type { ProcessedTx, Tx } from '@aztec/stdlib/tx';
 import { Attributes, type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client';
 
 import * as crypto from 'node:crypto';
 
-import type { ProverNodeMetrics } from '../metrics.js';
+import type { ProverNodeJobMetrics } from '../metrics.js';
 import type { ProverNodePublisher } from '../prover-node-publisher.js';
+import { type EpochProvingJobData, validateEpochProvingJobData } from './epoch-proving-job-data.js';
 
 /**
  * Job that grabs a range of blocks from the unfinalised chain from L1, gets their txs given their hashes,
@@ -36,21 +36,19 @@ export class EpochProvingJob implements Traceable {
   public readonly tracer: Tracer;
 
   constructor(
-    private dbProvider: ForkMerkleTreeOperations,
-    private epochNumber: bigint,
-    private blocks: L2Block[],
-    private txs: Tx[],
+    private data: EpochProvingJobData,
+    private dbProvider: Pick<ForkMerkleTreeOperations, 'fork'>,
     private prover: EpochProver,
     private publicProcessorFactory: PublicProcessorFactory,
-    private publisher: ProverNodePublisher,
-    private l2BlockSource: L2BlockSource,
-    private l1ToL2MessageSource: L1ToL2MessageSource,
-    private metrics: ProverNodeMetrics,
+    private publisher: Pick<ProverNodePublisher, 'submitEpochProof'>,
+    private l2BlockSource: L2BlockSource | undefined,
+    private metrics: ProverNodeJobMetrics,
     private deadline: Date | undefined,
     private config: { parallelBlockLimit: number } = { parallelBlockLimit: 32 },
   ) {
+    validateEpochProvingJobData(data);
     this.uuid = crypto.randomUUID();
-    this.tracer = metrics.client.getTracer('EpochProvingJob');
+    this.tracer = metrics.tracer;
   }
 
   public getId(): string {
@@ -62,18 +60,34 @@ export class EpochProvingJob implements Traceable {
   }
 
   public getEpochNumber(): bigint {
-    return this.epochNumber;
+    return this.data.epochNumber;
   }
 
   public getDeadline(): Date | undefined {
     return this.deadline;
   }
 
+  public getProvingData(): EpochProvingJobData {
+    return this.data;
+  }
+
+  private get epochNumber() {
+    return this.data.epochNumber;
+  }
+
+  private get blocks() {
+    return this.data.blocks;
+  }
+
+  private get txs() {
+    return this.data.txs;
+  }
+
   /**
    * Proves the given epoch and submits the proof to L1.
    */
   @trackSpan('EpochProvingJob.run', function () {
-    return { [Attributes.EPOCH_NUMBER]: Number(this.epochNumber) };
+    return { [Attributes.EPOCH_NUMBER]: Number(this.data.epochNumber) };
   })
   public async run() {
     this.scheduleDeadlineStop();
@@ -105,8 +119,8 @@ export class EpochProvingJob implements Traceable {
 
         const globalVariables = block.header.globalVariables;
         const txs = await this.getTxs(block);
-        const l1ToL2Messages = await this.getL1ToL2Messages(block);
-        const previousHeader = (await this.getBlockHeader(block.number - 1))!;
+        const l1ToL2Messages = this.getL1ToL2Messages(block);
+        const previousHeader = this.getBlockHeader(block.number - 1)!;
 
         this.log.verbose(`Starting processing block ${block.number}`, {
           number: block.number,
@@ -221,10 +235,16 @@ export class EpochProvingJob implements Traceable {
    * If those change, stops the proving job with a `rerun` state, so the node re-enqueues it.
    */
   private async scheduleEpochCheck() {
-    const intervalMs = Math.ceil((await this.l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
+    const l2BlockSource = this.l2BlockSource;
+    if (!l2BlockSource) {
+      this.log.warn(`No L2 block source available, skipping epoch check`);
+      return;
+    }
+
+    const intervalMs = Math.ceil((await l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
     this.epochCheckPromise = new RunningPromise(
       async () => {
-        const blocks = await this.l2BlockSource.getBlockHeadersForEpoch(this.epochNumber);
+        const blocks = await l2BlockSource.getBlockHeadersForEpoch(this.epochNumber);
         const blockHashes = await Promise.all(blocks.map(block => block.hash()));
         const thisBlockHashes = await Promise.all(this.blocks.map(block => block.hash()));
         if (
@@ -246,12 +266,22 @@ export class EpochProvingJob implements Traceable {
     this.log.verbose(`Scheduled epoch check for epoch ${this.epochNumber} every ${intervalMs}ms`);
   }
 
-  /* Returns the header for the given block number, or the genesis block for block zero. */
-  private async getBlockHeader(blockNumber: number) {
-    if (blockNumber === 0) {
-      return (await this.dbProvider.fork()).getInitialHeader();
+  /* Returns the header for the given block number based on the epoch proving job data. */
+  private getBlockHeader(blockNumber: number) {
+    const block = this.blocks.find(b => b.number === blockNumber);
+    if (block) {
+      return block.header;
     }
-    return this.l2BlockSource.getBlockHeader(blockNumber);
+
+    if (blockNumber === Number(this.data.previousBlockHeader.getBlockNumber())) {
+      return this.data.previousBlockHeader;
+    }
+
+    throw new Error(
+      `Block header not found for block number ${blockNumber} (got ${this.blocks
+        .map(b => b.number)
+        .join(', ')} and previous header ${this.data.previousBlockHeader.getBlockNumber()})`,
+    );
   }
 
   private async getTxs(block: L2Block): Promise<Tx[]> {
@@ -263,7 +293,7 @@ export class EpochProvingJob implements Traceable {
   }
 
   private getL1ToL2Messages(block: L2Block) {
-    return this.l1ToL2MessageSource.getL1ToL2Messages(BigInt(block.number));
+    return this.data.l1ToL2Messages[block.number];
   }
 
   private async processTxs(publicProcessor: PublicProcessor, txs: Tx[]): Promise<ProcessedTx[]> {

package/src/metrics.ts

@@ -1,107 +1,187 @@
+import type { RollupContract } from '@aztec/ethereum';
+import type { EthAddress } from '@aztec/foundation/eth-address';
 import { createLogger } from '@aztec/foundation/log';
 import type { L1PublishProofStats, L1PublishStats } from '@aztec/stdlib/stats';
 import {
   Attributes,
+  type BatchObservableResult,
   type Gauge,
   type Histogram,
+  type Meter,
   Metrics,
+  type ObservableGauge,
   type TelemetryClient,
+  type Tracer,
   type UpDownCounter,
   ValueType,
 } from '@aztec/telemetry-client';
 
-import { formatEther } from 'viem';
+import { formatEther, formatUnits } from 'viem';
 
-export class ProverNodeMetrics {
+export class ProverNodeJobMetrics {
   proverEpochExecutionDuration: Histogram;
   provingJobDuration: Histogram;
   provingJobBlocks: Gauge;
   provingJobTransactions: Gauge;
 
-  gasPrice: Histogram;
-  txCount: UpDownCounter;
-  txDuration: Histogram;
-  txGas: Histogram;
-  txCalldataSize: Histogram;
-  txCalldataGas: Histogram;
-  txBlobDataGasUsed: Histogram;
-  txBlobDataGasCost: Histogram;
-  txTotalFee: Histogram;
-
-  private senderBalance: Gauge;
-
   constructor(
-    public readonly client: TelemetryClient,
-    name = 'ProverNode',
+    private meter: Meter,
+    public readonly tracer: Tracer,
     private logger = createLogger('prover-node:publisher:metrics'),
   ) {
-    const meter = client.getMeter(name);
-    this.proverEpochExecutionDuration = meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, {
+    this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, {
       description: 'Duration of execution of an epoch by the prover',
       unit: 'ms',
       valueType: ValueType.INT,
     });
-    this.provingJobDuration = meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
+    this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
       description: 'Duration of proving job',
       unit: 's',
       valueType: ValueType.DOUBLE,
     });
-    this.provingJobBlocks = meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
+    this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
       description: 'Number of blocks in a proven epoch',
       valueType: ValueType.INT,
     });
-    this.provingJobTransactions = meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
+    this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
       description: 'Number of transactions in a proven epoch',
       valueType: ValueType.INT,
     });
+  }
+
+  public recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number) {
+    this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
+    this.provingJobDuration.record(totalTimeMs / 1000);
+    this.provingJobBlocks.record(Math.floor(numBlocks));
+    this.provingJobTransactions.record(Math.floor(numTxs));
+  }
+}
+
+export class ProverNodeRewardsMetrics {
+  private rewards: ObservableGauge;
+  private accumulatedRewards: UpDownCounter;
+  private prevEpoch = -1n;
+  private proofSubmissionWindow = 0n;
+
+  constructor(
+    private meter: Meter,
+    private coinbase: EthAddress,
+    private rollup: RollupContract,
+    private logger = createLogger('prover-node:publisher:metrics'),
+  ) {
+    this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH, {
+      valueType: ValueType.DOUBLE,
+      description: 'The rewards earned',
+    });
+
+    this.accumulatedRewards = this.meter.createUpDownCounter(Metrics.PROVER_NODE_REWARDS_TOTAL, {
+      valueType: ValueType.DOUBLE,
+      description: 'The rewards earned (total)',
+    });
+  }
+
+  public async start() {
+    this.proofSubmissionWindow = await this.rollup.getProofSubmissionWindow();
+    this.meter.addBatchObservableCallback(this.observe, [this.rewards]);
+  }
+
+  public stop() {
+    this.meter.removeBatchObservableCallback(this.observe, [this.rewards]);
+  }
+
+  private observe = async (observer: BatchObservableResult): Promise<void> => {
+    const slot = await this.rollup.getSlotNumber();
+
+    // look at the prev epoch so that we get an accurate value, after proof submission window has closed
+    if (slot > this.proofSubmissionWindow) {
+      const closedEpoch = await this.rollup.getEpochNumberForSlotNumber(slot - this.proofSubmissionWindow);
+      const rewards = await this.rollup.getSpecificProverRewardsForEpoch(closedEpoch, this.coinbase);
+
+      const fmt = parseFloat(formatUnits(rewards, 18));
+
+      observer.observe(this.rewards, fmt, {
+        [Attributes.COINBASE]: this.coinbase.toString(),
+      });
+
+      // only accumulate once per epoch
+      if (closedEpoch > this.prevEpoch) {
+        this.prevEpoch = closedEpoch;
+        this.accumulatedRewards.add(fmt, {
+          [Attributes.COINBASE]: this.coinbase.toString(),
+        });
+      }
+    }
+  };
+}
 
-    this.gasPrice = meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, {
+export class ProverNodePublisherMetrics {
+  gasPrice: Histogram;
+  txCount: UpDownCounter;
+  txDuration: Histogram;
+  txGas: Histogram;
+  txCalldataSize: Histogram;
+  txCalldataGas: Histogram;
+  txBlobDataGasUsed: Histogram;
+  txBlobDataGasCost: Histogram;
+  txTotalFee: Histogram;
+
+  private senderBalance: Gauge;
+  private meter: Meter;
+
+  constructor(
+    public readonly client: TelemetryClient,
+    name = 'ProverNode',
+    private logger = createLogger('prover-node:publisher:metrics'),
+  ) {
+    this.meter = client.getMeter(name);
+
+    this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, {
       description: 'The gas price used for transactions',
       unit: 'gwei',
       valueType: ValueType.DOUBLE,
     });
 
-    this.txCount = meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, {
+    this.txCount = this.meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, {
       description: 'The number of transactions processed',
     });
 
-    this.txDuration = meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, {
+    this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, {
       description: 'The duration of transaction processing',
       unit: 'ms',
       valueType: ValueType.INT,
     });
 
-    this.txGas = meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, {
+    this.txGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, {
       description: 'The gas consumed by transactions',
       unit: 'gas',
       valueType: ValueType.INT,
     });
 
-    this.txCalldataSize = meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, {
+    this.txCalldataSize = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, {
       description: 'The size of the calldata in transactions',
       unit: 'By',
       valueType: ValueType.INT,
     });
 
-    this.txCalldataGas = meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, {
+    this.txCalldataGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, {
       description: 'The gas consumed by the calldata in transactions',
       unit: 'gas',
       valueType: ValueType.INT,
     });
 
-    this.txBlobDataGasUsed = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, {
+    this.txBlobDataGasUsed = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, {
       description: 'The amount of blob gas used in transactions',
       unit: 'gas',
       valueType: ValueType.INT,
     });
 
-    this.txBlobDataGasCost = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, {
+    this.txBlobDataGasCost = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, {
       description: 'The gas cost of blobs in transactions',
       unit: 'gwei',
       valueType: ValueType.INT,
     });
 
-    this.txTotalFee = meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE, {
+    this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE, {
       description: 'How much L1 tx costs',
       unit: 'gwei',
       valueType: ValueType.DOUBLE,
@@ -112,7 +192,7 @@ export class ProverNodeMetrics {
       },
     });
 
-    this.senderBalance = meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, {
+    this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, {
       unit: 'eth',
       description: 'The balance of the sender address',
       valueType: ValueType.DOUBLE,
@@ -130,13 +210,6 @@ export class ProverNodeMetrics {
     this.recordTx(durationMs, stats);
   }
 
-  public recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number) {
-    this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
-    this.provingJobDuration.record(totalTimeMs / 1000);
-    this.provingJobBlocks.record(Math.floor(numBlocks));
-    this.provingJobTransactions.record(Math.floor(numTxs));
-  }
-
   public recordSenderBalance(wei: bigint, senderAddress: string) {
     const eth = parseFloat(formatEther(wei, 'wei'));
     this.senderBalance.record(eth, {

package/src/monitors/epoch-monitor.ts

@@ -11,7 +11,7 @@ import {
 } from '@aztec/telemetry-client';
 
 export interface EpochMonitorHandler {
-  handleEpochReadyToProve(epochNumber: bigint): Promise<void>;
+  handleEpochReadyToProve(epochNumber: bigint): Promise<boolean>;
 }
 
 /**
@@ -71,6 +71,7 @@ export class EpochMonitor implements Traceable {
   @trackSpan('EpochMonitor.work')
   public async work() {
     const { epochToProve, blockNumber, slotNumber } = await this.getEpochNumberToProve();
+    this.log.debug(`Epoch to prove: ${epochToProve}`, { blockNumber, slotNumber });
     if (epochToProve === undefined) {
       this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { blockNumber });
       return;
@@ -87,8 +88,9 @@ export class EpochMonitor implements Traceable {
     }
 
     this.log.debug(`Epoch ${epochToProve} is ready to be proven`);
-    await this.handler?.handleEpochReadyToProve(epochToProve);
-    this.latestEpochNumber = epochToProve;
+    if (await this.handler?.handleEpochReadyToProve(epochToProve)) {
+      this.latestEpochNumber = epochToProve;
+    }
   }
 
   private async getEpochNumberToProve() {

package/src/prover-coordination/combined-prover-coordination.ts

@@ -0,0 +1,160 @@
+import { asyncPool } from '@aztec/foundation/async-pool';
+import { createLogger } from '@aztec/foundation/log';
+import type { P2P } from '@aztec/p2p';
+import type { P2PClient, ProverCoordination } from '@aztec/stdlib/interfaces/server';
+import { type Tx, TxHash } from '@aztec/stdlib/tx';
+
+export type CombinedCoordinationOptions = {
+  // These options apply to http tx gathering only
+  txGatheringBatchSize: number;
+  txGatheringMaxParallelRequestsPerNode: number;
+};
+
+interface CoordinationPool {
+  getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
+  hasTxsInPool(txHashes: TxHash[]): Promise<boolean[]>;
+  getTxsByHashFromPool(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
+  addTxs(txs: Tx[]): Promise<void>;
+}
+
+export interface TxSource {
+  getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
+}
+
+// Wraps the p2p client into a coordination pool
+class P2PCoordinationPool implements CoordinationPool {
+  constructor(private readonly p2p: P2P) {}
+  getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
+    return this.p2p.getTxsByHash(txHashes);
+  }
+  hasTxsInPool(txHashes: TxHash[]): Promise<boolean[]> {
+    return this.p2p.hasTxsInPool(txHashes);
+  }
+  getTxsByHashFromPool(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
+    return this.p2p.getTxsByHashFromPool(txHashes);
+  }
+  addTxs(txs: Tx[]): Promise<void> {
+    return this.p2p.addTxs(txs);
+  }
+}
+
+// Wraps an in memory tx pool into a coordination pool. Used for testing when no p2p/tx pool is available.
+class InMemoryCoordinationPool implements CoordinationPool {
+  private txs: Map<string, Tx> = new Map();
+  getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
+    return Promise.resolve(txHashes.map(hash => this.txs.get(hash.toString())));
+  }
+  hasTxsInPool(txHashes: TxHash[]): Promise<boolean[]> {
+    return Promise.resolve(txHashes.map(hash => this.txs.has(hash.toString())));
+  }
+  getTxsByHashFromPool(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
+    return this.getTxsByHash(txHashes);
+  }
+  async addTxs(txs: Tx[]): Promise<void> {
+    const hashes = await Promise.all(txs.map(tx => tx.getTxHash()));
+    txs.forEach((tx, index) => this.txs.set(hashes[index].toString(), tx));
+  }
+}
+
+// Class to implement combined transaction retrieval from p2p and any available nodes
+export class CombinedProverCoordination implements ProverCoordination {
+  constructor(
+    public readonly p2p: P2P | undefined,
+    public readonly aztecNodes: TxSource[],
+    private readonly options: CombinedCoordinationOptions = {
+      txGatheringBatchSize: 10,
+      txGatheringMaxParallelRequestsPerNode: 10,
+    },
+    private readonly log = createLogger('prover-node:combined-prover-coordination'),
+  ) {}
+
+  public getP2PClient(): P2PClient | undefined {
+    return this.p2p;
+  }
+
+  public async getTxsByHash(txHashes: TxHash[]): Promise<Tx[]> {
+    const pool = this.p2p ? new P2PCoordinationPool(this.p2p) : new InMemoryCoordinationPool();
+    await this.#gatherTxs(txHashes, pool);
+    const availability = await pool.hasTxsInPool(txHashes);
+    const notFound = txHashes.filter((_, index) => !availability[index]);
+    if (notFound.length > 0) {
+      throw new Error(`Could not find txs: ${notFound.map(tx => tx.toString())}`);
+    }
+    const txs = await pool.getTxsByHashFromPool(txHashes);
+    return txs.filter(tx => tx !== undefined) as Tx[];
+  }
+
+  public async gatherTxs(txHashes: TxHash[]): Promise<void> {
+    const pool = this.p2p ? new P2PCoordinationPool(this.p2p) : new InMemoryCoordinationPool();
+    await this.#gatherTxs(txHashes, pool);
+  }
+
+  async #gatherTxs(txHashes: TxHash[], pool: CoordinationPool): Promise<void> {
+    const availability = await pool.hasTxsInPool(txHashes);
+    const notFound = txHashes.filter((_, index) => !availability[index]);
+    const txsToFind = new Set(notFound.map(tx => tx.toString()));
+    if (txsToFind.size === 0) {
+      this.log.info(`Check for ${txHashes.length} txs found all in the pool`);
+      return;
+    }
+    this.log.info(`Check for ${txHashes.length} txs found ${txsToFind.size} missing. Will gather from nodes and p2p`);
+    const originalToFind = txsToFind.size;
+    await this.#gatherTxsFromAllNodes(txsToFind, pool);
+    if (txsToFind.size === 0) {
+      this.log.info(`Found all ${originalToFind} txs directly from nodes`);
+      return;
+    }
+    const toFindFromP2P = txsToFind.size;
+    this.log.verbose(`Gathering ${toFindFromP2P} txs from p2p network`);
+    const foundFromP2P = await pool.getTxsByHash([...txsToFind].map(tx => TxHash.fromString(tx)));
+    const numFoundFromNodes = originalToFind - toFindFromP2P;
+    const numNotFound = toFindFromP2P - foundFromP2P.length;
+    if (numNotFound === 0) {
+      this.log.info(
+        `Found all ${originalToFind} txs. ${numFoundFromNodes} from nodes, ${foundFromP2P.length} from p2p`,
+      );
+      return;
+    }
+    this.log.warn(
+      `Failed to find ${numNotFound} txs from any source. Found ${foundFromP2P.length} from p2p and ${numFoundFromNodes} from nodes`,
+    );
+  }
+
+  async #gatherTxsFromAllNodes(txsToFind: Set<string>, pool: CoordinationPool) {
+    if (txsToFind.size === 0 || this.aztecNodes.length === 0) {
+      return;
+    }
+    await Promise.all(this.aztecNodes.map(aztecNode => this.#gatherTxsFromNode(txsToFind, aztecNode, pool)));
+  }
+
+  async #gatherTxsFromNode(txsToFind: Set<string>, aztecNode: TxSource, pool: CoordinationPool) {
+    const totalTxsRequired = txsToFind.size;
+
+    // It's possible that the set is empty as we already found the txs
+    if (totalTxsRequired === 0) {
+      return;
+    }
+    let totalTxsGathered = 0;
+
+    const batches: string[][] = [];
+    const allTxs: string[] = [...txsToFind];
+    while (allTxs.length) {
+      const batch = allTxs.splice(0, this.options.txGatheringBatchSize);
+      batches.push(batch);
+    }
+
+    await asyncPool(this.options.txGatheringMaxParallelRequestsPerNode, batches, async batch => {
+      try {
+        const txs = (await aztecNode.getTxsByHash(batch.map(b => TxHash.fromString(b)))).filter((tx): tx is Tx => !!tx);
+        const hashes = await Promise.all(txs.map(tx => tx.getTxHash()));
+        await pool.addTxs(txs);
+        hashes.forEach(hash => txsToFind.delete(hash.toString()));
+        totalTxsGathered += txs.length;
+      } catch (err) {
+        this.log.error(`Error gathering txs from aztec node: ${err}`);
+        return;
+      }
+    });
+    this.log.verbose(`Gathered ${totalTxsGathered} of ${totalTxsRequired} txs from a node`);
+  }
+}

package/src/prover-coordination/config.ts

@@ -1,14 +1,15 @@
 import { type ConfigMappingsType, getConfigFromMappings } from '@aztec/foundation/config';
 
 export type ProverCoordinationConfig = {
-  proverCoordinationNodeUrl: string | undefined;
+  proverCoordinationNodeUrls: string[];
 };
 
 export const proverCoordinationConfigMappings: ConfigMappingsType<ProverCoordinationConfig> = {
-  proverCoordinationNodeUrl: {
-    env: 'PROVER_COORDINATION_NODE_URL',
-    description: 'The URL of the tx provider node',
-    parseEnv: (val: string) => val,
+  proverCoordinationNodeUrls: {
+    env: 'PROVER_COORDINATION_NODE_URLS',
+    description: 'The URLs of the tx provider nodes',
+    parseEnv: (val: string) => val.split(',').map(url => url.trim().replace(/\/$/, '')),
+    defaultValue: [],
   },
 };