@aztec/prover-node 0.85.0-alpha-testnet.2 → 0.85.0-alpha-testnet.4

package/dest/job/epoch-proving-job.js

@@ -11,20 +11,18 @@ import { Timer } from '@aztec/foundation/timer';
 import { EpochProvingJobTerminalState } from '@aztec/stdlib/interfaces/server';
 import { Attributes, trackSpan } from '@aztec/telemetry-client';
 import * as crypto from 'node:crypto';
+import { validateEpochProvingJobData } from './epoch-proving-job-data.js';
 /**
  * Job that grabs a range of blocks from the unfinalised chain from L1, gets their txs given their hashes,
  * re-executes their public calls, generates a rollup proof, and submits it to L1. This job will update the
  * world state as part of public call execution via the public processor.
  */ export class EpochProvingJob {
+    data;
     dbProvider;
-    epochNumber;
-    blocks;
-    txs;
     prover;
     publicProcessorFactory;
     publisher;
     l2BlockSource;
-    l1ToL2MessageSource;
     metrics;
     deadline;
     config;
@@ -35,25 +33,23 @@ import * as crypto from 'node:crypto';
     epochCheckPromise;
     deadlineTimeoutHandler;
     tracer;
-    constructor(dbProvider, epochNumber, blocks, txs, prover, publicProcessorFactory, publisher, l2BlockSource, l1ToL2MessageSource, metrics, deadline, config = {
+    constructor(data, dbProvider, prover, publicProcessorFactory, publisher, l2BlockSource, metrics, deadline, config = {
         parallelBlockLimit: 32
     }){
+        this.data = data;
         this.dbProvider = dbProvider;
-        this.epochNumber = epochNumber;
-        this.blocks = blocks;
-        this.txs = txs;
         this.prover = prover;
         this.publicProcessorFactory = publicProcessorFactory;
         this.publisher = publisher;
         this.l2BlockSource = l2BlockSource;
-        this.l1ToL2MessageSource = l1ToL2MessageSource;
         this.metrics = metrics;
         this.deadline = deadline;
         this.config = config;
         this.state = 'initialized';
         this.log = createLogger('prover-node:epoch-proving-job');
+        validateEpochProvingJobData(data);
         this.uuid = crypto.randomUUID();
-        this.tracer = metrics.client.getTracer('EpochProvingJob');
+        this.tracer = metrics.tracer;
     }
     getId() {
         return this.uuid;
@@ -62,11 +58,23 @@ import * as crypto from 'node:crypto';
         return this.state;
     }
     getEpochNumber() {
-        return this.epochNumber;
+        return this.data.epochNumber;
     }
     getDeadline() {
         return this.deadline;
     }
+    getProvingData() {
+        return this.data;
+    }
+    get epochNumber() {
+        return this.data.epochNumber;
+    }
+    get blocks() {
+        return this.data.blocks;
+    }
+    get txs() {
+        return this.data.txs;
+    }
     /**
    * Proves the given epoch and submits the proof to L1.
    */ async run() {
@@ -97,8 +105,8 @@ import * as crypto from 'node:crypto';
                 this.checkState();
                 const globalVariables = block.header.globalVariables;
                 const txs = await this.getTxs(block);
-                const l1ToL2Messages = await this.getL1ToL2Messages(block);
-                const previousHeader = await this.getBlockHeader(block.number - 1);
+                const l1ToL2Messages = this.getL1ToL2Messages(block);
+                const previousHeader = this.getBlockHeader(block.number - 1);
                 this.log.verbose(`Starting processing block ${block.number}`, {
                     number: block.number,
                     blockHash: (await block.hash()).toString(),
@@ -217,9 +225,14 @@ import * as crypto from 'node:crypto';
    * Kicks off a running promise that queries the archiver for the set of L2 blocks of the current epoch.
    * If those change, stops the proving job with a `rerun` state, so the node re-enqueues it.
    */ async scheduleEpochCheck() {
-        const intervalMs = Math.ceil((await this.l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
+        const l2BlockSource = this.l2BlockSource;
+        if (!l2BlockSource) {
+            this.log.warn(`No L2 block source available, skipping epoch check`);
+            return;
+        }
+        const intervalMs = Math.ceil((await l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
         this.epochCheckPromise = new RunningPromise(async ()=>{
-            const blocks = await this.l2BlockSource.getBlockHeadersForEpoch(this.epochNumber);
+            const blocks = await l2BlockSource.getBlockHeadersForEpoch(this.epochNumber);
             const blockHashes = await Promise.all(blocks.map((block)=>block.hash()));
             const thisBlockHashes = await Promise.all(this.blocks.map((block)=>block.hash()));
             if (blocks.length !== this.blocks.length || !blockHashes.every((block, i)=>block.equals(thisBlockHashes[i]))) {
@@ -234,11 +247,15 @@ import * as crypto from 'node:crypto';
         }, this.log, intervalMs).start();
         this.log.verbose(`Scheduled epoch check for epoch ${this.epochNumber} every ${intervalMs}ms`);
     }
-    /* Returns the header for the given block number, or the genesis block for block zero. */ async getBlockHeader(blockNumber) {
-        if (blockNumber === 0) {
-            return (await this.dbProvider.fork()).getInitialHeader();
+    /* Returns the header for the given block number based on the epoch proving job data. */ getBlockHeader(blockNumber) {
+        const block = this.blocks.find((b)=>b.number === blockNumber);
+        if (block) {
+            return block.header;
+        }
+        if (blockNumber === Number(this.data.previousBlockHeader.getBlockNumber())) {
+            return this.data.previousBlockHeader;
         }
-        return this.l2BlockSource.getBlockHeader(blockNumber);
+        throw new Error(`Block header not found for block number ${blockNumber} (got ${this.blocks.map((b)=>b.number).join(', ')} and previous header ${this.data.previousBlockHeader.getBlockNumber()})`);
     }
     async getTxs(block) {
         const txHashes = block.body.txEffects.map((tx)=>tx.txHash.toBigInt());
@@ -249,7 +266,7 @@ import * as crypto from 'node:crypto';
         return txsAndHashes.filter((txAndHash)=>txHashes.includes(txAndHash.hash.toBigInt())).map((txAndHash)=>txAndHash.tx);
     }
     getL1ToL2Messages(block) {
-        return this.l1ToL2MessageSource.getL1ToL2Messages(BigInt(block.number));
+        return this.data.l1ToL2Messages[block.number];
     }
     async processTxs(publicProcessor, txs) {
         const { deadline } = this;
@@ -269,7 +286,7 @@ import * as crypto from 'node:crypto';
 _ts_decorate([
     trackSpan('EpochProvingJob.run', function() {
         return {
-            [Attributes.EPOCH_NUMBER]: Number(this.epochNumber)
+            [Attributes.EPOCH_NUMBER]: Number(this.data.epochNumber)
         };
     })
 ], EpochProvingJob.prototype, "run", null);

package/dest/metrics.d.ts

@@ -1,12 +1,35 @@
+import type { RollupContract } from '@aztec/ethereum';
+import type { EthAddress } from '@aztec/foundation/eth-address';
 import type { L1PublishProofStats } from '@aztec/stdlib/stats';
-import { type Gauge, type Histogram, type TelemetryClient, type UpDownCounter } from '@aztec/telemetry-client';
-export declare class ProverNodeMetrics {
-    readonly client: TelemetryClient;
+import { type Gauge, type Histogram, type Meter, type TelemetryClient, type Tracer, type UpDownCounter } from '@aztec/telemetry-client';
+export declare class ProverNodeJobMetrics {
+    private meter;
+    readonly tracer: Tracer;
     private logger;
     proverEpochExecutionDuration: Histogram;
     provingJobDuration: Histogram;
     provingJobBlocks: Gauge;
     provingJobTransactions: Gauge;
+    constructor(meter: Meter, tracer: Tracer, logger?: import("@aztec/foundation/log").Logger);
+    recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number): void;
+}
+export declare class ProverNodeRewardsMetrics {
+    private meter;
+    private coinbase;
+    private rollup;
+    private logger;
+    private rewards;
+    private accumulatedRewards;
+    private prevEpoch;
+    private proofSubmissionWindow;
+    constructor(meter: Meter, coinbase: EthAddress, rollup: RollupContract, logger?: import("@aztec/foundation/log").Logger);
+    start(): Promise<void>;
+    stop(): void;
+    private observe;
+}
+export declare class ProverNodePublisherMetrics {
+    readonly client: TelemetryClient;
+    private logger;
     gasPrice: Histogram;
     txCount: UpDownCounter;
     txDuration: Histogram;
@@ -17,10 +40,10 @@ export declare class ProverNodeMetrics {
     txBlobDataGasCost: Histogram;
     txTotalFee: Histogram;
     private senderBalance;
+    private meter;
     constructor(client: TelemetryClient, name?: string, logger?: import("@aztec/foundation/log").Logger);
     recordFailedTx(): void;
     recordSubmitProof(durationMs: number, stats: L1PublishProofStats): void;
-    recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number): void;
     recordSenderBalance(wei: bigint, senderAddress: string): void;
     private recordTx;
 }

package/dest/metrics.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,mBAAmB,EAAkB,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAEL,KAAK,KAAK,EACV,KAAK,SAAS,EAEd,KAAK,eAAe,EACpB,KAAK,aAAa,EAEnB,MAAM,yBAAyB,CAAC;AAIjC,qBAAa,iBAAiB;aAmBV,MAAM,EAAE,eAAe;IAEvC,OAAO,CAAC,MAAM;IApBhB,4BAA4B,EAAE,SAAS,CAAC;IACxC,kBAAkB,EAAE,SAAS,CAAC;IAC9B,gBAAgB,EAAE,KAAK,CAAC;IACxB,sBAAsB,EAAE,KAAK,CAAC;IAE9B,QAAQ,EAAE,SAAS,CAAC;IACpB,OAAO,EAAE,aAAa,CAAC;IACvB,UAAU,EAAE,SAAS,CAAC;IACtB,KAAK,EAAE,SAAS,CAAC;IACjB,cAAc,EAAE,SAAS,CAAC;IAC1B,aAAa,EAAE,SAAS,CAAC;IACzB,iBAAiB,EAAE,SAAS,CAAC;IAC7B,iBAAiB,EAAE,SAAS,CAAC;IAC7B,UAAU,EAAE,SAAS,CAAC;IAEtB,OAAO,CAAC,aAAa,CAAQ;gBAGX,MAAM,EAAE,eAAe,EACvC,IAAI,SAAe,EACX,MAAM,yCAAgD;IAsFhE,cAAc;IAOd,iBAAiB,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB;IAIzD,gBAAgB,CAAC,eAAe,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAOhG,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM;IAO7D,OAAO,CAAC,QAAQ;CAuCjB"}
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAEhE,OAAO,KAAK,EAAE,mBAAmB,EAAkB,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAGL,KAAK,KAAK,EACV,KAAK,SAAS,EACd,KAAK,KAAK,EAGV,KAAK,eAAe,EACpB,KAAK,MAAM,EACX,KAAK,aAAa,EAEnB,MAAM,yBAAyB,CAAC;AAIjC,qBAAa,oBAAoB;IAO7B,OAAO,CAAC,KAAK;aACG,MAAM,EAAE,MAAM;IAC9B,OAAO,CAAC,MAAM;IARhB,4BAA4B,EAAE,SAAS,CAAC;IACxC,kBAAkB,EAAE,SAAS,CAAC;IAC9B,gBAAgB,EAAE,KAAK,CAAC;IACxB,sBAAsB,EAAE,KAAK,CAAC;gBAGpB,KAAK,EAAE,KAAK,EACJ,MAAM,EAAE,MAAM,EACtB,MAAM,yCAAgD;IAsBzD,gBAAgB,CAAC,eAAe,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;CAMxG;AAED,qBAAa,wBAAwB;IAOjC,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IAThB,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,kBAAkB,CAAgB;IAC1C,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,qBAAqB,CAAM;gBAGzB,KAAK,EAAE,KAAK,EACZ,QAAQ,EAAE,UAAU,EACpB,MAAM,EAAE,cAAc,EACtB,MAAM,yCAAgD;IAanD,KAAK;IAKX,IAAI;IAIX,OAAO,CAAC,OAAO,CAsBb;CACH;AAED,qBAAa,0BAA0B;aAenB,MAAM,EAAE,eAAe;IAEvC,OAAO,CAAC,MAAM;IAhBhB,QAAQ,EAAE,SAAS,CAAC;IACpB,OAAO,EAAE,aAAa,CAAC;IACvB,UAAU,EAAE,SAAS,CAAC;IACtB,KAAK,EAAE,SAAS,CAAC;IACjB,cAAc,EAAE,SAAS,CAAC;IAC1B,aAAa,EAAE,SAAS,CAAC;IACzB,iBAAiB,EAAE,SAAS,CAAC;IAC7B,iBAAiB,EAAE,SAAS,CAAC;IAC7B,UAAU,EAAE,SAAS,CAAC;IAEtB,OAAO,CAAC,aAAa,CAAQ;IAC7B,OAAO,CAAC,KAAK,CAAQ;gBAGH,MAAM,EAAE,eAAe,EACvC,IAAI,SAAe,EACX,MAAM,yCAAgD;IAoEhE,cAAc;IAOd,iBAAiB,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB;IAIzD,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM;IAO7D,OAAO,CAAC,QAAQ;CAuCjB"}

package/dest/metrics.js

@@ -1,84 +1,158 @@
 import { createLogger } from '@aztec/foundation/log';
 import { Attributes, Metrics, ValueType } from '@aztec/telemetry-client';
-import { formatEther } from 'viem';
-export class ProverNodeMetrics {
-    client;
+import { formatEther, formatUnits } from 'viem';
+export class ProverNodeJobMetrics {
+    meter;
+    tracer;
     logger;
     proverEpochExecutionDuration;
     provingJobDuration;
     provingJobBlocks;
     provingJobTransactions;
-    gasPrice;
-    txCount;
-    txDuration;
-    txGas;
-    txCalldataSize;
-    txCalldataGas;
-    txBlobDataGasUsed;
-    txBlobDataGasCost;
-    txTotalFee;
-    senderBalance;
-    constructor(client, name = 'ProverNode', logger = createLogger('prover-node:publisher:metrics')){
-        this.client = client;
+    constructor(meter, tracer, logger = createLogger('prover-node:publisher:metrics')){
+        this.meter = meter;
+        this.tracer = tracer;
         this.logger = logger;
-        const meter = client.getMeter(name);
-        this.proverEpochExecutionDuration = meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, {
+        this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, {
             description: 'Duration of execution of an epoch by the prover',
             unit: 'ms',
             valueType: ValueType.INT
         });
-        this.provingJobDuration = meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
+        this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
             description: 'Duration of proving job',
             unit: 's',
             valueType: ValueType.DOUBLE
         });
-        this.provingJobBlocks = meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
+        this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
             description: 'Number of blocks in a proven epoch',
             valueType: ValueType.INT
         });
-        this.provingJobTransactions = meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
+        this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
             description: 'Number of transactions in a proven epoch',
             valueType: ValueType.INT
         });
-        this.gasPrice = meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, {
+    }
+    recordProvingJob(executionTimeMs, totalTimeMs, numBlocks, numTxs) {
+        this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
+        this.provingJobDuration.record(totalTimeMs / 1000);
+        this.provingJobBlocks.record(Math.floor(numBlocks));
+        this.provingJobTransactions.record(Math.floor(numTxs));
+    }
+}
+export class ProverNodeRewardsMetrics {
+    meter;
+    coinbase;
+    rollup;
+    logger;
+    rewards;
+    accumulatedRewards;
+    prevEpoch;
+    proofSubmissionWindow;
+    constructor(meter, coinbase, rollup, logger = createLogger('prover-node:publisher:metrics')){
+        this.meter = meter;
+        this.coinbase = coinbase;
+        this.rollup = rollup;
+        this.logger = logger;
+        this.prevEpoch = -1n;
+        this.proofSubmissionWindow = 0n;
+        this.observe = async (observer)=>{
+            const slot = await this.rollup.getSlotNumber();
+            // look at the prev epoch so that we get an accurate value, after proof submission window has closed
+            if (slot > this.proofSubmissionWindow) {
+                const closedEpoch = await this.rollup.getEpochNumberForSlotNumber(slot - this.proofSubmissionWindow);
+                const rewards = await this.rollup.getSpecificProverRewardsForEpoch(closedEpoch, this.coinbase);
+                const fmt = parseFloat(formatUnits(rewards, 18));
+                observer.observe(this.rewards, fmt, {
+                    [Attributes.COINBASE]: this.coinbase.toString()
+                });
+                // only accumulate once per epoch
+                if (closedEpoch > this.prevEpoch) {
+                    this.prevEpoch = closedEpoch;
+                    this.accumulatedRewards.add(fmt, {
+                        [Attributes.COINBASE]: this.coinbase.toString()
+                    });
+                }
+            }
+        };
+        this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH, {
+            valueType: ValueType.DOUBLE,
+            description: 'The rewards earned'
+        });
+        this.accumulatedRewards = this.meter.createUpDownCounter(Metrics.PROVER_NODE_REWARDS_TOTAL, {
+            valueType: ValueType.DOUBLE,
+            description: 'The rewards earned (total)'
+        });
+    }
+    async start() {
+        this.proofSubmissionWindow = await this.rollup.getProofSubmissionWindow();
+        this.meter.addBatchObservableCallback(this.observe, [
+            this.rewards
+        ]);
+    }
+    stop() {
+        this.meter.removeBatchObservableCallback(this.observe, [
+            this.rewards
+        ]);
+    }
+    observe;
+}
+export class ProverNodePublisherMetrics {
+    client;
+    logger;
+    gasPrice;
+    txCount;
+    txDuration;
+    txGas;
+    txCalldataSize;
+    txCalldataGas;
+    txBlobDataGasUsed;
+    txBlobDataGasCost;
+    txTotalFee;
+    senderBalance;
+    meter;
+    constructor(client, name = 'ProverNode', logger = createLogger('prover-node:publisher:metrics')){
+        this.client = client;
+        this.logger = logger;
+        this.meter = client.getMeter(name);
+        this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, {
             description: 'The gas price used for transactions',
             unit: 'gwei',
             valueType: ValueType.DOUBLE
         });
-        this.txCount = meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, {
+        this.txCount = this.meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, {
             description: 'The number of transactions processed'
         });
-        this.txDuration = meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, {
+        this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, {
             description: 'The duration of transaction processing',
             unit: 'ms',
             valueType: ValueType.INT
         });
-        this.txGas = meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, {
+        this.txGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, {
             description: 'The gas consumed by transactions',
             unit: 'gas',
             valueType: ValueType.INT
         });
-        this.txCalldataSize = meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, {
+        this.txCalldataSize = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, {
             description: 'The size of the calldata in transactions',
             unit: 'By',
             valueType: ValueType.INT
         });
-        this.txCalldataGas = meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, {
+        this.txCalldataGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, {
             description: 'The gas consumed by the calldata in transactions',
             unit: 'gas',
             valueType: ValueType.INT
         });
-        this.txBlobDataGasUsed = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, {
+        this.txBlobDataGasUsed = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, {
             description: 'The amount of blob gas used in transactions',
             unit: 'gas',
             valueType: ValueType.INT
         });
-        this.txBlobDataGasCost = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, {
+        this.txBlobDataGasCost = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, {
             description: 'The gas cost of blobs in transactions',
             unit: 'gwei',
             valueType: ValueType.INT
         });
-        this.txTotalFee = meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE, {
+        this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE, {
             description: 'How much L1 tx costs',
             unit: 'gwei',
             valueType: ValueType.DOUBLE,
@@ -104,7 +178,7 @@ export class ProverNodeMetrics {
                 ]
             }
         });
-        this.senderBalance = meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, {
+        this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, {
             unit: 'eth',
             description: 'The balance of the sender address',
             valueType: ValueType.DOUBLE
@@ -119,12 +193,6 @@ export class ProverNodeMetrics {
     recordSubmitProof(durationMs, stats) {
         this.recordTx(durationMs, stats);
     }
-    recordProvingJob(executionTimeMs, totalTimeMs, numBlocks, numTxs) {
-        this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
-        this.provingJobDuration.record(totalTimeMs / 1000);
-        this.provingJobBlocks.record(Math.floor(numBlocks));
-        this.provingJobTransactions.record(Math.floor(numTxs));
-    }
     recordSenderBalance(wei, senderAddress) {
         const eth = parseFloat(formatEther(wei, 'wei'));
         this.senderBalance.record(eth, {

package/dest/monitors/epoch-monitor.d.ts

@@ -2,7 +2,7 @@ import type { L2BlockSource } from '@aztec/stdlib/block';
 import { type L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
 import { type TelemetryClient, type Traceable, type Tracer } from '@aztec/telemetry-client';
 export interface EpochMonitorHandler {
-    handleEpochReadyToProve(epochNumber: bigint): Promise<void>;
+    handleEpochReadyToProve(epochNumber: bigint): Promise<boolean>;
 }
 /**
  * Fires an event when a new epoch ready to prove is detected.

package/dest/monitors/epoch-monitor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"epoch-monitor.d.ts","sourceRoot":"","sources":["../../src/monitors/epoch-monitor.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,KAAK,iBAAiB,EAAkB,MAAM,6BAA6B,CAAC;AACrF,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,SAAS,EACd,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAEjC,MAAM,WAAW,mBAAmB;IAClC,uBAAuB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7D;AAED;;;;;;;;;;GAUG;AACH,qBAAa,YAAa,YAAW,SAAS;IAS1C,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,OAAO;IAVjB,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,GAAG,CAA6C;IACxD,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B,OAAO,CAAC,OAAO,CAAkC;IACjD,OAAO,CAAC,iBAAiB,CAAqB;gBAG3B,aAAa,EAAE,aAAa,EAC5B,WAAW,EAAE,IAAI,CAAC,iBAAiB,EAAE,eAAe,CAAC,EAC9D,OAAO,EAAE;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAAE,EAC9C,SAAS,GAAE,eAAsC;WAM/B,MAAM,CACxB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAAE,EACtC,SAAS,GAAE,eAAsC,GAChD,OAAO,CAAC,YAAY,CAAC;IAKjB,KAAK,CAAC,OAAO,EAAE,mBAAmB;IAMzC,0BAA0B;IACnB,UAAU,CAAC,OAAO,EAAE,mBAAmB;IAIjC,IAAI;IAMJ,IAAI;YAsBH,qBAAqB;CAYpC"}
+{"version":3,"file":"epoch-monitor.d.ts","sourceRoot":"","sources":["../../src/monitors/epoch-monitor.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,KAAK,iBAAiB,EAAkB,MAAM,6BAA6B,CAAC;AACrF,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,SAAS,EACd,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAEjC,MAAM,WAAW,mBAAmB;IAClC,uBAAuB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAChE;AAED;;;;;;;;;;GAUG;AACH,qBAAa,YAAa,YAAW,SAAS;IAS1C,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,OAAO;IAVjB,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,GAAG,CAA6C;IACxD,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B,OAAO,CAAC,OAAO,CAAkC;IACjD,OAAO,CAAC,iBAAiB,CAAqB;gBAG3B,aAAa,EAAE,aAAa,EAC5B,WAAW,EAAE,IAAI,CAAC,iBAAiB,EAAE,eAAe,CAAC,EAC9D,OAAO,EAAE;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAAE,EAC9C,SAAS,GAAE,eAAsC;WAM/B,MAAM,CACxB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAAE,EACtC,SAAS,GAAE,eAAsC,GAChD,OAAO,CAAC,YAAY,CAAC;IAKjB,KAAK,CAAC,OAAO,EAAE,mBAAmB;IAMzC,0BAA0B;IACnB,UAAU,CAAC,OAAO,EAAE,mBAAmB;IAIjC,IAAI;IAMJ,IAAI;YAwBH,qBAAqB;CAYpC"}

package/dest/monitors/epoch-monitor.js

@@ -53,6 +53,10 @@ import { getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
     }
     async work() {
         const { epochToProve, blockNumber, slotNumber } = await this.getEpochNumberToProve();
+        this.log.debug(`Epoch to prove: ${epochToProve}`, {
+            blockNumber,
+            slotNumber
+        });
         if (epochToProve === undefined) {
             this.log.trace(`Next block to prove ${blockNumber} not yet mined`, {
                 blockNumber
@@ -77,8 +81,9 @@ import { getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
             return;
         }
         this.log.debug(`Epoch ${epochToProve} is ready to be proven`);
-        await this.handler?.handleEpochReadyToProve(epochToProve);
-        this.latestEpochNumber = epochToProve;
+        if (await this.handler?.handleEpochReadyToProve(epochToProve)) {
+            this.latestEpochNumber = epochToProve;
+        }
     }
     async getEpochNumberToProve() {
         const lastBlockProven = await this.l2BlockSource.getProvenBlockNumber();

package/dest/prover-coordination/combined-prover-coordination.d.ts

@@ -0,0 +1,22 @@
+import type { P2P } from '@aztec/p2p';
+import type { P2PClient, ProverCoordination } from '@aztec/stdlib/interfaces/server';
+import { type Tx, TxHash } from '@aztec/stdlib/tx';
+export type CombinedCoordinationOptions = {
+    txGatheringBatchSize: number;
+    txGatheringMaxParallelRequestsPerNode: number;
+};
+export interface TxSource {
+    getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
+}
+export declare class CombinedProverCoordination implements ProverCoordination {
+    #private;
+    readonly p2p: P2P | undefined;
+    readonly aztecNodes: TxSource[];
+    private readonly options;
+    private readonly log;
+    constructor(p2p: P2P | undefined, aztecNodes: TxSource[], options?: CombinedCoordinationOptions, log?: import("@aztec/foundation/log").Logger);
+    getP2PClient(): P2PClient | undefined;
+    getTxsByHash(txHashes: TxHash[]): Promise<Tx[]>;
+    gatherTxs(txHashes: TxHash[]): Promise<void>;
+}
+//# sourceMappingURL=combined-prover-coordination.d.ts.map

package/dest/prover-coordination/combined-prover-coordination.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"combined-prover-coordination.d.ts","sourceRoot":"","sources":["../../src/prover-coordination/combined-prover-coordination.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,KAAK,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AACrF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAEnD,MAAM,MAAM,2BAA2B,GAAG;IAExC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,qCAAqC,EAAE,MAAM,CAAC;CAC/C,CAAC;AASF,MAAM,WAAW,QAAQ;IACvB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;CAC/D;AAsCD,qBAAa,0BAA2B,YAAW,kBAAkB;;aAEjD,GAAG,EAAE,GAAG,GAAG,SAAS;aACpB,UAAU,EAAE,QAAQ,EAAE;IACtC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAIxB,OAAO,CAAC,QAAQ,CAAC,GAAG;gBANJ,GAAG,EAAE,GAAG,GAAG,SAAS,EACpB,UAAU,EAAE,QAAQ,EAAE,EACrB,OAAO,GAAE,2BAGzB,EACgB,GAAG,yCAA2D;IAG1E,YAAY,IAAI,SAAS,GAAG,SAAS;IAI/B,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,EAAE,CAAC;IAY/C,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAyE1D"}

package/dest/prover-coordination/combined-prover-coordination.js

@@ -0,0 +1,136 @@
+import { asyncPool } from '@aztec/foundation/async-pool';
+import { createLogger } from '@aztec/foundation/log';
+import { TxHash } from '@aztec/stdlib/tx';
+// Wraps the p2p client into a coordination pool
+class P2PCoordinationPool {
+    p2p;
+    constructor(p2p){
+        this.p2p = p2p;
+    }
+    getTxsByHash(txHashes) {
+        return this.p2p.getTxsByHash(txHashes);
+    }
+    hasTxsInPool(txHashes) {
+        return this.p2p.hasTxsInPool(txHashes);
+    }
+    getTxsByHashFromPool(txHashes) {
+        return this.p2p.getTxsByHashFromPool(txHashes);
+    }
+    addTxs(txs) {
+        return this.p2p.addTxs(txs);
+    }
+}
+// Wraps an in memory tx pool into a coordination pool. Used for testing when no p2p/tx pool is available.
+class InMemoryCoordinationPool {
+    txs = new Map();
+    getTxsByHash(txHashes) {
+        return Promise.resolve(txHashes.map((hash)=>this.txs.get(hash.toString())));
+    }
+    hasTxsInPool(txHashes) {
+        return Promise.resolve(txHashes.map((hash)=>this.txs.has(hash.toString())));
+    }
+    getTxsByHashFromPool(txHashes) {
+        return this.getTxsByHash(txHashes);
+    }
+    async addTxs(txs) {
+        const hashes = await Promise.all(txs.map((tx)=>tx.getTxHash()));
+        txs.forEach((tx, index)=>this.txs.set(hashes[index].toString(), tx));
+    }
+}
+// Class to implement combined transaction retrieval from p2p and any available nodes
+export class CombinedProverCoordination {
+    p2p;
+    aztecNodes;
+    options;
+    log;
+    constructor(p2p, aztecNodes, options = {
+        txGatheringBatchSize: 10,
+        txGatheringMaxParallelRequestsPerNode: 10
+    }, log = createLogger('prover-node:combined-prover-coordination')){
+        this.p2p = p2p;
+        this.aztecNodes = aztecNodes;
+        this.options = options;
+        this.log = log;
+    }
+    getP2PClient() {
+        return this.p2p;
+    }
+    async getTxsByHash(txHashes) {
+        const pool = this.p2p ? new P2PCoordinationPool(this.p2p) : new InMemoryCoordinationPool();
+        await this.#gatherTxs(txHashes, pool);
+        const availability = await pool.hasTxsInPool(txHashes);
+        const notFound = txHashes.filter((_, index)=>!availability[index]);
+        if (notFound.length > 0) {
+            throw new Error(`Could not find txs: ${notFound.map((tx)=>tx.toString())}`);
+        }
+        const txs = await pool.getTxsByHashFromPool(txHashes);
+        return txs.filter((tx)=>tx !== undefined);
+    }
+    async gatherTxs(txHashes) {
+        const pool = this.p2p ? new P2PCoordinationPool(this.p2p) : new InMemoryCoordinationPool();
+        await this.#gatherTxs(txHashes, pool);
+    }
+    async #gatherTxs(txHashes, pool) {
+        const availability = await pool.hasTxsInPool(txHashes);
+        const notFound = txHashes.filter((_, index)=>!availability[index]);
+        const txsToFind = new Set(notFound.map((tx)=>tx.toString()));
+        if (txsToFind.size === 0) {
+            this.log.info(`Check for ${txHashes.length} txs found all in the pool`);
+            return;
+        }
+        this.log.info(`Check for ${txHashes.length} txs found ${txsToFind.size} missing. Will gather from nodes and p2p`);
+        const originalToFind = txsToFind.size;
+        await this.#gatherTxsFromAllNodes(txsToFind, pool);
+        if (txsToFind.size === 0) {
+            this.log.info(`Found all ${originalToFind} txs directly from nodes`);
+            return;
+        }
+        const toFindFromP2P = txsToFind.size;
+        this.log.verbose(`Gathering ${toFindFromP2P} txs from p2p network`);
+        const foundFromP2P = await pool.getTxsByHash([
+            ...txsToFind
+        ].map((tx)=>TxHash.fromString(tx)));
+        const numFoundFromNodes = originalToFind - toFindFromP2P;
+        const numNotFound = toFindFromP2P - foundFromP2P.length;
+        if (numNotFound === 0) {
+            this.log.info(`Found all ${originalToFind} txs. ${numFoundFromNodes} from nodes, ${foundFromP2P.length} from p2p`);
+            return;
+        }
+        this.log.warn(`Failed to find ${numNotFound} txs from any source. Found ${foundFromP2P.length} from p2p and ${numFoundFromNodes} from nodes`);
+    }
+    async #gatherTxsFromAllNodes(txsToFind, pool) {
+        if (txsToFind.size === 0 || this.aztecNodes.length === 0) {
+            return;
+        }
+        await Promise.all(this.aztecNodes.map((aztecNode)=>this.#gatherTxsFromNode(txsToFind, aztecNode, pool)));
+    }
+    async #gatherTxsFromNode(txsToFind, aztecNode, pool) {
+        const totalTxsRequired = txsToFind.size;
+        // It's possible that the set is empty as we already found the txs
+        if (totalTxsRequired === 0) {
+            return;
+        }
+        let totalTxsGathered = 0;
+        const batches = [];
+        const allTxs = [
+            ...txsToFind
+        ];
+        while(allTxs.length){
+            const batch = allTxs.splice(0, this.options.txGatheringBatchSize);
+            batches.push(batch);
+        }
+        await asyncPool(this.options.txGatheringMaxParallelRequestsPerNode, batches, async (batch)=>{
+            try {
+                const txs = (await aztecNode.getTxsByHash(batch.map((b)=>TxHash.fromString(b)))).filter((tx)=>!!tx);
+                const hashes = await Promise.all(txs.map((tx)=>tx.getTxHash()));
+                await pool.addTxs(txs);
+                hashes.forEach((hash)=>txsToFind.delete(hash.toString()));
+                totalTxsGathered += txs.length;
+            } catch (err) {
+                this.log.error(`Error gathering txs from aztec node: ${err}`);
+                return;
+            }
+        });
+        this.log.verbose(`Gathered ${totalTxsGathered} of ${totalTxsRequired} txs from a node`);
+    }
+}

package/dest/prover-coordination/config.d.ts

@@ -1,6 +1,6 @@
 import { type ConfigMappingsType } from '@aztec/foundation/config';
 export type ProverCoordinationConfig = {
-    proverCoordinationNodeUrl: string | undefined;
+    proverCoordinationNodeUrls: string[];
 };
 export declare const proverCoordinationConfigMappings: ConfigMappingsType<ProverCoordinationConfig>;
 export declare function getTxProviderConfigFromEnv(): ProverCoordinationConfig;

package/dest/prover-coordination/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/prover-coordination/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,kBAAkB,EAAyB,MAAM,0BAA0B,CAAC;AAE1F,MAAM,MAAM,wBAAwB,GAAG;IACrC,yBAAyB,EAAE,MAAM,GAAG,SAAS,CAAC;CAC/C,CAAC;AAEF,eAAO,MAAM,gCAAgC,EAAE,kBAAkB,CAAC,wBAAwB,CAMzF,CAAC;AAEF,wBAAgB,0BAA0B,IAAI,wBAAwB,CAErE"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/prover-coordination/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,kBAAkB,EAAyB,MAAM,0BAA0B,CAAC;AAE1F,MAAM,MAAM,wBAAwB,GAAG;IACrC,0BAA0B,EAAE,MAAM,EAAE,CAAC;CACtC,CAAC;AAEF,eAAO,MAAM,gCAAgC,EAAE,kBAAkB,CAAC,wBAAwB,CAOzF,CAAC;AAEF,wBAAgB,0BAA0B,IAAI,wBAAwB,CAErE"}

package/dest/prover-coordination/config.js

@@ -1,9 +1,10 @@
 import { getConfigFromMappings } from '@aztec/foundation/config';
 export const proverCoordinationConfigMappings = {
-    proverCoordinationNodeUrl: {
-        env: 'PROVER_COORDINATION_NODE_URL',
-        description: 'The URL of the tx provider node',
-        parseEnv: (val)=>val
+    proverCoordinationNodeUrls: {
+        env: 'PROVER_COORDINATION_NODE_URLS',
+        description: 'The URLs of the tx provider nodes',
+        parseEnv: (val)=>val.split(',').map((url)=>url.trim().replace(/\/$/, '')),
+        defaultValue: []
     }
 };
 export function getTxProviderConfigFromEnv() {