@aztec/prover-node 0.0.0-test.1 → 0.0.1-fake-ceab37513c

package/dest/job/epoch-proving-job.js

@@ -4,57 +4,55 @@ function _ts_decorate(decorators, target, key, desc) {
     else for(var i = decorators.length - 1; i >= 0; i--)if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
     return c > 3 && r && Object.defineProperty(target, key, r), r;
 }
+import { BatchedBlob, Blob } from '@aztec/blob-lib';
+import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
 import { asyncPool } from '@aztec/foundation/async-pool';
+import { padArrayEnd } from '@aztec/foundation/collection';
+import { Fr } from '@aztec/foundation/fields';
 import { createLogger } from '@aztec/foundation/log';
-import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 import { Timer } from '@aztec/foundation/timer';
 import { EpochProvingJobTerminalState } from '@aztec/stdlib/interfaces/server';
+import { MerkleTreeId } from '@aztec/stdlib/trees';
 import { Attributes, trackSpan } from '@aztec/telemetry-client';
 import * as crypto from 'node:crypto';
+import { validateEpochProvingJobData } from './epoch-proving-job-data.js';
 /**
- * Job that grabs a range of blocks from the unfinalised chain from L1, gets their txs given their hashes,
+ * Job that grabs a range of blocks from the unfinalized chain from L1, gets their txs given their hashes,
  * re-executes their public calls, generates a rollup proof, and submits it to L1. This job will update the
  * world state as part of public call execution via the public processor.
  */ export class EpochProvingJob {
+    data;
     dbProvider;
-    epochNumber;
-    blocks;
-    txs;
     prover;
     publicProcessorFactory;
     publisher;
     l2BlockSource;
-    l1ToL2MessageSource;
     metrics;
     deadline;
     config;
-    cleanUp;
     state;
     log;
     uuid;
     runPromise;
+    epochCheckPromise;
     deadlineTimeoutHandler;
     tracer;
-    constructor(dbProvider, epochNumber, blocks, txs, prover, publicProcessorFactory, publisher, l2BlockSource, l1ToL2MessageSource, metrics, deadline, config = {
-        parallelBlockLimit: 32
-    }, cleanUp = ()=>Promise.resolve()){
+    constructor(data, dbProvider, prover, publicProcessorFactory, publisher, l2BlockSource, metrics, deadline, config){
+        this.data = data;
         this.dbProvider = dbProvider;
-        this.epochNumber = epochNumber;
-        this.blocks = blocks;
-        this.txs = txs;
         this.prover = prover;
         this.publicProcessorFactory = publicProcessorFactory;
         this.publisher = publisher;
         this.l2BlockSource = l2BlockSource;
-        this.l1ToL2MessageSource = l1ToL2MessageSource;
         this.metrics = metrics;
         this.deadline = deadline;
         this.config = config;
-        this.cleanUp = cleanUp;
         this.state = 'initialized';
         this.log = createLogger('prover-node:epoch-proving-job');
+        validateEpochProvingJobData(data);
         this.uuid = crypto.randomUUID();
-        this.tracer = metrics.client.getTracer('EpochProvingJob');
+        this.tracer = metrics.tracer;
     }
     getId() {
         return this.uuid;
@@ -63,12 +61,34 @@ import * as crypto from 'node:crypto';
         return this.state;
     }
     getEpochNumber() {
-        return this.epochNumber;
+        return this.data.epochNumber;
+    }
+    getDeadline() {
+        return this.deadline;
+    }
+    getProvingData() {
+        return this.data;
+    }
+    get epochNumber() {
+        return this.data.epochNumber;
+    }
+    get blocks() {
+        return this.data.blocks;
+    }
+    get txs() {
+        return this.data.txs;
+    }
+    get attestations() {
+        return this.data.attestations;
     }
     /**
    * Proves the given epoch and submits the proof to L1.
    */ async run() {
         this.scheduleDeadlineStop();
+        if (!this.config.skipEpochCheck) {
+            await this.scheduleEpochCheck();
+        }
+        const attestations = this.attestations.map((attestation)=>attestation.toViem());
         const epochNumber = Number(this.epochNumber);
         const epochSizeBlocks = this.blocks.length;
         const epochSizeTxs = this.blocks.reduce((total, current)=>total + current.body.txEffects.length, 0);
@@ -88,14 +108,16 @@ import * as crypto from 'node:crypto';
         const { promise, resolve } = promiseWithResolvers();
         this.runPromise = promise;
         try {
-            this.prover.startNewEpoch(epochNumber, fromBlock, epochSizeBlocks);
-            await this.prover.startTubeCircuits(this.txs);
-            await asyncPool(this.config.parallelBlockLimit, this.blocks, async (block)=>{
+            const allBlobs = (await Promise.all(this.blocks.map(async (block)=>await Blob.getBlobsPerBlock(block.body.toBlobFields())))).flat();
+            const finalBlobBatchingChallenges = await BatchedBlob.precomputeBatchedBlobChallenges(allBlobs);
+            this.prover.startNewEpoch(epochNumber, fromBlock, epochSizeBlocks, finalBlobBatchingChallenges);
+            await this.prover.startTubeCircuits(Array.from(this.txs.values()));
+            await asyncPool(this.config.parallelBlockLimit ?? 32, this.blocks, async (block)=>{
                 this.checkState();
                 const globalVariables = block.header.globalVariables;
-                const txs = await this.getTxs(block);
-                const l1ToL2Messages = await this.getL1ToL2Messages(block);
-                const previousHeader = await this.getBlockHeader(block.number - 1);
+                const txs = this.getTxs(block);
+                const l1ToL2Messages = this.getL1ToL2Messages(block);
+                const previousHeader = this.getBlockHeader(block.number - 1);
                 this.log.verbose(`Starting processing block ${block.number}`, {
                     number: block.number,
                     blockHash: (await block.hash()).toString(),
@@ -110,7 +132,7 @@ import * as crypto from 'node:crypto';
                 // Start block proving
                 await this.prover.startNewBlock(globalVariables, l1ToL2Messages, previousHeader);
                 // Process public fns
-                const db = await this.dbProvider.fork(block.number - 1);
+                const db = await this.createFork(block.number - 1, l1ToL2Messages);
                 const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, true);
                 const processed = await this.processTxs(publicProcessor, txs);
                 await this.prover.addTxs(processed);
@@ -125,19 +147,27 @@ import * as crypto from 'node:crypto';
             });
             const executionTime = timer.ms();
             this.progressState('awaiting-prover');
-            const { publicInputs, proof } = await this.prover.finaliseEpoch();
-            this.log.info(`Finalised proof for epoch ${epochNumber}`, {
+            const { publicInputs, proof, batchedBlobInputs } = await this.prover.finalizeEpoch();
+            this.log.info(`Finalized proof for epoch ${epochNumber}`, {
                 epochNumber,
                 uuid: this.uuid,
                 duration: timer.ms()
             });
             this.progressState('publishing-proof');
+            if (this.config.skipSubmitProof) {
+                this.log.info(`Proof publishing is disabled. Dropping valid proof for epoch ${epochNumber} (blocks ${fromBlock} to ${toBlock})`);
+                this.state = 'completed';
+                this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeBlocks, epochSizeTxs);
+                return;
+            }
             const success = await this.publisher.submitEpochProof({
                 fromBlock,
                 toBlock,
                 epochNumber,
                 publicInputs,
-                proof
+                proof,
+                batchedBlobInputs,
+                attestations
             });
             if (!success) {
                 throw new Error('Failed to submit epoch proof to L1');
@@ -152,7 +182,8 @@ import * as crypto from 'node:crypto';
             if (err && err.name === 'HaltExecutionError') {
                 this.log.warn(`Halted execution of epoch ${epochNumber} prover job`, {
                     uuid: this.uuid,
-                    epochNumber
+                    epochNumber,
+                    details: err.message
                 });
                 return;
             }
@@ -160,20 +191,35 @@ import * as crypto from 'node:crypto';
                 uuid: this.uuid,
                 epochNumber
             });
-            this.state = 'failed';
+            if (this.state === 'processing' || this.state === 'awaiting-prover' || this.state === 'publishing-proof') {
+                this.state = 'failed';
+            }
         } finally{
             clearTimeout(this.deadlineTimeoutHandler);
-            await this.cleanUp(this);
+            await this.epochCheckPromise?.stop();
             await this.prover.stop();
             resolve();
         }
     }
+    /**
+   * Create a new db fork for tx processing, inserting all L1 to L2.
+   * REFACTOR: The prover already spawns a db fork of its own for each block, so we may be able to do away with just one fork.
+   */ async createFork(blockNumber, l1ToL2Messages) {
+        const db = await this.dbProvider.fork(blockNumber);
+        const l1ToL2MessagesPadded = padArrayEnd(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 'Too many L1 to L2 messages');
+        this.log.verbose(`Creating fork at ${blockNumber} with ${l1ToL2Messages.length} L1 to L2 messages`, {
+            blockNumber,
+            l1ToL2Messages: l1ToL2MessagesPadded.map((m)=>m.toString())
+        });
+        await db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded);
+        return db;
+    }
     progressState(state) {
         this.checkState();
         this.state = state;
     }
     checkState() {
-        if (this.state === 'timed-out' || this.state === 'stopped' || this.state === 'failed') {
+        if (this.state === 'timed-out' || this.state === 'stopped' || this.state === 'failed' || this.state === 'reorg') {
             throw new HaltExecutionError(this.state);
         }
     }
@@ -209,22 +255,47 @@ import * as crypto from 'node:crypto';
             }, timeout);
         }
     }
-    /* Returns the header for the given block number, or the genesis block for block zero. */ async getBlockHeader(blockNumber) {
-        if (blockNumber === 0) {
-            return (await this.dbProvider.fork()).getInitialHeader();
+    /**
+   * Kicks off a running promise that queries the archiver for the set of L2 blocks of the current epoch.
+   * If those change, stops the proving job with a `rerun` state, so the node re-enqueues it.
+   */ async scheduleEpochCheck() {
+        const l2BlockSource = this.l2BlockSource;
+        if (!l2BlockSource) {
+            this.log.warn(`No L2 block source available, skipping epoch check`);
+            return;
+        }
+        const intervalMs = Math.ceil((await l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
+        this.epochCheckPromise = new RunningPromise(async ()=>{
+            const blocks = await l2BlockSource.getBlockHeadersForEpoch(this.epochNumber);
+            const blockHashes = await Promise.all(blocks.map((block)=>block.hash()));
+            const thisBlockHashes = await Promise.all(this.blocks.map((block)=>block.hash()));
+            if (blocks.length !== this.blocks.length || !blockHashes.every((block, i)=>block.equals(thisBlockHashes[i]))) {
+                this.log.warn('Epoch blocks changed underfoot', {
+                    uuid: this.uuid,
+                    epochNumber: this.epochNumber,
+                    oldBlockHashes: thisBlockHashes,
+                    newBlockHashes: blockHashes
+                });
+                void this.stop('reorg');
+            }
+        }, this.log, intervalMs).start();
+        this.log.verbose(`Scheduled epoch check for epoch ${this.epochNumber} every ${intervalMs}ms`);
+    }
+    /* Returns the header for the given block number based on the epoch proving job data. */ getBlockHeader(blockNumber) {
+        const block = this.blocks.find((b)=>b.number === blockNumber);
+        if (block) {
+            return block.header;
         }
-        return this.l2BlockSource.getBlockHeader(blockNumber);
+        if (blockNumber === Number(this.data.previousBlockHeader.getBlockNumber())) {
+            return this.data.previousBlockHeader;
+        }
+        throw new Error(`Block header not found for block number ${blockNumber} (got ${this.blocks.map((b)=>b.number).join(', ')} and previous header ${this.data.previousBlockHeader.getBlockNumber()})`);
     }
-    async getTxs(block) {
-        const txHashes = block.body.txEffects.map((tx)=>tx.txHash.toBigInt());
-        const txsAndHashes = await Promise.all(this.txs.map(async (tx)=>({
-                tx,
-                hash: await tx.getTxHash()
-            })));
-        return txsAndHashes.filter((txAndHash)=>txHashes.includes(txAndHash.hash.toBigInt())).map((txAndHash)=>txAndHash.tx);
+    getTxs(block) {
+        return block.body.txEffects.map((txEffect)=>this.txs.get(txEffect.txHash.toString()));
     }
     getL1ToL2Messages(block) {
-        return this.l1ToL2MessageSource.getL1ToL2Messages(BigInt(block.number));
+        return this.data.l1ToL2Messages[block.number];
     }
     async processTxs(publicProcessor, txs) {
         const { deadline } = this;
@@ -232,7 +303,8 @@ import * as crypto from 'node:crypto';
             deadline
         });
         if (failedTxs.length) {
-            throw new Error(`Txs failed processing: ${failedTxs.map(({ tx, error })=>`${tx.getTxHash()} (${error})`).join(', ')}`);
+            const failedTxHashes = await Promise.all(failedTxs.map(({ tx })=>tx.getTxHash()));
+            throw new Error(`Txs failed processing: ${failedTxs.map(({ error }, index)=>`${failedTxHashes[index]} (${error})`).join(', ')}`);
         }
         if (processedTxs.length !== txs.length) {
             throw new Error(`Failed to process all txs: processed ${processedTxs.length} out of ${txs.length}`);
@@ -243,13 +315,14 @@ import * as crypto from 'node:crypto';
 _ts_decorate([
     trackSpan('EpochProvingJob.run', function() {
         return {
-            [Attributes.EPOCH_NUMBER]: Number(this.epochNumber)
+            [Attributes.EPOCH_NUMBER]: Number(this.data.epochNumber)
         };
     })
 ], EpochProvingJob.prototype, "run", null);
 class HaltExecutionError extends Error {
+    state;
     constructor(state){
-        super(`Halted execution due to state ${state}`);
+        super(`Halted execution due to state ${state}`), this.state = state;
         this.name = 'HaltExecutionError';
     }
 }

package/dest/metrics.d.ts

@@ -1,12 +1,35 @@
+import type { RollupContract } from '@aztec/ethereum';
+import type { EthAddress } from '@aztec/foundation/eth-address';
 import type { L1PublishProofStats } from '@aztec/stdlib/stats';
-import { type Gauge, type Histogram, type TelemetryClient, type UpDownCounter } from '@aztec/telemetry-client';
-export declare class ProverNodeMetrics {
-    readonly client: TelemetryClient;
+import { type Gauge, type Histogram, type Meter, type TelemetryClient, type Tracer, type UpDownCounter } from '@aztec/telemetry-client';
+export declare class ProverNodeJobMetrics {
+    private meter;
+    readonly tracer: Tracer;
     private logger;
     proverEpochExecutionDuration: Histogram;
     provingJobDuration: Histogram;
     provingJobBlocks: Gauge;
     provingJobTransactions: Gauge;
+    constructor(meter: Meter, tracer: Tracer, logger?: import("@aztec/foundation/log").Logger);
+    recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number): void;
+}
+export declare class ProverNodeRewardsMetrics {
+    private meter;
+    private coinbase;
+    private rollup;
+    private logger;
+    private rewards;
+    private accumulatedRewards;
+    private prevEpoch;
+    private proofSubmissionEpochs;
+    constructor(meter: Meter, coinbase: EthAddress, rollup: RollupContract, logger?: import("@aztec/foundation/log").Logger);
+    start(): Promise<void>;
+    stop(): void;
+    private observe;
+}
+export declare class ProverNodePublisherMetrics {
+    readonly client: TelemetryClient;
+    private logger;
     gasPrice: Histogram;
     txCount: UpDownCounter;
     txDuration: Histogram;
@@ -15,11 +38,12 @@ export declare class ProverNodeMetrics {
     txCalldataGas: Histogram;
     txBlobDataGasUsed: Histogram;
     txBlobDataGasCost: Histogram;
+    txTotalFee: Histogram;
     private senderBalance;
+    private meter;
     constructor(client: TelemetryClient, name?: string, logger?: import("@aztec/foundation/log").Logger);
     recordFailedTx(): void;
     recordSubmitProof(durationMs: number, stats: L1PublishProofStats): void;
-    recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number): void;
     recordSenderBalance(wei: bigint, senderAddress: string): void;
     private recordTx;
 }

package/dest/metrics.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,mBAAmB,EAAkB,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAEL,KAAK,KAAK,EACV,KAAK,SAAS,EAEd,KAAK,eAAe,EACpB,KAAK,aAAa,EAEnB,MAAM,yBAAyB,CAAC;AAIjC,qBAAa,iBAAiB;aAkBV,MAAM,EAAE,eAAe;IAEvC,OAAO,CAAC,MAAM;IAnBhB,4BAA4B,EAAE,SAAS,CAAC;IACxC,kBAAkB,EAAE,SAAS,CAAC;IAC9B,gBAAgB,EAAE,KAAK,CAAC;IACxB,sBAAsB,EAAE,KAAK,CAAC;IAE9B,QAAQ,EAAE,SAAS,CAAC;IACpB,OAAO,EAAE,aAAa,CAAC;IACvB,UAAU,EAAE,SAAS,CAAC;IACtB,KAAK,EAAE,SAAS,CAAC;IACjB,cAAc,EAAE,SAAS,CAAC;IAC1B,aAAa,EAAE,SAAS,CAAC;IACzB,iBAAiB,EAAE,SAAS,CAAC;IAC7B,iBAAiB,EAAE,SAAS,CAAC;IAE7B,OAAO,CAAC,aAAa,CAAQ;gBAGX,MAAM,EAAE,eAAe,EACvC,IAAI,SAAe,EACX,MAAM,yCAAgD;IA2EhE,cAAc;IAOd,iBAAiB,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB;IAIzD,gBAAgB,CAAC,eAAe,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAOhG,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM;IAO7D,OAAO,CAAC,QAAQ;CA6BjB"}
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAEhE,OAAO,KAAK,EAAE,mBAAmB,EAAkB,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAGL,KAAK,KAAK,EACV,KAAK,SAAS,EACd,KAAK,KAAK,EAGV,KAAK,eAAe,EACpB,KAAK,MAAM,EACX,KAAK,aAAa,EAEnB,MAAM,yBAAyB,CAAC;AAIjC,qBAAa,oBAAoB;IAO7B,OAAO,CAAC,KAAK;aACG,MAAM,EAAE,MAAM;IAC9B,OAAO,CAAC,MAAM;IARhB,4BAA4B,EAAE,SAAS,CAAC;IACxC,kBAAkB,EAAE,SAAS,CAAC;IAC9B,gBAAgB,EAAE,KAAK,CAAC;IACxB,sBAAsB,EAAE,KAAK,CAAC;gBAGpB,KAAK,EAAE,KAAK,EACJ,MAAM,EAAE,MAAM,EACtB,MAAM,yCAAgD;IAsBzD,gBAAgB,CAAC,eAAe,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;CAMxG;AAED,qBAAa,wBAAwB;IAOjC,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IAThB,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,kBAAkB,CAAgB;IAC1C,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,qBAAqB,CAAK;gBAGxB,KAAK,EAAE,KAAK,EACZ,QAAQ,EAAE,UAAU,EACpB,MAAM,EAAE,cAAc,EACtB,MAAM,yCAAgD;IAanD,KAAK;IAMX,IAAI;IAIX,OAAO,CAAC,OAAO,CAwBb;CACH;AAED,qBAAa,0BAA0B;aAenB,MAAM,EAAE,eAAe;IAEvC,OAAO,CAAC,MAAM;IAhBhB,QAAQ,EAAE,SAAS,CAAC;IACpB,OAAO,EAAE,aAAa,CAAC;IACvB,UAAU,EAAE,SAAS,CAAC;IACtB,KAAK,EAAE,SAAS,CAAC;IACjB,cAAc,EAAE,SAAS,CAAC;IAC1B,aAAa,EAAE,SAAS,CAAC;IACzB,iBAAiB,EAAE,SAAS,CAAC;IAC7B,iBAAiB,EAAE,SAAS,CAAC;IAC7B,UAAU,EAAE,SAAS,CAAC;IAEtB,OAAO,CAAC,aAAa,CAAQ;IAC7B,OAAO,CAAC,KAAK,CAAQ;gBAGH,MAAM,EAAE,eAAe,EACvC,IAAI,SAAe,EACX,MAAM,yCAAgD;IAoEhE,cAAc;IAOd,iBAAiB,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB;IAIzD,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM;IAO7D,OAAO,CAAC,QAAQ;CAuCjB"}

package/dest/metrics.js

@@ -1,83 +1,187 @@
 import { createLogger } from '@aztec/foundation/log';
 import { Attributes, Metrics, ValueType } from '@aztec/telemetry-client';
-import { formatEther } from 'viem';
-export class ProverNodeMetrics {
-    client;
+import { formatEther, formatUnits } from 'viem';
+export class ProverNodeJobMetrics {
+    meter;
+    tracer;
     logger;
     proverEpochExecutionDuration;
     provingJobDuration;
     provingJobBlocks;
     provingJobTransactions;
-    gasPrice;
-    txCount;
-    txDuration;
-    txGas;
-    txCalldataSize;
-    txCalldataGas;
-    txBlobDataGasUsed;
-    txBlobDataGasCost;
-    senderBalance;
-    constructor(client, name = 'ProverNode', logger = createLogger('prover-node:publisher:metrics')){
-        this.client = client;
+    constructor(meter, tracer, logger = createLogger('prover-node:publisher:metrics')){
+        this.meter = meter;
+        this.tracer = tracer;
         this.logger = logger;
-        const meter = client.getMeter(name);
-        this.proverEpochExecutionDuration = meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, {
+        this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, {
             description: 'Duration of execution of an epoch by the prover',
             unit: 'ms',
             valueType: ValueType.INT
         });
-        this.provingJobDuration = meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
+        this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
             description: 'Duration of proving job',
             unit: 's',
             valueType: ValueType.DOUBLE
         });
-        this.provingJobBlocks = meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
+        this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
             description: 'Number of blocks in a proven epoch',
             valueType: ValueType.INT
         });
-        this.provingJobTransactions = meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
+        this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
             description: 'Number of transactions in a proven epoch',
             valueType: ValueType.INT
         });
-        this.gasPrice = meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, {
+    }
+    recordProvingJob(executionTimeMs, totalTimeMs, numBlocks, numTxs) {
+        this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
+        this.provingJobDuration.record(totalTimeMs / 1000);
+        this.provingJobBlocks.record(Math.floor(numBlocks));
+        this.provingJobTransactions.record(Math.floor(numTxs));
+    }
+}
+export class ProverNodeRewardsMetrics {
+    meter;
+    coinbase;
+    rollup;
+    logger;
+    rewards;
+    accumulatedRewards;
+    prevEpoch;
+    proofSubmissionEpochs;
+    constructor(meter, coinbase, rollup, logger = createLogger('prover-node:publisher:metrics')){
+        this.meter = meter;
+        this.coinbase = coinbase;
+        this.rollup = rollup;
+        this.logger = logger;
+        this.prevEpoch = -1n;
+        this.proofSubmissionEpochs = 0;
+        this.observe = async (observer)=>{
+            const epoch = await this.rollup.getCurrentEpochNumber();
+            if (epoch > this.proofSubmissionEpochs) {
+                // look at the prev epoch so that we get an accurate value, after proof submission window has closed
+                // For example, if proof submission window is 1 epoch, and we are in epoch 2, we should be looking at epoch 0.
+                // Similarly, if the proof submission window is 0, and we are in epoch 1, we should be looking at epoch 0.
+                const closedEpoch = epoch - BigInt(this.proofSubmissionEpochs) - 1n;
+                const rewards = await this.rollup.getSpecificProverRewardsForEpoch(closedEpoch, this.coinbase);
+                const fmt = parseFloat(formatUnits(rewards, 18));
+                observer.observe(this.rewards, fmt, {
+                    [Attributes.COINBASE]: this.coinbase.toString()
+                });
+                // only accumulate once per epoch
+                if (closedEpoch > this.prevEpoch) {
+                    this.prevEpoch = closedEpoch;
+                    this.accumulatedRewards.add(fmt, {
+                        [Attributes.COINBASE]: this.coinbase.toString()
+                    });
+                }
+            }
+        };
+        this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH, {
+            valueType: ValueType.DOUBLE,
+            description: 'The rewards earned'
+        });
+        this.accumulatedRewards = this.meter.createUpDownCounter(Metrics.PROVER_NODE_REWARDS_TOTAL, {
+            valueType: ValueType.DOUBLE,
+            description: 'The rewards earned (total)'
+        });
+    }
+    async start() {
+        const proofSubmissionEpochs = await this.rollup.getProofSubmissionEpochs();
+        this.proofSubmissionEpochs = Number(proofSubmissionEpochs);
+        this.meter.addBatchObservableCallback(this.observe, [
+            this.rewards
+        ]);
+    }
+    stop() {
+        this.meter.removeBatchObservableCallback(this.observe, [
+            this.rewards
+        ]);
+    }
+    observe;
+}
+export class ProverNodePublisherMetrics {
+    client;
+    logger;
+    gasPrice;
+    txCount;
+    txDuration;
+    txGas;
+    txCalldataSize;
+    txCalldataGas;
+    txBlobDataGasUsed;
+    txBlobDataGasCost;
+    txTotalFee;
+    senderBalance;
+    meter;
+    constructor(client, name = 'ProverNode', logger = createLogger('prover-node:publisher:metrics')){
+        this.client = client;
+        this.logger = logger;
+        this.meter = client.getMeter(name);
+        this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, {
             description: 'The gas price used for transactions',
             unit: 'gwei',
             valueType: ValueType.DOUBLE
         });
-        this.txCount = meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, {
+        this.txCount = this.meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, {
             description: 'The number of transactions processed'
         });
-        this.txDuration = meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, {
+        this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, {
             description: 'The duration of transaction processing',
             unit: 'ms',
             valueType: ValueType.INT
         });
-        this.txGas = meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, {
+        this.txGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, {
             description: 'The gas consumed by transactions',
             unit: 'gas',
             valueType: ValueType.INT
         });
-        this.txCalldataSize = meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, {
+        this.txCalldataSize = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, {
             description: 'The size of the calldata in transactions',
             unit: 'By',
             valueType: ValueType.INT
         });
-        this.txCalldataGas = meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, {
+        this.txCalldataGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, {
             description: 'The gas consumed by the calldata in transactions',
             unit: 'gas',
             valueType: ValueType.INT
         });
-        this.txBlobDataGasUsed = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, {
+        this.txBlobDataGasUsed = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, {
             description: 'The amount of blob gas used in transactions',
             unit: 'gas',
             valueType: ValueType.INT
         });
-        this.txBlobDataGasCost = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, {
+        this.txBlobDataGasCost = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, {
             description: 'The gas cost of blobs in transactions',
             unit: 'gwei',
             valueType: ValueType.INT
         });
-        this.senderBalance = meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, {
+        this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE, {
+            description: 'How much L1 tx costs',
+            unit: 'gwei',
+            valueType: ValueType.DOUBLE,
+            advice: {
+                explicitBucketBoundaries: [
+                    0.001,
+                    0.002,
+                    0.004,
+                    0.008,
+                    0.01,
+                    0.02,
+                    0.04,
+                    0.08,
+                    0.1,
+                    0.2,
+                    0.4,
+                    0.8,
+                    1,
+                    1.2,
+                    1.4,
+                    1.8,
+                    2
+                ]
+            }
+        });
+        this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, {
             unit: 'eth',
             description: 'The balance of the sender address',
             valueType: ValueType.DOUBLE
@@ -92,12 +196,6 @@ export class ProverNodeMetrics {
     recordSubmitProof(durationMs, stats) {
         this.recordTx(durationMs, stats);
     }
-    recordProvingJob(executionTimeMs, totalTimeMs, numBlocks, numTxs) {
-        this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
-        this.provingJobDuration.record(totalTimeMs / 1000);
-        this.provingJobBlocks.record(Math.floor(numBlocks));
-        this.provingJobTransactions.record(Math.floor(numTxs));
-    }
     recordSenderBalance(wei, senderAddress) {
         const eth = parseFloat(formatEther(wei, 'wei'));
         this.senderBalance.record(eth, {
@@ -122,7 +220,15 @@ export class ProverNodeMetrics {
         this.txBlobDataGasUsed.record(Number(stats.blobGasUsed), attributes);
         try {
             this.gasPrice.record(parseInt(formatEther(stats.gasPrice, 'gwei'), 10));
-        } catch (e) {
+        } catch  {
+        // ignore
+        }
+        const executionFee = stats.gasUsed * stats.gasPrice;
+        const blobFee = stats.blobGasUsed * stats.blobDataGas;
+        const totalFee = executionFee + blobFee;
+        try {
+            this.txTotalFee.record(parseFloat(formatEther(totalFee)));
+        } catch  {
         // ignore
         }
     }

package/dest/monitors/epoch-monitor.d.ts

@@ -2,7 +2,7 @@ import type { L2BlockSource } from '@aztec/stdlib/block';
 import { type L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
 import { type TelemetryClient, type Traceable, type Tracer } from '@aztec/telemetry-client';
 export interface EpochMonitorHandler {
-    handleEpochReadyToProve(epochNumber: bigint): Promise<void>;
+    handleEpochReadyToProve(epochNumber: bigint): Promise<boolean>;
 }
 /**
  * Fires an event when a new epoch ready to prove is detected.
@@ -26,9 +26,11 @@ export declare class EpochMonitor implements Traceable {
     private latestEpochNumber;
     constructor(l2BlockSource: L2BlockSource, l1Constants: Pick<L1RollupConstants, 'epochDuration'>, options: {
         pollingIntervalMs: number;
+        provingDelayMs?: number;
     }, telemetry?: TelemetryClient);
     static create(l2BlockSource: L2BlockSource, options: {
         pollingIntervalMs: number;
+        provingDelayMs?: number;
     }, telemetry?: TelemetryClient): Promise<EpochMonitor>;
     start(handler: EpochMonitorHandler): void;
     /** Exposed for testing */

package/dest/monitors/epoch-monitor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"epoch-monitor.d.ts","sourceRoot":"","sources":["../../src/monitors/epoch-monitor.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,KAAK,iBAAiB,EAAkB,MAAM,6BAA6B,CAAC;AACrF,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,SAAS,EACd,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAEjC,MAAM,WAAW,mBAAmB;IAClC,uBAAuB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7D;AAED;;;;;;;;;;GAUG;AACH,qBAAa,YAAa,YAAW,SAAS;IAS1C,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,OAAO;IAVjB,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,GAAG,CAA6C;IACxD,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B,OAAO,CAAC,OAAO,CAAkC;IACjD,OAAO,CAAC,iBAAiB,CAAqB;gBAG3B,aAAa,EAAE,aAAa,EAC5B,WAAW,EAAE,IAAI,CAAC,iBAAiB,EAAE,eAAe,CAAC,EAC9D,OAAO,EAAE;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAAE,EAC9C,SAAS,GAAE,eAAsC;WAM/B,MAAM,CACxB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAAE,EACtC,SAAS,GAAE,eAAsC,GAChD,OAAO,CAAC,YAAY,CAAC;IAKjB,KAAK,CAAC,OAAO,EAAE,mBAAmB;IAMzC,0BAA0B;IACnB,UAAU,CAAC,OAAO,EAAE,mBAAmB;IAIjC,IAAI;IAMJ,IAAI;YAsBH,qBAAqB;CAYpC"}
+{"version":3,"file":"epoch-monitor.d.ts","sourceRoot":"","sources":["../../src/monitors/epoch-monitor.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,KAAK,iBAAiB,EAAkB,MAAM,6BAA6B,CAAC;AACrF,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,SAAS,EACd,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAEjC,MAAM,WAAW,mBAAmB;IAClC,uBAAuB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAChE;AAED;;;;;;;;;;GAUG;AACH,qBAAa,YAAa,YAAW,SAAS;IAS1C,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,OAAO;IAVjB,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,GAAG,CAA6C;IACxD,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B,OAAO,CAAC,OAAO,CAAkC;IACjD,OAAO,CAAC,iBAAiB,CAAqB;gBAG3B,aAAa,EAAE,aAAa,EAC5B,WAAW,EAAE,IAAI,CAAC,iBAAiB,EAAE,eAAe,CAAC,EAC9D,OAAO,EAAE;QAAE,iBAAiB,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAA;KAAE,EACvE,SAAS,GAAE,eAAsC;WAS/B,MAAM,CACxB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE;QAAE,iBAAiB,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAA;KAAE,EAC/D,SAAS,GAAE,eAAsC,GAChD,OAAO,CAAC,YAAY,CAAC;IAKjB,KAAK,CAAC,OAAO,EAAE,mBAAmB;IAMzC,0BAA0B;IACnB,UAAU,CAAC,OAAO,EAAE,mBAAmB;IAIjC,IAAI;IAMJ,IAAI;YA6BH,qBAAqB;CAYpC"}