@aztec/prover-node 0.80.0 → 0.82.0

package/dest/job/epoch-proving-job.d.ts

@@ -1,6 +1,6 @@
 import type { PublicProcessorFactory } from '@aztec/simulator/server';
 import type { L2Block, L2BlockSource } from '@aztec/stdlib/block';
-import { type EpochProver, type EpochProvingJobState, type ForkMerkleTreeOperations } from '@aztec/stdlib/interfaces/server';
+import { type EpochProver, type EpochProvingJobState, EpochProvingJobTerminalState, type ForkMerkleTreeOperations } from '@aztec/stdlib/interfaces/server';
 import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
 import type { Tx } from '@aztec/stdlib/tx';
 import { type Traceable, type Tracer } from '@aztec/telemetry-client';
@@ -24,16 +24,16 @@ export declare class EpochProvingJob implements Traceable {
     private metrics;
     private deadline;
     private config;
-    private cleanUp;
     private state;
     private log;
     private uuid;
     private runPromise;
+    private epochCheckPromise;
     private deadlineTimeoutHandler;
     readonly tracer: Tracer;
     constructor(dbProvider: ForkMerkleTreeOperations, epochNumber: bigint, blocks: L2Block[], txs: Tx[], prover: EpochProver, publicProcessorFactory: PublicProcessorFactory, publisher: ProverNodePublisher, l2BlockSource: L2BlockSource, l1ToL2MessageSource: L1ToL2MessageSource, metrics: ProverNodeMetrics, deadline: Date | undefined, config?: {
         parallelBlockLimit: number;
-    }, cleanUp?: (job: EpochProvingJob) => Promise<void>);
+    });
     getId(): string;
     getState(): EpochProvingJobState;
     getEpochNumber(): bigint;
@@ -43,8 +43,13 @@ export declare class EpochProvingJob implements Traceable {
     run(): Promise<void>;
     private progressState;
     private checkState;
-    stop(state?: EpochProvingJobState): Promise<void>;
+    stop(state?: EpochProvingJobTerminalState): Promise<void>;
     private scheduleDeadlineStop;
+    /**
+     * Kicks off a running promise that queries the archiver for the set of L2 blocks of the current epoch.
+     * If those changed, stops the proving job with a `rerun` state, so the node re-enqueues it.
+     */
+    private scheduleEpochCheck;
     private getBlockHeader;
     private getTxs;
     private getL1ToL2Messages;

package/dest/job/epoch-proving-job.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"epoch-proving-job.d.ts","sourceRoot":"","sources":["../../src/job/epoch-proving-job.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAmB,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACvF,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,oBAAoB,EAEzB,KAAK,wBAAwB,EAC9B,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,KAAK,EAAe,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAc,KAAK,SAAS,EAAE,KAAK,MAAM,EAAa,MAAM,yBAAyB,CAAC;AAI7F,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAEvE;;;;GAIG;AACH,qBAAa,eAAgB,YAAW,SAAS;IAW7C,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,GAAG;IACX,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,sBAAsB;IAC9B,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,aAAa;IACrB,OAAO,CAAC,mBAAmB;IAC3B,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,OAAO;IAtBjB,OAAO,CAAC,KAAK,CAAuC;IACpD,OAAO,CAAC,GAAG,CAAiD;IAC5D,OAAO,CAAC,IAAI,CAAS;IAErB,OAAO,CAAC,UAAU,CAA4B;IAC9C,OAAO,CAAC,sBAAsB,CAA6B;IAE3D,SAAgB,MAAM,EAAE,MAAM,CAAC;gBAGrB,UAAU,EAAE,wBAAwB,EACpC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,OAAO,EAAE,EACjB,GAAG,EAAE,EAAE,EAAE,EACT,MAAM,EAAE,WAAW,EACnB,sBAAsB,EAAE,sBAAsB,EAC9C,SAAS,EAAE,mBAAmB,EAC9B,aAAa,EAAE,aAAa,EAC5B,mBAAmB,EAAE,mBAAmB,EACxC,OAAO,EAAE,iBAAiB,EAC1B,QAAQ,EAAE,IAAI,GAAG,SAAS,EAC1B,MAAM,GAAE;QAAE,kBAAkB,EAAE,MAAM,CAAA;KAA+B,EACnE,OAAO,GAAE,CAAC,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAA2B;IAM7E,KAAK,IAAI,MAAM;IAIf,QAAQ,IAAI,oBAAoB;IAIhC,cAAc,IAAI,MAAM;IAI/B;;OAEG;IAIU,GAAG;IAgGhB,OAAO,CAAC,aAAa;IAKrB,OAAO,CAAC,UAAU;IAML,IAAI,CAAC,KAAK,GAAE,oBAAgC;IASzD,OAAO,CAAC,oBAAoB;YAqBd,cAAc;YAOd,MAAM;IAQpB,OAAO,CAAC,iBAAiB;YAIX,UAAU;CAgBzB;AASD,OAAO,EAAE,KAAK,oBAAoB,EAAE,CAAC"}
+{"version":3,"file":"epoch-proving-job.d.ts","sourceRoot":"","sources":["../../src/job/epoch-proving-job.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAmB,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACvF,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,oBAAoB,EACzB,4BAA4B,EAC5B,KAAK,wBAAwB,EAC9B,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,KAAK,EAAe,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAc,KAAK,SAAS,EAAE,KAAK,MAAM,EAAa,MAAM,yBAAyB,CAAC;AAI7F,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAEvE;;;;GAIG;AACH,qBAAa,eAAgB,YAAW,SAAS;IAY7C,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,GAAG;IACX,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,sBAAsB;IAC9B,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,aAAa;IACrB,OAAO,CAAC,mBAAmB;IAC3B,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,MAAM;IAtBhB,OAAO,CAAC,KAAK,CAAuC;IACpD,OAAO,CAAC,GAAG,CAAiD;IAC5D,OAAO,CAAC,IAAI,CAAS;IAErB,OAAO,CAAC,UAAU,CAA4B;IAC9C,OAAO,CAAC,iBAAiB,CAA6B;IACtD,OAAO,CAAC,sBAAsB,CAA6B;IAE3D,SAAgB,MAAM,EAAE,MAAM,CAAC;gBAGrB,UAAU,EAAE,wBAAwB,EACpC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,OAAO,EAAE,EACjB,GAAG,EAAE,EAAE,EAAE,EACT,MAAM,EAAE,WAAW,EACnB,sBAAsB,EAAE,sBAAsB,EAC9C,SAAS,EAAE,mBAAmB,EAC9B,aAAa,EAAE,aAAa,EAC5B,mBAAmB,EAAE,mBAAmB,EACxC,OAAO,EAAE,iBAAiB,EAC1B,QAAQ,EAAE,IAAI,GAAG,SAAS,EAC1B,MAAM,GAAE;QAAE,kBAAkB,EAAE,MAAM,CAAA;KAA+B;IAMtE,KAAK,IAAI,MAAM;IAIf,QAAQ,IAAI,oBAAoB;IAIhC,cAAc,IAAI,MAAM;IAI/B;;OAEG;IAIU,GAAG;IAqGhB,OAAO,CAAC,aAAa;IAKrB,OAAO,CAAC,UAAU;IAML,IAAI,CAAC,KAAK,GAAE,4BAAwC;IASjE,OAAO,CAAC,oBAAoB;IAoB5B;;;OAGG;YACW,kBAAkB;YA2BlB,cAAc;YAOd,MAAM;IAQpB,OAAO,CAAC,iBAAiB;YAIX,UAAU;CAmBzB;AASD,OAAO,EAAE,KAAK,oBAAoB,EAAE,CAAC"}

package/dest/job/epoch-proving-job.js

@@ -6,7 +6,7 @@ function _ts_decorate(decorators, target, key, desc) {
 }
 import { asyncPool } from '@aztec/foundation/async-pool';
 import { createLogger } from '@aztec/foundation/log';
-import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 import { Timer } from '@aztec/foundation/timer';
 import { EpochProvingJobTerminalState } from '@aztec/stdlib/interfaces/server';
 import { Attributes, trackSpan } from '@aztec/telemetry-client';
@@ -28,16 +28,16 @@ import * as crypto from 'node:crypto';
     metrics;
     deadline;
     config;
-    cleanUp;
     state;
     log;
     uuid;
     runPromise;
+    epochCheckPromise;
     deadlineTimeoutHandler;
     tracer;
     constructor(dbProvider, epochNumber, blocks, txs, prover, publicProcessorFactory, publisher, l2BlockSource, l1ToL2MessageSource, metrics, deadline, config = {
         parallelBlockLimit: 32
-    }, cleanUp = ()=>Promise.resolve()){
+    }){
         this.dbProvider = dbProvider;
         this.epochNumber = epochNumber;
         this.blocks = blocks;
@@ -50,7 +50,6 @@ import * as crypto from 'node:crypto';
         this.metrics = metrics;
         this.deadline = deadline;
         this.config = config;
-        this.cleanUp = cleanUp;
         this.state = 'initialized';
         this.log = createLogger('prover-node:epoch-proving-job');
         this.uuid = crypto.randomUUID();
@@ -69,6 +68,7 @@ import * as crypto from 'node:crypto';
    * Proves the given epoch and submits the proof to L1.
    */ async run() {
         this.scheduleDeadlineStop();
+        await this.scheduleEpochCheck();
         const epochNumber = Number(this.epochNumber);
         const epochSizeBlocks = this.blocks.length;
         const epochSizeTxs = this.blocks.reduce((total, current)=>total + current.body.txEffects.length, 0);
@@ -152,7 +152,8 @@ import * as crypto from 'node:crypto';
             if (err && err.name === 'HaltExecutionError') {
                 this.log.warn(`Halted execution of epoch ${epochNumber} prover job`, {
                     uuid: this.uuid,
-                    epochNumber
+                    epochNumber,
+                    details: err.message
                 });
                 return;
             }
@@ -163,7 +164,7 @@ import * as crypto from 'node:crypto';
             this.state = 'failed';
         } finally{
             clearTimeout(this.deadlineTimeoutHandler);
-            await this.cleanUp(this);
+            await this.epochCheckPromise?.stop();
             await this.prover.stop();
             resolve();
         }
@@ -173,7 +174,7 @@ import * as crypto from 'node:crypto';
         this.state = state;
     }
     checkState() {
-        if (this.state === 'timed-out' || this.state === 'stopped' || this.state === 'failed') {
+        if (this.state === 'timed-out' || this.state === 'stopped' || this.state === 'failed' || this.state === 'reorg') {
             throw new HaltExecutionError(this.state);
         }
     }
@@ -209,6 +210,27 @@ import * as crypto from 'node:crypto';
             }, timeout);
         }
     }
+    /**
+   * Kicks off a running promise that queries the archiver for the set of L2 blocks of the current epoch.
+   * If those changed, stops the proving job with a `rerun` state, so the node re-enqueues it.
+   */ async scheduleEpochCheck() {
+        const intervalMs = Math.ceil((await this.l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
+        this.epochCheckPromise = new RunningPromise(async ()=>{
+            const blocks = await this.l2BlockSource.getBlockHeadersForEpoch(this.epochNumber);
+            const blockHashes = await Promise.all(blocks.map((block)=>block.hash()));
+            const thisBlockHashes = await Promise.all(this.blocks.map((block)=>block.hash()));
+            if (blocks.length !== this.blocks.length || !blockHashes.every((block, i)=>block.equals(thisBlockHashes[i]))) {
+                this.log.warn('Epoch blocks changed underfoot', {
+                    uuid: this.uuid,
+                    epochNumber: this.epochNumber,
+                    oldBlockHashes: thisBlockHashes,
+                    newBlockHashes: blockHashes
+                });
+                void this.stop('reorg');
+            }
+        }, this.log, intervalMs).start();
+        this.log.verbose(`Scheduled epoch check for epoch ${this.epochNumber} every ${intervalMs}ms`);
+    }
     /* Returns the header for the given block number, or the genesis block for block zero. */ async getBlockHeader(blockNumber) {
         if (blockNumber === 0) {
             return (await this.dbProvider.fork()).getInitialHeader();
@@ -232,7 +254,8 @@ import * as crypto from 'node:crypto';
             deadline
         });
         if (failedTxs.length) {
-            throw new Error(`Txs failed processing: ${failedTxs.map(({ tx, error })=>`${tx.getTxHash()} (${error})`).join(', ')}`);
+            const failedTxHashes = await Promise.all(failedTxs.map(({ tx })=>tx.getTxHash()));
+            throw new Error(`Txs failed processing: ${failedTxs.map(({ error }, index)=>`${failedTxHashes[index]} (${error})`).join(', ')}`);
         }
         if (processedTxs.length !== txs.length) {
             throw new Error(`Failed to process all txs: processed ${processedTxs.length} out of ${txs.length}`);
@@ -248,8 +271,9 @@ _ts_decorate([
     })
 ], EpochProvingJob.prototype, "run", null);
 class HaltExecutionError extends Error {
+    state;
     constructor(state){
-        super(`Halted execution due to state ${state}`);
+        super(`Halted execution due to state ${state}`), this.state = state;
         this.name = 'HaltExecutionError';
     }
 }

package/dest/prover-node-publisher.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"prover-node-publisher.d.ts","sourceRoot":"","sources":["../src/prover-node-publisher.ts"],"names":[],"mappings":"AAAA,OAAO,EAA6B,wBAAwB,EAAE,MAAM,kBAAkB,CAAC;AACvF,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGjE,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAC3D,OAAO,EAAE,EAAE,EAAE,MAAM,0BAA0B,CAAC;AAE9C,OAAO,EAAE,KAAK,KAAK,EAAqB,MAAM,6BAA6B,CAAC;AAI5E,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAC/E,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,KAAK,EAAE,YAAY,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAEjF,OAAO,EAAE,KAAK,eAAe,EAAsB,MAAM,yBAAyB,CAAC;AAMnF;;GAEG;AACH,sEAAsE;AACtE,MAAM,MAAM,sBAAsB,GAAG;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,EAAE,CAAC;IACpB,UAAU,EAAE,EAAE,CAAC;IACf,iBAAiB,EAAE,EAAE,CAAC;IACtB,YAAY,EAAE,EAAE,CAAC;IACjB,YAAY,EAAE,EAAE,CAAC;IACjB,OAAO,EAAE,EAAE,CAAC;IACZ,QAAQ,EAAE,EAAE,CAAC;IACb,IAAI,EAAE,KAAK,CAAC,YAAY,EAAE,OAAO,wBAAwB,CAAC,CAAC;IAC3D,KAAK,EAAE,KAAK,CAAC;CACd,CAAC;AAEF,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,kBAAkB,CAA4B;IACtD,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,OAAO,CAAoB;IAEnC,SAAS,CAAC,GAAG,yCAA+C;IAE5D,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC;IAEzC,SAAgB,SAAS,EAAE,SAAS,CAAC;gBAGnC,MAAM,EAAE,cAAc,GAAG,eAAe,EACxC,IAAI,EAAE;QACJ,cAAc,EAAE,cAAc,CAAC;QAC/B,SAAS,EAAE,SAAS,CAAC;QACrB,SAAS,CAAC,EAAE,eAAe,CAAC;KAC7B;IAYH;;;;;OAKG;IACI,SAAS;IAKhB,wDAAwD;IACjD,OAAO;IAIP,gBAAgB;IAIV,gBAAgB,CAAC,IAAI,EAAE;QAClC,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,sBAAsB,CAAC;QACrC,KAAK,EAAE,KAAK,CAAC;KACd,GAAG,OAAO,CAAC,OAAO,CAAC;YAgDN,4BAA4B;YAyD5B,sBAAsB;IAoDpC,OAAO,CAAC,uBAAuB;cA+Bf,kBAAkB;CAGnC"}
+{"version":3,"file":"prover-node-publisher.d.ts","sourceRoot":"","sources":["../src/prover-node-publisher.ts"],"names":[],"mappings":"AAAA,OAAO,EAA6B,wBAAwB,EAAE,MAAM,kBAAkB,CAAC;AACvF,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGjE,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAC3D,OAAO,EAAE,EAAE,EAAE,MAAM,0BAA0B,CAAC;AAE9C,OAAO,EAAE,KAAK,KAAK,EAAqB,MAAM,6BAA6B,CAAC;AAI5E,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAC/E,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,KAAK,EAAE,YAAY,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAEjF,OAAO,EAAE,KAAK,eAAe,EAAsB,MAAM,yBAAyB,CAAC;AAMnF;;GAEG;AACH,sEAAsE;AACtE,MAAM,MAAM,sBAAsB,GAAG;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,EAAE,CAAC;IACpB,UAAU,EAAE,EAAE,CAAC;IACf,iBAAiB,EAAE,EAAE,CAAC;IACtB,YAAY,EAAE,EAAE,CAAC;IACjB,YAAY,EAAE,EAAE,CAAC;IACjB,OAAO,EAAE,EAAE,CAAC;IACZ,QAAQ,EAAE,EAAE,CAAC;IACb,IAAI,EAAE,KAAK,CAAC,YAAY,EAAE,OAAO,wBAAwB,CAAC,CAAC;IAC3D,KAAK,EAAE,KAAK,CAAC;CACd,CAAC;AAEF,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,kBAAkB,CAA4B;IACtD,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,OAAO,CAAoB;IAEnC,SAAS,CAAC,GAAG,yCAA+C;IAE5D,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC;IAEzC,SAAgB,SAAS,EAAE,SAAS,CAAC;gBAGnC,MAAM,EAAE,cAAc,GAAG,eAAe,EACxC,IAAI,EAAE;QACJ,cAAc,EAAE,cAAc,CAAC;QAC/B,SAAS,EAAE,SAAS,CAAC;QACrB,SAAS,CAAC,EAAE,eAAe,CAAC;KAC7B;IAYH;;;;;OAKG;IACI,SAAS;IAKhB,wDAAwD;IACjD,OAAO;IAIP,gBAAgB;IAIV,gBAAgB,CAAC,IAAI,EAAE;QAClC,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,sBAAsB,CAAC;QACrC,KAAK,EAAE,KAAK,CAAC;KACd,GAAG,OAAO,CAAC,OAAO,CAAC;YAgDN,4BAA4B;YA2D5B,sBAAsB;IAoDpC,OAAO,CAAC,uBAAuB;cA+Bf,kBAAkB;CAGnC"}

package/dest/prover-node-publisher.js

@@ -93,14 +93,16 @@ export class ProverNodePublisher {
         const { fromBlock, toBlock, publicInputs, proof } = args;
         // Check that the block numbers match the expected epoch to be proven
         const { pendingBlockNumber: pending, provenBlockNumber: proven } = await this.rollupContract.getTips();
-        if (proven !== BigInt(fromBlock) - 1n) {
+        // Don't publish if proven is beyond our toBlock, pointless to do so
+        if (proven > BigInt(toBlock)) {
             throw new Error(`Cannot submit epoch proof for ${fromBlock}-${toBlock} as proven block is ${proven}`);
         }
+        // toBlock can't be greater than pending
         if (toBlock > pending) {
             throw new Error(`Cannot submit epoch proof for ${fromBlock}-${toBlock} as pending block is ${pending}`);
         }
         // Check the block hash and archive for the immediate block before the epoch
-        const blockLog = await this.rollupContract.getBlock(proven);
+        const blockLog = await this.rollupContract.getBlock(BigInt(fromBlock - 1));
         if (publicInputs.previousArchive.root.toString() !== blockLog.archive) {
             throw new Error(`Previous archive root mismatch: ${publicInputs.previousArchive.root.toString()} !== ${blockLog.archive}`);
         }

package/dest/prover-node.d.ts

@@ -37,11 +37,10 @@ export declare class ProverNode implements EpochMonitorHandler, ProverNodeApi, T
     protected readonly telemetryClient: TelemetryClient;
     private log;
     private dateProvider;
-    private latestEpochWeAreProving;
     private jobs;
-    private cachedEpochData;
     private options;
     private metrics;
+    private l1Metrics;
     private txFetcher;
     private lastBlockNumber;
     readonly tracer: Tracer;
@@ -62,14 +61,11 @@ export declare class ProverNode implements EpochMonitorHandler, ProverNodeApi, T
      * Stops the prover node and all its dependencies.
      */
     stop(): Promise<void>;
-    /**
-     * Creates a proof for a block range. Returns once the proof has been submitted to L1.
-     */
-    prove(epochNumber: number | bigint): Promise<void>;
     /**
      * Starts a proving process and returns immediately.
      */
     startProof(epochNumber: number | bigint): Promise<void>;
+    private runJob;
     /**
      * Returns the prover instance.
      */
@@ -95,7 +91,7 @@ export declare class ProverNode implements EpochMonitorHandler, ProverNodeApi, T
     private gatherBlocks;
     private gatherTxs;
     /** Extracted for testing purposes. */
-    protected doCreateEpochProvingJob(epochNumber: bigint, deadline: Date | undefined, blocks: L2Block[], txs: Tx[], publicProcessorFactory: PublicProcessorFactory, cleanUp: () => Promise<void>): EpochProvingJob;
+    protected doCreateEpochProvingJob(epochNumber: bigint, deadline: Date | undefined, blocks: L2Block[], txs: Tx[], publicProcessorFactory: PublicProcessorFactory): EpochProvingJob;
     /** Extracted for testing purposes. */
     protected triggerMonitors(): Promise<void>;
 }

package/dest/prover-node.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"prover-node.d.ts","sourceRoot":"","sources":["../src/prover-node.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAEjE,OAAO,EACL,KAAK,kBAAkB,EAEvB,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,OAAO,EACZ,KAAK,sBAAsB,EAE5B,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,KAAK,EAAE,EAAE,EAAU,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAEL,KAAK,eAAe,EACpB,KAAK,SAAS,EACd,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,eAAe,EAAE,KAAK,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAExF,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AACrF,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEtE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,yBAAyB,EAAE,MAAM,CAAC;IAClC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,8BAA8B,EAAE,MAAM,CAAC;CACxC,CAAC;AAEF;;;;;GAKG;AACH,qBAAa,UAAW,YAAW,mBAAmB,EAAE,aAAa,EAAE,SAAS;IAgB5E,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,kBAAkB;IAC7C,SAAS,CAAC,QAAQ,CAAC,SAAS,EAAE,mBAAmB;IACjD,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC;IAChE,SAAS,CAAC,QAAQ,CAAC,mBAAmB,EAAE,mBAAmB;IAC3D,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,kBAAkB;IACzD,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,sBAAsB;IACrD,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,kBAAkB,GAAG,KAAK,CAAC,OAAO,CAAC;IACpE,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,YAAY;IAE9C,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,eAAe;IAxBrD,OAAO,CAAC,GAAG,CAA+B;IAC1C,OAAO,CAAC,YAAY,CAAsB;IAE1C,OAAO,CAAC,uBAAuB,CAAqB;IACpD,OAAO,CAAC,IAAI,CAA2C;IACvD,OAAO,CAAC,eAAe,CAAgF;IACvG,OAAO,CAAC,OAAO,CAAoB;IACnC,OAAO,CAAC,OAAO,CAAoB;IAEnC,OAAO,CAAC,SAAS,CAAiB;IAClC,OAAO,CAAC,eAAe,CAAqB;IAE5C,SAAgB,MAAM,EAAE,MAAM,CAAC;gBAGV,MAAM,EAAE,kBAAkB,EAC1B,SAAS,EAAE,mBAAmB,EAC9B,aAAa,EAAE,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC,EAC7C,mBAAmB,EAAE,mBAAmB,EACxC,kBAAkB,EAAE,kBAAkB,EACtC,UAAU,EAAE,sBAAsB,EAClC,YAAY,EAAE,kBAAkB,GAAG,KAAK,CAAC,OAAO,CAAC,EACjD,aAAa,EAAE,YAAY,EAC9C,OAAO,GAAE,OAAO,CAAC,iBAAiB,CAAM,EACrB,eAAe,GAAE,eAAsC;IAiBrE,WAAW;IAIX,MAAM;IAQb;;;OAGG;IACG,uBAAuB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBjE;;;OAGG;IACH,KAAK;IAML;;OAEG;IACG,IAAI;IAcV;;OAEG;IACU,KAAK,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM;IAK/C;;OAEG;IACU,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM;IAKpD;;OAEG;IACI,SAAS;IAIhB;;OAEG;IACI,OAAO,IAAI,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,oBAAoB,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;cAUhF,qBAAqB,CACnC,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,oBAAoB,CAAA;KAAE,EAAE,CAAC;IAM5D,OAAO,CAAC,uBAAuB;YAMjB,gBAAgB;IAqC9B,OAAO,CAAC,cAAc;IAItB,kHAAkH;YAEpG,WAAW;YAeX,eAAe;YAQf,YAAY;YAQZ,SAAS;IAiBvB,sCAAsC;IACtC,SAAS,CAAC,uBAAuB,CAC/B,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,IAAI,GAAG,SAAS,EAC1B,MAAM,EAAE,OAAO,EAAE,EACjB,GAAG,EAAE,EAAE,EAAE,EACT,sBAAsB,EAAE,sBAAsB,EAC9C,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC;IAmB9B,sCAAsC;cACtB,eAAe;CAGhC"}
+{"version":3,"file":"prover-node.d.ts","sourceRoot":"","sources":["../src/prover-node.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAEjE,OAAO,EACL,KAAK,kBAAkB,EAEvB,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,OAAO,EACZ,KAAK,sBAAsB,EAE5B,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,KAAK,EAAE,EAAE,EAAU,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAGL,KAAK,eAAe,EACpB,KAAK,SAAS,EACd,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,eAAe,EAAE,KAAK,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAExF,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AACrF,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEtE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,yBAAyB,EAAE,MAAM,CAAC;IAClC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,8BAA8B,EAAE,MAAM,CAAC;CACxC,CAAC;AAEF;;;;;GAKG;AACH,qBAAa,UAAW,YAAW,mBAAmB,EAAE,aAAa,EAAE,SAAS;IAe5E,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,kBAAkB;IAC7C,SAAS,CAAC,QAAQ,CAAC,SAAS,EAAE,mBAAmB;IACjD,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC;IAChE,SAAS,CAAC,QAAQ,CAAC,mBAAmB,EAAE,mBAAmB;IAC3D,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,kBAAkB;IACzD,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,sBAAsB;IACrD,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,kBAAkB,GAAG,KAAK,CAAC,OAAO,CAAC;IACpE,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,YAAY;IAE9C,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,eAAe;IAvBrD,OAAO,CAAC,GAAG,CAA+B;IAC1C,OAAO,CAAC,YAAY,CAAsB;IAE1C,OAAO,CAAC,IAAI,CAA2C;IACvD,OAAO,CAAC,OAAO,CAAoB;IACnC,OAAO,CAAC,OAAO,CAAoB;IACnC,OAAO,CAAC,SAAS,CAAY;IAE7B,OAAO,CAAC,SAAS,CAAiB;IAClC,OAAO,CAAC,eAAe,CAAqB;IAE5C,SAAgB,MAAM,EAAE,MAAM,CAAC;gBAGV,MAAM,EAAE,kBAAkB,EAC1B,SAAS,EAAE,mBAAmB,EAC9B,aAAa,EAAE,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC,EAC7C,mBAAmB,EAAE,mBAAmB,EACxC,kBAAkB,EAAE,kBAAkB,EACtC,UAAU,EAAE,sBAAsB,EAClC,YAAY,EAAE,kBAAkB,GAAG,KAAK,CAAC,OAAO,CAAC,EACjD,aAAa,EAAE,YAAY,EAC9C,OAAO,GAAE,OAAO,CAAC,iBAAiB,CAAM,EACrB,eAAe,GAAE,eAAsC;IAqBrE,WAAW;IAIX,MAAM;IAQb;;;OAGG;IACG,uBAAuB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBjE;;;OAGG;IACH,KAAK;IAOL;;OAEG;IACG,IAAI;IAeV;;OAEG;IACU,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM;YAKtC,MAAM;IAkBpB;;OAEG;IACI,SAAS;IAIhB;;OAEG;IACI,OAAO,IAAI,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,oBAAoB,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;cAUhF,qBAAqB,CACnC,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,oBAAoB,CAAA;KAAE,EAAE,CAAC;IAM5D,OAAO,CAAC,uBAAuB;YAMjB,gBAAgB;IA+B9B,OAAO,CAAC,cAAc;IAItB,kHAAkH;YAEpG,WAAW;YAeX,eAAe;YAQf,YAAY;YAQZ,SAAS;IAiBvB,sCAAsC;IACtC,SAAS,CAAC,uBAAuB,CAC/B,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,IAAI,GAAG,SAAS,EAC1B,MAAM,EAAE,OAAO,EAAE,EACjB,GAAG,EAAE,EAAE,EAAE,EACT,sBAAsB,EAAE,sBAAsB;IAkBhD,sCAAsC;cACtB,eAAe;CAGhC"}

package/dest/prover-node.js

@@ -12,7 +12,7 @@ import { DateProvider } from '@aztec/foundation/timer';
 import { PublicProcessorFactory } from '@aztec/simulator/server';
 import { getTimestampRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
 import { EpochProvingJobTerminalState, tryStop } from '@aztec/stdlib/interfaces/server';
-import { Attributes, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
+import { Attributes, L1Metrics, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
 import { EpochProvingJob } from './job/epoch-proving-job.js';
 import { ProverNodeMetrics } from './metrics.js';
 /**
@@ -32,11 +32,10 @@ import { ProverNodeMetrics } from './metrics.js';
     telemetryClient;
     log;
     dateProvider;
-    latestEpochWeAreProving;
     jobs;
-    cachedEpochData;
     options;
     metrics;
+    l1Metrics;
     txFetcher;
     lastBlockNumber;
     tracer;
@@ -53,7 +52,9 @@ import { ProverNodeMetrics } from './metrics.js';
         this.log = createLogger('prover-node');
         this.dateProvider = new DateProvider();
         this.jobs = new Map();
-        this.cachedEpochData = undefined;
+        this.l1Metrics = new L1Metrics(telemetryClient.getMeter('ProverNodeL1Metrics'), publisher.l1TxUtils.publicClient, [
+            publisher.getSenderAddress()
+        ]);
         this.options = {
             pollingIntervalMs: 1_000,
             maxPendingJobs: 100,
@@ -82,13 +83,16 @@ import { ProverNodeMetrics } from './metrics.js';
    * @param epochNumber - The epoch number that was just completed.
    */ async handleEpochReadyToProve(epochNumber) {
         try {
-            this.log.debug('jobs', JSON.stringify(this.jobs, null, 2));
+            this.log.debug(`Running jobs as ${epochNumber} is ready to prove`, {
+                jobs: Array.from(this.jobs.values()).map((job)=>`${job.getEpochNumber()}:${job.getId()}`)
+            });
             const activeJobs = await this.getActiveJobsForEpoch(epochNumber);
             if (activeJobs.length > 0) {
-                this.log.info(`Not starting proof for ${epochNumber} since there are active jobs`);
+                this.log.warn(`Not starting proof for ${epochNumber} since there are active jobs for the epoch`, {
+                    activeJobs: activeJobs.map((job)=>job.uuid)
+                });
                 return;
             }
-            // TODO: we probably want to skip starting a proof if we are too far into the current epoch
             await this.startProof(epochNumber);
         } catch (err) {
             if (err instanceof EmptyEpochError) {
@@ -104,6 +108,7 @@ import { ProverNodeMetrics } from './metrics.js';
    */ start() {
         this.txFetcher.start();
         this.epochsMonitor.start(this);
+        this.l1Metrics.start();
         this.log.info('Started ProverNode', this.options);
     }
     /**
@@ -118,20 +123,35 @@ import { ProverNodeMetrics } from './metrics.js';
         await Promise.all(Array.from(this.jobs.values()).map((job)=>job.stop()));
         await this.worldState.stop();
         await tryStop(this.coordination);
+        this.l1Metrics.stop();
         await this.telemetryClient.stop();
         this.log.info('Stopped ProverNode');
     }
     /**
-   * Creates a proof for a block range. Returns once the proof has been submitted to L1.
-   */ async prove(epochNumber) {
-        const job = await this.createProvingJob(BigInt(epochNumber));
-        return job.run();
-    }
-    /**
    * Starts a proving process and returns immediately.
    */ async startProof(epochNumber) {
         const job = await this.createProvingJob(BigInt(epochNumber));
-        void job.run().catch((err)=>this.log.error(`Error proving epoch ${epochNumber}`, err));
+        void this.runJob(job);
+    }
+    async runJob(job) {
+        const ctx = {
+            id: job.getId(),
+            epochNumber: job.getEpochNumber()
+        };
+        try {
+            await job.run();
+            const state = job.getState();
+            if (state === 'reorg') {
+                this.log.warn(`Running new job for epoch ${job.getEpochNumber()} due to reorg`, ctx);
+                await this.createProvingJob(job.getEpochNumber());
+            } else {
+                this.log.verbose(`Job for ${job.getEpochNumber()} exited with state ${state}`, ctx);
+            }
+        } catch (err) {
+            this.log.error(`Error proving epoch ${job.getEpochNumber()}`, err, ctx);
+        } finally{
+            this.jobs.delete(job.getId());
+        }
     }
     /**
    * Returns the prover instance.
@@ -161,8 +181,7 @@ import { ProverNodeMetrics } from './metrics.js';
             throw new Error(`Maximum pending proving jobs ${this.options.maxPendingJobs} reached. Cannot create new job.`);
         }
         // Gather blocks for this epoch
-        const cachedEpochData = this.cachedEpochData?.epochNumber === epochNumber ? this.cachedEpochData : undefined;
-        const { blocks, txs } = cachedEpochData ?? await this.gatherEpochData(epochNumber);
+        const { blocks, txs } = await this.gatherEpochData(epochNumber);
         const fromBlock = blocks[0].number;
         const toBlock = blocks.at(-1).number;
         // Fast forward world state to right before the target block and get a fork
@@ -170,13 +189,9 @@ import { ProverNodeMetrics } from './metrics.js';
         await this.worldState.syncImmediate(toBlock);
         // Create a processor using the forked world state
         const publicProcessorFactory = new PublicProcessorFactory(this.contractDataSource, this.dateProvider, this.telemetryClient);
-        const cleanUp = ()=>{
-            this.jobs.delete(job.getId());
-            return Promise.resolve();
-        };
         const [_, endTimestamp] = getTimestampRangeForEpoch(epochNumber + 1n, await this.getL1Constants());
         const deadline = new Date(Number(endTimestamp) * 1000);
-        const job = this.doCreateEpochProvingJob(epochNumber, deadline, blocks, txs, publicProcessorFactory, cleanUp);
+        const job = this.doCreateEpochProvingJob(epochNumber, deadline, blocks, txs, publicProcessorFactory);
         this.jobs.set(job.getId(), job);
         return job;
     }
@@ -225,10 +240,10 @@ import { ProverNodeMetrics } from './metrics.js';
         const missingTxHashes = txsToFind.filter((txHashToFind)=>!txHashesFound.some((txHashFound)=>txHashToFind.equals(txHashFound))).join(', ');
         throw new Error(`Txs not found for epoch ${epochNumber}: ${missingTxHashes}`);
     }
-    /** Extracted for testing purposes. */ doCreateEpochProvingJob(epochNumber, deadline, blocks, txs, publicProcessorFactory, cleanUp) {
+    /** Extracted for testing purposes. */ doCreateEpochProvingJob(epochNumber, deadline, blocks, txs, publicProcessorFactory) {
         return new EpochProvingJob(this.worldState, epochNumber, blocks, txs, this.prover.createEpochProver(), publicProcessorFactory, this.publisher, this.l2BlockSource, this.l1ToL2MessageSource, this.metrics, deadline, {
             parallelBlockLimit: this.options.maxParallelBlocksPerEpoch
-        }, cleanUp);
+        });
     }
     /** Extracted for testing purposes. */ async triggerMonitors() {
         await this.epochsMonitor.work();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aztec/prover-node",
-  "version": "0.80.0",
+  "version": "0.82.0",
   "type": "module",
   "exports": {
     ".": "./dest/index.js",
@@ -53,24 +53,24 @@
     ]
   },
   "dependencies": {
-    "@aztec/archiver": "0.80.0",
-    "@aztec/bb-prover": "0.80.0",
-    "@aztec/blob-sink": "0.80.0",
-    "@aztec/constants": "0.80.0",
-    "@aztec/epoch-cache": "0.80.0",
-    "@aztec/ethereum": "0.80.0",
-    "@aztec/foundation": "0.80.0",
-    "@aztec/kv-store": "0.80.0",
-    "@aztec/l1-artifacts": "0.80.0",
-    "@aztec/noir-protocol-circuits-types": "0.80.0",
-    "@aztec/p2p": "0.80.0",
-    "@aztec/protocol-contracts": "0.80.0",
-    "@aztec/prover-client": "0.80.0",
-    "@aztec/sequencer-client": "0.80.0",
-    "@aztec/simulator": "0.80.0",
-    "@aztec/stdlib": "0.80.0",
-    "@aztec/telemetry-client": "0.80.0",
-    "@aztec/world-state": "0.80.0",
+    "@aztec/archiver": "0.82.0",
+    "@aztec/bb-prover": "0.82.0",
+    "@aztec/blob-sink": "0.82.0",
+    "@aztec/constants": "0.82.0",
+    "@aztec/epoch-cache": "0.82.0",
+    "@aztec/ethereum": "0.82.0",
+    "@aztec/foundation": "0.82.0",
+    "@aztec/kv-store": "0.82.0",
+    "@aztec/l1-artifacts": "0.82.0",
+    "@aztec/noir-protocol-circuits-types": "0.82.0",
+    "@aztec/p2p": "0.82.0",
+    "@aztec/protocol-contracts": "0.82.0",
+    "@aztec/prover-client": "0.82.0",
+    "@aztec/sequencer-client": "0.82.0",
+    "@aztec/simulator": "0.82.0",
+    "@aztec/stdlib": "0.82.0",
+    "@aztec/telemetry-client": "0.82.0",
+    "@aztec/world-state": "0.82.0",
     "source-map-support": "^0.5.21",
     "tslib": "^2.4.0",
     "viem": "2.23.7"

package/src/job/epoch-proving-job.ts

@@ -1,6 +1,6 @@
 import { asyncPool } from '@aztec/foundation/async-pool';
 import { createLogger } from '@aztec/foundation/log';
-import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 import { Timer } from '@aztec/foundation/timer';
 import type { PublicProcessor, PublicProcessorFactory } from '@aztec/simulator/server';
 import type { L2Block, L2BlockSource } from '@aztec/stdlib/block';
@@ -30,6 +30,7 @@ export class EpochProvingJob implements Traceable {
   private uuid: string;
 
   private runPromise: Promise<void> | undefined;
+  private epochCheckPromise: RunningPromise | undefined;
   private deadlineTimeoutHandler: NodeJS.Timeout | undefined;
 
   public readonly tracer: Tracer;
@@ -47,7 +48,6 @@ export class EpochProvingJob implements Traceable {
     private metrics: ProverNodeMetrics,
     private deadline: Date | undefined,
     private config: { parallelBlockLimit: number } = { parallelBlockLimit: 32 },
-    private cleanUp: (job: EpochProvingJob) => Promise<void> = () => Promise.resolve(),
   ) {
     this.uuid = crypto.randomUUID();
     this.tracer = metrics.client.getTracer('EpochProvingJob');
@@ -73,6 +73,7 @@ export class EpochProvingJob implements Traceable {
   })
   public async run() {
     this.scheduleDeadlineStop();
+    await this.scheduleEpochCheck();
 
     const epochNumber = Number(this.epochNumber);
     const epochSizeBlocks = this.blocks.length;
@@ -154,14 +155,18 @@ export class EpochProvingJob implements Traceable {
       this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeBlocks, epochSizeTxs);
     } catch (err: any) {
       if (err && err.name === 'HaltExecutionError') {
-        this.log.warn(`Halted execution of epoch ${epochNumber} prover job`, { uuid: this.uuid, epochNumber });
+        this.log.warn(`Halted execution of epoch ${epochNumber} prover job`, {
+          uuid: this.uuid,
+          epochNumber,
+          details: err.message,
+        });
         return;
       }
       this.log.error(`Error running epoch ${epochNumber} prover job`, err, { uuid: this.uuid, epochNumber });
       this.state = 'failed';
     } finally {
       clearTimeout(this.deadlineTimeoutHandler);
-      await this.cleanUp(this);
+      await this.epochCheckPromise?.stop();
       await this.prover.stop();
       resolve();
     }
@@ -173,12 +178,12 @@ export class EpochProvingJob implements Traceable {
   }
 
   private checkState() {
-    if (this.state === 'timed-out' || this.state === 'stopped' || this.state === 'failed') {
+    if (this.state === 'timed-out' || this.state === 'stopped' || this.state === 'failed' || this.state === 'reorg') {
       throw new HaltExecutionError(this.state);
     }
   }
 
-  public async stop(state: EpochProvingJobState = 'stopped') {
+  public async stop(state: EpochProvingJobTerminalState = 'stopped') {
     this.state = state;
     this.prover.cancel();
     // TODO(palla/prover): Stop the publisher as well
@@ -207,6 +212,36 @@ export class EpochProvingJob implements Traceable {
     }
   }
 
+  /**
+   * Kicks off a running promise that queries the archiver for the set of L2 blocks of the current epoch.
+   * If those changed, stops the proving job with a `rerun` state, so the node re-enqueues it.
+   */
+  private async scheduleEpochCheck() {
+    const intervalMs = Math.ceil((await this.l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
+    this.epochCheckPromise = new RunningPromise(
+      async () => {
+        const blocks = await this.l2BlockSource.getBlockHeadersForEpoch(this.epochNumber);
+        const blockHashes = await Promise.all(blocks.map(block => block.hash()));
+        const thisBlockHashes = await Promise.all(this.blocks.map(block => block.hash()));
+        if (
+          blocks.length !== this.blocks.length ||
+          !blockHashes.every((block, i) => block.equals(thisBlockHashes[i]))
+        ) {
+          this.log.warn('Epoch blocks changed underfoot', {
+            uuid: this.uuid,
+            epochNumber: this.epochNumber,
+            oldBlockHashes: thisBlockHashes,
+            newBlockHashes: blockHashes,
+          });
+          void this.stop('reorg');
+        }
+      },
+      this.log,
+      intervalMs,
+    ).start();
+    this.log.verbose(`Scheduled epoch check for epoch ${this.epochNumber} every ${intervalMs}ms`);
+  }
+
   /* Returns the header for the given block number, or the genesis block for block zero. */
   private async getBlockHeader(blockNumber: number) {
     if (blockNumber === 0) {
@@ -232,8 +267,11 @@ export class EpochProvingJob implements Traceable {
     const [processedTxs, failedTxs] = await publicProcessor.process(txs, { deadline });
 
     if (failedTxs.length) {
+      const failedTxHashes = await Promise.all(failedTxs.map(({ tx }) => tx.getTxHash()));
       throw new Error(
-        `Txs failed processing: ${failedTxs.map(({ tx, error }) => `${tx.getTxHash()} (${error})`).join(', ')}`,
+        `Txs failed processing: ${failedTxs
+          .map(({ error }, index) => `${failedTxHashes[index]} (${error})`)
+          .join(', ')}`,
       );
     }
 
@@ -246,7 +284,7 @@ export class EpochProvingJob implements Traceable {
 }
 
 class HaltExecutionError extends Error {
-  constructor(state: EpochProvingJobState) {
+  constructor(public readonly state: EpochProvingJobState) {
     super(`Halted execution due to state ${state}`);
     this.name = 'HaltExecutionError';
   }

package/src/prover-node-publisher.ts

@@ -150,15 +150,17 @@ export class ProverNodePublisher {
 
     // Check that the block numbers match the expected epoch to be proven
     const { pendingBlockNumber: pending, provenBlockNumber: proven } = await this.rollupContract.getTips();
-    if (proven !== BigInt(fromBlock) - 1n) {
+    // Don't publish if proven is beyond our toBlock, pointless to do so
+    if (proven > BigInt(toBlock)) {
       throw new Error(`Cannot submit epoch proof for ${fromBlock}-${toBlock} as proven block is ${proven}`);
     }
+    // toBlock can't be greater than pending
     if (toBlock > pending) {
       throw new Error(`Cannot submit epoch proof for ${fromBlock}-${toBlock} as pending block is ${pending}`);
     }
 
     // Check the block hash and archive for the immediate block before the epoch
-    const blockLog = await this.rollupContract.getBlock(proven);
+    const blockLog = await this.rollupContract.getBlock(BigInt(fromBlock - 1));
     if (publicInputs.previousArchive.root.toString() !== blockLog.archive) {
       throw new Error(
         `Previous archive root mismatch: ${publicInputs.previousArchive.root.toString()} !== ${blockLog.archive}`,

package/src/prover-node.ts

@@ -23,6 +23,7 @@ import type { P2PClientType } from '@aztec/stdlib/p2p';
 import type { Tx, TxHash } from '@aztec/stdlib/tx';
 import {
   Attributes,
+  L1Metrics,
   type TelemetryClient,
   type Traceable,
   type Tracer,
@@ -54,11 +55,10 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
   private log = createLogger('prover-node');
   private dateProvider = new DateProvider();
 
-  private latestEpochWeAreProving: bigint | undefined;
   private jobs: Map<string, EpochProvingJob> = new Map();
-  private cachedEpochData: { epochNumber: bigint; blocks: L2Block[]; txs: Tx[] } | undefined = undefined;
   private options: ProverNodeOptions;
   private metrics: ProverNodeMetrics;
+  private l1Metrics: L1Metrics;
 
   private txFetcher: RunningPromise;
   private lastBlockNumber: number | undefined;
@@ -77,6 +77,10 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
     options: Partial<ProverNodeOptions> = {},
     protected readonly telemetryClient: TelemetryClient = getTelemetryClient(),
   ) {
+    this.l1Metrics = new L1Metrics(telemetryClient.getMeter('ProverNodeL1Metrics'), publisher.l1TxUtils.publicClient, [
+      publisher.getSenderAddress(),
+    ]);
+
     this.options = {
       pollingIntervalMs: 1_000,
       maxPendingJobs: 100,
@@ -110,13 +114,16 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
    */
   async handleEpochReadyToProve(epochNumber: bigint): Promise<void> {
     try {
-      this.log.debug('jobs', JSON.stringify(this.jobs, null, 2));
+      this.log.debug(`Running jobs as ${epochNumber} is ready to prove`, {
+        jobs: Array.from(this.jobs.values()).map(job => `${job.getEpochNumber()}:${job.getId()}`),
+      });
       const activeJobs = await this.getActiveJobsForEpoch(epochNumber);
       if (activeJobs.length > 0) {
-        this.log.info(`Not starting proof for ${epochNumber} since there are active jobs`);
+        this.log.warn(`Not starting proof for ${epochNumber} since there are active jobs for the epoch`, {
+          activeJobs: activeJobs.map(job => job.uuid),
+        });
         return;
       }
-      // TODO: we probably want to skip starting a proof if we are too far into the current epoch
       await this.startProof(epochNumber);
     } catch (err) {
       if (err instanceof EmptyEpochError) {
@@ -134,6 +141,7 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
   start() {
     this.txFetcher.start();
     this.epochsMonitor.start(this);
+    this.l1Metrics.start();
     this.log.info('Started ProverNode', this.options);
   }
 
@@ -150,24 +158,35 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
     await Promise.all(Array.from(this.jobs.values()).map(job => job.stop()));
     await this.worldState.stop();
     await tryStop(this.coordination);
+    this.l1Metrics.stop();
     await this.telemetryClient.stop();
     this.log.info('Stopped ProverNode');
   }
 
-  /**
-   * Creates a proof for a block range. Returns once the proof has been submitted to L1.
-   */
-  public async prove(epochNumber: number | bigint) {
-    const job = await this.createProvingJob(BigInt(epochNumber));
-    return job.run();
-  }
-
   /**
    * Starts a proving process and returns immediately.
    */
   public async startProof(epochNumber: number | bigint) {
     const job = await this.createProvingJob(BigInt(epochNumber));
-    void job.run().catch(err => this.log.error(`Error proving epoch ${epochNumber}`, err));
+    void this.runJob(job);
+  }
+
+  private async runJob(job: EpochProvingJob) {
+    const ctx = { id: job.getId(), epochNumber: job.getEpochNumber() };
+    try {
+      await job.run();
+      const state = job.getState();
+      if (state === 'reorg') {
+        this.log.warn(`Running new job for epoch ${job.getEpochNumber()} due to reorg`, ctx);
+        await this.createProvingJob(job.getEpochNumber());
+      } else {
+        this.log.verbose(`Job for ${job.getEpochNumber()} exited with state ${state}`, ctx);
+      }
+    } catch (err) {
+      this.log.error(`Error proving epoch ${job.getEpochNumber()}`, err, ctx);
+    } finally {
+      this.jobs.delete(job.getId());
+    }
   }
 
   /**
@@ -210,8 +229,7 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
     }
 
     // Gather blocks for this epoch
-    const cachedEpochData = this.cachedEpochData?.epochNumber === epochNumber ? this.cachedEpochData : undefined;
-    const { blocks, txs } = cachedEpochData ?? (await this.gatherEpochData(epochNumber));
+    const { blocks, txs } = await this.gatherEpochData(epochNumber);
 
     const fromBlock = blocks[0].number;
     const toBlock = blocks.at(-1)!.number;
@@ -227,15 +245,10 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
       this.telemetryClient,
     );
 
-    const cleanUp = () => {
-      this.jobs.delete(job.getId());
-      return Promise.resolve();
-    };
-
     const [_, endTimestamp] = getTimestampRangeForEpoch(epochNumber + 1n, await this.getL1Constants());
     const deadline = new Date(Number(endTimestamp) * 1000);
 
-    const job = this.doCreateEpochProvingJob(epochNumber, deadline, blocks, txs, publicProcessorFactory, cleanUp);
+    const job = this.doCreateEpochProvingJob(epochNumber, deadline, blocks, txs, publicProcessorFactory);
     this.jobs.set(job.getId(), job);
     return job;
   }
@@ -302,7 +315,6 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
     blocks: L2Block[],
     txs: Tx[],
     publicProcessorFactory: PublicProcessorFactory,
-    cleanUp: () => Promise<void>,
   ) {
     return new EpochProvingJob(
       this.worldState,
@@ -317,7 +329,6 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
       this.metrics,
       deadline,
       { parallelBlockLimit: this.options.maxParallelBlocksPerEpoch },
-      cleanUp,
     );
   }