@aztec/prover-node 0.0.0-test.1 → 0.0.1-commit.017a351

package/src/job/epoch-proving-job.ts

@@ -1,56 +1,76 @@
 import { asyncPool } from '@aztec/foundation/async-pool';
-import { createLogger } from '@aztec/foundation/log';
-import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
+import { Fr } from '@aztec/foundation/curves/bn254';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
+import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 import { Timer } from '@aztec/foundation/timer';
+import { AVM_MAX_CONCURRENT_SIMULATIONS } from '@aztec/native';
+import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
+import { protocolContractsHash } from '@aztec/protocol-contracts';
+import { buildFinalBlobChallenges } from '@aztec/prover-client/helpers';
 import type { PublicProcessor, PublicProcessorFactory } from '@aztec/simulator/server';
+import { PublicSimulatorConfig } from '@aztec/stdlib/avm';
 import type { L2Block, L2BlockSource } from '@aztec/stdlib/block';
+import type { Checkpoint } from '@aztec/stdlib/checkpoint';
 import {
   type EpochProver,
   type EpochProvingJobState,
   EpochProvingJobTerminalState,
   type ForkMerkleTreeOperations,
 } from '@aztec/stdlib/interfaces/server';
-import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
+import { appendL1ToL2MessagesToTree } from '@aztec/stdlib/messaging';
+import { CheckpointConstantData } from '@aztec/stdlib/rollup';
 import type { ProcessedTx, Tx } from '@aztec/stdlib/tx';
 import { Attributes, type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client';
 
 import * as crypto from 'node:crypto';
 
-import type { ProverNodeMetrics } from '../metrics.js';
+import type { ProverNodeJobMetrics } from '../metrics.js';
 import type { ProverNodePublisher } from '../prover-node-publisher.js';
+import { type EpochProvingJobData, validateEpochProvingJobData } from './epoch-proving-job-data.js';
+
+export type EpochProvingJobOptions = {
+  parallelBlockLimit?: number;
+  skipEpochCheck?: boolean;
+  skipSubmitProof?: boolean;
+};
 
 /**
- * Job that grabs a range of blocks from the unfinalised chain from L1, gets their txs given their hashes,
+ * Job that grabs a range of blocks from the unfinalized chain from L1, gets their txs given their hashes,
  * re-executes their public calls, generates a rollup proof, and submits it to L1. This job will update the
  * world state as part of public call execution via the public processor.
  */
 export class EpochProvingJob implements Traceable {
   private state: EpochProvingJobState = 'initialized';
-  private log = createLogger('prover-node:epoch-proving-job');
+  private log: Logger;
   private uuid: string;
 
   private runPromise: Promise<void> | undefined;
+  private abortController = new AbortController();
+  private epochCheckPromise: RunningPromise | undefined;
   private deadlineTimeoutHandler: NodeJS.Timeout | undefined;
 
   public readonly tracer: Tracer;
 
   constructor(
-    private dbProvider: ForkMerkleTreeOperations,
-    private epochNumber: bigint,
-    private blocks: L2Block[],
-    private txs: Tx[],
+    private data: EpochProvingJobData,
+    private dbProvider: Pick<ForkMerkleTreeOperations, 'fork'>,
     private prover: EpochProver,
     private publicProcessorFactory: PublicProcessorFactory,
-    private publisher: ProverNodePublisher,
-    private l2BlockSource: L2BlockSource,
-    private l1ToL2MessageSource: L1ToL2MessageSource,
-    private metrics: ProverNodeMetrics,
+    private publisher: Pick<ProverNodePublisher, 'submitEpochProof' | 'analyzeEpochProofSubmission'>,
+    private l2BlockSource: L2BlockSource | undefined,
+    private metrics: ProverNodeJobMetrics,
     private deadline: Date | undefined,
-    private config: { parallelBlockLimit: number } = { parallelBlockLimit: 32 },
-    private cleanUp: (job: EpochProvingJob) => Promise<void> = () => Promise.resolve(),
+    private config: EpochProvingJobOptions,
+    bindings?: LoggerBindings,
   ) {
+    validateEpochProvingJobData(data);
     this.uuid = crypto.randomUUID();
-    this.tracer = metrics.client.getTracer('EpochProvingJob');
+    this.log = createLogger('prover-node:epoch-proving-job', {
+      ...bindings,
+      instanceId: `epoch-${data.epochNumber}`,
+    });
+    this.tracer = metrics.tracer;
   }
 
   public getId(): string {
@@ -61,27 +81,63 @@ export class EpochProvingJob implements Traceable {
     return this.state;
   }
 
-  public getEpochNumber(): bigint {
-    return this.epochNumber;
+  public getEpochNumber(): EpochNumber {
+    return this.data.epochNumber;
+  }
+
+  public getDeadline(): Date | undefined {
+    return this.deadline;
+  }
+
+  public getProvingData(): EpochProvingJobData {
+    return this.data;
+  }
+
+  private get epochNumber() {
+    return this.data.epochNumber;
+  }
+
+  private get checkpoints() {
+    return this.data.checkpoints;
+  }
+
+  private get txs() {
+    return this.data.txs;
+  }
+
+  private get attestations() {
+    return this.data.attestations;
   }
 
   /**
    * Proves the given epoch and submits the proof to L1.
    */
   @trackSpan('EpochProvingJob.run', function () {
-    return { [Attributes.EPOCH_NUMBER]: Number(this.epochNumber) };
+    return { [Attributes.EPOCH_NUMBER]: this.data.epochNumber };
   })
   public async run() {
     this.scheduleDeadlineStop();
+    if (!this.config.skipEpochCheck) {
+      await this.scheduleEpochCheck();
+    }
 
-    const epochNumber = Number(this.epochNumber);
-    const epochSizeBlocks = this.blocks.length;
-    const epochSizeTxs = this.blocks.reduce((total, current) => total + current.body.txEffects.length, 0);
-    const [fromBlock, toBlock] = [this.blocks[0].number, this.blocks.at(-1)!.number];
-    this.log.info(`Starting epoch ${epochNumber} proving job with blocks ${fromBlock} to ${toBlock}`, {
+    const attestations = this.attestations.map(attestation => attestation.toViem());
+    const epochNumber = this.epochNumber;
+    const epochSizeCheckpoints = this.checkpoints.length;
+    const epochSizeBlocks = this.checkpoints.reduce((accum, checkpoint) => accum + checkpoint.blocks.length, 0);
+    const epochSizeTxs = this.checkpoints.reduce(
+      (accum, checkpoint) =>
+        accum + checkpoint.blocks.reduce((accumC, block) => accumC + block.body.txEffects.length, 0),
+      0,
+    );
+    const fromCheckpoint = this.checkpoints[0].number;
+    const toCheckpoint = this.checkpoints.at(-1)!.number;
+    const fromBlock = this.checkpoints[0].blocks[0].number;
+    const toBlock = this.checkpoints.at(-1)!.blocks.at(-1)!.number;
+    this.log.info(`Starting epoch ${epochNumber} proving job with checkpoints ${fromCheckpoint} to ${toCheckpoint}`, {
       fromBlock,
       toBlock,
-      epochSizeBlocks,
+      epochSizeTxs,
       epochNumber,
       uuid: this.uuid,
     });
@@ -92,101 +148,281 @@ export class EpochProvingJob implements Traceable {
     this.runPromise = promise;
 
     try {
-      this.prover.startNewEpoch(epochNumber, fromBlock, epochSizeBlocks);
-      await this.prover.startTubeCircuits(this.txs);
+      const blobTimer = new Timer();
+      const blobFieldsPerCheckpoint = this.checkpoints.map(checkpoint => checkpoint.toBlobFields());
+      const finalBlobBatchingChallenges = await buildFinalBlobChallenges(blobFieldsPerCheckpoint);
+      this.metrics.recordBlobProcessing(blobTimer.ms());
 
-      await asyncPool(this.config.parallelBlockLimit, this.blocks, async block => {
-        this.checkState();
+      this.prover.startNewEpoch(epochNumber, epochSizeCheckpoints, finalBlobBatchingChallenges);
+      const chonkTimer = new Timer();
+      await this.prover.startChonkVerifierCircuits(Array.from(this.txs.values()));
+      this.metrics.recordChonkVerifier(chonkTimer.ms());
 
-        const globalVariables = block.header.globalVariables;
-        const txs = await this.getTxs(block);
-        const l1ToL2Messages = await this.getL1ToL2Messages(block);
-        const previousHeader = (await this.getBlockHeader(block.number - 1))!;
-
-        this.log.verbose(`Starting processing block ${block.number}`, {
-          number: block.number,
-          blockHash: (await block.hash()).toString(),
-          lastArchive: block.header.lastArchive.root,
-          noteHashTreeRoot: block.header.state.partial.noteHashTree.root,
-          nullifierTreeRoot: block.header.state.partial.nullifierTree.root,
-          publicDataTreeRoot: block.header.state.partial.publicDataTree.root,
-          previousHeader: previousHeader.hash(),
-          uuid: this.uuid,
-          ...globalVariables,
-        });
+      // Everything in the epoch should have the same chainId and version.
+      const { chainId, version } = this.checkpoints[0].blocks[0].header.globalVariables;
 
-        // Start block proving
-        await this.prover.startNewBlock(globalVariables, l1ToL2Messages, previousHeader);
-
-        // Process public fns
-        const db = await this.dbProvider.fork(block.number - 1);
-        const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, true);
-        const processed = await this.processTxs(publicProcessor, txs);
-        await this.prover.addTxs(processed);
-        await db.close();
-        this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, {
-          blockNumber: block.number,
-          blockHash: (await block.hash()).toString(),
+      const previousBlockHeaders = this.gatherPreviousBlockHeaders();
+
+      const allCheckpointsTimer = new Timer();
+
+      const parallelism = this.config.parallelBlockLimit
+        ? this.config.parallelBlockLimit
+        : AVM_MAX_CONCURRENT_SIMULATIONS > 0
+          ? AVM_MAX_CONCURRENT_SIMULATIONS
+          : this.checkpoints.length;
+
+      await this.processCheckpoints(parallelism, async checkpoint => {
+        this.checkState();
+        const checkpointTimer = new Timer();
+
+        const checkpointIndex = checkpoint.number - fromCheckpoint;
+        const checkpointConstants = CheckpointConstantData.from({
+          chainId,
+          version,
+          vkTreeRoot: getVKTreeRoot(),
+          protocolContractsHash: protocolContractsHash,
+          proverId: this.prover.getProverId().toField(),
+          slotNumber: checkpoint.header.slotNumber,
+          coinbase: checkpoint.header.coinbase,
+          feeRecipient: checkpoint.header.feeRecipient,
+          gasFees: checkpoint.header.gasFees,
+        });
+        const previousHeader = previousBlockHeaders[checkpointIndex];
+        const l1ToL2Messages = this.getL1ToL2Messages(checkpoint);
+
+        this.log.debug(`Starting processing checkpoint ${checkpoint.number}`, {
+          number: checkpoint.number,
+          checkpointHash: checkpoint.hash().toString(),
+          headerHash: checkpoint.header.hash().toString(),
+          numL1ToL2Messages: l1ToL2Messages.length,
+          previousBlockNumber: previousHeader.globalVariables.blockNumber,
           uuid: this.uuid,
         });
 
-        // Mark block as completed to pad it
-        await this.prover.setBlockCompleted(block.number, block.header);
+        await this.prover.startNewCheckpoint(
+          checkpointIndex,
+          checkpointConstants,
+          l1ToL2Messages,
+          checkpoint.blocks.length,
+          previousHeader,
+        );
+
+        for (let blockIndex = 0; blockIndex < checkpoint.blocks.length; blockIndex++) {
+          const blockTimer = new Timer();
+          const block = checkpoint.blocks[blockIndex];
+          const globalVariables = block.header.globalVariables;
+          const txs = this.getTxs(block);
+
+          this.log.verbose(`Starting processing block ${block.number}`, {
+            number: block.number,
+            blockHash: (await block.hash()).toString(),
+            lastArchive: block.header.lastArchive.root,
+            noteHashTreeRoot: block.header.state.partial.noteHashTree.root,
+            nullifierTreeRoot: block.header.state.partial.nullifierTree.root,
+            publicDataTreeRoot: block.header.state.partial.publicDataTree.root,
+            ...globalVariables,
+            numTxs: txs.length,
+          });
+
+          // Start block proving
+          await this.prover.startNewBlock(block.number, globalVariables.timestamp, txs.length);
+
+          // Process public fns. L1 to L2 messages are only inserted for the first block of a checkpoint,
+          // as the fork for subsequent blocks already includes them from the previous block's synced state.
+          {
+            await using db = await this.createFork(
+              BlockNumber(block.number - 1),
+              blockIndex === 0 ? l1ToL2Messages : undefined,
+            );
+            this.checkState();
+            const config = PublicSimulatorConfig.from({
+              proverId: this.prover.getProverId().toField(),
+              skipFeeEnforcement: false,
+              collectDebugLogs: false,
+              collectHints: true,
+              collectPublicInputs: true,
+              collectStatistics: false,
+            });
+            const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, config);
+            const processed = await this.processTxs(publicProcessor, txs);
+            this.checkState();
+            await this.prover.addTxs(processed);
+          }
+          this.checkState();
+          this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, {
+            blockNumber: block.number,
+            blockHash: (await block.hash()).toString(),
+            uuid: this.uuid,
+          });
+
+          // Mark block as completed to pad it
+          const expectedBlockHeader = block.header;
+          await this.prover.setBlockCompleted(block.number, expectedBlockHeader);
+          this.metrics.recordBlockProcessing(blockTimer.ms());
+        }
+        this.metrics.recordCheckpointProcessing(checkpointTimer.ms());
       });
+      this.metrics.recordAllCheckpointsProcessing(allCheckpointsTimer.ms());
 
       const executionTime = timer.ms();
 
       this.progressState('awaiting-prover');
-      const { publicInputs, proof } = await this.prover.finaliseEpoch();
-      this.log.info(`Finalised proof for epoch ${epochNumber}`, { epochNumber, uuid: this.uuid, duration: timer.ms() });
+      const { publicInputs, proof, batchedBlobInputs } = await this.prover.finalizeEpoch();
+      this.log.info(`Finalized proof for epoch ${epochNumber}`, { epochNumber, uuid: this.uuid, duration: timer.ms() });
 
       this.progressState('publishing-proof');
-      const success = await this.publisher.submitEpochProof({ fromBlock, toBlock, epochNumber, publicInputs, proof });
+
+      if (this.config.skipSubmitProof) {
+        this.log.info(
+          `Proof publishing is disabled. Analyzing estimated L1 fees for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`,
+        );
+        try {
+          await this.publisher.analyzeEpochProofSubmission({
+            fromCheckpoint,
+            toCheckpoint,
+            epochNumber,
+            publicInputs,
+            proof,
+            batchedBlobInputs,
+            attestations,
+          });
+        } catch (err) {
+          this.log.warn(`Failed to analyze estimated L1 fees for epoch ${epochNumber}`, err);
+        }
+        this.state = 'completed';
+        this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
+        return;
+      }
+
+      const success = await this.publisher.submitEpochProof({
+        fromCheckpoint,
+        toCheckpoint,
+        epochNumber,
+        publicInputs,
+        proof,
+        batchedBlobInputs,
+        attestations,
+      });
       if (!success) {
         throw new Error('Failed to submit epoch proof to L1');
       }
 
-      this.log.info(`Submitted proof for epoch ${epochNumber} (blocks ${fromBlock} to ${toBlock})`, {
+      this.log.info(`Submitted proof for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`, {
         epochNumber,
         uuid: this.uuid,
       });
       this.state = 'completed';
-      this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeBlocks, epochSizeTxs);
+      this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
     } catch (err: any) {
       if (err && err.name === 'HaltExecutionError') {
-        this.log.warn(`Halted execution of epoch ${epochNumber} prover job`, { uuid: this.uuid, epochNumber });
+        this.log.warn(`Halted execution of epoch ${epochNumber} prover job`, {
+          uuid: this.uuid,
+          epochNumber,
+          details: err.message,
+        });
         return;
       }
       this.log.error(`Error running epoch ${epochNumber} prover job`, err, { uuid: this.uuid, epochNumber });
-      this.state = 'failed';
+      if (this.state === 'processing' || this.state === 'awaiting-prover' || this.state === 'publishing-proof') {
+        this.state = 'failed';
+      }
     } finally {
       clearTimeout(this.deadlineTimeoutHandler);
-      await this.cleanUp(this);
+      await this.epochCheckPromise?.stop();
       await this.prover.stop();
       resolve();
     }
   }
 
+  /**
+   * Create a new db fork for tx processing, optionally inserting L1 to L2 messages.
+   * L1 to L2 messages should only be inserted for the first block in a checkpoint,
+   * as subsequent blocks' synced state already includes them.
+   * REFACTOR: The prover already spawns a db fork of its own for each block, so we may be able to do away with just one fork.
+   */
+  private async createFork(blockNumber: BlockNumber, l1ToL2Messages: Fr[] | undefined) {
+    this.log.verbose(`Creating fork at ${blockNumber}`, { blockNumber });
+    // temporary stack to control fork lifetime
+    await using cleanup = new AsyncDisposableStack();
+    const db = cleanup.use(await this.dbProvider.fork(blockNumber));
+
+    if (l1ToL2Messages !== undefined) {
+      this.log.verbose(`Inserting ${l1ToL2Messages.length} L1 to L2 messages in fork`, {
+        blockNumber,
+        l1ToL2Messages: l1ToL2Messages.map(m => m.toString()),
+      });
+      await appendL1ToL2MessagesToTree(db, l1ToL2Messages);
+    }
+
+    // everything run succesfully so we can release this stack and give control of the fork's lifetime to the caller
+    cleanup.move();
+    return db;
+  }
+
+  private async processCheckpoints(
+    parallelism: number,
+    processCheckpoint: (checkpoint: Checkpoint) => Promise<void>,
+  ): Promise<void> {
+    let hasError = false;
+    let firstError: unknown;
+
+    await asyncPool(Math.max(parallelism, 1), this.checkpoints, async checkpoint => {
+      if (hasError || this.abortController.signal.aborted) {
+        return;
+      }
+
+      try {
+        this.checkState();
+        await processCheckpoint(checkpoint);
+      } catch (err) {
+        if (!hasError) {
+          hasError = true;
+          firstError = err;
+          this.failProcessing();
+        }
+      }
+    });
+
+    if (hasError) {
+      throw firstError;
+    }
+
+    if (this.abortController.signal.aborted) {
+      this.checkState();
+    }
+  }
+
   private progressState(state: EpochProvingJobState) {
     this.checkState();
     this.state = state;
   }
 
   private checkState() {
-    if (this.state === 'timed-out' || this.state === 'stopped' || this.state === 'failed') {
+    if (this.state === 'timed-out' || this.state === 'stopped' || this.state === 'failed' || this.state === 'reorg') {
       throw new HaltExecutionError(this.state);
     }
   }
 
-  public async stop(state: EpochProvingJobState = 'stopped') {
+  public async stop(state: EpochProvingJobTerminalState = 'stopped') {
     this.state = state;
-    this.prover.cancel();
-    // TODO(palla/prover): Stop the publisher as well
+    this.interruptProcessing();
     if (this.runPromise) {
       await this.runPromise;
     }
   }
 
+  private failProcessing() {
+    if (!EpochProvingJobTerminalState.includes(this.state)) {
+      this.state = 'failed';
+    }
+    this.interruptProcessing();
+  }
+
+  private interruptProcessing() {
+    this.abortController.abort();
+    this.prover.cancel();
+  }
+
   private scheduleDeadlineStop() {
     const deadline = this.deadline;
     if (deadline) {
@@ -207,33 +443,73 @@ export class EpochProvingJob implements Traceable {
     }
   }
 
-  /* Returns the header for the given block number, or the genesis block for block zero. */
-  private async getBlockHeader(blockNumber: number) {
-    if (blockNumber === 0) {
-      return (await this.dbProvider.fork()).getInitialHeader();
+  /**
+   * Kicks off a running promise that queries the archiver for the set of L2 blocks of the current epoch.
+   * If those change, stops the proving job with a `rerun` state, so the node re-enqueues it.
+   */
+  private async scheduleEpochCheck() {
+    const l2BlockSource = this.l2BlockSource;
+    if (!l2BlockSource) {
+      this.log.warn(`No L2 block source available, skipping epoch check`);
+      return;
     }
-    return this.l2BlockSource.getBlockHeader(blockNumber);
+
+    const intervalMs = Math.ceil((await l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
+    this.epochCheckPromise = new RunningPromise(
+      async () => {
+        const blockHeaders = (
+          await l2BlockSource.getBlocksData({ epoch: this.epochNumber, onlyCheckpointed: true })
+        ).map(d => d.header);
+        const blockHashes = await Promise.all(blockHeaders.map(header => header.hash()));
+        const thisBlocks = this.checkpoints.flatMap(checkpoint => checkpoint.blocks);
+        const thisBlockHashes = await Promise.all(thisBlocks.map(block => block.hash()));
+        if (
+          blockHeaders.length !== thisBlocks.length ||
+          !blockHashes.every((block, i) => block.equals(thisBlockHashes[i]))
+        ) {
+          this.log.warn('Epoch blocks changed underfoot', {
+            uuid: this.uuid,
+            epochNumber: this.epochNumber,
+            oldBlockHashes: thisBlockHashes,
+            newBlockHashes: blockHashes,
+          });
+          void this.stop('reorg');
+        }
+      },
+      this.log,
+      intervalMs,
+    ).start();
+    this.log.verbose(`Scheduled epoch check for epoch ${this.epochNumber} every ${intervalMs}ms`);
   }
 
-  private async getTxs(block: L2Block): Promise<Tx[]> {
-    const txHashes = block.body.txEffects.map(tx => tx.txHash.toBigInt());
-    const txsAndHashes = await Promise.all(this.txs.map(async tx => ({ tx, hash: await tx.getTxHash() })));
-    return txsAndHashes
-      .filter(txAndHash => txHashes.includes(txAndHash.hash.toBigInt()))
-      .map(txAndHash => txAndHash.tx);
+  /* Returns the last block header in the previous checkpoint for all checkpoints in the epoch */
+  private gatherPreviousBlockHeaders() {
+    const lastBlocks = this.checkpoints.map(checkpoint => checkpoint.blocks.at(-1)!);
+    return [this.data.previousBlockHeader, ...lastBlocks.map(block => block.header).slice(0, -1)];
   }
 
-  private getL1ToL2Messages(block: L2Block) {
-    return this.l1ToL2MessageSource.getL1ToL2Messages(BigInt(block.number));
+  private getTxs(block: L2Block): Tx[] {
+    return block.body.txEffects.map(txEffect => this.txs.get(txEffect.txHash.toString())!);
+  }
+
+  private getL1ToL2Messages(checkpoint: Checkpoint) {
+    return this.data.l1ToL2Messages[checkpoint.number];
   }
 
   private async processTxs(publicProcessor: PublicProcessor, txs: Tx[]): Promise<ProcessedTx[]> {
     const { deadline } = this;
-    const [processedTxs, failedTxs] = await publicProcessor.process(txs, { deadline });
+    const [processedTxs, failedTxs] = await publicProcessor.process(txs, {
+      deadline,
+      signal: this.abortController.signal,
+    });
+    this.checkState();
 
     if (failedTxs.length) {
+      const failedTxHashes = await Promise.all(failedTxs.map(({ tx }) => tx.getTxHash()));
       throw new Error(
-        `Txs failed processing: ${failedTxs.map(({ tx, error }) => `${tx.getTxHash()} (${error})`).join(', ')}`,
+        `Txs failed processing: ${failedTxs
+          .map(({ error }, index) => `${failedTxHashes[index]} (${error})`)
+          .join(', ')}`,
       );
     }
 
@@ -246,7 +522,7 @@ export class EpochProvingJob implements Traceable {
 }
 
 class HaltExecutionError extends Error {
-  constructor(state: EpochProvingJobState) {
+  constructor(public readonly state: EpochProvingJobState) {
     super(`Halted execution due to state ${state}`);
     this.name = 'HaltExecutionError';
   }