@aztec/prover-client 0.0.1-commit.fcb71a6 → 0.0.1-commit.fffb133c

package/src/light/lightweight_checkpoint_builder.ts

@@ -1,18 +1,21 @@
 import { SpongeBlob, computeBlobsHashFromBlobs, encodeCheckpointEndMarker, getBlobsPerL1Block } from '@aztec/blob-lib';
 import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
-import type { CheckpointNumber } from '@aztec/foundation/branded-types';
+import { type CheckpointNumber, IndexWithinCheckpoint } from '@aztec/foundation/branded-types';
 import { padArrayEnd } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import { createLogger } from '@aztec/foundation/log';
-import { L2BlockNew } from '@aztec/stdlib/block';
+import { L2Block } from '@aztec/stdlib/block';
 import { Checkpoint } from '@aztec/stdlib/checkpoint';
 import type { MerkleTreeWriteOperations } from '@aztec/stdlib/interfaces/server';
-import { computeCheckpointOutHash, computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
+import {
+  accumulateCheckpointOutHashes,
+  computeCheckpointOutHash,
+  computeInHashFromL1ToL2Messages,
+} from '@aztec/stdlib/messaging';
 import { CheckpointHeader, computeBlockHeadersHash } from '@aztec/stdlib/rollup';
 import { AppendOnlyTreeSnapshot, MerkleTreeId } from '@aztec/stdlib/trees';
 import {
   type CheckpointGlobalVariables,
-  ContentCommitment,
   type GlobalVariables,
   type ProcessedTx,
   StateReference,
@@ -35,13 +38,14 @@ export class LightweightCheckpointBuilder {
 
   private lastArchives: AppendOnlyTreeSnapshot[] = [];
   private spongeBlob: SpongeBlob;
-  private blocks: L2BlockNew[] = [];
+  private blocks: L2Block[] = [];
   private blobFields: Fr[] = [];
 
   constructor(
     public readonly checkpointNumber: CheckpointNumber,
     public readonly constants: CheckpointGlobalVariables,
     public readonly l1ToL2Messages: Fr[],
+    private readonly previousCheckpointOutHashes: Fr[],
     public readonly db: MerkleTreeWriteOperations,
   ) {
     this.spongeBlob = SpongeBlob.init();
@@ -52,6 +56,7 @@ export class LightweightCheckpointBuilder {
     checkpointNumber: CheckpointNumber,
     constants: CheckpointGlobalVariables,
     l1ToL2Messages: Fr[],
+    previousCheckpointOutHashes: Fr[],
     db: MerkleTreeWriteOperations,
   ): Promise<LightweightCheckpointBuilder> {
     // Insert l1-to-l2 messages into the tree.
@@ -60,7 +65,73 @@ export class LightweightCheckpointBuilder {
       padArrayEnd<Fr, number>(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP),
     );
 
-    return new LightweightCheckpointBuilder(checkpointNumber, constants, l1ToL2Messages, db);
+    return new LightweightCheckpointBuilder(
+      checkpointNumber,
+      constants,
+      l1ToL2Messages,
+      previousCheckpointOutHashes,
+      db,
+    );
+  }
+
+  /**
+   * Resumes building a checkpoint from existing blocks. This is used for validator re-execution
+   * where blocks have already been built and their effects are already in the database.
+   * Unlike startNewCheckpoint, this does NOT append l1ToL2Messages to the tree since they
+   * were already added when the blocks were originally built.
+   */
+  static async resumeCheckpoint(
+    checkpointNumber: CheckpointNumber,
+    constants: CheckpointGlobalVariables,
+    l1ToL2Messages: Fr[],
+    previousCheckpointOutHashes: Fr[],
+    db: MerkleTreeWriteOperations,
+    existingBlocks: L2Block[],
+  ): Promise<LightweightCheckpointBuilder> {
+    const builder = new LightweightCheckpointBuilder(
+      checkpointNumber,
+      constants,
+      l1ToL2Messages,
+      previousCheckpointOutHashes,
+      db,
+    );
+
+    builder.logger.debug('Resuming checkpoint from existing blocks', {
+      checkpointNumber,
+      numExistingBlocks: existingBlocks.length,
+      blockNumbers: existingBlocks.map(b => b.header.getBlockNumber()),
+    });
+
+    // Validate block order and consistency
+    for (let i = 1; i < existingBlocks.length; i++) {
+      const prev = existingBlocks[i - 1];
+      const curr = existingBlocks[i];
+      if (curr.number !== prev.number + 1) {
+        throw new Error(`Non-sequential block numbers in resumeCheckpoint: ${prev.number} -> ${curr.number}`);
+      }
+      if (!prev.archive.root.equals(curr.header.lastArchive.root)) {
+        throw new Error(`Archive root mismatch between blocks ${prev.number} and ${curr.number}`);
+      }
+    }
+
+    for (let i = 0; i < existingBlocks.length; i++) {
+      const block = existingBlocks[i];
+      const isFirstBlock = i === 0;
+
+      if (isFirstBlock) {
+        builder.lastArchives.push(block.header.lastArchive);
+      }
+
+      builder.lastArchives.push(block.archive);
+
+      const blockBlobFields = block.toBlobFields();
+      await builder.spongeBlob.absorb(blockBlobFields);
+      builder.blobFields.push(...blockBlobFields);
+
+      builder.blocks.push(block);
+    }
+
+    return builder;
   }
 
   /**
@@ -71,7 +142,7 @@ export class LightweightCheckpointBuilder {
     globalVariables: GlobalVariables,
     txs: ProcessedTx[],
     opts: { insertTxsEffects?: boolean; expectedEndState?: StateReference } = {},
-  ): Promise<L2BlockNew> {
+  ): Promise<L2Block> {
     const isFirstBlock = this.blocks.length === 0;
 
     // Empty blocks are only allowed as the first block in a checkpoint
@@ -120,8 +191,8 @@ export class LightweightCheckpointBuilder {
     const newArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.db);
     this.lastArchives.push(newArchive);
 
-    const indexWithinCheckpoint = this.blocks.length;
-    const block = new L2BlockNew(newArchive, header, body, this.checkpointNumber, indexWithinCheckpoint);
+    const indexWithinCheckpoint = IndexWithinCheckpoint(this.blocks.length);
+    const block = new L2Block(newArchive, header, body, this.checkpointNumber, indexWithinCheckpoint);
     this.blocks.push(block);
 
     await this.spongeBlob.absorb(blockBlobFields);
@@ -157,9 +228,11 @@ export class LightweightCheckpointBuilder {
 
     const inHash = computeInHashFromL1ToL2Messages(this.l1ToL2Messages);
 
-    const outHash = computeCheckpointOutHash(blocks.map(block => block.body.txEffects.map(tx => tx.l2ToL1Msgs)));
-
     const { slotNumber, coinbase, feeRecipient, gasFees } = this.constants;
+    const checkpointOutHash = computeCheckpointOutHash(
+      blocks.map(block => block.body.txEffects.map(tx => tx.l2ToL1Msgs)),
+    );
+    const epochOutHash = accumulateCheckpointOutHashes([...this.previousCheckpointOutHashes, checkpointOutHash]);
 
     // TODO(palla/mbps): Should we source this from the constants instead?
     // timestamp of a checkpoint is the timestamp of the last block in the checkpoint.
@@ -169,7 +242,9 @@ export class LightweightCheckpointBuilder {
 
     const header = CheckpointHeader.from({
       lastArchiveRoot: this.lastArchives[0].root,
-      contentCommitment: new ContentCommitment(blobsHash, inHash, outHash),
+      blobsHash,
+      inHash,
+      epochOutHash,
       blockHeadersHash,
       slotNumber,
       timestamp,
@@ -187,6 +262,7 @@ export class LightweightCheckpointBuilder {
       this.checkpointNumber,
       this.constants,
       [...this.l1ToL2Messages],
+      [...this.previousCheckpointOutHashes],
       this.db,
     );
     clone.lastArchives = [...this.lastArchives];

package/src/mocks/test_context.ts

@@ -44,7 +44,9 @@ import { getEnvironmentConfig, getSimulator, makeCheckpointConstants, makeGlobal
 export class TestContext {
   private headers: Map<number, BlockHeader> = new Map();
   private checkpoints: Checkpoint[] = [];
+  private checkpointOutHashes: Fr[] = [];
   private nextCheckpointIndex = 0;
+  private nextCheckpointNumber = CheckpointNumber(1);
   private nextBlockNumber = 1;
   private epochNumber = 1;
   private feePayerBalance: Fr;
@@ -150,6 +152,7 @@ export class TestContext {
 
   public startNewEpoch() {
     this.checkpoints = [];
+    this.checkpointOutHashes = [];
     this.nextCheckpointIndex = 0;
     this.epochNumber++;
   }
@@ -187,7 +190,8 @@ export class TestContext {
     }
 
     const checkpointIndex = this.nextCheckpointIndex++;
-    const checkpointNumber = CheckpointNumber(checkpointIndex + 1);
+    const checkpointNumber = this.nextCheckpointNumber;
+    this.nextCheckpointNumber++;
     const slotNumber = checkpointNumber * 15; // times an arbitrary number to make it different to the checkpoint number
 
     const constants = makeCheckpointConstants(slotNumber, constantOpts);
@@ -204,6 +208,8 @@ export class TestContext {
 
     const startBlockNumber = this.nextBlockNumber;
     const previousBlockHeader = this.getBlockHeader(BlockNumber(startBlockNumber - 1));
+    // All blocks in the same slot/checkpoint share the same timestamp.
+    const timestamp = BigInt(slotNumber * 26);
 
     // Build global variables.
     const blockGlobalVariables = times(numBlocks, i =>
@@ -211,6 +217,7 @@ export class TestContext {
         coinbase: constants.coinbase,
         feeRecipient: constants.feeRecipient,
         gasFees: constants.gasFees,
+        timestamp,
       }),
     );
     this.nextBlockNumber += numBlocks;
@@ -240,10 +247,12 @@ export class TestContext {
     });
 
     const cleanFork = await this.worldState.fork();
+    const previousCheckpointOutHashes = this.checkpointOutHashes;
     const builder = await LightweightCheckpointBuilder.startNewCheckpoint(
       checkpointNumber,
       constants,
       l1ToL2Messages,
+      previousCheckpointOutHashes,
       cleanFork,
     );
 
@@ -269,6 +278,7 @@ export class TestContext {
 
     const checkpoint = await builder.completeCheckpoint();
     this.checkpoints.push(checkpoint);
+    this.checkpointOutHashes.push(checkpoint.getCheckpointOutHash());
 
     return {
       constants,

package/src/orchestrator/block-building-helpers.ts

@@ -97,7 +97,7 @@ export const insertSideEffectsAndBuildBaseRollupHints = runInSpan(
 
     const { nullifierInsertionResult, publicDataInsertionResult } = await insertSideEffects(tx, db);
 
-    const blockHash = await tx.data.constants.anchorBlockHeader.hash();
+    const blockHash = (await tx.data.constants.anchorBlockHeader.hash()).toField();
     const anchorBlockArchiveSiblingPath = (
       await getMembershipWitnessFor(blockHash, MerkleTreeId.ARCHIVE, ARCHIVE_HEIGHT, db)
     ).siblingPath;
@@ -282,7 +282,7 @@ export const buildHeaderFromCircuitOutputs = runInSpan(
       chainId: constants.chainId,
       version: constants.version,
       blockNumber: BlockNumber(blockRootRollupOutput.previousArchive.nextAvailableLeafIndex),
-      timestamp: blockRootRollupOutput.endTimestamp,
+      timestamp: blockRootRollupOutput.timestamp,
       slotNumber: constants.slotNumber,
       coinbase: constants.coinbase,
       feeRecipient: constants.feeRecipient,

package/src/orchestrator/checkpoint-proving-state.ts

@@ -11,6 +11,7 @@ import {
   type L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH,
   type NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH,
   NUM_MSGS_PER_BASE_PARITY,
+  OUT_HASH_TREE_HEIGHT,
 } from '@aztec/constants';
 import { BlockNumber } from '@aztec/foundation/branded-types';
 import { padArrayEnd } from '@aztec/foundation/collection';
@@ -19,6 +20,7 @@ import { Fr } from '@aztec/foundation/curves/bn254';
 import type { Tuple } from '@aztec/foundation/serialize';
 import { type TreeNodeLocation, UnbalancedTreeStore } from '@aztec/foundation/trees';
 import type { PublicInputsAndRecursiveProof } from '@aztec/stdlib/interfaces/server';
+import { computeCheckpointOutHash } from '@aztec/stdlib/messaging';
 import { ParityBasePrivateInputs } from '@aztec/stdlib/parity';
 import {
   BlockMergeRollupPrivateInputs,
@@ -38,6 +40,11 @@ import { accumulateBlobs, buildBlobHints, toProofData } from './block-building-h
 import { BlockProvingState, type ProofState } from './block-proving-state.js';
 import type { EpochProvingState } from './epoch-proving-state.js';
 
+type OutHashHint = {
+  treeSnapshot: AppendOnlyTreeSnapshot;
+  siblingPath: Tuple<Fr, typeof OUT_HASH_TREE_HEIGHT>;
+};
+
 export class CheckpointProvingState {
   private blockProofs: UnbalancedTreeStore<
     ProofState<BlockRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>
@@ -46,6 +53,11 @@ export class CheckpointProvingState {
     | ProofState<CheckpointRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>
     | undefined;
   private blocks: (BlockProvingState | undefined)[] = [];
+  private previousOutHashHint: OutHashHint | undefined;
+  private outHash: Fr | undefined;
+  // The snapshot and sibling path after the checkpoint's out hash is inserted.
+  // Stored here to be retrieved for the next checkpoint when it's added.
+  private newOutHashHint: OutHashHint | undefined;
   private startBlobAccumulator: BatchedBlobAccumulator | undefined;
   private endBlobAccumulator: BatchedBlobAccumulator | undefined;
   private blobFields: Fr[] | undefined;
@@ -195,6 +207,35 @@ export class CheckpointProvingState {
     return new ParityBasePrivateInputs(messages, this.constants.vkTreeRoot);
   }
 
+  public setOutHashHint(hint: OutHashHint) {
+    this.previousOutHashHint = hint;
+  }
+
+  public getOutHashHint() {
+    return this.previousOutHashHint;
+  }
+
+  public accumulateBlockOutHashes() {
+    if (this.isAcceptingBlocks() || this.blocks.some(b => !b?.hasEndState())) {
+      return;
+    }
+
+    if (!this.outHash) {
+      const messagesPerBlock = this.blocks.map(b => b!.getTxEffects().map(tx => tx.l2ToL1Msgs));
+      this.outHash = computeCheckpointOutHash(messagesPerBlock);
+    }
+
+    return this.outHash;
+  }
+
+  public setOutHashHintForNextCheckpoint(hint: OutHashHint) {
+    this.newOutHashHint = hint;
+  }
+
+  public getOutHashHintForNextCheckpoint() {
+    return this.newOutHashHint;
+  }
+
   public async accumulateBlobs(startBlobAccumulator: BatchedBlobAccumulator) {
     if (this.isAcceptingBlocks() || this.blocks.some(b => !b?.hasEndState())) {
       return;
@@ -236,6 +277,9 @@ export class CheckpointProvingState {
     if (proofs.length !== nonEmptyProofs.length) {
       throw new Error('At least one child is not ready for the checkpoint root rollup.');
     }
+    if (!this.previousOutHashHint) {
+      throw new Error('Out hash hint is not set.');
+    }
     if (!this.startBlobAccumulator) {
       throw new Error('Start blob accumulator is not set.');
     }
@@ -248,6 +292,8 @@ export class CheckpointProvingState {
     const hints = CheckpointRootRollupHints.from({
       previousBlockHeader: this.headerOfLastBlockInPreviousCheckpoint,
       previousArchiveSiblingPath: this.lastArchiveSiblingPath,
+      previousOutHash: this.previousOutHashHint.treeSnapshot,
+      newOutHashSiblingPath: this.previousOutHashHint.siblingPath,
       startBlobAccumulator: this.startBlobAccumulator.toBlobAccumulator(),
       finalBlobChallenges: this.finalBlobBatchingChallenges,
       blobFields: padArrayEnd(blobFields, Fr.ZERO, FIELDS_PER_BLOB * BLOBS_PER_CHECKPOINT),
@@ -273,7 +319,7 @@ export class CheckpointProvingState {
 
   public isReadyForCheckpointRoot() {
     const allChildProofsReady = this.#getChildProofsForRoot().every(p => !!p);
-    return allChildProofsReady && !!this.startBlobAccumulator;
+    return allChildProofsReady && !!this.previousOutHashHint && !!this.startBlobAccumulator;
   }
 
   public verifyState() {

package/src/orchestrator/epoch-proving-state.ts

@@ -1,14 +1,20 @@
 import { BatchedBlob, BatchedBlobAccumulator, type FinalBlobBatchingChallenges } from '@aztec/blob-lib';
-import type {
-  ARCHIVE_HEIGHT,
-  L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH,
-  NESTED_RECURSIVE_PROOF_LENGTH,
-  NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH,
+import {
+  type ARCHIVE_HEIGHT,
+  type L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH,
+  type NESTED_RECURSIVE_PROOF_LENGTH,
+  type NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH,
+  OUT_HASH_TREE_HEIGHT,
 } from '@aztec/constants';
 import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
-import type { Fr } from '@aztec/foundation/curves/bn254';
+import { Fr } from '@aztec/foundation/curves/bn254';
 import type { Tuple } from '@aztec/foundation/serialize';
-import { type TreeNodeLocation, UnbalancedTreeStore } from '@aztec/foundation/trees';
+import {
+  MerkleTreeCalculator,
+  type TreeNodeLocation,
+  UnbalancedTreeStore,
+  shaMerkleHash,
+} from '@aztec/foundation/trees';
 import type { PublicInputsAndRecursiveProof } from '@aztec/stdlib/interfaces/server';
 import type { Proof } from '@aztec/stdlib/proofs';
 import {
@@ -20,7 +26,7 @@ import {
   RootRollupPrivateInputs,
   type RootRollupPublicInputs,
 } from '@aztec/stdlib/rollup';
-import type { AppendOnlyTreeSnapshot, MerkleTreeId } from '@aztec/stdlib/trees';
+import { AppendOnlyTreeSnapshot, type MerkleTreeId } from '@aztec/stdlib/trees';
 import type { BlockHeader } from '@aztec/stdlib/tx';
 
 import { toProofData } from './block-building-helpers.js';
@@ -212,6 +218,48 @@ export class EpochProvingState {
     this.checkpointPaddingProof = { provingOutput };
   }
 
+  public async accumulateCheckpointOutHashes() {
+    const treeCalculator = await MerkleTreeCalculator.create(OUT_HASH_TREE_HEIGHT, undefined, (left, right) =>
+      Promise.resolve(shaMerkleHash(left, right)),
+    );
+
+    const computeOutHashHint = async (leaves: Fr[]) => {
+      const tree = await treeCalculator.computeTree(leaves.map(l => l.toBuffer()));
+      const nextAvailableLeafIndex = leaves.length;
+      return {
+        treeSnapshot: new AppendOnlyTreeSnapshot(Fr.fromBuffer(tree.root), nextAvailableLeafIndex),
+        siblingPath: tree.getSiblingPath(nextAvailableLeafIndex).map(Fr.fromBuffer) as Tuple<
+          Fr,
+          typeof OUT_HASH_TREE_HEIGHT
+        >,
+      };
+    };
+
+    let hint = this.checkpoints[0]?.getOutHashHint();
+    const outHashes = [];
+    for (let i = 0; i < this.totalNumCheckpoints; i++) {
+      const checkpoint = this.checkpoints[i];
+      if (!checkpoint) {
+        break;
+      }
+
+      // If hints are not set yet, it must be the first checkpoint. Compute the hints with an empty tree.
+      hint ??= await computeOutHashHint([]);
+      checkpoint.setOutHashHint(hint);
+
+      // Get the out hash for this checkpoint.
+      const outHash = checkpoint.accumulateBlockOutHashes();
+      if (!outHash) {
+        break;
+      }
+      outHashes.push(outHash);
+
+      // Get or create hints for the next checkpoint.
+      hint = checkpoint.getOutHashHintForNextCheckpoint() ?? (await computeOutHashHint(outHashes));
+      checkpoint.setOutHashHintForNextCheckpoint(hint);
+    }
+  }
+
   public async setBlobAccumulators() {
     let previousAccumulator = this.startBlobAccumulator;
     // Accumulate blobs as far as we can for this epoch.

package/src/orchestrator/orchestrator.ts

@@ -73,6 +73,11 @@ import { TxProvingState } from './tx-proving-state.js';
 
 const logger = createLogger('prover-client:orchestrator');
 
+type WorldStateFork = {
+  fork: MerkleTreeWriteOperations;
+  cleanupPromise: Promise<void> | undefined;
+};
+
 /**
  * Implements an event driven proving scheduler to build the recursive proof tree. The idea being:
  * 1. Transactions are provided to the scheduler post simulation.
@@ -94,12 +99,13 @@ export class ProvingOrchestrator implements EpochProver {
   private provingPromise: Promise<ProvingResult> | undefined = undefined;
   private metrics: ProvingOrchestratorMetrics;
   // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
-  private dbs: Map<BlockNumber, MerkleTreeWriteOperations> = new Map();
+  private dbs: Map<BlockNumber, WorldStateFork> = new Map();
 
   constructor(
     private dbProvider: ReadonlyWorldStateAccess & ForkMerkleTreeOperations,
     private prover: ServerCircuitProver,
     private readonly proverId: EthAddress,
+    private readonly cancelJobsOnStop: boolean = false,
     telemetryClient: TelemetryClient = getTelemetryClient(),
   ) {
     this.metrics = new ProvingOrchestratorMetrics(telemetryClient, 'ProvingOrchestrator');
@@ -113,6 +119,10 @@ export class ProvingOrchestrator implements EpochProver {
     return this.proverId;
   }
 
+  public getNumActiveForks() {
+    return this.dbs.size;
+  }
+
   public stop(): Promise<void> {
     this.cancel();
     return Promise.resolve();
@@ -143,6 +153,14 @@ export class ProvingOrchestrator implements EpochProver {
     this.provingPromise = promise;
   }
 
+  /**
+   * Starts a new checkpoint.
+   * @param checkpointIndex - The index of the checkpoint in the epoch.
+   * @param constants - The constants for this checkpoint.
+   * @param l1ToL2Messages - The set of L1 to L2 messages to be inserted at the beginning of this checkpoint.
+   * @param totalNumBlocks - The total number of blocks expected in the checkpoint (must be at least one).
+   * @param headerOfLastBlockInPreviousCheckpoint - The header of the last block in the previous checkpoint.
+   */
   public async startNewCheckpoint(
     checkpointIndex: number,
     constants: CheckpointConstantData,
@@ -163,7 +181,7 @@ export class ProvingOrchestrator implements EpochProver {
     const db = await this.dbProvider.fork(lastBlockNumber);
 
     const firstBlockNumber = BlockNumber(lastBlockNumber + 1);
-    this.dbs.set(firstBlockNumber, db);
+    this.dbs.set(firstBlockNumber, { fork: db, cleanupPromise: undefined });
 
     // Get archive sibling path before any block in this checkpoint lands.
     const lastArchiveSiblingPath = await getLastSiblingPath(MerkleTreeId.ARCHIVE, db);
@@ -221,9 +239,9 @@ export class ProvingOrchestrator implements EpochProver {
     if (!this.dbs.has(blockNumber)) {
       // Fork world state at the end of the immediately previous block
       const db = await this.dbProvider.fork(BlockNumber(blockNumber - 1));
-      this.dbs.set(blockNumber, db);
+      this.dbs.set(blockNumber, { fork: db, cleanupPromise: undefined });
     }
-    const db = this.dbs.get(blockNumber)!;
+    const db = this.dbs.get(blockNumber)!.fork;
 
     // Get archive snapshot and sibling path before any txs in this block lands.
     const lastArchiveTreeSnapshot = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
@@ -255,7 +273,8 @@ export class ProvingOrchestrator implements EpochProver {
       await endSpongeBlob.absorb(blockEndBlobFields);
       blockProvingState.setEndSpongeBlob(endSpongeBlob);
 
-      // And also try to accumulate the blobs as far as we can:
+      // Try to accumulate the out hashes and blobs as far as we can:
+      await this.provingState.accumulateCheckpointOutHashes();
       await this.provingState.setBlobAccumulators();
     }
   }
@@ -297,7 +316,7 @@ export class ProvingOrchestrator implements EpochProver {
 
     logger.info(`Adding ${txs.length} transactions to block ${blockNumber}`);
 
-    const db = this.dbs.get(blockNumber)!;
+    const db = this.dbs.get(blockNumber)!.fork;
     const lastArchive = provingState.lastArchiveTreeSnapshot;
     const newL1ToL2MessageTreeSnapshot = provingState.newL1ToL2MessageTreeSnapshot;
     const spongeBlobState = provingState.getStartSpongeBlob().clone();
@@ -352,7 +371,8 @@ export class ProvingOrchestrator implements EpochProver {
 
     provingState.setEndSpongeBlob(spongeBlobState);
 
-    // Txs have been added to the block. Now try to accumulate the blobs as far as we can:
+    // Txs have been added to the block. Now try to accumulate the out hashes and blobs as far as we can:
+    await this.provingState.accumulateCheckpointOutHashes();
     await this.provingState.setBlobAccumulators();
   }
 
@@ -425,7 +445,7 @@ export class ProvingOrchestrator implements EpochProver {
     }
 
     // Get db for this block
-    const db = this.dbs.get(provingState.blockNumber)!;
+    const db = this.dbs.get(provingState.blockNumber)!.fork;
 
     // Update the archive tree, so we're ready to start processing the next block:
     logger.verbose(
@@ -461,7 +481,7 @@ export class ProvingOrchestrator implements EpochProver {
 
     // Get db for this block
     const blockNumber = provingState.blockNumber;
-    const db = this.dbs.get(blockNumber)!;
+    const db = this.dbs.get(blockNumber)!.fork;
 
     const newArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
     const syncedArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.dbProvider.getSnapshot(blockNumber));
@@ -486,20 +506,19 @@ export class ProvingOrchestrator implements EpochProver {
     // is aborted and never reaches this point, it will leak the fork. We need to add a global cleanup,
     // but have to make sure it only runs once all operations are completed, otherwise some function here
     // will attempt to access the fork after it was closed.
-    logger.debug(`Cleaning up world state fork for ${blockNumber}`);
-    void this.dbs
-      .get(blockNumber)
-      ?.close()
-      .then(() => this.dbs.delete(blockNumber))
-      .catch(err => logger.error(`Error closing db for block ${blockNumber}`, err));
+    void this.cleanupDBFork(blockNumber);
   }
 
   /**
-   * Cancel any further proving
+   * Cancel any further proving.
+   * If cancelJobsOnStop is true, aborts all pending jobs with the broker (which marks them as 'Aborted').
+   * If cancelJobsOnStop is false (default), jobs remain in the broker queue and can be reused on restart/reorg.
    */
   public cancel() {
-    for (const controller of this.pendingProvingJobs) {
-      controller.abort();
+    if (this.cancelJobsOnStop) {
+      for (const controller of this.pendingProvingJobs) {
+        controller.abort();
+      }
     }
 
     this.provingState?.cancel();
@@ -534,6 +553,24 @@ export class ProvingOrchestrator implements EpochProver {
     return epochProofResult;
   }
 
+  private async cleanupDBFork(blockNumber: BlockNumber): Promise<void> {
+    logger.debug(`Cleaning up world state fork for ${blockNumber}`);
+    const fork = this.dbs.get(blockNumber);
+    if (!fork) {
+      return;
+    }
+
+    try {
+      if (!fork.cleanupPromise) {
+        fork.cleanupPromise = fork.fork.close();
+      }
+      await fork.cleanupPromise;
+      this.dbs.delete(blockNumber);
+    } catch (err) {
+      logger.error(`Error closing db for block ${blockNumber}`, err);
+    }
+  }
+
   /**
    * Enqueue a job to be scheduled
    * @param provingState - The proving state object being operated on
@@ -851,19 +888,22 @@ export class ProvingOrchestrator implements EpochProver {
         },
       ),
       async result => {
-        // If the proofs were slower than the block header building, then we need to try validating the block header hashes here.
-        await this.verifyBuiltBlockAgainstSyncedState(provingState);
-
         logger.debug(`Completed ${rollupType} proof for block ${provingState.blockNumber}`);
 
         const leafLocation = provingState.setBlockRootRollupProof(result);
         const checkpointProvingState = provingState.parentCheckpoint;
 
+        // If the proofs were slower than the block header building, then we need to try validating the block header hashes here.
+        await this.verifyBuiltBlockAgainstSyncedState(provingState);
+
         if (checkpointProvingState.totalNumBlocks === 1) {
           this.checkAndEnqueueCheckpointRootRollup(checkpointProvingState);
         } else {
           this.checkAndEnqueueNextBlockMergeRollup(checkpointProvingState, leafLocation);
         }
+
+        // We are finished with the block at this point, ensure the fork is cleaned up
+        void this.cleanupDBFork(provingState.blockNumber);
       },
     );
   }
@@ -1219,9 +1259,9 @@ export class ProvingOrchestrator implements EpochProver {
       },
     );
 
-    this.deferredProving(provingState, doAvmProving, proofAndVk => {
+    this.deferredProving(provingState, doAvmProving, proof => {
       logger.debug(`Proven VM for tx index: ${txIndex}`);
-      txProvingState.setAvmProof(proofAndVk);
+      txProvingState.setAvmProof(proof);
       this.checkAndEnqueueBaseRollup(provingState, txIndex);
     });
   }

package/src/orchestrator/orchestrator_metrics.ts

@@ -1,4 +1,4 @@
-import { type Histogram, Metrics, type TelemetryClient, type Tracer, ValueType } from '@aztec/telemetry-client';
+import { type Histogram, Metrics, type TelemetryClient, type Tracer } from '@aztec/telemetry-client';
 
 export class ProvingOrchestratorMetrics {
   public readonly tracer: Tracer;
@@ -9,11 +9,7 @@ export class ProvingOrchestratorMetrics {
     this.tracer = client.getTracer(name);
     const meter = client.getMeter(name);
 
-    this.baseRollupInputsDuration = meter.createHistogram(Metrics.PROVING_ORCHESTRATOR_BASE_ROLLUP_INPUTS_DURATION, {
-      unit: 'ms',
-      description: 'Duration to build base rollup inputs',
-      valueType: ValueType.INT,
-    });
+    this.baseRollupInputsDuration = meter.createHistogram(Metrics.PROVING_ORCHESTRATOR_BASE_ROLLUP_INPUTS_DURATION);
   }
 
   recordBaseRollupInputs(durationMs: number) {