@aztec/prover-client 3.0.0-nightly.20250917 → 3.0.0-nightly.20250919

package/dest/orchestrator/orchestrator.js

@@ -6,7 +6,7 @@ function _ts_decorate(decorators, target, key, desc) {
 }
 import { BlobAccumulatorPublicInputs } from '@aztec/blob-lib';
 import { L1_TO_L2_MSG_SUBTREE_HEIGHT, L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, NUM_BASE_PARITY_PER_ROOT_PARITY } from '@aztec/constants';
-import { padArrayEnd, times } from '@aztec/foundation/collection';
+import { padArrayEnd } from '@aztec/foundation/collection';
 import { AbortError } from '@aztec/foundation/error';
 import { Fr } from '@aztec/foundation/fields';
 import { createLogger } from '@aztec/foundation/log';
@@ -14,16 +14,13 @@ import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { assertLength } from '@aztec/foundation/serialize';
 import { pushTestData } from '@aztec/foundation/testing';
 import { elapsed } from '@aztec/foundation/timer';
-import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
 import { readAvmMinimalPublicTxInputsFromFile } from '@aztec/simulator/public/fixtures';
-import { L2Block } from '@aztec/stdlib/block';
-import { BaseParityInputs } from '@aztec/stdlib/parity';
-import { EmptyBlockRootRollupInputs, PrivateBaseRollupInputs, SingleTxBlockRootRollupInputs } from '@aztec/stdlib/rollup';
+import { createBlockEndMarker } from '@aztec/stdlib/block';
+import { BlockRootEmptyTxFirstRollupPrivateInputs, BlockRootFirstRollupPrivateInputs, BlockRootSingleTxFirstRollupPrivateInputs, BlockRootSingleTxRollupPrivateInputs, CheckpointRootSingleBlockRollupPrivateInputs, PrivateTxBaseRollupPrivateInputs } from '@aztec/stdlib/rollup';
 import { MerkleTreeId } from '@aztec/stdlib/trees';
-import { toNumBlobFields } from '@aztec/stdlib/tx';
 import { Attributes, getTelemetryClient, trackSpan, wrapCallbackInSpan } from '@aztec/telemetry-client';
 import { inspect } from 'util';
-import { buildHeaderAndBodyFromTxs, getLastSiblingPath, getPublicTubePrivateInputsFromTx, getRootTreeSiblingPath, getSubtreeSiblingPath, getTreeSnapshot, insertSideEffectsAndBuildBaseRollupHints, validatePartialState, validateTx } from './block-building-helpers.js';
+import { buildBlockHeaderFromTxs, buildHeaderFromCircuitOutputs, getLastSiblingPath, getPublicTubePrivateInputsFromTx, getRootTreeSiblingPath, getSubtreeSiblingPath, getTreeSnapshot, insertSideEffectsAndBuildBaseRollupHints, validatePartialState, validateTx } from './block-building-helpers.js';
 import { EpochProvingState } from './epoch-proving-state.js';
 import { ProvingOrchestratorMetrics } from './orchestrator_metrics.js';
 import { TxProvingState } from './tx-proving-state.js';
@@ -68,51 +65,92 @@ const logger = createLogger('prover-client:orchestrator');
         this.cancel();
         return Promise.resolve();
     }
-    startNewEpoch(epochNumber, firstBlockNumber, totalNumBlocks, finalBlobBatchingChallenges) {
+    startNewEpoch(epochNumber, firstCheckpointNumber, totalNumCheckpoints, finalBlobBatchingChallenges) {
+        if (this.provingState?.verifyState()) {
+            throw new Error(`Cannot start epoch ${epochNumber} when epoch ${this.provingState.epochNumber} is still being processed.`);
+        }
         const { promise: _promise, resolve, reject } = promiseWithResolvers();
         const promise = _promise.catch((reason)=>({
                 status: 'failure',
                 reason
             }));
-        if (totalNumBlocks <= 0 || !Number.isInteger(totalNumBlocks)) {
-            throw new Error(`Invalid number of blocks for epoch (got ${totalNumBlocks})`);
-        }
-        logger.info(`Starting epoch ${epochNumber} with ${totalNumBlocks} blocks`);
-        this.provingState = new EpochProvingState(epochNumber, firstBlockNumber, totalNumBlocks, finalBlobBatchingChallenges, resolve, reject);
+        logger.info(`Starting epoch ${epochNumber} with ${totalNumCheckpoints} checkpoints.`);
+        this.provingState = new EpochProvingState(epochNumber, firstCheckpointNumber, totalNumCheckpoints, finalBlobBatchingChallenges, (provingState)=>this.checkAndEnqueueCheckpointRootRollup(provingState), resolve, reject);
         this.provingPromise = promise;
     }
+    async startNewCheckpoint(constants, l1ToL2Messages, totalNumBlocks, totalNumBlobFields, headerOfLastBlockInPreviousCheckpoint) {
+        if (!this.provingState) {
+            throw new Error('Empty epoch proving state. Call startNewEpoch before starting a checkpoint.');
+        }
+        if (!this.provingState.isAcceptingCheckpoints()) {
+            throw new Error(`Epoch not accepting further checkpoints.`);
+        }
+        // Fork world state at the end of the immediately previous block.
+        const lastBlockNumber = headerOfLastBlockInPreviousCheckpoint.globalVariables.blockNumber;
+        const db = await this.dbProvider.fork(lastBlockNumber);
+        const firstBlockNumber = lastBlockNumber + 1;
+        this.dbs.set(firstBlockNumber, db);
+        // Get archive sibling path before any block in this checkpoint lands.
+        const lastArchiveSiblingPath = await getLastSiblingPath(MerkleTreeId.ARCHIVE, db);
+        // Insert all the l1 to l2 messages into the db. And get the states before and after the insertion.
+        const { lastL1ToL2MessageTreeSnapshot, lastL1ToL2MessageSubtreeSiblingPath, newL1ToL2MessageTreeSnapshot, newL1ToL2MessageSubtreeSiblingPath } = await this.updateL1ToL2MessageTree(l1ToL2Messages, db);
+        this.provingState.startNewCheckpoint(constants, totalNumBlocks, totalNumBlobFields, headerOfLastBlockInPreviousCheckpoint, lastArchiveSiblingPath, l1ToL2Messages, lastL1ToL2MessageTreeSnapshot, lastL1ToL2MessageSubtreeSiblingPath, newL1ToL2MessageTreeSnapshot, newL1ToL2MessageSubtreeSiblingPath);
+    }
     /**
    * Starts off a new block
-   * @param globalVariables - The global variables for the block
-   * @param l1ToL2Messages - The l1 to l2 messages for the block
-   * @returns A proving ticket, containing a promise notifying of proving completion
-   */ async startNewBlock(globalVariables, l1ToL2Messages, previousBlockHeader) {
+   * @param blockNumber - The block number
+   * @param timestamp - The timestamp of the block. This is only required for constructing the private inputs for the
+   * block that doesn't have any txs.
+   * @param totalNumTxs - The total number of txs in the block
+   */ async startNewBlock(blockNumber, timestamp, totalNumTxs) {
         if (!this.provingState) {
-            throw new Error(`Invalid proving state, call startNewEpoch before starting a block`);
-        }
-        if (!this.provingState?.isAcceptingBlocks()) {
-            throw new Error(`Epoch not accepting further blocks`);
-        }
-        logger.info(`Starting block ${globalVariables.blockNumber} for slot ${globalVariables.slotNumber.toNumber()}`);
-        // Fork world state at the end of the immediately previous block
-        const db = await this.dbProvider.fork(globalVariables.blockNumber - 1);
-        this.dbs.set(globalVariables.blockNumber, db);
-        // we start the block by enqueueing all of the base parity circuits
-        const { l1ToL2MessageTreeSnapshot, l1ToL2MessageSubtreeSiblingPath, l1ToL2MessageTreeSnapshotAfterInsertion, baseParityInputs } = await this.prepareBaseParityInputs(l1ToL2Messages, db);
-        // Get archive snapshot before this block lands
-        const lastArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
-        const lastArchiveSiblingPath = await getLastSiblingPath(MerkleTreeId.ARCHIVE, db);
-        const newArchiveSiblingPath = await getRootTreeSiblingPath(MerkleTreeId.ARCHIVE, db);
-        const blockProvingState = this.provingState.startNewBlock(globalVariables, l1ToL2Messages, l1ToL2MessageTreeSnapshot, l1ToL2MessageSubtreeSiblingPath, l1ToL2MessageTreeSnapshotAfterInsertion, lastArchive, lastArchiveSiblingPath, newArchiveSiblingPath, previousBlockHeader, this.proverId);
-        // Enqueue base parity circuits for the block
-        for(let i = 0; i < baseParityInputs.length; i++){
-            this.enqueueBaseParityCircuit(blockProvingState, baseParityInputs[i], i);
+            throw new Error('Empty epoch proving state. Call startNewEpoch before starting a block.');
+        }
+        const checkpointProvingState = this.provingState.getCheckpointProvingStateByBlockNumber(blockNumber);
+        if (!checkpointProvingState) {
+            throw new Error(`Checkpoint not started. Call startNewCheckpoint first.`);
+        }
+        if (!checkpointProvingState.isAcceptingBlocks()) {
+            throw new Error(`Checkpoint not accepting further blocks.`);
+        }
+        const constants = checkpointProvingState.constants;
+        logger.info(`Starting block ${blockNumber} for slot ${constants.slotNumber.toNumber()}.`);
+        // Fork the db only when it's not already set. The db for the first block is set in `startNewCheckpoint`.
+        if (!this.dbs.has(blockNumber)) {
+            // Fork world state at the end of the immediately previous block
+            const db = await this.dbProvider.fork(blockNumber - 1);
+            this.dbs.set(blockNumber, db);
+        }
+        const db = this.dbs.get(blockNumber);
+        // Get archive snapshot and sibling path before any txs in this block lands.
+        const lastArchiveTreeSnapshot = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
+        const lastArchiveSiblingPath = await getRootTreeSiblingPath(MerkleTreeId.ARCHIVE, db);
+        const blockProvingState = checkpointProvingState.startNewBlock(blockNumber, timestamp, totalNumTxs, lastArchiveTreeSnapshot, lastArchiveSiblingPath);
+        // Enqueue base parity circuits for the first block in the checkpoint.
+        if (blockProvingState.index === 0) {
+            for(let i = 0; i < NUM_BASE_PARITY_PER_ROOT_PARITY; i++){
+                this.enqueueBaseParityCircuit(checkpointProvingState, blockProvingState, i);
+            }
+        }
+        // Because `addTxs` won't be called for a block without txs, and that's where the sponge blob state is computed.
+        // We need to set its end sponge blob here, which will become the start sponge blob for the next block.
+        if (totalNumTxs === 0) {
+            const endSpongeBlob = blockProvingState.getStartSpongeBlob().clone();
+            await endSpongeBlob.absorb([
+                createBlockEndMarker(0)
+            ]);
+            blockProvingState.setEndSpongeBlob(endSpongeBlob);
+            // And also try to accumulate the blobs as far as we can:
+            await this.provingState.setBlobAccumulators();
         }
     }
     /**
    * The interface to add simulated transactions to the scheduler. This can only be called once per block.
    * @param txs - The transactions to be proven
    */ async addTxs(txs) {
+        if (!this.provingState) {
+            throw new Error(`Empty epoch proving state. Call startNewEpoch before adding txs.`);
+        }
         if (!txs.length) {
             // To avoid an ugly throw below. If we require an empty block, we can just call setBlockCompleted
             // on a block with no txs. We cannot do that here because we cannot find the blockNumber without any txs.
@@ -120,16 +158,21 @@ const logger = createLogger('prover-client:orchestrator');
             return;
         }
         const blockNumber = txs[0].globalVariables.blockNumber;
-        const provingState = this.provingState?.getBlockProvingStateByBlockNumber(blockNumber);
+        const provingState = this.provingState.getBlockProvingStateByBlockNumber(blockNumber);
         if (!provingState) {
-            throw new Error(`Block proving state for ${blockNumber} not found`);
+            throw new Error(`Proving state for block ${blockNumber} not found. Call startNewBlock first.`);
+        }
+        if (provingState.totalNumTxs !== txs.length) {
+            throw new Error(`Block ${blockNumber} should be filled with ${provingState.totalNumTxs} txs. Received ${txs.length} txs.`);
         }
-        if (provingState.totalNumTxs) {
+        if (!provingState.isAcceptingTxs()) {
             throw new Error(`Block ${blockNumber} has been initialized with transactions.`);
         }
-        const numBlobFields = toNumBlobFields(txs);
-        provingState.startNewBlock(txs.length, numBlobFields);
-        logger.info(`Adding ${txs.length} transactions with ${numBlobFields} blob fields to block ${provingState.blockNumber}`);
+        logger.info(`Adding ${txs.length} transactions to block ${blockNumber}`);
+        const db = this.dbs.get(blockNumber);
+        const lastArchive = provingState.lastArchiveTreeSnapshot;
+        const newL1ToL2MessageTreeSnapshot = provingState.newL1ToL2MessageTreeSnapshot;
+        const spongeBlobState = provingState.getStartSpongeBlob().clone();
         for (const tx of txs){
             try {
                 if (!provingState.verifyState()) {
@@ -137,7 +180,12 @@ const logger = createLogger('prover-client:orchestrator');
                 }
                 validateTx(tx);
                 logger.info(`Received transaction: ${tx.hash}`);
-                const [hints, treeSnapshots] = await this.prepareTransaction(tx, provingState);
+                const startSpongeBlob = spongeBlobState.clone();
+                const [hints, treeSnapshots] = await this.prepareBaseRollupInputs(tx, lastArchive, newL1ToL2MessageTreeSnapshot, startSpongeBlob, db);
+                if (!provingState.verifyState()) {
+                    throw new Error(`Unable to add transaction, preparing base inputs failed`);
+                }
+                await spongeBlobState.absorb(tx.txEffect.toBlobFields());
                 const txProvingState = new TxProvingState(tx, hints, treeSnapshots);
                 const txIndex = provingState.addNewTx(txProvingState);
                 if (txProvingState.requireAvmProof) {
@@ -154,13 +202,19 @@ const logger = createLogger('prover-client:orchestrator');
                 });
             }
         }
+        await spongeBlobState.absorb([
+            createBlockEndMarker(txs.length)
+        ]);
+        provingState.setEndSpongeBlob(spongeBlobState);
+        // Txs have been added to the block. Now try to accumulate the blobs as far as we can:
+        await this.provingState.setBlobAccumulators();
     }
     /**
    * Kickstarts tube circuits for the specified txs. These will be used during epoch proving.
    * Note that if the tube circuits are not started this way, they will be started nontheless after processing.
    */ startTubeCircuits(txs) {
         if (!this.provingState?.verifyState()) {
-            throw new Error(`Invalid proving state, call startNewEpoch before starting tube circuits`);
+            throw new Error(`Empty epoch proving state. call startNewEpoch before starting tube circuits.`);
         }
         const publicTxs = txs.filter((tx)=>tx.data.forPublic);
         for (const tx of publicTxs){
@@ -171,7 +225,7 @@ const logger = createLogger('prover-client:orchestrator');
             this.doEnqueueTube(txHash, privateInputs, (proof)=>{
                 tubeProof.resolve(proof);
             });
-            this.provingState?.cachedTubeProofs.set(txHash, tubeProof.promise);
+            this.provingState.cachedTubeProofs.set(txHash, tubeProof.promise);
         }
         return Promise.resolve();
     }
@@ -183,57 +237,82 @@ const logger = createLogger('prover-client:orchestrator');
         if (!provingState) {
             throw new Error(`Block proving state for ${blockNumber} not found`);
         }
-        if (!provingState.spongeBlobState) {
-            // If we are completing an empty block, initialize the provingState.
-            // We will have 0 txs and no blob fields.
-            provingState.startNewBlock(0, 0);
+        // Abort with specific error for the block if there's one.
+        const error = provingState.getError();
+        if (error) {
+            throw new Error(`Block proving failed: ${error}`);
         }
+        // Abort if the proving state is not valid due to errors occurred elsewhere.
         if (!provingState.verifyState()) {
-            throw new Error(`Block proving failed: ${provingState.error}`);
+            throw new Error(`Invalid proving state when completing block ${blockNumber}.`);
+        }
+        if (provingState.isAcceptingTxs()) {
+            throw new Error(`Block ${blockNumber} is still accepting txs. Call setBlockCompleted after all txs have been added.`);
         }
         // And build the block header
         logger.verbose(`Block ${blockNumber} completed. Assembling header.`);
-        await this.buildBlock(provingState, expectedHeader);
-        logger.debug(`Accumulating blobs for ${blockNumber}`);
-        await this.provingState?.setBlobAccumulators(blockNumber);
-        // If the proofs were faster than the block building, then we need to try the block root rollup again here
-        await this.checkAndEnqueueBlockRootRollup(provingState);
-        return provingState.block;
-    }
-    /** Returns the block as built for a given index. */ getBlock(index) {
-        const block = this.provingState?.blocks[index]?.block;
-        if (!block) {
-            throw new Error(`Block at index ${index} not available`);
-        }
-        return block;
-    }
-    async buildBlock(provingState, expectedHeader) {
-        // Collect all new nullifiers, commitments, and contracts from all txs in this block to build body
-        const txs = provingState.allTxs.map((a)=>a.processedTx);
+        const header = await this.buildL2BlockHeader(provingState, expectedHeader);
+        await this.verifyBuiltBlockAgainstSyncedState(provingState);
+        return header;
+    }
+    async buildL2BlockHeader(provingState, expectedHeader) {
+        // Collect all txs in this block to build the header. The function calling this has made sure that all txs have been added.
+        const txs = provingState.getProcessedTxs();
+        const startSpongeBlob = provingState.getStartSpongeBlob();
         // Get db for this block
         const db = this.dbs.get(provingState.blockNumber);
         // Given we've applied every change from this block, now assemble the block header
         // and update the archive tree, so we're ready to start processing the next block
-        const { header, body } = await buildHeaderAndBodyFromTxs(txs, provingState.globalVariables, provingState.newL1ToL2Messages, db);
+        const header = await buildBlockHeaderFromTxs(txs, provingState.getGlobalVariables(), startSpongeBlob, db);
         if (expectedHeader && !header.equals(expectedHeader)) {
             logger.error(`Block header mismatch: header=${header} expectedHeader=${expectedHeader}`);
             throw new Error('Block header mismatch');
         }
         logger.verbose(`Updating archive tree with block ${provingState.blockNumber} header ${(await header.hash()).toString()}`);
         await db.updateArchive(header);
-        // Assemble the L2 block
-        const newArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
-        const l2Block = new L2Block(newArchive, header, body);
-        await this.verifyBuiltBlockAgainstSyncedState(l2Block, newArchive);
-        logger.verbose(`Orchestrator finalized block ${l2Block.number}`);
-        provingState.setBlock(l2Block);
+        provingState.setBuiltBlockHeader(header);
+        return header;
     }
     // Flagged as protected to disable in certain unit tests
-    async verifyBuiltBlockAgainstSyncedState(l2Block, newArchive) {
-        const syncedArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.dbProvider.getSnapshot(l2Block.number));
+    async verifyBuiltBlockAgainstSyncedState(provingState) {
+        const builtBlockHeader = provingState.getBuiltBlockHeader();
+        if (!builtBlockHeader) {
+            logger.debug('Block header not built yet, skipping header check.');
+            return;
+        }
+        const output = provingState.getBlockRootRollupOutput();
+        if (!output) {
+            logger.debug('Block root rollup proof not built yet, skipping header check.');
+            return;
+        }
+        const header = await buildHeaderFromCircuitOutputs(output);
+        if (!(await header.hash()).equals(await builtBlockHeader.hash())) {
+            logger.error(`Block header mismatch.\nCircuit: ${inspect(header)}\nComputed: ${inspect(builtBlockHeader)}`);
+            provingState.reject(`Block header hash mismatch.`);
+            return;
+        }
+        // Get db for this block
+        const blockNumber = provingState.blockNumber;
+        const db = this.dbs.get(blockNumber);
+        const newArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
+        const syncedArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.dbProvider.getSnapshot(blockNumber));
         if (!syncedArchive.equals(newArchive)) {
-            throw new Error(`Archive tree mismatch for block ${l2Block.number}: world state synced to ${inspect(syncedArchive)} but built ${inspect(newArchive)}`);
+            logger.error(`Archive tree mismatch for block ${blockNumber}: world state synced to ${inspect(syncedArchive)} but built ${inspect(newArchive)}`);
+            provingState.reject(`Archive tree mismatch.`);
+            return;
         }
+        const circuitArchive = output.newArchive;
+        if (!newArchive.equals(circuitArchive)) {
+            logger.error(`New archive mismatch.\nCircuit: ${output.newArchive}\nComputed: ${newArchive}`);
+            provingState.reject(`New archive mismatch.`);
+            return;
+        }
+        // TODO(palla/prover): This closes the fork only on the happy path. If this epoch orchestrator
+        // is aborted and never reaches this point, it will leak the fork. We need to add a global cleanup,
+        // but have to make sure it only runs once all operations are completed, otherwise some function here
+        // will attempt to access the fork after it was closed.
+        logger.debug(`Cleaning up world state fork for ${blockNumber}`);
+        void this.dbs.get(blockNumber)?.close().then(()=>this.dbs.delete(blockNumber)).catch((err)=>logger.error(`Error closing db for block ${blockNumber}`, err));
     }
     /**
    * Cancel any further proving
@@ -253,14 +332,7 @@ const logger = createLogger('prover-client:orchestrator');
         if (result.status === 'failure') {
             throw new Error(`Epoch proving failed: ${result.reason}`);
         }
-        // TODO(MW): Move this? Requires async and don't want to force root methods to be async
-        // TODO(MW): EpochProvingState uses this.blocks.filter(b => !!b).length as total blocks, use this below:
-        const finalBlock = this.provingState.blocks[this.provingState.totalNumBlocks - 1];
-        if (!finalBlock || !finalBlock.endBlobAccumulator) {
-            throw new Error(`Epoch's final block not ready for finalize`);
-        }
-        const finalBatchedBlob = await finalBlock.endBlobAccumulator.finalize();
-        this.provingState.setFinalBatchedBlob(finalBatchedBlob);
+        await this.provingState.finalizeBatchedBlob();
         const epochProofResult = this.provingState.getEpochProofResult();
         pushTestData('epochProofResult', {
             proof: epochProofResult.proof.toString(),
@@ -269,24 +341,12 @@ const logger = createLogger('prover-client:orchestrator');
         return epochProofResult;
     }
     /**
-   * Starts the proving process for the given transaction and adds it to our state
-   * @param tx - The transaction whose proving we wish to commence
-   * @param provingState - The proving state being worked on
-   */ async prepareTransaction(tx, provingState) {
-        const txInputs = await this.prepareBaseRollupInputs(provingState, tx);
-        if (!txInputs) {
-            // This should not be possible
-            throw new Error(`Unable to add transaction, preparing base inputs failed`);
-        }
-        return txInputs;
-    }
-    /**
    * Enqueue a job to be scheduled
    * @param provingState - The proving state object being operated on
    * @param jobType - The type of job to be queued
    * @param job - The actual job, returns a promise notifying of the job's completion
    */ deferredProving(provingState, request, callback) {
-        if (!provingState?.verifyState()) {
+        if (!provingState.verifyState()) {
             logger.debug(`Not enqueuing job, state no longer valid`);
             return;
         }
@@ -301,7 +361,7 @@ const logger = createLogger('prover-client:orchestrator');
                     return;
                 }
                 const result = await request(controller.signal);
-                if (!provingState?.verifyState()) {
+                if (!provingState.verifyState()) {
                     logger.debug(`State no longer valid, discarding result`);
                     return;
                 }
@@ -329,29 +389,26 @@ const logger = createLogger('prover-client:orchestrator');
         // let the callstack unwind before adding the job to the queue
         setImmediate(()=>void safeJob());
     }
-    async prepareBaseParityInputs(l1ToL2Messages, db) {
+    async updateL1ToL2MessageTree(l1ToL2Messages, db) {
         const l1ToL2MessagesPadded = padArrayEnd(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 'Too many L1 to L2 messages');
-        const baseParityInputs = times(NUM_BASE_PARITY_PER_ROOT_PARITY, (i)=>BaseParityInputs.fromSlice(l1ToL2MessagesPadded, i, getVKTreeRoot()));
-        const l1ToL2MessageTreeSnapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, db);
-        const l1ToL2MessageSubtreeSiblingPath = assertLength(await getSubtreeSiblingPath(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, L1_TO_L2_MSG_SUBTREE_HEIGHT, db), L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH);
+        const lastL1ToL2MessageTreeSnapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, db);
+        const lastL1ToL2MessageSubtreeSiblingPath = assertLength(await getSubtreeSiblingPath(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, L1_TO_L2_MSG_SUBTREE_HEIGHT, db), L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH);
         // Update the local trees to include the new l1 to l2 messages
         await db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded);
-        const l1ToL2MessageTreeSnapshotAfterInsertion = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, db);
+        const newL1ToL2MessageTreeSnapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, db);
+        const newL1ToL2MessageSubtreeSiblingPath = assertLength(await getSubtreeSiblingPath(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, L1_TO_L2_MSG_SUBTREE_HEIGHT, db), L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH);
         return {
-            l1ToL2MessageTreeSnapshot,
-            l1ToL2MessageSubtreeSiblingPath,
-            l1ToL2MessageTreeSnapshotAfterInsertion,
-            baseParityInputs
+            lastL1ToL2MessageTreeSnapshot,
+            lastL1ToL2MessageSubtreeSiblingPath,
+            newL1ToL2MessageTreeSnapshot,
+            newL1ToL2MessageSubtreeSiblingPath
         };
     }
     // Updates the merkle trees for a transaction. The first enqueued job for a transaction
-    async prepareBaseRollupInputs(provingState, tx) {
-        if (!provingState.verifyState() || !provingState.spongeBlobState) {
-            logger.debug('Not preparing base rollup inputs, state invalid');
-            return;
-        }
-        const db = this.dbs.get(provingState.blockNumber);
-        const [ms, hints] = await elapsed(insertSideEffectsAndBuildBaseRollupHints(tx, provingState.globalVariables, provingState.l1ToL2MessageTreeSnapshotAfterInsertion, db, provingState.spongeBlobState));
+    async prepareBaseRollupInputs(tx, lastArchive, newL1ToL2MessageTreeSnapshot, startSpongeBlob, db) {
+        // We build the base rollup inputs using a mock proof and verification key.
+        // These will be overwritten later once we have proven the tube circuit and any public kernels
+        const [ms, hints] = await elapsed(insertSideEffectsAndBuildBaseRollupHints(tx, lastArchive, newL1ToL2MessageTreeSnapshot, startSpongeBlob, this.proverId.toField(), db));
         this.metrics.recordBaseRollupInputs(ms);
         const promises = [
             MerkleTreeId.NOTE_HASH_TREE,
@@ -367,10 +424,6 @@ const logger = createLogger('prover-client:orchestrator');
                 obj.key,
                 obj.value
             ]));
-        if (!provingState.verifyState()) {
-            logger.debug(`Discarding proving job, state no longer valid`);
-            return;
-        }
         return [
             hints,
             treeSnapshots
@@ -383,27 +436,31 @@ const logger = createLogger('prover-client:orchestrator');
             logger.debug('Not running base rollup, state invalid');
             return;
         }
+        if (!provingState.tryStartProvingBase(txIndex)) {
+            logger.debug(`Base rollup for tx ${txIndex} already started.`);
+            return;
+        }
         const txProvingState = provingState.getTxProvingState(txIndex);
         const { processedTx } = txProvingState;
         const { rollupType, inputs } = txProvingState.getBaseRollupTypeAndInputs();
         logger.debug(`Enqueuing deferred proving base rollup for ${processedTx.hash.toString()}`);
-        this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, `ProvingOrchestrator.prover.${inputs instanceof PrivateBaseRollupInputs ? 'getPrivateBaseRollupProof' : 'getPublicBaseRollupProof'}`, {
+        this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, `ProvingOrchestrator.prover.${inputs instanceof PrivateTxBaseRollupPrivateInputs ? 'getPrivateTxBaseRollupProof' : 'getPublicTxBaseRollupProof'}`, {
             [Attributes.TX_HASH]: processedTx.hash.toString(),
             [Attributes.PROTOCOL_CIRCUIT_NAME]: rollupType
         }, (signal)=>{
-            if (inputs instanceof PrivateBaseRollupInputs) {
-                return this.prover.getPrivateBaseRollupProof(inputs, signal, provingState.epochNumber);
+            if (inputs instanceof PrivateTxBaseRollupPrivateInputs) {
+                return this.prover.getPrivateTxBaseRollupProof(inputs, signal, provingState.epochNumber);
             } else {
-                return this.prover.getPublicBaseRollupProof(inputs, signal, provingState.epochNumber);
+                return this.prover.getPublicTxBaseRollupProof(inputs, signal, provingState.epochNumber);
             }
-        }), async (result)=>{
+        }), (result)=>{
             logger.debug(`Completed proof for ${rollupType} for tx ${processedTx.hash.toString()}`);
-            validatePartialState(result.inputs.end, txProvingState.treeSnapshots);
+            validatePartialState(result.inputs.endPartialState, txProvingState.treeSnapshots);
             const leafLocation = provingState.setBaseRollupProof(txIndex, result);
             if (provingState.totalNumTxs === 1) {
-                await this.checkAndEnqueueBlockRootRollup(provingState);
+                this.checkAndEnqueueBlockRootRollup(provingState);
             } else {
-                await this.checkAndEnqueueNextMergeRollup(provingState, leafLocation);
+                this.checkAndEnqueueNextMergeRollup(provingState, leafLocation);
             }
         });
     }
@@ -436,14 +493,14 @@ const logger = createLogger('prover-client:orchestrator');
         this.doEnqueueTube(txHash, txProvingState.getPublicTubePrivateInputs(), handleResult);
     }
     doEnqueueTube(txHash, inputs, handler, provingState = this.provingState) {
-        if (!provingState?.verifyState()) {
+        if (!provingState.verifyState()) {
             logger.debug('Not running tube circuit, state invalid');
             return;
         }
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getPublicTubeProof', {
             [Attributes.TX_HASH]: txHash,
-            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'public-tube'
-        }, (signal)=>this.prover.getPublicTubeProof(inputs, signal, this.provingState.epochNumber)), handler);
+            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'tube-public'
+        }, (signal)=>this.prover.getPublicTubeProof(inputs, signal, provingState.epochNumber)), handler);
     }
     // Executes the merge rollup circuit and stored the output as intermediate state for the parent merge/block root circuit
     // Enqueues the next level of merge if all inputs are available
@@ -452,74 +509,73 @@ const logger = createLogger('prover-client:orchestrator');
             logger.debug('Not running merge rollup. State no longer valid.');
             return;
         }
+        if (!provingState.tryStartProvingMerge(location)) {
+            logger.debug('Merge rollup already started.');
+            return;
+        }
         const inputs = provingState.getMergeRollupInputs(location);
-        this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getMergeRollupProof', {
-            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'merge-rollup'
-        }, (signal)=>this.prover.getMergeRollupProof(inputs, signal, provingState.epochNumber)), async (result)=>{
+        this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getTxMergeRollupProof', {
+            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'rollup-tx-merge'
+        }, (signal)=>this.prover.getTxMergeRollupProof(inputs, signal, provingState.epochNumber)), (result)=>{
             provingState.setMergeRollupProof(location, result);
-            await this.checkAndEnqueueNextMergeRollup(provingState, location);
+            this.checkAndEnqueueNextMergeRollup(provingState, location);
         });
     }
     // Executes the block root rollup circuit
-    async enqueueBlockRootRollup(provingState) {
+    enqueueBlockRootRollup(provingState) {
         if (!provingState.verifyState()) {
             logger.debug('Not running block root rollup, state no longer valid');
             return;
         }
-        provingState.blockRootRollupStarted = true;
-        const { rollupType, inputs } = await provingState.getBlockRootRollupTypeAndInputs();
-        logger.debug(`Enqueuing ${rollupType} for block ${provingState.blockNumber} with ${provingState.newL1ToL2Messages.length} l1 to l2 msgs.`);
+        if (!provingState.tryStartProvingBlockRoot()) {
+            logger.debug('Block root rollup already started.');
+            return;
+        }
+        const { rollupType, inputs } = provingState.getBlockRootRollupTypeAndInputs();
+        logger.debug(`Enqueuing ${rollupType} for block ${provingState.blockNumber}.`);
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getBlockRootRollupProof', {
             [Attributes.PROTOCOL_CIRCUIT_NAME]: rollupType
         }, (signal)=>{
-            if (inputs instanceof EmptyBlockRootRollupInputs) {
-                return this.prover.getEmptyBlockRootRollupProof(inputs, signal, provingState.epochNumber);
-            } else if (inputs instanceof SingleTxBlockRootRollupInputs) {
-                return this.prover.getSingleTxBlockRootRollupProof(inputs, signal, provingState.epochNumber);
+            if (inputs instanceof BlockRootFirstRollupPrivateInputs) {
+                return this.prover.getBlockRootFirstRollupProof(inputs, signal, provingState.epochNumber);
+            } else if (inputs instanceof BlockRootSingleTxFirstRollupPrivateInputs) {
+                return this.prover.getBlockRootSingleTxFirstRollupProof(inputs, signal, provingState.epochNumber);
+            } else if (inputs instanceof BlockRootEmptyTxFirstRollupPrivateInputs) {
+                return this.prover.getBlockRootEmptyTxFirstRollupProof(inputs, signal, provingState.epochNumber);
+            } else if (inputs instanceof BlockRootSingleTxRollupPrivateInputs) {
+                return this.prover.getBlockRootSingleTxRollupProof(inputs, signal, provingState.epochNumber);
             } else {
                 return this.prover.getBlockRootRollupProof(inputs, signal, provingState.epochNumber);
             }
         }), async (result)=>{
-            provingState.setBlockRootRollupProof(result);
-            const header = await provingState.buildHeaderFromProvingOutputs();
-            if (!(await header.hash()).equals(await provingState.block.header.hash())) {
-                logger.error(`Block header mismatch.\nCircuit: ${inspect(header)}\nComputed: ${inspect(provingState.block.header)}`);
-                provingState.reject(`Block header hash mismatch.`);
-            }
-            const dbArchiveRoot = provingState.block.archive.root;
-            const circuitArchiveRoot = result.inputs.newArchive.root;
-            if (!dbArchiveRoot.equals(circuitArchiveRoot)) {
-                logger.error(`New archive root mismatch.\nCircuit: ${result.inputs.newArchive.root}\nComputed: ${dbArchiveRoot}`);
-                provingState.reject(`New archive root mismatch.`);
-            }
-            const endBlobAccumulatorPublicInputs = BlobAccumulatorPublicInputs.fromBatchedBlobAccumulator(provingState.endBlobAccumulator);
-            const circuitEndBlobAccumulatorState = result.inputs.blobPublicInputs.endBlobAccumulator;
-            if (!circuitEndBlobAccumulatorState.equals(endBlobAccumulatorPublicInputs)) {
-                logger.error(`Blob accumulator state mismatch.\nCircuit: ${inspect(circuitEndBlobAccumulatorState)}\nComputed: ${inspect(endBlobAccumulatorPublicInputs)}`);
-                provingState.reject(`Blob accumulator state mismatch.`);
-            }
-            logger.debug(`Completed ${rollupType} proof for block ${provingState.block.number}`);
-            // validatePartialState(result.inputs.end, tx.treeSnapshots); // TODO(palla/prover)
-            const epochProvingState = this.provingState;
-            const leafLocation = epochProvingState.setBlockRootRollupProof(provingState.index, result);
-            if (epochProvingState.totalNumBlocks === 1) {
-                this.enqueueEpochPadding(epochProvingState);
+            // If the proofs were slower than the block header building, then we need to try validating the block header hashes here.
+            await this.verifyBuiltBlockAgainstSyncedState(provingState);
+            logger.debug(`Completed ${rollupType} proof for block ${provingState.blockNumber}`);
+            const leafLocation = provingState.setBlockRootRollupProof(result);
+            const checkpointProvingState = provingState.parentCheckpoint;
+            if (checkpointProvingState.totalNumBlocks === 1) {
+                this.checkAndEnqueueCheckpointRootRollup(checkpointProvingState);
             } else {
-                this.checkAndEnqueueNextBlockMergeRollup(epochProvingState, leafLocation);
+                this.checkAndEnqueueNextBlockMergeRollup(checkpointProvingState, leafLocation);
             }
         });
     }
     // Executes the base parity circuit and stores the intermediate state for the root parity circuit
     // Enqueues the root parity circuit if all inputs are available
-    enqueueBaseParityCircuit(provingState, inputs, index) {
+    enqueueBaseParityCircuit(checkpointProvingState, provingState, baseParityIndex) {
         if (!provingState.verifyState()) {
             logger.debug('Not running base parity. State no longer valid.');
             return;
         }
+        if (!provingState.tryStartProvingBaseParity(baseParityIndex)) {
+            logger.warn(`Base parity ${baseParityIndex} already started.`);
+            return;
+        }
+        const inputs = checkpointProvingState.getBaseParityInputs(baseParityIndex);
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getBaseParityProof', {
-            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'base-parity'
+            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'parity-base'
         }, (signal)=>this.prover.getBaseParityProof(inputs, signal, provingState.epochNumber)), (provingOutput)=>{
-            provingState.setBaseParityProof(index, provingOutput);
+            provingState.setBaseParityProof(baseParityIndex, provingOutput);
             this.checkAndEnqueueRootParityCircuit(provingState);
         });
     }
@@ -536,12 +592,16 @@ const logger = createLogger('prover-client:orchestrator');
             logger.debug('Not running root parity. State no longer valid.');
             return;
         }
-        const inputs = provingState.getRootParityInputs();
+        if (!provingState.tryStartProvingRootParity()) {
+            logger.debug('Root parity already started.');
+            return;
+        }
+        const inputs = provingState.getParityRootInputs();
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getRootParityProof', {
-            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'root-parity'
-        }, (signal)=>this.prover.getRootParityProof(inputs, signal, provingState.epochNumber)), async (result)=>{
+            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'parity-root'
+        }, (signal)=>this.prover.getRootParityProof(inputs, signal, provingState.epochNumber)), (result)=>{
             provingState.setRootParityProof(result);
-            await this.checkAndEnqueueBlockRootRollup(provingState);
+            this.checkAndEnqueueBlockRootRollup(provingState);
         });
     }
     // Executes the block merge rollup circuit and stored the output as intermediate state for the parent merge/block root circuit
@@ -551,26 +611,90 @@ const logger = createLogger('prover-client:orchestrator');
             logger.debug('Not running block merge rollup. State no longer valid.');
             return;
         }
+        if (!provingState.tryStartProvingBlockMerge(location)) {
+            logger.debug('Block merge rollup already started.');
+            return;
+        }
         const inputs = provingState.getBlockMergeRollupInputs(location);
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getBlockMergeRollupProof', {
-            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'block-merge-rollup'
+            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'rollup-block-merge'
         }, (signal)=>this.prover.getBlockMergeRollupProof(inputs, signal, provingState.epochNumber)), (result)=>{
             provingState.setBlockMergeRollupProof(location, result);
             this.checkAndEnqueueNextBlockMergeRollup(provingState, location);
         });
     }
+    enqueueCheckpointRootRollup(provingState) {
+        if (!provingState.verifyState()) {
+            logger.debug('Not running checkpoint root rollup. State no longer valid.');
+            return;
+        }
+        if (!provingState.tryStartProvingCheckpointRoot()) {
+            logger.debug('Checkpoint root rollup already started.');
+            return;
+        }
+        const rollupType = provingState.getCheckpointRootRollupType();
+        logger.debug(`Enqueuing ${rollupType} for checkpoint ${provingState.index}.`);
+        this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getCheckpointRootRollupProof', {
+            [Attributes.PROTOCOL_CIRCUIT_NAME]: rollupType
+        }, async (signal)=>{
+            const inputs = await provingState.getCheckpointRootRollupInputs();
+            if (inputs instanceof CheckpointRootSingleBlockRollupPrivateInputs) {
+                return this.prover.getCheckpointRootSingleBlockRollupProof(inputs, signal, provingState.epochNumber);
+            } else {
+                return this.prover.getCheckpointRootRollupProof(inputs, signal, provingState.epochNumber);
+            }
+        }), (result)=>{
+            const computedEndBlobAccumulatorState = BlobAccumulatorPublicInputs.fromBatchedBlobAccumulator(provingState.getEndBlobAccumulator());
+            const circuitEndBlobAccumulatorState = result.inputs.endBlobAccumulator;
+            if (!circuitEndBlobAccumulatorState.equals(computedEndBlobAccumulatorState)) {
+                logger.error(`Blob accumulator state mismatch.\nCircuit: ${inspect(circuitEndBlobAccumulatorState)}\nComputed: ${inspect(computedEndBlobAccumulatorState)}`);
+                provingState.reject(`Blob accumulator state mismatch.`);
+                return;
+            }
+            logger.debug(`Completed ${rollupType} proof for checkpoint ${provingState.index}.`);
+            const leafLocation = provingState.setCheckpointRootRollupProof(result);
+            const epochProvingState = provingState.parentEpoch;
+            if (epochProvingState.totalNumCheckpoints === 1) {
+                this.enqueueEpochPadding(epochProvingState);
+            } else {
+                this.checkAndEnqueueNextCheckpointMergeRollup(epochProvingState, leafLocation);
+            }
+        });
+    }
+    enqueueCheckpointMergeRollup(provingState, location) {
+        if (!provingState.verifyState()) {
+            logger.debug('Not running checkpoint merge rollup. State no longer valid.');
+            return;
+        }
+        if (!provingState.tryStartProvingCheckpointMerge(location)) {
+            logger.debug('Checkpoint merge rollup already started.');
+            return;
+        }
+        const inputs = provingState.getCheckpointMergeRollupInputs(location);
+        this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getCheckpointMergeRollupProof', {
+            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'rollup-checkpoint-merge'
+        }, (signal)=>this.prover.getCheckpointMergeRollupProof(inputs, signal, provingState.epochNumber)), (result)=>{
+            logger.debug('Completed proof for checkpoint merge rollup.');
+            provingState.setCheckpointMergeRollupProof(location, result);
+            this.checkAndEnqueueNextCheckpointMergeRollup(provingState, location);
+        });
+    }
     enqueueEpochPadding(provingState) {
         if (!provingState.verifyState()) {
             logger.debug('Not running epoch padding. State no longer valid.');
             return;
         }
+        if (!provingState.tryStartProvingPaddingCheckpoint()) {
+            logger.debug('Padding checkpoint already started.');
+            return;
+        }
         logger.debug('Padding epoch proof with a padding block root proof.');
-        const inputs = provingState.getPaddingBlockRootInputs();
-        this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getPaddingBlockRootRollupProof', {
-            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'padding-block-root-rollup'
-        }, (signal)=>this.prover.getPaddingBlockRootRollupProof(inputs, signal, provingState.epochNumber)), (result)=>{
-            logger.debug('Completed proof for padding block root.');
-            provingState.setPaddingBlockRootProof(result);
+        const inputs = provingState.getPaddingCheckpointInputs();
+        this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getCheckpointPaddingRollupProof', {
+            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'rollup-checkpoint-padding'
+        }, (signal)=>this.prover.getCheckpointPaddingRollupProof(inputs, signal, provingState.epochNumber)), (result)=>{
+            logger.debug('Completed proof for padding checkpoint.');
+            provingState.setCheckpointPaddingProof(result);
             this.checkAndEnqueueRootRollup(provingState);
         });
     }
@@ -583,7 +707,7 @@ const logger = createLogger('prover-client:orchestrator');
         logger.debug(`Preparing root rollup`);
         const inputs = provingState.getRootRollupInputs();
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getRootRollupProof', {
-            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'root-rollup'
+            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'rollup-root'
         }, (signal)=>this.prover.getRootRollupProof(inputs, signal, provingState.epochNumber)), (result)=>{
             logger.verbose(`Orchestrator completed root rollup for epoch ${provingState.epochNumber}`);
             provingState.setRootRollupProof(result);
@@ -592,36 +716,23 @@ const logger = createLogger('prover-client:orchestrator');
             });
         });
     }
-    async checkAndEnqueueNextMergeRollup(provingState, currentLocation) {
+    checkAndEnqueueNextMergeRollup(provingState, currentLocation) {
         if (!provingState.isReadyForMergeRollup(currentLocation)) {
             return;
         }
         const parentLocation = provingState.getParentLocation(currentLocation);
         if (parentLocation.level === 0) {
-            await this.checkAndEnqueueBlockRootRollup(provingState);
+            this.checkAndEnqueueBlockRootRollup(provingState);
         } else {
             this.enqueueMergeRollup(provingState, parentLocation);
         }
     }
-    async checkAndEnqueueBlockRootRollup(provingState) {
-        const blockNumber = provingState.blockNumber;
-        // Accumulate as far as we can, in case blocks came in out of order and we are behind:
-        await this.provingState?.setBlobAccumulators(blockNumber);
+    checkAndEnqueueBlockRootRollup(provingState) {
         if (!provingState.isReadyForBlockRootRollup()) {
             logger.debug('Not ready for block root rollup');
             return;
         }
-        if (provingState.blockRootRollupStarted) {
-            logger.debug('Block root rollup already started');
-            return;
-        }
-        // TODO(palla/prover): This closes the fork only on the happy path. If this epoch orchestrator
-        // is aborted and never reaches this point, it will leak the fork. We need to add a global cleanup,
-        // but have to make sure it only runs once all operations are completed, otherwise some function here
-        // will attempt to access the fork after it was closed.
-        logger.debug(`Cleaning up world state fork for ${blockNumber}`);
-        void this.dbs.get(blockNumber)?.close().then(()=>this.dbs.delete(blockNumber)).catch((err)=>logger.error(`Error closing db for block ${blockNumber}`, err));
-        await this.enqueueBlockRootRollup(provingState);
+        this.enqueueBlockRootRollup(provingState);
     }
     checkAndEnqueueNextBlockMergeRollup(provingState, currentLocation) {
         if (!provingState.isReadyForBlockMerge(currentLocation)) {
@@ -629,11 +740,28 @@ const logger = createLogger('prover-client:orchestrator');
         }
         const parentLocation = provingState.getParentLocation(currentLocation);
         if (parentLocation.level === 0) {
-            this.checkAndEnqueueRootRollup(provingState);
+            this.checkAndEnqueueCheckpointRootRollup(provingState);
         } else {
             this.enqueueBlockMergeRollup(provingState, parentLocation);
         }
     }
+    checkAndEnqueueCheckpointRootRollup(provingState) {
+        if (!provingState.isReadyForCheckpointRoot()) {
+            return;
+        }
+        this.enqueueCheckpointRootRollup(provingState);
+    }
+    checkAndEnqueueNextCheckpointMergeRollup(provingState, currentLocation) {
+        if (!provingState.isReadyForCheckpointMerge(currentLocation)) {
+            return;
+        }
+        const parentLocation = provingState.getParentLocation(currentLocation);
+        if (parentLocation.level === 0) {
+            this.checkAndEnqueueRootRollup(provingState);
+        } else {
+            this.enqueueCheckpointMergeRollup(provingState, parentLocation);
+        }
+    }
     checkAndEnqueueRootRollup(provingState) {
         if (!provingState.isReadyForRootRollup()) {
             logger.debug('Not ready for root rollup');
@@ -697,8 +825,8 @@ const logger = createLogger('prover-client:orchestrator');
     }
 }
 _ts_decorate([
-    trackSpan('ProvingOrchestrator.startNewBlock', (globalVariables)=>({
-            [Attributes.BLOCK_NUMBER]: globalVariables.blockNumber
+    trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
+            [Attributes.BLOCK_NUMBER]: blockNumber
         }))
 ], ProvingOrchestrator.prototype, "startNewBlock", null);
 _ts_decorate([
@@ -715,7 +843,7 @@ _ts_decorate([
         }))
 ], ProvingOrchestrator.prototype, "setBlockCompleted", null);
 _ts_decorate([
-    trackSpan('ProvingOrchestrator.prepareBaseRollupInputs', (_, tx)=>({
+    trackSpan('ProvingOrchestrator.prepareBaseRollupInputs', (tx)=>({
             [Attributes.TX_HASH]: tx.hash.toString()
         }))
 ], ProvingOrchestrator.prototype, "prepareBaseRollupInputs", null);