@aztec/prover-client 0.0.1-commit.96bb3f7 → 0.0.1-commit.993d240

package/dest/orchestrator/orchestrator.js

@@ -371,16 +371,13 @@ function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
     return (_apply_decs_2203_r = applyDecs2203RFactory())(targetClass, memberDecs, classDecs, parentClass);
 }
 var _dec, _dec1, _dec2, _dec3, _dec4, _initProto;
-import { L1_TO_L2_MSG_SUBTREE_HEIGHT, L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH, NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, NUM_BASE_PARITY_PER_ROOT_PARITY } from '@aztec/constants';
+import { L1_TO_L2_MSG_SUBTREE_HEIGHT, L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH, NUM_BASE_PARITY_PER_ROOT_PARITY } from '@aztec/constants';
 import { BlockNumber } from '@aztec/foundation/branded-types';
-import { padArrayEnd } from '@aztec/foundation/collection';
-import { Fr } from '@aztec/foundation/curves/bn254';
-import { AbortError } from '@aztec/foundation/error';
-import { createLogger } from '@aztec/foundation/log';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { assertLength } from '@aztec/foundation/serialize';
 import { pushTestData } from '@aztec/foundation/testing';
 import { elapsed } from '@aztec/foundation/timer';
+import { appendL1ToL2MessagesToTree } from '@aztec/stdlib/messaging';
 import { BlockRootEmptyTxFirstRollupPrivateInputs, BlockRootFirstRollupPrivateInputs, BlockRootSingleTxFirstRollupPrivateInputs, BlockRootSingleTxRollupPrivateInputs, CheckpointRootSingleBlockRollupPrivateInputs, PrivateTxBaseRollupPrivateInputs } from '@aztec/stdlib/rollup';
 import { MerkleTreeId } from '@aztec/stdlib/trees';
 import { Attributes, getTelemetryClient, trackSpan, wrapCallbackInSpan } from '@aztec/telemetry-client';
@@ -388,8 +385,8 @@ import { inspect } from 'util';
 import { buildHeaderFromCircuitOutputs, getLastSiblingPath, getPublicChonkVerifierPrivateInputsFromTx, getRootTreeSiblingPath, getSubtreeSiblingPath, getTreeSnapshot, insertSideEffectsAndBuildBaseRollupHints, validatePartialState, validateTx } from './block-building-helpers.js';
 import { EpochProvingState } from './epoch-proving-state.js';
 import { ProvingOrchestratorMetrics } from './orchestrator_metrics.js';
+import { TopTreeProvingScheduler } from './top-tree-proving-scheduler.js';
 import { TxProvingState } from './tx-proving-state.js';
-const logger = createLogger('prover-client:orchestrator');
 _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
         [Attributes.BLOCK_NUMBER]: blockNumber
     })), _dec1 = trackSpan('ProvingOrchestrator.addTxs', (txs)=>({
@@ -410,10 +407,10 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
  * The proving implementation is determined by the provided prover. This could be for example a local prover or a remote prover pool.
  */ /**
  * The orchestrator, managing the flow of recursive proving operations required to build the rollup proof tree.
- */ export class ProvingOrchestrator {
+ */ export class ProvingOrchestrator extends TopTreeProvingScheduler {
     dbProvider;
-    prover;
     proverId;
+    cancelJobsOnStop;
     static{
         ({ e: [_initProto] } = _apply_decs_2203_r(this, [
             [
@@ -444,19 +441,11 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
         ], []));
     }
     provingState;
-    pendingProvingJobs;
     provingPromise;
     metrics;
-    // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
     dbs;
-    constructor(dbProvider, prover, proverId, telemetryClient = getTelemetryClient()){
-        this.dbProvider = dbProvider;
-        this.prover = prover;
-        this.proverId = proverId;
-        this.provingState = (_initProto(this), undefined);
-        this.pendingProvingJobs = [];
-        this.provingPromise = undefined;
-        this.dbs = new Map();
+    constructor(dbProvider, prover, proverId, cancelJobsOnStop = false, enqueueConcurrency, telemetryClient = getTelemetryClient(), bindings){
+        super(prover, enqueueConcurrency, 'prover-client:orchestrator', bindings), this.dbProvider = dbProvider, this.proverId = proverId, this.cancelJobsOnStop = cancelJobsOnStop, this.provingState = (_initProto(this), undefined), this.provingPromise = undefined, this.dbs = new Map();
         this.metrics = new ProvingOrchestratorMetrics(telemetryClient, 'ProvingOrchestrator');
     }
     get tracer() {
@@ -465,9 +454,21 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     getProverId() {
         return this.proverId;
     }
-    stop() {
+    getNumActiveForks() {
+        return this.dbs.size;
+    }
+    cancelInternal() {
         this.cancel();
-        return Promise.resolve();
+    }
+    wrapCircuitCall(circuitName, fn) {
+        return wrapCallbackInSpan(this.tracer, `ProvingOrchestrator.prover.${circuitName}`, {
+            [Attributes.PROTOCOL_CIRCUIT_NAME]: circuitName
+        }, fn);
+    }
+    onRootRollupComplete(state) {
+        state.resolve({
+            status: 'success'
+        });
     }
     startNewEpoch(epochNumber, totalNumCheckpoints, finalBlobBatchingChallenges) {
         if (this.provingState?.verifyState()) {
@@ -478,11 +479,18 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
                 status: 'failure',
                 reason
             }));
-        logger.info(`Starting epoch ${epochNumber} with ${totalNumCheckpoints} checkpoints.`);
+        this.logger.info(`Starting epoch ${epochNumber} with ${totalNumCheckpoints} checkpoints.`);
         this.provingState = new EpochProvingState(epochNumber, totalNumCheckpoints, finalBlobBatchingChallenges, (provingState)=>this.checkAndEnqueueCheckpointRootRollup(provingState), resolve, reject);
         this.provingPromise = promise;
     }
-    async startNewCheckpoint(checkpointIndex, constants, l1ToL2Messages, totalNumBlocks, headerOfLastBlockInPreviousCheckpoint) {
+    /**
+   * Starts a new checkpoint.
+   * @param checkpointIndex - The index of the checkpoint in the epoch.
+   * @param constants - The constants for this checkpoint.
+   * @param l1ToL2Messages - The set of L1 to L2 messages to be inserted at the beginning of this checkpoint.
+   * @param totalNumBlocks - The total number of blocks expected in the checkpoint (must be at least one).
+   * @param headerOfLastBlockInPreviousCheckpoint - The header of the last block in the previous checkpoint.
+   */ async startNewCheckpoint(checkpointIndex, constants, l1ToL2Messages, totalNumBlocks, headerOfLastBlockInPreviousCheckpoint) {
         if (!this.provingState) {
             throw new Error('Empty epoch proving state. Call startNewEpoch before starting a checkpoint.');
         }
@@ -518,14 +526,14 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
             throw new Error(`Checkpoint not accepting further blocks.`);
         }
         const constants = checkpointProvingState.constants;
-        logger.info(`Starting block ${blockNumber} for slot ${constants.slotNumber}.`);
+        this.logger.info(`Starting block ${blockNumber} for slot ${constants.slotNumber}.`);
         // Fork the db only when it's not already set. The db for the first block is set in `startNewCheckpoint`.
         if (!this.dbs.has(blockNumber)) {
             // Fork world state at the end of the immediately previous block
             const db = await this.dbProvider.fork(BlockNumber(blockNumber - 1));
             this.dbs.set(blockNumber, db);
         }
-        const db = this.dbs.get(blockNumber);
+        const db = this.getDbForBlock(blockNumber);
         // Get archive snapshot and sibling path before any txs in this block lands.
         const lastArchiveTreeSnapshot = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
         const lastArchiveSiblingPath = await getRootTreeSiblingPath(MerkleTreeId.ARCHIVE, db);
@@ -545,7 +553,8 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
             const blockEndBlobFields = blockProvingState.getBlockEndBlobFields();
             await endSpongeBlob.absorb(blockEndBlobFields);
             blockProvingState.setEndSpongeBlob(endSpongeBlob);
-            // And also try to accumulate the blobs as far as we can:
+            // Try to accumulate the out hashes and blobs as far as we can:
+            await this.provingState.accumulateCheckpointOutHashes();
             await this.provingState.setBlobAccumulators();
         }
     }
@@ -559,7 +568,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
         if (!txs.length) {
             // To avoid an ugly throw below. If we require an empty block, we can just call setBlockCompleted
             // on a block with no txs. We cannot do that here because we cannot find the blockNumber without any txs.
-            logger.warn(`Provided no txs to orchestrator addTxs.`);
+            this.logger.warn(`Provided no txs to orchestrator addTxs.`);
             return;
         }
         const blockNumber = BlockNumber(txs[0].globalVariables.blockNumber);
@@ -573,8 +582,8 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
         if (!provingState.isAcceptingTxs()) {
             throw new Error(`Block ${blockNumber} has been initialized with transactions.`);
         }
-        logger.info(`Adding ${txs.length} transactions to block ${blockNumber}`);
-        const db = this.dbs.get(blockNumber);
+        this.logger.info(`Adding ${txs.length} transactions to block ${blockNumber}`);
+        const db = this.getDbForBlock(blockNumber);
         const lastArchive = provingState.lastArchiveTreeSnapshot;
         const newL1ToL2MessageTreeSnapshot = provingState.newL1ToL2MessageTreeSnapshot;
         const spongeBlobState = provingState.getStartSpongeBlob().clone();
@@ -584,7 +593,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
                     throw new Error(`Invalid proving state when adding a tx`);
                 }
                 validateTx(tx);
-                logger.debug(`Received transaction: ${tx.hash}`);
+                this.logger.debug(`Received transaction: ${tx.hash}`);
                 const startSpongeBlob = spongeBlobState.clone();
                 const [hints, treeSnapshots] = await this.prepareBaseRollupInputs(tx, lastArchive, newL1ToL2MessageTreeSnapshot, startSpongeBlob, db);
                 if (!provingState.verifyState()) {
@@ -595,10 +604,10 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
                 const txIndex = provingState.addNewTx(txProvingState);
                 if (txProvingState.requireAvmProof) {
                     this.getOrEnqueueChonkVerifier(provingState, txIndex);
-                    logger.debug(`Enqueueing public VM for tx ${txIndex}`);
+                    this.logger.debug(`Enqueueing public VM for tx ${txIndex}`);
                     this.enqueueVM(provingState, txIndex);
                 } else {
-                    logger.debug(`Enqueueing base rollup for private-only tx ${txIndex}`);
+                    this.logger.debug(`Enqueueing base rollup for private-only tx ${txIndex}`);
                     this.enqueueBaseRollup(provingState, txIndex);
                 }
             } catch (err) {
@@ -612,7 +621,8 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
         const blockEndBlobFields = provingState.getBlockEndBlobFields();
         await spongeBlobState.absorb(blockEndBlobFields);
         provingState.setEndSpongeBlob(spongeBlobState);
-        // Txs have been added to the block. Now try to accumulate the blobs as far as we can:
+        // Txs have been added to the block. Now try to accumulate the out hashes and blobs as far as we can:
+        await this.provingState.accumulateCheckpointOutHashes();
         await this.provingState.setBlobAccumulators();
     }
     /**
@@ -627,7 +637,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
             const txHash = tx.getTxHash().toString();
             const privateInputs = getPublicChonkVerifierPrivateInputsFromTx(tx, this.proverId.toField());
             const tubeProof = promiseWithResolvers();
-            logger.debug(`Starting chonk verifier circuit for tx ${txHash}`);
+            this.logger.debug(`Starting chonk verifier circuit for tx ${txHash}`);
             this.doEnqueueChonkVerifier(txHash, privateInputs, (proof)=>{
                 tubeProof.resolve(proof);
             });
@@ -656,17 +666,23 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
             throw new Error(`Block ${blockNumber} is still accepting txs. Call setBlockCompleted after all txs have been added.`);
         }
         // Given we've applied every change from this block, now assemble the block header:
-        logger.verbose(`Block ${blockNumber} completed. Assembling header.`);
+        this.logger.verbose(`Block ${blockNumber} completed. Assembling header.`);
         const header = await provingState.buildBlockHeader();
         if (expectedHeader && !header.equals(expectedHeader)) {
-            logger.error(`Block header mismatch: header=${header} expectedHeader=${expectedHeader}`);
+            this.logger.error(`Block header mismatch: header=${header} expectedHeader=${expectedHeader}`);
             throw new Error('Block header mismatch');
         }
-        // Get db for this block
-        const db = this.dbs.get(provingState.blockNumber);
-        // Update the archive tree, so we're ready to start processing the next block:
-        logger.verbose(`Updating archive tree with block ${provingState.blockNumber} header ${(await header.hash()).toString()}`);
-        await db.updateArchive(header);
+        // Get db for this block and remove from map — no other code should use it after this point.
+        const db = this.getDbForBlock(provingState.blockNumber);
+        this.dbs.delete(provingState.blockNumber);
+        // Update the archive tree, capture the snapshot, and close the fork deterministically.
+        try {
+            this.logger.verbose(`Updating archive tree with block ${provingState.blockNumber} header ${(await header.hash()).toString()}`);
+            await db.updateArchive(header);
+            provingState.setBuiltArchive(await getTreeSnapshot(MerkleTreeId.ARCHIVE, db));
+        } finally{
+            await db.close();
+        }
         await this.verifyBuiltBlockAgainstSyncedState(provingState);
         return header;
     }
@@ -674,50 +690,57 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     async verifyBuiltBlockAgainstSyncedState(provingState) {
         const builtBlockHeader = provingState.getBuiltBlockHeader();
         if (!builtBlockHeader) {
-            logger.debug('Block header not built yet, skipping header check.');
+            this.logger.debug('Block header not built yet, skipping header check.');
             return;
         }
         const output = provingState.getBlockRootRollupOutput();
         if (!output) {
-            logger.debug('Block root rollup proof not built yet, skipping header check.');
+            this.logger.debug('Block root rollup proof not built yet, skipping header check.');
+            return;
+        }
+        const newArchive = provingState.getBuiltArchive();
+        if (!newArchive) {
+            this.logger.debug('Archive snapshot not yet captured, skipping header check.');
             return;
         }
         const header = await buildHeaderFromCircuitOutputs(output);
         if (!(await header.hash()).equals(await builtBlockHeader.hash())) {
-            logger.error(`Block header mismatch.\nCircuit: ${inspect(header)}\nComputed: ${inspect(builtBlockHeader)}`);
+            this.logger.error(`Block header mismatch.\nCircuit: ${inspect(header)}\nComputed: ${inspect(builtBlockHeader)}`);
             provingState.reject(`Block header hash mismatch.`);
             return;
         }
-        // Get db for this block
         const blockNumber = provingState.blockNumber;
-        const db = this.dbs.get(blockNumber);
-        const newArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
         const syncedArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.dbProvider.getSnapshot(blockNumber));
         if (!syncedArchive.equals(newArchive)) {
-            logger.error(`Archive tree mismatch for block ${blockNumber}: world state synced to ${inspect(syncedArchive)} but built ${inspect(newArchive)}`);
+            this.logger.error(`Archive tree mismatch for block ${blockNumber}: world state synced to ${inspect(syncedArchive)} but built ${inspect(newArchive)}`);
             provingState.reject(`Archive tree mismatch.`);
             return;
         }
         const circuitArchive = output.newArchive;
         if (!newArchive.equals(circuitArchive)) {
-            logger.error(`New archive mismatch.\nCircuit: ${output.newArchive}\nComputed: ${newArchive}`);
+            this.logger.error(`New archive mismatch.\nCircuit: ${output.newArchive}\nComputed: ${newArchive}`);
             provingState.reject(`New archive mismatch.`);
             return;
         }
-        // TODO(palla/prover): This closes the fork only on the happy path. If this epoch orchestrator
-        // is aborted and never reaches this point, it will leak the fork. We need to add a global cleanup,
-        // but have to make sure it only runs once all operations are completed, otherwise some function here
-        // will attempt to access the fork after it was closed.
-        logger.debug(`Cleaning up world state fork for ${blockNumber}`);
-        void this.dbs.get(blockNumber)?.close().then(()=>this.dbs.delete(blockNumber)).catch((err)=>logger.error(`Error closing db for block ${blockNumber}`, err));
     }
     /**
-   * Cancel any further proving
+   * Cancel any further proving.
+   * If cancelJobsOnStop is true, aborts all pending jobs with the broker (which marks them as 'Aborted').
+   * If cancelJobsOnStop is false (default), jobs remain in the broker queue and can be reused on restart/reorg.
    */ cancel() {
-        for (const controller of this.pendingProvingJobs){
-            controller.abort();
-        }
+        this.resetSchedulerState(this.cancelJobsOnStop);
         this.provingState?.cancel();
+        for (const [blockNumber, db] of this.dbs.entries()){
+            void db.close().catch((err)=>this.logger.error(`Error closing db for block ${blockNumber}`, err));
+        }
+        this.dbs.clear();
+    }
+    getDbForBlock(blockNumber) {
+        const db = this.dbs.get(blockNumber);
+        if (!db) {
+            throw new Error(`World state fork for block ${blockNumber} not found.`);
+        }
+        return db;
     }
     /**
    * Returns the proof for the current epoch.
@@ -737,61 +760,11 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
         });
         return epochProofResult;
     }
-    /**
-   * Enqueue a job to be scheduled
-   * @param provingState - The proving state object being operated on
-   * @param jobType - The type of job to be queued
-   * @param job - The actual job, returns a promise notifying of the job's completion
-   */ deferredProving(provingState, request, callback) {
-        if (!provingState.verifyState()) {
-            logger.debug(`Not enqueuing job, state no longer valid`);
-            return;
-        }
-        const controller = new AbortController();
-        this.pendingProvingJobs.push(controller);
-        // We use a 'safeJob'. We don't want promise rejections in the proving pool, we want to capture the error here
-        // and reject the proving job whilst keeping the event loop free of rejections
-        const safeJob = async ()=>{
-            try {
-                // there's a delay between enqueueing this job and it actually running
-                if (controller.signal.aborted) {
-                    return;
-                }
-                const result = await request(controller.signal);
-                if (!provingState.verifyState()) {
-                    logger.debug(`State no longer valid, discarding result`);
-                    return;
-                }
-                // we could have been cancelled whilst waiting for the result
-                // and the prover ignored the signal. Drop the result in that case
-                if (controller.signal.aborted) {
-                    return;
-                }
-                await callback(result);
-            } catch (err) {
-                if (err instanceof AbortError) {
-                    // operation was cancelled, probably because the block was cancelled
-                    // drop this result
-                    return;
-                }
-                logger.error(`Error thrown when proving job`, err);
-                provingState.reject(`${err}`);
-            } finally{
-                const index = this.pendingProvingJobs.indexOf(controller);
-                if (index > -1) {
-                    this.pendingProvingJobs.splice(index, 1);
-                }
-            }
-        };
-        // let the callstack unwind before adding the job to the queue
-        setImmediate(()=>void safeJob());
-    }
     async updateL1ToL2MessageTree(l1ToL2Messages, db) {
-        const l1ToL2MessagesPadded = padArrayEnd(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 'Too many L1 to L2 messages');
         const lastL1ToL2MessageTreeSnapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, db);
         const lastL1ToL2MessageSubtreeRootSiblingPath = assertLength(await getSubtreeSiblingPath(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, L1_TO_L2_MSG_SUBTREE_HEIGHT, db), L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH);
         // Update the local trees to include the new l1 to l2 messages
-        await db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded);
+        await appendL1ToL2MessagesToTree(db, l1ToL2Messages);
         const newL1ToL2MessageTreeSnapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, db);
         const newL1ToL2MessageSubtreeRootSiblingPath = assertLength(await getSubtreeSiblingPath(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, L1_TO_L2_MSG_SUBTREE_HEIGHT, db), L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH);
         return {
@@ -830,17 +803,17 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     // Executes the next level of merge if all inputs are available
     enqueueBaseRollup(provingState, txIndex) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running base rollup, state invalid');
+            this.logger.debug('Not running base rollup, state invalid');
             return;
         }
         if (!provingState.tryStartProvingBase(txIndex)) {
-            logger.debug(`Base rollup for tx ${txIndex} already started.`);
+            this.logger.debug(`Base rollup for tx ${txIndex} already started.`);
             return;
         }
         const txProvingState = provingState.getTxProvingState(txIndex);
         const { processedTx } = txProvingState;
         const { rollupType, inputs } = txProvingState.getBaseRollupTypeAndInputs();
-        logger.debug(`Enqueuing deferred proving base rollup for ${processedTx.hash.toString()}`);
+        this.logger.debug(`Enqueuing deferred proving base rollup for ${processedTx.hash.toString()}`);
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, `ProvingOrchestrator.prover.${inputs instanceof PrivateTxBaseRollupPrivateInputs ? 'getPrivateTxBaseRollupProof' : 'getPublicTxBaseRollupProof'}`, {
             [Attributes.TX_HASH]: processedTx.hash.toString(),
             [Attributes.PROTOCOL_CIRCUIT_NAME]: rollupType
@@ -851,7 +824,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
                 return this.prover.getPublicTxBaseRollupProof(inputs, signal, provingState.epochNumber);
             }
         }), (result)=>{
-            logger.debug(`Completed proof for ${rollupType} for tx ${processedTx.hash.toString()}`);
+            this.logger.debug(`Completed proof for ${rollupType} for tx ${processedTx.hash.toString()}`);
             validatePartialState(result.inputs.endTreeSnapshots, txProvingState.treeSnapshots);
             const leafLocation = provingState.setBaseRollupProof(txIndex, result);
             if (provingState.totalNumTxs === 1) {
@@ -865,14 +838,13 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     // Once completed, will enqueue the the public tx base rollup.
     getOrEnqueueChonkVerifier(provingState, txIndex) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running chonk verifier circuit, state invalid');
+            this.logger.debug('Not running chonk verifier circuit, state invalid');
             return;
         }
         const txProvingState = provingState.getTxProvingState(txIndex);
         const txHash = txProvingState.processedTx.hash.toString();
-        NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH;
         const handleResult = (result)=>{
-            logger.debug(`Got chonk verifier proof for tx index: ${txIndex}`, {
+            this.logger.debug(`Got chonk verifier proof for tx index: ${txIndex}`, {
                 txHash
             });
             txProvingState.setPublicChonkVerifierProof(result);
@@ -880,18 +852,18 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
             this.checkAndEnqueueBaseRollup(provingState, txIndex);
         };
         if (this.provingState?.cachedChonkVerifierProofs.has(txHash)) {
-            logger.debug(`Chonk verifier proof already enqueued for tx index: ${txIndex}`, {
+            this.logger.debug(`Chonk verifier proof already enqueued for tx index: ${txIndex}`, {
                 txHash
             });
             void this.provingState.cachedChonkVerifierProofs.get(txHash).then(handleResult);
             return;
         }
-        logger.debug(`Enqueuing chonk verifier circuit for tx index: ${txIndex}`);
+        this.logger.debug(`Enqueuing chonk verifier circuit for tx index: ${txIndex}`);
         this.doEnqueueChonkVerifier(txHash, txProvingState.getPublicChonkVerifierPrivateInputs(), handleResult);
     }
     doEnqueueChonkVerifier(txHash, inputs, handler, provingState = this.provingState) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running chonk verifier circuit, state invalid');
+            this.logger.debug('Not running chonk verifier circuit, state invalid');
             return;
         }
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getPublicChonkVerifierProof', {
@@ -903,11 +875,11 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     // Enqueues the next level of merge if all inputs are available
     enqueueMergeRollup(provingState, location) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running merge rollup. State no longer valid.');
+            this.logger.debug('Not running merge rollup. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingMerge(location)) {
-            logger.debug('Merge rollup already started.');
+            this.logger.debug('Merge rollup already started.');
             return;
         }
         const inputs = provingState.getMergeRollupInputs(location);
@@ -921,15 +893,15 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     // Executes the block root rollup circuit
     enqueueBlockRootRollup(provingState) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running block root rollup, state no longer valid');
+            this.logger.debug('Not running block root rollup, state no longer valid');
             return;
         }
         if (!provingState.tryStartProvingBlockRoot()) {
-            logger.debug('Block root rollup already started.');
+            this.logger.debug('Block root rollup already started.');
             return;
         }
         const { rollupType, inputs } = provingState.getBlockRootRollupTypeAndInputs();
-        logger.debug(`Enqueuing ${rollupType} for block ${provingState.blockNumber}.`);
+        this.logger.debug(`Enqueuing ${rollupType} for block ${provingState.blockNumber}.`);
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getBlockRootRollupProof', {
             [Attributes.PROTOCOL_CIRCUIT_NAME]: rollupType
         }, (signal)=>{
@@ -945,15 +917,20 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
                 return this.prover.getBlockRootRollupProof(inputs, signal, provingState.epochNumber);
             }
         }), async (result)=>{
-            // If the proofs were slower than the block header building, then we need to try validating the block header hashes here.
-            await this.verifyBuiltBlockAgainstSyncedState(provingState);
-            logger.debug(`Completed ${rollupType} proof for block ${provingState.blockNumber}`);
+            this.logger.debug(`Completed ${rollupType} proof for block ${provingState.blockNumber}`, {
+                blockNumber: provingState.blockNumber,
+                checkpointIndex: provingState.parentCheckpoint.index,
+                ...result.inputs.toInspect()
+            });
             const leafLocation = provingState.setBlockRootRollupProof(result);
             const checkpointProvingState = provingState.parentCheckpoint;
+            // Verification is called from both here and setBlockCompleted. Whichever runs last
+            // will be the first to see all three pieces (header, proof output, archive) and run the checks.
+            await this.verifyBuiltBlockAgainstSyncedState(provingState);
             if (checkpointProvingState.totalNumBlocks === 1) {
-                this.checkAndEnqueueCheckpointRootRollup(checkpointProvingState);
+                await this.checkAndEnqueueCheckpointRootRollup(checkpointProvingState);
             } else {
-                this.checkAndEnqueueNextBlockMergeRollup(checkpointProvingState, leafLocation);
+                await this.checkAndEnqueueNextBlockMergeRollup(checkpointProvingState, leafLocation);
             }
         });
     }
@@ -961,11 +938,11 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     // Enqueues the root parity circuit if all inputs are available
     enqueueBaseParityCircuit(checkpointProvingState, provingState, baseParityIndex) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running base parity. State no longer valid.');
+            this.logger.debug('Not running base parity. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingBaseParity(baseParityIndex)) {
-            logger.warn(`Base parity ${baseParityIndex} already started.`);
+            this.logger.warn(`Base parity ${baseParityIndex} already started.`);
             return;
         }
         const inputs = checkpointProvingState.getBaseParityInputs(baseParityIndex);
@@ -986,11 +963,11 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     // Enqueues the root rollup proof if all inputs are available
     enqueueRootParityCircuit(provingState) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running root parity. State no longer valid.');
+            this.logger.debug('Not running root parity. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingRootParity()) {
-            logger.debug('Root parity already started.');
+            this.logger.debug('Root parity already started.');
             return;
         }
         const inputs = provingState.getParityRootInputs();
@@ -1005,33 +982,38 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     // Enqueues the next level of merge if all inputs are available
     enqueueBlockMergeRollup(provingState, location) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running block merge rollup. State no longer valid.');
+            this.logger.debug('Not running block merge rollup. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingBlockMerge(location)) {
-            logger.debug('Block merge rollup already started.');
+            this.logger.debug('Block merge rollup already started.');
             return;
         }
         const inputs = provingState.getBlockMergeRollupInputs(location);
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getBlockMergeRollupProof', {
             [Attributes.PROTOCOL_CIRCUIT_NAME]: 'rollup-block-merge'
-        }, (signal)=>this.prover.getBlockMergeRollupProof(inputs, signal, provingState.epochNumber)), (result)=>{
+        }, (signal)=>this.prover.getBlockMergeRollupProof(inputs, signal, provingState.epochNumber)), async (result)=>{
+            this.logger.debug(`Completed block merge rollup proof for checkpoint ${provingState.index}`, {
+                checkpointIndex: provingState.index,
+                mergeLocation: location,
+                ...result.inputs.toInspect()
+            });
             provingState.setBlockMergeRollupProof(location, result);
-            this.checkAndEnqueueNextBlockMergeRollup(provingState, location);
+            await this.checkAndEnqueueNextBlockMergeRollup(provingState, location);
         });
     }
-    enqueueCheckpointRootRollup(provingState) {
+    async enqueueCheckpointRootRollup(provingState) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running checkpoint root rollup. State no longer valid.');
+            this.logger.debug('Not running checkpoint root rollup. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingCheckpointRoot()) {
-            logger.debug('Checkpoint root rollup already started.');
+            this.logger.debug('Checkpoint root rollup already started.');
             return;
         }
         const rollupType = provingState.getCheckpointRootRollupType();
-        logger.debug(`Enqueuing ${rollupType} for checkpoint ${provingState.index}.`);
-        const inputs = provingState.getCheckpointRootRollupInputs();
+        this.logger.debug(`Enqueuing ${rollupType} for checkpoint ${provingState.index}.`);
+        const inputs = await provingState.getCheckpointRootRollupInputs();
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getCheckpointRootRollupProof', {
             [Attributes.PROTOCOL_CIRCUIT_NAME]: rollupType
         }, (signal)=>{
@@ -1044,11 +1026,14 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
             const computedEndBlobAccumulatorState = provingState.getEndBlobAccumulator().toBlobAccumulator();
             const circuitEndBlobAccumulatorState = result.inputs.endBlobAccumulator;
             if (!circuitEndBlobAccumulatorState.equals(computedEndBlobAccumulatorState)) {
-                logger.error(`Blob accumulator state mismatch.\nCircuit: ${inspect(circuitEndBlobAccumulatorState)}\nComputed: ${inspect(computedEndBlobAccumulatorState)}`);
+                this.logger.error(`Blob accumulator state mismatch.\nCircuit: ${inspect(circuitEndBlobAccumulatorState)}\nComputed: ${inspect(computedEndBlobAccumulatorState)}`);
                 provingState.reject(`Blob accumulator state mismatch.`);
                 return;
             }
-            logger.debug(`Completed ${rollupType} proof for checkpoint ${provingState.index}.`);
+            this.logger.debug(`Completed ${rollupType} proof for checkpoint ${provingState.index}`, {
+                checkpointIndex: provingState.index,
+                ...result.inputs.toInspect()
+            });
             const leafLocation = provingState.setCheckpointRootRollupProof(result);
             const epochProvingState = provingState.parentEpoch;
             if (epochProvingState.totalNumCheckpoints === 1) {
@@ -1058,61 +1043,6 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
             }
         });
     }
-    enqueueCheckpointMergeRollup(provingState, location) {
-        if (!provingState.verifyState()) {
-            logger.debug('Not running checkpoint merge rollup. State no longer valid.');
-            return;
-        }
-        if (!provingState.tryStartProvingCheckpointMerge(location)) {
-            logger.debug('Checkpoint merge rollup already started.');
-            return;
-        }
-        const inputs = provingState.getCheckpointMergeRollupInputs(location);
-        this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getCheckpointMergeRollupProof', {
-            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'rollup-checkpoint-merge'
-        }, (signal)=>this.prover.getCheckpointMergeRollupProof(inputs, signal, provingState.epochNumber)), (result)=>{
-            logger.debug('Completed proof for checkpoint merge rollup.');
-            provingState.setCheckpointMergeRollupProof(location, result);
-            this.checkAndEnqueueNextCheckpointMergeRollup(provingState, location);
-        });
-    }
-    enqueueEpochPadding(provingState) {
-        if (!provingState.verifyState()) {
-            logger.debug('Not running epoch padding. State no longer valid.');
-            return;
-        }
-        if (!provingState.tryStartProvingPaddingCheckpoint()) {
-            logger.debug('Padding checkpoint already started.');
-            return;
-        }
-        logger.debug('Padding epoch proof with a padding block root proof.');
-        const inputs = provingState.getPaddingCheckpointInputs();
-        this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getCheckpointPaddingRollupProof', {
-            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'rollup-checkpoint-padding'
-        }, (signal)=>this.prover.getCheckpointPaddingRollupProof(inputs, signal, provingState.epochNumber)), (result)=>{
-            logger.debug('Completed proof for padding checkpoint.');
-            provingState.setCheckpointPaddingProof(result);
-            this.checkAndEnqueueRootRollup(provingState);
-        });
-    }
-    // Executes the root rollup circuit
-    enqueueRootRollup(provingState) {
-        if (!provingState.verifyState()) {
-            logger.debug('Not running root rollup, state no longer valid');
-            return;
-        }
-        logger.debug(`Preparing root rollup`);
-        const inputs = provingState.getRootRollupInputs();
-        this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getRootRollupProof', {
-            [Attributes.PROTOCOL_CIRCUIT_NAME]: 'rollup-root'
-        }, (signal)=>this.prover.getRootRollupProof(inputs, signal, provingState.epochNumber)), (result)=>{
-            logger.verbose(`Orchestrator completed root rollup for epoch ${provingState.epochNumber}`);
-            provingState.setRootRollupProof(result);
-            provingState.resolve({
-                status: 'success'
-            });
-        });
-    }
     checkAndEnqueueNextMergeRollup(provingState, currentLocation) {
         if (!provingState.isReadyForMergeRollup(currentLocation)) {
             return;
@@ -1126,45 +1056,27 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     }
     checkAndEnqueueBlockRootRollup(provingState) {
         if (!provingState.isReadyForBlockRootRollup()) {
-            logger.debug('Not ready for block root rollup');
+            this.logger.debug('Not ready for block root rollup');
             return;
         }
         this.enqueueBlockRootRollup(provingState);
     }
-    checkAndEnqueueNextBlockMergeRollup(provingState, currentLocation) {
+    async checkAndEnqueueNextBlockMergeRollup(provingState, currentLocation) {
         if (!provingState.isReadyForBlockMerge(currentLocation)) {
             return;
         }
         const parentLocation = provingState.getParentLocation(currentLocation);
         if (parentLocation.level === 0) {
-            this.checkAndEnqueueCheckpointRootRollup(provingState);
+            await this.checkAndEnqueueCheckpointRootRollup(provingState);
         } else {
             this.enqueueBlockMergeRollup(provingState, parentLocation);
         }
     }
-    checkAndEnqueueCheckpointRootRollup(provingState) {
+    async checkAndEnqueueCheckpointRootRollup(provingState) {
         if (!provingState.isReadyForCheckpointRoot()) {
             return;
         }
-        this.enqueueCheckpointRootRollup(provingState);
-    }
-    checkAndEnqueueNextCheckpointMergeRollup(provingState, currentLocation) {
-        if (!provingState.isReadyForCheckpointMerge(currentLocation)) {
-            return;
-        }
-        const parentLocation = provingState.getParentLocation(currentLocation);
-        if (parentLocation.level === 0) {
-            this.checkAndEnqueueRootRollup(provingState);
-        } else {
-            this.enqueueCheckpointMergeRollup(provingState, parentLocation);
-        }
-    }
-    checkAndEnqueueRootRollup(provingState) {
-        if (!provingState.isReadyForRootRollup()) {
-            logger.debug('Not ready for root rollup');
-            return;
-        }
-        this.enqueueRootRollup(provingState);
+        await this.enqueueCheckpointRootRollup(provingState);
     }
     /**
    * Executes the VM circuit for a public function, will enqueue the corresponding kernel if the
@@ -1173,7 +1085,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
    * @param txIndex - The index of the transaction being proven
    */ enqueueVM(provingState, txIndex) {
         if (!provingState.verifyState()) {
-            logger.debug(`Not running VM circuit as state is no longer valid`);
+            this.logger.debug(`Not running VM circuit as state is no longer valid`);
             return;
         }
         const txProvingState = provingState.getTxProvingState(txIndex);
@@ -1184,7 +1096,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
             return await this.prover.getAvmProof(inputs, signal, provingState.epochNumber);
         });
         this.deferredProving(provingState, doAvmProving, (proof)=>{
-            logger.debug(`Proven VM for tx index: ${txIndex}`);
+            this.logger.debug(`Proven VM for tx index: ${txIndex}`);
             txProvingState.setAvmProof(proof);
             this.checkAndEnqueueBaseRollup(provingState, txIndex);
         });
@@ -1195,7 +1107,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
             return;
         }
         // We must have completed all proving (chonk verifier proof and (if required) vm proof are generated), we now move to the base rollup.
-        logger.debug(`Public functions completed for tx ${txIndex} enqueueing base rollup`);
+        this.logger.debug(`Public functions completed for tx ${txIndex} enqueueing base rollup`);
         this.enqueueBaseRollup(provingState, txIndex);
     }
 }