@aztec/prover-client 0.0.1-commit.6d3c34e → 0.0.1-commit.7035c9bd6

package/dest/orchestrator/orchestrator.js

@@ -378,7 +378,9 @@ import { Fr } from '@aztec/foundation/curves/bn254';
 import { AbortError } from '@aztec/foundation/error';
 import { createLogger } from '@aztec/foundation/log';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { SerialQueue } from '@aztec/foundation/queue';
 import { assertLength } from '@aztec/foundation/serialize';
+import { sleep } from '@aztec/foundation/sleep';
 import { pushTestData } from '@aztec/foundation/testing';
 import { elapsed } from '@aztec/foundation/timer';
 import { BlockRootEmptyTxFirstRollupPrivateInputs, BlockRootFirstRollupPrivateInputs, BlockRootSingleTxFirstRollupPrivateInputs, BlockRootSingleTxRollupPrivateInputs, CheckpointRootSingleBlockRollupPrivateInputs, PrivateTxBaseRollupPrivateInputs } from '@aztec/stdlib/rollup';
@@ -389,7 +391,6 @@ import { buildHeaderFromCircuitOutputs, getLastSiblingPath, getPublicChonkVerifi
 import { EpochProvingState } from './epoch-proving-state.js';
 import { ProvingOrchestratorMetrics } from './orchestrator_metrics.js';
 import { TxProvingState } from './tx-proving-state.js';
-const logger = createLogger('prover-client:orchestrator');
 _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
         [Attributes.BLOCK_NUMBER]: blockNumber
     })), _dec1 = trackSpan('ProvingOrchestrator.addTxs', (txs)=>({
@@ -414,6 +415,8 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     dbProvider;
     prover;
     proverId;
+    cancelJobsOnStop;
+    enqueueConcurrency;
     static{
         ({ e: [_initProto] } = _apply_decs_2203_r(this, [
             [
@@ -449,15 +452,22 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     metrics;
     // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
     dbs;
-    constructor(dbProvider, prover, proverId, telemetryClient = getTelemetryClient()){
+    logger;
+    deferredJobQueue;
+    constructor(dbProvider, prover, proverId, cancelJobsOnStop = false, enqueueConcurrency, telemetryClient = getTelemetryClient(), bindings){
         this.dbProvider = dbProvider;
         this.prover = prover;
         this.proverId = proverId;
+        this.cancelJobsOnStop = cancelJobsOnStop;
+        this.enqueueConcurrency = enqueueConcurrency;
         this.provingState = (_initProto(this), undefined);
         this.pendingProvingJobs = [];
         this.provingPromise = undefined;
         this.dbs = new Map();
+        this.deferredJobQueue = new SerialQueue();
+        this.logger = createLogger('prover-client:orchestrator', bindings);
         this.metrics = new ProvingOrchestratorMetrics(telemetryClient, 'ProvingOrchestrator');
+        this.deferredJobQueue.start(this.enqueueConcurrency);
     }
     get tracer() {
         return this.metrics.tracer;
@@ -465,9 +475,14 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     getProverId() {
         return this.proverId;
     }
-    stop() {
+    getNumActiveForks() {
+        return this.dbs.size;
+    }
+    async stop() {
+        // Grab the old queue before cancel() replaces it, so we can await its draining.
+        const oldQueue = this.deferredJobQueue;
         this.cancel();
-        return Promise.resolve();
+        await oldQueue.cancel();
     }
     startNewEpoch(epochNumber, totalNumCheckpoints, finalBlobBatchingChallenges) {
         if (this.provingState?.verifyState()) {
@@ -478,11 +493,18 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
                 status: 'failure',
                 reason
             }));
-        logger.info(`Starting epoch ${epochNumber} with ${totalNumCheckpoints} checkpoints.`);
+        this.logger.info(`Starting epoch ${epochNumber} with ${totalNumCheckpoints} checkpoints.`);
         this.provingState = new EpochProvingState(epochNumber, totalNumCheckpoints, finalBlobBatchingChallenges, (provingState)=>this.checkAndEnqueueCheckpointRootRollup(provingState), resolve, reject);
         this.provingPromise = promise;
     }
-    async startNewCheckpoint(checkpointIndex, constants, l1ToL2Messages, totalNumBlocks, headerOfLastBlockInPreviousCheckpoint) {
+    /**
+   * Starts a new checkpoint.
+   * @param checkpointIndex - The index of the checkpoint in the epoch.
+   * @param constants - The constants for this checkpoint.
+   * @param l1ToL2Messages - The set of L1 to L2 messages to be inserted at the beginning of this checkpoint.
+   * @param totalNumBlocks - The total number of blocks expected in the checkpoint (must be at least one).
+   * @param headerOfLastBlockInPreviousCheckpoint - The header of the last block in the previous checkpoint.
+   */ async startNewCheckpoint(checkpointIndex, constants, l1ToL2Messages, totalNumBlocks, headerOfLastBlockInPreviousCheckpoint) {
         if (!this.provingState) {
             throw new Error('Empty epoch proving state. Call startNewEpoch before starting a checkpoint.');
         }
@@ -518,14 +540,14 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
             throw new Error(`Checkpoint not accepting further blocks.`);
         }
         const constants = checkpointProvingState.constants;
-        logger.info(`Starting block ${blockNumber} for slot ${constants.slotNumber}.`);
+        this.logger.info(`Starting block ${blockNumber} for slot ${constants.slotNumber}.`);
         // Fork the db only when it's not already set. The db for the first block is set in `startNewCheckpoint`.
         if (!this.dbs.has(blockNumber)) {
             // Fork world state at the end of the immediately previous block
             const db = await this.dbProvider.fork(BlockNumber(blockNumber - 1));
             this.dbs.set(blockNumber, db);
         }
-        const db = this.dbs.get(blockNumber);
+        const db = this.getDbForBlock(blockNumber);
         // Get archive snapshot and sibling path before any txs in this block lands.
         const lastArchiveTreeSnapshot = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
         const lastArchiveSiblingPath = await getRootTreeSiblingPath(MerkleTreeId.ARCHIVE, db);
@@ -545,7 +567,8 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
             const blockEndBlobFields = blockProvingState.getBlockEndBlobFields();
             await endSpongeBlob.absorb(blockEndBlobFields);
             blockProvingState.setEndSpongeBlob(endSpongeBlob);
-            // And also try to accumulate the blobs as far as we can:
+            // Try to accumulate the out hashes and blobs as far as we can:
+            await this.provingState.accumulateCheckpointOutHashes();
             await this.provingState.setBlobAccumulators();
         }
     }
@@ -559,7 +582,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
         if (!txs.length) {
             // To avoid an ugly throw below. If we require an empty block, we can just call setBlockCompleted
             // on a block with no txs. We cannot do that here because we cannot find the blockNumber without any txs.
-            logger.warn(`Provided no txs to orchestrator addTxs.`);
+            this.logger.warn(`Provided no txs to orchestrator addTxs.`);
             return;
         }
         const blockNumber = BlockNumber(txs[0].globalVariables.blockNumber);
@@ -573,8 +596,8 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
         if (!provingState.isAcceptingTxs()) {
             throw new Error(`Block ${blockNumber} has been initialized with transactions.`);
         }
-        logger.info(`Adding ${txs.length} transactions to block ${blockNumber}`);
-        const db = this.dbs.get(blockNumber);
+        this.logger.info(`Adding ${txs.length} transactions to block ${blockNumber}`);
+        const db = this.getDbForBlock(blockNumber);
         const lastArchive = provingState.lastArchiveTreeSnapshot;
         const newL1ToL2MessageTreeSnapshot = provingState.newL1ToL2MessageTreeSnapshot;
         const spongeBlobState = provingState.getStartSpongeBlob().clone();
@@ -584,7 +607,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
                     throw new Error(`Invalid proving state when adding a tx`);
                 }
                 validateTx(tx);
-                logger.debug(`Received transaction: ${tx.hash}`);
+                this.logger.debug(`Received transaction: ${tx.hash}`);
                 const startSpongeBlob = spongeBlobState.clone();
                 const [hints, treeSnapshots] = await this.prepareBaseRollupInputs(tx, lastArchive, newL1ToL2MessageTreeSnapshot, startSpongeBlob, db);
                 if (!provingState.verifyState()) {
@@ -595,10 +618,10 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
                 const txIndex = provingState.addNewTx(txProvingState);
                 if (txProvingState.requireAvmProof) {
                     this.getOrEnqueueChonkVerifier(provingState, txIndex);
-                    logger.debug(`Enqueueing public VM for tx ${txIndex}`);
+                    this.logger.debug(`Enqueueing public VM for tx ${txIndex}`);
                     this.enqueueVM(provingState, txIndex);
                 } else {
-                    logger.debug(`Enqueueing base rollup for private-only tx ${txIndex}`);
+                    this.logger.debug(`Enqueueing base rollup for private-only tx ${txIndex}`);
                     this.enqueueBaseRollup(provingState, txIndex);
                 }
             } catch (err) {
@@ -612,7 +635,8 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
         const blockEndBlobFields = provingState.getBlockEndBlobFields();
         await spongeBlobState.absorb(blockEndBlobFields);
         provingState.setEndSpongeBlob(spongeBlobState);
-        // Txs have been added to the block. Now try to accumulate the blobs as far as we can:
+        // Txs have been added to the block. Now try to accumulate the out hashes and blobs as far as we can:
+        await this.provingState.accumulateCheckpointOutHashes();
         await this.provingState.setBlobAccumulators();
     }
     /**
@@ -627,7 +651,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
             const txHash = tx.getTxHash().toString();
             const privateInputs = getPublicChonkVerifierPrivateInputsFromTx(tx, this.proverId.toField());
             const tubeProof = promiseWithResolvers();
-            logger.debug(`Starting chonk verifier circuit for tx ${txHash}`);
+            this.logger.debug(`Starting chonk verifier circuit for tx ${txHash}`);
             this.doEnqueueChonkVerifier(txHash, privateInputs, (proof)=>{
                 tubeProof.resolve(proof);
             });
@@ -656,17 +680,23 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
             throw new Error(`Block ${blockNumber} is still accepting txs. Call setBlockCompleted after all txs have been added.`);
         }
         // Given we've applied every change from this block, now assemble the block header:
-        logger.verbose(`Block ${blockNumber} completed. Assembling header.`);
+        this.logger.verbose(`Block ${blockNumber} completed. Assembling header.`);
         const header = await provingState.buildBlockHeader();
         if (expectedHeader && !header.equals(expectedHeader)) {
-            logger.error(`Block header mismatch: header=${header} expectedHeader=${expectedHeader}`);
+            this.logger.error(`Block header mismatch: header=${header} expectedHeader=${expectedHeader}`);
             throw new Error('Block header mismatch');
         }
-        // Get db for this block
-        const db = this.dbs.get(provingState.blockNumber);
-        // Update the archive tree, so we're ready to start processing the next block:
-        logger.verbose(`Updating archive tree with block ${provingState.blockNumber} header ${(await header.hash()).toString()}`);
-        await db.updateArchive(header);
+        // Get db for this block and remove from map — no other code should use it after this point.
+        const db = this.getDbForBlock(provingState.blockNumber);
+        this.dbs.delete(provingState.blockNumber);
+        // Update the archive tree, capture the snapshot, and close the fork deterministically.
+        try {
+            this.logger.verbose(`Updating archive tree with block ${provingState.blockNumber} header ${(await header.hash()).toString()}`);
+            await db.updateArchive(header);
+            provingState.setBuiltArchive(await getTreeSnapshot(MerkleTreeId.ARCHIVE, db));
+        } finally{
+            await db.close();
+        }
         await this.verifyBuiltBlockAgainstSyncedState(provingState);
         return header;
     }
@@ -674,50 +704,65 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     async verifyBuiltBlockAgainstSyncedState(provingState) {
         const builtBlockHeader = provingState.getBuiltBlockHeader();
         if (!builtBlockHeader) {
-            logger.debug('Block header not built yet, skipping header check.');
+            this.logger.debug('Block header not built yet, skipping header check.');
             return;
         }
         const output = provingState.getBlockRootRollupOutput();
         if (!output) {
-            logger.debug('Block root rollup proof not built yet, skipping header check.');
+            this.logger.debug('Block root rollup proof not built yet, skipping header check.');
+            return;
+        }
+        const newArchive = provingState.getBuiltArchive();
+        if (!newArchive) {
+            this.logger.debug('Archive snapshot not yet captured, skipping header check.');
             return;
         }
         const header = await buildHeaderFromCircuitOutputs(output);
         if (!(await header.hash()).equals(await builtBlockHeader.hash())) {
-            logger.error(`Block header mismatch.\nCircuit: ${inspect(header)}\nComputed: ${inspect(builtBlockHeader)}`);
+            this.logger.error(`Block header mismatch.\nCircuit: ${inspect(header)}\nComputed: ${inspect(builtBlockHeader)}`);
             provingState.reject(`Block header hash mismatch.`);
             return;
         }
-        // Get db for this block
         const blockNumber = provingState.blockNumber;
-        const db = this.dbs.get(blockNumber);
-        const newArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
         const syncedArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.dbProvider.getSnapshot(blockNumber));
         if (!syncedArchive.equals(newArchive)) {
-            logger.error(`Archive tree mismatch for block ${blockNumber}: world state synced to ${inspect(syncedArchive)} but built ${inspect(newArchive)}`);
+            this.logger.error(`Archive tree mismatch for block ${blockNumber}: world state synced to ${inspect(syncedArchive)} but built ${inspect(newArchive)}`);
             provingState.reject(`Archive tree mismatch.`);
             return;
         }
         const circuitArchive = output.newArchive;
         if (!newArchive.equals(circuitArchive)) {
-            logger.error(`New archive mismatch.\nCircuit: ${output.newArchive}\nComputed: ${newArchive}`);
+            this.logger.error(`New archive mismatch.\nCircuit: ${output.newArchive}\nComputed: ${newArchive}`);
             provingState.reject(`New archive mismatch.`);
             return;
         }
-        // TODO(palla/prover): This closes the fork only on the happy path. If this epoch orchestrator
-        // is aborted and never reaches this point, it will leak the fork. We need to add a global cleanup,
-        // but have to make sure it only runs once all operations are completed, otherwise some function here
-        // will attempt to access the fork after it was closed.
-        logger.debug(`Cleaning up world state fork for ${blockNumber}`);
-        void this.dbs.get(blockNumber)?.close().then(()=>this.dbs.delete(blockNumber)).catch((err)=>logger.error(`Error closing db for block ${blockNumber}`, err));
     }
     /**
-   * Cancel any further proving
+   * Cancel any further proving.
+   * If cancelJobsOnStop is true, aborts all pending jobs with the broker (which marks them as 'Aborted').
+   * If cancelJobsOnStop is false (default), jobs remain in the broker queue and can be reused on restart/reorg.
    */ cancel() {
-        for (const controller of this.pendingProvingJobs){
-            controller.abort();
+        void this.deferredJobQueue.cancel();
+        // Recreate the queue so it can accept jobs for subsequent epochs.
+        this.deferredJobQueue = new SerialQueue();
+        this.deferredJobQueue.start(this.enqueueConcurrency);
+        if (this.cancelJobsOnStop) {
+            for (const controller of this.pendingProvingJobs){
+                controller.abort();
+            }
         }
         this.provingState?.cancel();
+        for (const [blockNumber, db] of this.dbs.entries()){
+            void db.close().catch((err)=>this.logger.error(`Error closing db for block ${blockNumber}`, err));
+        }
+        this.dbs.clear();
+    }
+    getDbForBlock(blockNumber) {
+        const db = this.dbs.get(blockNumber);
+        if (!db) {
+            throw new Error(`World state fork for block ${blockNumber} not found.`);
+        }
+        return db;
     }
     /**
    * Returns the proof for the current epoch.
@@ -744,7 +789,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
    * @param job - The actual job, returns a promise notifying of the job's completion
    */ deferredProving(provingState, request, callback) {
         if (!provingState.verifyState()) {
-            logger.debug(`Not enqueuing job, state no longer valid`);
+            this.logger.debug(`Not enqueuing job, state no longer valid`);
             return;
         }
         const controller = new AbortController();
@@ -759,7 +804,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
                 }
                 const result = await request(controller.signal);
                 if (!provingState.verifyState()) {
-                    logger.debug(`State no longer valid, discarding result`);
+                    this.logger.debug(`State no longer valid, discarding result`);
                     return;
                 }
                 // we could have been cancelled whilst waiting for the result
@@ -774,7 +819,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
                     // drop this result
                     return;
                 }
-                logger.error(`Error thrown when proving job`, err);
+                this.logger.error(`Error thrown when proving job`, err);
                 provingState.reject(`${err}`);
             } finally{
                 const index = this.pendingProvingJobs.indexOf(controller);
@@ -783,8 +828,11 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
                 }
             }
         };
-        // let the callstack unwind before adding the job to the queue
-        setImmediate(()=>void safeJob());
+        void this.deferredJobQueue.put(async ()=>{
+            void safeJob();
+            // we yield here to the macro task queue such to give Nodejs a chance to run other operatoins in between enqueues
+            await sleep(0);
+        });
     }
     async updateL1ToL2MessageTree(l1ToL2Messages, db) {
         const l1ToL2MessagesPadded = padArrayEnd(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 'Too many L1 to L2 messages');
@@ -830,17 +878,17 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     // Executes the next level of merge if all inputs are available
     enqueueBaseRollup(provingState, txIndex) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running base rollup, state invalid');
+            this.logger.debug('Not running base rollup, state invalid');
             return;
         }
         if (!provingState.tryStartProvingBase(txIndex)) {
-            logger.debug(`Base rollup for tx ${txIndex} already started.`);
+            this.logger.debug(`Base rollup for tx ${txIndex} already started.`);
             return;
         }
         const txProvingState = provingState.getTxProvingState(txIndex);
         const { processedTx } = txProvingState;
         const { rollupType, inputs } = txProvingState.getBaseRollupTypeAndInputs();
-        logger.debug(`Enqueuing deferred proving base rollup for ${processedTx.hash.toString()}`);
+        this.logger.debug(`Enqueuing deferred proving base rollup for ${processedTx.hash.toString()}`);
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, `ProvingOrchestrator.prover.${inputs instanceof PrivateTxBaseRollupPrivateInputs ? 'getPrivateTxBaseRollupProof' : 'getPublicTxBaseRollupProof'}`, {
             [Attributes.TX_HASH]: processedTx.hash.toString(),
             [Attributes.PROTOCOL_CIRCUIT_NAME]: rollupType
@@ -851,7 +899,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
                 return this.prover.getPublicTxBaseRollupProof(inputs, signal, provingState.epochNumber);
             }
         }), (result)=>{
-            logger.debug(`Completed proof for ${rollupType} for tx ${processedTx.hash.toString()}`);
+            this.logger.debug(`Completed proof for ${rollupType} for tx ${processedTx.hash.toString()}`);
             validatePartialState(result.inputs.endTreeSnapshots, txProvingState.treeSnapshots);
             const leafLocation = provingState.setBaseRollupProof(txIndex, result);
             if (provingState.totalNumTxs === 1) {
@@ -865,14 +913,14 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     // Once completed, will enqueue the the public tx base rollup.
     getOrEnqueueChonkVerifier(provingState, txIndex) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running chonk verifier circuit, state invalid');
+            this.logger.debug('Not running chonk verifier circuit, state invalid');
             return;
         }
         const txProvingState = provingState.getTxProvingState(txIndex);
         const txHash = txProvingState.processedTx.hash.toString();
         NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH;
         const handleResult = (result)=>{
-            logger.debug(`Got chonk verifier proof for tx index: ${txIndex}`, {
+            this.logger.debug(`Got chonk verifier proof for tx index: ${txIndex}`, {
                 txHash
             });
             txProvingState.setPublicChonkVerifierProof(result);
@@ -880,18 +928,18 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
             this.checkAndEnqueueBaseRollup(provingState, txIndex);
         };
         if (this.provingState?.cachedChonkVerifierProofs.has(txHash)) {
-            logger.debug(`Chonk verifier proof already enqueued for tx index: ${txIndex}`, {
+            this.logger.debug(`Chonk verifier proof already enqueued for tx index: ${txIndex}`, {
                 txHash
             });
             void this.provingState.cachedChonkVerifierProofs.get(txHash).then(handleResult);
             return;
         }
-        logger.debug(`Enqueuing chonk verifier circuit for tx index: ${txIndex}`);
+        this.logger.debug(`Enqueuing chonk verifier circuit for tx index: ${txIndex}`);
         this.doEnqueueChonkVerifier(txHash, txProvingState.getPublicChonkVerifierPrivateInputs(), handleResult);
     }
     doEnqueueChonkVerifier(txHash, inputs, handler, provingState = this.provingState) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running chonk verifier circuit, state invalid');
+            this.logger.debug('Not running chonk verifier circuit, state invalid');
             return;
         }
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getPublicChonkVerifierProof', {
@@ -903,11 +951,11 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     // Enqueues the next level of merge if all inputs are available
     enqueueMergeRollup(provingState, location) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running merge rollup. State no longer valid.');
+            this.logger.debug('Not running merge rollup. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingMerge(location)) {
-            logger.debug('Merge rollup already started.');
+            this.logger.debug('Merge rollup already started.');
             return;
         }
         const inputs = provingState.getMergeRollupInputs(location);
@@ -921,15 +969,15 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     // Executes the block root rollup circuit
     enqueueBlockRootRollup(provingState) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running block root rollup, state no longer valid');
+            this.logger.debug('Not running block root rollup, state no longer valid');
             return;
         }
         if (!provingState.tryStartProvingBlockRoot()) {
-            logger.debug('Block root rollup already started.');
+            this.logger.debug('Block root rollup already started.');
             return;
         }
         const { rollupType, inputs } = provingState.getBlockRootRollupTypeAndInputs();
-        logger.debug(`Enqueuing ${rollupType} for block ${provingState.blockNumber}.`);
+        this.logger.debug(`Enqueuing ${rollupType} for block ${provingState.blockNumber}.`);
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getBlockRootRollupProof', {
             [Attributes.PROTOCOL_CIRCUIT_NAME]: rollupType
         }, (signal)=>{
@@ -945,15 +993,16 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
                 return this.prover.getBlockRootRollupProof(inputs, signal, provingState.epochNumber);
             }
         }), async (result)=>{
-            // If the proofs were slower than the block header building, then we need to try validating the block header hashes here.
-            await this.verifyBuiltBlockAgainstSyncedState(provingState);
-            logger.debug(`Completed ${rollupType} proof for block ${provingState.blockNumber}`);
+            this.logger.debug(`Completed ${rollupType} proof for block ${provingState.blockNumber}`);
             const leafLocation = provingState.setBlockRootRollupProof(result);
             const checkpointProvingState = provingState.parentCheckpoint;
+            // Verification is called from both here and setBlockCompleted. Whichever runs last
+            // will be the first to see all three pieces (header, proof output, archive) and run the checks.
+            await this.verifyBuiltBlockAgainstSyncedState(provingState);
             if (checkpointProvingState.totalNumBlocks === 1) {
-                this.checkAndEnqueueCheckpointRootRollup(checkpointProvingState);
+                await this.checkAndEnqueueCheckpointRootRollup(checkpointProvingState);
             } else {
-                this.checkAndEnqueueNextBlockMergeRollup(checkpointProvingState, leafLocation);
+                await this.checkAndEnqueueNextBlockMergeRollup(checkpointProvingState, leafLocation);
             }
         });
     }
@@ -961,11 +1010,11 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     // Enqueues the root parity circuit if all inputs are available
     enqueueBaseParityCircuit(checkpointProvingState, provingState, baseParityIndex) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running base parity. State no longer valid.');
+            this.logger.debug('Not running base parity. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingBaseParity(baseParityIndex)) {
-            logger.warn(`Base parity ${baseParityIndex} already started.`);
+            this.logger.warn(`Base parity ${baseParityIndex} already started.`);
             return;
         }
         const inputs = checkpointProvingState.getBaseParityInputs(baseParityIndex);
@@ -986,11 +1035,11 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     // Enqueues the root rollup proof if all inputs are available
     enqueueRootParityCircuit(provingState) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running root parity. State no longer valid.');
+            this.logger.debug('Not running root parity. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingRootParity()) {
-            logger.debug('Root parity already started.');
+            this.logger.debug('Root parity already started.');
             return;
         }
         const inputs = provingState.getParityRootInputs();
@@ -1005,33 +1054,33 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     // Enqueues the next level of merge if all inputs are available
     enqueueBlockMergeRollup(provingState, location) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running block merge rollup. State no longer valid.');
+            this.logger.debug('Not running block merge rollup. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingBlockMerge(location)) {
-            logger.debug('Block merge rollup already started.');
+            this.logger.debug('Block merge rollup already started.');
             return;
         }
         const inputs = provingState.getBlockMergeRollupInputs(location);
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getBlockMergeRollupProof', {
             [Attributes.PROTOCOL_CIRCUIT_NAME]: 'rollup-block-merge'
-        }, (signal)=>this.prover.getBlockMergeRollupProof(inputs, signal, provingState.epochNumber)), (result)=>{
+        }, (signal)=>this.prover.getBlockMergeRollupProof(inputs, signal, provingState.epochNumber)), async (result)=>{
             provingState.setBlockMergeRollupProof(location, result);
-            this.checkAndEnqueueNextBlockMergeRollup(provingState, location);
+            await this.checkAndEnqueueNextBlockMergeRollup(provingState, location);
         });
     }
-    enqueueCheckpointRootRollup(provingState) {
+    async enqueueCheckpointRootRollup(provingState) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running checkpoint root rollup. State no longer valid.');
+            this.logger.debug('Not running checkpoint root rollup. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingCheckpointRoot()) {
-            logger.debug('Checkpoint root rollup already started.');
+            this.logger.debug('Checkpoint root rollup already started.');
             return;
         }
         const rollupType = provingState.getCheckpointRootRollupType();
-        logger.debug(`Enqueuing ${rollupType} for checkpoint ${provingState.index}.`);
-        const inputs = provingState.getCheckpointRootRollupInputs();
+        this.logger.debug(`Enqueuing ${rollupType} for checkpoint ${provingState.index}.`);
+        const inputs = await provingState.getCheckpointRootRollupInputs();
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getCheckpointRootRollupProof', {
             [Attributes.PROTOCOL_CIRCUIT_NAME]: rollupType
         }, (signal)=>{
@@ -1044,11 +1093,11 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
             const computedEndBlobAccumulatorState = provingState.getEndBlobAccumulator().toBlobAccumulator();
             const circuitEndBlobAccumulatorState = result.inputs.endBlobAccumulator;
             if (!circuitEndBlobAccumulatorState.equals(computedEndBlobAccumulatorState)) {
-                logger.error(`Blob accumulator state mismatch.\nCircuit: ${inspect(circuitEndBlobAccumulatorState)}\nComputed: ${inspect(computedEndBlobAccumulatorState)}`);
+                this.logger.error(`Blob accumulator state mismatch.\nCircuit: ${inspect(circuitEndBlobAccumulatorState)}\nComputed: ${inspect(computedEndBlobAccumulatorState)}`);
                 provingState.reject(`Blob accumulator state mismatch.`);
                 return;
             }
-            logger.debug(`Completed ${rollupType} proof for checkpoint ${provingState.index}.`);
+            this.logger.debug(`Completed ${rollupType} proof for checkpoint ${provingState.index}.`);
             const leafLocation = provingState.setCheckpointRootRollupProof(result);
             const epochProvingState = provingState.parentEpoch;
             if (epochProvingState.totalNumCheckpoints === 1) {
@@ -1060,37 +1109,37 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     }
     enqueueCheckpointMergeRollup(provingState, location) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running checkpoint merge rollup. State no longer valid.');
+            this.logger.debug('Not running checkpoint merge rollup. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingCheckpointMerge(location)) {
-            logger.debug('Checkpoint merge rollup already started.');
+            this.logger.debug('Checkpoint merge rollup already started.');
             return;
         }
         const inputs = provingState.getCheckpointMergeRollupInputs(location);
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getCheckpointMergeRollupProof', {
             [Attributes.PROTOCOL_CIRCUIT_NAME]: 'rollup-checkpoint-merge'
         }, (signal)=>this.prover.getCheckpointMergeRollupProof(inputs, signal, provingState.epochNumber)), (result)=>{
-            logger.debug('Completed proof for checkpoint merge rollup.');
+            this.logger.debug('Completed proof for checkpoint merge rollup.');
             provingState.setCheckpointMergeRollupProof(location, result);
             this.checkAndEnqueueNextCheckpointMergeRollup(provingState, location);
         });
     }
     enqueueEpochPadding(provingState) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running epoch padding. State no longer valid.');
+            this.logger.debug('Not running epoch padding. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingPaddingCheckpoint()) {
-            logger.debug('Padding checkpoint already started.');
+            this.logger.debug('Padding checkpoint already started.');
             return;
         }
-        logger.debug('Padding epoch proof with a padding block root proof.');
+        this.logger.debug('Padding epoch proof with a padding block root proof.');
         const inputs = provingState.getPaddingCheckpointInputs();
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getCheckpointPaddingRollupProof', {
             [Attributes.PROTOCOL_CIRCUIT_NAME]: 'rollup-checkpoint-padding'
         }, (signal)=>this.prover.getCheckpointPaddingRollupProof(inputs, signal, provingState.epochNumber)), (result)=>{
-            logger.debug('Completed proof for padding checkpoint.');
+            this.logger.debug('Completed proof for padding checkpoint.');
             provingState.setCheckpointPaddingProof(result);
             this.checkAndEnqueueRootRollup(provingState);
         });
@@ -1098,15 +1147,15 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     // Executes the root rollup circuit
     enqueueRootRollup(provingState) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running root rollup, state no longer valid');
+            this.logger.debug('Not running root rollup, state no longer valid');
             return;
         }
-        logger.debug(`Preparing root rollup`);
+        this.logger.debug(`Preparing root rollup`);
         const inputs = provingState.getRootRollupInputs();
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getRootRollupProof', {
             [Attributes.PROTOCOL_CIRCUIT_NAME]: 'rollup-root'
         }, (signal)=>this.prover.getRootRollupProof(inputs, signal, provingState.epochNumber)), (result)=>{
-            logger.verbose(`Orchestrator completed root rollup for epoch ${provingState.epochNumber}`);
+            this.logger.verbose(`Orchestrator completed root rollup for epoch ${provingState.epochNumber}`);
             provingState.setRootRollupProof(result);
             provingState.resolve({
                 status: 'success'
@@ -1126,27 +1175,27 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     }
     checkAndEnqueueBlockRootRollup(provingState) {
         if (!provingState.isReadyForBlockRootRollup()) {
-            logger.debug('Not ready for block root rollup');
+            this.logger.debug('Not ready for block root rollup');
             return;
         }
         this.enqueueBlockRootRollup(provingState);
     }
-    checkAndEnqueueNextBlockMergeRollup(provingState, currentLocation) {
+    async checkAndEnqueueNextBlockMergeRollup(provingState, currentLocation) {
         if (!provingState.isReadyForBlockMerge(currentLocation)) {
             return;
         }
         const parentLocation = provingState.getParentLocation(currentLocation);
         if (parentLocation.level === 0) {
-            this.checkAndEnqueueCheckpointRootRollup(provingState);
+            await this.checkAndEnqueueCheckpointRootRollup(provingState);
         } else {
             this.enqueueBlockMergeRollup(provingState, parentLocation);
         }
     }
-    checkAndEnqueueCheckpointRootRollup(provingState) {
+    async checkAndEnqueueCheckpointRootRollup(provingState) {
         if (!provingState.isReadyForCheckpointRoot()) {
             return;
         }
-        this.enqueueCheckpointRootRollup(provingState);
+        await this.enqueueCheckpointRootRollup(provingState);
     }
     checkAndEnqueueNextCheckpointMergeRollup(provingState, currentLocation) {
         if (!provingState.isReadyForCheckpointMerge(currentLocation)) {
@@ -1161,7 +1210,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
     }
     checkAndEnqueueRootRollup(provingState) {
         if (!provingState.isReadyForRootRollup()) {
-            logger.debug('Not ready for root rollup');
+            this.logger.debug('Not ready for root rollup');
             return;
         }
         this.enqueueRootRollup(provingState);
@@ -1173,7 +1222,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
    * @param txIndex - The index of the transaction being proven
    */ enqueueVM(provingState, txIndex) {
         if (!provingState.verifyState()) {
-            logger.debug(`Not running VM circuit as state is no longer valid`);
+            this.logger.debug(`Not running VM circuit as state is no longer valid`);
             return;
         }
         const txProvingState = provingState.getTxProvingState(txIndex);
@@ -1184,7 +1233,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
             return await this.prover.getAvmProof(inputs, signal, provingState.epochNumber);
         });
         this.deferredProving(provingState, doAvmProving, (proof)=>{
-            logger.debug(`Proven VM for tx index: ${txIndex}`);
+            this.logger.debug(`Proven VM for tx index: ${txIndex}`);
             txProvingState.setAvmProof(proof);
             this.checkAndEnqueueBaseRollup(provingState, txIndex);
         });
@@ -1195,7 +1244,7 @@ _dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
             return;
         }
         // We must have completed all proving (chonk verifier proof and (if required) vm proof are generated), we now move to the base rollup.
-        logger.debug(`Public functions completed for tx ${txIndex} enqueueing base rollup`);
+        this.logger.debug(`Public functions completed for tx ${txIndex} enqueueing base rollup`);
         this.enqueueBaseRollup(provingState, txIndex);
     }
 }