@aztec/prover-client 0.0.1-commit.9b94fc1 → 0.0.1-commit.9badcec54

package/src/orchestrator/orchestrator.ts

@@ -6,23 +6,25 @@ import {
   NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP,
   NUM_BASE_PARITY_PER_ROOT_PARITY,
 } from '@aztec/constants';
-import { EpochNumber } from '@aztec/foundation/branded-types';
+import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
 import { padArrayEnd } from '@aztec/foundation/collection';
+import { Fr } from '@aztec/foundation/curves/bn254';
 import { AbortError } from '@aztec/foundation/error';
-import { Fr } from '@aztec/foundation/fields';
-import { createLogger } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { SerialQueue } from '@aztec/foundation/queue';
 import { assertLength } from '@aztec/foundation/serialize';
+import { sleep } from '@aztec/foundation/sleep';
 import { pushTestData } from '@aztec/foundation/testing';
 import { elapsed } from '@aztec/foundation/timer';
 import type { TreeNodeLocation } from '@aztec/foundation/trees';
-import { readAvmMinimalPublicTxInputsFromFile } from '@aztec/simulator/public/fixtures';
 import { EthAddress } from '@aztec/stdlib/block';
 import type {
   EpochProver,
   ForkMerkleTreeOperations,
   MerkleTreeWriteOperations,
   PublicInputsAndRecursiveProof,
+  ReadonlyWorldStateAccess,
   ServerCircuitProver,
 } from '@aztec/stdlib/interfaces/server';
 import type { Proof } from '@aztec/stdlib/proofs';
@@ -71,8 +73,6 @@ import { EpochProvingState, type ProvingResult, type TreeSnapshots } from './epo
 import { ProvingOrchestratorMetrics } from './orchestrator_metrics.js';
 import { TxProvingState } from './tx-proving-state.js';
 
-const logger = createLogger('prover-client:orchestrator');
-
 /**
  * Implements an event driven proving scheduler to build the recursive proof tree. The idea being:
  * 1. Transactions are provided to the scheduler post simulation.
@@ -93,15 +93,23 @@ export class ProvingOrchestrator implements EpochProver {
 
   private provingPromise: Promise<ProvingResult> | undefined = undefined;
   private metrics: ProvingOrchestratorMetrics;
-  private dbs: Map<number, MerkleTreeWriteOperations> = new Map();
+  // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
+  private dbs: Map<BlockNumber, MerkleTreeWriteOperations> = new Map();
+  private logger: Logger;
+  private deferredJobQueue = new SerialQueue();
 
   constructor(
-    private dbProvider: ForkMerkleTreeOperations,
+    private dbProvider: ReadonlyWorldStateAccess & ForkMerkleTreeOperations,
     private prover: ServerCircuitProver,
     private readonly proverId: EthAddress,
+    private readonly cancelJobsOnStop: boolean = false,
+    private readonly enqueueConcurrency: number,
     telemetryClient: TelemetryClient = getTelemetryClient(),
+    bindings?: LoggerBindings,
   ) {
+    this.logger = createLogger('prover-client:orchestrator', bindings);
     this.metrics = new ProvingOrchestratorMetrics(telemetryClient, 'ProvingOrchestrator');
+    this.deferredJobQueue.start(this.enqueueConcurrency);
   }
 
   get tracer(): Tracer {
@@ -112,9 +120,15 @@ export class ProvingOrchestrator implements EpochProver {
     return this.proverId;
   }
 
-  public stop(): Promise<void> {
+  public getNumActiveForks() {
+    return this.dbs.size;
+  }
+
+  public async stop(): Promise<void> {
+    // Grab the old queue before cancel() replaces it, so we can await its draining.
+    const oldQueue = this.deferredJobQueue;
     this.cancel();
-    return Promise.resolve();
+    await oldQueue.cancel();
   }
 
   public startNewEpoch(
@@ -130,7 +144,7 @@ export class ProvingOrchestrator implements EpochProver {
 
     const { promise: _promise, resolve, reject } = promiseWithResolvers<ProvingResult>();
     const promise = _promise.catch((reason): ProvingResult => ({ status: 'failure', reason }));
-    logger.info(`Starting epoch ${epochNumber} with ${totalNumCheckpoints} checkpoints.`);
+    this.logger.info(`Starting epoch ${epochNumber} with ${totalNumCheckpoints} checkpoints.`);
     this.provingState = new EpochProvingState(
       epochNumber,
       totalNumCheckpoints,
@@ -142,6 +156,14 @@ export class ProvingOrchestrator implements EpochProver {
     this.provingPromise = promise;
   }
 
+  /**
+   * Starts a new checkpoint.
+   * @param checkpointIndex - The index of the checkpoint in the epoch.
+   * @param constants - The constants for this checkpoint.
+   * @param l1ToL2Messages - The set of L1 to L2 messages to be inserted at the beginning of this checkpoint.
+   * @param totalNumBlocks - The total number of blocks expected in the checkpoint (must be at least one).
+   * @param headerOfLastBlockInPreviousCheckpoint - The header of the last block in the previous checkpoint.
+   */
   public async startNewCheckpoint(
     checkpointIndex: number,
     constants: CheckpointConstantData,
@@ -161,7 +183,7 @@ export class ProvingOrchestrator implements EpochProver {
     const lastBlockNumber = headerOfLastBlockInPreviousCheckpoint.globalVariables.blockNumber;
     const db = await this.dbProvider.fork(lastBlockNumber);
 
-    const firstBlockNumber = lastBlockNumber + 1;
+    const firstBlockNumber = BlockNumber(lastBlockNumber + 1);
     this.dbs.set(firstBlockNumber, db);
 
     // Get archive sibling path before any block in this checkpoint lands.
@@ -199,7 +221,7 @@ export class ProvingOrchestrator implements EpochProver {
   @trackSpan('ProvingOrchestrator.startNewBlock', blockNumber => ({
     [Attributes.BLOCK_NUMBER]: blockNumber,
   }))
-  public async startNewBlock(blockNumber: number, timestamp: UInt64, totalNumTxs: number) {
+  public async startNewBlock(blockNumber: BlockNumber, timestamp: UInt64, totalNumTxs: number) {
     if (!this.provingState) {
       throw new Error('Empty epoch proving state. Call startNewEpoch before starting a block.');
     }
@@ -214,15 +236,15 @@ export class ProvingOrchestrator implements EpochProver {
     }
 
     const constants = checkpointProvingState.constants;
-    logger.info(`Starting block ${blockNumber} for slot ${constants.slotNumber}.`);
+    this.logger.info(`Starting block ${blockNumber} for slot ${constants.slotNumber}.`);
 
     // Fork the db only when it's not already set. The db for the first block is set in `startNewCheckpoint`.
     if (!this.dbs.has(blockNumber)) {
       // Fork world state at the end of the immediately previous block
-      const db = await this.dbProvider.fork(blockNumber - 1);
+      const db = await this.dbProvider.fork(BlockNumber(blockNumber - 1));
       this.dbs.set(blockNumber, db);
     }
-    const db = this.dbs.get(blockNumber)!;
+    const db = this.getDbForBlock(blockNumber);
 
     // Get archive snapshot and sibling path before any txs in this block lands.
     const lastArchiveTreeSnapshot = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
@@ -254,7 +276,8 @@ export class ProvingOrchestrator implements EpochProver {
       await endSpongeBlob.absorb(blockEndBlobFields);
       blockProvingState.setEndSpongeBlob(endSpongeBlob);
 
-      // And also try to accumulate the blobs as far as we can:
+      // Try to accumulate the out hashes and blobs as far as we can:
+      await this.provingState.accumulateCheckpointOutHashes();
       await this.provingState.setBlobAccumulators();
     }
   }
@@ -274,11 +297,11 @@ export class ProvingOrchestrator implements EpochProver {
     if (!txs.length) {
       // To avoid an ugly throw below. If we require an empty block, we can just call setBlockCompleted
       // on a block with no txs. We cannot do that here because we cannot find the blockNumber without any txs.
-      logger.warn(`Provided no txs to orchestrator addTxs.`);
+      this.logger.warn(`Provided no txs to orchestrator addTxs.`);
       return;
     }
 
-    const blockNumber = txs[0].globalVariables.blockNumber;
+    const blockNumber = BlockNumber(txs[0].globalVariables.blockNumber);
     const provingState = this.provingState.getBlockProvingStateByBlockNumber(blockNumber!);
     if (!provingState) {
       throw new Error(`Proving state for block ${blockNumber} not found. Call startNewBlock first.`);
@@ -294,9 +317,9 @@ export class ProvingOrchestrator implements EpochProver {
       throw new Error(`Block ${blockNumber} has been initialized with transactions.`);
     }
 
-    logger.info(`Adding ${txs.length} transactions to block ${blockNumber}`);
+    this.logger.info(`Adding ${txs.length} transactions to block ${blockNumber}`);
 
-    const db = this.dbs.get(blockNumber)!;
+    const db = this.getDbForBlock(blockNumber);
     const lastArchive = provingState.lastArchiveTreeSnapshot;
     const newL1ToL2MessageTreeSnapshot = provingState.newL1ToL2MessageTreeSnapshot;
     const spongeBlobState = provingState.getStartSpongeBlob().clone();
@@ -309,7 +332,7 @@ export class ProvingOrchestrator implements EpochProver {
 
         validateTx(tx);
 
-        logger.info(`Received transaction: ${tx.hash}`);
+        this.logger.debug(`Received transaction: ${tx.hash}`);
 
         const startSpongeBlob = spongeBlobState.clone();
         const [hints, treeSnapshots] = await this.prepareBaseRollupInputs(
@@ -330,10 +353,10 @@ export class ProvingOrchestrator implements EpochProver {
         const txIndex = provingState.addNewTx(txProvingState);
         if (txProvingState.requireAvmProof) {
           this.getOrEnqueueChonkVerifier(provingState, txIndex);
-          logger.debug(`Enqueueing public VM for tx ${txIndex}`);
+          this.logger.debug(`Enqueueing public VM for tx ${txIndex}`);
           this.enqueueVM(provingState, txIndex);
         } else {
-          logger.debug(`Enqueueing base rollup for private-only tx ${txIndex}`);
+          this.logger.debug(`Enqueueing base rollup for private-only tx ${txIndex}`);
           this.enqueueBaseRollup(provingState, txIndex);
         }
       } catch (err: any) {
@@ -351,7 +374,8 @@ export class ProvingOrchestrator implements EpochProver {
 
     provingState.setEndSpongeBlob(spongeBlobState);
 
-    // Txs have been added to the block. Now try to accumulate the blobs as far as we can:
+    // Txs have been added to the block. Now try to accumulate the out hashes and blobs as far as we can:
+    await this.provingState.accumulateCheckpointOutHashes();
     await this.provingState.setBlobAccumulators();
   }
 
@@ -375,7 +399,7 @@ export class ProvingOrchestrator implements EpochProver {
             typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH
           >
         >();
-      logger.debug(`Starting chonk verifier circuit for tx ${txHash}`);
+      this.logger.debug(`Starting chonk verifier circuit for tx ${txHash}`);
       this.doEnqueueChonkVerifier(txHash, privateInputs, proof => {
         tubeProof.resolve(proof);
       });
@@ -388,10 +412,10 @@ export class ProvingOrchestrator implements EpochProver {
    * Marks the block as completed.
    * Computes the block header and updates the archive tree.
    */
-  @trackSpan('ProvingOrchestrator.setBlockCompleted', (blockNumber: number) => ({
+  @trackSpan('ProvingOrchestrator.setBlockCompleted', (blockNumber: BlockNumber) => ({
     [Attributes.BLOCK_NUMBER]: blockNumber,
   }))
-  public async setBlockCompleted(blockNumber: number, expectedHeader?: BlockHeader): Promise<BlockHeader> {
+  public async setBlockCompleted(blockNumber: BlockNumber, expectedHeader?: BlockHeader): Promise<BlockHeader> {
     const provingState = this.provingState?.getBlockProvingStateByBlockNumber(blockNumber);
     if (!provingState) {
       throw new Error(`Block proving state for ${blockNumber} not found`);
@@ -415,22 +439,28 @@ export class ProvingOrchestrator implements EpochProver {
     }
 
     // Given we've applied every change from this block, now assemble the block header:
-    logger.verbose(`Block ${blockNumber} completed. Assembling header.`);
+    this.logger.verbose(`Block ${blockNumber} completed. Assembling header.`);
     const header = await provingState.buildBlockHeader();
 
     if (expectedHeader && !header.equals(expectedHeader)) {
-      logger.error(`Block header mismatch: header=${header} expectedHeader=${expectedHeader}`);
+      this.logger.error(`Block header mismatch: header=${header} expectedHeader=${expectedHeader}`);
       throw new Error('Block header mismatch');
     }
 
-    // Get db for this block
-    const db = this.dbs.get(provingState.blockNumber)!;
+    // Get db for this block and remove from map — no other code should use it after this point.
+    const db = this.getDbForBlock(provingState.blockNumber);
+    this.dbs.delete(provingState.blockNumber);
 
-    // Update the archive tree, so we're ready to start processing the next block:
-    logger.verbose(
-      `Updating archive tree with block ${provingState.blockNumber} header ${(await header.hash()).toString()}`,
-    );
-    await db.updateArchive(header);
+    // Update the archive tree, capture the snapshot, and close the fork deterministically.
+    try {
+      this.logger.verbose(
+        `Updating archive tree with block ${provingState.blockNumber} header ${(await header.hash()).toString()}`,
+      );
+      await db.updateArchive(header);
+      provingState.setBuiltArchive(await getTreeSnapshot(MerkleTreeId.ARCHIVE, db));
+    } finally {
+      await db.close();
+    }
 
     await this.verifyBuiltBlockAgainstSyncedState(provingState);
 
@@ -441,31 +471,34 @@ export class ProvingOrchestrator implements EpochProver {
   protected async verifyBuiltBlockAgainstSyncedState(provingState: BlockProvingState) {
     const builtBlockHeader = provingState.getBuiltBlockHeader();
     if (!builtBlockHeader) {
-      logger.debug('Block header not built yet, skipping header check.');
+      this.logger.debug('Block header not built yet, skipping header check.');
       return;
     }
 
     const output = provingState.getBlockRootRollupOutput();
     if (!output) {
-      logger.debug('Block root rollup proof not built yet, skipping header check.');
+      this.logger.debug('Block root rollup proof not built yet, skipping header check.');
+      return;
+    }
+
+    const newArchive = provingState.getBuiltArchive();
+    if (!newArchive) {
+      this.logger.debug('Archive snapshot not yet captured, skipping header check.');
       return;
     }
+
     const header = await buildHeaderFromCircuitOutputs(output);
 
     if (!(await header.hash()).equals(await builtBlockHeader.hash())) {
-      logger.error(`Block header mismatch.\nCircuit: ${inspect(header)}\nComputed: ${inspect(builtBlockHeader)}`);
+      this.logger.error(`Block header mismatch.\nCircuit: ${inspect(header)}\nComputed: ${inspect(builtBlockHeader)}`);
       provingState.reject(`Block header hash mismatch.`);
       return;
     }
 
-    // Get db for this block
     const blockNumber = provingState.blockNumber;
-    const db = this.dbs.get(blockNumber)!;
-
-    const newArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
     const syncedArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.dbProvider.getSnapshot(blockNumber));
     if (!syncedArchive.equals(newArchive)) {
-      logger.error(
+      this.logger.error(
         `Archive tree mismatch for block ${blockNumber}: world state synced to ${inspect(
           syncedArchive,
         )} but built ${inspect(newArchive)}`,
@@ -476,32 +509,43 @@ export class ProvingOrchestrator implements EpochProver {
 
     const circuitArchive = output.newArchive;
     if (!newArchive.equals(circuitArchive)) {
-      logger.error(`New archive mismatch.\nCircuit: ${output.newArchive}\nComputed: ${newArchive}`);
+      this.logger.error(`New archive mismatch.\nCircuit: ${output.newArchive}\nComputed: ${newArchive}`);
       provingState.reject(`New archive mismatch.`);
       return;
     }
-
-    // TODO(palla/prover): This closes the fork only on the happy path. If this epoch orchestrator
-    // is aborted and never reaches this point, it will leak the fork. We need to add a global cleanup,
-    // but have to make sure it only runs once all operations are completed, otherwise some function here
-    // will attempt to access the fork after it was closed.
-    logger.debug(`Cleaning up world state fork for ${blockNumber}`);
-    void this.dbs
-      .get(blockNumber)
-      ?.close()
-      .then(() => this.dbs.delete(blockNumber))
-      .catch(err => logger.error(`Error closing db for block ${blockNumber}`, err));
   }
 
   /**
-   * Cancel any further proving
+   * Cancel any further proving.
+   * If cancelJobsOnStop is true, aborts all pending jobs with the broker (which marks them as 'Aborted').
+   * If cancelJobsOnStop is false (default), jobs remain in the broker queue and can be reused on restart/reorg.
    */
   public cancel() {
-    for (const controller of this.pendingProvingJobs) {
-      controller.abort();
+    void this.deferredJobQueue.cancel();
+    // Recreate the queue so it can accept jobs for subsequent epochs.
+    this.deferredJobQueue = new SerialQueue();
+    this.deferredJobQueue.start(this.enqueueConcurrency);
+
+    if (this.cancelJobsOnStop) {
+      for (const controller of this.pendingProvingJobs) {
+        controller.abort();
+      }
     }
 
     this.provingState?.cancel();
+
+    for (const [blockNumber, db] of this.dbs.entries()) {
+      void db.close().catch(err => this.logger.error(`Error closing db for block ${blockNumber}`, err));
+    }
+    this.dbs.clear();
+  }
+
+  private getDbForBlock(blockNumber: BlockNumber): MerkleTreeWriteOperations {
+    const db = this.dbs.get(blockNumber);
+    if (!db) {
+      throw new Error(`World state fork for block ${blockNumber} not found.`);
+    }
+    return db;
   }
 
   /**
@@ -545,7 +589,7 @@ export class ProvingOrchestrator implements EpochProver {
     callback: (result: T) => void | Promise<void>,
   ) {
     if (!provingState.verifyState()) {
-      logger.debug(`Not enqueuing job, state no longer valid`);
+      this.logger.debug(`Not enqueuing job, state no longer valid`);
       return;
     }
 
@@ -563,7 +607,7 @@ export class ProvingOrchestrator implements EpochProver {
 
         const result = await request(controller.signal);
         if (!provingState.verifyState()) {
-          logger.debug(`State no longer valid, discarding result`);
+          this.logger.debug(`State no longer valid, discarding result`);
           return;
         }
 
@@ -581,7 +625,7 @@ export class ProvingOrchestrator implements EpochProver {
           return;
         }
 
-        logger.error(`Error thrown when proving job`, err);
+        this.logger.error(`Error thrown when proving job`, err);
         provingState!.reject(`${err}`);
       } finally {
         const index = this.pendingProvingJobs.indexOf(controller);
@@ -591,8 +635,11 @@ export class ProvingOrchestrator implements EpochProver {
       }
     };
 
-    // let the callstack unwind before adding the job to the queue
-    setImmediate(() => void safeJob());
+    void this.deferredJobQueue.put(async () => {
+      void safeJob();
+      // we yield here to the macro task queue such to give Nodejs a chance to run other operatoins in between enqueues
+      await sleep(0);
+    });
   }
 
   private async updateL1ToL2MessageTree(l1ToL2Messages: Fr[], db: MerkleTreeWriteOperations) {
@@ -666,12 +713,12 @@ export class ProvingOrchestrator implements EpochProver {
   // Executes the next level of merge if all inputs are available
   private enqueueBaseRollup(provingState: BlockProvingState, txIndex: number) {
     if (!provingState.verifyState()) {
-      logger.debug('Not running base rollup, state invalid');
+      this.logger.debug('Not running base rollup, state invalid');
       return;
     }
 
     if (!provingState.tryStartProvingBase(txIndex)) {
-      logger.debug(`Base rollup for tx ${txIndex} already started.`);
+      this.logger.debug(`Base rollup for tx ${txIndex} already started.`);
       return;
     }
 
@@ -679,7 +726,7 @@ export class ProvingOrchestrator implements EpochProver {
     const { processedTx } = txProvingState;
     const { rollupType, inputs } = txProvingState.getBaseRollupTypeAndInputs();
 
-    logger.debug(`Enqueuing deferred proving base rollup for ${processedTx.hash.toString()}`);
+    this.logger.debug(`Enqueuing deferred proving base rollup for ${processedTx.hash.toString()}`);
 
     this.deferredProving(
       provingState,
@@ -703,7 +750,7 @@ export class ProvingOrchestrator implements EpochProver {
         },
       ),
       result => {
-        logger.debug(`Completed proof for ${rollupType} for tx ${processedTx.hash.toString()}`);
+        this.logger.debug(`Completed proof for ${rollupType} for tx ${processedTx.hash.toString()}`);
         validatePartialState(result.inputs.endTreeSnapshots, txProvingState.treeSnapshots);
         const leafLocation = provingState.setBaseRollupProof(txIndex, result);
         if (provingState.totalNumTxs === 1) {
@@ -719,7 +766,7 @@ export class ProvingOrchestrator implements EpochProver {
   // Once completed, will enqueue the the public tx base rollup.
   private getOrEnqueueChonkVerifier(provingState: BlockProvingState, txIndex: number) {
     if (!provingState.verifyState()) {
-      logger.debug('Not running chonk verifier circuit, state invalid');
+      this.logger.debug('Not running chonk verifier circuit, state invalid');
       return;
     }
 
@@ -732,19 +779,19 @@ export class ProvingOrchestrator implements EpochProver {
         typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH
       >,
     ) => {
-      logger.debug(`Got chonk verifier proof for tx index: ${txIndex}`, { txHash });
+      this.logger.debug(`Got chonk verifier proof for tx index: ${txIndex}`, { txHash });
       txProvingState.setPublicChonkVerifierProof(result);
       this.provingState?.cachedChonkVerifierProofs.delete(txHash);
       this.checkAndEnqueueBaseRollup(provingState, txIndex);
     };
 
     if (this.provingState?.cachedChonkVerifierProofs.has(txHash)) {
-      logger.debug(`Chonk verifier proof already enqueued for tx index: ${txIndex}`, { txHash });
+      this.logger.debug(`Chonk verifier proof already enqueued for tx index: ${txIndex}`, { txHash });
       void this.provingState!.cachedChonkVerifierProofs.get(txHash)!.then(handleResult);
       return;
     }
 
-    logger.debug(`Enqueuing chonk verifier circuit for tx index: ${txIndex}`);
+    this.logger.debug(`Enqueuing chonk verifier circuit for tx index: ${txIndex}`);
     this.doEnqueueChonkVerifier(txHash, txProvingState.getPublicChonkVerifierPrivateInputs(), handleResult);
   }
 
@@ -760,7 +807,7 @@ export class ProvingOrchestrator implements EpochProver {
     provingState: EpochProvingState | BlockProvingState = this.provingState!,
   ) {
     if (!provingState.verifyState()) {
-      logger.debug('Not running chonk verifier circuit, state invalid');
+      this.logger.debug('Not running chonk verifier circuit, state invalid');
       return;
     }
 
@@ -783,12 +830,12 @@ export class ProvingOrchestrator implements EpochProver {
   // Enqueues the next level of merge if all inputs are available
   private enqueueMergeRollup(provingState: BlockProvingState, location: TreeNodeLocation) {
     if (!provingState.verifyState()) {
-      logger.debug('Not running merge rollup. State no longer valid.');
+      this.logger.debug('Not running merge rollup. State no longer valid.');
       return;
     }
 
     if (!provingState.tryStartProvingMerge(location)) {
-      logger.debug('Merge rollup already started.');
+      this.logger.debug('Merge rollup already started.');
       return;
     }
 
@@ -814,18 +861,18 @@ export class ProvingOrchestrator implements EpochProver {
   // Executes the block root rollup circuit
   private enqueueBlockRootRollup(provingState: BlockProvingState) {
     if (!provingState.verifyState()) {
-      logger.debug('Not running block root rollup, state no longer valid');
+      this.logger.debug('Not running block root rollup, state no longer valid');
       return;
     }
 
     if (!provingState.tryStartProvingBlockRoot()) {
-      logger.debug('Block root rollup already started.');
+      this.logger.debug('Block root rollup already started.');
       return;
     }
 
     const { rollupType, inputs } = provingState.getBlockRootRollupTypeAndInputs();
 
-    logger.debug(`Enqueuing ${rollupType} for block ${provingState.blockNumber}.`);
+    this.logger.debug(`Enqueuing ${rollupType} for block ${provingState.blockNumber}.`);
 
     this.deferredProving(
       provingState,
@@ -850,18 +897,19 @@ export class ProvingOrchestrator implements EpochProver {
         },
       ),
       async result => {
-        // If the proofs were slower than the block header building, then we need to try validating the block header hashes here.
-        await this.verifyBuiltBlockAgainstSyncedState(provingState);
-
-        logger.debug(`Completed ${rollupType} proof for block ${provingState.blockNumber}`);
+        this.logger.debug(`Completed ${rollupType} proof for block ${provingState.blockNumber}`);
 
         const leafLocation = provingState.setBlockRootRollupProof(result);
         const checkpointProvingState = provingState.parentCheckpoint;
 
+        // Verification is called from both here and setBlockCompleted. Whichever runs last
+        // will be the first to see all three pieces (header, proof output, archive) and run the checks.
+        await this.verifyBuiltBlockAgainstSyncedState(provingState);
+
         if (checkpointProvingState.totalNumBlocks === 1) {
-          this.checkAndEnqueueCheckpointRootRollup(checkpointProvingState);
+          await this.checkAndEnqueueCheckpointRootRollup(checkpointProvingState);
         } else {
-          this.checkAndEnqueueNextBlockMergeRollup(checkpointProvingState, leafLocation);
+          await this.checkAndEnqueueNextBlockMergeRollup(checkpointProvingState, leafLocation);
         }
       },
     );
@@ -875,12 +923,12 @@ export class ProvingOrchestrator implements EpochProver {
     baseParityIndex: number,
   ) {
     if (!provingState.verifyState()) {
-      logger.debug('Not running base parity. State no longer valid.');
+      this.logger.debug('Not running base parity. State no longer valid.');
       return;
     }
 
     if (!provingState.tryStartProvingBaseParity(baseParityIndex)) {
-      logger.warn(`Base parity ${baseParityIndex} already started.`);
+      this.logger.warn(`Base parity ${baseParityIndex} already started.`);
       return;
     }
 
@@ -915,12 +963,12 @@ export class ProvingOrchestrator implements EpochProver {
   // Enqueues the root rollup proof if all inputs are available
   private enqueueRootParityCircuit(provingState: BlockProvingState) {
     if (!provingState.verifyState()) {
-      logger.debug('Not running root parity. State no longer valid.');
+      this.logger.debug('Not running root parity. State no longer valid.');
       return;
     }
 
     if (!provingState.tryStartProvingRootParity()) {
-      logger.debug('Root parity already started.');
+      this.logger.debug('Root parity already started.');
       return;
     }
 
@@ -947,12 +995,12 @@ export class ProvingOrchestrator implements EpochProver {
   // Enqueues the next level of merge if all inputs are available
   private enqueueBlockMergeRollup(provingState: CheckpointProvingState, location: TreeNodeLocation) {
     if (!provingState.verifyState()) {
-      logger.debug('Not running block merge rollup. State no longer valid.');
+      this.logger.debug('Not running block merge rollup. State no longer valid.');
       return;
     }
 
     if (!provingState.tryStartProvingBlockMerge(location)) {
-      logger.debug('Block merge rollup already started.');
+      this.logger.debug('Block merge rollup already started.');
       return;
     }
 
@@ -967,29 +1015,29 @@ export class ProvingOrchestrator implements EpochProver {
         },
         signal => this.prover.getBlockMergeRollupProof(inputs, signal, provingState.epochNumber),
       ),
-      result => {
+      async result => {
         provingState.setBlockMergeRollupProof(location, result);
-        this.checkAndEnqueueNextBlockMergeRollup(provingState, location);
+        await this.checkAndEnqueueNextBlockMergeRollup(provingState, location);
       },
     );
   }
 
-  private enqueueCheckpointRootRollup(provingState: CheckpointProvingState) {
+  private async enqueueCheckpointRootRollup(provingState: CheckpointProvingState) {
     if (!provingState.verifyState()) {
-      logger.debug('Not running checkpoint root rollup. State no longer valid.');
+      this.logger.debug('Not running checkpoint root rollup. State no longer valid.');
       return;
     }
 
     if (!provingState.tryStartProvingCheckpointRoot()) {
-      logger.debug('Checkpoint root rollup already started.');
+      this.logger.debug('Checkpoint root rollup already started.');
       return;
     }
 
     const rollupType = provingState.getCheckpointRootRollupType();
 
-    logger.debug(`Enqueuing ${rollupType} for checkpoint ${provingState.index}.`);
+    this.logger.debug(`Enqueuing ${rollupType} for checkpoint ${provingState.index}.`);
 
-    const inputs = provingState.getCheckpointRootRollupInputs();
+    const inputs = await provingState.getCheckpointRootRollupInputs();
 
     this.deferredProving(
       provingState,
@@ -1011,7 +1059,7 @@ export class ProvingOrchestrator implements EpochProver {
         const computedEndBlobAccumulatorState = provingState.getEndBlobAccumulator()!.toBlobAccumulator();
         const circuitEndBlobAccumulatorState = result.inputs.endBlobAccumulator;
         if (!circuitEndBlobAccumulatorState.equals(computedEndBlobAccumulatorState)) {
-          logger.error(
+          this.logger.error(
             `Blob accumulator state mismatch.\nCircuit: ${inspect(circuitEndBlobAccumulatorState)}\nComputed: ${inspect(
               computedEndBlobAccumulatorState,
             )}`,
@@ -1020,7 +1068,7 @@ export class ProvingOrchestrator implements EpochProver {
           return;
         }
 
-        logger.debug(`Completed ${rollupType} proof for checkpoint ${provingState.index}.`);
+        this.logger.debug(`Completed ${rollupType} proof for checkpoint ${provingState.index}.`);
 
         const leafLocation = provingState.setCheckpointRootRollupProof(result);
         const epochProvingState = provingState.parentEpoch;
@@ -1036,12 +1084,12 @@ export class ProvingOrchestrator implements EpochProver {
 
   private enqueueCheckpointMergeRollup(provingState: EpochProvingState, location: TreeNodeLocation) {
     if (!provingState.verifyState()) {
-      logger.debug('Not running checkpoint merge rollup. State no longer valid.');
+      this.logger.debug('Not running checkpoint merge rollup. State no longer valid.');
       return;
     }
 
     if (!provingState.tryStartProvingCheckpointMerge(location)) {
-      logger.debug('Checkpoint merge rollup already started.');
+      this.logger.debug('Checkpoint merge rollup already started.');
       return;
     }
 
@@ -1058,7 +1106,7 @@ export class ProvingOrchestrator implements EpochProver {
         signal => this.prover.getCheckpointMergeRollupProof(inputs, signal, provingState.epochNumber),
       ),
       result => {
-        logger.debug('Completed proof for checkpoint merge rollup.');
+        this.logger.debug('Completed proof for checkpoint merge rollup.');
         provingState.setCheckpointMergeRollupProof(location, result);
         this.checkAndEnqueueNextCheckpointMergeRollup(provingState, location);
       },
@@ -1067,16 +1115,16 @@ export class ProvingOrchestrator implements EpochProver {
 
   private enqueueEpochPadding(provingState: EpochProvingState) {
     if (!provingState.verifyState()) {
-      logger.debug('Not running epoch padding. State no longer valid.');
+      this.logger.debug('Not running epoch padding. State no longer valid.');
       return;
     }
 
     if (!provingState.tryStartProvingPaddingCheckpoint()) {
-      logger.debug('Padding checkpoint already started.');
+      this.logger.debug('Padding checkpoint already started.');
       return;
     }
 
-    logger.debug('Padding epoch proof with a padding block root proof.');
+    this.logger.debug('Padding epoch proof with a padding block root proof.');
 
     const inputs = provingState.getPaddingCheckpointInputs();
 
@@ -1091,7 +1139,7 @@ export class ProvingOrchestrator implements EpochProver {
         signal => this.prover.getCheckpointPaddingRollupProof(inputs, signal, provingState.epochNumber),
       ),
       result => {
-        logger.debug('Completed proof for padding checkpoint.');
+        this.logger.debug('Completed proof for padding checkpoint.');
         provingState.setCheckpointPaddingProof(result);
         this.checkAndEnqueueRootRollup(provingState);
       },
@@ -1101,11 +1149,11 @@ export class ProvingOrchestrator implements EpochProver {
   // Executes the root rollup circuit
   private enqueueRootRollup(provingState: EpochProvingState) {
     if (!provingState.verifyState()) {
-      logger.debug('Not running root rollup, state no longer valid');
+      this.logger.debug('Not running root rollup, state no longer valid');
       return;
     }
 
-    logger.debug(`Preparing root rollup`);
+    this.logger.debug(`Preparing root rollup`);
 
     const inputs = provingState.getRootRollupInputs();
 
@@ -1120,7 +1168,7 @@ export class ProvingOrchestrator implements EpochProver {
         signal => this.prover.getRootRollupProof(inputs, signal, provingState.epochNumber),
       ),
       result => {
-        logger.verbose(`Orchestrator completed root rollup for epoch ${provingState.epochNumber}`);
+        this.logger.verbose(`Orchestrator completed root rollup for epoch ${provingState.epochNumber}`);
         provingState.setRootRollupProof(result);
         provingState.resolve({ status: 'success' });
       },
@@ -1142,32 +1190,35 @@ export class ProvingOrchestrator implements EpochProver {
 
   private checkAndEnqueueBlockRootRollup(provingState: BlockProvingState) {
     if (!provingState.isReadyForBlockRootRollup()) {
-      logger.debug('Not ready for block root rollup');
+      this.logger.debug('Not ready for block root rollup');
       return;
     }
 
     this.enqueueBlockRootRollup(provingState);
   }
 
-  private checkAndEnqueueNextBlockMergeRollup(provingState: CheckpointProvingState, currentLocation: TreeNodeLocation) {
+  private async checkAndEnqueueNextBlockMergeRollup(
+    provingState: CheckpointProvingState,
+    currentLocation: TreeNodeLocation,
+  ) {
     if (!provingState.isReadyForBlockMerge(currentLocation)) {
       return;
     }
 
     const parentLocation = provingState.getParentLocation(currentLocation);
     if (parentLocation.level === 0) {
-      this.checkAndEnqueueCheckpointRootRollup(provingState);
+      await this.checkAndEnqueueCheckpointRootRollup(provingState);
     } else {
       this.enqueueBlockMergeRollup(provingState, parentLocation);
     }
   }
 
-  private checkAndEnqueueCheckpointRootRollup(provingState: CheckpointProvingState) {
+  private async checkAndEnqueueCheckpointRootRollup(provingState: CheckpointProvingState) {
     if (!provingState.isReadyForCheckpointRoot()) {
       return;
     }
 
-    this.enqueueCheckpointRootRollup(provingState);
+    await this.enqueueCheckpointRootRollup(provingState);
   }
 
   private checkAndEnqueueNextCheckpointMergeRollup(provingState: EpochProvingState, currentLocation: TreeNodeLocation) {
@@ -1185,7 +1236,7 @@ export class ProvingOrchestrator implements EpochProver {
 
   private checkAndEnqueueRootRollup(provingState: EpochProvingState) {
     if (!provingState.isReadyForRootRollup()) {
-      logger.debug('Not ready for root rollup');
+      this.logger.debug('Not ready for root rollup');
       return;
     }
 
@@ -1200,14 +1251,12 @@ export class ProvingOrchestrator implements EpochProver {
    */
   private enqueueVM(provingState: BlockProvingState, txIndex: number) {
     if (!provingState.verifyState()) {
-      logger.debug(`Not running VM circuit as state is no longer valid`);
+      this.logger.debug(`Not running VM circuit as state is no longer valid`);
       return;
     }
 
     const txProvingState = provingState.getTxProvingState(txIndex);
 
-    // This function tries to do AVM proving. If there is a failure, it fakes the proof unless AVM_PROVING_STRICT is defined.
-    // Nothing downstream depends on the AVM proof yet. So having this mode lets us incrementally build the AVM circuit.
     const doAvmProving = wrapCallbackInSpan(
       this.tracer,
       'ProvingOrchestrator.prover.getAvmProof',
@@ -1216,36 +1265,13 @@ export class ProvingOrchestrator implements EpochProver {
       },
       async (signal: AbortSignal) => {
         const inputs = txProvingState.getAvmInputs();
-        try {
-          // TODO(#14234)[Unconditional PIs validation]: Remove the whole try-catch logic and
-          // just keep the next line but removing the second argument (false).
-          return await this.prover.getAvmProof(inputs, false, signal, provingState.epochNumber);
-        } catch (err) {
-          if (process.env.AVM_PROVING_STRICT) {
-            logger.error(`Error thrown when proving AVM circuit with AVM_PROVING_STRICT on`, err);
-            throw err;
-          } else {
-            logger.warn(
-              `Error thrown when proving AVM circuit but AVM_PROVING_STRICT is off. Use snapshotted
-               AVM inputs and carrying on. ${inspect(err)}.`,
-            );
-
-            try {
-              this.metrics.incAvmFallback();
-              const snapshotAvmPrivateInputs = readAvmMinimalPublicTxInputsFromFile();
-              return await this.prover.getAvmProof(snapshotAvmPrivateInputs, true, signal, provingState.epochNumber);
-            } catch (err) {
-              logger.error(`Error thrown when proving snapshotted AVM inputs.`, err);
-              throw err;
-            }
-          }
-        }
+        return await this.prover.getAvmProof(inputs, signal, provingState.epochNumber);
       },
     );
 
-    this.deferredProving(provingState, doAvmProving, proofAndVk => {
-      logger.debug(`Proven VM for tx index: ${txIndex}`);
-      txProvingState.setAvmProof(proofAndVk);
+    this.deferredProving(provingState, doAvmProving, proof => {
+      this.logger.debug(`Proven VM for tx index: ${txIndex}`);
+      txProvingState.setAvmProof(proof);
       this.checkAndEnqueueBaseRollup(provingState, txIndex);
     });
   }
@@ -1257,7 +1283,7 @@ export class ProvingOrchestrator implements EpochProver {
     }
 
     // We must have completed all proving (chonk verifier proof and (if required) vm proof are generated), we now move to the base rollup.
-    logger.debug(`Public functions completed for tx ${txIndex} enqueueing base rollup`);
+    this.logger.debug(`Public functions completed for tx ${txIndex} enqueueing base rollup`);
 
     this.enqueueBaseRollup(provingState, txIndex);
   }